SQLite

Check-in [ba6bf33147]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge the branch-3.28a fixes into branch-3.28.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | branch-3.28
Files: files | file ages | folders
SHA3-256: ba6bf331476d0217f4132b73cb3da559e75bfb21856ec94f82c0f0150a53592b
User & Date: drh 2022-08-10 17:03:43.321
Context
2022-09-01
10:41
In the query planner, add a heuristic that will reduce the cost of a full table scan for a materialized view or subquery if the full scan is the outer-most loop. This is shown to speed up some queries. (check-in: e3754cc188 user: drh tags: branch-3.28)
2022-08-10
17:03
Merge the branch-3.28a fixes into branch-3.28. (check-in: ba6bf33147 user: drh tags: branch-3.28)
2022-08-09
20:22
Fix a rounding error caused by scalar->logarithm->scalar conversion when using stat4 data to estimate some range scans. (check-in: 68d86f2b20 user: drh tags: branch-3.28)
2021-07-13
15:30
Remove two incorrect assert() statements from the logic used to derive column names and types from subqueries. This allows the SQL associated with CVE-2020-13871 (ticket [c8d3b9f0a750a529]) to be tested. (Closed-Leaf check-in: d2e6722037 user: dan tags: branch-3.28a)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/select.c.
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
  char const *zOrigDb = 0;
  char const *zOrigTab = 0;
  char const *zOrigCol = 0;
#endif

  assert( pExpr!=0 );
  assert( pNC->pSrcList!=0 );
  assert( pExpr->op!=TK_AGG_COLUMN );  /* This routine runes before aggregates
                                       ** are processed */
  switch( pExpr->op ){
    case TK_COLUMN: {
      /* The expression is a column. Locate the table the column is being
      ** extracted from in NameContext.pSrcList. This table may be real
      ** database table or a subquery.
      */
      Table *pTab = 0;            /* Table structure column is extracted from */







<
<







1639
1640
1641
1642
1643
1644
1645


1646
1647
1648
1649
1650
1651
1652
  char const *zOrigDb = 0;
  char const *zOrigTab = 0;
  char const *zOrigCol = 0;
#endif

  assert( pExpr!=0 );
  assert( pNC->pSrcList!=0 );


  switch( pExpr->op ){
    case TK_COLUMN: {
      /* The expression is a column. Locate the table the column is being
      ** extracted from in NameContext.pSrcList. This table may be real
      ** database table or a subquery.
      */
      Table *pTab = 0;            /* Table structure column is extracted from */
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
      /* If the column contains an "AS <name>" phrase, use <name> as the name */
    }else{
      Expr *pColExpr = sqlite3ExprSkipCollate(pEList->a[i].pExpr);
      while( pColExpr->op==TK_DOT ){
        pColExpr = pColExpr->pRight;
        assert( pColExpr!=0 );
      }
      assert( pColExpr->op!=TK_AGG_COLUMN );
      if( pColExpr->op==TK_COLUMN ){
        /* For columns use the column name name */
        int iCol = pColExpr->iColumn;
        Table *pTab = pColExpr->y.pTab;
        assert( pTab!=0 );
        if( iCol<0 ) iCol = pTab->iPKey;
        zName = iCol>=0 ? pTab->aCol[iCol].zName : "rowid";







<







1960
1961
1962
1963
1964
1965
1966

1967
1968
1969
1970
1971
1972
1973
      /* If the column contains an "AS <name>" phrase, use <name> as the name */
    }else{
      Expr *pColExpr = sqlite3ExprSkipCollate(pEList->a[i].pExpr);
      while( pColExpr->op==TK_DOT ){
        pColExpr = pColExpr->pRight;
        assert( pColExpr!=0 );
      }

      if( pColExpr->op==TK_COLUMN ){
        /* For columns use the column name name */
        int iCol = pColExpr->iColumn;
        Table *pTab = pColExpr->y.pTab;
        assert( pTab!=0 );
        if( iCol<0 ) iCol = pTab->iPKey;
        zName = iCol>=0 ? pTab->aCol[iCol].zName : "rowid";
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
        pLimit = p->pLimit;
        p->pLimit = 0;
        uniondest.eDest = op;
        ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE",
                          selectOpName(p->op)));
        rc = sqlite3Select(pParse, p, &uniondest);
        testcase( rc!=SQLITE_OK );
        /* Query flattening in sqlite3Select() might refill p->pOrderBy.
        ** Be sure to delete p->pOrderBy, therefore, to avoid a memory leak. */
        sqlite3ExprListDelete(db, p->pOrderBy);
        pDelete = p->pPrior;
        p->pPrior = pPrior;
        p->pOrderBy = 0;
        if( p->op==TK_UNION ){
          p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow);
        }
        sqlite3ExprDelete(db, p->pLimit);







<
<
|







2685
2686
2687
2688
2689
2690
2691


2692
2693
2694
2695
2696
2697
2698
2699
        pLimit = p->pLimit;
        p->pLimit = 0;
        uniondest.eDest = op;
        ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE",
                          selectOpName(p->op)));
        rc = sqlite3Select(pParse, p, &uniondest);
        testcase( rc!=SQLITE_OK );


        assert( p->pOrderBy==0 );
        pDelete = p->pPrior;
        p->pPrior = pPrior;
        p->pOrderBy = 0;
        if( p->op==TK_UNION ){
          p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow);
        }
        sqlite3ExprDelete(db, p->pLimit);
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
    **   SELECT a+5, b*10 FROM (SELECT x*3 AS a, y+10 AS b FROM t1) WHERE a>b;
    **   \                     \_____________ subquery __________/          /
    **    \_____________________ outer query ______________________________/
    **
    ** We look at every expression in the outer query and every place we see
    ** "a" we substitute "x*3" and every place we see "b" we substitute "y+10".
    */
    if( pSub->pOrderBy ){
      /* At this point, any non-zero iOrderByCol values indicate that the
      ** ORDER BY column expression is identical to the iOrderByCol'th
      ** expression returned by SELECT statement pSub. Since these values
      ** do not necessarily correspond to columns in SELECT statement pParent,
      ** zero them before transfering the ORDER BY clause.
      **
      ** Not doing this may cause an error if a subsequent call to this







|







4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
    **   SELECT a+5, b*10 FROM (SELECT x*3 AS a, y+10 AS b FROM t1) WHERE a>b;
    **   \                     \_____________ subquery __________/          /
    **    \_____________________ outer query ______________________________/
    **
    ** We look at every expression in the outer query and every place we see
    ** "a" we substitute "x*3" and every place we see "b" we substitute "y+10".
    */
    if( pSub->pOrderBy && (pParent->selFlags & SF_NoopOrderBy)==0 ){
      /* At this point, any non-zero iOrderByCol values indicate that the
      ** ORDER BY column expression is identical to the iOrderByCol'th
      ** expression returned by SELECT statement pSub. Since these values
      ** do not necessarily correspond to columns in SELECT statement pParent,
      ** zero them before transfering the ORDER BY clause.
      **
      ** Not doing this may cause an error if a subsequent call to this
5640
5641
5642
5643
5644
5645
5646

5647
5648
5649
5650
5651
5652
5653
           pDest->eDest==SRT_Queue  || pDest->eDest==SRT_DistFifo ||
           pDest->eDest==SRT_DistQueue || pDest->eDest==SRT_Fifo);
    /* If ORDER BY makes no difference in the output then neither does
    ** DISTINCT so it can be removed too. */
    sqlite3ExprListDelete(db, p->pOrderBy);
    p->pOrderBy = 0;
    p->selFlags &= ~SF_Distinct;

  }
  sqlite3SelectPrep(pParse, p, 0);
  if( pParse->nErr || db->mallocFailed ){
    goto select_end;
  }
  assert( p->pEList!=0 );
#if SELECTTRACE_ENABLED







>







5635
5636
5637
5638
5639
5640
5641
5642
5643
5644
5645
5646
5647
5648
5649
           pDest->eDest==SRT_Queue  || pDest->eDest==SRT_DistFifo ||
           pDest->eDest==SRT_DistQueue || pDest->eDest==SRT_Fifo);
    /* If ORDER BY makes no difference in the output then neither does
    ** DISTINCT so it can be removed too. */
    sqlite3ExprListDelete(db, p->pOrderBy);
    p->pOrderBy = 0;
    p->selFlags &= ~SF_Distinct;
    p->selFlags |= SF_NoopOrderBy;
  }
  sqlite3SelectPrep(pParse, p, 0);
  if( pParse->nErr || db->mallocFailed ){
    goto select_end;
  }
  assert( p->pEList!=0 );
#if SELECTTRACE_ENABLED
Changes to src/sqliteInt.h.
2885
2886
2887
2888
2889
2890
2891

2892
2893
2894
2895
2896
2897
2898
#define SF_MinMaxAgg      0x01000  /* Aggregate containing min() or max() */
#define SF_Recursive      0x02000  /* The recursive part of a recursive CTE */
#define SF_FixedLimit     0x04000  /* nSelectRow set by a constant LIMIT */
#define SF_MaybeConvert   0x08000  /* Need convertCompoundSelectToSubquery() */
#define SF_Converted      0x10000  /* By convertCompoundSelectToSubquery() */
#define SF_IncludeHidden  0x20000  /* Include hidden columns in output */
#define SF_ComplexResult  0x40000  /* Result contains subquery or function */


/*
** The results of a SELECT can be distributed in several ways, as defined
** by one of the following macros.  The "SRT" prefix means "SELECT Result
** Type".
**
**     SRT_Union       Store results as a key in a temporary index







>







2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
#define SF_MinMaxAgg      0x01000  /* Aggregate containing min() or max() */
#define SF_Recursive      0x02000  /* The recursive part of a recursive CTE */
#define SF_FixedLimit     0x04000  /* nSelectRow set by a constant LIMIT */
#define SF_MaybeConvert   0x08000  /* Need convertCompoundSelectToSubquery() */
#define SF_Converted      0x10000  /* By convertCompoundSelectToSubquery() */
#define SF_IncludeHidden  0x20000  /* Include hidden columns in output */
#define SF_ComplexResult  0x40000  /* Result contains subquery or function */
#define SF_NoopOrderBy   0x0400000 /* ORDER BY is ignored for this query */

/*
** The results of a SELECT can be distributed in several ways, as defined
** by one of the following macros.  The "SRT" prefix means "SELECT Result
** Type".
**
**     SRT_Union       Store results as a key in a temporary index
Changes to test/colname.test.
395
396
397
398
399
400
401






402
403
404
405
406
407
408
} {Bbb 123}
ifcapable vtab {
  do_execsql_test colname-9.320 {
    CREATE TABLE t2 AS SELECT BBb FROM (SELECT aaa AS Bbb FROM t1);
    SELECT name FROM pragma_table_info('t2');
  } {Bbb}
}







# Issue detected by OSSFuzz on 2017-12-24 (Christmas Eve)
# caused by check-in https://sqlite.org/src/info/6b2ff26c25
#
# Prior to being fixed, the following CREATE TABLE was dereferencing
# a NULL pointer and segfaulting.
#







>
>
>
>
>
>







395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
} {Bbb 123}
ifcapable vtab {
  do_execsql_test colname-9.320 {
    CREATE TABLE t2 AS SELECT BBb FROM (SELECT aaa AS Bbb FROM t1);
    SELECT name FROM pragma_table_info('t2');
  } {Bbb}
}
do_execsql_test colname-9.330 { -- added 2019-08-10 to invalidate
  DROP TABLE IF EXISTS t1;      -- a couple assert()s that were
  CREATE TABLE t1(a);           -- added by ticket 3b44500725
  INSERT INTO t1 VALUES(17),(2),(99),(-3),(7);
  SELECT (SELECT avg(a) UNION SELECT min(a) OVER()) FROM t1;
} {17}

# Issue detected by OSSFuzz on 2017-12-24 (Christmas Eve)
# caused by check-in https://sqlite.org/src/info/6b2ff26c25
#
# Prior to being fixed, the following CREATE TABLE was dereferencing
# a NULL pointer and segfaulting.
#
Changes to test/selectA.test.
1442
1443
1444
1445
1446
1447
1448





















1449
1450
  DROP TABLE IF EXISTS t2;
  CREATE TABLE t1(a INTEGER);
  CREATE TABLE t2(b TEXT);
  INSERT INTO t2(b) VALUES('12345');
  SELECT * FROM (SELECT a FROM t1 UNION SELECT b FROM t2) WHERE a=a;
} {12345}























finish_test







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
  DROP TABLE IF EXISTS t2;
  CREATE TABLE t1(a INTEGER);
  CREATE TABLE t2(b TEXT);
  INSERT INTO t2(b) VALUES('12345');
  SELECT * FROM (SELECT a FROM t1 UNION SELECT b FROM t2) WHERE a=a;
} {12345}

# 2020-06-15 ticket 8f157e8010b22af0
#
reset_db
do_execsql_test 7.1 {
  CREATE TABLE t1(c1);     INSERT INTO t1 VALUES(12),(123),(1234),(NULL),('abc');
  CREATE TABLE t2(c2);     INSERT INTO t2 VALUES(44),(55),(123);
  CREATE TABLE t3(c3,c4);  INSERT INTO t3 VALUES(66,1),(123,2),(77,3);
  CREATE VIEW t4 AS SELECT c3 FROM t3;
  CREATE VIEW t5 AS SELECT c3 FROM t3 ORDER BY c4;
}
do_execsql_test 7.2 {
  SELECT * FROM t1, t2 WHERE c1=(SELECT 123 INTERSECT SELECT c2 FROM t4) AND c1=123;
} {123 123}
do_execsql_test 7.3 {
  SELECT * FROM t1, t2 WHERE c1=(SELECT 123 INTERSECT SELECT c2 FROM t5) AND c1=123;
} {123 123}
do_execsql_test 7.4 {
  CREATE TABLE a(b);
  CREATE VIEW c(d) AS SELECT b FROM a ORDER BY b;
  SELECT sum(d) OVER( PARTITION BY(SELECT 0 FROM c JOIN a WHERE b =(SELECT b INTERSECT SELECT d FROM c) AND b = 123)) FROM c;
} {}

finish_test
Changes to test/window1.test.
1163
1164
1165
1166
1167
1168
1169
1170



1171



1172







  13 M cc NULL JM | 
  3 C cc 1 {} | 
  4 D cc 8.25 {} | 
  12 L cc 'xyZ' L | 
  11 K cc 'xyz' K |
}

finish_test






















|
>
>
>
|
>
>
>
|
>
>
>
>
>
>
>
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
  13 M cc NULL JM | 
  3 C cc 1 {} | 
  4 D cc 8.25 {} | 
  12 L cc 'xyZ' L | 
  11 K cc 'xyz' K |
}


#-------------------------------------------------------------------------
# Test that the SQL in ticket [c8d3b9f0a75] - CVE-2020-13871 - does not
# cause a problem for this version.
#
reset_db
do_execsql_test 30.0 {
  CREATE TABLE a(b);
}

do_execsql_test 30.1 {
  SELECT(SELECT b FROM a GROUP BY b HAVING(NULL AND b IN((SELECT COUNT() OVER(ORDER BY b) = lead(b) OVER(ORDER BY 3.100000 * SUM(DISTINCT CASE WHEN b LIKE 'SM PACK' THEN b * b ELSE 0 END) / b))))) FROM a EXCEPT SELECT b FROM a ORDER BY b, b, b;
}


finish_test