SQLite

Check-in [3a5860d86f]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add the TK_IF_NULL_ROW opcode to deal with non-NULL result columns in the result set of a view or subquery on the RHS of a LEFT JOIN that gets flattened.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | left-join-view
Files: files | file ages | folders
SHA3-256: 3a5860d86fadcf924316707918bf283d26c53b1473e5e67f5cff59d18c2a7742
User & Date: drh 2017-04-14 19:03:10.224
Context
2017-04-18
11:20
Improved optimizations of views as the right operand of a LEFT JOIN. (check-in: 41c27bc0ff user: drh tags: trunk)
2017-04-14
19:03
Add the TK_IF_NULL_ROW opcode to deal with non-NULL result columns in the result set of a view or subquery on the RHS of a LEFT JOIN that gets flattened. (Closed-Leaf check-in: 3a5860d86f user: drh tags: left-join-view)
17:18
An initial attempt to optimize VIEWs that occur as the right operand of a LEFT JOIN. This particular check-in does not work correctly because it does not deal with the case of columns in the VIEW that return non-NULL even when all columns in the table of the VIEW are NULL because of the LEFT JOIN. (check-in: 1838a59c8a user: drh tags: left-join-view)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/expr.c.
3884
3885
3886
3887
3888
3889
3890











3891
3892
3893
3894
3895
3896
3897
      break;
    }

    case TK_VECTOR: {
      sqlite3ErrorMsg(pParse, "row value misused");
      break;
    }












    /*
    ** Form A:
    **   CASE x WHEN e1 THEN r1 WHEN e2 THEN r2 ... WHEN eN THEN rN ELSE y END
    **
    ** Form B:
    **   CASE WHEN e1 THEN r1 WHEN e2 THEN r2 ... WHEN eN THEN rN ELSE y END







>
>
>
>
>
>
>
>
>
>
>







3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
      break;
    }

    case TK_VECTOR: {
      sqlite3ErrorMsg(pParse, "row value misused");
      break;
    }

    case TK_IF_NULL_ROW: {
      int addrINR;
      addrINR = sqlite3VdbeAddOp1(v, OP_IfNullRow, pExpr->iTable);
      sqlite3ExprCachePush(pParse);
      inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target);
      sqlite3ExprCachePop(pParse);
      sqlite3VdbeJumpHere(v, addrINR);
      sqlite3VdbeChangeP3(v, addrINR, inReg);
      break;
    }

    /*
    ** Form A:
    **   CASE x WHEN e1 THEN r1 WHEN e2 THEN r2 ... WHEN eN THEN rN ELSE y END
    **
    ** Form B:
    **   CASE WHEN e1 THEN r1 WHEN e2 THEN r2 ... WHEN eN THEN rN ELSE y END
Changes to src/parse.y.
188
189
190
191
192
193
194

















195
196
197
198
199
200
201
    A = 0;
    sqlite3ErrorMsg(pParse, "unknown table option: %.*s", X.n, X.z);
  }
}
columnlist ::= columnlist COMMA columnname carglist.
columnlist ::= columnname carglist.
columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,&A,&Y);}


















// Define operator precedence early so that this is the first occurrence
// of the operator tokens in the grammer.  Keeping the operators together
// causes them to be assigned integer values that are close together,
// which keeps parser tables smaller.
//
// The token values assigned to these symbols is determined by the order







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
    A = 0;
    sqlite3ErrorMsg(pParse, "unknown table option: %.*s", X.n, X.z);
  }
}
columnlist ::= columnlist COMMA columnname carglist.
columnlist ::= columnname carglist.
columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,&A,&Y);}

// The following directive causes tokens ABORT, AFTER, ASC, etc. to
// fallback to ID if they will not parse as their original value.
// This obviates the need for the "id" nonterminal.
//
%fallback ID
  ABORT ACTION AFTER ANALYZE ASC ATTACH BEFORE BEGIN BY CASCADE CAST COLUMNKW
  CONFLICT DATABASE DEFERRED DESC DETACH EACH END EXCLUSIVE EXPLAIN FAIL FOR
  IGNORE IMMEDIATE INITIALLY INSTEAD LIKE_KW MATCH NO PLAN
  QUERY KEY OF OFFSET PRAGMA RAISE RECURSIVE RELEASE REPLACE RESTRICT ROW
  ROLLBACK SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL WITH WITHOUT
%ifdef SQLITE_OMIT_COMPOUND_SELECT
  EXCEPT INTERSECT UNION
%endif SQLITE_OMIT_COMPOUND_SELECT
  REINDEX RENAME CTIME_KW IF
  .
%wildcard ANY.

// Define operator precedence early so that this is the first occurrence
// of the operator tokens in the grammer.  Keeping the operators together
// causes them to be assigned integer values that are close together,
// which keeps parser tables smaller.
//
// The token values assigned to these symbols is determined by the order
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
%right BITNOT.

// An IDENTIFIER can be a generic identifier, or one of several
// keywords.  Any non-standard keyword can also be an identifier.
//
%token_class id  ID|INDEXED.

// The following directive causes tokens ABORT, AFTER, ASC, etc. to
// fallback to ID if they will not parse as their original value.
// This obviates the need for the "id" nonterminal.
//
%fallback ID
  ABORT ACTION AFTER ANALYZE ASC ATTACH BEFORE BEGIN BY CASCADE CAST COLUMNKW
  CONFLICT DATABASE DEFERRED DESC DETACH EACH END EXCLUSIVE EXPLAIN FAIL FOR
  IGNORE IMMEDIATE INITIALLY INSTEAD LIKE_KW MATCH NO PLAN
  QUERY KEY OF OFFSET PRAGMA RAISE RECURSIVE RELEASE REPLACE RESTRICT ROW
  ROLLBACK SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL WITH WITHOUT
%ifdef SQLITE_OMIT_COMPOUND_SELECT
  EXCEPT INTERSECT UNION
%endif SQLITE_OMIT_COMPOUND_SELECT
  REINDEX RENAME CTIME_KW IF
  .
%wildcard ANY.


// And "ids" is an identifer-or-string.
//
%token_class ids  ID|STRING.

// The name of a column or table can be any of the following:
//







<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<







235
236
237
238
239
240
241

















242
243
244
245
246
247
248
%right BITNOT.

// An IDENTIFIER can be a generic identifier, or one of several
// keywords.  Any non-standard keyword can also be an identifier.
//
%token_class id  ID|INDEXED.



















// And "ids" is an identifer-or-string.
//
%token_class ids  ID|STRING.

// The name of a column or table can be any of the following:
//
Changes to src/select.c.
3150
3151
3152
3153
3154
3155
3156
3157

3158
3159
3160
3161
3162
3163
3164
**
** All references to columns in table iTable are to be replaced by corresponding
** expressions in pEList.
*/
typedef struct SubstContext {
  Parse *pParse;            /* The parsing context */
  int iTable;               /* Replace references to this table */
  int iNewTable;            /* New table number, or -1 */

  ExprList *pEList;         /* Replacement expressions */
} SubstContext;

/* Forward Declarations */
static void substExprList(SubstContext*, ExprList*);
static void substSelect(SubstContext*, Select*, int);








|
>







3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
**
** All references to columns in table iTable are to be replaced by corresponding
** expressions in pEList.
*/
typedef struct SubstContext {
  Parse *pParse;            /* The parsing context */
  int iTable;               /* Replace references to this table */
  int iNewTable;            /* New table number */
  int isLeftJoin;           /* Add TK_IF_NULL_ROW opcodes on each replacement */
  ExprList *pEList;         /* Replacement expressions */
} SubstContext;

/* Forward Declarations */
static void substExprList(SubstContext*, ExprList*);
static void substSelect(SubstContext*, Select*, int);

3185
3186
3187
3188
3189
3190
3191

3192
3193
3194
3195
3196
3197







3198
3199
3200
3201
3202
3203
3204
  }
  if( pExpr->op==TK_COLUMN && pExpr->iTable==pSubst->iTable ){
    if( pExpr->iColumn<0 ){
      pExpr->op = TK_NULL;
    }else{
      Expr *pNew;
      Expr *pCopy = pSubst->pEList->a[pExpr->iColumn].pExpr;

      assert( pSubst->pEList!=0 && pExpr->iColumn<pSubst->pEList->nExpr );
      assert( pExpr->pLeft==0 && pExpr->pRight==0 );
      if( sqlite3ExprIsVector(pCopy) ){
        sqlite3VectorErrorMsg(pSubst->pParse, pCopy);
      }else{
        sqlite3 *db = pSubst->pParse->db;







        pNew = sqlite3ExprDup(db, pCopy, 0);
        if( pNew && (pExpr->flags & EP_FromJoin) ){
          pNew->iRightJoinTable = pExpr->iRightJoinTable;
          pNew->flags |= EP_FromJoin;
        }
        sqlite3ExprDelete(db, pExpr);
        pExpr = pNew;







>






>
>
>
>
>
>
>







3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
  }
  if( pExpr->op==TK_COLUMN && pExpr->iTable==pSubst->iTable ){
    if( pExpr->iColumn<0 ){
      pExpr->op = TK_NULL;
    }else{
      Expr *pNew;
      Expr *pCopy = pSubst->pEList->a[pExpr->iColumn].pExpr;
      Expr ifNullRow;
      assert( pSubst->pEList!=0 && pExpr->iColumn<pSubst->pEList->nExpr );
      assert( pExpr->pLeft==0 && pExpr->pRight==0 );
      if( sqlite3ExprIsVector(pCopy) ){
        sqlite3VectorErrorMsg(pSubst->pParse, pCopy);
      }else{
        sqlite3 *db = pSubst->pParse->db;
        if( pSubst->isLeftJoin && pCopy->op!=TK_COLUMN ){
          memset(&ifNullRow, 0, sizeof(ifNullRow));
          ifNullRow.op = TK_IF_NULL_ROW;
          ifNullRow.pLeft = pCopy;
          ifNullRow.iTable = pSubst->iNewTable;
          pCopy = &ifNullRow;
        }
        pNew = sqlite3ExprDup(db, pCopy, 0);
        if( pNew && (pExpr->flags & EP_FromJoin) ){
          pNew->iRightJoinTable = pExpr->iRightJoinTable;
          pNew->flags |= EP_FromJoin;
        }
        sqlite3ExprDelete(db, pExpr);
        pExpr = pNew;
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
**   (1)  The subquery and the outer query do not both use aggregates.
**
**   (2)  The subquery is not an aggregate or (2a) the outer query is not a join
**        and (2b) the outer query does not use subqueries other than the one
**        FROM-clause subquery that is a candidate for flattening.  (2b is
**        due to ticket [2f7170d73bf9abf80] from 2015-02-09.)
**
**   (3)  The subquery is not the right operand of a left outer join
**        or (3b) the subquery is not a join and (3c) does not contain any
**        result-set columns that could be non-NULL even if columns of the
**        subquery table are NULL.
**        to be more restrictive - disallowing subquery if it was the
**        right operand of a left join.  See tickets #306 and #3300. Relaxed
**        to allow simple subqueries as the right operand of a left join
**        on 2017-04-14.)
**
**   (4)  The subquery is not DISTINCT.
**
**  (**)  At one point restrictions (4) and (5) defined a subset of DISTINCT
**        sub-queries that were excluded from this optimization. Restriction 
**        (4) has since been expanded to exclude all DISTINCT subqueries.
**
**   (6)  The subquery does not use aggregates or the outer query is not
**        DISTINCT.
**
**   (7)  The subquery has a FROM clause.  TODO:  For subqueries without
**        A FROM clause, consider adding a FROM close with the special
**        table sqlite_once that consists of a single row containing a
**        single NULL.
**
**   (8)  The subquery does not use LIMIT or the outer query is not a join.
**
**   (9)  The subquery does not use LIMIT or the outer query does not use
**        aggregates.







|
|
<
<
<
<
<
<











|







3293
3294
3295
3296
3297
3298
3299
3300
3301






3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
**   (1)  The subquery and the outer query do not both use aggregates.
**
**   (2)  The subquery is not an aggregate or (2a) the outer query is not a join
**        and (2b) the outer query does not use subqueries other than the one
**        FROM-clause subquery that is a candidate for flattening.  (2b is
**        due to ticket [2f7170d73bf9abf80] from 2015-02-09.)
**
**   (3)  The subquery is not the right operand of a LEFT JOIN
**        or the subquery is not itself a join.






**
**   (4)  The subquery is not DISTINCT.
**
**  (**)  At one point restrictions (4) and (5) defined a subset of DISTINCT
**        sub-queries that were excluded from this optimization. Restriction 
**        (4) has since been expanded to exclude all DISTINCT subqueries.
**
**   (6)  The subquery does not use aggregates or the outer query is not
**        DISTINCT.
**
**   (7)  The subquery has a FROM clause.  TODO:  For subqueries without
**        A FROM clause, consider adding a FROM clause with the special
**        table sqlite_once that consists of a single row containing a
**        single NULL.
**
**   (8)  The subquery does not use LIMIT or the outer query is not a join.
**
**   (9)  The subquery does not use LIMIT or the outer query does not use
**        aggregates.
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
  **
  ** If we flatten the above, we would get
  **
  **         (t1 LEFT OUTER JOIN t2) JOIN t3
  **
  ** which is not at all the same thing.
  **
  ** See also tickets #306 and #3300.
  */
  if( (pSubitem->fg.jointype & JT_OUTER)!=0 ){
    isLeftJoin = 1;
    if( pSubSrc->nSrc>1 ){
      return 0; /* Restriction (3b) */
    }
    /* TBD: Deal with the case of result-set expressions that are non-NULL even
    ** when the RHS table of a LEFT JOIN is NULL. */
  }

  /* Restriction 17: If the sub-query is a compound SELECT, then it must
  ** use only the UNION ALL operator. And none of the simple select queries
  ** that make up the compound SELECT are allowed to be aggregate or distinct
  ** queries.
  */







|




|

<
<







3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503


3504
3505
3506
3507
3508
3509
3510
  **
  ** If we flatten the above, we would get
  **
  **         (t1 LEFT OUTER JOIN t2) JOIN t3
  **
  ** which is not at all the same thing.
  **
  ** See also tickets #306, #350, and #3300.
  */
  if( (pSubitem->fg.jointype & JT_OUTER)!=0 ){
    isLeftJoin = 1;
    if( pSubSrc->nSrc>1 ){
      return 0; /* Restriction (3) */
    }


  }

  /* Restriction 17: If the sub-query is a compound SELECT, then it must
  ** use only the UNION ALL operator. And none of the simple select queries
  ** that make up the compound SELECT are allowed to be aggregate or distinct
  ** queries.
  */
3770
3771
3772
3773
3774
3775
3776

3777
3778
3779
3780
3781
3782
3783
      pParent->pWhere = sqlite3ExprAnd(db, pWhere, pParent->pWhere);
    }
    if( db->mallocFailed==0 ){
      SubstContext x;
      x.pParse = pParse;
      x.iTable = iParent;
      x.iNewTable = iNewParent;

      x.pEList = pSub->pEList;
      substSelect(&x, pParent, 0);
    }
  
    /* The flattened query is distinct if either the inner or the
    ** outer query is distinct. 
    */







>







3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
      pParent->pWhere = sqlite3ExprAnd(db, pWhere, pParent->pWhere);
    }
    if( db->mallocFailed==0 ){
      SubstContext x;
      x.pParse = pParse;
      x.iTable = iParent;
      x.iNewTable = iNewParent;
      x.isLeftJoin = isLeftJoin;
      x.pEList = pSub->pEList;
      substSelect(&x, pParent, 0);
    }
  
    /* The flattened query is distinct if either the inner or the
    ** outer query is distinct. 
    */
3879
3880
3881
3882
3883
3884
3885

3886
3887
3888
3889
3890
3891
3892
    nChng++;
    while( pSubq ){
      SubstContext x;
      pNew = sqlite3ExprDup(pParse->db, pWhere, 0);
      x.pParse = pParse;
      x.iTable = iCursor;
      x.iNewTable = iCursor;

      x.pEList = pSubq->pEList;
      pNew = substExpr(&x, pNew);
      pSubq->pWhere = sqlite3ExprAnd(pParse->db, pSubq->pWhere, pNew);
      pSubq = pSubq->pPrior;
    }
  }
  return nChng;







>







3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
    nChng++;
    while( pSubq ){
      SubstContext x;
      pNew = sqlite3ExprDup(pParse->db, pWhere, 0);
      x.pParse = pParse;
      x.iTable = iCursor;
      x.iNewTable = iCursor;
      x.isLeftJoin = 0;
      x.pEList = pSubq->pEList;
      pNew = substExpr(&x, pNew);
      pSubq->pWhere = sqlite3ExprAnd(pParse->db, pSubq->pWhere, pNew);
      pSubq = pSubq->pPrior;
    }
  }
  return nChng;
Changes to src/treeview.c.
465
466
467
468
469
470
471





472
473
474
475
476
477
478
      sqlite3TreeViewBareExprList(pView, pExpr->x.pList, "VECTOR");
      break;
    }
    case TK_SELECT_COLUMN: {
      sqlite3TreeViewLine(pView, "SELECT-COLUMN %d", pExpr->iColumn);
      sqlite3TreeViewSelect(pView, pExpr->pLeft->x.pSelect, 0);
      break;





    }
    default: {
      sqlite3TreeViewLine(pView, "op=%d", pExpr->op);
      break;
    }
  }
  if( zBinOp ){







>
>
>
>
>







465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
      sqlite3TreeViewBareExprList(pView, pExpr->x.pList, "VECTOR");
      break;
    }
    case TK_SELECT_COLUMN: {
      sqlite3TreeViewLine(pView, "SELECT-COLUMN %d", pExpr->iColumn);
      sqlite3TreeViewSelect(pView, pExpr->pLeft->x.pSelect, 0);
      break;
    }
    case TK_IF_NULL_ROW: {
      sqlite3TreeViewLine(pView, "IF-NULL-ROW %d", pExpr->iTable);
      sqlite3TreeViewExpr(pView, pExpr->pLeft, 0);
      break;
    }
    default: {
      sqlite3TreeViewLine(pView, "op=%d", pExpr->op);
      break;
    }
  }
  if( zBinOp ){
Changes to src/vdbe.c.
2424
2425
2426
2427
2428
2429
2430

















2431
2432
2433
2434
2435
2436
2437
  pIn1 = &aMem[pOp->p1];
  VdbeBranchTaken( (pIn1->flags & MEM_Null)==0, 2);
  if( (pIn1->flags & MEM_Null)==0 ){
    goto jump_to_p2;
  }
  break;
}


















/* Opcode: Column P1 P2 P3 P4 P5
** Synopsis: r[P3]=PX
**
** Interpret the data that cursor P1 points to as a structure built using
** the MakeRecord instruction.  (See the MakeRecord opcode for additional
** information about the format of the data.)  Extract the P2-th column







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
  pIn1 = &aMem[pOp->p1];
  VdbeBranchTaken( (pIn1->flags & MEM_Null)==0, 2);
  if( (pIn1->flags & MEM_Null)==0 ){
    goto jump_to_p2;
  }
  break;
}

/* Opcode: IfNullRow P1 P2 P3 * *
** Synopsis: if P1.nullRow then r[P3]=NULL, goto P2
**
** Check the cursor P1 to see if it is currently pointing at a NULL row.
** If it is, then set register P3 to NULL and jump immediately to P2.
** If P1 is not on a NULL row, then fall through without making any
** changes.
*/
case OP_IfNullRow: {         /* jump */
  assert( pOp->p1>=0 && pOp->p1<p->nCursor );
  if( p->apCsr[pOp->p1]->nullRow ){
    sqlite3VdbeMemSetNull(aMem + pOp->p3);
    goto jump_to_p2;
  }
  break;
}

/* Opcode: Column P1 P2 P3 P4 P5
** Synopsis: r[P3]=PX
**
** Interpret the data that cursor P1 points to as a structure built using
** the MakeRecord instruction.  (See the MakeRecord opcode for additional
** information about the format of the data.)  Extract the P2-th column
Changes to src/where.c.
4972
4973
4974
4975
4976
4977
4978


4979
4980
4981
4982
4983
4984
4985
            pOp->p1 = pLevel->iIdxCur;
          }
          assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 || x>=0 
              || pWInfo->eOnePass );
        }else if( pOp->opcode==OP_Rowid ){
          pOp->p1 = pLevel->iIdxCur;
          pOp->opcode = OP_IdxRowid;


        }
      }
    }
  }

  /* Final cleanup
  */







>
>







4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
            pOp->p1 = pLevel->iIdxCur;
          }
          assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 || x>=0 
              || pWInfo->eOnePass );
        }else if( pOp->opcode==OP_Rowid ){
          pOp->p1 = pLevel->iIdxCur;
          pOp->opcode = OP_IdxRowid;
        }else if( pOp->opcode==OP_IfNullRow ){
          pOp->p1 = pLevel->iIdxCur;
        }
      }
    }
  }

  /* Final cleanup
  */
Changes to tool/addopcodes.tcl.
35
36
37
38
39
40
41

42
43
44
45
46
47
48
  AGG_FUNCTION
  AGG_COLUMN
  UMINUS
  UPLUS
  REGISTER
  VECTOR
  SELECT_COLUMN

  ASTERISK
  SPAN
  SPACE
  ILLEGAL
}
if {[lrange $extras end-1 end]!="SPACE ILLEGAL"} {
  error "SPACE and ILLEGAL must be the last two token codes and they\







>







35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
  AGG_FUNCTION
  AGG_COLUMN
  UMINUS
  UPLUS
  REGISTER
  VECTOR
  SELECT_COLUMN
  IF_NULL_ROW
  ASTERISK
  SPAN
  SPACE
  ILLEGAL
}
if {[lrange $extras end-1 end]!="SPACE ILLEGAL"} {
  error "SPACE and ILLEGAL must be the last two token codes and they\