SQLite

Check-in [581e3d4988]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix the cursor hint mechanism so that it does the right thing for indexed lookups.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | cursor-hints
Files: files | file ages | folders
SHA1: 581e3d4988e98975fea5daaeb9f854c54a4976b7
User & Date: drh 2015-08-14 18:50:04.420
Context
2015-08-14
20:08
Updated testing of cursor-hints. Remove the test_cursorhint.c file and associated logic in the core and do tests based purely on the newly enhanced EXPLAIN output. (check-in: bf383e665a user: drh tags: cursor-hints)
18:50
Fix the cursor hint mechanism so that it does the right thing for indexed lookups. (check-in: 581e3d4988 user: drh tags: cursor-hints)
15:05
Refactor the sqlite3BtreeCursorHint() interface for improved maintainability. (check-in: fc3fb5cd0d user: drh tags: cursor-hints)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/sqliteInt.h.
2990
2991
2992
2993
2994
2995
2996

2997
2998
2999
3000
3001
3002
3003
  u8 eCode;                                 /* A small processing code */
  union {                                   /* Extra data for callback */
    NameContext *pNC;                          /* Naming context */
    int n;                                     /* A counter */
    int iCur;                                  /* A cursor number */
    SrcList *pSrcList;                         /* FROM clause */
    struct SrcCount *pSrcCount;                /* Counting column references */

  } u;
};

/* Forward declarations */
int sqlite3WalkExpr(Walker*, Expr*);
int sqlite3WalkExprList(Walker*, ExprList*);
int sqlite3WalkSelect(Walker*, Select*);







>







2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
  u8 eCode;                                 /* A small processing code */
  union {                                   /* Extra data for callback */
    NameContext *pNC;                          /* Naming context */
    int n;                                     /* A counter */
    int iCur;                                  /* A cursor number */
    SrcList *pSrcList;                         /* FROM clause */
    struct SrcCount *pSrcCount;                /* Counting column references */
    struct CCurHint *pCCurHint;                /* Used by codeCursorHint() */
  } u;
};

/* Forward declarations */
int sqlite3WalkExpr(Walker*, Expr*);
int sqlite3WalkExprList(Walker*, ExprList*);
int sqlite3WalkSelect(Walker*, Select*);
Changes to src/vdbeaux.c.
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
    pMem->u.i = pOp->p2;                          /* P2 */
    pMem++;

    pMem->flags = MEM_Int;
    pMem->u.i = pOp->p3;                          /* P3 */
    pMem++;

    if( sqlite3VdbeMemClearAndResize(pMem, 32) ){ /* P4 */
      assert( p->db->mallocFailed );
      return SQLITE_ERROR;
    }
    pMem->flags = MEM_Str|MEM_Term;
    zP4 = displayP4(pOp, pMem->z, 32);
    if( zP4!=pMem->z ){
      sqlite3VdbeMemSetStr(pMem, zP4, -1, SQLITE_UTF8, 0);
    }else{
      assert( pMem->z!=0 );
      pMem->n = sqlite3Strlen30(pMem->z);
      pMem->enc = SQLITE_UTF8;
    }







|




|







1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
    pMem->u.i = pOp->p2;                          /* P2 */
    pMem++;

    pMem->flags = MEM_Int;
    pMem->u.i = pOp->p3;                          /* P3 */
    pMem++;

    if( sqlite3VdbeMemClearAndResize(pMem, 100) ){ /* P4 */
      assert( p->db->mallocFailed );
      return SQLITE_ERROR;
    }
    pMem->flags = MEM_Str|MEM_Term;
    zP4 = displayP4(pOp, pMem->z, pMem->szMalloc);
    if( zP4!=pMem->z ){
      sqlite3VdbeMemSetStr(pMem, zP4, -1, SQLITE_UTF8, 0);
    }else{
      assert( pMem->z!=0 );
      pMem->n = sqlite3Strlen30(pMem->z);
      pMem->enc = SQLITE_UTF8;
    }
Changes to src/where.c.
4213
4214
4215
4216
4217
4218
4219



4220
4221
4222
4223
4224
4225
4226
        Bitmask b = pTabItem->colUsed;
        int n = 0;
        for(; b; b=b>>1, n++){}
        sqlite3VdbeChangeP4(v, sqlite3VdbeCurrentAddr(v)-1, 
                            SQLITE_INT_TO_PTR(n), P4_INT32);
        assert( n<=pTab->nCol );
      }



#ifdef SQLITE_ENABLE_COLUMN_USED_MASK
      sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, pTabItem->iCursor, 0, 0,
                            (const u8*)&pTabItem->colUsed, P4_INT64);
#endif
    }else{
      sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName);
    }







>
>
>







4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
        Bitmask b = pTabItem->colUsed;
        int n = 0;
        for(; b; b=b>>1, n++){}
        sqlite3VdbeChangeP4(v, sqlite3VdbeCurrentAddr(v)-1, 
                            SQLITE_INT_TO_PTR(n), P4_INT32);
        assert( n<=pTab->nCol );
      }
#ifdef SQLITE_ENABLE_CURSOR_HINTS
      if( pLoop->u.btree.pIndex!=0 ) sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ);
#endif
#ifdef SQLITE_ENABLE_COLUMN_USED_MASK
      sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, pTabItem->iCursor, 0, 0,
                            (const u8*)&pTabItem->colUsed, P4_INT64);
#endif
    }else{
      sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName);
    }
Changes to src/wherecode.c.
584
585
586
587
588
589
590










591
592



















593
594
595

596
597
598
599
600
601





602
603
604

605

606
607
608
609
610
611
612





613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637

638
639
640
641


642
643
644
645






646
647
648
649
650
651
652






653
654
655
656
657
658
659
660
661
662
663


664
665
666
667
668
669
670
            || pTerm->pWC->pWInfo->pParse->db->mallocFailed );
    pOp->p3 = pLevel->iLikeRepCntr;
    pOp->p5 = 1;
  }
}

#ifdef SQLITE_ENABLE_CURSOR_HINTS











/*



















** This function is called on every node of an expression tree used as an
** argument to the OP_CursorHint instruction. If the node is a TK_COLUMN
** that accesses any cursor other than (pWalker->u.n), do the following:

**
**   1) allocate a register and code an OP_Column instruction to read 
**      the specified column into the new register, and
**
**   2) transform the expression node to a TK_REGISTER node that reads 
**      from the newly populated register.





*/
static int codeCursorHintFixExpr(Walker *pWalker, Expr *pExpr){
  int rc = WRC_Continue;

  if( pExpr->op==TK_COLUMN && pExpr->iTable!=pWalker->u.n ){

    Vdbe *v = pWalker->pParse->pVdbe;
    int reg = ++pWalker->pParse->nMem;   /* Register for column value */
    sqlite3ExprCodeGetColumnOfTable(
        v, pExpr->pTab, pExpr->iTable, pExpr->iColumn, reg
    );
    pExpr->op = TK_REGISTER;
    pExpr->iTable = reg;





  }else if( pExpr->op==TK_AGG_FUNCTION ){
    /* An aggregate function in the WHERE clause of a query means this must
    ** be a correlated sub-query, and expression pExpr is an aggregate from
    ** the parent context. Do not walk the function arguments in this case.
    **
    ** todo: It should be possible to replace this node with a TK_REGISTER
    ** expression, as the result of the expression must be stored in a 
    ** register at this point. The same holds for TK_AGG_COLUMN nodes. */
    rc = WRC_Prune;
  }
  return rc;
}

/*
** Insert an OP_CursorHint instruction if it is appropriate to do so.
*/
static void codeCursorHint(
  WhereInfo *pWInfo,
  int iLevel
){
  Parse *pParse = pWInfo->pParse;
  sqlite3 *db = pParse->db;
  Vdbe *v = pParse->pVdbe;
  WhereLevel *pLevel;
  Expr *pExpr = 0;

  int iCur;
  WhereClause *pWC;
  WhereTerm *pTerm;
  int i;



  if( OptimizationDisabled(db, SQLITE_CursorHints) ) return;
  pLevel = &pWInfo->a[iLevel];
  iCur = pWInfo->pTabList->a[pLevel->iFrom].iCursor;






  pWC = &pWInfo->sWC;
  for(i=0; i<pWC->nTerm; i++){
    pTerm = &pWC->a[i];
    if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue;
    if( pTerm->prereqAll & pLevel->notReady ) continue;
    if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) continue;
    if( sqlite3ExprContainsSubquery(pTerm->pExpr) ) continue;






    pExpr = sqlite3ExprAnd(db, pExpr, sqlite3ExprDup(db, pTerm->pExpr, 0));
  }
  if( pExpr!=0 ){
    const char *a = (const char*)pExpr;
    Walker sWalker;
    memset(&sWalker, 0, sizeof(sWalker));
    sWalker.xExprCallback = codeCursorHintFixExpr;
    sWalker.pParse = pParse;
    sWalker.u.n = pLevel->iTabCur;
    sqlite3WalkExpr(&sWalker, pExpr);
    sqlite3VdbeAddOp4(v, OP_CursorHint, pLevel->iTabCur, iCur, 0, a, P4_EXPR);


  }
}
#else
# define codeCursorHint(A,B)  /* No-op */
#endif /* SQLITE_ENABLE_CURSOR_HINTS */

/*







>
>
>
>
>
>
>
>
>
>


>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


|
>






>
>
>
>
>



>
|
>
|
|
|
|
|
|
|
>
>
>
>
>


















|




<

>




>
>


|
|
>
>
>
>
>
>







>
>
>
>
>
>



<
<
<

<
<

|
>
>







584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677

678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711



712


713
714
715
716
717
718
719
720
721
722
723
            || pTerm->pWC->pWInfo->pParse->db->mallocFailed );
    pOp->p3 = pLevel->iLikeRepCntr;
    pOp->p5 = 1;
  }
}

#ifdef SQLITE_ENABLE_CURSOR_HINTS
/*
** Information is passed from codeCursorHint() down to individual nodes of
** the expression tree (by sqlite3WalkExpr()) using an instance of this
** structure.
*/
struct CCurHint {
  int iTabCur;    /* Cursor for the main table */
  int iIdxCur;    /* Cursor for the index, if pIdx!=0.  Unused otherwise */
  Index *pIdx;    /* The index used to access the table */
};

/*
** This function is called for every node of an expression that is a candidate
** for a cursor hint on an index cursor.  For TK_COLUMN nodes that reference
** the table CCurHint.iTabCur, verify that the same column can be
** accessed through the index.  If it cannot, then set pWalker->eCode to 1.
*/
static int codeCursorHintCheckExpr(Walker *pWalker, Expr *pExpr){
  struct CCurHint *pHint = pWalker->u.pCCurHint;
  assert( pHint->pIdx!=0 );
  if( pExpr->op==TK_COLUMN
   && pExpr->iTable==pHint->iTabCur
   && sqlite3ColumnOfIndex(pHint->pIdx, pExpr->iColumn)<0
  ){
    pWalker->eCode = 1;
  }
  return WRC_Continue;
}


/*
** This function is called on every node of an expression tree used as an
** argument to the OP_CursorHint instruction. If the node is a TK_COLUMN
** that accesses any table other than the one identified by
** CCurHint.iTabCur, then do the following:
**
**   1) allocate a register and code an OP_Column instruction to read 
**      the specified column into the new register, and
**
**   2) transform the expression node to a TK_REGISTER node that reads 
**      from the newly populated register.
**
** Also, if the node is a TK_COLUMN that does access the table idenified
** by pCCurHint.iTabCur, and an index is being used (which we will
** know because CCurHint.pIdx!=0) then transform the TK_COLUMN into
** an access of the index rather than the original table.
*/
static int codeCursorHintFixExpr(Walker *pWalker, Expr *pExpr){
  int rc = WRC_Continue;
  struct CCurHint *pHint = pWalker->u.pCCurHint;
  if( pExpr->op==TK_COLUMN ){
    if( pExpr->iTable!=pHint->iTabCur ){
      Vdbe *v = pWalker->pParse->pVdbe;
      int reg = ++pWalker->pParse->nMem;   /* Register for column value */
      sqlite3ExprCodeGetColumnOfTable(
          v, pExpr->pTab, pExpr->iTable, pExpr->iColumn, reg
      );
      pExpr->op = TK_REGISTER;
      pExpr->iTable = reg;
    }else if( pHint->pIdx!=0 ){
      pExpr->iTable = pHint->iIdxCur;
      pExpr->iColumn = sqlite3ColumnOfIndex(pHint->pIdx, pExpr->iColumn);
      assert( pExpr->iColumn>=0 );
    }
  }else if( pExpr->op==TK_AGG_FUNCTION ){
    /* An aggregate function in the WHERE clause of a query means this must
    ** be a correlated sub-query, and expression pExpr is an aggregate from
    ** the parent context. Do not walk the function arguments in this case.
    **
    ** todo: It should be possible to replace this node with a TK_REGISTER
    ** expression, as the result of the expression must be stored in a 
    ** register at this point. The same holds for TK_AGG_COLUMN nodes. */
    rc = WRC_Prune;
  }
  return rc;
}

/*
** Insert an OP_CursorHint instruction if it is appropriate to do so.
*/
static void codeCursorHint(
  WhereInfo *pWInfo,
  WhereLevel *pLevel
){
  Parse *pParse = pWInfo->pParse;
  sqlite3 *db = pParse->db;
  Vdbe *v = pParse->pVdbe;

  Expr *pExpr = 0;
  WhereLoop *pLoop = pLevel->pWLoop;
  int iCur;
  WhereClause *pWC;
  WhereTerm *pTerm;
  int i;
  struct CCurHint sHint;
  Walker sWalker;

  if( OptimizationDisabled(db, SQLITE_CursorHints) ) return;
  iCur = pLevel->iTabCur;
  assert( iCur==pWInfo->pTabList->a[pLevel->iFrom].iCursor );
  sHint.iTabCur = iCur;
  sHint.iIdxCur = pLevel->iIdxCur;
  sHint.pIdx = pLoop->u.btree.pIndex;
  memset(&sWalker, 0, sizeof(sWalker));
  sWalker.pParse = pParse;
  sWalker.u.pCCurHint = &sHint;
  pWC = &pWInfo->sWC;
  for(i=0; i<pWC->nTerm; i++){
    pTerm = &pWC->a[i];
    if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue;
    if( pTerm->prereqAll & pLevel->notReady ) continue;
    if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) continue;
    if( sqlite3ExprContainsSubquery(pTerm->pExpr) ) continue;
    if( sHint.pIdx!=0 ){
      sWalker.eCode = 0;
      sWalker.xExprCallback = codeCursorHintCheckExpr;
      sqlite3WalkExpr(&sWalker, pTerm->pExpr);
      if( sWalker.eCode ) continue;
    }
    pExpr = sqlite3ExprAnd(db, pExpr, sqlite3ExprDup(db, pTerm->pExpr, 0));
  }
  if( pExpr!=0 ){



    sWalker.xExprCallback = codeCursorHintFixExpr;


    sqlite3WalkExpr(&sWalker, pExpr);
    sqlite3VdbeAddOp4(v, OP_CursorHint, 
                      (sHint.pIdx ? sHint.iIdxCur : sHint.iTabCur), 0, 0,
                      (const char*)pExpr, P4_EXPR);
  }
}
#else
# define codeCursorHint(A,B)  /* No-op */
#endif /* SQLITE_ENABLE_CURSOR_HINTS */

/*
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
    if( pLoop->wsFlags & WHERE_TOP_LIMIT ) pEnd = pLoop->aLTerm[j++];
    assert( pStart!=0 || pEnd!=0 );
    if( bRev ){
      pTerm = pStart;
      pStart = pEnd;
      pEnd = pTerm;
    }
    codeCursorHint(pWInfo, iLevel);
    if( pStart ){
      Expr *pX;             /* The expression that defines the start bound */
      int r1, rTemp;        /* Registers for holding the start boundary */

      /* The following constant maps TK_xx codes into corresponding 
      ** seek opcodes.  It depends on a particular ordering of TK_xx
      */







|







879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
    if( pLoop->wsFlags & WHERE_TOP_LIMIT ) pEnd = pLoop->aLTerm[j++];
    assert( pStart!=0 || pEnd!=0 );
    if( bRev ){
      pTerm = pStart;
      pStart = pEnd;
      pEnd = pTerm;
    }
    codeCursorHint(pWInfo, pLevel);
    if( pStart ){
      Expr *pX;             /* The expression that defines the start bound */
      int r1, rTemp;        /* Registers for holding the start boundary */

      /* The following constant maps TK_xx codes into corresponding 
      ** seek opcodes.  It depends on a particular ordering of TK_xx
      */
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
    testcase( pRangeEnd && (pRangeEnd->eOperator & WO_LE)!=0 );
    testcase( pRangeEnd && (pRangeEnd->eOperator & WO_GE)!=0 );
    startEq = !pRangeStart || pRangeStart->eOperator & (WO_LE|WO_GE);
    endEq =   !pRangeEnd || pRangeEnd->eOperator & (WO_LE|WO_GE);
    start_constraints = pRangeStart || nEq>0;

    /* Seek the index cursor to the start of the range. */
    codeCursorHint(pWInfo, iLevel);
    nConstraint = nEq;
    if( pRangeStart ){
      Expr *pRight = pRangeStart->pExpr->pRight;
      sqlite3ExprCode(pParse, pRight, regBase+nEq);
      whereLikeOptimizationStringFixup(v, pLevel, pRangeStart);
      if( (pRangeStart->wtFlags & TERM_VNULL)==0
       && sqlite3ExprCanBeNull(pRight)







|







1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
    testcase( pRangeEnd && (pRangeEnd->eOperator & WO_LE)!=0 );
    testcase( pRangeEnd && (pRangeEnd->eOperator & WO_GE)!=0 );
    startEq = !pRangeStart || pRangeStart->eOperator & (WO_LE|WO_GE);
    endEq =   !pRangeEnd || pRangeEnd->eOperator & (WO_LE|WO_GE);
    start_constraints = pRangeStart || nEq>0;

    /* Seek the index cursor to the start of the range. */
    codeCursorHint(pWInfo, pLevel);
    nConstraint = nEq;
    if( pRangeStart ){
      Expr *pRight = pRangeStart->pExpr->pRight;
      sqlite3ExprCode(pParse, pRight, regBase+nEq);
      whereLikeOptimizationStringFixup(v, pLevel, pRangeStart);
      if( (pRangeStart->wtFlags & TERM_VNULL)==0
       && sqlite3ExprCanBeNull(pRight)
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
    static const u8 aStart[] = { OP_Rewind, OP_Last };
    assert( bRev==0 || bRev==1 );
    if( pTabItem->isRecursive ){
      /* Tables marked isRecursive have only a single row that is stored in
      ** a pseudo-cursor.  No need to Rewind or Next such cursors. */
      pLevel->op = OP_Noop;
    }else{
      codeCursorHint(pWInfo, iLevel);
      pLevel->op = aStep[bRev];
      pLevel->p1 = iCur;
      pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk);
      VdbeCoverageIf(v, bRev==0);
      VdbeCoverageIf(v, bRev!=0);
      pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP;
    }







|







1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
    static const u8 aStart[] = { OP_Rewind, OP_Last };
    assert( bRev==0 || bRev==1 );
    if( pTabItem->isRecursive ){
      /* Tables marked isRecursive have only a single row that is stored in
      ** a pseudo-cursor.  No need to Rewind or Next such cursors. */
      pLevel->op = OP_Noop;
    }else{
      codeCursorHint(pWInfo, pLevel);
      pLevel->op = aStep[bRev];
      pLevel->p1 = iCur;
      pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk);
      VdbeCoverageIf(v, bRev==0);
      VdbeCoverageIf(v, bRev!=0);
      pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP;
    }