SQLite

Changes On Branch vtab-IN-opt
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Changes In Branch vtab-IN-opt Excluding Merge-Ins

This is equivalent to a diff from 3c15a9bf to 5893e972

2016-03-03
20:42
Allow the left-hand side of IN operators on virtual tables to have the aConstraintUsage[].omit flag clear. (check-in: 3eb74342 user: drh tags: trunk)
17:54
Merge 3.11.1 changes into trunk. (check-in: 7abc2dd9 user: drh tags: trunk)
2016-03-02
17:57
Add new test script bestindex1.test to the utf16 permutation. (Closed-Leaf check-in: 5893e972 user: dan tags: vtab-IN-opt)
16:13
Add an extra test for the change on this branch. (check-in: d2d28251 user: dan tags: vtab-IN-opt)
16:01
Enhance test_bestindex.c so that it can be used to test plans generated by xBestIndex. (check-in: 3c15a9bf user: dan tags: trunk)
00:58
Add the optional -DSERIES_OMIT_CONSTRAINT_VERIFY=0 option to the series.c extension that implements the generate_series() virtual table. (check-in: 3d9daa92 user: drh tags: trunk)

Changes to Makefile.in.

1046
1047
1048
1049
1050
1051
1052

1053
1054
1055
1056
1057
1058
1059
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060







+







# fixture.  Otherwise link against libsqlite3.la.  (This distinction is
# necessary because the test fixture requires non-API symbols which are
# hidden when the library is built via the amalgamation).
#
TESTFIXTURE_FLAGS  = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE 
TESTFIXTURE_FLAGS += -DBUILD_sqlite
TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1

TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la
TESTFIXTURE_SRC1 = sqlite3.c
TESTFIXTURE_SRC = $(TESTSRC) $(TOP)/src/tclsqlite.c
TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION))

testfixture$(TEXE):	$(TESTFIXTURE_SRC)

Changes to Makefile.msc.

1921
1922
1923
1924
1925
1926
1927

1928
1929
1930
1931
1932
1933
1934
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935







+







# fixture.  Otherwise link against libsqlite3.lib.  (This distinction is
# necessary because the test fixture requires non-API symbols which are
# hidden when the library is built via the amalgamation).
#
TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE=""
TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CORE $(NO_WARN)
TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1

TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2)
TESTFIXTURE_SRC1 = $(TESTEXT) $(SQLITE3C)
!IF $(USE_AMALGAMATION)==0
TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0)
!ELSE
TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC1)

Changes to ext/misc/series.c.

213
214
215
216
217
218
219
220
221


222
223
224
225


226
227
228
229
230
231
232
213
214
215
216
217
218
219


220
221
222
223


224
225
226
227
228
229
230
231
232







-
-
+
+


-
-
+
+







  if( pCur->isDesc ){
    return pCur->iValue < pCur->mnValue;
  }else{
    return pCur->iValue > pCur->mxValue;
  }
}

/* True to omit run-time checking of the start=, stop=, and/or step= 
** parameters.  The only reason to not omit these is for testing the
/* True to cause run-time checking of the start=, stop=, and/or step= 
** parameters.  The only reason to do this is for testing the
** constraint checking logic for virtual tables in the SQLite core.
*/
#ifndef SERIES_OMIT_CONSTRAINT_VERIFY
# define SERIES_OMIT_CONSTRAINT_VERIFY 1
#ifndef SQLITE_SERIES_CONSTRAINT_VERIFY
# define SQLITE_SERIES_CONSTRAINT_VERIFY 0
#endif

/*
** This method is called to "rewind" the series_cursor object back
** to the first row of output.  This method is always called at least
** once prior to any call to seriesColumn() or seriesRowid() or 
** seriesEof().
328
329
330
331
332
333
334
335

336
337
338
339

340
341
342
343

344
345
346
347
348
349
350
328
329
330
331
332
333
334

335
336
337
338

339
340
341
342

343
344
345
346
347
348
349
350







-
+



-
+



-
+







        stepIdx = i;
        idxNum |= 4;
        break;
    }
  }
  if( startIdx>=0 ){
    pIdxInfo->aConstraintUsage[startIdx].argvIndex = ++nArg;
    pIdxInfo->aConstraintUsage[startIdx].omit = SERIES_OMIT_CONSTRAINT_VERIFY;
    pIdxInfo->aConstraintUsage[startIdx].omit= !SQLITE_SERIES_CONSTRAINT_VERIFY;
  }
  if( stopIdx>=0 ){
    pIdxInfo->aConstraintUsage[stopIdx].argvIndex = ++nArg;
    pIdxInfo->aConstraintUsage[stopIdx].omit = SERIES_OMIT_CONSTRAINT_VERIFY;
    pIdxInfo->aConstraintUsage[stopIdx].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY;
  }
  if( stepIdx>=0 ){
    pIdxInfo->aConstraintUsage[stepIdx].argvIndex = ++nArg;
    pIdxInfo->aConstraintUsage[stepIdx].omit = SERIES_OMIT_CONSTRAINT_VERIFY;
    pIdxInfo->aConstraintUsage[stepIdx].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY;
  }
  if( (idxNum & 3)==3 ){
    /* Both start= and stop= boundaries are available.  This is the 
    ** the preferred case */
    pIdxInfo->estimatedCost = (double)(2 - ((idxNum&4)!=0));
    pIdxInfo->estimatedRows = 1000;
    if( pIdxInfo->nOrderBy==1 ){

Changes to main.mk.

722
723
724
725
726
727
728
729


730
731
732
733
734
735
736
722
723
724
725
726
727
728

729
730
731
732
733
734
735
736
737







-
+
+








sqlite3_analyzer$(EXE): sqlite3_analyzer.c
	$(TCCX) $(TCL_FLAGS) sqlite3_analyzer.c -o $@ $(LIBTCL) $(THREADLIB) 

# Rules to build the 'testfixture' application.
#
TESTFIXTURE_FLAGS  = -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE 
TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE
TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1

testfixture$(EXE): $(TESTSRC2) libsqlite3.a $(TESTSRC) $(TOP)/src/tclsqlite.c
	$(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS)                  \
		$(TESTSRC) $(TESTSRC2) $(TOP)/src/tclsqlite.c                \
		-o testfixture$(EXE) $(LIBTCL) libsqlite3.a $(THREADLIB)

amalgamation-testfixture$(EXE): sqlite3.c $(TESTSRC) $(TOP)/src/tclsqlite.c

Changes to src/where.c.

2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2894
2895
2896
2897
2898
2899
2900







2901
2902
2903
2904
2905
2906
2907







-
-
-
-
-
-
-







        assert( iTerm<pNew->nLSlot );
        pNew->aLTerm[iTerm] = pTerm;
        if( iTerm>mxTerm ) mxTerm = iTerm;
        testcase( iTerm==15 );
        testcase( iTerm==16 );
        if( iTerm<16 && pUsage[i].omit ) pNew->u.vtab.omitMask |= 1<<iTerm;
        if( (pTerm->eOperator & WO_IN)!=0 ){
          if( pUsage[i].omit==0 ){
            /* Do not attempt to use an IN constraint if the virtual table
            ** says that the equivalent EQ constraint cannot be safely omitted.
            ** If we do attempt to use such a constraint, some rows might be
            ** repeated in the output. */
            break;
          }
          /* A virtual table that is constrained by an IN clause may not
          ** consume the ORDER BY clause because (1) the order of IN terms
          ** is not necessarily related to the order of output terms and
          ** (2) Multiple outputs from a single IN value will not merge
          ** together.  */
          pIdxInfo->orderByConsumed = 0;
          pIdxInfo->idxFlags &= ~SQLITE_INDEX_SCAN_UNIQUE;

Changes to src/wherecode.c.

870
871
872
873
874
875
876

877
878
879
880
881
882
883
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884







+







  if(  (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){
    /* Case 1:  The table is a virtual-table.  Use the VFilter and VNext
    **          to access the data.
    */
    int iReg;   /* P3 Value for OP_VFilter */
    int addrNotFound;
    int nConstraint = pLoop->nLTerm;
    int iIn;    /* Counter for IN constraints */

    sqlite3ExprCachePush(pParse);
    iReg = sqlite3GetTempRange(pParse, nConstraint+2);
    addrNotFound = pLevel->addrBrk;
    for(j=0; j<nConstraint; j++){
      int iTarget = iReg+j+2;
      pTerm = pLoop->aLTerm[j];
892
893
894
895
896
897
898




899
900
901
902
903
904
905
906






































907
908
909
910
911
912
913
893
894
895
896
897
898
899
900
901
902
903








904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948







+
+
+
+
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+







    sqlite3VdbeAddOp2(v, OP_Integer, pLoop->u.vtab.idxNum, iReg);
    sqlite3VdbeAddOp2(v, OP_Integer, nConstraint, iReg+1);
    sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrNotFound, iReg,
                      pLoop->u.vtab.idxStr,
                      pLoop->u.vtab.needFree ? P4_MPRINTF : P4_STATIC);
    VdbeCoverage(v);
    pLoop->u.vtab.needFree = 0;
    pLevel->p1 = iCur;
    pLevel->op = pWInfo->eOnePass ? OP_Noop : OP_VNext;
    pLevel->p2 = sqlite3VdbeCurrentAddr(v);
    iIn = pLevel->u.in.nIn;
    for(j=0; j<nConstraint && j<16; j++){
      if( (pLoop->u.vtab.omitMask>>j)&1 ){
        disableTerm(pLevel, pLoop->aLTerm[j]);
      }
    }
    pLevel->p1 = iCur;
    pLevel->op = pWInfo->eOnePass ? OP_Noop : OP_VNext;
    pLevel->p2 = sqlite3VdbeCurrentAddr(v);
    for(j=nConstraint-1; j>=0; j--){
      pTerm = pLoop->aLTerm[j];
      if( j<16 && (pLoop->u.vtab.omitMask>>j)&1 ){
        disableTerm(pLevel, pTerm);
      }else if( (pTerm->eOperator & WO_IN)!=0 ){
        Expr *pCompare;  /* The comparison operator */
        Expr *pRight;    /* RHS of the comparison */
        VdbeOp *pOp;     /* Opcode to access the value of the IN constraint */

        /* Reload the constraint value into reg[iReg+j+2].  The same value
        ** was loaded into the same register prior to the OP_VFilter, but
        ** the xFilter implementation might have changed the datatype or
        ** encoding of the value in the register, so it *must* be reloaded. */
        assert( pLevel->u.in.aInLoop!=0 || db->mallocFailed );
        if( pLevel->u.in.aInLoop!=0 ){
          assert( iIn>0 );
          pOp = sqlite3VdbeGetOp(v, pLevel->u.in.aInLoop[--iIn].addrInTop);
          assert( pOp->opcode==OP_Column || pOp->opcode==OP_Rowid );
          assert( pOp->opcode!=OP_Column || pOp->p3==iReg+j+2 );
          assert( pOp->opcode!=OP_Rowid || pOp->p2==iReg+j+2 );
          testcase( pOp->opcode==OP_Rowid );
          sqlite3VdbeAddOp3(v, pOp->opcode, pOp->p1, pOp->p2, pOp->p3);
        }

        /* Generate code that will continue to the next row if 
        ** the IN constraint is not satisfied */
        pCompare = sqlite3PExpr(pParse, TK_EQ, 0, 0, 0);
        assert( pCompare!=0 || db->mallocFailed );
        if( pCompare ){
          pCompare->pLeft = pTerm->pExpr->pLeft;
          pCompare->pRight = pRight = sqlite3Expr(db, TK_REGISTER, 0);
          if( pRight ) pRight->iTable = iReg+j+2;
          sqlite3ExprIfFalse(pParse, pCompare, pLevel->addrCont, 0);
          pCompare->pLeft = 0;
          sqlite3ExprDelete(db, pCompare);
        }
      }
    }
    sqlite3ReleaseTempRange(pParse, iReg, nConstraint+2);
    sqlite3ExprCachePop(pParse);
  }else
#endif /* SQLITE_OMIT_VIRTUALTABLE */

  if( (pLoop->wsFlags & WHERE_IPK)!=0
   && (pLoop->wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_EQ))!=0

Changes to test/bestindex1.test.

135
136
137
138
139
140
141
142
143





144
145
146






147
148
149
150
151

152
153
154
155
156
135
136
137
138
139
140
141


142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159

160
161
162
163
164
165







-
-
+
+
+
+
+



+
+
+
+
+
+




-
+





  do_execsql_test 2.2.$mode.3 {SELECT rowid FROM t1} {1 2 3 4}
  do_execsql_test 2.2.$mode.4 {SELECT rowid FROM t1 WHERE a='two'} {2} 

  do_execsql_test 2.2.$mode.5 {
    SELECT rowid FROM t1 WHERE a IN ('one', 'four') ORDER BY +rowid
  } {1 4} 

  set plan(use) {0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:SELECT * FROM t1x}}
  set plan(use2) {0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:SELECT * FROM t1x}}
  set plan(use) {
    0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:SELECT * FROM t1x WHERE a='%1%'}
    0 0 0 {EXECUTE LIST SUBQUERY 1}
    0 0 0 {USE TEMP B-TREE FOR ORDER BY}
  }
  set plan(omit) {
    0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:SELECT * FROM t1x WHERE a='%1%'}
    0 0 0 {EXECUTE LIST SUBQUERY 1}
    0 0 0 {USE TEMP B-TREE FOR ORDER BY}
  }
  set plan(use2) {
    0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:SELECT * FROM t1x}
    0 0 0 {EXECUTE LIST SUBQUERY 1}
    0 0 0 {USE TEMP B-TREE FOR ORDER BY}
  }

  do_eqp_test 2.2.$mode.6 { 
    SELECT rowid FROM t1 WHERE a IN ('one', 'four') ORDER BY +rowid
  } [concat $plan($mode) {0 0 0 {USE TEMP B-TREE FOR ORDER BY}}]
  } $plan($mode)
}

finish_test


Changes to test/permutations.test.

608
609
610
611
612
613
614

615
616
617
618
619
620
621
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622







+







    memdb.test minmax.test misc1.test misc2.test misc3.test notnull.test
    null.test progress.test quote.test rowid.test select1.test select2.test
    select3.test select4.test select5.test select6.test sort.test 
    subselect.test tableapi.test table.test temptable.test
    trace.test trigger1.test trigger2.test trigger3.test
    trigger4.test types2.test types.test unique.test update.test
    vacuum.test view.test where.test
    bestindex1.test
}

# Run some tests in exclusive locking mode.
#
test_suite "exclusive" -description {
  Run tests in exclusive locking mode.
} -presql {

Changes to test/tabfunc01.test.

115
116
117
118
119
120
121















122
123
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138







+
+
+
+
+
+
+
+
+
+
+
+
+
+
+


  SELECT * FROM temp.generate_series(1,4)
} {1 {no such table: temp.generate_series}}
do_catchsql_test tabfunc01-4.3 {
  ATTACH ':memory:' AS aux1;
  CREATE TABLE aux1.t1(a,b,c);
  SELECT * FROM aux1.generate_series(1,4)
} {1 {no such table: aux1.generate_series}}

# The next series of tests is verifying that virtual table are able
# to optimize the IN operator, even on terms that are not marked "omit".
# When the generate_series virtual table is compiled for the testfixture,
# the special -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 option is used, which
# causes the xBestIndex method of generate_series to leave the
# sqlite3_index_constraint_usage.omit flag set to 0, which should cause
# the SQLite core to verify the start=, stop=, and step= constraints on
# each step of output.  At one point, the IN operator could not be used
# by virtual tables unless omit was set.
#
do_execsql_test tabfunc01-500 {
  SELECT * FROM generate_series WHERE start IN (1,7) AND stop=20 AND step=10
  ORDER BY +1;
} {1 7 11 17}

finish_test