Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Forward port the skip-ahead-distinct branch which was abandoned for some reason that I do not recall. This port should have been achived by a merge of trunk into the previous head of skip-ahead-distinct, but that did not work. So I had to manually "rebase" the changes. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | skip-ahead-distinct |
Files: | files | file ages | folders |
SHA3-256: |
132339a1fb0b9664df4d3eefbed6e77e |
User & Date: | drh 2017-04-13 01:19:30.439 |
Context
2017-04-13
| ||
13:01 | Only use the skip-ahead-distinct optimization if the index has been analyzed and we know that a skip-head is likely to skip over at least 11 rows. The magic number 11 was determined by experimentation. (check-in: 0cf16decd5 user: drh tags: skip-ahead-distinct) | |
01:19 | Forward port the skip-ahead-distinct branch which was abandoned for some reason that I do not recall. This port should have been achived by a merge of trunk into the previous head of skip-ahead-distinct, but that did not work. So I had to manually "rebase" the changes. (check-in: 132339a1fb user: drh tags: skip-ahead-distinct) | |
00:12 | Fix a regression caused by the fix for ticket [6c9b5514077fed34551f98e64c09a1] - control characters allowed in JSON. (check-in: 8e7b611863 user: drh tags: trunk) | |
Changes
Changes to src/select.c.
︙ | ︙ | |||
1686 1687 1688 1689 1690 1691 1692 | }else{ Expr *pColExpr = p; /* The expression that is the result column name */ Table *pTab; /* Table associated with this expression */ while( pColExpr->op==TK_DOT ){ pColExpr = pColExpr->pRight; assert( pColExpr!=0 ); } | | | 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 | }else{ Expr *pColExpr = p; /* The expression that is the result column name */ Table *pTab; /* Table associated with this expression */ while( pColExpr->op==TK_DOT ){ pColExpr = pColExpr->pRight; assert( pColExpr!=0 ); } if( pColExpr->op==TK_COLUMN && pColExpr->pTab!=0 ){ /* For columns use the column name name */ int iCol = pColExpr->iColumn; pTab = pColExpr->pTab; if( iCol<0 ) iCol = pTab->iPKey; zName = iCol>=0 ? pTab->aCol[iCol].zName : "rowid"; }else if( pColExpr->op==TK_ID ){ assert( !ExprHasProperty(pColExpr, EP_IntValue) ); |
︙ | ︙ |
Changes to src/sqliteInt.h.
︙ | ︙ | |||
1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 | #define SQLITE_DistinctOpt 0x0020 /* DISTINCT using indexes */ #define SQLITE_CoverIdxScan 0x0040 /* Covering index scans */ #define SQLITE_OrderByIdxJoin 0x0080 /* ORDER BY of joins via index */ #define SQLITE_SubqCoroutine 0x0100 /* Evaluate subqueries as coroutines */ #define SQLITE_Transitive 0x0200 /* Transitive constraints */ #define SQLITE_OmitNoopJoin 0x0400 /* Omit unused tables in joins */ #define SQLITE_Stat34 0x0800 /* Use STAT3 or STAT4 data */ #define SQLITE_CursorHints 0x2000 /* Add OP_CursorHint opcodes */ #define SQLITE_AllOpts 0xffff /* All optimizations */ /* ** Macros for testing whether or not optimizations are enabled or disabled. */ #define OptimizationDisabled(db, mask) (((db)->dbOptFlags&(mask))!=0) | > | 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 | #define SQLITE_DistinctOpt 0x0020 /* DISTINCT using indexes */ #define SQLITE_CoverIdxScan 0x0040 /* Covering index scans */ #define SQLITE_OrderByIdxJoin 0x0080 /* ORDER BY of joins via index */ #define SQLITE_SubqCoroutine 0x0100 /* Evaluate subqueries as coroutines */ #define SQLITE_Transitive 0x0200 /* Transitive constraints */ #define SQLITE_OmitNoopJoin 0x0400 /* Omit unused tables in joins */ #define SQLITE_Stat34 0x0800 /* Use STAT3 or STAT4 data */ #define SQLITE_SkipAhead 0x1000 /* Skip ahead on DISTINCT */ #define SQLITE_CursorHints 0x2000 /* Add OP_CursorHint opcodes */ #define SQLITE_AllOpts 0xffff /* All optimizations */ /* ** Macros for testing whether or not optimizations are enabled or disabled. */ #define OptimizationDisabled(db, mask) (((db)->dbOptFlags&(mask))!=0) |
︙ | ︙ |
Changes to src/where.c.
︙ | ︙ | |||
4754 4755 4756 4757 4758 4759 4760 4761 4762 4763 4764 4765 4766 4767 | assert( iIndexCur>=0 ); if( op ){ sqlite3VdbeAddOp3(v, op, iIndexCur, pIx->tnum, iDb); sqlite3VdbeSetP4KeyInfo(pParse, pIx); if( (pLoop->wsFlags & WHERE_CONSTRAINT)!=0 && (pLoop->wsFlags & (WHERE_COLUMN_RANGE|WHERE_SKIPSCAN))==0 && (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 ){ sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ); /* Hint to COMDB2 */ } VdbeComment((v, "%s", pIx->zName)); #ifdef SQLITE_ENABLE_COLUMN_USED_MASK { u64 colUsed = 0; | > | 4754 4755 4756 4757 4758 4759 4760 4761 4762 4763 4764 4765 4766 4767 4768 | assert( iIndexCur>=0 ); if( op ){ sqlite3VdbeAddOp3(v, op, iIndexCur, pIx->tnum, iDb); sqlite3VdbeSetP4KeyInfo(pParse, pIx); if( (pLoop->wsFlags & WHERE_CONSTRAINT)!=0 && (pLoop->wsFlags & (WHERE_COLUMN_RANGE|WHERE_SKIPSCAN))==0 && (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 && pWInfo->eDistinct!=WHERE_DISTINCT_ORDERED ){ sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ); /* Hint to COMDB2 */ } VdbeComment((v, "%s", pIx->zName)); #ifdef SQLITE_ENABLE_COLUMN_USED_MASK { u64 colUsed = 0; |
︙ | ︙ | |||
4844 4845 4846 4847 4848 4849 4850 | sqlite3ExprCacheClear(pParse); for(i=pWInfo->nLevel-1; i>=0; i--){ int addr; pLevel = &pWInfo->a[i]; pLoop = pLevel->pWLoop; sqlite3VdbeResolveLabel(v, pLevel->addrCont); if( pLevel->op!=OP_Noop ){ | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | | | | | > | 4845 4846 4847 4848 4849 4850 4851 4852 4853 4854 4855 4856 4857 4858 4859 4860 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870 4871 4872 4873 4874 4875 4876 4877 4878 4879 4880 4881 4882 4883 4884 4885 4886 4887 4888 4889 4890 4891 4892 4893 4894 4895 4896 4897 4898 4899 4900 4901 4902 4903 4904 4905 4906 4907 4908 4909 4910 4911 | sqlite3ExprCacheClear(pParse); for(i=pWInfo->nLevel-1; i>=0; i--){ int addr; pLevel = &pWInfo->a[i]; pLoop = pLevel->pWLoop; sqlite3VdbeResolveLabel(v, pLevel->addrCont); if( pLevel->op!=OP_Noop ){ #ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT int n = -1; int j, k, op; int r1 = pParse->nMem+1; if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED && (pLoop->wsFlags & WHERE_INDEXED)!=0 && OptimizationEnabled(db, SQLITE_SkipAhead) ){ /* This is the Skip-ahead optimization. When doing a DISTINCT query ** that has WHERE_DISTINCT_ORDERED, use OP_SkipGT/OP_SkipLT to skip ** over all duplicate entries, rather than visiting all duplicates ** using OP_Next/OP_Prev. */ ExprList *pX = pWInfo->pResultSet; Index *pIdx = pLoop->u.btree.pIndex; for(j=0; j<pX->nExpr; j++){ Expr *pE = sqlite3ExprSkipCollate(pX->a[j].pExpr); if( pE->op==TK_COLUMN ){ if( pE->iTable!=pLevel->iTabCur ) continue; k = 1+sqlite3ColumnOfIndex(pIdx, pE->iColumn); if( k>n ) n = k; }else if( pIdx->aColExpr ){ for(k=n+1; k<pIdx->nKeyCol; k++){ Expr *pI = pIdx->aColExpr->a[k].pExpr; if( pI && sqlite3ExprCompare(pE,pI,0)<2 ){ n = k+1; break; } } } } } if( n>0 ){ for(j=0; j<n; j++){ sqlite3VdbeAddOp3(v, OP_Column, pLevel->iIdxCur, j, r1+j); } pParse->nMem += n; op = pLevel->op==OP_Prev ? OP_SeekLT : OP_SeekGT; k = sqlite3VdbeAddOp4Int(v, op, pLevel->iIdxCur, 0, r1, n); VdbeCoverageIf(v, op==OP_SeekLT); VdbeCoverageIf(v, op==OP_SeekGT); sqlite3VdbeAddOp2(v, OP_Goto, 1, pLevel->p2); sqlite3VdbeJumpHere(v, k); }else #endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */ { /* The common case: Advance to the next row */ sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3); sqlite3VdbeChangeP5(v, pLevel->p5); VdbeCoverage(v); VdbeCoverageIf(v, pLevel->op==OP_Next); VdbeCoverageIf(v, pLevel->op==OP_Prev); VdbeCoverageIf(v, pLevel->op==OP_VNext); } } if( pLoop->wsFlags & WHERE_IN_ABLE && pLevel->u.in.nIn>0 ){ struct InLoop *pIn; int j; sqlite3VdbeResolveLabel(v, pLevel->addrNxt); for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){ sqlite3VdbeJumpHere(v, pIn->addrInTop+1); |
︙ | ︙ |
Added test/distinct2.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 | # 2016-04-15 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # This file implements regression tests for SQLite library. The # focus of this script is DISTINCT queries using the skip-ahead # optimization. # set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix distinct2 do_execsql_test 100 { CREATE TABLE t1(x INTEGER PRIMARY KEY); INSERT INTO t1 VALUES(0),(1),(2); CREATE TABLE t2 AS SELECT DISTINCT a.x AS aa, b.x AS bb FROM t1 a, t1 b; SELECT *, '|' FROM t2 ORDER BY aa, bb; } {0 0 | 0 1 | 0 2 | 1 0 | 1 1 | 1 2 | 2 0 | 2 1 | 2 2 |} do_execsql_test 110 { DROP TABLE t2; CREATE TABLE t2 AS SELECT DISTINCT a.x AS aa, b.x AS bb FROM t1 a, t1 b WHERE a.x IN t1 AND b.x IN t1; SELECT *, '|' FROM t2 ORDER BY aa, bb; } {0 0 | 0 1 | 0 2 | 1 0 | 1 1 | 1 2 | 2 0 | 2 1 | 2 2 |} do_execsql_test 120 { CREATE TABLE t102 (i0 TEXT UNIQUE NOT NULL); INSERT INTO t102 VALUES ('0'),('1'),('2'); DROP TABLE t2; CREATE TABLE t2 AS SELECT DISTINCT * FROM t102 AS t0 JOIN t102 AS t4 ON (t2.i0 IN t102) NATURAL JOIN t102 AS t3 JOIN t102 AS t1 ON (t0.i0 IN t102) JOIN t102 AS t2 ON (t2.i0=+t0.i0 OR (t0.i0<>500 AND t2.i0=t1.i0)); SELECT *, '|' FROM t2 ORDER BY 1, 2, 3, 4, 5; } {0 0 0 0 | 0 0 1 0 | 0 0 1 1 | 0 0 2 0 | 0 0 2 2 | 0 1 0 0 | 0 1 1 0 | 0 1 1 1 | 0 1 2 0 | 0 1 2 2 | 0 2 0 0 | 0 2 1 0 | 0 2 1 1 | 0 2 2 0 | 0 2 2 2 | 1 0 0 0 | 1 0 0 1 | 1 0 1 1 | 1 0 2 1 | 1 0 2 2 | 1 1 0 0 | 1 1 0 1 | 1 1 1 1 | 1 1 2 1 | 1 1 2 2 | 1 2 0 0 | 1 2 0 1 | 1 2 1 1 | 1 2 2 1 | 1 2 2 2 | 2 0 0 0 | 2 0 0 2 | 2 0 1 1 | 2 0 1 2 | 2 0 2 2 | 2 1 0 0 | 2 1 0 2 | 2 1 1 1 | 2 1 1 2 | 2 1 2 2 | 2 2 0 0 | 2 2 0 2 | 2 2 1 1 | 2 2 1 2 | 2 2 2 2 |} do_execsql_test 400 { CREATE TABLE t4(a,b,c,d,e,f,g,h,i,j); INSERT INTO t4 VALUES(0,1,2,3,4,5,6,7,8,9); INSERT INTO t4 SELECT * FROM t4; INSERT INTO t4 SELECT * FROM t4; CREATE INDEX t4x ON t4(c,d,e); SELECT DISTINCT a,b,c FROM t4 WHERE a=0 AND b=1; } {0 1 2} do_execsql_test 410 { SELECT DISTINCT a,b,c,d FROM t4 WHERE a=0 AND b=1; } {0 1 2 3} do_execsql_test 411 { SELECT DISTINCT d,a,b,c FROM t4 WHERE a=0 AND b=1; } {3 0 1 2} do_execsql_test 420 { SELECT DISTINCT a,b,c,d,e FROM t4 WHERE a=0 AND b=1; } {0 1 2 3 4} do_execsql_test 430 { SELECT DISTINCT a,b,c,d,e,f FROM t4 WHERE a=0 AND b=1; } {0 1 2 3 4 5} do_execsql_test 500 { CREATE TABLE t5(a INT, b INT); CREATE UNIQUE INDEX t5x ON t5(a+b); INSERT INTO t5(a,b) VALUES(0,0),(1,0),(1,1),(0,3); CREATE TEMP TABLE out AS SELECT DISTINCT a+b FROM t5; SELECT * FROM out ORDER BY 1; } {0 1 2 3} do_execsql_test 600 { CREATE TABLE t6a(x INTEGER PRIMARY KEY); INSERT INTO t6a VALUES(1); CREATE TABLE t6b(y INTEGER PRIMARY KEY); INSERT INTO t6b VALUES(2),(3); SELECT DISTINCT x, x FROM t6a, t6b; } {1 1} finish_test |