/ Check-in [0ad0f8d7]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Changes toward being able to process indexes on expressions. Not there yet - this check-in is just movement in that direction. Some tests are failing.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | index-expr
Files: files | file ages | folders
SHA1: 0ad0f8d77d8f95ca2ffb7745d18219f5e87dc89c
User & Date: drh 2015-08-25 00:27:06
Original Comment: Changes toward being abld to process indexes on expressions. Not there yet - this check-in is just movement in that direction. Some tests are failing.
Context
2015-08-25
16:57
Add code to maintain indexes with expression arguments across DELETE, INSERT, and UPDATE statements. Legacy tests pass, but the new code paths are still largely untested. The query planner currently makes no effort to use expression indexes. check-in: efaabdb7 user: drh tags: index-expr
00:27
Changes toward being able to process indexes on expressions. Not there yet - this check-in is just movement in that direction. Some tests are failing. check-in: 0ad0f8d7 user: drh tags: index-expr
2015-08-24
20:54
Remove some redundant code: Call sqlite3ResolveExprListNames() rather than calling sqlite3ResolveExprNames() in a loop - in two places. check-in: bdaf6646 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/build.c.

636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
....
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
....
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122


3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138

3139

3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152

3153
3154
3155
3156
3157
3158
3159

  /* Delete the Table structure itself.
  */
  sqlite3DeleteColumnNames(db, pTable);
  sqlite3DbFree(db, pTable->zName);
  sqlite3DbFree(db, pTable->zColAff);
  sqlite3SelectDelete(db, pTable->pSelect);
#ifndef SQLITE_OMIT_CHECK
  sqlite3ExprListDelete(db, pTable->pCheck);
#endif
#ifndef SQLITE_OMIT_VIRTUALTABLE
  sqlite3VtabClear(db, pTable);
#endif
  sqlite3DbFree(db, pTable);

  /* Verify that no lookaside memory was used by schema tables */
  assert( nLookaside==0 || nLookaside==db->lookaside.nOut );
................................................................................
  DbFixer sFix;        /* For assigning database names to pTable */
  int sortOrderMask;   /* 1 to honor DESC in index.  0 to ignore. */
  sqlite3 *db = pParse->db;
  Db *pDb;             /* The specific table containing the indexed database */
  int iDb;             /* Index of the database that is being written */
  Token *pName = 0;    /* Unqualified name of the index to create */
  struct ExprList_item *pListItem; /* For looping over pList */
  const Column *pTabCol;           /* A column in the table */
  int nExtra = 0;                  /* Space allocated for zExtra[] */
  int nExtraCol;                   /* Number of extra columns needed */
  char *zExtra = 0;                /* Extra space after the Index object */
  Index *pPk = 0;      /* PRIMARY KEY index for WITHOUT ROWID tables */

  if( db->mallocFailed || IN_DECLARE_VTAB || pParse->nErr>0 ){
    goto exit_create_index;
................................................................................
  ** TODO:  Add a test to make sure that the same column is not named
  ** more than once within the same index.  Only the first instance of
  ** the column will ever be used by the optimizer.  Note that using the
  ** same column more than once cannot be an error because that would 
  ** break backwards compatibility - it needs to be a warning.
  */
  for(i=0, pListItem=pList->a; i<pList->nExpr; i++, pListItem++){
    const char *zColName;
    Expr *pCExpr;
    int requestedSortOrder;
    char *zColl;                   /* Collation sequence name */



    pCExpr = sqlite3ExprSkipCollate(pListItem->pExpr);
    if( pCExpr->op!=TK_ID ){
      sqlite3ErrorMsg(pParse, "indexes on expressions not yet supported");
      continue;
    }
    zColName = pCExpr->u.zToken;
    for(j=0, pTabCol=pTab->aCol; j<pTab->nCol; j++, pTabCol++){
      if( sqlite3StrICmp(zColName, pTabCol->zName)==0 ) break;
    }
    if( j>=pTab->nCol ){
      sqlite3ErrorMsg(pParse, "table %s has no column named %s",
        pTab->zName, zColName);
      pParse->checkSchema = 1;
      goto exit_create_index;
    }
    assert( j<=0x7fff );

    pIndex->aiColumn[i] = (i16)j;

    if( pListItem->pExpr->op==TK_COLLATE ){
      int nColl;
      zColl = pListItem->pExpr->u.zToken;
      nColl = sqlite3Strlen30(zColl) + 1;
      assert( nExtra>=nColl );
      memcpy(zExtra, zColl, nColl);
      zColl = zExtra;
      zExtra += nColl;
      nExtra -= nColl;
    }else{
      zColl = pTab->aCol[j].zColl;
      if( !zColl ) zColl = "BINARY";
    }

    if( !db->init.busy && !sqlite3LocateCollSeq(pParse, zColl) ){
      goto exit_create_index;
    }
    pIndex->azColl[i] = zColl;
    requestedSortOrder = pListItem->sortOrder & sortOrderMask;
    pIndex->aSortOrder[i] = (u8)requestedSortOrder;
    if( pTab->aCol[j].notNull==0 ) pIndex->uniqNotNull = 0;







<

<







 







<







 







<




>
>

|



|
<
<
<
<
<
<
<
<
<

>

>









|

<

>







636
637
638
639
640
641
642

643

644
645
646
647
648
649
650
....
2885
2886
2887
2888
2889
2890
2891

2892
2893
2894
2895
2896
2897
2898
....
3108
3109
3110
3111
3112
3113
3114

3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126









3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141

3142
3143
3144
3145
3146
3147
3148
3149
3150

  /* Delete the Table structure itself.
  */
  sqlite3DeleteColumnNames(db, pTable);
  sqlite3DbFree(db, pTable->zName);
  sqlite3DbFree(db, pTable->zColAff);
  sqlite3SelectDelete(db, pTable->pSelect);

  sqlite3ExprListDelete(db, pTable->pCheck);

#ifndef SQLITE_OMIT_VIRTUALTABLE
  sqlite3VtabClear(db, pTable);
#endif
  sqlite3DbFree(db, pTable);

  /* Verify that no lookaside memory was used by schema tables */
  assert( nLookaside==0 || nLookaside==db->lookaside.nOut );
................................................................................
  DbFixer sFix;        /* For assigning database names to pTable */
  int sortOrderMask;   /* 1 to honor DESC in index.  0 to ignore. */
  sqlite3 *db = pParse->db;
  Db *pDb;             /* The specific table containing the indexed database */
  int iDb;             /* Index of the database that is being written */
  Token *pName = 0;    /* Unqualified name of the index to create */
  struct ExprList_item *pListItem; /* For looping over pList */

  int nExtra = 0;                  /* Space allocated for zExtra[] */
  int nExtraCol;                   /* Number of extra columns needed */
  char *zExtra = 0;                /* Extra space after the Index object */
  Index *pPk = 0;      /* PRIMARY KEY index for WITHOUT ROWID tables */

  if( db->mallocFailed || IN_DECLARE_VTAB || pParse->nErr>0 ){
    goto exit_create_index;
................................................................................
  ** TODO:  Add a test to make sure that the same column is not named
  ** more than once within the same index.  Only the first instance of
  ** the column will ever be used by the optimizer.  Note that using the
  ** same column more than once cannot be an error because that would 
  ** break backwards compatibility - it needs to be a warning.
  */
  for(i=0, pListItem=pList->a; i<pList->nExpr; i++, pListItem++){

    Expr *pCExpr;
    int requestedSortOrder;
    char *zColl;                   /* Collation sequence name */

    sqlite3ResolveSelfReference(pParse, pTab, NC_IdxExpr, pListItem->pExpr, 0);
    if( pParse->nErr ) goto exit_create_index;
    pCExpr = sqlite3ExprSkipCollate(pListItem->pExpr);
    if( pCExpr->op!=TK_COLUMN ){
      sqlite3ErrorMsg(pParse, "indexes on expressions not yet supported");
      continue;
    }
    j = pCExpr->iColumn;









    assert( j<=0x7fff );
    if( j<0 ) j = pTab->iPKey;
    pIndex->aiColumn[i] = (i16)j;
    zColl = 0;
    if( pListItem->pExpr->op==TK_COLLATE ){
      int nColl;
      zColl = pListItem->pExpr->u.zToken;
      nColl = sqlite3Strlen30(zColl) + 1;
      assert( nExtra>=nColl );
      memcpy(zExtra, zColl, nColl);
      zColl = zExtra;
      zExtra += nColl;
      nExtra -= nColl;
    }else if( j>=0 ){
      zColl = pTab->aCol[j].zColl;

    }
    if( !zColl ) zColl = "BINARY";
    if( !db->init.busy && !sqlite3LocateCollSeq(pParse, zColl) ){
      goto exit_create_index;
    }
    pIndex->azColl[i] = zColl;
    requestedSortOrder = pListItem->sortOrder & sortOrderMask;
    pIndex->aSortOrder[i] = (u8)requestedSortOrder;
    if( pTab->aCol[j].notNull==0 ) pIndex->uniqNotNull = 0;

Changes to src/resolve.c.

543
544
545
546
547
548
549
550
551
552
553
554
555
556


557

558


559
560
561
562
563

564
565
566
567
568
569
570
571
572
573

574
575
576
577
578
579
580
581
582
583
584
585
586
...
657
658
659
660
661
662
663


664
665
666
667
668
669
670
...
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
...
736
737
738
739
740
741
742


743
744
745
746
747
748
749
...
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
....
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
    }
    ExprSetProperty(p, EP_Resolved);
  }
  return p;
}

/*
** Report an error that an expression is not valid for a partial index WHERE
** clause.
*/
static void notValidPartIdxWhere(
  Parse *pParse,       /* Leave error message here */
  NameContext *pNC,    /* The name context */
  const char *zMsg     /* Type of error */


){

  if( (pNC->ncFlags & NC_PartIdx)!=0 ){


    sqlite3ErrorMsg(pParse, "%s prohibited in partial index WHERE clauses",
                    zMsg);
  }
}


#ifndef SQLITE_OMIT_CHECK
/*
** Report an error that an expression is not valid for a CHECK constraint.
*/
static void notValidCheckConstraint(
  Parse *pParse,       /* Leave error message here */
  NameContext *pNC,    /* The name context */
  const char *zMsg     /* Type of error */
){
  if( (pNC->ncFlags & NC_IsCheck)!=0 ){

    sqlite3ErrorMsg(pParse,"%s prohibited in CHECK constraints", zMsg);
  }
}
#else
# define notValidCheckConstraint(P,N,M)
#endif

/*
** Expression p should encode a floating point value between 1.0 and 0.0.
** Return 1024 times this value.  Or return -1 if p is not a floating point
** value between 1.0 and 0.0.
*/
static int exprProbability(Expr *p){
................................................................................
    case TK_DOT: {
      const char *zColumn;
      const char *zTable;
      const char *zDb;
      Expr *pRight;

      /* if( pSrcList==0 ) break; */


      pRight = pExpr->pRight;
      if( pRight->op==TK_ID ){
        zDb = 0;
        zTable = pExpr->pLeft->u.zToken;
        zColumn = pRight->u.zToken;
      }else{
        assert( pRight->op==TK_DOT );
................................................................................
      int auth;                   /* Authorization to use the function */
      int nId;                    /* Number of characters in function name */
      const char *zId;            /* The function name. */
      FuncDef *pDef;              /* Information about the function */
      u8 enc = ENC(pParse->db);   /* The database encoding */

      assert( !ExprHasProperty(pExpr, EP_xIsSelect) );
      notValidPartIdxWhere(pParse, pNC, "functions");
      zId = pExpr->u.zToken;
      nId = sqlite3Strlen30(zId);
      pDef = sqlite3FindFunction(pParse->db, zId, nId, n, enc, 0);
      if( pDef==0 ){
        pDef = sqlite3FindFunction(pParse->db, zId, nId, -2, enc, 0);
        if( pDef==0 ){
          no_such_func = 1;
................................................................................
          }
          pExpr->op = TK_NULL;
          return WRC_Prune;
        }
#endif
        if( pDef->funcFlags & SQLITE_FUNC_CONSTANT ){
          ExprSetProperty(pExpr,EP_ConstFunc);


        }
      }
      if( is_agg && (pNC->ncFlags & NC_AllowAgg)==0 ){
        sqlite3ErrorMsg(pParse, "misuse of aggregate function %.*s()", nId,zId);
        pNC->nErr++;
        is_agg = 0;
      }else if( no_such_func && pParse->db->init.busy==0 ){
................................................................................
    case TK_SELECT:
    case TK_EXISTS:  testcase( pExpr->op==TK_EXISTS );
#endif
    case TK_IN: {
      testcase( pExpr->op==TK_IN );
      if( ExprHasProperty(pExpr, EP_xIsSelect) ){
        int nRef = pNC->nRef;
        notValidCheckConstraint(pParse, pNC, "subqueries");
        notValidPartIdxWhere(pParse, pNC, "subqueries");
        sqlite3WalkSelect(pWalker, pExpr->x.pSelect);
        assert( pNC->nRef>=nRef );
        if( nRef!=pNC->nRef ){
          ExprSetProperty(pExpr, EP_VarSelect);
        }
      }
      break;
    }
    case TK_VARIABLE: {
      notValidCheckConstraint(pParse, pNC, "parameters");
      notValidPartIdxWhere(pParse, pNC, "parameters");
      break;
    }
  }
  return (pParse->nErr || pParse->db->mallocFailed) ? WRC_Abort : WRC_Continue;
}

/*
................................................................................
** is set to -1 and the Expr.iColumn value is set to the column number.
**
** Any errors cause an error message to be set in pParse.
*/
void sqlite3ResolveSelfReference(
  Parse *pParse,      /* Parsing context */
  Table *pTab,        /* The table being referenced */
  int type,           /* NC_IsCheck or NC_PartIdx */
  Expr *pExpr,        /* Expression to resolve.  May be NULL. */
  ExprList *pList     /* Expression list to resolve.  May be NUL. */
){
  SrcList sSrc;                   /* Fake SrcList for pParse->pNewTable */
  NameContext sNC;                /* Name context for pParse->pNewTable */

  assert( type==NC_IsCheck || type==NC_PartIdx );
  memset(&sNC, 0, sizeof(sNC));
  memset(&sSrc, 0, sizeof(sSrc));
  sSrc.nSrc = 1;
  sSrc.a[0].zName = pTab->zName;
  sSrc.a[0].pTab = pTab;
  sSrc.a[0].iCursor = -1;
  sNC.pParse = pParse;
  sNC.pSrcList = &sSrc;
  sNC.ncFlags = type;
  if( sqlite3ResolveExprNames(&sNC, pExpr) ) return;
  if( pList ) sqlite3ResolveExprListNames(&sNC, pList);
}







|
|

|


|
>
>

>
|
>
>
|
<
<
<
<
>

<
<
<
<
<
<
<
<
|
>
|


<
<
<







 







>
>







 







|







 







>
>







 







<
|









<
|







 







|






|












543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564




565
566








567
568
569
570
571



572
573
574
575
576
577
578
...
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
...
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
...
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
...
778
779
780
781
782
783
784

785
786
787
788
789
790
791
792
793
794

795
796
797
798
799
800
801
802
....
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
    }
    ExprSetProperty(p, EP_Resolved);
  }
  return p;
}

/*
** Report an error that an expression is not valid for some set of
** pNC->ncFlags values determined by validMask.  If 
*/
static void notValid(
  Parse *pParse,       /* Leave error message here */
  NameContext *pNC,    /* The name context */
  const char *zMsg,    /* Type of error */
  int validMask,       /* Set of contexts for which prohibited */
  int okForInit        /* No error if pParse->db->init.busy is true */
){
  assert( (validMask&~(NC_IsCheck|NC_PartIdx|NC_IdxExpr))==0 );
  if( (pNC->ncFlags & validMask)!=0 
   && (pParse->db->init.busy==0 || !okForInit)
  ){
    const char *zIn = "partial index WHERE clauses";




    if( pNC->ncFlags & NC_IdxExpr )      zIn = "index expressions";
#ifndef SQLITE_OMIT_CHECK








    else if( pNC->ncFlags & NC_IsCheck ) zIn = "CHECK constraints";
#endif
    sqlite3ErrorMsg(pParse, "%s prohibited in %s", zMsg, zIn);
  }
}




/*
** Expression p should encode a floating point value between 1.0 and 0.0.
** Return 1024 times this value.  Or return -1 if p is not a floating point
** value between 1.0 and 0.0.
*/
static int exprProbability(Expr *p){
................................................................................
    case TK_DOT: {
      const char *zColumn;
      const char *zTable;
      const char *zDb;
      Expr *pRight;

      /* if( pSrcList==0 ) break; */
      notValid(pParse, pNC, "the \".\" operator", NC_IdxExpr, 0);
      notValid(pParse, pNC, "the \".\" operator", NC_PartIdx|NC_IsCheck, 1);
      pRight = pExpr->pRight;
      if( pRight->op==TK_ID ){
        zDb = 0;
        zTable = pExpr->pLeft->u.zToken;
        zColumn = pRight->u.zToken;
      }else{
        assert( pRight->op==TK_DOT );
................................................................................
      int auth;                   /* Authorization to use the function */
      int nId;                    /* Number of characters in function name */
      const char *zId;            /* The function name. */
      FuncDef *pDef;              /* Information about the function */
      u8 enc = ENC(pParse->db);   /* The database encoding */

      assert( !ExprHasProperty(pExpr, EP_xIsSelect) );
      notValid(pParse, pNC, "functions", NC_PartIdx, 0);
      zId = pExpr->u.zToken;
      nId = sqlite3Strlen30(zId);
      pDef = sqlite3FindFunction(pParse->db, zId, nId, n, enc, 0);
      if( pDef==0 ){
        pDef = sqlite3FindFunction(pParse->db, zId, nId, -2, enc, 0);
        if( pDef==0 ){
          no_such_func = 1;
................................................................................
          }
          pExpr->op = TK_NULL;
          return WRC_Prune;
        }
#endif
        if( pDef->funcFlags & SQLITE_FUNC_CONSTANT ){
          ExprSetProperty(pExpr,EP_ConstFunc);
        }else{
          notValid(pParse, pNC, "non-deterministic functions", NC_IdxExpr, 0);
        }
      }
      if( is_agg && (pNC->ncFlags & NC_AllowAgg)==0 ){
        sqlite3ErrorMsg(pParse, "misuse of aggregate function %.*s()", nId,zId);
        pNC->nErr++;
        is_agg = 0;
      }else if( no_such_func && pParse->db->init.busy==0 ){
................................................................................
    case TK_SELECT:
    case TK_EXISTS:  testcase( pExpr->op==TK_EXISTS );
#endif
    case TK_IN: {
      testcase( pExpr->op==TK_IN );
      if( ExprHasProperty(pExpr, EP_xIsSelect) ){
        int nRef = pNC->nRef;

        notValid(pParse, pNC, "subqueries", NC_IsCheck|NC_PartIdx|NC_IdxExpr,0);
        sqlite3WalkSelect(pWalker, pExpr->x.pSelect);
        assert( pNC->nRef>=nRef );
        if( nRef!=pNC->nRef ){
          ExprSetProperty(pExpr, EP_VarSelect);
        }
      }
      break;
    }
    case TK_VARIABLE: {

      notValid(pParse, pNC, "parameters", NC_IsCheck|NC_PartIdx|NC_IdxExpr, 0);
      break;
    }
  }
  return (pParse->nErr || pParse->db->mallocFailed) ? WRC_Abort : WRC_Continue;
}

/*
................................................................................
** is set to -1 and the Expr.iColumn value is set to the column number.
**
** Any errors cause an error message to be set in pParse.
*/
void sqlite3ResolveSelfReference(
  Parse *pParse,      /* Parsing context */
  Table *pTab,        /* The table being referenced */
  int type,           /* NC_IsCheck or NC_PartIdx or NC_IdxExpr */
  Expr *pExpr,        /* Expression to resolve.  May be NULL. */
  ExprList *pList     /* Expression list to resolve.  May be NUL. */
){
  SrcList sSrc;                   /* Fake SrcList for pParse->pNewTable */
  NameContext sNC;                /* Name context for pParse->pNewTable */

  assert( type==NC_IsCheck || type==NC_PartIdx || type==NC_IdxExpr );
  memset(&sNC, 0, sizeof(sNC));
  memset(&sSrc, 0, sizeof(sSrc));
  sSrc.nSrc = 1;
  sSrc.a[0].zName = pTab->zName;
  sSrc.a[0].pTab = pTab;
  sSrc.a[0].iCursor = -1;
  sNC.pParse = pParse;
  sNC.pSrcList = &sSrc;
  sNC.ncFlags = type;
  if( sqlite3ResolveExprNames(&sNC, pExpr) ) return;
  if( pList ) sqlite3ResolveExprListNames(&sNC, pList);
}

Changes to src/sqliteInt.h.

2387
2388
2389
2390
2391
2392
2393

2394
2395
2396
2397
2398
2399
2400
** 
*/
#define NC_AllowAgg  0x0001  /* Aggregate functions are allowed here */
#define NC_HasAgg    0x0002  /* One or more aggregate functions seen */
#define NC_IsCheck   0x0004  /* True if resolving names in a CHECK constraint */
#define NC_InAggFunc 0x0008  /* True if analyzing arguments to an agg func */
#define NC_PartIdx   0x0010  /* True if resolving a partial index WHERE */

#define NC_MinMaxAgg 0x1000  /* min/max aggregates seen.  See note above */

/*
** An instance of the following structure contains all information
** needed to generate code for a single SELECT statement.
**
** nLimit is set to -1 if there is no LIMIT clause.  nOffset is set to 0.







>







2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
** 
*/
#define NC_AllowAgg  0x0001  /* Aggregate functions are allowed here */
#define NC_HasAgg    0x0002  /* One or more aggregate functions seen */
#define NC_IsCheck   0x0004  /* True if resolving names in a CHECK constraint */
#define NC_InAggFunc 0x0008  /* True if analyzing arguments to an agg func */
#define NC_PartIdx   0x0010  /* True if resolving a partial index WHERE */
#define NC_IdxExpr   0x0020  /* True if resolving columns of CREATE INDEX */
#define NC_MinMaxAgg 0x1000  /* min/max aggregates seen.  See note above */

/*
** An instance of the following structure contains all information
** needed to generate code for a single SELECT statement.
**
** nLimit is set to -1 if there is no LIMIT clause.  nOffset is set to 0.