/ Check-in [8aede091]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge the latest enhancements from trunk.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | lsm-vtab
Files: files | file ages | folders
SHA1: 8aede091c4740511d11ea14da253fe39bbfe75a6
User & Date: drh 2015-11-19 19:31:33
Context
2016-02-22
13:01
Merge up to trunk. check-in: f9e5fb88 user: drh tags: lsm-vtab
2015-11-19
19:31
Merge the latest enhancements from trunk. check-in: 8aede091 user: drh tags: lsm-vtab
19:27
Work toward more flexible typing for keys and values. check-in: 5c79f531 user: drh tags: lsm-vtab
18:11
Fix problems with the way the IsHiddenColumn() macro is defined. check-in: 126b998c user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/lsm1/Makefile.

     1      1   #!/usr/bin/make
     2      2   #
     3      3   # This is a temporary makefile for use during experimental development.
     4      4   # Replace with something more portable, if the experiments actually work out.
     5      5   #
     6      6   CC     = gcc
     7         -CFLAGS =-g -fPIC -Wall -I. -I/home/drh/sqlite/bld
            7  +CFLAGS =-g -fPIC -Wall -I. -I../..
     8      8   
     9      9   LSMOBJ    = \
    10     10     lsm_ckpt.o \
    11     11     lsm_file.o \
    12     12     lsm_log.o \
    13     13     lsm_main.o \
    14     14     lsm_mem.o \

Changes to src/build.c.

  1045   1045   
  1046   1046     /* If an error occurs, we jump here */
  1047   1047   begin_table_error:
  1048   1048     sqlite3DbFree(db, zName);
  1049   1049     return;
  1050   1050   }
  1051   1051   
  1052         -/*
  1053         -** This macro is used to compare two strings in a case-insensitive manner.
  1054         -** It is slightly faster than calling sqlite3StrICmp() directly, but
  1055         -** produces larger code.
  1056         -**
  1057         -** WARNING: This macro is not compatible with the strcmp() family. It
  1058         -** returns true if the two strings are equal, otherwise false.
         1052  +/* Set properties of a table column based on the (magical)
         1053  +** name of the column.
  1059   1054   */
  1060         -#define STRICMP(x, y) (\
  1061         -sqlite3UpperToLower[*(unsigned char *)(x)]==   \
  1062         -sqlite3UpperToLower[*(unsigned char *)(y)]     \
  1063         -&& sqlite3StrICmp((x)+1,(y)+1)==0 )
         1055  +void sqlite3ColumnPropertiesFromName(Table *pTab, Column *pCol){
         1056  +#if SQLITE_ENABLE_HIDDEN_COLUMNS
         1057  +  if( sqlite3_strnicmp(pCol->zName, "__hidden__", 10)==0 ){
         1058  +    pCol->colFlags |= COLFLAG_HIDDEN;
         1059  +  }else if( pTab && pCol!=pTab->aCol && (pCol[-1].colFlags & COLFLAG_HIDDEN) ){
         1060  +    pTab->tabFlags |= TF_OOOHidden;
         1061  +  }
         1062  +#endif
         1063  +}
         1064  +
  1064   1065   
  1065   1066   /*
  1066   1067   ** Add a new column to the table currently being constructed.
  1067   1068   **
  1068   1069   ** The parser calls this routine once for each column declaration
  1069   1070   ** in a CREATE TABLE statement.  sqlite3StartTable() gets called
  1070   1071   ** first to get things going.  Then this routine is called for each
................................................................................
  1082   1083       sqlite3ErrorMsg(pParse, "too many columns on %s", p->zName);
  1083   1084       return;
  1084   1085     }
  1085   1086   #endif
  1086   1087     z = sqlite3NameFromToken(db, pName);
  1087   1088     if( z==0 ) return;
  1088   1089     for(i=0; i<p->nCol; i++){
  1089         -    if( STRICMP(z, p->aCol[i].zName) ){
         1090  +    if( sqlite3_stricmp(z, p->aCol[i].zName)==0 ){
  1090   1091         sqlite3ErrorMsg(pParse, "duplicate column name: %s", z);
  1091   1092         sqlite3DbFree(db, z);
  1092   1093         return;
  1093   1094       }
  1094   1095     }
  1095   1096     if( (p->nCol & 0x7)==0 ){
  1096   1097       Column *aNew;
................................................................................
  1100   1101         return;
  1101   1102       }
  1102   1103       p->aCol = aNew;
  1103   1104     }
  1104   1105     pCol = &p->aCol[p->nCol];
  1105   1106     memset(pCol, 0, sizeof(p->aCol[0]));
  1106   1107     pCol->zName = z;
         1108  +  sqlite3ColumnPropertiesFromName(p, pCol);
  1107   1109    
  1108   1110     /* If there is no type specified, columns have the default affinity
  1109   1111     ** 'BLOB'. If there is a type specified, then sqlite3AddColumnType() will
  1110   1112     ** be called next to set pCol->affinity correctly.
  1111   1113     */
  1112   1114     pCol->affinity = SQLITE_AFF_BLOB;
  1113   1115     pCol->szEst = 1;

Changes to src/delete.c.

   102    102     if( pFrom ){
   103    103       assert( pFrom->nSrc==1 );
   104    104       pFrom->a[0].zName = sqlite3DbStrDup(db, pView->zName);
   105    105       pFrom->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zName);
   106    106       assert( pFrom->a[0].pOn==0 );
   107    107       assert( pFrom->a[0].pUsing==0 );
   108    108     }
   109         -  pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, 0, 0, 0);
          109  +  pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, 
          110  +                          SF_IncludeHidden, 0, 0);
   110    111     sqlite3SelectDestInit(&dest, SRT_EphemTab, iCur);
   111    112     sqlite3Select(pParse, pSel, &dest);
   112    113     sqlite3SelectDelete(db, pSel);
   113    114   }
   114    115   #endif /* !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) */
   115    116   
   116    117   #if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY)

Changes to src/insert.c.

   732    732     if( pColumn==0 && nColumn>0 ){
   733    733       ipkColumn = pTab->iPKey;
   734    734     }
   735    735   
   736    736     /* Make sure the number of columns in the source data matches the number
   737    737     ** of columns to be inserted into the table.
   738    738     */
   739         -  if( IsVirtual(pTab) ){
   740         -    for(i=0; i<pTab->nCol; i++){
   741         -      nHidden += (IsHiddenColumn(&pTab->aCol[i]) ? 1 : 0);
   742         -    }
          739  +  for(i=0; i<pTab->nCol; i++){
          740  +    nHidden += (IsHiddenColumn(&pTab->aCol[i]) ? 1 : 0);
   743    741     }
   744    742     if( pColumn==0 && nColumn && nColumn!=(pTab->nCol-nHidden) ){
   745    743       sqlite3ErrorMsg(pParse, 
   746    744          "table %S has %d columns but %d values were supplied",
   747    745          pTabList, 0, pTab->nCol-nHidden, nColumn);
   748    746       goto insert_cleanup;
   749    747     }
................................................................................
   831    829       /* Cannot have triggers on a virtual table. If it were possible,
   832    830       ** this block would have to account for hidden column.
   833    831       */
   834    832       assert( !IsVirtual(pTab) );
   835    833   
   836    834       /* Create the new column data
   837    835       */
   838         -    for(i=0; i<pTab->nCol; i++){
   839         -      if( pColumn==0 ){
   840         -        j = i;
   841         -      }else{
          836  +    for(i=j=0; i<pTab->nCol; i++){
          837  +      if( pColumn ){
   842    838           for(j=0; j<pColumn->nId; j++){
   843    839             if( pColumn->a[j].idx==i ) break;
   844    840           }
   845    841         }
   846         -      if( (!useTempTable && !pList) || (pColumn && j>=pColumn->nId) ){
          842  +      if( (!useTempTable && !pList) || (pColumn && j>=pColumn->nId)
          843  +            || (pColumn==0 && IsOrdinaryHiddenColumn(&pTab->aCol[i])) ){
   847    844           sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, regCols+i+1);
   848    845         }else if( useTempTable ){
   849    846           sqlite3VdbeAddOp3(v, OP_Column, srcTab, j, regCols+i+1); 
   850    847         }else{
   851    848           assert( pSelect==0 ); /* Otherwise useTempTable is true */
   852    849           sqlite3ExprCodeAndCache(pParse, pList->a[j].pExpr, regCols+i+1);
   853    850         }
          851  +      if( pColumn==0 && !IsOrdinaryHiddenColumn(&pTab->aCol[i]) ) j++;
   854    852       }
   855    853   
   856    854       /* If this is an INSERT on a view with an INSTEAD OF INSERT trigger,
   857    855       ** do not attempt any conversions before assembling the record.
   858    856       ** If this is a real table, attempt conversions as required by the
   859    857       ** table column affinities.
   860    858       */
................................................................................
   930    928           ** taking up data space with information that will never be used.
   931    929           ** As there may be shallow copies of this value, make it a soft-NULL */
   932    930           sqlite3VdbeAddOp1(v, OP_SoftNull, iRegStore);
   933    931           continue;
   934    932         }
   935    933         if( pColumn==0 ){
   936    934           if( IsHiddenColumn(&pTab->aCol[i]) ){
   937         -          assert( IsVirtual(pTab) );
   938    935             j = -1;
   939    936             nHidden++;
   940    937           }else{
   941    938             j = i - nHidden;
   942    939           }
   943    940         }else{
   944    941           for(j=0; j<pColumn->nId; j++){
................................................................................
  1872   1869     }
  1873   1870     pEList = pSelect->pEList;
  1874   1871     assert( pEList!=0 );
  1875   1872     if( pEList->nExpr!=1 ){
  1876   1873       return 0;   /* The result set must have exactly one column */
  1877   1874     }
  1878   1875     assert( pEList->a[0].pExpr );
  1879         -  if( pEList->a[0].pExpr->op!=TK_ALL ){
         1876  +  if( pEList->a[0].pExpr->op!=TK_ASTERISK ){
  1880   1877       return 0;   /* The result set must be the special operator "*" */
  1881   1878     }
  1882   1879   
  1883   1880     /* At this point we have established that the statement is of the
  1884   1881     ** correct syntactic form to participate in this optimization.  Now
  1885   1882     ** we have to check the semantics.
  1886   1883     */
................................................................................
  1908   1905     }
  1909   1906     if( pDest->iPKey!=pSrc->iPKey ){
  1910   1907       return 0;   /* Both tables must have the same INTEGER PRIMARY KEY */
  1911   1908     }
  1912   1909     for(i=0; i<pDest->nCol; i++){
  1913   1910       Column *pDestCol = &pDest->aCol[i];
  1914   1911       Column *pSrcCol = &pSrc->aCol[i];
         1912  +#ifdef SQLITE_ENABLE_HIDDEN_COLUMNS
         1913  +    if( (db->flags & SQLITE_Vacuum)==0 
         1914  +     && (pDestCol->colFlags | pSrcCol->colFlags) & COLFLAG_HIDDEN 
         1915  +    ){
         1916  +      return 0;    /* Neither table may have __hidden__ columns */
         1917  +    }
         1918  +#endif
  1915   1919       if( pDestCol->affinity!=pSrcCol->affinity ){
  1916   1920         return 0;    /* Affinity must be the same on all columns */
  1917   1921       }
  1918   1922       if( !xferCompatibleCollation(pDestCol->zColl, pSrcCol->zColl) ){
  1919   1923         return 0;    /* Collating sequence must be the same on all columns */
  1920   1924       }
  1921   1925       if( pDestCol->notNull && !pSrcCol->notNull ){

Changes to src/os_unix.c.

  3345   3345         if( rc!=4 || memcmp(oldCntr, &((char*)pBuf)[24-offset], 4)!=0 ){
  3346   3346           pFile->transCntrChng = 1;  /* The transaction counter has changed */
  3347   3347         }
  3348   3348       }
  3349   3349     }
  3350   3350   #endif
  3351   3351   
  3352         -#if SQLITE_MAX_MMAP_SIZE>0
         3352  +#if defined(SQLITE_MMAP_READWRITE) && SQLITE_MAX_MMAP_SIZE>0
  3353   3353     /* Deal with as much of this write request as possible by transfering
  3354   3354     ** data from the memory mapping using memcpy().  */
  3355   3355     if( offset<pFile->mmapSize ){
  3356   3356       if( offset+amt <= pFile->mmapSize ){
  3357   3357         memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, amt);
  3358   3358         return SQLITE_OK;
  3359   3359       }else{
................................................................................
  4770   4770     assert( pFd->nFetchOut==0 );
  4771   4771     assert( nNew>pFd->mmapSize );
  4772   4772     assert( nNew<=pFd->mmapSizeMax );
  4773   4773     assert( nNew>0 );
  4774   4774     assert( pFd->mmapSizeActual>=pFd->mmapSize );
  4775   4775     assert( MAP_FAILED!=0 );
  4776   4776   
         4777  +#ifdef SQLITE_MMAP_READWRITE
  4777   4778     if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE;
         4779  +#endif
  4778   4780   
  4779   4781     if( pOrig ){
  4780   4782   #if HAVE_MREMAP
  4781   4783       i64 nReuse = pFd->mmapSize;
  4782   4784   #else
  4783   4785       const int szSyspage = osGetpagesize();
  4784   4786       i64 nReuse = (pFd->mmapSize & ~(szSyspage-1));

Changes to src/os_win.c.

  2598   2598     SimulateIOError(return SQLITE_IOERR_WRITE);
  2599   2599     SimulateDiskfullError(return SQLITE_FULL);
  2600   2600   
  2601   2601     OSTRACE(("WRITE pid=%lu, pFile=%p, file=%p, buffer=%p, amount=%d, "
  2602   2602              "offset=%lld, lock=%d\n", osGetCurrentProcessId(), pFile,
  2603   2603              pFile->h, pBuf, amt, offset, pFile->locktype));
  2604   2604   
  2605         -#if SQLITE_MAX_MMAP_SIZE>0
         2605  +#if defined(SQLITE_MMAP_READWRITE) && SQLITE_MAX_MMAP_SIZE>0
  2606   2606     /* Deal with as much of this write request as possible by transfering
  2607   2607     ** data from the memory mapping using memcpy().  */
  2608   2608     if( offset<pFile->mmapSize ){
  2609   2609       if( offset+amt <= pFile->mmapSize ){
  2610   2610         memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, amt);
  2611   2611         OSTRACE(("WRITE-MMAP pid=%lu, pFile=%p, file=%p, rc=SQLITE_OK\n",
  2612   2612                  osGetCurrentProcessId(), pFile, pFile->h));
................................................................................
  4092   4092     }
  4093   4093     if( nMap!=pFd->mmapSize ){
  4094   4094       void *pNew = 0;
  4095   4095       DWORD protect = PAGE_READONLY;
  4096   4096       DWORD flags = FILE_MAP_READ;
  4097   4097   
  4098   4098       winUnmapfile(pFd);
         4099  +#ifdef SQLITE_MMAP_READWRITE
  4099   4100       if( (pFd->ctrlFlags & WINFILE_RDONLY)==0 ){
  4100   4101         protect = PAGE_READWRITE;
  4101   4102         flags |= FILE_MAP_WRITE;
  4102   4103       }
         4104  +#endif
  4103   4105   #if SQLITE_OS_WINRT
  4104   4106       pFd->hMap = osCreateFileMappingFromApp(pFd->h, NULL, protect, nMap, NULL);
  4105   4107   #elif defined(SQLITE_WIN32_HAS_WIDE)
  4106   4108       pFd->hMap = osCreateFileMappingW(pFd->h, NULL, protect,
  4107   4109                                   (DWORD)((nMap>>32) & 0xffffffff),
  4108   4110                                   (DWORD)(nMap & 0xffffffff), NULL);
  4109   4111   #elif defined(SQLITE_WIN32_HAS_ANSI)

Changes to src/parse.y.

   542    542   distinct(A) ::= DISTINCT.   {A = SF_Distinct;}
   543    543   distinct(A) ::= ALL.        {A = SF_All;}
   544    544   distinct(A) ::= .           {A = 0;}
   545    545   
   546    546   // selcollist is a list of expressions that are to become the return
   547    547   // values of the SELECT statement.  The "*" in statements like
   548    548   // "SELECT * FROM ..." is encoded as a special expression with an
   549         -// opcode of TK_ALL.
          549  +// opcode of TK_ASTERISK.
   550    550   //
   551    551   %type selcollist {ExprList*}
   552    552   %destructor selcollist {sqlite3ExprListDelete(pParse->db, $$);}
   553    553   %type sclp {ExprList*}
   554    554   %destructor sclp {sqlite3ExprListDelete(pParse->db, $$);}
   555    555   sclp(A) ::= selcollist(X) COMMA.             {A = X;}
   556    556   sclp(A) ::= .                                {A = 0;}
   557    557   selcollist(A) ::= sclp(P) expr(X) as(Y).     {
   558    558      A = sqlite3ExprListAppend(pParse, P, X.pExpr);
   559    559      if( Y.n>0 ) sqlite3ExprListSetName(pParse, A, &Y, 1);
   560    560      sqlite3ExprListSetSpan(pParse,A,&X);
   561    561   }
   562    562   selcollist(A) ::= sclp(P) STAR. {
   563         -  Expr *p = sqlite3Expr(pParse->db, TK_ALL, 0);
          563  +  Expr *p = sqlite3Expr(pParse->db, TK_ASTERISK, 0);
   564    564     A = sqlite3ExprListAppend(pParse, P, p);
   565    565   }
   566    566   selcollist(A) ::= sclp(P) nm(X) DOT STAR(Y). {
   567         -  Expr *pRight = sqlite3PExpr(pParse, TK_ALL, 0, 0, &Y);
          567  +  Expr *pRight = sqlite3PExpr(pParse, TK_ASTERISK, 0, 0, &Y);
   568    568     Expr *pLeft = sqlite3PExpr(pParse, TK_ID, 0, 0, &X);
   569    569     Expr *pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight, 0);
   570    570     A = sqlite3ExprListAppend(pParse,P, pDot);
   571    571   }
   572    572   
   573    573   // An option "AS <id>" phrase that can follow one of the expressions that
   574    574   // define the result set, or one of the tables in the FROM clause.

Changes to src/select.c.

   114    114     pNew = sqlite3DbMallocZero(db, sizeof(*pNew) );
   115    115     if( pNew==0 ){
   116    116       assert( db->mallocFailed );
   117    117       pNew = &standin;
   118    118       memset(pNew, 0, sizeof(*pNew));
   119    119     }
   120    120     if( pEList==0 ){
   121         -    pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db,TK_ALL,0));
          121  +    pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db,TK_ASTERISK,0));
   122    122     }
   123    123     pNew->pEList = pEList;
   124    124     if( pSrc==0 ) pSrc = sqlite3DbMallocZero(db, sizeof(*pSrc));
   125    125     pNew->pSrc = pSrc;
   126    126     pNew->pWhere = pWhere;
   127    127     pNew->pGroupBy = pGroupBy;
   128    128     pNew->pHaving = pHaving;
................................................................................
  1655   1655           for(j=nName-1; j>0 && sqlite3Isdigit(zName[j]); j--){}
  1656   1656           if( zName[j]==':' ) nName = j;
  1657   1657         }
  1658   1658         zName = sqlite3MPrintf(db, "%.*z:%u", nName, zName, ++cnt);
  1659   1659         if( cnt>3 ) sqlite3_randomness(sizeof(cnt), &cnt);
  1660   1660       }
  1661   1661       pCol->zName = zName;
         1662  +    sqlite3ColumnPropertiesFromName(0, pCol);
  1662   1663       if( zName && sqlite3HashInsert(&ht, zName, pCol)==pCol ){
  1663   1664         db->mallocFailed = 1;
  1664   1665       }
  1665   1666     }
  1666   1667     sqlite3HashClear(&ht);
  1667   1668     if( db->mallocFailed ){
  1668   1669       for(j=0; j<i; j++){
................................................................................
  3933   3934     pNew = sqlite3DbMallocZero(db, sizeof(*pNew) );
  3934   3935     if( pNew==0 ) return WRC_Abort;
  3935   3936     memset(&dummy, 0, sizeof(dummy));
  3936   3937     pNewSrc = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&dummy,pNew,0,0);
  3937   3938     if( pNewSrc==0 ) return WRC_Abort;
  3938   3939     *pNew = *p;
  3939   3940     p->pSrc = pNewSrc;
  3940         -  p->pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ALL, 0));
         3941  +  p->pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ASTERISK, 0));
  3941   3942     p->op = TK_SELECT;
  3942   3943     p->pWhere = 0;
  3943   3944     pNew->pGroupBy = 0;
  3944   3945     pNew->pHaving = 0;
  3945   3946     pNew->pOrderBy = 0;
  3946   3947     p->pPrior = 0;
  3947   3948     p->pNext = 0;
................................................................................
  4274   4275     if( db->mallocFailed || sqliteProcessJoin(pParse, p) ){
  4275   4276       return WRC_Abort;
  4276   4277     }
  4277   4278   
  4278   4279     /* For every "*" that occurs in the column list, insert the names of
  4279   4280     ** all columns in all tables.  And for every TABLE.* insert the names
  4280   4281     ** of all columns in TABLE.  The parser inserted a special expression
  4281         -  ** with the TK_ALL operator for each "*" that it found in the column list.
  4282         -  ** The following code just has to locate the TK_ALL expressions and expand
  4283         -  ** each one to the list of all columns in all tables.
         4282  +  ** with the TK_ASTERISK operator for each "*" that it found in the column
         4283  +  ** list.  The following code just has to locate the TK_ASTERISK
         4284  +  ** expressions and expand each one to the list of all columns in
         4285  +  ** all tables.
  4284   4286     **
  4285   4287     ** The first loop just checks to see if there are any "*" operators
  4286   4288     ** that need expanding.
  4287   4289     */
  4288   4290     for(k=0; k<pEList->nExpr; k++){
  4289   4291       pE = pEList->a[k].pExpr;
  4290         -    if( pE->op==TK_ALL ) break;
         4292  +    if( pE->op==TK_ASTERISK ) break;
  4291   4293       assert( pE->op!=TK_DOT || pE->pRight!=0 );
  4292   4294       assert( pE->op!=TK_DOT || (pE->pLeft!=0 && pE->pLeft->op==TK_ID) );
  4293         -    if( pE->op==TK_DOT && pE->pRight->op==TK_ALL ) break;
         4295  +    if( pE->op==TK_DOT && pE->pRight->op==TK_ASTERISK ) break;
  4294   4296     }
  4295   4297     if( k<pEList->nExpr ){
  4296   4298       /*
  4297   4299       ** If we get here it means the result set contains one or more "*"
  4298   4300       ** operators that need to be expanded.  Loop through each expression
  4299   4301       ** in the result set and expand them one by one.
  4300   4302       */
................................................................................
  4304   4306       int longNames = (flags & SQLITE_FullColNames)!=0
  4305   4307                         && (flags & SQLITE_ShortColNames)==0;
  4306   4308   
  4307   4309       for(k=0; k<pEList->nExpr; k++){
  4308   4310         pE = a[k].pExpr;
  4309   4311         pRight = pE->pRight;
  4310   4312         assert( pE->op!=TK_DOT || pRight!=0 );
  4311         -      if( pE->op!=TK_ALL && (pE->op!=TK_DOT || pRight->op!=TK_ALL) ){
         4313  +      if( pE->op!=TK_ASTERISK
         4314  +       && (pE->op!=TK_DOT || pRight->op!=TK_ASTERISK)
         4315  +      ){
  4312   4316           /* This particular expression does not need to be expanded.
  4313   4317           */
  4314   4318           pNew = sqlite3ExprListAppend(pParse, pNew, a[k].pExpr);
  4315   4319           if( pNew ){
  4316   4320             pNew->a[pNew->nExpr-1].zName = a[k].zName;
  4317   4321             pNew->a[pNew->nExpr-1].zSpan = a[k].zSpan;
  4318   4322             a[k].zName = 0;
................................................................................
  4356   4360               assert( zName );
  4357   4361               if( zTName && pSub
  4358   4362                && sqlite3MatchSpanName(pSub->pEList->a[j].zSpan, 0, zTName, 0)==0
  4359   4363               ){
  4360   4364                 continue;
  4361   4365               }
  4362   4366   
  4363         -            /* If a column is marked as 'hidden' (currently only possible
  4364         -            ** for virtual tables), do not include it in the expanded
  4365         -            ** result-set list.
         4367  +            /* If a column is marked as 'hidden', omit it from the expanded
         4368  +            ** result-set list unless the SELECT has the SF_IncludeHidden
         4369  +            ** bit set.
  4366   4370               */
  4367         -            if( IsHiddenColumn(&pTab->aCol[j]) ){
  4368         -              assert(IsVirtual(pTab));
         4371  +            if( (p->selFlags & SF_IncludeHidden)==0
         4372  +             && IsHiddenColumn(&pTab->aCol[j]) 
         4373  +            ){
  4369   4374                 continue;
  4370   4375               }
  4371   4376               tableSeen = 1;
  4372   4377   
  4373   4378               if( i>0 && zTName==0 ){
  4374   4379                 if( (pFrom->fg.jointype & JT_NATURAL)!=0
  4375   4380                   && tableAndColumnIndex(pTabList, i, zName, 0, 0)

Changes to src/shell.c.

  2556   2556        { "number of triggers:",
  2557   2557          "SELECT count(*) FROM %s WHERE type='trigger'" },
  2558   2558        { "number of views:",
  2559   2559          "SELECT count(*) FROM %s WHERE type='view'" },
  2560   2560        { "schema size:",
  2561   2561          "SELECT total(length(sql)) FROM %s" },
  2562   2562     };
  2563         -  sqlite3_file *pFile;
         2563  +  sqlite3_file *pFile = 0;
  2564   2564     int i;
  2565   2565     char *zSchemaTab;
  2566   2566     char *zDb = nArg>=2 ? azArg[1] : "main";
  2567   2567     unsigned char aHdr[100];
  2568   2568     open_db(p, 0);
  2569   2569     if( p->db==0 ) return 1;
  2570   2570     sqlite3_file_control(p->db, zDb, SQLITE_FCNTL_FILE_POINTER, &pFile);

Changes to src/sqliteInt.h.

  1669   1669     Schema *pSchema;     /* Schema that contains this table */
  1670   1670     Table *pNextZombie;  /* Next on the Parse.pZombieTab list */
  1671   1671   };
  1672   1672   
  1673   1673   /*
  1674   1674   ** Allowed values for Table.tabFlags.
  1675   1675   **
  1676         -** TF_OOOHidden applies to virtual tables that have hidden columns that are
         1676  +** TF_OOOHidden applies to tables or view that have hidden columns that are
  1677   1677   ** followed by non-hidden columns.  Example:  "CREATE VIRTUAL TABLE x USING
  1678   1678   ** vtab1(a HIDDEN, b);".  Since "b" is a non-hidden column but "a" is hidden,
  1679   1679   ** the TF_OOOHidden attribute would apply in this case.  Such tables require
  1680   1680   ** special handling during INSERT processing.
  1681   1681   */
  1682   1682   #define TF_Readonly        0x01    /* Read-only system table */
  1683   1683   #define TF_Ephemeral       0x02    /* An ephemeral table */
................................................................................
  1692   1692   /*
  1693   1693   ** Test to see whether or not a table is a virtual table.  This is
  1694   1694   ** done as a macro so that it will be optimized out when virtual
  1695   1695   ** table support is omitted from the build.
  1696   1696   */
  1697   1697   #ifndef SQLITE_OMIT_VIRTUALTABLE
  1698   1698   #  define IsVirtual(X)      (((X)->tabFlags & TF_Virtual)!=0)
  1699         -#  define IsHiddenColumn(X) (((X)->colFlags & COLFLAG_HIDDEN)!=0)
  1700   1699   #else
  1701   1700   #  define IsVirtual(X)      0
  1702         -#  define IsHiddenColumn(X) 0
  1703   1701   #endif
         1702  +
         1703  +/*
         1704  +** Macros to determine if a column is hidden.  IsOrdinaryHiddenColumn()
         1705  +** only works for non-virtual tables (ordinary tables and views) and is
         1706  +** always false unless SQLITE_ENABLE_HIDDEN_COLUMNS is defined.  The
         1707  +** IsHiddenColumn() macro is general purpose.
         1708  +*/
         1709  +#if defined(SQLITE_ENABLE_HIDDEN_COLUMNS)
         1710  +#  define IsHiddenColumn(X)         (((X)->colFlags & COLFLAG_HIDDEN)!=0)
         1711  +#  define IsOrdinaryHiddenColumn(X) (((X)->colFlags & COLFLAG_HIDDEN)!=0)
         1712  +#elif !defined(SQLITE_OMIT_VIRTUALTABLE)
         1713  +#  define IsHiddenColumn(X)         (((X)->colFlags & COLFLAG_HIDDEN)!=0)
         1714  +#  define IsOrdinaryHiddenColumn(X) 0
         1715  +#else
         1716  +#  define IsHiddenColumn(X)         0
         1717  +#  define IsOrdinaryHiddenColumn(X) 0
         1718  +#endif
         1719  +
  1704   1720   
  1705   1721   /* Does the table have a rowid */
  1706   1722   #define HasRowid(X)     (((X)->tabFlags & TF_WithoutRowid)==0)
  1707   1723   #define VisibleRowid(X) (((X)->tabFlags & TF_NoVisibleRowid)==0)
  1708   1724   
  1709   1725   /*
  1710   1726   ** Each foreign key constraint is an instance of the following structure.
................................................................................
  2491   2507   #define SF_Values          0x0100  /* Synthesized from VALUES clause */
  2492   2508   #define SF_MultiValue      0x0200  /* Single VALUES term with multiple rows */
  2493   2509   #define SF_NestedFrom      0x0400  /* Part of a parenthesized FROM clause */
  2494   2510   #define SF_MaybeConvert    0x0800  /* Need convertCompoundSelectToSubquery() */
  2495   2511   #define SF_MinMaxAgg       0x1000  /* Aggregate containing min() or max() */
  2496   2512   #define SF_Recursive       0x2000  /* The recursive part of a recursive CTE */
  2497   2513   #define SF_Converted       0x4000  /* By convertCompoundSelectToSubquery() */
         2514  +#define SF_IncludeHidden   0x8000  /* Include hidden columns in output */
  2498   2515   
  2499   2516   
  2500   2517   /*
  2501   2518   ** The results of a SELECT can be distributed in several ways, as defined
  2502   2519   ** by one of the following macros.  The "SRT" prefix means "SELECT Result
  2503   2520   ** Type".
  2504   2521   **
................................................................................
  3310   3327   void sqlite3DeleteColumnNames(sqlite3*,Table*);
  3311   3328   int sqlite3ColumnsFromExprList(Parse*,ExprList*,i16*,Column**);
  3312   3329   Table *sqlite3ResultSetOfSelect(Parse*,Select*);
  3313   3330   void sqlite3OpenMasterTable(Parse *, int);
  3314   3331   Index *sqlite3PrimaryKeyIndex(Table*);
  3315   3332   i16 sqlite3ColumnOfIndex(Index*, i16);
  3316   3333   void sqlite3StartTable(Parse*,Token*,Token*,int,int,int,int);
         3334  +void sqlite3ColumnPropertiesFromName(Table*, Column*);
  3317   3335   void sqlite3AddColumn(Parse*,Token*);
  3318   3336   void sqlite3AddNotNull(Parse*, int);
  3319   3337   void sqlite3AddPrimaryKey(Parse*, ExprList*, int, int, int);
  3320   3338   void sqlite3AddCheckConstraint(Parse*, Expr*);
  3321   3339   void sqlite3AddColumnType(Parse*,Token*);
  3322   3340   void sqlite3AddDefaultValue(Parse*,ExprSpan*);
  3323   3341   void sqlite3AddCollateType(Parse*, Token*);

Changes to src/test1.c.

  2234   2234   */
  2235   2235   static int test_config_sqllog(
  2236   2236     void * clientData,
  2237   2237     Tcl_Interp *interp,
  2238   2238     int objc,
  2239   2239     Tcl_Obj *CONST objv[]
  2240   2240   ){
  2241         -  sqlite3_stmt *pStmt;            /* First argument */
  2242   2241     if( objc!=1 ){
  2243   2242       Tcl_WrongNumArgs(interp, 1, objv, "");
  2244   2243       return TCL_ERROR;
  2245   2244     }
  2246   2245     sqlite3_config(SQLITE_CONFIG_SQLLOG, 0, 0);
  2247   2246     return TCL_OK;
  2248   2247   }
  2249   2248   #endif
         2249  +
         2250  +/*
         2251  +** Usage: vfs_current_time_int64
         2252  +**
         2253  +** Return the value returned by the default VFS's xCurrentTimeInt64 method.
         2254  +*/
         2255  +static int vfsCurrentTimeInt64(
         2256  +  void * clientData,
         2257  +  Tcl_Interp *interp,
         2258  +  int objc,
         2259  +  Tcl_Obj *CONST objv[]
         2260  +){
         2261  +  i64 t;
         2262  +  sqlite3_vfs *pVfs = sqlite3_vfs_find(0);
         2263  +  if( objc!=1 ){
         2264  +    Tcl_WrongNumArgs(interp, 1, objv, "");
         2265  +    return TCL_ERROR;
         2266  +  }
         2267  +  pVfs->xCurrentTimeInt64(pVfs, &t);
         2268  +  Tcl_SetObjResult(interp, Tcl_NewWideIntObj(t));
         2269  +  return TCL_OK;
         2270  +}
  2250   2271   
  2251   2272   /*
  2252   2273   ** Usage:  sqlite3_next_stmt  DB  STMT
  2253   2274   **
  2254   2275   ** Return the next statment in sequence after STMT.
  2255   2276   */
  2256   2277   static int test_next_stmt(
................................................................................
  7057   7078   #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
  7058   7079        { "sqlite3_stmt_scanstatus",       test_stmt_scanstatus,   0 },
  7059   7080        { "sqlite3_stmt_scanstatus_reset", test_stmt_scanstatus_reset,   0 },
  7060   7081   #endif
  7061   7082   #ifdef SQLITE_ENABLE_SQLLOG
  7062   7083        { "sqlite3_config_sqllog",         test_config_sqllog,   0 },
  7063   7084   #endif
  7064         -
         7085  +     { "vfs_current_time_int64",           vfsCurrentTimeInt64,   0 },
  7065   7086     };
  7066   7087     static int bitmask_size = sizeof(Bitmask)*8;
  7067   7088     static int longdouble_size = sizeof(LONGDOUBLE_TYPE);
  7068   7089     int i;
  7069   7090     extern int sqlite3_sync_count, sqlite3_fullsync_count;
  7070   7091     extern int sqlite3_opentemp_count;
  7071   7092     extern int sqlite3_like_count;

Changes to src/test_config.c.

   120    120   #endif
   121    121   
   122    122   #ifdef SQLITE_ENABLE_CURSOR_HINTS
   123    123     Tcl_SetVar2(interp, "sqlite_options", "cursorhints", "1", TCL_GLOBAL_ONLY);
   124    124   #else
   125    125     Tcl_SetVar2(interp, "sqlite_options", "cursorhints", "0", TCL_GLOBAL_ONLY);
   126    126   #endif
          127  +
          128  +#ifdef SQLITE_ENABLE_HIDDEN_COLUMNS
          129  +  Tcl_SetVar2(interp, "sqlite_options", "hiddencolumns", "1", TCL_GLOBAL_ONLY);
          130  +#else
          131  +  Tcl_SetVar2(interp, "sqlite_options", "hiddencolumns", "0", TCL_GLOBAL_ONLY);
          132  +#endif
   127    133   
   128    134   #ifdef SQLITE_ENABLE_MEMSYS3
   129    135     Tcl_SetVar2(interp, "sqlite_options", "mem3", "1", TCL_GLOBAL_ONLY);
   130    136   #else
   131    137     Tcl_SetVar2(interp, "sqlite_options", "mem3", "0", TCL_GLOBAL_ONLY);
   132    138   #endif
   133    139   

Added test/hidden.test.

            1  +# 2015 November 18
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +# Test the __hidden__ hack.
           13  +#
           14  +
           15  +set testdir [file dirname $argv0]
           16  +source $testdir/tester.tcl
           17  +set testprefix hidden
           18  +
           19  +ifcapable !hiddencolumns {
           20  +  finish_test
           21  +  return
           22  +}
           23  +
           24  +do_execsql_test 1.1 {
           25  +  CREATE TABLE t1(__hidden__a, b);
           26  +  INSERT INTO t1 VALUES('1');
           27  +  INSERT INTO t1(__hidden__a, b) VALUES('x', 'y');
           28  +} {}
           29  +
           30  +do_execsql_test 1.2 {
           31  +  SELECT * FROM t1;
           32  +} {1 y}
           33  +
           34  +do_execsql_test 1.3 {
           35  +  SELECT __hidden__a, * FROM t1;
           36  +} {{} 1 x y}
           37  +
           38  +foreach {tn view} {
           39  +  1 { CREATE VIEW v1(a, b, __hidden__c) AS SELECT a, b, c FROM x1 }
           40  +  2 { CREATE VIEW v1 AS SELECT a, b, c AS __hidden__c FROM x1 }
           41  +} {
           42  +  do_execsql_test 2.$tn.1 {
           43  +    DROP TABLE IF EXISTS x1;
           44  +    CREATE TABLE x1(a, b, c);
           45  +    INSERT INTO x1 VALUES(1, 2, 3);
           46  +  }
           47  +
           48  +  catchsql { DROP VIEW v1 }
           49  +  execsql $view
           50  +
           51  +  do_execsql_test 2.$tn.2 {
           52  +    SELECT a, b, __hidden__c FROM v1;
           53  +  } {1 2 3}
           54  +  
           55  +  do_execsql_test 2.$tn.3 {
           56  +    SELECT * FROM v1;
           57  +  } {1 2}
           58  +  
           59  +  do_execsql_test 2.$tn.4 {
           60  +    CREATE TRIGGER tr1 INSTEAD OF INSERT ON v1 BEGIN
           61  +      INSERT INTO x1 VALUES(new.a, new.b, new.__hidden__c);
           62  +    END;
           63  +  
           64  +    INSERT INTO v1 VALUES(4, 5);
           65  +    SELECT * FROM x1;
           66  +  } {1 2 3 4 5 {}}
           67  +  
           68  +  do_execsql_test 2.$tn.5 {
           69  +    INSERT INTO v1(a, b, __hidden__c) VALUES(7, 8, 9);
           70  +    SELECT * FROM x1;
           71  +  } {1 2 3 4 5 {} 7 8 9}
           72  +}
           73  +
           74  +#-------------------------------------------------------------------------
           75  +# Test INSERT INTO ... SELECT ... statements that write to tables with
           76  +# hidden columns.
           77  +#
           78  +do_execsql_test 3.1 {
           79  +  CREATE TABLE t4(a, __hidden__b, c);
           80  +  INSERT INTO t4 SELECT 1, 2;
           81  +  SELECT a, __hidden__b, c FROM t4;
           82  +} {1 {} 2}
           83  +
           84  +do_execsql_test 3.2.1 {
           85  +  CREATE TABLE t5(__hidden__a, b, c);
           86  +  CREATE TABLE t6(__hidden__a, b, c);
           87  +  INSERT INTO t6(__hidden__a, b, c) VALUES(1, 2, 3);
           88  +  INSERT INTO t6(__hidden__a, b, c) VALUES(4, 5, 6);
           89  +  INSERT INTO t6(__hidden__a, b, c) VALUES(7, 8, 9);
           90  +}
           91  +
           92  +do_execsql_test 3.2.2 {
           93  +  INSERT INTO t5 SELECT * FROM t6;
           94  +  SELECT * FROM t5;
           95  +} {2 3   5 6   8 9}
           96  +
           97  +do_execsql_test 3.2.3 {
           98  +  SELECT __hidden__a FROM t5;
           99  +} {{} {} {}}
          100  +
          101  +
          102  +do_execsql_test 3.3.1 {
          103  +  CREATE TABLE t5a(a, b, __hidden__c);
          104  +  CREATE TABLE t6a(a, b, __hidden__c);
          105  +  INSERT INTO t6a(a, b, __hidden__c) VALUES(1, 2, 3);
          106  +  INSERT INTO t6a(a, b, __hidden__c) VALUES(4, 5, 6);
          107  +  INSERT INTO t6a(a, b, __hidden__c) VALUES(7, 8, 9);
          108  +}
          109  +
          110  +do_execsql_test 3.3.2 {
          111  +  INSERT INTO t5a SELECT * FROM t6a;
          112  +  SELECT * FROM t5a;
          113  +} {1 2   4 5   7 8}
          114  +
          115  +do_execsql_test 3.3.3 {
          116  +  SELECT __hidden__c FROM t5a;
          117  +} {{} {} {}}
          118  +
          119  +do_execsql_test 3.4.1 {
          120  +  CREATE TABLE t5b(a, __hidden__b, c);
          121  +  CREATE TABLE t6b(a, b, __hidden__c);
          122  +  INSERT INTO t6b(a, b, __hidden__c) VALUES(1, 2, 3);
          123  +  INSERT INTO t6b(a, b, __hidden__c) VALUES(4, 5, 6);
          124  +  INSERT INTO t6b(a, b, __hidden__c) VALUES(7, 8, 9);
          125  +}
          126  +
          127  +do_execsql_test 3.4.2 {
          128  +  INSERT INTO t5b SELECT * FROM t6b;
          129  +  SELECT * FROM t5b;
          130  +} {1 2   4 5   7 8}
          131  +
          132  +do_execsql_test 3.4.3 {
          133  +  SELECT __hidden__b FROM t5b;
          134  +} {{} {} {}}
          135  +
          136  +#-------------------------------------------------------------------------
          137  +# Test VACUUM
          138  +#
          139  +reset_db
          140  +do_execsql_test 4.1 {
          141  +  CREATE TABLE t1(a, __hidden__b, c UNIQUE);
          142  +  INSERT INTO t1(a, __hidden__b, c) VALUES(1, 2, 3);
          143  +  INSERT INTO t1(a, __hidden__b, c) VALUES(4, 5, 6);
          144  +  INSERT INTO t1(a, __hidden__b, c) VALUES(7, 8, 9);
          145  +  DELETE FROM t1 WHERE __hidden__b = 5;
          146  +  SELECT rowid, a, __hidden__b, c FROM t1;
          147  +} {1 1 2 3   3 7 8 9}
          148  +do_execsql_test 4.2 {
          149  +  VACUUM;
          150  +  SELECT rowid, a, __hidden__b, c FROM t1;
          151  +} {1 1 2 3   3 7 8 9}
          152  +
          153  +finish_test

Changes to test/releasetest.tcl.

   115    115       -DSQLITE_TCL_DEFAULT_FULLMUTEX=1
   116    116       -DSQLITE_ENABLE_FTS3=1
   117    117       -DSQLITE_ENABLE_RTREE=1
   118    118       -DSQLITE_ENABLE_MEMSYS5=1
   119    119       -DSQLITE_ENABLE_MEMSYS3=1
   120    120       -DSQLITE_ENABLE_COLUMN_METADATA=1
   121    121       -DSQLITE_ENABLE_STAT4
          122  +    -DSQLITE_ENABLE_HIDDEN_COLUMNS
   122    123       -DSQLITE_MAX_ATTACHED=125
   123    124     }
   124    125     "Fast-One" {
   125    126       -O6
   126    127       -DSQLITE_ENABLE_FTS4=1
   127    128       -DSQLITE_ENABLE_RTREE=1
   128    129       -DSQLITE_ENABLE_STAT4
................................................................................
   141    142       -DSQLITE_ENABLE_ATOMIC_WRITE=1
   142    143       -DSQLITE_ENABLE_IOTRACE=1
   143    144       -DSQLITE_ENABLE_MEMORY_MANAGEMENT=1
   144    145       -DSQLITE_MAX_PAGE_SIZE=4096
   145    146       -DSQLITE_OMIT_LOAD_EXTENSION=1
   146    147       -DSQLITE_OMIT_PROGRESS_CALLBACK=1
   147    148       -DSQLITE_OMIT_VIRTUALTABLE=1
          149  +    -DSQLITE_ENABLE_HIDDEN_COLUMNS
   148    150       -DSQLITE_TEMP_STORE=3
   149    151       --enable-json1
   150    152     }
   151    153     "Device-Two" {
   152    154       -DSQLITE_4_BYTE_ALIGNED_MALLOC=1
   153    155       -DSQLITE_DEFAULT_AUTOVACUUM=1
   154    156       -DSQLITE_DEFAULT_CACHE_SIZE=1000
................................................................................
   209    211       -DSQLITE_OMIT_LOOKASIDE=1
   210    212       -DHAVE_USLEEP=1
   211    213     }
   212    214     "Valgrind" {
   213    215       -DSQLITE_ENABLE_STAT4
   214    216       -DSQLITE_ENABLE_FTS4
   215    217       -DSQLITE_ENABLE_RTREE
          218  +    -DSQLITE_ENABLE_HIDDEN_COLUMNS
   216    219       --enable-json1
   217    220     }
   218    221   
   219    222     # The next group of configurations are used only by the
   220    223     # Failure-Detection platform.  They are all the same, but we need
   221    224     # different names for them all so that they results appear in separate
   222    225     # subdirectories.

Changes to tool/addopcodes.tcl.

    33     33     FUNCTION
    34     34     COLUMN
    35     35     AGG_FUNCTION
    36     36     AGG_COLUMN
    37     37     UMINUS
    38     38     UPLUS
    39     39     REGISTER
           40  +  ASTERISK
    40     41     SPACE
    41     42     ILLEGAL
    42     43   }
    43     44   if {[lrange $extras end-1 end]!="SPACE ILLEGAL"} {
    44     45     error "SPACE and ILLEGAL must be the last two token codes and they\
    45     46            must be in that order"
    46     47   }

Changes to tool/mkkeywordhash.c.

   324    324   }
   325    325   
   326    326   /*
   327    327   ** This routine does the work.  The generated code is printed on standard
   328    328   ** output.
   329    329   */
   330    330   int main(int argc, char **argv){
   331         -  int i, j, k, h, m;
          331  +  int i, j, k, h;
   332    332     int bestSize, bestCount;
   333    333     int count;
   334    334     int nChar;
   335    335     int totalLen = 0;
   336    336     int aHash[1000];  /* 1000 is much bigger than nKeyword */
   337    337     char zText[2000];
   338    338