/ Check-in [842b2116]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Reduce the amount of heap required to store many schemas by storing each column datatype appended to the column name, rather than as a separate allocation.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | schema-storage
Files: files | file ages | folders
SHA1: 842b21162713bb141b845b01c136457a31af4ab0
User & Date: drh 2016-02-29 15:53:11
Context
2016-03-03
21:22
Reduce the amount of heap memory required to store many schemas by storing the column datatype appended to the column name (with a \000 separator), rather than in separate memory allocation. check-in: 16fbf2e1 user: drh tags: trunk
2016-02-29
15:53
Reduce the amount of heap required to store many schemas by storing each column datatype appended to the column name, rather than as a separate allocation. Closed-Leaf check-in: 842b2116 user: drh tags: schema-storage
13:44
Very minor improvement to the performance and reduction in size to the parser by capturing the name and datatype of table columns in a single grammar rule reduction. check-in: 4b55c520 user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/alter.c.

   782    782       goto exit_begin_add_column;
   783    783     }
   784    784     memcpy(pNew->aCol, pTab->aCol, sizeof(Column)*pNew->nCol);
   785    785     for(i=0; i<pNew->nCol; i++){
   786    786       Column *pCol = &pNew->aCol[i];
   787    787       pCol->zName = sqlite3DbStrDup(db, pCol->zName);
   788    788       pCol->zColl = 0;
   789         -    pCol->zType = 0;
   790    789       pCol->pDflt = 0;
   791    790     }
   792    791     pNew->pSchema = db->aDb[iDb].pSchema;
   793    792     pNew->addColOffset = pTab->addColOffset;
   794    793     pNew->nRef = 1;
   795    794   
   796    795     /* Begin a transaction and increment the schema cookie.  */

Changes to src/build.c.

   567    567     int i;
   568    568     Column *pCol;
   569    569     assert( pTable!=0 );
   570    570     if( (pCol = pTable->aCol)!=0 ){
   571    571       for(i=0; i<pTable->nCol; i++, pCol++){
   572    572         sqlite3DbFree(db, pCol->zName);
   573    573         sqlite3ExprDelete(db, pCol->pDflt);
   574         -      sqlite3DbFree(db, pCol->zType);
   575    574         sqlite3DbFree(db, pCol->zColl);
   576    575       }
   577    576       sqlite3DbFree(db, pTable->aCol);
   578    577     }
   579    578   }
   580    579   
   581    580   /*
................................................................................
  1038   1037   ** first to get things going.  Then this routine is called for each
  1039   1038   ** column.
  1040   1039   */
  1041   1040   void sqlite3AddColumn(Parse *pParse, Token *pName, Token *pType){
  1042   1041     Table *p;
  1043   1042     int i;
  1044   1043     char *z;
         1044  +  char *zType;
  1045   1045     Column *pCol;
  1046   1046     sqlite3 *db = pParse->db;
  1047   1047     if( (p = pParse->pNewTable)==0 ) return;
  1048   1048   #if SQLITE_MAX_COLUMN
  1049   1049     if( p->nCol+1>db->aLimit[SQLITE_LIMIT_COLUMN] ){
  1050   1050       sqlite3ErrorMsg(pParse, "too many columns on %s", p->zName);
  1051   1051       return;
  1052   1052     }
  1053   1053   #endif
  1054         -  z = sqlite3NameFromToken(db, pName);
         1054  +  z = sqlite3DbMallocRaw(db, pName->n + pType->n + 2);
  1055   1055     if( z==0 ) return;
         1056  +  memcpy(z, pName->z, pName->n);
         1057  +  z[pName->n] = 0;
         1058  +  sqlite3Dequote(z);
         1059  +  zType = z + sqlite3Strlen30(z) + 1;
         1060  +  memcpy(zType, pType->z, pType->n);
         1061  +  zType[pType->n] = 0;
  1056   1062     for(i=0; i<p->nCol; i++){
  1057   1063       if( sqlite3_stricmp(z, p->aCol[i].zName)==0 ){
  1058   1064         sqlite3ErrorMsg(pParse, "duplicate column name: %s", z);
  1059   1065         sqlite3DbFree(db, z);
  1060   1066         return;
  1061   1067       }
  1062   1068     }
................................................................................
  1076   1082    
  1077   1083     if( pType->n==0 ){
  1078   1084       /* If there is no type specified, columns have the default affinity
  1079   1085       ** 'BLOB'. */
  1080   1086       pCol->affinity = SQLITE_AFF_BLOB;
  1081   1087       pCol->szEst = 1;
  1082   1088     }else{
  1083         -    pCol->zType = sqlite3NameFromToken(pParse->db, pType);
  1084         -    pCol->affinity = sqlite3AffinityType(pCol->zType, &pCol->szEst);
         1089  +    pCol->affinity = sqlite3AffinityType(zType, &pCol->szEst);
  1085   1090     }
  1086   1091     p->nCol++;
  1087   1092     pParse->constraintName.n = 0;
  1088   1093   }
  1089   1094   
  1090   1095   /*
  1091   1096   ** This routine is called by the parser while in the middle of
................................................................................
  1273   1278     Parse *pParse,    /* Parsing context */
  1274   1279     ExprList *pList,  /* List of field names to be indexed */
  1275   1280     int onError,      /* What to do with a uniqueness conflict */
  1276   1281     int autoInc,      /* True if the AUTOINCREMENT keyword is present */
  1277   1282     int sortOrder     /* SQLITE_SO_ASC or SQLITE_SO_DESC */
  1278   1283   ){
  1279   1284     Table *pTab = pParse->pNewTable;
  1280         -  char *zType = 0;
         1285  +  const char *zName = 0;
  1281   1286     int iCol = -1, i;
  1282   1287     int nTerm;
  1283   1288     if( pTab==0 || IN_DECLARE_VTAB ) goto primary_key_exit;
  1284   1289     if( pTab->tabFlags & TF_HasPrimaryKey ){
  1285   1290       sqlite3ErrorMsg(pParse, 
  1286   1291         "table \"%s\" has more than one primary key", pTab->zName);
  1287   1292       goto primary_key_exit;
  1288   1293     }
  1289   1294     pTab->tabFlags |= TF_HasPrimaryKey;
  1290   1295     if( pList==0 ){
  1291   1296       iCol = pTab->nCol - 1;
  1292   1297       pTab->aCol[iCol].colFlags |= COLFLAG_PRIMKEY;
  1293         -    zType = pTab->aCol[iCol].zType;
         1298  +    zName = pTab->aCol[iCol].zName;
  1294   1299       nTerm = 1;
  1295   1300     }else{
  1296   1301       nTerm = pList->nExpr;
  1297   1302       for(i=0; i<nTerm; i++){
  1298   1303         Expr *pCExpr = sqlite3ExprSkipCollate(pList->a[i].pExpr);
  1299   1304         assert( pCExpr!=0 );
  1300   1305         sqlite3StringToId(pCExpr);
  1301   1306         if( pCExpr->op==TK_ID ){
  1302   1307           const char *zCName = pCExpr->u.zToken;
  1303   1308           for(iCol=0; iCol<pTab->nCol; iCol++){
  1304   1309             if( sqlite3StrICmp(zCName, pTab->aCol[iCol].zName)==0 ){
  1305   1310               pTab->aCol[iCol].colFlags |= COLFLAG_PRIMKEY;
  1306         -            zType = pTab->aCol[iCol].zType;
         1311  +            zName = pTab->aCol[iCol].zName;
  1307   1312               break;
  1308   1313             }
  1309   1314           }
  1310   1315         }
  1311   1316       }
  1312   1317     }
  1313   1318     if( nTerm==1
  1314         -   && zType && sqlite3StrICmp(zType, "INTEGER")==0
         1319  +   && zName
         1320  +   && sqlite3StrICmp(sqlite3StrNext(zName), "INTEGER")==0
  1315   1321      && sortOrder!=SQLITE_SO_DESC
  1316   1322     ){
  1317   1323       pTab->iPKey = iCol;
  1318   1324       pTab->keyConf = (u8)onError;
  1319   1325       assert( autoInc==0 || autoInc==1 );
  1320   1326       pTab->tabFlags |= autoInc*TF_Autoincrement;
  1321   1327       if( pList ) pParse->iPkSortOrder = pList->a[0].sortOrder;

Changes to src/main.c.

  3331   3331     **     1. The specified column name was rowid", "oid" or "_rowid_" 
  3332   3332     **        and there is no explicitly declared IPK column. 
  3333   3333     **
  3334   3334     **     2. The table is not a view and the column name identified an 
  3335   3335     **        explicitly declared column. Copy meta information from *pCol.
  3336   3336     */ 
  3337   3337     if( pCol ){
  3338         -    zDataType = pCol->zType;
         3338  +    zDataType = sqlite3StrNext(pCol->zName);
         3339  +    if( zDataType[0]==0 ) zDataType = 0;
  3339   3340       zCollSeq = pCol->zColl;
  3340   3341       notnull = pCol->notNull!=0;
  3341   3342       primarykey  = (pCol->colFlags & COLFLAG_PRIMKEY)!=0;
  3342   3343       autoinc = pTab->iPKey==iCol && (pTab->tabFlags & TF_Autoincrement)!=0;
  3343   3344     }else{
  3344   3345       zDataType = "INTEGER";
  3345   3346       primarykey = 1;

Changes to src/pragma.c.

  1061   1061         Column *pCol;
  1062   1062         Index *pPk = sqlite3PrimaryKeyIndex(pTab);
  1063   1063         pParse->nMem = 6;
  1064   1064         sqlite3CodeVerifySchema(pParse, iDb);
  1065   1065         setAllColumnNames(v, 6, azCol); assert( 6==ArraySize(azCol) );
  1066   1066         sqlite3ViewGetColumnNames(pParse, pTab);
  1067   1067         for(i=0, pCol=pTab->aCol; i<pTab->nCol; i++, pCol++){
         1068  +        const char *zName;
  1068   1069           if( IsHiddenColumn(pCol) ){
  1069   1070             nHidden++;
  1070   1071             continue;
  1071   1072           }
  1072   1073           if( (pCol->colFlags & COLFLAG_PRIMKEY)==0 ){
  1073   1074             k = 0;
  1074   1075           }else if( pPk==0 ){
  1075   1076             k = 1;
  1076   1077           }else{
  1077   1078             for(k=1; k<=pTab->nCol && pPk->aiColumn[k-1]!=i; k++){}
  1078   1079           }
  1079   1080           assert( pCol->pDflt==0 || pCol->pDflt->op==TK_SPAN );
         1081  +        zName = pCol->zName;
  1080   1082           sqlite3VdbeMultiLoad(v, 1, "issisi",
  1081   1083                  i-nHidden,
  1082         -               pCol->zName,
  1083         -               pCol->zType ? pCol->zType : "",
         1084  +               zName,
         1085  +               sqlite3StrNext(zName),
  1084   1086                  pCol->notNull ? 1 : 0,
  1085   1087                  pCol->pDflt ? pCol->pDflt->u.zToken : 0,
  1086   1088                  k);
  1087   1089           sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 6);
  1088   1090         }
  1089   1091       }
  1090   1092     }

Changes to src/select.c.

  1425   1425           if( iCol<0 ) iCol = pTab->iPKey;
  1426   1426           assert( iCol==-1 || (iCol>=0 && iCol<pTab->nCol) );
  1427   1427   #ifdef SQLITE_ENABLE_COLUMN_METADATA
  1428   1428           if( iCol<0 ){
  1429   1429             zType = "INTEGER";
  1430   1430             zOrigCol = "rowid";
  1431   1431           }else{
  1432         -          zType = pTab->aCol[iCol].zType;
  1433   1432             zOrigCol = pTab->aCol[iCol].zName;
         1433  +          zType = sqlite3StrNext(zOrigCol);
  1434   1434             estWidth = pTab->aCol[iCol].szEst;
  1435   1435           }
  1436   1436           zOrigTab = pTab->zName;
  1437   1437           if( pNC->pParse ){
  1438   1438             int iDb = sqlite3SchemaToIndex(pNC->pParse->db, pTab->pSchema);
  1439   1439             zOrigDb = pNC->pParse->db->aDb[iDb].zName;
  1440   1440           }
  1441   1441   #else
  1442   1442           if( iCol<0 ){
  1443   1443             zType = "INTEGER";
  1444   1444           }else{
  1445         -          zType = pTab->aCol[iCol].zType;
         1445  +          zType = sqlite3StrNext(pTab->aCol[iCol].zName);
  1446   1446             estWidth = pTab->aCol[iCol].szEst;
  1447   1447           }
  1448   1448   #endif
  1449   1449         }
  1450   1450         break;
  1451   1451       }
  1452   1452   #ifndef SQLITE_OMIT_SUBQUERY
................................................................................
  1723   1723     assert( pTab->nCol==pSelect->pEList->nExpr || db->mallocFailed );
  1724   1724     if( db->mallocFailed ) return;
  1725   1725     memset(&sNC, 0, sizeof(sNC));
  1726   1726     sNC.pSrcList = pSelect->pSrc;
  1727   1727     a = pSelect->pEList->a;
  1728   1728     for(i=0, pCol=pTab->aCol; i<pTab->nCol; i++, pCol++){
  1729   1729       p = a[i].pExpr;
  1730         -    if( pCol->zType==0 ){
  1731         -      pCol->zType = sqlite3DbStrDup(db, 
  1732         -                        columnType(&sNC, p,0,0,0, &pCol->szEst));
  1733         -    }
         1730  +    columnType(&sNC, p, 0, 0, 0, &pCol->szEst);
  1734   1731       szAll += pCol->szEst;
  1735   1732       pCol->affinity = sqlite3ExprAffinity(p);
  1736   1733       if( pCol->affinity==0 ) pCol->affinity = SQLITE_AFF_BLOB;
  1737   1734       pColl = sqlite3ExprCollSeq(pParse, p);
  1738   1735       if( pColl && pCol->zColl==0 ){
  1739   1736         pCol->zColl = sqlite3DbStrDup(db, pColl->zName);
  1740   1737       }

Changes to src/sqliteInt.h.

  1539   1539   };
  1540   1540   
  1541   1541   /*
  1542   1542   ** information about each column of an SQL table is held in an instance
  1543   1543   ** of this structure.
  1544   1544   */
  1545   1545   struct Column {
  1546         -  char *zName;     /* Name of this column */
         1546  +  char *zName;     /* Name of this column, \000, then the type */
  1547   1547     Expr *pDflt;     /* Default value of this column */
  1548         -  char *zType;     /* Data type for this column */
  1549   1548     char *zColl;     /* Collating sequence.  If NULL, use the default */
  1550   1549     u8 notNull;      /* An OE_ code for handling a NOT NULL constraint */
  1551   1550     char affinity;   /* One of the SQLITE_AFF_... values */
  1552   1551     u8 szEst;        /* Estimated size of value in this column. sizeof(INT)==1 */
  1553   1552     u8 colFlags;     /* Boolean properties.  See COLFLAG_ defines below */
  1554   1553   };
  1555   1554   
................................................................................
  3254   3253   #endif
  3255   3254   
  3256   3255   /*
  3257   3256   ** Internal function prototypes
  3258   3257   */
  3259   3258   int sqlite3StrICmp(const char*,const char*);
  3260   3259   int sqlite3Strlen30(const char*);
         3260  +const char *sqlite3StrNext(const char*);
  3261   3261   #define sqlite3StrNICmp sqlite3_strnicmp
  3262   3262   
  3263   3263   int sqlite3MallocInit(void);
  3264   3264   void sqlite3MallocEnd(void);
  3265   3265   void *sqlite3Malloc(u64);
  3266   3266   void *sqlite3MallocZero(u64);
  3267   3267   void *sqlite3DbMallocZero(sqlite3*, u64);

Changes to src/util.c.

   104    104   ** than the actual length of the string.  For very long strings (greater
   105    105   ** than 1GiB) the value returned might be less than the true string length.
   106    106   */
   107    107   int sqlite3Strlen30(const char *z){
   108    108     if( z==0 ) return 0;
   109    109     return 0x3fffffff & (int)strlen(z);
   110    110   }
          111  +
          112  +/*
          113  +** The string z[] is followed immediately by another string.  Return
          114  +** a poiner to that other string.
          115  +*/
          116  +const char *sqlite3StrNext(const char *z){
          117  +  return z + strlen(z) + 1;
          118  +}
   111    119   
   112    120   /*
   113    121   ** Set the current error code to err_code and clear any prior error message.
   114    122   */
   115    123   void sqlite3Error(sqlite3 *db, int err_code){
   116    124     assert( db!=0 );
   117    125     db->errCode = err_code;

Changes to src/vtab.c.

   560    560         ** columns of the table to see if any of them contain the token "hidden".
   561    561         ** If so, set the Column COLFLAG_HIDDEN flag and remove the token from
   562    562         ** the type string.  */
   563    563         pVTable->pNext = pTab->pVTable;
   564    564         pTab->pVTable = pVTable;
   565    565   
   566    566         for(iCol=0; iCol<pTab->nCol; iCol++){
   567         -        char *zType = pTab->aCol[iCol].zType;
          567  +        char *zType = (char*)sqlite3StrNext(pTab->aCol[iCol].zName);
   568    568           int nType;
   569    569           int i = 0;
   570         -        if( !zType ){
          570  +        if( !zType[0] ){
   571    571             pTab->tabFlags |= oooHidden;
   572    572             continue;
   573    573           }
   574    574           nType = sqlite3Strlen30(zType);
   575    575           if( sqlite3StrNICmp("hidden", zType, 6)||(zType[6] && zType[6]!=' ') ){
   576    576             for(i=0; i<nType; i++){
   577    577               if( (0==sqlite3StrNICmp(" hidden", &zType[i], 7))