/ Check-in [221aee72]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add the {quote: StrAccum} object for accumulating strings. Revamp xprintf to use the new object. Rewrite the group_concat() function to use the new object. Productize and test the group_concat() function. (CVS 4578)
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 221aee72be040769e8026b91648f03c6366a8821
User & Date: drh 2007-11-28 22:36:41
Context
2007-11-29
17:05
Optimisations for expressions of the form "<value> IN (SELECT <column> FROM )". (CVS 4579)
check-in: 56d0e326 user: danielk1977 tags: trunk
2007-11-28
22:36
Add the {quote: StrAccum} object for accumulating strings. Revamp xprintf to use the new object. Rewrite the group_concat() function to use the new object. Productize and test the group_concat() function. (CVS 4578) check-in: 221aee72 user: drh tags: trunk
16:19
Change the BTree so that it uses the Pagers temporary page space when reorganizing the rows on a page, rather than mallocing for space of its own. In this way, we avoid having to deal with a malloc failure deep down inside the page reorganizer. Ticket #2806 (CVS 4577) check-in: 98960132 user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/func.c.

    12     12   ** This file contains the C functions that implement various SQL
    13     13   ** functions of SQLite.  
    14     14   **
    15     15   ** There is only one exported symbol in this file - the function
    16     16   ** sqliteRegisterBuildinFunctions() found at the bottom of the file.
    17     17   ** All other code has file scope.
    18     18   **
    19         -** $Id: func.c,v 1.176 2007/11/01 17:38:31 drh Exp $
           19  +** $Id: func.c,v 1.177 2007/11/28 22:36:41 drh Exp $
    20     20   */
    21     21   #include "sqliteInt.h"
    22     22   #include <ctype.h>
    23     23   #include <stdlib.h>
    24     24   #include <assert.h>
    25     25   #include "vdbeInt.h"
    26     26   
................................................................................
  1308   1308       if( pRes->flags ){
  1309   1309         sqlite3_result_value(context, pRes);
  1310   1310       }
  1311   1311       sqlite3VdbeMemRelease(pRes);
  1312   1312     }
  1313   1313   }
  1314   1314   
  1315         -#ifdef SQLITE_GROUP_CONCAT
  1316   1315   /*
  1317   1316   ** group_concat(EXPR, ?SEPARATOR?)
  1318   1317   */
  1319   1318   static void groupConcatStep(
  1320   1319     sqlite3_context *context,
  1321   1320     int argc,
  1322   1321     sqlite3_value **argv
  1323   1322   ){
  1324   1323     const char *zVal;
  1325         -  char **pzAccumulator;
         1324  +  StrAccum *pAccum;
  1326   1325     const char *zSep;
         1326  +  int nVal, nSep;
  1327   1327     if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return;
  1328         -  zVal = sqlite3_value_text(argv[0]);
  1329         -  pzAccumulator = (char**)sqlite3_aggregate_context(context, sizeof(char*));
  1330         -  if( pzAccumulator ){
  1331         -    if( *pzAccumulator==0 ){
  1332         -      *pzAccumulator = sqlite3_mprintf("%s", zVal);
  1333         -    }else{
         1328  +  pAccum = (StrAccum*)sqlite3_aggregate_context(context, sizeof(*pAccum));
         1329  +
         1330  +  if( pAccum ){
         1331  +    pAccum->useMalloc = 1;
         1332  +    if( pAccum->nChar ){
  1334   1333         if( argc==2 ){
  1335         -        zSep = sqlite3_value_text(argv[1]);
         1334  +        zSep = (char*)sqlite3_value_text(argv[1]);
         1335  +        nSep = sqlite3_value_bytes(argv[1]);
  1336   1336         }else{
  1337   1337           zSep = ",";
         1338  +        nSep = 1;
  1338   1339         }
  1339         -      *pzAccumulator = sqlite3_mprintf("%z%s%s", *pzAccumulator, zSep, zVal);
         1340  +      sqlite3StrAccumAppend(pAccum, zSep, nSep);
  1340   1341       }
         1342  +    zVal = (char*)sqlite3_value_text(argv[0]);
         1343  +    nVal = sqlite3_value_bytes(argv[0]);
         1344  +    sqlite3StrAccumAppend(pAccum, zVal, nVal);
  1341   1345     }
  1342   1346   }
  1343   1347   static void groupConcatFinalize(sqlite3_context *context){
  1344         -  char **pzAccum;
  1345         -  pzAccum = sqlite3_aggregate_context(context, 0);
  1346         -  if( pzAccum ){
  1347         -    sqlite3_result_text(context, *pzAccum, -1, sqlite3_free);
         1348  +  StrAccum *pAccum;
         1349  +  pAccum = sqlite3_aggregate_context(context, 0);
         1350  +  if( pAccum ){
         1351  +    if( pAccum->tooBig ){
         1352  +      sqlite3_result_error_toobig(context);
         1353  +    }else if( pAccum->mallocFailed ){
         1354  +      sqlite3_result_error_nomem(context);
         1355  +    }else{    
         1356  +      sqlite3_result_text(context, sqlite3StrAccumFinish(pAccum), -1, 
         1357  +                          sqlite3_free);
         1358  +    }
  1348   1359     }
  1349   1360   }
  1350         -#endif /*SQLITE_GROUP_CONCAT*/
  1351   1361   
  1352   1362   /*
  1353   1363   ** This function registered all of the above C functions as SQL
  1354   1364   ** functions.  This should be the only routine in this file with
  1355   1365   ** external linkage.
  1356   1366   */
  1357   1367   void sqlite3RegisterBuiltinFunctions(sqlite3 *db){
................................................................................
  1423   1433       { "min",    1, 0, 1, minmaxStep,   minMaxFinalize },
  1424   1434       { "max",    1, 1, 1, minmaxStep,   minMaxFinalize },
  1425   1435       { "sum",    1, 0, 0, sumStep,      sumFinalize    },
  1426   1436       { "total",  1, 0, 0, sumStep,      totalFinalize    },
  1427   1437       { "avg",    1, 0, 0, sumStep,      avgFinalize    },
  1428   1438       { "count",  0, 0, 0, countStep,    countFinalize  },
  1429   1439       { "count",  1, 0, 0, countStep,    countFinalize  },
  1430         -#ifdef SQLITE_GROUP_CONCAT
  1431   1440       { "group_concat", 1, 0, 0, groupConcatStep, groupConcatFinalize },
  1432   1441       { "group_concat", 2, 0, 0, groupConcatStep, groupConcatFinalize },
  1433         -#endif
  1434   1442     };
  1435   1443     int i;
  1436   1444   
  1437   1445     for(i=0; i<sizeof(aFuncs)/sizeof(aFuncs[0]); i++){
  1438   1446       void *pArg;
  1439   1447       u8 argType = aFuncs[i].argType;
  1440   1448       if( argType==0xff ){

Changes to src/printf.c.

   161    161     digit = (int)*val;
   162    162     d = digit;
   163    163     digit += '0';
   164    164     *val = (*val - d)*10.0;
   165    165     return digit;
   166    166   }
   167    167   #endif /* SQLITE_OMIT_FLOATING_POINT */
          168  +
          169  +/*
          170  +** Append N space characters to the given string buffer.
          171  +*/
          172  +static void appendSpace(StrAccum *pAccum, int N){
          173  +  static const char zSpaces[] = "                             ";
          174  +  while( N>=sizeof(zSpaces)-1 ){
          175  +    sqlite3StrAccumAppend(pAccum, zSpaces, sizeof(zSpaces)-1);
          176  +    N -= sizeof(zSpaces)-1;
          177  +  }
          178  +  if( N>0 ){
          179  +    sqlite3StrAccumAppend(pAccum, zSpaces, N);
          180  +  }
          181  +}
   168    182   
   169    183   /*
   170    184   ** On machines with a small stack size, you can redefine the
   171    185   ** SQLITE_PRINT_BUF_SIZE to be less than 350.  But beware - for
   172    186   ** smaller values some %f conversions may go into an infinite loop.
   173    187   */
   174    188   #ifndef SQLITE_PRINT_BUF_SIZE
................................................................................
   199    213   **          The return value is the total number of characters sent to
   200    214   **          the function "func".  Returns -1 on a error.
   201    215   **
   202    216   ** Note that the order in which automatic variables are declared below
   203    217   ** seems to make a big difference in determining how fast this beast
   204    218   ** will run.
   205    219   */
   206         -static int vxprintf(
   207         -  void (*func)(void*,const char*,int),     /* Consumer of text */
   208         -  void *arg,                         /* First argument to the consumer */
          220  +static void vxprintf(
          221  +  StrAccum *pAccum,                  /* Accumulate results here */
   209    222     int useExtended,                   /* Allow extended %-conversions */
   210    223     const char *fmt,                   /* Format string */
   211    224     va_list ap                         /* arguments */
   212    225   ){
   213    226     int c;                     /* Next character in the format string */
   214    227     char *bufpt;               /* Pointer to the conversion buffer */
   215    228     int precision;             /* Precision of the current field */
   216    229     int length;                /* Length of the field */
   217    230     int idx;                   /* A general purpose loop counter */
   218         -  int count;                 /* Total number of characters output */
   219    231     int width;                 /* Width of the current field */
   220    232     etByte flag_leftjustify;   /* True if "-" flag is present */
   221    233     etByte flag_plussign;      /* True if "+" flag is present */
   222    234     etByte flag_blanksign;     /* True if " " flag is present */
   223    235     etByte flag_alternateform; /* True if "#" flag is present */
   224    236     etByte flag_altform2;      /* True if "!" flag is present */
   225    237     etByte flag_zeropad;       /* True if field width constant starts with zero */
................................................................................
   230    242     LONGDOUBLE_TYPE realvalue; /* Value for real types */
   231    243     const et_info *infop;      /* Pointer to the appropriate info structure */
   232    244     char buf[etBUFSIZE];       /* Conversion buffer */
   233    245     char prefix;               /* Prefix character.  "+" or "-" or " " or '\0'. */
   234    246     etByte errorflag = 0;      /* True if an error is encountered */
   235    247     etByte xtype;              /* Conversion paradigm */
   236    248     char *zExtra;              /* Extra memory used for etTCLESCAPE conversions */
   237         -  static const char spaces[] =
   238         -   "                                                                         ";
   239         -#define etSPACESIZE (sizeof(spaces)-1)
   240    249   #ifndef SQLITE_OMIT_FLOATING_POINT
   241    250     int  exp, e2;              /* exponent of real numbers */
   242    251     double rounder;            /* Used for rounding floating point values */
   243    252     etByte flag_dp;            /* True if decimal point should be shown */
   244    253     etByte flag_rtz;           /* True if trailing zeros should be removed */
   245    254     etByte flag_exp;           /* True to force display of the exponent */
   246    255     int nsd;                   /* Number of significant digits returned */
   247    256   #endif
   248    257   
   249         -  func(arg,"",0);
   250         -  count = length = 0;
          258  +  length = 0;
   251    259     bufpt = 0;
   252    260     for(; (c=(*fmt))!=0; ++fmt){
   253    261       if( c!='%' ){
   254    262         int amt;
   255    263         bufpt = (char *)fmt;
   256    264         amt = 1;
   257    265         while( (c=(*++fmt))!='%' && c!=0 ) amt++;
   258         -      (*func)(arg,bufpt,amt);
   259         -      count += amt;
          266  +      sqlite3StrAccumAppend(pAccum, bufpt, amt);
   260    267         if( c==0 ) break;
   261    268       }
   262    269       if( (c=(*++fmt))==0 ){
   263    270         errorflag = 1;
   264         -      (*func)(arg,"%",1);
   265         -      count++;
          271  +      sqlite3StrAccumAppend(pAccum, "%", 1);
   266    272         break;
   267    273       }
   268    274       /* Find out what flags are present */
   269    275       flag_leftjustify = flag_plussign = flag_blanksign = 
   270    276        flag_alternateform = flag_altform2 = flag_zeropad = 0;
   271    277       done = 0;
   272    278       do{
................................................................................
   332    338       infop = 0;
   333    339       for(idx=0; idx<etNINFO; idx++){
   334    340         if( c==fmtinfo[idx].fmttype ){
   335    341           infop = &fmtinfo[idx];
   336    342           if( useExtended || (infop->flags & FLAG_INTERN)==0 ){
   337    343             xtype = infop->type;
   338    344           }else{
   339         -          return -1;
          345  +          return;
   340    346           }
   341    347           break;
   342    348         }
   343    349       }
   344    350       zExtra = 0;
   345    351       if( infop==0 ){
   346         -      return -1;
          352  +      return;
   347    353       }
   348    354   
   349    355   
   350    356       /* Limit the precision to prevent overflowing buf[] during conversion */
   351    357       if( precision>etBUFSIZE-40 && (infop->flags & FLAG_STRING)==0 ){
   352    358         precision = etBUFSIZE-40;
   353    359       }
................................................................................
   577    583             i = prefix!=0;
   578    584             while( nPad-- ) bufpt[i++] = '0';
   579    585             length = width;
   580    586           }
   581    587   #endif
   582    588           break;
   583    589         case etSIZE:
   584         -        *(va_arg(ap,int*)) = count;
          590  +        *(va_arg(ap,int*)) = pAccum->nChar;
   585    591           length = width = 0;
   586    592           break;
   587    593         case etPERCENT:
   588    594           buf[0] = '%';
   589    595           bufpt = buf;
   590    596           length = 1;
   591    597           break;
................................................................................
   623    629           for(i=n=0; (ch=escarg[i])!=0; i++){
   624    630             if( ch==q )  n++;
   625    631           }
   626    632           needQuote = !isnull && xtype==etSQLESCAPE2;
   627    633           n += i + 1 + needQuote*2;
   628    634           if( n>etBUFSIZE ){
   629    635             bufpt = zExtra = sqlite3_malloc( n );
   630         -          if( bufpt==0 ) return -1;
          636  +          if( bufpt==0 ) return;
   631    637           }else{
   632    638             bufpt = buf;
   633    639           }
   634    640           j = 0;
   635    641           if( needQuote ) bufpt[j++] = q;
   636    642           for(i=0; (ch=escarg[i])!=0; i++){
   637    643             bufpt[j++] = ch;
................................................................................
   643    649           /* The precision is ignored on %q and %Q */
   644    650           /* if( precision>=0 && precision<length ) length = precision; */
   645    651           break;
   646    652         }
   647    653         case etTOKEN: {
   648    654           Token *pToken = va_arg(ap, Token*);
   649    655           if( pToken && pToken->z ){
   650         -          (*func)(arg, (char*)pToken->z, pToken->n);
          656  +          sqlite3StrAccumAppend(pAccum, (const char*)pToken->z, pToken->n);
   651    657           }
   652    658           length = width = 0;
   653    659           break;
   654    660         }
   655    661         case etSRCLIST: {
   656    662           SrcList *pSrc = va_arg(ap, SrcList*);
   657    663           int k = va_arg(ap, int);
   658    664           struct SrcList_item *pItem = &pSrc->a[k];
   659    665           assert( k>=0 && k<pSrc->nSrc );
   660    666           if( pItem->zDatabase && pItem->zDatabase[0] ){
   661         -          (*func)(arg, pItem->zDatabase, strlen(pItem->zDatabase));
   662         -          (*func)(arg, ".", 1);
          667  +          sqlite3StrAccumAppend(pAccum, pItem->zDatabase, -1);
          668  +          sqlite3StrAccumAppend(pAccum, ".", 1);
   663    669           }
   664         -        (*func)(arg, pItem->zName, strlen(pItem->zName));
          670  +        sqlite3StrAccumAppend(pAccum, pItem->zName, -1);
   665    671           length = width = 0;
   666    672           break;
   667    673         }
   668    674       }/* End switch over the format type */
   669    675       /*
   670    676       ** The text of the conversion is pointed to by "bufpt" and is
   671    677       ** "length" characters long.  The field width is "width".  Do
   672    678       ** the output.
   673    679       */
   674    680       if( !flag_leftjustify ){
   675    681         register int nspace;
   676    682         nspace = width-length;
   677    683         if( nspace>0 ){
   678         -        count += nspace;
   679         -        while( nspace>=etSPACESIZE ){
   680         -          (*func)(arg,spaces,etSPACESIZE);
   681         -          nspace -= etSPACESIZE;
   682         -        }
   683         -        if( nspace>0 ) (*func)(arg,spaces,nspace);
          684  +        appendSpace(pAccum, nspace);
   684    685         }
   685    686       }
   686    687       if( length>0 ){
   687         -      (*func)(arg,bufpt,length);
   688         -      count += length;
          688  +      sqlite3StrAccumAppend(pAccum, bufpt, length);
   689    689       }
   690    690       if( flag_leftjustify ){
   691    691         register int nspace;
   692    692         nspace = width-length;
   693    693         if( nspace>0 ){
   694         -        count += nspace;
   695         -        while( nspace>=etSPACESIZE ){
   696         -          (*func)(arg,spaces,etSPACESIZE);
   697         -          nspace -= etSPACESIZE;
   698         -        }
   699         -        if( nspace>0 ) (*func)(arg,spaces,nspace);
          694  +        appendSpace(pAccum, nspace);
   700    695         }
   701    696       }
   702    697       if( zExtra ){
   703    698         sqlite3_free(zExtra);
   704    699       }
   705    700     }/* End for loop over the format string */
   706         -  return errorflag ? -1 : count;
   707    701   } /* End of function */
   708    702   
   709         -
   710         -/* This structure is used to store state information about the
   711         -** write to memory that is currently in progress.
          703  +/*
          704  +** Append N bytes of text from z to the StrAccum object.
   712    705   */
   713         -struct sgMprintf {
   714         -  char *zBase;     /* A base allocation */
   715         -  char *zText;     /* The string collected so far */
   716         -  int  nChar;      /* Length of the string so far */
   717         -  int  nTotal;     /* Output size if unconstrained */
   718         -  int  nAlloc;     /* Amount of space allocated in zText */
   719         -  void *(*xRealloc)(void*,int);  /* Function used to realloc memory */
   720         -  int  iMallocFailed;            /* True if xRealloc() has failed */
   721         -};
          706  +void sqlite3StrAccumAppend(StrAccum *p, const char *z, int N){
          707  +  if( p->tooBig | p->mallocFailed ){
          708  +    return;
          709  +  }
          710  +  if( N<0 ){
          711  +    N = strlen(z);
          712  +  }
          713  +  if( N==0 ){
          714  +    return;
          715  +  }
          716  +  if( p->nChar+N >= p->nAlloc ){
          717  +    char *zNew;
          718  +    if( !p->useMalloc ){
          719  +      p->tooBig = 1;
          720  +      N = p->nAlloc - p->nChar - 1;
          721  +      if( N<=0 ){
          722  +        return;
          723  +      }
          724  +    }else{
          725  +      p->nAlloc += p->nAlloc + N + 1;
          726  +      if( p->nAlloc > SQLITE_MAX_LENGTH ){
          727  +        p->nAlloc = SQLITE_MAX_LENGTH;
          728  +        if( p->nChar+N >= p->nAlloc ){
          729  +          sqlite3StrAccumReset(p);
          730  +          p->tooBig = 1;
          731  +          return;
          732  +        }
          733  +      }
          734  +      zNew = sqlite3_malloc( p->nAlloc );
          735  +      if( zNew ){
          736  +        memcpy(zNew, p->zText, p->nChar);
          737  +        sqlite3StrAccumReset(p);
          738  +        p->zText = zNew;
          739  +      }else{
          740  +        p->mallocFailed = 1;
          741  +        sqlite3StrAccumReset(p);
          742  +        return;
          743  +      }
          744  +    }
          745  +  }
          746  +  memcpy(&p->zText[p->nChar], z, N);
          747  +  p->nChar += N;
          748  +}
   722    749   
   723         -/* 
   724         -** This function implements the callback from vxprintf. 
   725         -**
   726         -** This routine add nNewChar characters of text in zNewText to
   727         -** the sgMprintf structure pointed to by "arg".
          750  +/*
          751  +** Finish off a string by making sure it is zero-terminated.
          752  +** Return a pointer to the resulting string.  Return a NULL
          753  +** pointer if any kind of error was encountered.
   728    754   */
   729         -static void mout(void *arg, const char *zNewText, int nNewChar){
   730         -  struct sgMprintf *pM = (struct sgMprintf*)arg;
   731         -  if( pM->iMallocFailed ) return;
   732         -  pM->nTotal += nNewChar;
   733         -  if( pM->zText ){
   734         -    if( pM->nChar + nNewChar + 1 > pM->nAlloc ){
   735         -      if( pM->xRealloc==0 ){
   736         -        nNewChar =  pM->nAlloc - pM->nChar - 1;
          755  +char *sqlite3StrAccumFinish(StrAccum *p){
          756  +  if( p->zText ){
          757  +    p->zText[p->nChar] = 0;
          758  +    if( p->useMalloc && p->zText==p->zBase ){
          759  +      p->zText = sqlite3_malloc( p->nChar+1 );
          760  +      if( p->zText ){
          761  +        memcpy(p->zText, p->zBase, p->nChar+1);
   737    762         }else{
   738         -        int nAlloc = pM->nChar + nNewChar*2 + 1;
   739         -        if( pM->zText==pM->zBase ){
   740         -          pM->zText = pM->xRealloc(0, nAlloc);
   741         -          if( pM->zText==0 ){
   742         -            pM->nAlloc = 0;
   743         -            pM->iMallocFailed = 1;
   744         -            return;
   745         -          }else if( pM->nChar ){
   746         -            memcpy(pM->zText, pM->zBase, pM->nChar);
   747         -          }
   748         -        }else{
   749         -          char *zNew;
   750         -          zNew = pM->xRealloc(pM->zText, nAlloc);
   751         -          if( zNew ){
   752         -            pM->zText = zNew;
   753         -          }else{
   754         -            pM->iMallocFailed = 1;
   755         -            pM->xRealloc(pM->zText, 0);
   756         -            pM->zText = 0;
   757         -            pM->nAlloc = 0;
   758         -            return;
   759         -          }
   760         -        }
   761         -        pM->nAlloc = nAlloc;
          763  +        p->mallocFailed = 1;
   762    764         }
   763    765       }
   764         -    if( nNewChar>0 ){
   765         -      memcpy(&pM->zText[pM->nChar], zNewText, nNewChar);
   766         -      pM->nChar += nNewChar;
   767         -    }
   768         -    pM->zText[pM->nChar] = 0;
          766  +  }
          767  +  return p->zText;
          768  +}
          769  +
          770  +/*
          771  +** Reset an StrAccum string.  Reclaim all malloced memory.
          772  +*/
          773  +void sqlite3StrAccumReset(StrAccum *p){
          774  +  if( p->zText!=p->zBase ){
          775  +    sqlite3_free(p->zText);
          776  +    p->zText = 0;
   769    777     }
   770    778   }
   771    779   
   772    780   /*
   773         -** This routine is a wrapper around xprintf() that invokes mout() as
   774         -** the consumer.  
          781  +** Initialize a string accumulator
   775    782   */
   776         -static char *base_vprintf(
   777         -  void *(*xRealloc)(void*, int),  /* realloc() function. May be NULL */
   778         -  int useInternal,                /* Use internal %-conversions if true */
   779         -  char *zInitBuf,                 /* Initially write here, before mallocing */
   780         -  int nInitBuf,                   /* Size of zInitBuf[] */
   781         -  const char *zFormat,            /* format string */
   782         -  va_list ap                      /* arguments */
   783         -){
   784         -  struct sgMprintf sM;
   785         -  sM.zBase = sM.zText = zInitBuf;
   786         -  sM.nChar = sM.nTotal = 0;
   787         -  sM.nAlloc = nInitBuf;
   788         -  sM.xRealloc = xRealloc;
   789         -  sM.iMallocFailed = 0;
   790         -  vxprintf(mout, &sM, useInternal, zFormat, ap);
   791         -  assert(sM.iMallocFailed==0 || sM.zText==0);
   792         -  if( xRealloc && !sM.iMallocFailed ){
   793         -    if( sM.zText==sM.zBase ){
   794         -      sM.zText = xRealloc(0, sM.nChar+1);
   795         -      if( sM.zText ){
   796         -        memcpy(sM.zText, sM.zBase, sM.nChar+1);
   797         -      }
   798         -    }else if( sM.nAlloc>sM.nChar+10 ){
   799         -      char *zNew;
   800         -      sqlite3MallocBenignFailure(1);
   801         -      zNew = xRealloc(sM.zText, sM.nChar+1);
   802         -      if( zNew ){
   803         -        sM.zText = zNew;
   804         -      }
   805         -    }
   806         -  }
   807         -  return sM.zText;
   808         -}
   809         -
   810         -/*
   811         -** Realloc that is a real function, not a macro.
   812         -*/
   813         -static void *printf_realloc(void *old, int size){
   814         -  return sqlite3_realloc(old, size);
          783  +static void sqlite3StrAccumInit(StrAccum *p, char *zBase, int n){
          784  +  p->zText = p->zBase = zBase;
          785  +  p->nChar = 0;
          786  +  p->nAlloc = n;
          787  +  p->useMalloc = 1;
          788  +  p->tooBig = 0;
          789  +  p->mallocFailed = 0;
   815    790   }
   816    791   
   817    792   /*
   818    793   ** Print into memory obtained from sqliteMalloc().  Use the internal
   819    794   ** %-conversion extensions.
   820    795   */
   821    796   char *sqlite3VMPrintf(sqlite3 *db, const char *zFormat, va_list ap){
   822    797     char *z;
   823    798     char zBase[SQLITE_PRINT_BUF_SIZE];
   824         -  z = base_vprintf(printf_realloc, 1, zBase, sizeof(zBase), zFormat, ap);
   825         -  if( z==0 && db!=0 ){
          799  +  StrAccum acc;
          800  +  sqlite3StrAccumInit(&acc, zBase, sizeof(zBase));
          801  +  vxprintf(&acc, 1, zFormat, ap);
          802  +  z = sqlite3StrAccumFinish(&acc);
          803  +  if( acc.mallocFailed && db ){
   826    804       db->mallocFailed = 1;
   827    805     }
   828    806     return z;
   829    807   }
   830    808   
   831    809   /*
   832    810   ** Print into memory obtained from sqliteMalloc().  Use the internal
   833    811   ** %-conversion extensions.
   834    812   */
   835    813   char *sqlite3MPrintf(sqlite3 *db, const char *zFormat, ...){
   836    814     va_list ap;
   837    815     char *z;
   838         -  char zBase[SQLITE_PRINT_BUF_SIZE];
   839    816     va_start(ap, zFormat);
   840         -  z = base_vprintf(printf_realloc, 1, zBase, sizeof(zBase), zFormat, ap);
          817  +  z = sqlite3VMPrintf(db, zFormat, ap);
   841    818     va_end(ap);
   842         -  if( z==0 && db!=0 ){
   843         -    db->mallocFailed = 1;
   844         -  }
   845    819     return z;
   846    820   }
   847    821   
   848    822   /*
   849    823   ** Print into memory obtained from sqlite3_malloc().  Omit the internal
   850    824   ** %-conversion extensions.
   851    825   */
   852    826   char *sqlite3_vmprintf(const char *zFormat, va_list ap){
          827  +  char *z;
   853    828     char zBase[SQLITE_PRINT_BUF_SIZE];
   854         -  return base_vprintf(sqlite3_realloc, 0, zBase, sizeof(zBase), zFormat, ap);
          829  +  StrAccum acc;
          830  +  sqlite3StrAccumInit(&acc, zBase, sizeof(zBase));
          831  +  vxprintf(&acc, 0, zFormat, ap);
          832  +  z = sqlite3StrAccumFinish(&acc);
          833  +  return z;
   855    834   }
   856    835   
   857    836   /*
   858    837   ** Print into memory obtained from sqlite3_malloc()().  Omit the internal
   859    838   ** %-conversion extensions.
   860    839   */
   861    840   char *sqlite3_mprintf(const char *zFormat, ...){
................................................................................
   872    851   ** current locale settings.  This is important for SQLite because we
   873    852   ** are not able to use a "," as the decimal point in place of "." as
   874    853   ** specified by some locales.
   875    854   */
   876    855   char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){
   877    856     char *z;
   878    857     va_list ap;
          858  +  StrAccum acc;
   879    859   
   880    860     if( n<=0 ){
   881    861       return zBuf;
   882    862     }
   883         -  zBuf[0] = 0;
          863  +  sqlite3StrAccumInit(&acc, zBuf, n);
          864  +  acc.useMalloc = 0;
   884    865     va_start(ap,zFormat);
   885         -  z = base_vprintf(0, 0, zBuf, n, zFormat, ap);
          866  +  vxprintf(&acc, 0, zFormat, ap);
   886    867     va_end(ap);
          868  +  z = sqlite3StrAccumFinish(&acc);
   887    869     return z;
   888    870   }
   889    871   
   890    872   #if defined(SQLITE_TEST) || defined(SQLITE_DEBUG) || defined(SQLITE_MEMDEBUG)
   891    873   /*
   892    874   ** A version of printf() that understands %lld.  Used for debugging.
   893    875   ** The printf() built into some versions of windows does not understand %lld
   894    876   ** and segfaults if you give it a long long int.
   895    877   */
   896    878   void sqlite3DebugPrintf(const char *zFormat, ...){
   897         -  extern int getpid(void);
   898    879     va_list ap;
          880  +  StrAccum acc;
   899    881     char zBuf[500];
   900         -  va_start(ap, zFormat);
   901         -  base_vprintf(0, 0, zBuf, sizeof(zBuf), zFormat, ap);
          882  +  sqlite3StrAccumInit(&acc, zBuf, sizeof(zBuf));
          883  +  acc.useMalloc = 0;
          884  +  va_start(ap,zFormat);
          885  +  vxprintf(&acc, 0, zFormat, ap);
   902    886     va_end(ap);
   903    887     fprintf(stdout,"%s", zBuf);
   904    888     fflush(stdout);
   905    889   }
   906    890   #endif

Changes to src/sqliteInt.h.

     7      7   **    May you do good and not evil.
     8      8   **    May you find forgiveness for yourself and forgive others.
     9      9   **    May you share freely, never taking more than you give.
    10     10   **
    11     11   *************************************************************************
    12     12   ** Internal interface definitions for SQLite.
    13     13   **
    14         -** @(#) $Id: sqliteInt.h,v 1.620 2007/11/27 02:38:01 drh Exp $
           14  +** @(#) $Id: sqliteInt.h,v 1.621 2007/11/28 22:36:41 drh Exp $
    15     15   */
    16     16   #ifndef _SQLITEINT_H_
    17     17   #define _SQLITEINT_H_
    18     18   
    19     19   /*
    20     20   ** The macro unlikely() is a hint that surrounds a boolean
    21     21   ** expression that is usually false.  Macro likely() surrounds
................................................................................
   359    359   typedef struct KeyClass KeyClass;
   360    360   typedef struct KeyInfo KeyInfo;
   361    361   typedef struct Module Module;
   362    362   typedef struct NameContext NameContext;
   363    363   typedef struct Parse Parse;
   364    364   typedef struct Select Select;
   365    365   typedef struct SrcList SrcList;
          366  +typedef struct StrAccum StrAccum;
   366    367   typedef struct Table Table;
   367    368   typedef struct TableLock TableLock;
   368    369   typedef struct Token Token;
   369    370   typedef struct TriggerStack TriggerStack;
   370    371   typedef struct TriggerStep TriggerStep;
   371    372   typedef struct Trigger Trigger;
   372    373   typedef struct WhereInfo WhereInfo;
................................................................................
  1569   1570   struct DbFixer {
  1570   1571     Parse *pParse;      /* The parsing context.  Error messages written here */
  1571   1572     const char *zDb;    /* Make sure all objects are contained in this database */
  1572   1573     const char *zType;  /* Type of the container - used for error messages */
  1573   1574     const Token *pName; /* Name of the container - used for error messages */
  1574   1575   };
  1575   1576   
         1577  +/*
         1578  +** An objected used to accumulate the text of a string where we
         1579  +** do not necessarily know how big the string will be in the end.
         1580  +*/
         1581  +struct StrAccum {
         1582  +  char *zBase;     /* A base allocation.  Not from malloc. */
         1583  +  char *zText;     /* The string collected so far */
         1584  +  int  nChar;      /* Length of the string so far */
         1585  +  int  nAlloc;     /* Amount of space allocated in zText */
         1586  +  u8   mallocFailed;   /* Becomes true if any memory allocation fails */
         1587  +  u8   useMalloc;      /* True if zText is enlargable using realloc */
         1588  +  u8   tooBig;         /* Becomes true if string size exceeds limits */
         1589  +};
         1590  +
  1576   1591   /*
  1577   1592   ** A pointer to this structure is used to communicate information
  1578   1593   ** from sqlite3Init and OP_ParseSchema into the sqlite3InitCallback.
  1579   1594   */
  1580   1595   typedef struct {
  1581   1596     sqlite3 *db;        /* The database being initialized */
  1582   1597     int iDb;            /* 0 for main database.  1 for TEMP, 2.. for ATTACHed */
................................................................................
  1874   1889   KeyInfo *sqlite3IndexKeyinfo(Parse *, Index *);
  1875   1890   int sqlite3CreateFunc(sqlite3 *, const char *, int, int, void *, 
  1876   1891     void (*)(sqlite3_context*,int,sqlite3_value **),
  1877   1892     void (*)(sqlite3_context*,int,sqlite3_value **), void (*)(sqlite3_context*));
  1878   1893   int sqlite3ApiExit(sqlite3 *db, int);
  1879   1894   void sqlite3AbortOtherActiveVdbes(sqlite3 *, Vdbe *);
  1880   1895   int sqlite3OpenTempDatabase(Parse *);
         1896  +
         1897  +void sqlite3StrAccumAppend(StrAccum*,const char*,int);
         1898  +char *sqlite3StrAccumFinish(StrAccum*);
         1899  +void sqlite3StrAccumReset(StrAccum*);
  1881   1900   
  1882   1901   
  1883   1902   /*
  1884   1903   ** The interface to the LEMON-generated parser
  1885   1904   */
  1886   1905   void *sqlite3ParserAlloc(void*(*)(size_t));
  1887   1906   void sqlite3ParserFree(void*, void(*)(void*));

Changes to test/func.test.

     7      7   #    May you find forgiveness for yourself and forgive others.
     8      8   #    May you share freely, never taking more than you give.
     9      9   #
    10     10   #***********************************************************************
    11     11   # This file implements regression tests for SQLite library.  The
    12     12   # focus of this file is testing built-in functions.
    13     13   #
    14         -# $Id: func.test,v 1.70 2007/10/20 15:41:58 drh Exp $
           14  +# $Id: func.test,v 1.71 2007/11/28 22:36:41 drh Exp $
    15     15   
    16     16   set testdir [file dirname $argv0]
    17     17   source $testdir/tester.tcl
    18     18   
    19     19   # Create a table to work with.
    20     20   #
    21     21   do_test func-0.0 {
................................................................................
   879    879   #
   880    880   do_test func-23.1 {
   881    881     sqlite3_create_aggregate db
   882    882     execsql {
   883    883       SELECT legacy_count() FROM t6;
   884    884     }
   885    885   } {3}
          886  +
          887  +# The group_concat() function.
          888  +#
          889  +do_test func-24.1 {
          890  +  execsql {
          891  +    SELECT group_concat(t1) FROM tbl1
          892  +  }
          893  +} {this,program,is,free,software}
          894  +do_test func-24.2 {
          895  +  execsql {
          896  +    SELECT group_concat(t1,' ') FROM tbl1
          897  +  }
          898  +} {{this program is free software}}
          899  +do_test func-24.3 {
          900  +  execsql {
          901  +    SELECT group_concat(t1,' ' || rowid || ' ') FROM tbl1
          902  +  }
          903  +} {{this 2 program 3 is 4 free 5 software}}
          904  +do_test func-24.4 {
          905  +  execsql {
          906  +    SELECT group_concat(NULL,t1) FROM tbl1
          907  +  }
          908  +} {{}}
          909  +do_test func-24.5 {
          910  +  execsql {
          911  +    SELECT group_concat(t1,NULL) FROM tbl1
          912  +  }
          913  +} {thisprogramisfreesoftware}
   886    914   
   887    915   finish_test

Changes to test/malloc.test.

    12     12   # This file attempts to check the behavior of the SQLite library in 
    13     13   # an out-of-memory situation. When compiled with -DSQLITE_DEBUG=1, 
    14     14   # the SQLite library accepts a special command (sqlite3_memdebug_fail N C)
    15     15   # which causes the N-th malloc to fail.  This special feature is used
    16     16   # to see what happens in the library if a malloc were to really fail
    17     17   # due to an out-of-memory situation.
    18     18   #
    19         -# $Id: malloc.test,v 1.52 2007/11/16 14:55:46 danielk1977 Exp $
           19  +# $Id: malloc.test,v 1.53 2007/11/28 22:36:41 drh Exp $
    20     20   
    21     21   set testdir [file dirname $argv0]
    22     22   source $testdir/tester.tcl
    23     23   
    24     24   # Only run these tests if memory debugging is turned on.
    25     25   #
    26     26   ifcapable !memdebug {
................................................................................
    47     47       );
    48     48       CREATE INDEX i1 ON t1(a,b);
    49     49       INSERT INTO t1 VALUES(1,2.3,4.5,'hi',x'746865726500');
    50     50       INSERT INTO t1 VALUES(6,7.0,0.8,'hello','out yonder');
    51     51       SELECT * FROM t1;
    52     52       SELECT avg(b) FROM t1 GROUP BY a HAVING b>20.0;
    53     53       DELETE FROM t1 WHERE a IN (SELECT min(a) FROM t1);
    54         -    SELECT count(*) FROM t1;
           54  +    SELECT count(*), group_concat(e) FROM t1;
    55     55     } 
    56     56   }
    57     57   
    58     58   # Ensure that no file descriptors were leaked.
    59     59   do_test malloc-1.X {
    60     60     catch {db close}
    61     61     set sqlite_open_file_count

Changes to test/sqllimits1.test.

     8      8   #    May you share freely, never taking more than you give.
     9      9   #
    10     10   #***********************************************************************
    11     11   #
    12     12   # This file contains tests to verify that the limits defined in
    13     13   # sqlite source file limits.h are enforced.
    14     14   #
    15         -# $Id: sqllimits1.test,v 1.19 2007/10/09 08:29:33 danielk1977 Exp $
           15  +# $Id: sqllimits1.test,v 1.20 2007/11/28 22:36:41 drh Exp $
    16     16   
    17     17   set testdir [file dirname $argv0]
    18     18   source $testdir/tester.tcl
    19     19   
    20     20   # Test organization:
    21     21   #
    22     22   #     sqllimits-1.*:  SQLITE_MAX_LENGTH
................................................................................
   123    123   } {}
   124    124   do_test sqllimits-1.14.2 {
   125    125     sqlite3_step $::STMT 
   126    126   } {SQLITE_ERROR}
   127    127   do_test sqllimits-1.14.3 {
   128    128     sqlite3_finalize $::STMT 
   129    129   } {SQLITE_TOOBIG}
          130  +
          131  +do_test sqllimits-1.15 {
          132  +  execsql {
          133  +    CREATE TABLE t4(x);
          134  +    INSERT INTO t4 VALUES(1);
          135  +    INSERT INTO t4 VALUES(2);
          136  +    INSERT INTO t4 SELECT 2+x FROM t4;
          137  +  }
          138  +  catchsql {
          139  +    SELECT group_concat(hex(randomblob(20000))) FROM t4;
          140  +  }
          141  +} {1 {string or blob too big}}
          142  +db eval {DROP TABLE t4}
   130    143   
   131    144   #--------------------------------------------------------------------
   132    145   # Test cases sqllimits-2.* test that the SQLITE_MAX_SQL_LENGTH limit
   133    146   # is enforced.
   134    147   #
   135    148   do_test sqllimits-2.1 {
   136    149     set    sql "SELECT 1 WHERE 1==1"