/ Check-in [bd3ce723]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add stdio-like I/O interfaces to the test_quota VFS. This is a prototype change for discussion and is mostly untested. This is an alternative to adding stdio-like I/O interfaces in the core. There is no guarantee that this code will make it into the trunk. If it does get to trunk, there could be many changes to the interface first.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | quota-stdio
Files: files | file ages | folders
SHA1:bd3ce723f1b5be52be46ede8614ca316f56e7e6f
User & Date: drh 2011-12-01 18:44:21
Context
2011-12-01
20:48
Add test logic and some test cases. check-in: a4730586 user: drh tags: quota-stdio
18:44
Add stdio-like I/O interfaces to the test_quota VFS. This is a prototype change for discussion and is mostly untested. This is an alternative to adding stdio-like I/O interfaces in the core. There is no guarantee that this code will make it into the trunk. If it does get to trunk, there could be many changes to the interface first. check-in: bd3ce723 user: drh tags: quota-stdio
2011-11-29
15:40
Remove unused fields from the Parse object. Documentation and formatting improvements on data structure definitions. check-in: 431556ca user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/test_quota.c.

    23     23   ** However, before returning SQLITE_FULL, the write requests invoke
    24     24   ** a callback function that is configurable for each quota group.
    25     25   ** This callback has the opportunity to enlarge the quota.  If the
    26     26   ** callback does enlarge the quota such that the total size of all
    27     27   ** files within the group is less than the new quota, then the write
    28     28   ** continues as if nothing had happened.
    29     29   */
    30         -#include "sqlite3.h"
           30  +#include "test_quota.h"
    31     31   #include <string.h>
    32     32   #include <assert.h>
    33     33   
    34     34   /*
    35     35   ** For an build without mutexes, no-op the mutex calls.
    36     36   */
    37     37   #if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE==0
................................................................................
   106    106   ** VFS is appended to this structure.
   107    107   */
   108    108   struct quotaConn {
   109    109     sqlite3_file base;              /* Base class - must be first */
   110    110     quotaFile *pFile;               /* The underlying file */
   111    111     /* The underlying VFS sqlite3_file is appended to this object */
   112    112   };
          113  +
          114  +/*
          115  +** An instance of the following object records the state of an
          116  +** open file.  This object is opaque to all users - the internal
          117  +** structure is only visible to the functions below.
          118  +*/
          119  +struct quota_FILE {
          120  +  FILE *f;                /* Open stdio file pointer */
          121  +  sqlite3_int64 iOfst;    /* Current offset into the file */
          122  +  quotaFile *pFile;       /* The file record in the quota system */
          123  +};
          124  +
   113    125   
   114    126   /************************* Global Variables **********************************/
   115    127   /*
   116    128   ** All global variables used by this file are containing within the following
   117    129   ** gQuota structure.
   118    130   */
   119    131   static struct {
................................................................................
   309    321     quotaConn *p = (quotaConn*)pConn;
   310    322     return (sqlite3_file*)&p[1];
   311    323   }
   312    324   
   313    325   /* Find a file in a quota group and return a pointer to that file.
   314    326   ** Return NULL if the file is not in the group.
   315    327   */
   316         -static quotaFile *quotaFindFile(quotaGroup *pGroup, const char *zName){
          328  +static quotaFile *quotaFindFile(
          329  +  quotaGroup *pGroup,     /* Group in which to look for the file */
          330  +  const char *zName,      /* Full pathname of the file */
          331  +  int createFlag          /* Try to create the file if not found */
          332  +){
   317    333     quotaFile *pFile = pGroup->pFiles;
   318    334     while( pFile && strcmp(pFile->zFilename, zName)!=0 ){
   319    335       pFile = pFile->pNext;
          336  +  }
          337  +  if( pFile==0 && createFlag ){
          338  +    int nName = strlen(zName);
          339  +    pFile = (quotaFile *)sqlite3_malloc( sizeof(*pFile) + nName + 1 );
          340  +    if( pFile ){
          341  +      memset(pFile, 0, sizeof(*pFile));
          342  +      pFile->zFilename = (char*)&pFile[1];
          343  +      memcpy(pFile->zFilename, zName, nName+1);
          344  +      pFile->pNext = pGroup->pFiles;
          345  +      if( pGroup->pFiles ) pGroup->pFiles->ppPrev = &pFile->pNext;
          346  +      pFile->ppPrev = &pGroup->pFiles;
          347  +      pGroup->pFiles = pFile;
          348  +      pFile->pGroup = pGroup;
          349  +    }
   320    350     }
   321    351     return pFile;
   322    352   }
   323    353   
   324    354   /************************* VFS Method Wrappers *****************************/
   325    355   /*
   326    356   ** This is the xOpen method used for the "quota" VFS.
................................................................................
   360    390     }else{
   361    391       /* If we get to this point, it means the file needs to be quota tracked.
   362    392       */
   363    393       pQuotaOpen = (quotaConn*)pConn;
   364    394       pSubOpen = quotaSubOpen(pConn);
   365    395       rc = pOrigVfs->xOpen(pOrigVfs, zName, pSubOpen, flags, pOutFlags);
   366    396       if( rc==SQLITE_OK ){
   367         -      pFile = quotaFindFile(pGroup, zName);
          397  +      pFile = quotaFindFile(pGroup, zName, 1);
   368    398         if( pFile==0 ){
   369         -        int nName = strlen(zName);
   370         -        pFile = (quotaFile *)sqlite3_malloc( sizeof(*pFile) + nName + 1 );
   371         -        if( pFile==0 ){
   372         -          quotaLeave();
   373         -          pSubOpen->pMethods->xClose(pSubOpen);
   374         -          return SQLITE_NOMEM;
   375         -        }
   376         -        memset(pFile, 0, sizeof(*pFile));
   377         -        pFile->zFilename = (char*)&pFile[1];
   378         -        memcpy(pFile->zFilename, zName, nName+1);
   379         -        pFile->pNext = pGroup->pFiles;
   380         -        if( pGroup->pFiles ) pGroup->pFiles->ppPrev = &pFile->pNext;
   381         -        pFile->ppPrev = &pGroup->pFiles;
   382         -        pGroup->pFiles = pFile;
   383         -        pFile->pGroup = pGroup;
   384         -        pFile->deleteOnClose = (flags & SQLITE_OPEN_DELETEONCLOSE)!=0;
          399  +        quotaLeave();
          400  +        pSubOpen->pMethods->xClose(pSubOpen);
          401  +        return SQLITE_NOMEM;
   385    402         }
          403  +      pFile->deleteOnClose = (flags & SQLITE_OPEN_DELETEONCLOSE)!=0;
   386    404         pFile->nRef++;
   387    405         pQuotaOpen->pFile = pFile;
   388    406         if( pSubOpen->pMethods->iVersion==1 ){
   389    407           pQuotaOpen->base.pMethods = &gQuota.sIoMethodsV1;
   390    408         }else{
   391    409           pQuotaOpen->base.pMethods = &gQuota.sIoMethodsV2;
   392    410         }
................................................................................
   419    437     /* If the file just deleted is a member of a quota group, then remove
   420    438     ** it from that quota group.
   421    439     */
   422    440     if( rc==SQLITE_OK ){
   423    441       quotaEnter();
   424    442       pGroup = quotaGroupFind(zName);
   425    443       if( pGroup ){
   426         -      pFile = quotaFindFile(pGroup, zName);
          444  +      pFile = quotaFindFile(pGroup, zName, 0);
   427    445         if( pFile ){
   428    446           if( pFile->nRef ){
   429    447             pFile->deleteOnClose = 1;
   430    448           }else{
   431    449             quotaRemoveFile(pFile);
   432    450             quotaGroupDeref(pGroup);
   433    451           }
................................................................................
   819    837       fd->pMethods->xClose(fd);
   820    838     }else if( rc==SQLITE_CANTOPEN ){
   821    839       quotaGroup *pGroup;
   822    840       quotaFile *pFile;
   823    841       quotaEnter();
   824    842       pGroup = quotaGroupFind(zFull);
   825    843       if( pGroup ){
   826         -      pFile = quotaFindFile(pGroup, zFull);
          844  +      pFile = quotaFindFile(pGroup, zFull, 0);
   827    845         if( pFile ) quotaRemoveFile(pFile);
   828    846       }
   829    847       quotaLeave();
   830    848     }
   831    849     sqlite3_free(fd);
   832    850     return rc;
   833    851   }
          852  +
          853  +/*
          854  +** Open a potentially quotaed file for I/O.
          855  +*/
          856  +quota_FILE *sqlite3_quota_fopen(const char *zFilename, const char *zMode){
          857  +  quota_FILE *p = 0;
          858  +  char *zFull = 0;
          859  +  int rc;
          860  +  quotaGroup *pGroup;
          861  +  quotaFile *pFile;
          862  +
          863  +  p = sqlite3_malloc(gQuota.sThisVfs.mxPathname + 1);
          864  +  if( p==0 ) return 0;
          865  +  zFull = (char*)&p[1];
          866  +  rc = gQuota.pOrigVfs->xFullPathname(gQuota.pOrigVfs, zFilename,
          867  +                                      gQuota.sThisVfs.mxPathname+1, zFull);
          868  +  if( rc ) goto quota_fopen_error;
          869  +  p = sqlite3_malloc(sizeof(*p));
          870  +  if( p==0 ) goto quota_fopen_error;
          871  +  memset(p, 0, sizeof(*p));
          872  +  p->f = fopen(zFull, zMode);
          873  +  if( p->f==0 ) goto quota_fopen_error;
          874  +  quotaEnter();
          875  +  pGroup = quotaGroupFind(zFull);
          876  +  if( pGroup ){
          877  +    pFile = quotaFindFile(pGroup, zFull, 1);
          878  +    if( pFile==0 ){
          879  +      quotaLeave();
          880  +      goto quota_fopen_error;
          881  +    }
          882  +    pFile->nRef++;
          883  +    p->pFile = pFile;
          884  +  }
          885  +  quotaLeave();
          886  +  sqlite3_free(zFull);
          887  +  return p;
          888  +
          889  +quota_fopen_error:
          890  +  sqlite3_free(zFull);
          891  +  if( p && p->f ) fclose(p->f);
          892  +  sqlite3_free(p);
          893  +  return 0;
          894  +}
          895  +
          896  +/*
          897  +** Read content from a quota_FILE
          898  +*/
          899  +size_t sqlite3_quota_fread(
          900  +  void *pBuf,            /* Store the content here */
          901  +  size_t size,           /* Size of each element */
          902  +  size_t nmemb,          /* Number of elements to read */
          903  +  quota_FILE *p          /* Read from this quota_FILE object */
          904  +){
          905  +  return fread(pBuf, size, nmemb, p->f);
          906  +}
          907  +
          908  +/*
          909  +** Write content into a quota_FILE.  Invoke the quota callback and block
          910  +** the write if we exceed quota.
          911  +*/
          912  +size_t sqlite3_quota_fwrite(
          913  +  void *pBuf,            /* Take content to write from here */
          914  +  size_t size,           /* Size of each element */
          915  +  size_t nmemb,          /* Number of elements */
          916  +  quota_FILE *p          /* Write to this quota_FILE objecct */
          917  +){
          918  +  sqlite3_int64 iOfst;
          919  +  sqlite3_int64 iEnd;
          920  +  sqlite3_int64 szNew;
          921  +  quotaFile *pFile;
          922  +  
          923  +  iOfst = ftell(p->f);
          924  +  iEnd = iOfst + size*nmemb;
          925  +  pFile = p->pFile;
          926  +  if( pFile->iSize<iEnd ){
          927  +    quotaGroup *pGroup = pFile->pGroup;
          928  +    quotaEnter();
          929  +    szNew = pGroup->iSize - pFile->iSize + iEnd;
          930  +    if( szNew>pGroup->iLimit && pGroup->iLimit>0 ){
          931  +      if( pGroup->xCallback ){
          932  +        pGroup->xCallback(pFile->zFilename, &pGroup->iLimit, szNew, 
          933  +                          pGroup->pArg);
          934  +      }
          935  +      if( szNew>pGroup->iLimit && pGroup->iLimit>0 ){
          936  +        iEnd = pGroup->iLimit - pGroup->iSize + pFile->iSize;
          937  +        nmemb = (iEnd - iOfst)/size;
          938  +        iEnd = iOfst + size*nmemb;
          939  +        szNew = pGroup->iSize - pFile->iSize + iEnd;
          940  +      }
          941  +    }
          942  +    pGroup->iSize = szNew;
          943  +    pFile->iSize = iEnd;
          944  +    quotaLeave();
          945  +  }
          946  +  return fwrite(pBuf, size, nmemb, p->f);
          947  +}
          948  +
          949  +/*
          950  +** Close an open quota_FILE stream.
          951  +*/
          952  +int sqlite3_quota_fclose(quota_FILE *p){
          953  +  int rc;
          954  +  quotaFile *pFile;
          955  +  rc = fclose(p->f);
          956  +  pFile = p->pFile;
          957  +  quotaEnter();
          958  +  pFile->nRef--;
          959  +  if( pFile->nRef==0 ){
          960  +    quotaGroup *pGroup = pFile->pGroup;
          961  +    if( pFile->deleteOnClose ) quotaRemoveFile(pFile);
          962  +    quotaGroupDeref(pGroup);
          963  +  }
          964  +  quotaLeave();
          965  +  sqlite3_free(p);
          966  +  return rc;
          967  +}
          968  +
          969  +/*
          970  +** Seek on a quota_FILE stream.
          971  +*/
          972  +int sqlite3_quota_fseek(quota_FILE *p, long offset, int whence){
          973  +  return fseek(p->f, offset, whence);
          974  +}
          975  +
          976  +/*
          977  +** rewind a quota_FILE stream.
          978  +*/
          979  +void sqlite3_quota_rewind(quota_FILE *p){
          980  +  rewind(p->f);
          981  +}
          982  +
          983  +/*
          984  +** Tell the current location of a quota_FILE stream.
          985  +*/
          986  +long sqlite3_quota_ftell(quota_FILE *p){
          987  +  return ftell(p->f);
          988  +}
          989  +
          990  +/*
          991  +** Remove a file.  Update quotas accordingly.
          992  +*/
          993  +int sqlite3_quota_remove(const char *zFilename){
          994  +  int rc = remove(zFilename);
          995  +  sqlite3_quota_file(zFilename);
          996  +  return rc;
          997  +}
   834    998   
   835    999     
   836   1000   /***************************** Test Code ***********************************/
   837   1001   #ifdef SQLITE_TEST
   838   1002   #include <tcl.h>
   839   1003   
   840   1004   /*

Added src/test_quota.h.

            1  +/*
            2  +** 2011 December 1
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +**
           13  +** This file contains the interface definition for the quota a VFS shim.
           14  +**
           15  +** This particular shim enforces a quota system on files.  One or more
           16  +** database files are in a "quota group" that is defined by a GLOB
           17  +** pattern.  A quota is set for the combined size of all files in the
           18  +** the group.  A quota of zero means "no limit".  If the total size
           19  +** of all files in the quota group is greater than the limit, then
           20  +** write requests that attempt to enlarge a file fail with SQLITE_FULL.
           21  +**
           22  +** However, before returning SQLITE_FULL, the write requests invoke
           23  +** a callback function that is configurable for each quota group.
           24  +** This callback has the opportunity to enlarge the quota.  If the
           25  +** callback does enlarge the quota such that the total size of all
           26  +** files within the group is less than the new quota, then the write
           27  +** continues as if nothing had happened.
           28  +*/
           29  +#ifndef _QUOTA_H_
           30  +#include "sqlite3.h"
           31  +#include <stdio.h>
           32  +
           33  +/*
           34  +** Initialize the quota VFS shim.  Use the VFS named zOrigVfsName
           35  +** as the VFS that does the actual work.  Use the default if
           36  +** zOrigVfsName==NULL.  
           37  +**
           38  +** The quota VFS shim is named "quota".  It will become the default
           39  +** VFS if makeDefault is non-zero.
           40  +**
           41  +** THIS ROUTINE IS NOT THREADSAFE.  Call this routine exactly once
           42  +** during start-up.
           43  +*/
           44  +int sqlite3_quota_initialize(const char *zOrigVfsName, int makeDefault);
           45  +
           46  +/*
           47  +** Shutdown the quota system.
           48  +**
           49  +** All SQLite database connections must be closed before calling this
           50  +** routine.
           51  +**
           52  +** THIS ROUTINE IS NOT THREADSAFE.  Call this routine exactly once while
           53  +** shutting down in order to free all remaining quota groups.
           54  +*/
           55  +int sqlite3_quota_shutdown(void);
           56  +
           57  +/*
           58  +** Create or destroy a quota group.
           59  +**
           60  +** The quota group is defined by the zPattern.  When calling this routine
           61  +** with a zPattern for a quota group that already exists, this routine
           62  +** merely updates the iLimit, xCallback, and pArg values for that quota
           63  +** group.  If zPattern is new, then a new quota group is created.
           64  +**
           65  +** The zPattern is always compared against the full pathname of the file.
           66  +** Even if APIs are called with relative pathnames, SQLite converts the
           67  +** name to a full pathname before comparing it against zPattern.  zPattern
           68  +** is a standard glob pattern with the following matching rules:
           69  +**
           70  +**      '*'       Matches any sequence of zero or more characters.
           71  +**
           72  +**      '?'       Matches exactly one character.
           73  +**
           74  +**     [...]      Matches one character from the enclosed list of
           75  +**                characters.
           76  +**
           77  +**     [^...]     Matches one character not in the enclosed list.
           78  +**
           79  +** Note that, unlike unix shell globbing, the directory separator "/"
           80  +** can match a wildcard.  So, for example, the pattern "/abc/xyz/" "*"
           81  +** matches any files anywhere in the directory hierarchy beneath
           82  +** /abc/xyz
           83  +**
           84  +** If the iLimit for a quota group is set to zero, then the quota group
           85  +** is disabled and will be deleted when the last database connection using
           86  +** the quota group is closed.
           87  +**
           88  +** Calling this routine on a zPattern that does not exist and with a
           89  +** zero iLimit is a no-op.
           90  +**
           91  +** A quota group must exist with a non-zero iLimit prior to opening
           92  +** database connections if those connections are to participate in the
           93  +** quota group.  Creating a quota group does not affect database connections
           94  +** that are already open.
           95  +*/
           96  +int sqlite3_quota_set(
           97  +  const char *zPattern,           /* The filename pattern */
           98  +  sqlite3_int64 iLimit,           /* New quota to set for this quota group */
           99  +  void (*xCallback)(              /* Callback invoked when going over quota */
          100  +     const char *zFilename,         /* Name of file whose size increases */
          101  +     sqlite3_int64 *piLimit,        /* IN/OUT: The current limit */
          102  +     sqlite3_int64 iSize,           /* Total size of all files in the group */
          103  +     void *pArg                     /* Client data */
          104  +  ),
          105  +  void *pArg,                     /* client data passed thru to callback */
          106  +  void (*xDestroy)(void*)         /* Optional destructor for pArg */
          107  +);
          108  +
          109  +/*
          110  +** Bring the named file under quota management.  Or if it is already under
          111  +** management, update its size.
          112  +*/
          113  +int sqlite3_quota_file(const char *zFilename);
          114  +
          115  +/*
          116  +** The following object serves the same role as FILE in the standard C
          117  +** library.  It represents an open connection to a file on disk for I/O.
          118  +*/
          119  +typedef struct quota_FILE quota_FILE;
          120  +
          121  +/*
          122  +** Create a new quota_FILE object used to read and/or write to the
          123  +** file zFilename.  The zMode parameter is as with standard library zMode.
          124  +*/
          125  +quota_FILE *sqlite3_quota_fopen(const char *zFilename, const char *zMode);
          126  +
          127  +/*
          128  +** Perform I/O against a quota_FILE object.  When doing writes, the
          129  +** quota mechanism may result in a short write, in order to prevent
          130  +** the sum of sizes of all files from going over quota.
          131  +*/
          132  +size_t sqlite3_quota_fread(void*, size_t, size_t, quota_FILE*);
          133  +size_t sqlite3_quota_fwrite(void*, size_t, size_t, quota_FILE*);
          134  +
          135  +/*
          136  +** Close a quota_FILE object and free all associated resources.  The
          137  +** file remains under quota management.
          138  +*/
          139  +int sqlite3_quota_fclose(quota_FILE*);
          140  +
          141  +/*
          142  +** Move the read/write pointer for a quota_FILE object.  Or tell the
          143  +** current location of the read/write pointer.
          144  +*/
          145  +int sqlite3_quota_fseek(quota_FILE*, long, int);
          146  +void sqlite3_quota_rewind(quota_FILE*);
          147  +long sqlite3_quota_ftell(quota_FILE*);
          148  +
          149  +/*
          150  +** Delete a file from the disk.  If that file is under quota management,
          151  +** then adjust quotas accordingly.
          152  +**
          153  +** The file being deleted must not be open for reading or writing or as
          154  +** a database when it is deleted.
          155  +*/
          156  +int sqlite3_quota_remove(const char *zFilename);
          157  +
          158  +#endif /* _QUOTA_H_ */