/ Check-in [de804f4c]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Simplifications to the sorter to support full-coverage testing.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: de804f4c90f02ca98991da185ed5e28bdd319e92
User & Date: drh 2012-08-07 22:53:01
Context
2012-08-08
10:14
Change to securedel2.test so that it works even if SQLITE_DEFAULT_AUTOVACUUM=1 is defined. check-in: 1e6f5ea4 user: dan tags: trunk
2012-08-07
22:53
Simplifications to the sorter to support full-coverage testing. check-in: de804f4c user: drh tags: trunk
17:41
Add extra tests for secure-delete mode. check-in: e380cd3c user: dan tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/vdbesort.c.

   121    121     u8 *aAlloc;                     /* Allocated space */
   122    122     u8 *aKey;                       /* Pointer to current key */
   123    123     u8 *aBuffer;                    /* Current read buffer */
   124    124     int nBuffer;                    /* Size of read buffer in bytes */
   125    125   };
   126    126   
   127    127   /*
   128         -** An instance of this structure is used to separate the stream of records
          128  +** An instance of this structure is used to organize the stream of records
   129    129   ** being written to files by the merge-sort code into aligned, page-sized
   130         -** blocks.
          130  +** blocks.  Doing all I/O in aligned page-sized blocks helps I/O to go
          131  +** faster on many operating systems.
   131    132   */
   132    133   struct FileWriter {
          134  +  int eFWErr;                     /* Non-zero if in an error state */
   133    135     u8 *aBuffer;                    /* Pointer to write buffer */
   134    136     int nBuffer;                    /* Size of write buffer in bytes */
   135    137     int iBufStart;                  /* First byte of buffer to write */
   136    138     int iBufEnd;                    /* Last byte of buffer to write */
   137    139     i64 iWriteOff;                  /* Offset of start of buffer in file */
   138    140     sqlite3_file *pFile;            /* File to write to */
   139    141   };
................................................................................
   260    262   static int vdbeSorterIterVarint(sqlite3 *db, VdbeSorterIter *p, u64 *pnOut){
   261    263     int iBuf;
   262    264   
   263    265     iBuf = p->iReadOff % p->nBuffer;
   264    266     if( iBuf && (p->nBuffer-iBuf)>=9 ){
   265    267       p->iReadOff += sqlite3GetVarint(&p->aBuffer[iBuf], pnOut);
   266    268     }else{
   267         -    u8 aVarint[9];
   268         -    int i;
   269         -    for(i=0; i<sizeof(aVarint); i++){
   270         -      u8 *a;
   271         -      int rc = vdbeSorterIterRead(db, p, 1, &a);
          269  +    u8 aVarint[16], *a;
          270  +    int i = 0, rc;
          271  +    do{
          272  +      rc = vdbeSorterIterRead(db, p, 1, &a);
   272    273         if( rc ) return rc;
   273         -      aVarint[i] = *a;
   274         -      if( (aVarint[i] & 0x80)==0 ) break;
   275         -    }
          274  +      aVarint[(i++)&0xf] = a[0];
          275  +    }while( (a[0]&0x80)!=0 );
   276    276       sqlite3GetVarint(aVarint, pnOut);
   277    277     }
   278    278   
   279    279     return SQLITE_OK;
   280    280   }
   281    281   
   282    282   
................................................................................
   610    610     sqlite3_free(aSlot);
   611    611     return SQLITE_OK;
   612    612   }
   613    613   
   614    614   /*
   615    615   ** Initialize a file-writer object.
   616    616   */
   617         -static int fileWriterInit(
          617  +static void fileWriterInit(
   618    618     sqlite3 *db,                    /* Database (for malloc) */
   619    619     sqlite3_file *pFile,            /* File to write to */
   620    620     FileWriter *p,                  /* Object to populate */
   621    621     i64 iStart                      /* Offset of pFile to begin writing at */
   622    622   ){
   623    623     int nBuf = sqlite3BtreeGetPageSize(db->aDb[0].pBt);
   624    624   
   625    625     memset(p, 0, sizeof(FileWriter));
   626    626     p->aBuffer = (u8 *)sqlite3DbMallocRaw(db, nBuf);
   627         -  if( !p->aBuffer ) return SQLITE_NOMEM;
   628         -
   629         -  p->iBufEnd = p->iBufStart = (iStart % nBuf);
   630         -  p->iWriteOff = iStart - p->iBufStart;
   631         -  p->nBuffer = nBuf;
   632         -  p->pFile = pFile;
   633         -  return SQLITE_OK;
          627  +  if( !p->aBuffer ){
          628  +    p->eFWErr = SQLITE_NOMEM;
          629  +  }else{
          630  +    p->iBufEnd = p->iBufStart = (iStart % nBuf);
          631  +    p->iWriteOff = iStart - p->iBufStart;
          632  +    p->nBuffer = nBuf;
          633  +    p->pFile = pFile;
          634  +  }
   634    635   }
   635    636   
   636    637   /*
   637    638   ** Write nData bytes of data to the file-write object. Return SQLITE_OK
   638    639   ** if successful, or an SQLite error code if an error occurs.
   639    640   */
   640         -static int fileWriterWrite(FileWriter *p, u8 *pData, int nData){
          641  +static void fileWriterWrite(FileWriter *p, u8 *pData, int nData){
   641    642     int nRem = nData;
   642         -  while( nRem>0 ){
          643  +  while( nRem>0 && p->eFWErr==0 ){
   643    644       int nCopy = nRem;
   644    645       if( nCopy>(p->nBuffer - p->iBufEnd) ){
   645    646         nCopy = p->nBuffer - p->iBufEnd;
   646    647       }
   647    648   
   648    649       memcpy(&p->aBuffer[p->iBufEnd], &pData[nData-nRem], nCopy);
   649    650       p->iBufEnd += nCopy;
   650    651       if( p->iBufEnd==p->nBuffer ){
   651         -      int rc = sqlite3OsWrite(p->pFile, 
          652  +      p->eFWErr = sqlite3OsWrite(p->pFile, 
   652    653             &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart, 
   653    654             p->iWriteOff + p->iBufStart
   654    655         );
   655         -      if( rc!=SQLITE_OK ) return rc;
   656    656         p->iBufStart = p->iBufEnd = 0;
   657    657         p->iWriteOff += p->nBuffer;
   658    658       }
   659    659       assert( p->iBufEnd<p->nBuffer );
   660    660   
   661    661       nRem -= nCopy;
   662    662     }
   663         -
   664         -  return SQLITE_OK;
   665    663   }
   666    664   
   667    665   /*
   668    666   ** Flush any buffered data to disk and clean up the file-writer object.
   669    667   ** The results of using the file-writer after this call are undefined.
   670    668   ** Return SQLITE_OK if flushing the buffered data succeeds or is not 
   671    669   ** required. Otherwise, return an SQLite error code.
   672    670   **
   673    671   ** Before returning, set *piEof to the offset immediately following the
   674    672   ** last byte written to the file.
   675    673   */
   676    674   static int fileWriterFinish(sqlite3 *db, FileWriter *p, i64 *piEof){
   677         -  int rc = SQLITE_OK;
   678         -  if( p->aBuffer && p->iBufEnd>p->iBufStart ){
   679         -    rc = sqlite3OsWrite(p->pFile, 
          675  +  int rc;
          676  +  if( p->eFWErr==0 && ALWAYS(p->aBuffer) && p->iBufEnd>p->iBufStart ){
          677  +    p->eFWErr = sqlite3OsWrite(p->pFile, 
   680    678           &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart, 
   681    679           p->iWriteOff + p->iBufStart
   682    680       );
   683    681     }
   684    682     *piEof = (p->iWriteOff + p->iBufEnd);
   685    683     sqlite3DbFree(db, p->aBuffer);
          684  +  rc = p->eFWErr;
   686    685     memset(p, 0, sizeof(FileWriter));
   687    686     return rc;
   688    687   }
   689    688   
   690    689   /*
   691    690   ** Write value iVal encoded as a varint to the file-write object. Return 
   692    691   ** SQLITE_OK if successful, or an SQLite error code if an error occurs.
   693    692   */
   694         -static int fileWriterWriteVarint(FileWriter *p, u64 iVal){
          693  +static void fileWriterWriteVarint(FileWriter *p, u64 iVal){
   695    694     int nByte; 
   696    695     u8 aByte[10];
   697    696     nByte = sqlite3PutVarint(aByte, iVal);
   698         -  return fileWriterWrite(p, aByte, nByte);
          697  +  fileWriterWrite(p, aByte, nByte);
   699    698   }
   700    699   
   701    700   /*
   702    701   ** Write the current contents of the in-memory linked-list to a PMA. Return
   703    702   ** SQLITE_OK if successful, or an SQLite error code otherwise.
   704    703   **
   705    704   ** The format of a PMA is:
................................................................................
   709    708   **
   710    709   **     * One or more records packed end-to-end in order of ascending keys. 
   711    710   **       Each record consists of a varint followed by a blob of data (the 
   712    711   **       key). The varint is the number of bytes in the blob of data.
   713    712   */
   714    713   static int vdbeSorterListToPMA(sqlite3 *db, const VdbeCursor *pCsr){
   715    714     int rc = SQLITE_OK;             /* Return code */
   716         -  int rc2;                        /* fileWriterFinish return code */
   717    715     VdbeSorter *pSorter = pCsr->pSorter;
   718    716     FileWriter writer;
   719    717   
   720    718     memset(&writer, 0, sizeof(FileWriter));
   721    719   
   722    720     if( pSorter->nInMemory==0 ){
   723    721       assert( pSorter->pRecord==0 );
................................................................................
   730    728     if( rc==SQLITE_OK && pSorter->pTemp1==0 ){
   731    729       rc = vdbeSorterOpenTempFile(db, &pSorter->pTemp1);
   732    730       assert( rc!=SQLITE_OK || pSorter->pTemp1 );
   733    731       assert( pSorter->iWriteOff==0 );
   734    732       assert( pSorter->nPMA==0 );
   735    733     }
   736    734   
   737         -  if( rc==SQLITE_OK ){
   738         -    rc = fileWriterInit(db, pSorter->pTemp1, &writer, pSorter->iWriteOff);
   739         -  }
   740         -
   741    735     if( rc==SQLITE_OK ){
   742    736       SorterRecord *p;
   743    737       SorterRecord *pNext = 0;
   744    738   
   745         -
          739  +    fileWriterInit(db, pSorter->pTemp1, &writer, pSorter->iWriteOff);
   746    740       pSorter->nPMA++;
   747         -    rc = fileWriterWriteVarint(&writer, pSorter->nInMemory);
   748         -    for(p=pSorter->pRecord; rc==SQLITE_OK && p; p=pNext){
          741  +    fileWriterWriteVarint(&writer, pSorter->nInMemory);
          742  +    for(p=pSorter->pRecord; p; p=pNext){
   749    743         pNext = p->pNext;
   750         -      rc = fileWriterWriteVarint(&writer, p->nVal);
   751         -      if( rc==SQLITE_OK ){
   752         -        rc = fileWriterWrite(&writer, p->pVal, p->nVal);
   753         -      }
   754         -
          744  +      fileWriterWriteVarint(&writer, p->nVal);
          745  +      fileWriterWrite(&writer, p->pVal, p->nVal);
   755    746         sqlite3DbFree(db, p);
   756    747       }
   757         -
   758    748       pSorter->pRecord = p;
          749  +    rc = fileWriterFinish(db, &writer, &pSorter->iWriteOff);
   759    750     }
   760    751   
   761         -  rc2 = fileWriterFinish(db, &writer, &pSorter->iWriteOff);
   762         -  if( rc==SQLITE_OK ) rc = rc2;
   763         -
   764    752     return rc;
   765    753   }
   766    754   
   767    755   /*
   768    756   ** Add a record to the sorter.
   769    757   */
   770    758   int sqlite3VdbeSorterWrite(
................................................................................
   916    904   
   917    905         /* Open the second temp file, if it is not already open. */
   918    906         if( pTemp2==0 ){
   919    907           assert( iWrite2==0 );
   920    908           rc = vdbeSorterOpenTempFile(db, &pTemp2);
   921    909         }
   922    910   
   923         -      if( rc==SQLITE_OK ){
   924         -        rc = fileWriterInit(db, pTemp2, &writer, iWrite2);
   925         -      }
   926         -      if( rc==SQLITE_OK ){
   927         -        rc = fileWriterWriteVarint(&writer, nWrite);
   928         -      }
   929         -
   930    911         if( rc==SQLITE_OK ){
   931    912           int bEof = 0;
          913  +        fileWriterInit(db, pTemp2, &writer, iWrite2);
          914  +        fileWriterWriteVarint(&writer, nWrite);
   932    915           while( rc==SQLITE_OK && bEof==0 ){
   933    916             VdbeSorterIter *pIter = &pSorter->aIter[ pSorter->aTree[1] ];
   934    917             assert( pIter->pFile );
   935    918   
   936         -          rc = fileWriterWriteVarint(&writer, pIter->nKey);
   937         -          if( rc==SQLITE_OK ){
   938         -            rc = fileWriterWrite(&writer, pIter->aKey, pIter->nKey);
   939         -          }
   940         -          if( rc==SQLITE_OK ){
   941         -            rc = sqlite3VdbeSorterNext(db, pCsr, &bEof);
   942         -          }
          919  +          fileWriterWriteVarint(&writer, pIter->nKey);
          920  +          fileWriterWrite(&writer, pIter->aKey, pIter->nKey);
          921  +          rc = sqlite3VdbeSorterNext(db, pCsr, &bEof);
   943    922           }
          923  +        rc2 = fileWriterFinish(db, &writer, &iWrite2);
          924  +        if( rc==SQLITE_OK ) rc = rc2;
   944    925         }
   945         -
   946         -      rc2 = fileWriterFinish(db, &writer, &iWrite2);
   947         -      if( rc==SQLITE_OK ) rc = rc2;
   948    926       }
   949    927   
   950    928       if( pSorter->nPMA<=SORTER_MAX_MERGE_COUNT ){
   951    929         break;
   952    930       }else{
   953    931         sqlite3_file *pTmp = pSorter->pTemp1;
   954    932         pSorter->nPMA = iNew;