SQLite: Check-in [5ffec5db]

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview

Comment:	In the LSM extension, ensure that empty space on pages is zeroed before they are written to disk. This helps with compressed databases, and stops valgrind complaining about uninitialized bytes and write().
Downloads:	Tarball \| ZIP archive \| SQL archive
Timelines:	family \| ancestors \| descendants \| both \| trunk
Files:	files \| file ages \| folders
SHA3-256:	5ffec5db33137251090d45e2ca7e3e7823e3215ae89822d905923424ceba2ac2
User & Date:	dan 2017-11-16 20:48:47
Original Comment:	Ensure that empty space on pages is zeroed before they are written to disk. This helps with compressed databases, and stops valgrind complaining about uninitialized bytes and write().

Context

2017-11-17
13:21		Do not count temporary databases that have been attached using ATTACH when figuring out if a master-journal file is required by a transaction. check-in: 93e012a3 user: dan tags: trunk
08:20		Avoid creating a master journal if all or all but one of the databases in the transaction is a temp file. Closed-Leaf check-in: 355d1232 user: dan tags: master-journal-temp-files
2017-11-16
20:48		In the LSM extension, ensure that empty space on pages is zeroed before they are written to disk. This helps with compressed databases, and stops valgrind complaining about uninitialized bytes and write(). check-in: 5ffec5db user: dan tags: trunk
19:04		Add a missing "finish_test" to the end of the stmtvtab1.test script. check-in: e0b5c058 user: drh tags: trunk

Changes

Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/lsm1/lsm_sorted.c.

** The following macros are used to access a page footer.
*/
#define SEGMENT_NRECORD_OFFSET(pgsz)        ((pgsz) - 2)
#define SEGMENT_FLAGS_OFFSET(pgsz)          ((pgsz) - 2 - 2)
#define SEGMENT_POINTER_OFFSET(pgsz)        ((pgsz) - 2 - 2 - 8)
#define SEGMENT_CELLPTR_OFFSET(pgsz, iCell) ((pgsz) - 2 - 2 - 8 - 2 - (iCell)*2)

#define SEGMENT_EOF(pgsz, nEntry) SEGMENT_CELLPTR_OFFSET(pgsz, nEntry)

#define SEGMENT_BTREE_FLAG     0x0001
#define PGFTR_SKIP_NEXT_FLAG   0x0002
#define PGFTR_SKIP_THIS_FLAG   0x0004


#ifndef LSM_SEGMENTPTR_FREE_THRESHOLD
................................................................................
    nHdr = 1 + lsmVarintLen32(iRPtr) + lsmVarintLen32(nKey);
    if( rtIsWrite(eType) ) nHdr += lsmVarintLen32(nVal);

    /* If the entire header will not fit on page pPg, or if page pPg is 
    ** marked read-only, advance to the next page of the output run. */
    iOff = pMerge->iOutputOff;
    if( iOff<0 || pPg==0 || iOff+nHdr > SEGMENT_EOF(nData, nRec+1) ){





      iFPtr = (int)*pMW->pCsr->pPrevMergePtr;
      iRPtr = iPtr - iFPtr;
      iOff = 0;
      nRec = 0;
      rc = mergeWorkerNextPage(pMW, iFPtr);
      pPg = pMW->pPage;
    }
................................................................................
  int i;                          /* Iterator variable */
  int rc = *pRc;
  MultiCursor *pCsr = pMW->pCsr;

  /* Unless the merge has finished, save the cursor position in the
  ** Merge.aInput[] array. See function mergeWorkerInit() for the 
  ** code to restore a cursor position based on aInput[].  */
  if( rc==LSM_OK && pCsr && lsmMCursorValid(pCsr) ){
    Merge *pMerge = pMW->pLevel->pMerge;

    int bBtree = (pCsr->pBtCsr!=0);
    int iPtr;

    /* pMerge->nInput==0 indicates that this is a FlushTree() operation. */
    assert( pMerge->nInput==0 || pMW->pLevel->nRight>0 );
    assert( pMerge->nInput==0 || pMerge->nInput==(pCsr->nPtr+bBtree) );

    for(i=0; i<(pMerge->nInput-bBtree); i++){
      SegmentPtr *pPtr = &pCsr->aPtr[i];
      if( pPtr->pPg ){
        pMerge->aInput[i].iPg = lsmFsPageNumber(pPtr->pPg);
        pMerge->aInput[i].iCell = pPtr->iCell;
      }else{
        pMerge->aInput[i].iPg = 0;
        pMerge->aInput[i].iCell = 0;
      }
    }
    if( bBtree && pMerge->nInput ){
      assert( i==pCsr->nPtr );
      btreeCursorPosition(pCsr->pBtCsr, &pMerge->aInput[i]);
    }

    /* Store the location of the split-key */
    iPtr = pCsr->aTree[1] - CURSOR_DATA_SEGMENT;
    if( iPtr<pCsr->nPtr ){
      pMerge->splitkey = pMerge->aInput[iPtr];
    }else{
      btreeCursorSplitkey(pCsr->pBtCsr, &pMerge->splitkey);












    }
    
    pMerge->iOutputOff = -1;
  }

  lsmMCursorClose(pCsr, 0);








|







 







>
>
>
>
>







 







|

>
|
|

|
|
|

|
|
|
|
|
|
|
|
|
|
|
|
|
|

|
|
|
|
|
|
>
>
>
>
>
>
>
>
>
>
>
>

** The following macros are used to access a page footer.
*/
#define SEGMENT_NRECORD_OFFSET(pgsz)        ((pgsz) - 2)
#define SEGMENT_FLAGS_OFFSET(pgsz)          ((pgsz) - 2 - 2)
#define SEGMENT_POINTER_OFFSET(pgsz)        ((pgsz) - 2 - 2 - 8)
#define SEGMENT_CELLPTR_OFFSET(pgsz, iCell) ((pgsz) - 2 - 2 - 8 - 2 - (iCell)*2)

#define SEGMENT_EOF(pgsz, nEntry) SEGMENT_CELLPTR_OFFSET(pgsz, nEntry-1)

#define SEGMENT_BTREE_FLAG     0x0001
#define PGFTR_SKIP_NEXT_FLAG   0x0002
#define PGFTR_SKIP_THIS_FLAG   0x0004


#ifndef LSM_SEGMENTPTR_FREE_THRESHOLD
................................................................................
    nHdr = 1 + lsmVarintLen32(iRPtr) + lsmVarintLen32(nKey);
    if( rtIsWrite(eType) ) nHdr += lsmVarintLen32(nVal);

    /* If the entire header will not fit on page pPg, or if page pPg is 
    ** marked read-only, advance to the next page of the output run. */
    iOff = pMerge->iOutputOff;
    if( iOff<0 || pPg==0 || iOff+nHdr > SEGMENT_EOF(nData, nRec+1) ){
      if( iOff>=0 && pPg ){
        /* Zero any free space on the page */
        assert( aData );
        memset(&aData[iOff], 0, SEGMENT_EOF(nData, nRec)-iOff);
      }
      iFPtr = (int)*pMW->pCsr->pPrevMergePtr;
      iRPtr = iPtr - iFPtr;
      iOff = 0;
      nRec = 0;
      rc = mergeWorkerNextPage(pMW, iFPtr);
      pPg = pMW->pPage;
    }
................................................................................
  int i;                          /* Iterator variable */
  int rc = *pRc;
  MultiCursor *pCsr = pMW->pCsr;

  /* Unless the merge has finished, save the cursor position in the
  ** Merge.aInput[] array. See function mergeWorkerInit() for the 
  ** code to restore a cursor position based on aInput[].  */
  if( rc==LSM_OK && pCsr ){
    Merge *pMerge = pMW->pLevel->pMerge;
    if( lsmMCursorValid(pCsr) ){
      int bBtree = (pCsr->pBtCsr!=0);
      int iPtr;

      /* pMerge->nInput==0 indicates that this is a FlushTree() operation. */
      assert( pMerge->nInput==0 || pMW->pLevel->nRight>0 );
      assert( pMerge->nInput==0 || pMerge->nInput==(pCsr->nPtr+bBtree) );

      for(i=0; i<(pMerge->nInput-bBtree); i++){
        SegmentPtr *pPtr = &pCsr->aPtr[i];
        if( pPtr->pPg ){
          pMerge->aInput[i].iPg = lsmFsPageNumber(pPtr->pPg);
          pMerge->aInput[i].iCell = pPtr->iCell;
        }else{
          pMerge->aInput[i].iPg = 0;
          pMerge->aInput[i].iCell = 0;
        }
      }
      if( bBtree && pMerge->nInput ){
        assert( i==pCsr->nPtr );
        btreeCursorPosition(pCsr->pBtCsr, &pMerge->aInput[i]);
      }

      /* Store the location of the split-key */
      iPtr = pCsr->aTree[1] - CURSOR_DATA_SEGMENT;
      if( iPtr<pCsr->nPtr ){
        pMerge->splitkey = pMerge->aInput[iPtr];
      }else{
        btreeCursorSplitkey(pCsr->pBtCsr, &pMerge->splitkey);
      }
    }

    /* Zero any free space left on the final page. This helps with
    ** compression if using a compression hook. And prevents valgrind
    ** from complaining about uninitialized byte passed to write(). */
    if( pMW->pPage ){
      int nData;
      u8 *aData = fsPageData(pMW->pPage, &nData);
      int iOff = pMerge->iOutputOff;
      int iEof = SEGMENT_EOF(nData, pageGetNRec(aData, nData));
      memset(&aData[iOff], 0, iEof - iOff);
    }
    
    pMerge->iOutputOff = -1;
  }

  lsmMCursorClose(pCsr, 0);