SQLite4  Check-in [94016fe5e8]

         random.o resolve.o rowset.o rtree.o select.o status.o \
         tokenize.o trigger.o \
         update.o util.o varint.o \
         vdbeapi.o vdbeaux.o vdbecodec.o vdbecursor.o \
         vdbemem.o vdbetrace.o \
         walker.o where.o utf.o

LIBOBJ += bt_unix.o bt_pager.o bt_main.o bt_varint.o kvbt.o bt_lock.o bt_log.o

# All of the source code files.
#
SRC = \
  $(TOP)/src/alter.c \
  $(TOP)/src/analyze.c \
  $(TOP)/src/attach.c \

*/
int sqlite4BtPagerSetCookie(BtPager*, u32 iVal);
int sqlite4BtPagerGetCookie(BtPager*, u32 *piVal);

/*
** Return a pointer to a buffer containing the name of the pager log file.
*/
#define BT_PAGERFILE_DATABASE 0
#define BT_PAGERFILE_LOG      1
#define BT_PAGERFILE_SHM      2
const char *sqlite4BtPagerFilename(BtPager*, int ePagerfile);

/*
** End of bt_pager.c interface.
*************************************************************************/

/*************************************************************************
** File-system interface.

/*
** End of bt_log.c interface.
*************************************************************************/

/*************************************************************************
** Interface to bt_lock.c functionality.
*/
typedef struct BtShared BtShared;
typedef struct BtLock BtLock;
struct BtLock {
  /* These three are set by the bt_pager module and thereafter used by 
  ** both bt_lock and bt_pager. */
  sqlite4_env *pEnv;              /* SQLite environment */
  bt_env *pVfs;                   /* Bt environment */
  bt_file *pFd;                   /* Database file descriptor */

  /* These are used only by the bt_lock module. */
  BtShared *pShared;              /* Shared by all handles on this file */
  BtLock *pNext;                  /* Next connection using pShared */
};

/* Connect and disconnect procedures */
int sqlite4BtLockConnect(BtLock*, int (*xRecover)(BtLock*));
int sqlite4BtLockDisconnect(BtLock*, int(*xCkpt)(BtLock*), int(*xDel)(BtLock*));

/* Obtain and release the WRITER lock */

**    May you share freely, never taking more than you give.
**
*************************************************************************
*/

#include "btInt.h"

#include "sqliteInt.h"

#include <string.h>
#include <assert.h>
#include <stdio.h>

typedef struct BtFile BtFile;

/*
** Global data. All global variables used by code in this file are grouped
** into the following structure instance.
**
** pDatabase:
**   Linked list of all Database objects allocated within this process.
**   This list may not be traversed without holding the global mutex (see
**   functions enterGlobalMutex() and leaveGlobalMutex()).
*/
static struct SharedData {
  BtShared *pDatabase;            /* Linked list of all Database objects */
} gShared;

struct BtFile {
  BtFile *pNext;
  bt_file *pFd;
};

struct BtShared {
  /* Protected by the global mutex (see btLockMutexEnter()/Leave()) */
  char *zName;                    /* Canonical path to database file */
  int nName;                      /* strlen(zName) */
  int nRef;                       /* Number of handles open on this file */
  BtShared *pNext;                /* Next BtShared structure in global list */

  /* Protected by the local mutex (pClientMutex) */
  sqlite4_mutex *pClientMutex;    /* Protects the apShmChunk[] and pConn */
  int nShmChunk;                  /* Number of entries in apShmChunk[] array */
  u8 **apShmChunk;                /* Array of "shared" memory regions */
  BtLock *pLock;                  /* List of connnections to this db */

  /* Multi-process mode stuff */
  int bMultiProc;                 /* True if running in multi-process mode */
  int bReadonly;                  /* True if Database.pFile is read-only */
  bt_file *pFile;                 /* Used for locks/shm in multi-proc mode */
  BtFile *pBtFile;                /* List of deferred closes */
};

/*
** Grab the global mutex that protects the linked list of BtShared
** objects.
*/
static void btLockMutexEnter(){
  /* todo... */
}

/*
** Relinquish the mutex obtained by calling btLockMutexEnter().
*/
static void btLockMutexLeave(){
  /* todo... */
}

/*
** Connect to the database as a read/write connection. If recovery
** is required (i.e. if this is the first connection to the db), invoke 
** the xRecover() method.
**
** Return SQLITE4_OK if successful, or an SQLite4 error code if an
** error occurs.
*/
int sqlite4BtLockConnect(BtLock *p, int (*xRecover)(BtLock*)){
  sqlite4_env *pEnv = p->pEnv;
  int rc = SQLITE4_OK;
  const char *zName;
  int nName;
  BtShared *pShared;

  zName = sqlite4BtPagerFilename((BtPager*)p, BT_PAGERFILE_DATABASE);
  nName = strlen(zName);

  btLockMutexEnter();
  for(pShared=gShared.pDatabase; pShared; pShared=pShared->pNext){
    if( pShared->nName==nName && 0==memcmp(zName, pShared->zName, nName) ){
      break;
    }
  }

  if( pShared==0 ){
    sqlite4_mutex *pMutex;
    pShared = (BtShared*)sqlite4_malloc(pEnv, sizeof(BtShared) + nName + 1);
    pMutex = sqlite4_mutex_alloc(pEnv, SQLITE4_MUTEX_RECURSIVE);

    if( pShared==0 || pMutex==0 ){
      sqlite4_free(pEnv, pShared);
      sqlite4_mutex_free(pMutex);
      pShared = 0;
      pMutex = 0;
      rc = btErrorBkpt(SQLITE4_NOMEM);
    }else{
      memset(pShared, 0, sizeof(BtShared));
      pShared->nName = nName;
      pShared->zName = (char *)&pShared[1];
      memcpy(pShared->zName, zName, nName+1);
      pShared->pNext = gShared.pDatabase;
      pShared->pClientMutex = pMutex;
      gShared.pDatabase = pShared;
    }
  }

  if( rc==SQLITE4_OK ){
    pShared->nRef++;
  }
  btLockMutexLeave();

  /* Add this connection to the linked list at BtShared.pLock */
  if( rc==SQLITE4_OK ){
    sqlite4_mutex_enter(pShared->pClientMutex);
    p->pNext = pShared->pLock;
    pShared->pLock = p;
    sqlite4_mutex_leave(pShared->pClientMutex);
    p->pShared = pShared;
  }

  if( rc==SQLITE4_OK ){
    rc = xRecover(p);
  }

  return rc;
}

/*
** Disconnect the read/write connection passed as the first argument
** from the database.
**
** If a checkpoint is required (i.e. if this is the last read/write

** that an error occurred while checkpointing or deleting the log file.
*/
int sqlite4BtLockDisconnect(
  BtLock *p,                      /* Locker handle */
  int (*xCkpt)(BtLock*),          /* Callback to checkpoint database */
  int (*xDel)(BtLock*)            /* Callback to delete wal+shm files */
){
  BtShared *pShared = p->pShared;
  BtLock **pp;
  int rc;                         /* Return code */
  rc = xCkpt(p);
  if( rc==SQLITE4_OK ){
    rc = xDel(p);
  }

  sqlite4_mutex_enter(pShared->pClientMutex);
  for(pp=&p->pShared->pLock; *pp!=p; pp=&(*pp)->pNext);
  *pp = (*pp)->pNext;
  sqlite4_mutex_leave(pShared->pClientMutex);

  btLockMutexEnter();
  pShared->nRef--;
  if( pShared->nRef==0 ){
    BtShared **ppS;
    for(ppS=&gShared.pDatabase; *ppS!=pShared; ppS=&(*ppS)->pNext);
    *ppS = (*ppS)->pNext;

    sqlite4_mutex_free(pShared->pClientMutex);
    sqlite4_free(p->pEnv, pShared);
  }
  btLockMutexLeave();
  return rc;
}

int sqlite4BtLockCheckpoint(BtLock *p, int (*xCkpt)(BtLock*)){
  return xCkpt(p);
}

/* Obtain a READER lock. 
**
** Argument aLog points to an array of 6 frame addresses. These are the 
** first and last frames in each of log regions A, B and C. Argument 
** aLock points to the array of read-lock slots in shared memory.
*/
int sqlite4BtLockReader(BtLock *pLock, u32 *aLog, u32 *aLock){
  return SQLITE4_OK;
}

int sqlite4BtLockShmMap(BtLock *pLock, int iChunk, int nByte, u8 **ppOut){
  int rc = SQLITE4_OK;
  BtShared *pShared = pLock->pShared;
  u8 *pOut = 0;

  assert( pShared->bMultiProc==0 && pShared->bReadonly==0 );

  sqlite4_mutex_enter(pShared->pClientMutex);
  if( pShared->nShmChunk<=iChunk ){
    u8 **apNew;
    int nNew = iChunk+1;
    int nByte = sizeof(u8*)*nNew;

    apNew = (u8**)sqlite4_realloc(pLock->pEnv, pShared->apShmChunk, nByte);
    if( apNew==0 ){
      rc = btErrorBkpt(SQLITE4_NOMEM);
    }else{
      memset(&apNew[pShared->nShmChunk],0,nByte-sizeof(u8*)*pShared->nShmChunk);
      pShared->nShmChunk = nNew;
      pShared->apShmChunk = apNew;
    }
  }

  if( rc==SQLITE4_OK ){
    if( pShared->apShmChunk[iChunk]==0 ){
      pShared->apShmChunk[iChunk] = (u8*)sqlite4_malloc(pLock->pEnv, nByte);
    }
    pOut = pShared->apShmChunk[iChunk];
    if( pOut==0 ){
      rc = btErrorBkpt(SQLITE4_NOMEM);
    }
  }
  sqlite4_mutex_leave(pShared->pClientMutex);
  
  *ppOut = pOut;
  return rc;
}

*/

#include "btInt.h"

#include <string.h>
#include <assert.h>
#include <stdio.h>
#include <stddef.h>

/* Magic values identifying WAL file header */
#define BT_WAL_MAGIC   0xBEE1CA62
#define BT_WAL_VERSION 0x00000001

typedef struct BtWalHdr BtWalHdr;
typedef struct BtShmHdr BtShmHdr;
typedef struct BtCkptHdr BtCkptHdr;
typedef struct BtFrameHdr BtFrameHdr;
typedef struct BtShm BtShm;

/*
** WAL file header. All u32 fields are stored in big-endian order on
** disk. A single WAL file may contain two of these headers - one at
** byte offset 0 and the other at offset <sector-size> (the start of
** the second disk sector in the file, according to the xSectorSize
** VFS method).

** from the start of it. The two hash-slot arrays remain the same size, 
** but the number of entries in the page-number array is reduced to the
** value defined below.
*/
#define HASHTABLE_NFRAME_ONE (HASHTABLE_NFRAME - (sizeof(BtShm)/sizeof(u32)))

/*
** Number of slots in a hash table.
*/
#define HASHTABLE_NSLOT (HASHTABLE_NFRAME * 2)

#define HASHTABLE_OFFSET_1 (HASHTABLE_NFRAME * sizeof(u32))
#define HASHTABLE_OFFSET_2 (HASHTABLE_OFFSET_1+HASHTABLE_NSLOT*sizeof(ht_slot))

/*
** Used to calculate hash keys.
*/
#define HASHTABLE_KEY_MUL 383


/*

** also run database recovery before returning. In this case, the caller
** has already obtained the required locks.
*/
int sqlite4BtLogOpen(BtPager *pPager, int bRecover, BtLog **ppLog){
  BtLock *pLock = (BtLock*)pPager;
  bt_env *pVfs = pLock->pVfs;
  sqlite4_env *pEnv = pLock->pEnv;
  int rc;                         /* Return code */
  const char *zWal;               /* Name of log file to open */
  BtLog *pLog;                    /* Log handle to return */

  pLog = sqlite4_malloc(pEnv, sizeof(BtLog));
  if( pLog==0 ){
    rc = SQLITE4_NOMEM;
    goto open_out;
  }
  memset(pLog, 0, sizeof(BtLog));
  pLog->pLock = (BtLock*)pPager;

  zWal = sqlite4BtPagerFilename(pPager, BT_PAGERFILE_LOG);
  rc = pVfs->xOpen(pEnv, pVfs, zWal, 0, &pLog->pFd);

  if( rc==SQLITE4_OK && bRecover ){
    /* TODO: Run recovery... */
  }

 open_out:
  if( rc!=SQLITE4_OK ){
    sqlite4_free(pEnv, pLog);
    pLog = 0;
  }
  *ppLog = pLog;
  return rc;
}

/*
** Close the log file handle BtLog*. 
*/
int sqlite4BtLogClose(BtLog *pLog, int bCleanup){
  int rc = SQLITE4_OK;
  sqlite4_env *pEnv = pLog->pLock->pEnv;
  bt_env *pVfs = pLog->pLock->pVfs;

  pVfs->xClose(pLog->pFd);
  sqlite4_free(pEnv, pLog);

  if( bCleanup ){
    BtPager *pPager = (BtPager*)pLog->pLock;
    const char *zWal = sqlite4BtPagerFilename(pPager, BT_PAGERFILE_LOG);
    rc = pVfs->xUnlink(pEnv, pVfs, zWal);
  }

  return rc;
}

/*
** Attempt to read data for page pgno from the log file. If successful,
** the data is written into buffer aData[] (which must be at least as
** large as a database page). In this case SQLITE4_OK is returned.
**
** If the log does not contain any version of page pgno, SQLITE4_NOTFOUND
** is returned and the contents of buffer aData[] are not modified.
**
** If any other error occurs, an SQLite4 error code is returned. The final
** state of buffer aData[] is undefined in this case.
*/
int sqlite4BtLogRead(BtLog *pLog, u32 pgno, u8 *aData){
  return SQLITE4_OK;
}

/*
** Return true if log is completely empty (as it is if a file zero bytes
** in size has been opened or created).
*/
static int btLogIsEmpty(BtLog *pLog){

static int btLogWriteHeader(BtLog *pLog, int iHdr, BtWalHdr *pHdr){
  int rc;                         /* Return code */
  i64 iOff;                       /* File offset to write to */
  assert( iHdr==0 || iHdr==1 );

  /* Calculate a checksum for the header */
  btLogChecksum(1, (u8*)pHdr, offsetof(BtWalHdr, aCksum), 0, pHdr->aCksum);

  /* Write the object to disk */
  iOff = iHdr * pLog->snapshot.nSector;
  rc = btLogWriteData(pLog, iOff, (u8*)pHdr, sizeof(BtWalHdr));

  return rc;
}

/*
** Ensure that shared-memory chunk iChunk is mapped and available in
** the BtLog.apShm[] array. If an error occurs, return an SQLite4 error
** code. Otherwise, SQLITE4_OK.
*/
static int btLogMapShm(BtLog *pLog, int iChunk){
  int rc = SQLITE4_OK;

  if( pLog->nShm<=iChunk ){
    sqlite4_env *pEnv = pLog->pLock->pEnv;
    u8 **apNew;
    int nNew = iChunk+1;

    apNew = (u8**)sqlite4_malloc(pEnv, sizeof(u8*)*nNew);
    if( apNew==0 ) return btErrorBkpt(SQLITE4_NOMEM);
    memcpy(apNew, pLog->apShm, sizeof(u8*)*pLog->nShm);
    memset(&apNew[pLog->nShm], 0, (nNew-pLog->nShm) * sizeof(u8*));
    pLog->nShm = nNew;
    pLog->apShm = apNew;
  }

/*
** Locate the iHash'th hash table in shared memory. Return it.
*/
static int btLogFindHash(
  BtLog *pLog,                    /* Log handle */
  int iHash,                      /* Hash table (numbered from 0) to find */
  ht_slot **paHash,               /* OUT: Pointer to hash slots */
  u32 **paPgno,                   /* OUT: Pointer to page number array */
  u32 *piZero                     /* OUT: Frame associated with *paPgno[0] */
){
  int rc;

  rc = btLogMapShm(pLog, iHash);
  if( rc==SQLITE4_OK ){
    u8 *aChunk = pLog->apShm[iHash];
    u32 *aPgno;
    u32 iZero;

    *paHash = (ht_slot*)&aChunk[HASHTABLE_OFFSET_1];
    if( iHash==0 ){
      aPgno = (u32*)&aChunk[sizeof(BtShm)];
      iZero = 1;
    }else{
      aPgno = (u32*)aChunk;
      iZero = 1 + HASHTABLE_NFRAME_ONE + (HASHTABLE_NFRAME * (iHash-1));
    }
    *paPgno = aPgno;
    *piZero = iZero;
  }

  return rc;
}

static BtShm *btLogShm(BtLog *pLog){
  return (BtShm*)(pLog->apShm[0]);
}

/*
** Return the index of the hash table that contains the entry for frame
** iFrame. 
*/
static int btLogFrameHash(BtLog *pLog, u32 iFrame){
  if( iFrame<=HASHTABLE_NFRAME_ONE ) return 0;
  return 1 + ((iFrame - HASHTABLE_NFRAME_ONE - 1) / HASHTABLE_NFRAME);
}

/*
** Return a hash key for page number pgno.
*/
static int btLogHashKey(BtLog *pLog, u32 pgno){
  assert( pgno>=1 );
  return ((pgno * HASHTABLE_KEY_MUL) % HASHTABLE_NSLOT);
}
static int btLogHashNext(BtLog *pLog, int iSlot){
  return ((iSlot + 1) % HASHTABLE_NSLOT);
}

/*
** Add an entry mapping database page pgno to log frame iFrame to the
** the shared hash table. Return SQLITE4_OK if successful, or an SQLite4
** error code if an error occurs.
*/
static int btLogHashInsert(BtLog *pLog, u32 pgno, u32 iFrame){
  int iHash;                      /* Index of hash table to update */
  int rc = SQLITE4_OK;            /* Return code */
  ht_slot *aHash;                 /* Hash slots */
  u32 *aPgno;                     /* Page array for updated hash table */
  u32 iZero;                      /* Zero-offset of updated hash table */

  assert( iFrame>=1 && pgno>=1 );
  assert( iFrame==(pLog->snapshot.aLog[5]+1) );

  /* Find the required hash table */
  iHash = btLogFrameHash(pLog, iFrame);
  rc = btLogFindHash(pLog, iHash, &aHash, &aPgno, &iZero);

  /* Update the hash table */
  if( rc==SQLITE4_OK ){
    int iSlot;
    int nCollide = HASHTABLE_NSLOT*2;

    aPgno[iFrame-iZero] = pgno;

    for(iSlot=btLogHashKey(pLog,pgno); ; iSlot=btLogHashNext(pLog, iSlot)){
      if( aHash[iSlot]==0 ){
        aHash[iSlot] = (iFrame-iZero+1);
        break;
      }
      if( (nCollide--)==0 ) return btErrorBkpt(SQLITE4_CORRUPT);
    }
  }

  return rc;
}

static int btLogUpdateSharedHdr(BtLog *pLog){
  bt_env *pVfs = pLog->pLock->pVfs;
  BtShmHdr *p = &pLog->snapshot;
  BtShm *pShm = btLogShm(pLog);

  /* Calculate a checksum for the private snapshot object. */
  btLogChecksum(1, (u8*)p, offsetof(BtShmHdr, aCksum), 0, p->aCksum);

  /* Update the shared object. */
  pVfs->xShmBarrier(pLog->pFd);
  memcpy(&pShm->hdr1, p, sizeof(BtShmHdr));
  pVfs->xShmBarrier(pLog->pFd);
  memcpy(&pShm->hdr2, p, sizeof(BtShmHdr));

  return SQLITE4_OK;
}

/*
** Return the offset of frame iFrame within the log file.
*/
static i64 btLogFrameOffset(BtLog *pLog, int pgsz, u32 iFrame){
  return (i64)pLog->snapshot.nSector*2 + (i64)(iFrame-1) * (i64)pgsz;
}

/*
** Write a frame to the log file.
*/
int sqlite4BtLogWrite(BtLog *pLog, u32 pgno, u8 *aData, int bCommit){
  const int pgsz = sqlite4BtPagerPagesize((BtPager*)(pLog->pLock));
  int rc = SQLITE4_OK;
  u32 iFrame;                     /* Write this frame (numbered from 1) */
  BtFrameHdr frame;               /* Header for new frame */
  u32 *a;                         /* Pointer to cksum of previous frame */
  i64 iOff;                       /* Offset of log file to write to */

  /* Handle a special case - if the log file is completely empty then
  ** this writer must write the first header into the WAL file. */
  if( btLogIsEmpty(pLog) ){
    BtWalHdr hdr;
    memset(&hdr, 0, sizeof(BtWalHdr));

    hdr.iMagic = BT_WAL_MAGIC;
    hdr.iVersion = BT_WAL_VERSION;
    hdr.nSector = pLog->snapshot.nSector;
    hdr.iSalt1 = 22;
    hdr.iSalt2 = 23;
    hdr.iFirstFrame = 1;

    rc = btLogWriteHeader(pLog, 0, &hdr);
    if( rc!=SQLITE4_OK ) return rc;
  }

  /* Figure out where exactly to write the new data */
  iFrame = pLog->snapshot.aLog[5] + 1;
  iOff = btLogFrameOffset(pLog, pgsz, iFrame);

  /* Populate the frame header object. */
  memset(&frame, 0, sizeof(frame));
  frame.pgno = pgno;
  frame.ctrl = (bCommit ? 0x80000000 : 0x00000000);
  a = pLog->snapshot.aFrameCksum;
  btLogChecksum(1, (u8*)&frame, offsetof(BtFrameHdr, aCksum), a, frame.aCksum);
  btLogChecksum(1, aData, pgsz, frame.aCksum, frame.aCksum);

  /* Write the header and page record to the log file. */
  rc = btLogWriteData(pLog, iOff, (u8*)&frame, sizeof(frame));
  if( rc==SQLITE4_OK ){
    rc = btLogWriteData(pLog, iOff+sizeof(frame), aData, pgsz);
  }

  /* Update the wal index hash tables with the (pgno -> iFrame) record. */
  if( rc==SQLITE4_OK ){
    rc = btLogHashInsert(pLog, pgno, iFrame);

int sqlite4BtLogSnapshotClose(BtLog *pLog){
  return SQLITE4_OK;
}

int sqlite4BtLogSnapshotWritable(BtLog *pLog){
  return 1;
}

int sqlite4BtLogCheckpoint(BtLog *pLog){
}

};

/*
** Pager object.
*/
struct BtPager {
  BtLock btl;                     /* Variables shared with bt_lock module */
  BtLog *pLog;                    /* Logging module */
  int iTransactionLevel;          /* Current transaction level (see bt.h) */
  char *zFile;                    /* Database file name */
  int nFile;                      /* Length of string zFile in bytes */
  BtPageHash hash;                /* Hash table */
  BtPage *pDirty;                 /* List of all dirty pages */
  BtDbhdr dbhdr;
  int nTotalRef;

  int rc;
  rc = sqlite4BtLogOpen(p, 1, &p->pLog);
  return rc;
}

static int btCheckpoint(BtLock *pLock){
  BtPager *p = (BtPager*)pLock;
  return sqlite4BtLogCheckpoint(p->pLog);
}

static int btCleanup(BtLock *pLock){
  BtPager *p = (BtPager*)pLock;
  int rc = sqlite4BtLogClose(p->pLog, 1);
  return rc;
}

  }

  /* If the database file was successfully opened, connect to the
  ** database and open the database logger. */
  if( rc==SQLITE4_OK ){
    rc = sqlite4BtLockConnect((BtLock*)p, btRecover);
    if( rc==SQLITE4_OK && p->pLog==0 ){
      rc = sqlite4BtLogOpen(p, 0, &p->pLog);
    }
  }

  assert( (rc==SQLITE4_OK)==(p->btl.pFd!=0 && p->pLog!=0) );
  return rc;
}

*/
int sqlite4BtPagerGetCookie(BtPager *p, u32 *piVal){
  assert( p->iTransactionLevel>=1 );
  *piVal = p->dbhdr.cookie;
  return SQLITE4_OK;
}

const char *sqlite4BtPagerFilename(BtPager *p, int ePagerfile){
  const char *zTail;

  switch( ePagerfile ){
    case BT_PAGERFILE_DATABASE:
      zTail = "";
      break;

    case BT_PAGERFILE_LOG:
      zTail = "-wal";
      break;

    default:
      assert( ePagerfile==BT_PAGERFILE_SHM );
      zTail = "-shm";
      break;
  }
  memcpy(&p->zFile[p->nFile], zTail, strlen(zTail)+1);
  return p->zFile;
}

#ifndef NDEBUG
int sqlite4BtPagerRefcount(BtPager *p){
  return p->nTotalRef;
}
#endif