SQLite

Check-in [13e7a82422]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add the new xShmMap (formerly xShmPage) to os_win.c.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | experimental
Files: files | file ages | folders
SHA1: 13e7a8242206bca4b5bf356ef074e66474d39609
User & Date: dan 2010-06-14 16:16:34.000
Context
2010-06-15
13:56
Changes to the way tcl test scripts work. No changes to production code. (Closed-Leaf check-in: 2c5e48a485 user: dan tags: experimental)
2010-06-14
17:09
Merge the experimental shared-memory mmap-by-chunk changes into the trunk. (check-in: f295e7ed5f user: drh tags: trunk)
16:16
Add the new xShmMap (formerly xShmPage) to os_win.c. (check-in: 13e7a82422 user: dan tags: experimental)
14:07
Remove xShmGet/Size/Release from the sqlite3_vfs structure. Change the name of xShmPage to xShmMap. Remove some code that is now unused from os_unix.c and some of the test VFS implementations. (check-in: fc0cabc15c user: dan tags: experimental)
Changes
Side-by-Side Diff Ignore Whitespace Patch
Changes to src/os_unix.c.
3569
3570
3571
3572
3573
3574
3575
3576

3577
3578
3579
3580
3581
3582
3583
3569
3570
3571
3572
3573
3574
3575

3576
3577
3578
3579
3580
3581
3582
3583







-
+







** been allocated, it is allocated by this function.
**
** If the shared-memory region has already been allocated or is allocated by
** this call as described above, then it is mapped into this processes 
** address space (if it is not already), *pp is set to point to the mapped 
** memory and SQLITE_OK returned.
*/
static int unixShmPage(
static int unixShmMap(
  sqlite3_file *fd,               /* Handle open on database file */
  int iRegion,                    /* Region to retrieve */
  int szRegion,                   /* Size of regions */
  int isWrite,                    /* True to extend file if necessary */
  void volatile **pp              /* OUT: Mapped memory */
){
  unixFile *pDbFd = (unixFile*)fd;
3651
3652
3653
3654
3655
3656
3657
3658

3659
3660
3661
3662
3663
3664
3665
3651
3652
3653
3654
3655
3656
3657

3658
3659
3660
3661
3662
3663
3664
3665







-
+







}

#else
# define unixShmOpen    0
# define unixShmLock    0
# define unixShmBarrier 0
# define unixShmClose   0
# define unixShmPage    0
# define unixShmMap     0
#endif /* #ifndef SQLITE_OMIT_WAL */

/*
** Here ends the implementation of all sqlite3_file methods.
**
********************** End sqlite3_file Methods *******************************
******************************************************************************/
3713
3714
3715
3716
3717
3718
3719
3720

3721
3722
3723
3724
3725
3726
3727
3713
3714
3715
3716
3717
3718
3719

3720
3721
3722
3723
3724
3725
3726
3727







-
+







   unixFileControl,            /* xFileControl */                            \
   unixSectorSize,             /* xSectorSize */                             \
   unixDeviceCharacteristics,  /* xDeviceCapabilities */                     \
   unixShmOpen,                /* xShmOpen */                                \
   unixShmLock,                /* xShmLock */                                \
   unixShmBarrier,             /* xShmBarrier */                             \
   unixShmClose,               /* xShmClose */                               \
   unixShmPage                 /* xShmPage */                                \
   unixShmMap                  /* xShmMap */                                 \
};                                                                           \
static const sqlite3_io_methods *FINDER##Impl(const char *z, unixFile *p){   \
  UNUSED_PARAMETER(z); UNUSED_PARAMETER(p);                                  \
  return &METHOD;                                                            \
}                                                                            \
static const sqlite3_io_methods *(*const FINDER)(const char*,unixFile *p)    \
    = FINDER##Impl;
Changes to src/os_win.c.
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221




1222



1223
1224

1225
1226
1227
1228
1229
1230
1231
1232
1212
1213
1214
1215
1216
1217
1218

1219
1220
1221
1222
1223
1224

1225
1226
1227
1228

1229

1230
1231
1232
1233
1234
1235
1236







-


+
+
+
+
-
+
+
+

-
+
-







** reverse order that they are acquired.  mutexBuf is always acquired
** first and released last.  This invariant is check by asserting
** sqlite3_mutex_notheld() on mutex whenever mutexBuf is acquired or
** released.
*/
struct winShmNode {
  sqlite3_mutex *mutex;      /* Mutex to access this object */
  sqlite3_mutex *mutexBuf;   /* Mutex to access zBuf[] */
  char *zFilename;           /* Name of the file */
  winFile hFile;             /* File handle from winOpen */

  int szRegion;              /* Size of shared-memory regions */
  int nRegion;               /* Size of array apRegion */
  struct ShmRegion {
  HANDLE hMap;               /* File handle from CreateFileMapping */
    HANDLE hMap;             /* File handle from CreateFileMapping */
    void *pMap;
  } *aRegion;
  DWORD lastErrno;           /* The Windows errno from the last I/O error */
  int szMap;                 /* Size of the mapping of file into memory */

  char *pMMapBuf;            /* Where currently mmapped().  NULL if unmapped */
  int nRef;                  /* Number of winShm objects pointing to this */
  winShm *pFirst;            /* All winShm objects pointing to this */
  winShmNode *pNext;         /* Next in list of all winShmNode objects */
#ifdef SQLITE_DEBUG
  u8 nextShmId;              /* Next available winShm.id value */
#endif
};
1321
1322
1323
1324
1325
1326
1327

1328
1329
1330
1331


1332
1333
1334

1335
1336
1337
1338
1339
1340

1341
1342
1343
1344
1345
1346
1347
1325
1326
1327
1328
1329
1330
1331
1332
1333



1334
1335



1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350







+

-
-
-
+
+
-
-
-
+






+







static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){
  winShmNode **pp;
  winShmNode *p;
  assert( winShmMutexHeld() );
  pp = &winShmNodeList;
  while( (p = *pp)!=0 ){
    if( p->nRef==0 ){
      int i;
      if( p->mutex ) sqlite3_mutex_free(p->mutex);
      if( p->mutexBuf ) sqlite3_mutex_free(p->mutexBuf);
      if( p->pMMapBuf ){
        UnmapViewOfFile(p->pMMapBuf);
      for(i=0; i<p->nRegion; i++){
        UnmapViewOfFile(p->aRegion[i].pMap);
      }
      if( INVALID_HANDLE_VALUE != p->hMap ){
        CloseHandle(p->hMap);
        CloseHandle(p->aRegion[i].hMap);
      }
      if( p->hFile.h != INVALID_HANDLE_VALUE ) {
        winClose((sqlite3_file *)&p->hFile);
      }
      if( deleteFlag ) winDelete(pVfs, p->zFilename, 0);
      *pp = p->pNext;
      sqlite3_free(p->aRegion);
      sqlite3_free(p);
    }else{
      pp = &p->pNext;
    }
  }
}

1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1403
1404
1405
1406
1407
1408
1409


1410
1411
1412
1413
1414
1415
1416





1417
1418
1419
1420
1421
1422
1423







-
-







-
-
-
-
-







    if( sqlite3StrICmp(pShmNode->zFilename, pNew->zFilename)==0 ) break;
  }
  if( pShmNode ){
    sqlite3_free(pNew);
  }else{
    pShmNode = pNew;
    pNew = 0;
    pShmNode->pMMapBuf = NULL;
    pShmNode->hMap = INVALID_HANDLE_VALUE;
    ((winFile*)(&pShmNode->hFile))->h = INVALID_HANDLE_VALUE;
    pShmNode->pNext = winShmNodeList;
    winShmNodeList = pShmNode;

    pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST);
    if( pShmNode->mutex==0 ){
      rc = SQLITE_NOMEM;
      goto shm_open_err;
    }
    pShmNode->mutexBuf = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST);
    if( pShmNode->mutexBuf==0 ){
      rc = SQLITE_NOMEM;
      goto shm_open_err;
    }
    rc = winOpen(pDbFd->pVfs,
                 pShmNode->zFilename,             /* Name of the file (UTF-8) */
                 (sqlite3_file*)&pShmNode->hFile,  /* File handle here */
                 SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, /* Mode flags */
                 0);
1503
1504
1505
1506
1507
1508
1509

1510



1511
1512
1513
1514
1515

1516
1517
1518
1519
1520
1521
1522

1523
1524
1525
1526
1527
1528
1529

1530
1531
1532
1533
1534
1535
1536
1537
1538

1539
1540
1541
1542
1543
1544
1545


1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557

1558
1559
1560
1561
1562
1563
1564
1565
1566
1567

1568
1569
1570
1571
1572
1573


1574
1575
1576
1577
1578
1579
1580
1581






1582
1583
1584
1585
1586
1587
1588
1589
1590

1591
1592


1593
1594
1595



1596
1597
1598
1599
1600
1601
1602
1603
1604
1605












1606
1607
1608
1609
1610
1611
1612
1613


















1614
1615
1616
1617
1618
1619
1620












1621
1622
1623





1624
1625
1626
1627
1628
1629
1630
1631
1632









1633
1634
1635
1636
1637
1638
1639


1640
1641

1642
1643
1644
1645
1646
1647









1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1499
1500
1501
1502
1503
1504
1505
1506

1507
1508
1509
1510




1511
1512






1513







1514









1515







1516
1517










1518

1519










1520






1521
1522


1523





1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535



1536


1537
1538



1539
1540
1541










1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553








1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571







1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583



1584
1585
1586
1587
1588









1589
1590
1591
1592
1593
1594
1595
1596
1597







1598
1599


1600






1601
1602
1603
1604
1605
1606
1607
1608
1609
1610


1611
1612
1613
1614






















1615
1616
1617
1618
1619
1620
1621







+
-
+
+
+

-
-
-
-
+

-
-
-
-
-
-
+
-
-
-
-
-
-
-
+
-
-
-
-
-
-
-
-
-
+
-
-
-
-
-
-
-
+
+
-
-
-
-
-
-
-
-
-
-

-
+
-
-
-
-
-
-
-
-
-
-
+
-
-
-
-
-
-
+
+
-
-

-
-
-
-
-
+
+
+
+
+
+






-
-
-
+
-
-
+
+
-
-
-
+
+
+
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
+
+
+
+
+
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
-
-
-
-
-
-
-
+
+
-
-
+
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+

-
-




-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-







  }
  winShmLeaveMutex();

  return SQLITE_OK;
}

/*
** This function is called to obtain a pointer to region iRegion of the 
** Increase the size of the underlying storage for a shared-memory segment.
** shared-memory associated with the database file fd. Shared-memory regions 
** are numbered starting from zero. Each shared-memory region is szRegion 
** bytes in size.
**
** The reqSize parameter is the new requested minimum size of the underlying
** shared memory.  This routine may choose to make the shared memory larger
** than this value (for example to round the shared memory size up to an
** operating-system dependent page size.)
** If an error occurs, an error code is returned and *pp is set to NULL.
**
** This routine will only grow the size of shared memory.  A request for
** a smaller size is a no-op.
*/
static int winShmSize(
  sqlite3_file *fd,         /* Database holding the shared memory */
  int reqSize,              /* Requested size.  -1 for query only */
** Otherwise, if the isWrite parameter is 0 and the requested shared-memory
  int *pNewSize             /* Write new size here */
){
  winFile *pDbFd = (winFile*)fd;
  winShm *p = pDbFd->pShm;
  winShmNode *pShmNode = p->pShmNode;
  int rc = SQLITE_OK;

** region has not been allocated (by any client, including one running in a
  *pNewSize = 0;
  if( reqSize>=0 ){
    sqlite3_int64 sz;
    rc = winFileSize((sqlite3_file *)&pShmNode->hFile, &sz);
    if( SQLITE_OK==rc && reqSize>sz ){
      rc = winTruncate((sqlite3_file *)&pShmNode->hFile, reqSize);
    }
  }
  if( SQLITE_OK==rc ){
** separate process), then *pp is set to NULL and SQLITE_OK returned. If 
    sqlite3_int64 sz;
    rc = winFileSize((sqlite3_file *)&pShmNode->hFile, &sz);
    if( SQLITE_OK==rc ){
      *pNewSize = (int)sz;
    }else{
      rc = SQLITE_IOERR;
    }
** isWrite is non-zero and the requested shared-memory region has not yet 
** been allocated, it is allocated by this function.
  }
  return rc;
}


/*
** Map the shared storage into memory.  The minimum size of the
** mapping should be reqMapSize if reqMapSize is positive.  If
** reqMapSize is zero or negative, the implementation can choose
** whatever mapping size is convenient.
**
** *ppBuf is made to point to the memory which is a mapping of the
** If the shared-memory region has already been allocated or is allocated by
** underlying storage.  A mutex is acquired to prevent other threads
** from running while *ppBuf is in use in order to prevent other threads
** remapping *ppBuf out from under this thread.  The winShmRelease()
** call will release the mutex.  However, if the lock state is CHECKPOINT,
** the mutex is not acquired because CHECKPOINT will never remap the
** buffer.  RECOVER might remap, though, so CHECKPOINT will acquire
** the mutex if and when it promotes to RECOVER.
**
** RECOVER needs to be atomic.  The same mutex that prevents *ppBuf from
** being remapped also prevents more than one thread from being in
** this call as described above, then it is mapped into this processes 
** RECOVER at a time.  But, RECOVER sometimes wants to remap itself.
** To prevent RECOVER from losing its lock while remapping, the
** mutex is not released by winShmRelease() when in RECOVER.
**
** *pNewMapSize is set to the size of the mapping.
**
** address space (if it is not already), *pp is set to point to the mapped 
** memory and SQLITE_OK returned.
** *ppBuf and *pNewMapSize might be NULL and zero if no space has
** yet been allocated to the underlying storage.
*/
static int winShmGet(
  sqlite3_file *fd,        /* The database file holding the shared memory */
  int reqMapSize,          /* Requested size of mapping. -1 means don't care */
  int *pNewMapSize,        /* Write new size of mapping here */
  void volatile **ppBuf    /* Write mapping buffer origin here */
static int winShmMap(
  sqlite3_file *fd,               /* Handle open on database file */
  int iRegion,                    /* Region to retrieve */
  int szRegion,                   /* Size of regions */
  int isWrite,                    /* True to extend file if necessary */
  void volatile **pp              /* OUT: Mapped memory */
){
  winFile *pDbFd = (winFile*)fd;
  winShm *p = pDbFd->pShm;
  winShmNode *pShmNode = p->pShmNode;
  int rc = SQLITE_OK;

  if( p->hasMutexBuf==0 ){
    assert( sqlite3_mutex_notheld(pShmNode->mutex) );
    sqlite3_mutex_enter(pShmNode->mutexBuf);
  sqlite3_mutex_enter(pShmNode->mutex);
    p->hasMutexBuf = 1;
  }
  assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 );

  sqlite3_mutex_enter(pShmNode->mutex);
  if( pShmNode->szMap==0 || reqMapSize>pShmNode->szMap ){
    int actualSize;
  if( pShmNode->nRegion<=iRegion ){
    struct ShmRegion *apNew;           /* New aRegion[] array */
    int nByte = (iRegion+1)*szRegion;  /* Minimum required file size */
    if( winShmSize(fd, -1, &actualSize)==SQLITE_OK
     && reqMapSize<actualSize
    ){
      reqMapSize = actualSize;
    }
    if( pShmNode->pMMapBuf ){
      if( !UnmapViewOfFile(pShmNode->pMMapBuf) ){
        pShmNode->lastErrno = GetLastError();
        rc = SQLITE_IOERR;
      }
    sqlite3_int64 sz;                  /* Current size of wal-index file */

    pShmNode->szRegion = szRegion;

    /* The requested region is not mapped into this processes address space.
    ** Check to see if it has been allocated (i.e. if the wal-index file is
    ** large enough to contain the requested region).
    */
    rc = winFileSize((sqlite3_file *)&pShmNode->hFile, &sz);
    if( rc!=SQLITE_OK ){
      goto shmpage_out;
    }
      CloseHandle(pShmNode->hMap);
      pShmNode->hMap = INVALID_HANDLE_VALUE;
    }
    if( SQLITE_OK == rc ){
      pShmNode->pMMapBuf = 0;
      if( reqMapSize == 0 ){
        /* can't create 0 byte file mapping in Windows */
        pShmNode->szMap = 0;

    if( sz<nByte ){
      /* The requested memory region does not exist. If isWrite is set to
      ** zero, exit early. *pp will be set to NULL and SQLITE_OK returned.
      **
      ** Alternatively, if isWrite is non-zero, use ftruncate() to allocate
      ** the requested memory region.
      */
      if( !isWrite ) goto shmpage_out;
      rc = winTruncate((sqlite3_file *)&pShmNode->hFile, nByte);
      if( rc!=SQLITE_OK ){
        goto shmpage_out;
      }
    }

    /* Map the requested memory region into this processes address space. */
    apNew = (struct ShmRegion *)sqlite3_realloc(
        pShmNode->aRegion, (iRegion+1)*sizeof(apNew[0])
      }else{
        /* create the file mapping object */
        if( INVALID_HANDLE_VALUE == pShmNode->hMap ){
          /* TBD provide an object name to each file
          ** mapping so it can be re-used across processes.
          */
          pShmNode->hMap = CreateFileMapping(pShmNode->hFile.h,
    );
    if( !apNew ){
      rc = SQLITE_IOERR_NOMEM;
      goto shmpage_out;
    }
    pShmNode->aRegion = apNew;

    while( pShmNode->nRegion<=iRegion ){
      HANDLE hMap;                /* file-mapping handle */
      void *pMap = 0;             /* Mapped memory region */
     
      hMap = CreateFileMapping(pShmNode->hFile.h, 
                                          NULL,
                                          PAGE_READWRITE,
                                          0,
          NULL, PAGE_READWRITE, 0, nByte, NULL
      );
      if( hMap ){
        pMap = MapViewOfFile(hMap, FILE_MAP_WRITE | FILE_MAP_READ,
            0, 0, nByte
                                          reqMapSize,
                                          NULL);
        }
        if( NULL==pShmNode->hMap ){
          pShmNode->lastErrno = GetLastError();
          rc = SQLITE_IOERR;
          pShmNode->szMap = 0;
          pShmNode->hMap = INVALID_HANDLE_VALUE;
        }else{
        );
      }
      if( !pMap ){
        pShmNode->lastErrno = GetLastError();
        rc = SQLITE_IOERR;
        if( hMap ) CloseHandle(hMap);
        goto shmpage_out;
      }

          pShmNode->pMMapBuf = MapViewOfFile(pShmNode->hMap,
                                          FILE_MAP_WRITE | FILE_MAP_READ,
                                          0,
                                          0,
                                          reqMapSize);
          if( !pShmNode->pMMapBuf ){
            pShmNode->lastErrno = GetLastError();
      pShmNode->aRegion[pShmNode->nRegion].pMap = pMap;
      pShmNode->aRegion[pShmNode->nRegion].hMap = hMap;
            rc = SQLITE_IOERR;
            pShmNode->szMap = 0;
      pShmNode->nRegion++;
          }else{
            pShmNode->szMap = reqMapSize;
          }
        }
      }
    }
    }
  }

shmpage_out:
  if( pShmNode->nRegion>iRegion ){
    char *p = (char *)pShmNode->aRegion[iRegion].pMap;
    *pp = (void *)&p[iRegion*szRegion];
  }else{
    *pp = 0;
  }
  *pNewMapSize = pShmNode->szMap;
  *ppBuf = pShmNode->pMMapBuf;
  sqlite3_mutex_leave(pShmNode->mutex);
  return rc;
}

/*
** Release the lock held on the shared memory segment so that other
** threads are free to resize it if necessary.
**
** If the lock is not currently held, this routine is a harmless no-op.
**
** If the shared-memory object is in lock state RECOVER, then we do not
** really want to release the lock, so in that case too, this routine
** is a no-op.
*/
static int winShmRelease(sqlite3_file *fd){
  winFile *pDbFd = (winFile*)fd;
  winShm *p = pDbFd->pShm;
  if( p->hasMutexBuf ){
    winShmNode *pShmNode = p->pShmNode;
    assert( sqlite3_mutex_notheld(pShmNode->mutex) );
    sqlite3_mutex_leave(pShmNode->mutexBuf);
    p->hasMutexBuf = 0;
  }
  return SQLITE_OK;
}

/*
** Change the lock state for a shared-memory segment.
*/
static int winShmLock(
  sqlite3_file *fd,          /* Database file holding the shared memory */
  int ofst,                  /* First lock to acquire or release */
  int n,                     /* Number of locks to acquire or release */
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764


1765
1766
1767
1768
1769
1770
1771
1690
1691
1692
1693
1694
1695
1696



1697
1698

1699
1700
1701
1702
1703
1704
1705
1706
1707







-
-
-


-
+
+







  winLock,
  winUnlock,
  winCheckReservedLock,
  winFileControl,
  winSectorSize,
  winDeviceCharacteristics,
  winShmOpen,              /* xShmOpen */
  winShmSize,              /* xShmSize */
  winShmGet,               /* xShmGet */
  winShmRelease,           /* xShmRelease */
  winShmLock,              /* xShmLock */
  winShmBarrier,           /* xShmBarrier */
  winShmClose              /* xShmClose */
  winShmClose,             /* xShmClose */
  winShmMap                /* xShmMap */
};

/***************************************************************************
** Here ends the I/O methods that form the sqlite3_io_methods object.
**
** The next block of code implements the VFS methods.
****************************************************************************/