/ Check-in [c3191fc8]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add the "mutexfree-shmlock" patch to this branch.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | begin-concurrent-pnu-wal2
Files: files | file ages | folders
SHA3-256: c3191fc8fcb0775f87f55cb8124e0c36caa54993c2a5fe770d437e4f6ee72d77
User & Date: dan 2018-12-18 19:46:50
Wiki:begin-concurrent-pnu-wal2
Context
2019-01-02
19:46
Merge changes from begin-concurrent-wal2. Also modify the unix-excl shm-locks on this branch so that it is not an error to attempt to release an EXCLUSIVE lock that is not held. check-in: 86685679 user: dan tags: begin-concurrent-pnu-wal2
2018-12-18
19:46
Add the "mutexfree-shmlock" patch to this branch. check-in: c3191fc8 user: dan tags: begin-concurrent-pnu-wal2
18:16
Fix a test script problem in wal2savepoint.test. check-in: e388d029 user: dan tags: begin-concurrent-pnu-wal2
2018-12-10
16:52
Make SQLITE_MFS_NSHARD a compile time setting. check-in: b9a74151 user: dan tags: mutexfree-shmlock
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/os_unix.c.

    42     42   **   *  Locking primitives for the proxy uber-locking-method. (MacOSX only)
    43     43   **   *  Definitions of sqlite3_vfs objects for all locking methods
    44     44   **      plus implementations of sqlite3_os_init() and sqlite3_os_end().
    45     45   */
    46     46   #include "sqliteInt.h"
    47     47   #if SQLITE_OS_UNIX              /* This file is used on unix only */
    48     48   
           49  +/* Turn this feature on in all builds for now */
           50  +#define SQLITE_MUTEXFREE_SHMLOCK 1
           51  +#define SQLITE_MFS_EXCLUSIVE     255
           52  +#ifndef SQLITE_MFS_NSHARD
           53  +# define SQLITE_MFS_NSHARD       8
           54  +#endif
           55  +#if SQLITE_MFS_NSHARD<1
           56  +# error "SQLITE_MFS_NSHARD must be greater than 0"
           57  +#endif
           58  +
    49     59   /*
    50     60   ** There are various methods for file locking used for concurrency
    51     61   ** control:
    52     62   **
    53     63   **   1. POSIX locking (the default),
    54     64   **   2. No locking,
    55     65   **   3. Dot-file locking,
................................................................................
  4257   4267     int nRef;                  /* Number of unixShm objects pointing to this */
  4258   4268     unixShm *pFirst;           /* All unixShm objects pointing to this */
  4259   4269   #ifdef SQLITE_DEBUG
  4260   4270     u8 exclMask;               /* Mask of exclusive locks held */
  4261   4271     u8 sharedMask;             /* Mask of shared locks held */
  4262   4272     u8 nextShmId;              /* Next available unixShm.id value */
  4263   4273   #endif
         4274  +
         4275  +#ifdef SQLITE_MUTEXFREE_SHMLOCK
         4276  +  /* In unix-excl mode, if SQLITE_MUTEXFREE_SHMLOCK is defined, all locks
         4277  +  ** are stored in the following 64-bit value. There are in total 8 
         4278  +  ** shm-locking slots, each of which are assigned 8-bits from the 64-bit
         4279  +  ** value. The least-significant 8 bits correspond to shm-locking slot
         4280  +  ** 0, and so on.
         4281  +  **
         4282  +  ** If the 8-bits corresponding to a shm-locking locking slot are set to
         4283  +  ** 0xFF, then a write-lock is held on the slot. Or, if they are set to
         4284  +  ** a non-zero value smaller than 0xFF, then they represent the total 
         4285  +  ** number of read-locks held on the slot. There is no way to distinguish
         4286  +  ** between a write-lock and 255 read-locks.  */
         4287  +  struct LockingSlot {
         4288  +    u32 nLock;
         4289  +    u64 aPadding[7];
         4290  +  } aMFSlot[3 + SQLITE_MFS_NSHARD*5];
         4291  +#endif
  4264   4292   };
         4293  +
         4294  +/*
         4295  +** Atomic CAS primitive used in multi-process mode. Equivalent to:
         4296  +** 
         4297  +**   int unixCompareAndSwap(u32 *ptr, u32 oldval, u32 newval){
         4298  +**     if( *ptr==oldval ){
         4299  +**       *ptr = newval;
         4300  +**       return 1;
         4301  +**     }
         4302  +**     return 0;
         4303  +**   }
         4304  +*/
         4305  +#define unixCompareAndSwap(ptr,oldval,newval) \
         4306  +    __sync_bool_compare_and_swap(ptr,oldval,newval)
         4307  +
  4265   4308   
  4266   4309   /*
  4267   4310   ** Structure used internally by this VFS to record the state of an
  4268   4311   ** open shared memory connection.
  4269   4312   **
  4270   4313   ** The following fields are initialized when this object is created and
  4271   4314   ** are read-only thereafter:
................................................................................
  4279   4322   struct unixShm {
  4280   4323     unixShmNode *pShmNode;     /* The underlying unixShmNode object */
  4281   4324     unixShm *pNext;            /* Next unixShm with the same unixShmNode */
  4282   4325     u8 hasMutex;               /* True if holding the unixShmNode->pShmMutex */
  4283   4326     u8 id;                     /* Id of this connection within its unixShmNode */
  4284   4327     u16 sharedMask;            /* Mask of shared locks held */
  4285   4328     u16 exclMask;              /* Mask of exclusive locks held */
         4329  +#ifdef SQLITE_MUTEXFREE_SHMLOCK
         4330  +  u8 aMFCurrent[8];          /* Current slot used for each shared lock */
         4331  +#endif
  4286   4332   };
  4287   4333   
  4288   4334   /*
  4289   4335   ** Constants used for locking
  4290   4336   */
  4291   4337   #define UNIX_SHM_BASE   ((22+SQLITE_SHM_NLOCK)*4)         /* first lock byte */
  4292   4338   #define UNIX_SHM_DMS    (UNIX_SHM_BASE+SQLITE_SHM_NLOCK)  /* deadman switch */
................................................................................
  4783   4829       *pp = 0;
  4784   4830     }
  4785   4831     if( pShmNode->isReadonly && rc==SQLITE_OK ) rc = SQLITE_READONLY;
  4786   4832     sqlite3_mutex_leave(pShmNode->pShmMutex);
  4787   4833     return rc;
  4788   4834   }
  4789   4835   
         4836  +#ifdef SQLITE_MUTEXFREE_SHMLOCK
         4837  +static int unixMutexFreeShmlock(
         4838  +  unixFile *pFd,             /* Database file holding the shared memory */
         4839  +  int ofst,                  /* First lock to acquire or release */
         4840  +  int n,                     /* Number of locks to acquire or release */
         4841  +  int flags                  /* What to do with the lock */
         4842  +){
         4843  +  struct LockMapEntry {
         4844  +    int iFirst;
         4845  +    int nSlot;
         4846  +  } aMap[9] = {
         4847  +    { 0, 1 },
         4848  +    { 1, 1 },
         4849  +    { 2, 1 },
         4850  +    { 3+0*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD },
         4851  +    { 3+1*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD },
         4852  +    { 3+2*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD },
         4853  +    { 3+3*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD },
         4854  +    { 3+4*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD },
         4855  +    { 3+5*SQLITE_MFS_NSHARD, 0 },
         4856  +  };
         4857  +
         4858  +  unixShm *p = pFd->pShm;               /* The shared memory being locked */
         4859  +  unixShm *pX;                          /* For looping over all siblings */
         4860  +  unixShmNode *pShmNode = p->pShmNode;  /* The underlying file iNode */
         4861  +  int rc = SQLITE_OK;
         4862  +  int iIncr;
         4863  +  u16 mask;                             /* Mask of locks to take or release */
         4864  +
         4865  +  if( flags & SQLITE_SHM_SHARED ){
         4866  +    /* SHARED locks */
         4867  +    u32 iOld, iNew, *ptr;
         4868  +    int iIncr = -1;
         4869  +    if( (flags & SQLITE_SHM_UNLOCK)==0 ){
         4870  +      p->aMFCurrent[ofst] = (p->aMFCurrent[ofst] + 1) % aMap[ofst].nSlot;
         4871  +      iIncr = 1;
         4872  +    }
         4873  +    ptr = &pShmNode->aMFSlot[aMap[ofst].iFirst + p->aMFCurrent[ofst]].nLock;
         4874  +    do {
         4875  +      iOld = *ptr;
         4876  +      iNew = iOld + iIncr;
         4877  +      if( iNew>SQLITE_MFS_EXCLUSIVE ){
         4878  +        return SQLITE_BUSY;
         4879  +      }
         4880  +    }while( 0==unixCompareAndSwap(ptr, iOld, iNew) );
         4881  +  }else{
         4882  +    /* EXCLUSIVE locks */
         4883  +    int iFirst = aMap[ofst].iFirst;
         4884  +    int iLast = aMap[ofst+n].iFirst;
         4885  +    int i;
         4886  +    for(i=iFirst; i<iLast; i++){
         4887  +      u32 *ptr = &pShmNode->aMFSlot[i].nLock;
         4888  +      if( flags & SQLITE_SHM_UNLOCK ){
         4889  +        assert( (*ptr)==SQLITE_MFS_EXCLUSIVE );
         4890  +        *ptr = 0;
         4891  +      }else{
         4892  +        u32 iOld;
         4893  +        do {
         4894  +          iOld = *ptr;
         4895  +          if( iOld>0 ){
         4896  +            while( i>iFirst ){
         4897  +              i--;
         4898  +              pShmNode->aMFSlot[i].nLock = 0;
         4899  +            }
         4900  +            return SQLITE_BUSY;
         4901  +          }
         4902  +        }while( 0==unixCompareAndSwap(ptr, iOld, SQLITE_MFS_EXCLUSIVE) );
         4903  +      }
         4904  +    }
         4905  +  }
         4906  +
         4907  +  return SQLITE_OK;
         4908  +}
         4909  +#else
         4910  +# define unixMutexFreeShmlock(a,b,c,d) SQLITE_OK
         4911  +#endif
         4912  +
  4790   4913   /*
  4791   4914   ** Change the lock state for a shared-memory segment.
  4792   4915   **
  4793   4916   ** Note that the relationship between SHAREd and EXCLUSIVE locks is a little
  4794   4917   ** different here than in posix.  In xShmLock(), one can go from unlocked
  4795   4918   ** to shared and back or from unlocked to exclusive and back.  But one may
  4796   4919   ** not go from shared to exclusive or from exclusive to shared.
................................................................................
  4815   4938     assert( flags==(SQLITE_SHM_LOCK | SQLITE_SHM_SHARED)
  4816   4939          || flags==(SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE)
  4817   4940          || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED)
  4818   4941          || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE) );
  4819   4942     /* assert( n==1 || (flags & SQLITE_SHM_EXCLUSIVE)!=0 ); */
  4820   4943     assert( pShmNode->hShm>=0 || pDbFd->pInode->bProcessLock==1 );
  4821   4944     assert( pShmNode->hShm<0 || pDbFd->pInode->bProcessLock==0 );
         4945  +
         4946  +  if( pDbFd->pInode->bProcessLock ){
         4947  +    return unixMutexFreeShmlock(pDbFd, ofst, n, flags);
         4948  +  }
  4822   4949   
  4823   4950     mask = (1<<(ofst+n)) - (1<<ofst);
  4824   4951     assert( n>1 || mask==(1<<ofst) );
         4952  +  if( flags & SQLITE_SHM_LOCK ){
         4953  +    assert( !(flags&SQLITE_SHM_SHARED) || (p->sharedMask&mask)==0 );
         4954  +    assert( !(flags&SQLITE_SHM_EXCLUSIVE) || !(p->exclMask&mask) );
         4955  +  }else{
         4956  +    assert( !(flags&SQLITE_SHM_SHARED) || (p->sharedMask&mask)==mask );
         4957  +    assert( !(flags&SQLITE_SHM_EXCLUSIVE) || (p->exclMask&mask)==mask );
         4958  +  }
         4959  +
  4825   4960     sqlite3_mutex_enter(pShmNode->pShmMutex);
  4826   4961     if( flags & SQLITE_SHM_UNLOCK ){
  4827   4962       u16 allMask = 0; /* Mask of locks held by siblings */
  4828   4963   
  4829   4964       /* See if any siblings hold this same lock */
  4830   4965       for(pX=pShmNode->pFirst; pX; pX=pX->pNext){
  4831   4966         if( pX==p ) continue;
................................................................................
  4907   5042   ** All loads and stores begun before the barrier must complete before
  4908   5043   ** any load or store begun after the barrier.
  4909   5044   */
  4910   5045   static void unixShmBarrier(
  4911   5046     sqlite3_file *fd                /* Database file holding the shared memory */
  4912   5047   ){
  4913   5048     UNUSED_PARAMETER(fd);
         5049  +#ifdef SQLITE_MUTEXFREE_SHMLOCK
         5050  +  __sync_synchronize();
         5051  +#else
  4914   5052     sqlite3MemoryBarrier();         /* compiler-defined memory barrier */
  4915   5053     assert( fd->pMethods->xLock==nolockLock 
  4916   5054          || unixFileMutexNotheld((unixFile*)fd) 
  4917   5055     );
  4918   5056     unixEnterMutex();               /* Also mutex, for redundancy */
  4919   5057     unixLeaveMutex();
         5058  +#endif
  4920   5059   }
  4921   5060   
  4922   5061   /*
  4923   5062   ** Close a connection to shared-memory.  Delete the underlying 
  4924   5063   ** storage if deleteFlag is true.
  4925   5064   **
  4926   5065   ** If there is no shared memory associated with the connection then this

Changes to src/test_superlock.c.

    37     37   ** An instance of the following structure is allocated for each active
    38     38   ** superlock. The opaque handle returned by sqlite3demo_superlock() is
    39     39   ** actually a pointer to an instance of this structure.
    40     40   */
    41     41   struct Superlock {
    42     42     sqlite3 *db;                    /* Database handle used to lock db */
    43     43     int bWal;                       /* True if db is a WAL database */
           44  +  int bRecoveryLocked;            /* True if WAL RECOVERY lock is held */
           45  +  int bReaderLocked;              /* True if WAL READER locks are held */
    44     46   };
    45     47   typedef struct Superlock Superlock;
    46     48   
    47     49   /*
    48     50   ** The pCtx pointer passed to this function is actually a pointer to a
    49     51   ** SuperlockBusy structure. Invoke the busy-handler function encapsulated
    50     52   ** by the structure and return the result.
................................................................................
   103    105   }
   104    106   
   105    107   /*
   106    108   ** Obtain the extra locks on the database file required for WAL databases.
   107    109   ** Invoke the supplied busy-handler as required.
   108    110   */
   109    111   static int superlockWalLock(
   110         -  sqlite3 *db,                    /* Database handle open on WAL database */
          112  +  Superlock *pLock,               /* Superlock handle */
   111    113     SuperlockBusy *pBusy            /* Busy handler wrapper object */
   112    114   ){
   113    115     int rc;                         /* Return code */
   114    116     sqlite3_file *fd = 0;           /* Main database file handle */
   115    117     void volatile *p = 0;           /* Pointer to first page of shared memory */
          118  +  sqlite3 *db = pLock->db;
   116    119   
   117    120     /* Obtain a pointer to the sqlite3_file object open on the main db file. */
   118    121     rc = sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, (void *)&fd);
   119    122     if( rc!=SQLITE_OK ) return rc;
   120    123   
   121    124     /* Obtain the "recovery" lock. Normally, this lock is only obtained by
   122    125     ** clients running database recovery.  
   123    126     */
          127  +  assert( pLock->bRecoveryLocked==0 );
   124    128     rc = superlockShmLock(fd, 2, 1, pBusy);
   125    129     if( rc!=SQLITE_OK ) return rc;
          130  +  pLock->bRecoveryLocked = 1;
   126    131   
   127    132     /* Zero the start of the first shared-memory page. This means that any
   128    133     ** clients that open read or write transactions from this point on will
   129    134     ** have to run recovery before proceeding. Since they need the "recovery"
   130    135     ** lock that this process is holding to do that, no new read or write
   131    136     ** transactions may now be opened. Nor can a checkpoint be run, for the
   132    137     ** same reason.
................................................................................
   135    140     if( rc!=SQLITE_OK ) return rc;
   136    141     memset((void *)p, 0, 32);
   137    142   
   138    143     /* Obtain exclusive locks on all the "read-lock" slots. Once these locks
   139    144     ** are held, it is guaranteed that there are no active reader, writer or 
   140    145     ** checkpointer clients.
   141    146     */
          147  +  assert( pLock->bReaderLocked==0 );
   142    148     rc = superlockShmLock(fd, 3, SQLITE_SHM_NLOCK-3, pBusy);
          149  +  if( rc==SQLITE_OK ) pLock->bReaderLocked = 1;
   143    150     return rc;
   144    151   }
   145    152   
   146    153   /*
   147    154   ** Release a superlock held on a database file. The argument passed to 
   148    155   ** this function must have been obtained from a successful call to
   149    156   ** sqlite3demo_superlock().
................................................................................
   152    159     Superlock *p = (Superlock *)pLock;
   153    160     if( p->bWal ){
   154    161       int rc;                         /* Return code */
   155    162       int flags = SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE;
   156    163       sqlite3_file *fd = 0;
   157    164       rc = sqlite3_file_control(p->db, "main", SQLITE_FCNTL_FILE_POINTER, (void *)&fd);
   158    165       if( rc==SQLITE_OK ){
   159         -      fd->pMethods->xShmLock(fd, 2, 1, flags);
   160         -      fd->pMethods->xShmLock(fd, 3, SQLITE_SHM_NLOCK-3, flags);
          166  +      if( p->bRecoveryLocked ){
          167  +        fd->pMethods->xShmLock(fd, 2, 1, flags);
          168  +        p->bRecoveryLocked = 0;
          169  +      }
          170  +      if( p->bReaderLocked ){
          171  +        fd->pMethods->xShmLock(fd, 3, SQLITE_SHM_NLOCK-3, flags);
          172  +        p->bReaderLocked = 0;
          173  +      }
   161    174       }
   162    175     }
   163    176     sqlite3_close(p->db);
   164    177     sqlite3_free(p);
   165    178   }
   166    179   
   167    180   /*
................................................................................
   228    241     ** to drop the WAL read and write locks currently held. Otherwise, the
   229    242     ** new WAL locks may conflict with the old.
   230    243     */
   231    244     if( rc==SQLITE_OK ){
   232    245       if( SQLITE_OK==(rc = superlockIsWal(pLock)) && pLock->bWal ){
   233    246         rc = sqlite3_exec(pLock->db, "COMMIT", 0, 0, 0);
   234    247         if( rc==SQLITE_OK ){
   235         -        rc = superlockWalLock(pLock->db, &busy);
          248  +        rc = superlockWalLock(pLock, &busy);
   236    249         }
   237    250       }
   238    251     }
   239    252   
   240    253     if( rc!=SQLITE_OK ){
   241    254       sqlite3demo_superunlock(pLock);
   242    255       *ppLock = 0;

Changes to src/test_vfs.c.

  1558   1558   
  1559   1559     return TCL_OK;
  1560   1560   
  1561   1561    bad_args:
  1562   1562     Tcl_WrongNumArgs(interp, 1, objv, "VFSNAME ?-noshm BOOL? ?-fullshm BOOL? ?-default BOOL? ?-mxpathname INT? ?-szosfile INT? ?-iversion INT?");
  1563   1563     return TCL_ERROR;
  1564   1564   }
         1565  +
         1566  +extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb);
         1567  +extern const char *sqlite3ErrName(int);
         1568  +
         1569  +/*
         1570  +** tclcmd: vfs_shmlock DB DBNAME (shared|exclusive) (lock|unlock) OFFSET N
         1571  +*/
         1572  +static int SQLITE_TCLAPI test_vfs_shmlock(
         1573  +  void * clientData,
         1574  +  Tcl_Interp *interp,
         1575  +  int objc,
         1576  +  Tcl_Obj *CONST objv[]
         1577  +){
         1578  +  const char *azArg1[] = {"shared", "exclusive", 0};
         1579  +  const char *azArg2[] = {"lock", "unlock", 0};
         1580  +  sqlite3 *db = 0;
         1581  +  int rc = SQLITE_OK;
         1582  +  const char *zDbname = 0;
         1583  +  int iArg1 = 0;
         1584  +  int iArg2 = 0;
         1585  +  int iOffset = 0;
         1586  +  int n = 0;
         1587  +  sqlite3_file *pFd;
         1588  +
         1589  +  if( objc!=7 ){
         1590  +    Tcl_WrongNumArgs(interp, 1, objv, 
         1591  +        "DB DBNAME (shared|exclusive) (lock|unlock) OFFSET N"
         1592  +    );
         1593  +    return TCL_ERROR;
         1594  +  }
         1595  +
         1596  +  zDbname = Tcl_GetString(objv[2]);
         1597  +  if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) 
         1598  +   || Tcl_GetIndexFromObj(interp, objv[3], azArg1, "ARG", 0, &iArg1) 
         1599  +   || Tcl_GetIndexFromObj(interp, objv[4], azArg2, "ARG", 0, &iArg2) 
         1600  +   || Tcl_GetIntFromObj(interp, objv[5], &iOffset)
         1601  +   || Tcl_GetIntFromObj(interp, objv[6], &n)
         1602  +  ){
         1603  +    return TCL_ERROR;
         1604  +  }
         1605  +
         1606  +  sqlite3_file_control(db, zDbname, SQLITE_FCNTL_FILE_POINTER, (void*)&pFd);
         1607  +  if( pFd==0 ){
         1608  +    return TCL_ERROR;
         1609  +  }
         1610  +  rc = pFd->pMethods->xShmLock(pFd, iOffset, n, 
         1611  +      (iArg1==0 ? SQLITE_SHM_SHARED : SQLITE_SHM_EXCLUSIVE)
         1612  +    | (iArg2==0 ? SQLITE_SHM_LOCK : SQLITE_SHM_UNLOCK)
         1613  +  );
         1614  +  Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
         1615  +  return TCL_OK;
         1616  +}
         1617  +
  1565   1618   
  1566   1619   int Sqlitetestvfs_Init(Tcl_Interp *interp){
  1567   1620     Tcl_CreateObjCommand(interp, "testvfs", testvfs_cmd, 0, 0);
         1621  +  Tcl_CreateObjCommand(interp, "vfs_shmlock", test_vfs_shmlock, 0, 0);
  1568   1622     return TCL_OK;
  1569   1623   }
  1570   1624   
  1571   1625   #endif

Changes to src/wal.c.

  3408   3408         **         checkpoint need not have completed for this to cause problems.
  3409   3409         */
  3410   3410         volatile WalCkptInfo *pInfo = walCkptInfo(pWal);
  3411   3411   
  3412   3412         assert( pWal->readLock>0 || pWal->hdr.mxFrame==0 );
  3413   3413         assert( pInfo->aReadMark[pWal->readLock]<=pSnapshot->mxFrame );
  3414   3414   
  3415         -      /* It is possible that there is a checkpointer thread running 
  3416         -      ** concurrent with this code. If this is the case, it may be that the
  3417         -      ** checkpointer has already determined that it will checkpoint 
         3415  +      /* If it were possible for a checkpointer thread to run concurrent 
         3416  +      ** with this code, it would be a problem. In this case, it could be
         3417  +      ** that the checkpointer has already determined that it will checkpoint 
  3418   3418         ** snapshot X, where X is later in the wal file than pSnapshot, but 
  3419   3419         ** has not yet set the pInfo->nBackfillAttempted variable to indicate 
  3420         -      ** its intent. To avoid the race condition this leads to, ensure that
  3421         -      ** there is no checkpointer process by taking a shared CKPT lock 
  3422         -      ** before checking pInfo->nBackfillAttempted.  
  3423         -      **
  3424         -      ** TODO: Does the aReadMark[] lock prevent a checkpointer from doing
  3425         -      **       this already?
  3426         -      */
  3427         -      rc = walLockShared(pWal, WAL_CKPT_LOCK);
         3420  +      ** its intent. Fortunately this is not possible, as the call to
         3421  +      ** sqlite3WalSnapshotOpen() that sets pWal->pSnapshot also takes a
         3422  +      ** SHARED lock on the checkpointer slot.  */
  3428   3423   
  3429   3424         if( rc==SQLITE_OK ){
  3430   3425           /* Check that the wal file has not been wrapped. Assuming that it has
  3431   3426           ** not, also check that no checkpointer has attempted to checkpoint any
  3432   3427           ** frames beyond pSnapshot->mxFrame. If either of these conditions are
  3433   3428           ** true, return SQLITE_ERROR_SNAPSHOT. Otherwise, overwrite pWal->hdr
  3434   3429           ** with *pSnapshot and set *pChanged as appropriate for opening the
................................................................................
  3439   3434             assert( pWal->readLock>0 );
  3440   3435             memcpy(&pWal->hdr, pSnapshot, sizeof(WalIndexHdr));
  3441   3436             *pChanged = bChanged;
  3442   3437           }else{
  3443   3438             rc = SQLITE_ERROR_SNAPSHOT;
  3444   3439           }
  3445   3440   
  3446         -        /* Release the shared CKPT lock obtained above. */
  3447         -        walUnlockShared(pWal, WAL_CKPT_LOCK);
  3448   3441           pWal->minFrame = 1;
  3449   3442         }
  3450   3443   
  3451   3444   
  3452   3445         if( rc!=SQLITE_OK ){
  3453   3446           sqlite3WalEndReadTransaction(pWal);
  3454   3447         }

Changes to test/lock_common.tcl.

    11     11   # This file contains code used by several different test scripts. The
    12     12   # code in this file allows testfixture to control another process (or
    13     13   # processes) to test locking.
    14     14   #
    15     15   
    16     16   proc do_multiclient_test {varname script} {
    17     17   
    18         -  foreach code [list {
           18  +  foreach {tn code} [list 1 {
    19     19       if {[info exists ::G(valgrind)]} { db close ; continue }
    20     20       set ::code2_chan [launch_testfixture]
    21     21       set ::code3_chan [launch_testfixture]
    22     22       proc code2 {tcl} { testfixture $::code2_chan $tcl }
    23     23       proc code3 {tcl} { testfixture $::code3_chan $tcl }
    24         -    set tn 1
    25         -  } {
           24  +  } 2 {
    26     25       proc code2 {tcl} { uplevel #0 $tcl }
    27     26       proc code3 {tcl} { uplevel #0 $tcl }
    28         -    set tn 2
    29     27     }] {
           28  +    # Do not run multi-process tests with the unix-excl VFS.
           29  +    #
           30  +    if {$tn==1 && [permutation]=="unix-excl"} continue
           31  +
    30     32       faultsim_delete_and_reopen
    31     33   
    32     34       proc code1 {tcl} { uplevel #0 $tcl }
    33     35     
    34     36       # Open connections [db2] and [db3]. Depending on which iteration this
    35     37       # is, the connections may be created in this interpreter, or in 
    36     38       # interpreters running in other OS processes. As such, the [db2] and [db3]

Changes to test/permutations.test.

   612    612     Run some tests using the "test_onefile.c" demo
   613    613   } -initialize {
   614    614     set ::G(perm:sqlite3_args) [list -vfs fs]
   615    615   } -files {
   616    616     conflict.test  insert.test   insert2.test  insert3.test
   617    617     rollback.test  select1.test  select2.test  select3.test
   618    618   }
          619  +
          620  +# Run some tests using the "unix-excl" VFS.
          621  +#
          622  +test_suite "unix-excl" -description {
          623  +  Run some tests using the "unix-excl" VFS
          624  +} -initialize {
          625  +  set ::G(perm:sqlite3_args) [list -vfs unix-excl]
          626  +} -files {
          627  +  shmlock.test
          628  +}
   619    629   
   620    630   # Run some tests using UTF-16 databases.
   621    631   #
   622    632   test_suite "utf16" -description {
   623    633     Run tests using UTF-16 databases
   624    634   } -presql {
   625    635     pragma encoding = 'UTF-16'

Added test/shmlock.test.

            1  +# 2018 December 6
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +
           13  +set testdir [file dirname $argv0]
           14  +source $testdir/tester.tcl
           15  +
           16  +set testprefix shmlock
           17  +
           18  +ifcapable !wal {finish_test ; return }
           19  +
           20  +sqlite3 db2 test.db
           21  +sqlite3 db3 test.db
           22  +
           23  +do_execsql_test 1.0 {
           24  +  PRAGMA journal_mode = wal;
           25  +  CREATE TABLE t1(a, b);
           26  +  INSERT INTO t1 VALUES(1, 2);
           27  +} {wal}
           28  +do_test 1.1 { execsql { SELECT * FROM t1 } db2 } {1 2}
           29  +do_test 1.2 { execsql { SELECT * FROM t1 } db3 } {1 2}
           30  +
           31  +foreach {tn dbhandle cmd res} {
           32  +  1    db  {shared    lock   7 1}    OK
           33  +  2    db2 {exclusive lock   7 1}    BUSY
           34  +  3    db  {shared    unlock 7 1}    OK
           35  +  4    db2 {exclusive lock   7 1}    OK
           36  +  5    db  {shared    lock   7 1}    BUSY
           37  +  6    db  {exclusive lock   7 1}    BUSY
           38  +  7    db2 {exclusive unlock 7 1}    OK
           39  +
           40  +  8    db  {exclusive lock   0 8}    OK
           41  +  9    db  {exclusive unlock 0 8}    OK
           42  +  10   db2 {exclusive lock   0 8}    OK
           43  +  11   db2 {exclusive unlock 0 8}    OK
           44  +
           45  +  12   db  {shared    lock   0 1}    OK
           46  +  13   db2 {shared    lock   0 1}    OK
           47  +  14   db3 {shared    lock   0 1}    OK
           48  +  15   db3 {shared    unlock 0 1}    OK
           49  +  16   db3 {exclusive lock   0 1}    BUSY
           50  +  17   db2 {shared    unlock 0 1}    OK
           51  +  18   db3 {exclusive lock   0 1}    BUSY
           52  +  19   db  {shared    unlock 0 1}    OK
           53  +  20   db3 {exclusive lock   0 1}    OK
           54  +  21   db3 {exclusive unlock 0 1}    OK
           55  +
           56  +  22   db  {shared    lock   3 1}    OK
           57  +  23   db2 {exclusive lock   2 2}    BUSY
           58  +  24   db  {shared    lock   2 1}    OK
           59  +  25   db2 {exclusive lock   0 5}    BUSY
           60  +  26   db2 {exclusive lock   0 4}    BUSY
           61  +  27   db2 {exclusive lock   0 3}    BUSY
           62  +  28   db  {shared    unlock 3 1}    OK
           63  +  29   db2 {exclusive lock   2 2}    BUSY
           64  +  28   db  {shared    unlock 2 1}    OK
           65  +  29   db2 {exclusive lock   2 2}    OK
           66  +  29   db2 {exclusive unlock 2 2}    OK
           67  +} {
           68  +  do_test 1.3.$tn [list vfs_shmlock $dbhandle main {*}$cmd] "SQLITE_$res"
           69  +}
           70  +
           71  +db  close
           72  +db2 close
           73  +db3 close
           74  +
           75  +if {[permutation]=="unix-excl"} {
           76  +  do_test 2.0 {
           77  +    for {set i 0} {$i < 256} {incr i} { 
           78  +      sqlite3 db$i test.db 
           79  +      execsql { SELECT * FROM t1 } db$i
           80  +    }
           81  +    for {set i 0} {$i < 255} {incr i} { 
           82  +      set rc [vfs_shmlock db$i main shared lock 4 1]
           83  +      if {$rc != "SQLITE_OK"} { error $rc }
           84  +    }
           85  +
           86  +    vfs_shmlock db255 main shared lock 4 1
           87  +  } {SQLITE_BUSY}
           88  +
           89  +  do_test 2.1 { vfs_shmlock db255 main exclusive lock   4 1 } SQLITE_BUSY
           90  +  do_test 2.2 { vfs_shmlock db0   main shared    unlock 4 1 } SQLITE_OK
           91  +  do_test 2.3 { vfs_shmlock db255 main shared    lock   4 1 } SQLITE_OK
           92  +  do_test 2.4 { vfs_shmlock db255 main shared    unlock 4 1 } SQLITE_OK
           93  +  do_test 2.5 { vfs_shmlock db255 main exclusive lock   4 1 } SQLITE_BUSY
           94  +
           95  +  do_test 2.6 {
           96  +    for {set i 1} {$i < 255} {incr i} { 
           97  +      set rc [vfs_shmlock db255 main exclusive lock 4 1]
           98  +      if {$rc != "SQLITE_BUSY"} { error $rc }
           99  +      set rc [vfs_shmlock db$i main shared unlock 4 1]
          100  +      if {$rc != "SQLITE_OK"} { error $rc }
          101  +    }
          102  +
          103  +    vfs_shmlock db255 main exclusive lock 4 1
          104  +  } {SQLITE_OK}
          105  +
          106  +  vfs_shmlock db255 main exclusive unlock 4 1
          107  +
          108  +  for {set i 0} {$i < 256} {incr i} {
          109  +    db$i close
          110  +  }
          111  +}
          112  +
          113  +sqlite3 db0 test.db
          114  +sqlite3 db1 test.db
          115  +do_test 3.1 { execsql { SELECT * FROM t1 } db0 } {1 2}
          116  +do_test 3.2 { execsql { SELECT * FROM t1 } db1 } {1 2}
          117  +
          118  +set L(0) {n n n n n n n n}
          119  +set L(1) {n n n n n n n n}
          120  +proc random_lock_test {idx} {
          121  +  global L
          122  +  set iSlot [expr int(rand()*8)]
          123  +  if {[expr int(rand()*2)]} {
          124  +    # Unlock operation
          125  +    if {[lindex $L($idx) $iSlot]!="n"} {
          126  +      vfs_shmlock db$idx main [lindex $L($idx) $iSlot] unlock $iSlot 1
          127  +      lset L($idx) $iSlot n
          128  +    }
          129  +  } else {
          130  +    # Lock operation
          131  +    if {[lindex $L($idx) $iSlot]=="n"} {
          132  +      set locktype [lindex {e s} [expr int(rand()*2)]]
          133  +      set n 1
          134  +      if {$locktype=="e"} {
          135  +        for {set l $iSlot} {$l<8 && [lindex $L($idx) $l]=="n"} {incr l} {}
          136  +        set n [expr int(rand()*($l-$iSlot))+1]
          137  +        # puts "iSlot=$iSlot l=$l L=$L($idx)"
          138  +        # puts "$iSlot $n"
          139  +      }
          140  +      set res [vfs_shmlock db$idx main $locktype lock $iSlot $n]
          141  +
          142  +      set bBusy 0
          143  +      for {set i $iSlot} {$i<($iSlot+$n)} {incr i} {
          144  +        set other [lindex $L([expr ($idx+1)%2]) $i]
          145  +        if {($other!="n" && $locktype=="e")||($other=="e" && $locktype=="s")} {
          146  +          if {$res != "SQLITE_BUSY"} { error "BUSY not detected" }
          147  +          set bBusy 1
          148  +          break
          149  +        } 
          150  +      }
          151  +
          152  +      if {$bBusy==0} {
          153  +        if {$res != "SQLITE_OK"} { error "BUSY false-positive" }
          154  +        for {set i $iSlot} {$i<($iSlot+$n)} {incr i} {
          155  +          lset L($idx) $i $locktype
          156  +        }
          157  +      }
          158  +    }
          159  +  }
          160  +}
          161  +
          162  +set nStep 100000
          163  +for {set i 0} {$i < $nStep} {incr i} {
          164  +  random_lock_test 0
          165  +  random_lock_test 1
          166  +}
          167  +
          168  +db0 close
          169  +db1 close
          170  +
          171  +finish_test
          172  +
          173  +

Changes to test/wal.test.

  1293   1293   #
  1294   1294   #   3. Using connection 1, checkpoint the database. Make sure all
  1295   1295   #      the data is present and the database is not corrupt.
  1296   1296   #
  1297   1297   # At one point, SQLite was failing to grow the mapping of the wal-index
  1298   1298   # file in step 3 and the checkpoint was corrupting the database file.
  1299   1299   #
  1300         -do_test wal-20.1 {
  1301         -  catch {db close}
  1302         -  forcedelete test.db test.db-wal test.db-journal
  1303         -  sqlite3 db test.db
  1304         -  execsql {
  1305         -    PRAGMA journal_mode = WAL;
  1306         -    CREATE TABLE t1(x);
  1307         -    INSERT INTO t1 VALUES(randomblob(900));
  1308         -    SELECT count(*) FROM t1;
  1309         -  }
  1310         -} {wal 1}
  1311         -do_test wal-20.2 {
  1312         -  set ::buddy [launch_testfixture]
  1313         -  testfixture $::buddy {
         1300  +if {[permutation]!="unix-excl"} {
         1301  +  do_test wal-20.1 {
         1302  +    catch {db close}
         1303  +    forcedelete test.db test.db-wal test.db-journal
         1304  +    sqlite3 db test.db
         1305  +    execsql {
         1306  +      PRAGMA journal_mode = WAL;
         1307  +      CREATE TABLE t1(x);
         1308  +      INSERT INTO t1 VALUES(randomblob(900));
         1309  +      SELECT count(*) FROM t1;
         1310  +    }
         1311  +  } {wal 1}
         1312  +  do_test wal-20.2 {
         1313  +    set ::buddy [launch_testfixture]
         1314  +    testfixture $::buddy {
         1315  +      sqlite3 db test.db
         1316  +      db transaction { db eval {
         1317  +        PRAGMA wal_autocheckpoint = 0;
         1318  +        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 2 */
         1319  +        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 4 */
         1320  +        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 8 */
         1321  +        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 16 */
         1322  +        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 32 */
         1323  +        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 64 */
         1324  +        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 128 */
         1325  +        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 256 */
         1326  +        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 512 */
         1327  +        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 1024 */
         1328  +        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 2048 */
         1329  +        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 4096 */
         1330  +        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 8192 */
         1331  +        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 16384 */
         1332  +      } }
         1333  +    }
         1334  +  } {0}
         1335  +  do_test wal-20.3 {
         1336  +    close $::buddy
         1337  +    execsql { PRAGMA wal_checkpoint }
         1338  +    execsql { SELECT count(*) FROM t1 }
         1339  +  } {16384}
         1340  +  do_test wal-20.4 {
         1341  +    db close
  1314   1342       sqlite3 db test.db
  1315         -    db transaction { db eval {
  1316         -      PRAGMA wal_autocheckpoint = 0;
  1317         -      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 2 */
  1318         -      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 4 */
  1319         -      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 8 */
  1320         -      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 16 */
  1321         -      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 32 */
  1322         -      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 64 */
  1323         -      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 128 */
  1324         -      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 256 */
  1325         -      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 512 */
  1326         -      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 1024 */
  1327         -      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 2048 */
  1328         -      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 4096 */
  1329         -      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 8192 */
  1330         -      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 16384 */
  1331         -    } }
  1332         -  }
  1333         -} {0}
  1334         -do_test wal-20.3 {
  1335         -  close $::buddy
  1336         -  execsql { PRAGMA wal_checkpoint }
  1337         -  execsql { SELECT count(*) FROM t1 }
  1338         -} {16384}
  1339         -do_test wal-20.4 {
  1340         -  db close
  1341         -  sqlite3 db test.db
  1342         -  execsql { SELECT count(*) FROM t1 }
  1343         -} {16384}
  1344         -integrity_check wal-20.5
         1343  +    execsql { SELECT count(*) FROM t1 }
         1344  +  } {16384}
         1345  +  integrity_check wal-20.5
         1346  +}
  1345   1347   
  1346   1348   catch { db2 close }
  1347   1349   catch { db close }
  1348   1350   
  1349   1351   do_test wal-21.1 {
  1350   1352     faultsim_delete_and_reopen
  1351   1353     execsql {