/ Check-in [d81708f7]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Remove the BtreeMutexArray object - use the Vdbe.btreeMask field to accomplish the same result. Add a generation counter to btree mutexes in order to assert that mutexes are never temporarily dropped over a range of instructions in order to do deadlock avoidance in some subroutine. Lock all btrees in any Vdbe program that uses OP_ParseSchema.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: d81708f7d1eee399bfe76f6b8dac950a85dc2582
User & Date: drh 2011-04-04 00:14:43
Context
2011-04-04
03:27
Suppress unused parameter warnings in sqlite3VdbeEnter() and related routines. check-in: f8e98ab3 user: drh tags: trunk
00:14
Remove the BtreeMutexArray object - use the Vdbe.btreeMask field to accomplish the same result. Add a generation counter to btree mutexes in order to assert that mutexes are never temporarily dropped over a range of instructions in order to do deadlock avoidance in some subroutine. Lock all btrees in any Vdbe program that uses OP_ParseSchema. check-in: d81708f7 user: drh tags: trunk
2011-04-03
18:19
Make sure that the constant 1 is cast to yDbType before shifting to create an attached database mask. This check-in is a follow-up and fix to the [7aaf8772274422] change that increases the maximum number of attached databases from 30 to 62. check-in: e2a09ea7 user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/btmutex.c.

    35     35   }
    36     36   
    37     37   /*
    38     38   ** Release the BtShared mutex associated with B-Tree handle p and
    39     39   ** clear the p->locked boolean.
    40     40   */
    41     41   static void unlockBtreeMutex(Btree *p){
           42  +  BtShared *pBt = p->pBt;
    42     43     assert( p->locked==1 );
    43         -  assert( sqlite3_mutex_held(p->pBt->mutex) );
           44  +  assert( sqlite3_mutex_held(pBt->mutex) );
    44     45     assert( sqlite3_mutex_held(p->db->mutex) );
    45         -  assert( p->db==p->pBt->db );
           46  +  assert( p->db==pBt->db );
    46     47   
    47         -  sqlite3_mutex_leave(p->pBt->mutex);
           48  +  pBt->iMutexCounter++;
           49  +  sqlite3_mutex_leave(pBt->mutex);
    48     50     p->locked = 0;
    49     51   }
           52  +
           53  +#ifdef SQLITE_DEBUG
           54  +/*
           55  +** Return the number of times that the mutex has been exited for
           56  +** the given btree.
           57  +**
           58  +** This is a small circular counter that wraps around to zero on
           59  +** overflow.  It is used only for sanity checking - to verify that
           60  +** mutexes are held continously by asserting that the value of
           61  +** this counter at the beginning of a region is the same as at
           62  +** the end.
           63  +*/
           64  +u32 sqlite3BtreeMutexCounter(Btree *p){
           65  +  assert( p->locked==1 || p->sharable==0 );
           66  +  return p->pBt->iMutexCounter;
           67  +}
           68  +#endif
    50     69   
    51     70   /*
    52     71   ** Enter a mutex on the given BTree object.
    53     72   **
    54     73   ** If the object is not sharable, then no mutex is ever required
    55     74   ** and this routine is a no-op.  The underlying mutex is non-recursive.
    56     75   ** But we keep a reference count in Btree.wantToLock so the behavior
................................................................................
    87    106     /* Unless the database is sharable and unlocked, then BtShared.db
    88    107     ** should already be set correctly. */
    89    108     assert( (p->locked==0 && p->sharable) || p->pBt->db==p->db );
    90    109   
    91    110     if( !p->sharable ) return;
    92    111     p->wantToLock++;
    93    112     if( p->locked ) return;
          113  +
          114  +  /* Increment the mutex counter on all locked btrees in the same
          115  +  ** database connection.  This simulates the unlocking that would
          116  +  ** occur on a worst-case mutex dead-lock avoidance scenario.
          117  +  */
          118  +#ifdef SQLITE_DEBUG
          119  +  {
          120  +    int ii;
          121  +    sqlite3 *db = p->db;
          122  +    Btree *pOther;
          123  +    for(ii=0; ii<db->nDb; ii++){
          124  +      if( ii==1 ) continue;
          125  +      pOther = db->aDb[ii].pBt;
          126  +      if( pOther==0 || pOther->sharable==0 || pOther->locked==0 ) continue;
          127  +      pOther->pBt->iMutexCounter++;
          128  +    }
          129  +  }
          130  +#endif
    94    131   
    95    132     /* In most cases, we should be able to acquire the lock we
    96    133     ** want without having to go throught the ascending lock
    97    134     ** procedure that follows.  Just be sure not to block.
    98    135     */
    99    136     if( sqlite3_mutex_try(p->pBt->mutex)==SQLITE_OK ){
   100    137       p->pBt->db = p->db;
................................................................................
   247    284         return 0;
   248    285       }
   249    286     }
   250    287     return 1;
   251    288   }
   252    289   #endif /* NDEBUG */
   253    290   
          291  +#else /* SQLITE_THREADSAFE>0 above.  SQLITE_THREADSAFE==0 below */
   254    292   /*
   255         -** Add a new Btree pointer to a BtreeMutexArray. 
   256         -** if the pointer can possibly be shared with
   257         -** another database connection.
          293  +** The following are special cases for mutex enter routines for use
          294  +** in single threaded applications that use shared cache.  Except for
          295  +** these two routines, all mutex operations are no-ops in that case and
          296  +** are null #defines in btree.h.
   258    297   **
   259         -** The pointers are kept in sorted order by pBtree->pBt.  That
   260         -** way when we go to enter all the mutexes, we can enter them
   261         -** in order without every having to backup and retry and without
   262         -** worrying about deadlock.
   263         -**
   264         -** The number of shared btrees will always be small (usually 0 or 1)
   265         -** so an insertion sort is an adequate algorithm here.
          298  +** If shared cache is disabled, then all btree mutex routines, including
          299  +** the ones below, are no-ops and are null #defines in btree.h.
   266    300   */
   267         -void sqlite3BtreeMutexArrayInsert(BtreeMutexArray *pArray, Btree *pBtree){
   268         -  int i, j;
   269         -  BtShared *pBt;
   270         -  if( pBtree==0 || pBtree->sharable==0 ) return;
   271         -#ifndef NDEBUG
   272         -  {
   273         -    for(i=0; i<pArray->nMutex; i++){
   274         -      assert( pArray->aBtree[i]!=pBtree );
   275         -    }
   276         -  }
   277         -#endif
   278         -  assert( pArray->nMutex>=0 );
   279         -  assert( pArray->nMutex<ArraySize(pArray->aBtree)-1 );
   280         -  pBt = pBtree->pBt;
   281         -  for(i=0; i<pArray->nMutex; i++){
   282         -    assert( pArray->aBtree[i]!=pBtree );
   283         -    if( pArray->aBtree[i]->pBt>pBt ){
   284         -      for(j=pArray->nMutex; j>i; j--){
   285         -        pArray->aBtree[j] = pArray->aBtree[j-1];
   286         -      }
   287         -      pArray->aBtree[i] = pBtree;
   288         -      pArray->nMutex++;
   289         -      return;
   290         -    }
   291         -  }
   292         -  pArray->aBtree[pArray->nMutex++] = pBtree;
   293         -}
   294    301   
   295         -/*
   296         -** Enter the mutex of every btree in the array.  This routine is
   297         -** called at the beginning of sqlite3VdbeExec().  The mutexes are
   298         -** exited at the end of the same function.
   299         -*/
   300         -void sqlite3BtreeMutexArrayEnter(BtreeMutexArray *pArray){
   301         -  int i;
   302         -  for(i=0; i<pArray->nMutex; i++){
   303         -    Btree *p = pArray->aBtree[i];
   304         -    /* Some basic sanity checking */
   305         -    assert( i==0 || pArray->aBtree[i-1]->pBt<p->pBt );
   306         -    assert( !p->locked || p->wantToLock>0 );
   307         -
   308         -    /* We should already hold a lock on the database connection */
   309         -    assert( sqlite3_mutex_held(p->db->mutex) );
   310         -
   311         -    /* The Btree is sharable because only sharable Btrees are entered
   312         -    ** into the array in the first place. */
   313         -    assert( p->sharable );
   314         -
   315         -    p->wantToLock++;
   316         -    if( !p->locked ){
   317         -      lockBtreeMutex(p);
   318         -    }
   319         -  }
   320         -}
   321         -
   322         -/*
   323         -** Leave the mutex of every btree in the group.
   324         -*/
   325         -void sqlite3BtreeMutexArrayLeave(BtreeMutexArray *pArray){
   326         -  int i;
   327         -  for(i=0; i<pArray->nMutex; i++){
   328         -    Btree *p = pArray->aBtree[i];
   329         -    /* Some basic sanity checking */
   330         -    assert( i==0 || pArray->aBtree[i-1]->pBt<p->pBt );
   331         -    assert( p->locked );
   332         -    assert( p->wantToLock>0 );
   333         -
   334         -    /* We should already hold a lock on the database connection */
   335         -    assert( sqlite3_mutex_held(p->db->mutex) );
   336         -
   337         -    p->wantToLock--;
   338         -    if( p->wantToLock==0 ){
   339         -      unlockBtreeMutex(p);
   340         -    }
   341         -  }
   342         -}
   343         -
   344         -#else
   345    302   void sqlite3BtreeEnter(Btree *p){
   346    303     p->pBt->db = p->db;
   347    304   }
   348    305   void sqlite3BtreeEnterAll(sqlite3 *db){
   349    306     int i;
   350    307     for(i=0; i<db->nDb; i++){
   351    308       Btree *p = db->aDb[i].pBt;

Changes to src/btree.h.

    35     35   
    36     36   /*
    37     37   ** Forward declarations of structure
    38     38   */
    39     39   typedef struct Btree Btree;
    40     40   typedef struct BtCursor BtCursor;
    41     41   typedef struct BtShared BtShared;
    42         -typedef struct BtreeMutexArray BtreeMutexArray;
    43         -
    44         -/*
    45         -** This structure records all of the Btrees that need to hold
    46         -** a mutex before we enter sqlite3VdbeExec().  The Btrees are
    47         -** are placed in aBtree[] in order of aBtree[]->pBt.  That way,
    48         -** we can always lock and unlock them all quickly.
    49         -*/
    50         -struct BtreeMutexArray {
    51         -  int nMutex;
    52         -  Btree *aBtree[SQLITE_MAX_ATTACHED+1];
    53         -};
    54     42   
    55     43   
    56     44   int sqlite3BtreeOpen(
    57     45     const char *zFilename,   /* Name of database file to open */
    58     46     sqlite3 *db,             /* Associated database connection */
    59     47     Btree **ppBtree,         /* Return open Btree* here */
    60     48     int flags,               /* Flags */
................................................................................
   224    212   #endif
   225    213   
   226    214   #if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE
   227    215     void sqlite3BtreeLeave(Btree*);
   228    216     void sqlite3BtreeEnterCursor(BtCursor*);
   229    217     void sqlite3BtreeLeaveCursor(BtCursor*);
   230    218     void sqlite3BtreeLeaveAll(sqlite3*);
   231         -  void sqlite3BtreeMutexArrayEnter(BtreeMutexArray*);
   232         -  void sqlite3BtreeMutexArrayLeave(BtreeMutexArray*);
   233         -  void sqlite3BtreeMutexArrayInsert(BtreeMutexArray*, Btree*);
   234    219   #ifndef NDEBUG
   235    220     /* These routines are used inside assert() statements only. */
   236    221     int sqlite3BtreeHoldsMutex(Btree*);
   237    222     int sqlite3BtreeHoldsAllMutexes(sqlite3*);
          223  +  u32 sqlite3BtreeMutexCounter(Btree*);
   238    224   #endif
   239    225   #else
   240    226   
   241    227   # define sqlite3BtreeLeave(X)
          228  +# define sqlite3BtreeMutexCounter(X) 0
   242    229   # define sqlite3BtreeEnterCursor(X)
   243    230   # define sqlite3BtreeLeaveCursor(X)
   244    231   # define sqlite3BtreeLeaveAll(X)
   245         -# define sqlite3BtreeMutexArrayEnter(X)
   246         -# define sqlite3BtreeMutexArrayLeave(X)
   247         -# define sqlite3BtreeMutexArrayInsert(X,Y)
   248    232   
   249    233   # define sqlite3BtreeHoldsMutex(X) 1
   250    234   # define sqlite3BtreeHoldsAllMutexes(X) 1
   251    235   #endif
   252    236   
   253    237   
   254    238   #endif /* _BTREE_H_ */

Changes to src/btreeInt.h.

   422    422     u16 minLeaf;          /* Minimum local payload in a LEAFDATA table */
   423    423     u32 pageSize;         /* Total number of bytes on a page */
   424    424     u32 usableSize;       /* Number of usable bytes on each page */
   425    425     int nTransaction;     /* Number of open transactions (read + write) */
   426    426     u32 nPage;            /* Number of pages in the database */
   427    427     void *pSchema;        /* Pointer to space allocated by sqlite3BtreeSchema() */
   428    428     void (*xFreeSchema)(void*);  /* Destructor for BtShared.pSchema */
   429         -  sqlite3_mutex *mutex; /* Non-recursive mutex required to access this struct */
          429  +  sqlite3_mutex *mutex; /* Non-recursive mutex required to access this object */
   430    430     Bitvec *pHasContent;  /* Set of pages moved to free-list this transaction */
   431    431   #ifndef SQLITE_OMIT_SHARED_CACHE
   432    432     int nRef;             /* Number of references to this structure */
   433    433     BtShared *pNext;      /* Next on a list of sharable BtShared structs */
   434    434     BtLock *pLock;        /* List of locks held on this shared-btree struct */
   435    435     Btree *pWriter;       /* Btree with currently open write transaction */
   436    436     u8 isExclusive;       /* True if pWriter has an EXCLUSIVE lock on the db */
   437    437     u8 isPending;         /* If waiting for read-locks to clear */
          438  +  u16 iMutexCounter;    /* The number of mutex_leave(mutex) calls */
   438    439   #endif
   439    440     u8 *pTmpSpace;        /* BtShared.pageSize bytes of space for tmp use */
   440    441   };
   441    442   
   442    443   /*
   443    444   ** An instance of the following structure is used to hold information
   444    445   ** about a cell.  The parseCellPtr() function fills in this structure

Changes to src/vdbe.c.

   567    567   #ifdef VDBE_PROFILE
   568    568     u64 start;                 /* CPU clock count at start of opcode */
   569    569     int origPc;                /* Program counter at start of opcode */
   570    570   #endif
   571    571     /*** INSERT STACK UNION HERE ***/
   572    572   
   573    573     assert( p->magic==VDBE_MAGIC_RUN );  /* sqlite3_step() verifies this */
   574         -  sqlite3VdbeMutexArrayEnter(p);
          574  +  sqlite3VdbeEnter(p);
   575    575     if( p->rc==SQLITE_NOMEM ){
   576    576       /* This happens if a malloc() inside a call to sqlite3_column_text() or
   577    577       ** sqlite3_column_text16() failed.  */
   578    578       goto no_mem;
   579    579     }
   580    580     assert( p->rc==SQLITE_OK || p->rc==SQLITE_BUSY );
   581    581     p->rc = SQLITE_OK;
................................................................................
  1390   1390     if( ctx.pFunc->flags & SQLITE_FUNC_NEEDCOLL ){
  1391   1391       assert( pOp>aOp );
  1392   1392       assert( pOp[-1].p4type==P4_COLLSEQ );
  1393   1393       assert( pOp[-1].opcode==OP_CollSeq );
  1394   1394       ctx.pColl = pOp[-1].p4.pColl;
  1395   1395     }
  1396   1396     (*ctx.pFunc->xFunc)(&ctx, n, apVal); /* IMP: R-24505-23230 */
         1397  +  sqlite3VdbeMutexResync(p);
  1397   1398     if( db->mallocFailed ){
  1398   1399       /* Even though a malloc() has failed, the implementation of the
  1399   1400       ** user function may have called an sqlite3_result_XXX() function
  1400   1401       ** to return a value. The following call releases any resources
  1401   1402       ** associated with such a value.
  1402   1403       */
  1403   1404       sqlite3VdbeMemRelease(&ctx.s);
  1404   1405       goto no_mem;
  1405   1406     }
         1407  +
         1408  +  /* The app-defined function has done something that as caused this
         1409  +  ** statement to expire.  (Perhaps the function called sqlite3_exec()
         1410  +  ** with a CREATE TABLE statement.)
         1411  +  */
         1412  +#if 0
         1413  +  if( p->expired ){
         1414  +    rc = SQLITE_ABORT;
         1415  +    break;
         1416  +  }
         1417  +#endif
  1406   1418   
  1407   1419     /* If any auxiliary data functions have been called by this user function,
  1408   1420     ** immediately call the destructor for any non-static values.
  1409   1421     */
  1410   1422     if( ctx.pVdbeFunc ){
  1411   1423       sqlite3VdbeDeleteAuxData(ctx.pVdbeFunc, pOp->p1);
  1412   1424       pOp->p4.pVdbeFunc = ctx.pVdbeFunc;
................................................................................
  2646   2658             if( rc!=SQLITE_OK ){
  2647   2659               goto abort_due_to_error;
  2648   2660             }
  2649   2661           }
  2650   2662           if( p1==SAVEPOINT_ROLLBACK && (db->flags&SQLITE_InternChanges)!=0 ){
  2651   2663             sqlite3ExpirePreparedStatements(db);
  2652   2664             sqlite3ResetInternalSchema(db, 0);
         2665  +          sqlite3VdbeMutexResync(p);
  2653   2666             db->flags = (db->flags | SQLITE_InternChanges);
  2654   2667           }
  2655   2668         }
  2656   2669     
  2657   2670         /* Regardless of whether this is a RELEASE or ROLLBACK, destroy all 
  2658   2671         ** savepoints nested inside of the savepoint being operated on. */
  2659   2672         while( db->pSavepoint!=pSavepoint ){
................................................................................
  2936   2949       ** discard the database schema, as the user code implementing the
  2937   2950       ** v-table would have to be ready for the sqlite3_vtab structure itself
  2938   2951       ** to be invalidated whenever sqlite3_step() is called from within 
  2939   2952       ** a v-table method.
  2940   2953       */
  2941   2954       if( db->aDb[pOp->p1].pSchema->schema_cookie!=iMeta ){
  2942   2955         sqlite3ResetInternalSchema(db, pOp->p1);
         2956  +      sqlite3VdbeMutexResync(p);
  2943   2957       }
  2944   2958   
  2945   2959       p->expired = 1;
  2946   2960       rc = SQLITE_SCHEMA;
  2947   2961     }
  2948   2962     break;
  2949   2963   }
................................................................................
  4617   4631   ** then runs the new virtual machine.  It is thus a re-entrant opcode.
  4618   4632   */
  4619   4633   case OP_ParseSchema: {
  4620   4634     int iDb;
  4621   4635     const char *zMaster;
  4622   4636     char *zSql;
  4623   4637     InitData initData;
         4638  +
         4639  +  /* Any prepared statement that invokes this opcode will hold mutexes
         4640  +  ** on every btree.  This is a prerequisite for invoking 
         4641  +  ** sqlite3InitCallback().
         4642  +  */
         4643  +#ifdef SQLITE_DEBUG
         4644  +  for(iDb=0; iDb<db->nDb; iDb++){
         4645  +    assert( iDb==1 || sqlite3BtreeHoldsMutex(db->aDb[iDb].pBt) );
         4646  +  }
         4647  +#endif
         4648  +  assert( p->btreeMask == ~(yDbMask)0 );
         4649  +
  4624   4650   
  4625   4651     iDb = pOp->p1;
  4626   4652     assert( iDb>=0 && iDb<db->nDb );
  4627         -
  4628         -  /* When this opcode is invoked, it is guaranteed that the b-tree mutex
  4629         -  ** is held and the schema is loaded for database iDb. However, at the 
  4630         -  ** start of the sqlite3_exec() call below, SQLite will invoke 
  4631         -  ** sqlite3BtreeEnterAll(). If all mutexes are not already held, the iDb 
  4632         -  ** mutex may be temporarily released to avoid deadlock. If this happens, 
  4633         -  ** then some other thread may delete the in-memory schema of database iDb 
  4634         -  ** before the SQL statement runs. The schema will not be reloaded because 
  4635         -  ** the db->init.busy flag is set. This can result in a "no such table: 
  4636         -  ** sqlite_master" or "malformed database schema" error being returned to 
  4637         -  ** the user. 
  4638         -  **
  4639         -  ** To avoid this, obtain all mutexes and check that no other thread has
  4640         -  ** deleted the schema before calling sqlite3_exec(). If we find that the
  4641         -  ** another thread has deleted the schema, there is no need to update it.
  4642         -  ** The updated schema will be loaded from disk when it is next required.
  4643         -  */
  4644         -  assert( sqlite3BtreeHoldsMutex(db->aDb[iDb].pBt) );
  4645   4653     assert( DbHasProperty(db, iDb, DB_SchemaLoaded) );
  4646         -  sqlite3BtreeEnterAll(db);
  4647         -  if( DbHasProperty(db, iDb, DB_SchemaLoaded) ){
         4654  +  /* Used to be a conditional */ {
  4648   4655       zMaster = SCHEMA_TABLE(iDb);
  4649   4656       initData.db = db;
  4650   4657       initData.iDb = pOp->p1;
  4651   4658       initData.pzErrMsg = &p->zErrMsg;
  4652   4659       zSql = sqlite3MPrintf(db,
  4653   4660          "SELECT name, rootpage, sql FROM '%q'.%s WHERE %s ORDER BY rowid",
  4654   4661          db->aDb[iDb].zName, zMaster, pOp->p4.z);
................................................................................
  4661   4668         assert( !db->mallocFailed );
  4662   4669         rc = sqlite3_exec(db, zSql, sqlite3InitCallback, &initData, 0);
  4663   4670         if( rc==SQLITE_OK ) rc = initData.rc;
  4664   4671         sqlite3DbFree(db, zSql);
  4665   4672         db->init.busy = 0;
  4666   4673       }
  4667   4674     }
  4668         -  sqlite3BtreeLeaveAll(db);
  4669   4675     if( rc==SQLITE_NOMEM ){
  4670   4676       goto no_mem;
  4671   4677     }
  4672   4678     break;  
  4673   4679   }
  4674   4680   
  4675   4681   #if !defined(SQLITE_OMIT_ANALYZE)
................................................................................
  5186   5192     if( ctx.pFunc->flags & SQLITE_FUNC_NEEDCOLL ){
  5187   5193       assert( pOp>p->aOp );
  5188   5194       assert( pOp[-1].p4type==P4_COLLSEQ );
  5189   5195       assert( pOp[-1].opcode==OP_CollSeq );
  5190   5196       ctx.pColl = pOp[-1].p4.pColl;
  5191   5197     }
  5192   5198     (ctx.pFunc->xStep)(&ctx, n, apVal); /* IMP: R-24505-23230 */
         5199  +  sqlite3VdbeMutexResync(p);
  5193   5200     if( ctx.isError ){
  5194   5201       sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&ctx.s));
  5195   5202       rc = ctx.isError;
  5196   5203     }
         5204  +
         5205  +  /* The app-defined function has done something that as caused this
         5206  +  ** statement to expire.  (Perhaps the function called sqlite3_exec()
         5207  +  ** with a CREATE TABLE statement.)
         5208  +  */
         5209  +#if 0
         5210  +  if( p->expired ){
         5211  +    rc = SQLITE_ABORT;
         5212  +    break;
         5213  +  }
         5214  +#endif
         5215  +
  5197   5216     sqlite3VdbeMemRelease(&ctx.s);
         5217  +
  5198   5218     break;
  5199   5219   }
  5200   5220   
  5201   5221   /* Opcode: AggFinal P1 P2 * P4 *
  5202   5222   **
  5203   5223   ** Execute the finalizer function for an aggregate.  P1 is
  5204   5224   ** the memory location that is the accumulator for the aggregate.
................................................................................
  5212   5232   */
  5213   5233   case OP_AggFinal: {
  5214   5234     Mem *pMem;
  5215   5235     assert( pOp->p1>0 && pOp->p1<=p->nMem );
  5216   5236     pMem = &aMem[pOp->p1];
  5217   5237     assert( (pMem->flags & ~(MEM_Null|MEM_Agg))==0 );
  5218   5238     rc = sqlite3VdbeMemFinalize(pMem, pOp->p4.pFunc);
         5239  +  sqlite3VdbeMutexResync(p);
  5219   5240     if( rc ){
  5220   5241       sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(pMem));
         5242  +  }else if( p->expired ){
         5243  +    rc = SQLITE_ABORT;
  5221   5244     }
  5222   5245     sqlite3VdbeChangeEncoding(pMem, encoding);
  5223   5246     UPDATE_MAX_BLOBSIZE(pMem);
  5224   5247     if( sqlite3VdbeMemTooBig(pMem) ){
  5225   5248       goto too_big;
  5226   5249     }
  5227   5250     break;
................................................................................
  5290   5313          || eNew==PAGER_JOURNALMODE_WAL
  5291   5314          || eNew==PAGER_JOURNALMODE_QUERY
  5292   5315     );
  5293   5316     assert( pOp->p1>=0 && pOp->p1<db->nDb );
  5294   5317   
  5295   5318     /* This opcode is used in two places: PRAGMA journal_mode and ATTACH.
  5296   5319     ** In PRAGMA journal_mode, the sqlite3VdbeUsesBtree() routine is called
  5297         -  ** when the statment is prepared and so p->aMutex.nMutex>0.  All mutexes
         5320  +  ** when the statement is prepared and so p->btreeMask!=0.  All mutexes
  5298   5321     ** are already acquired.  But when used in ATTACH, sqlite3VdbeUsesBtree()
  5299   5322     ** is not called when the statement is prepared because it requires the
  5300   5323     ** iDb index of the database as a parameter, and the database has not
  5301   5324     ** yet been attached so that index is unavailable.  We have to wait
  5302   5325     ** until runtime (now) to get the mutex on the newly attached database.
  5303   5326     ** No other mutexes are required by the ATTACH command so this is safe
  5304   5327     ** to do.
  5305   5328     */
  5306         -  assert( (p->btreeMask & (((yDbMask)1)<<pOp->p1))!=0 || p->aMutex.nMutex==0 );
  5307         -  if( p->aMutex.nMutex==0 ){
         5329  +  if( p->btreeMask==0 ){
  5308   5330       /* This occurs right after ATTACH.  Get a mutex on the newly ATTACHed
  5309   5331       ** database. */
  5310   5332       sqlite3VdbeUsesBtree(p, pOp->p1);
  5311         -    sqlite3VdbeMutexArrayEnter(p);
         5333  +    sqlite3VdbeEnter(p);
  5312   5334     }
  5313   5335   
  5314   5336     pBt = db->aDb[pOp->p1].pBt;
  5315   5337     pPager = sqlite3BtreePager(pBt);
  5316   5338     eOld = sqlite3PagerGetJournalMode(pPager);
  5317   5339     if( eNew==PAGER_JOURNALMODE_QUERY ) eNew = eOld;
  5318   5340     if( !sqlite3PagerOkToChangeJournalMode(pPager) ) eNew = eOld;
................................................................................
  5942   5964     p->rc = rc;
  5943   5965     testcase( sqlite3GlobalConfig.xLog!=0 );
  5944   5966     sqlite3_log(rc, "statement aborts at %d: [%s] %s", 
  5945   5967                      pc, p->zSql, p->zErrMsg);
  5946   5968     sqlite3VdbeHalt(p);
  5947   5969     if( rc==SQLITE_IOERR_NOMEM ) db->mallocFailed = 1;
  5948   5970     rc = SQLITE_ERROR;
  5949         -  if( resetSchemaOnFault ) sqlite3ResetInternalSchema(db, 0);
         5971  +  if( resetSchemaOnFault ){
         5972  +    sqlite3ResetInternalSchema(db, 0);
         5973  +    sqlite3VdbeMutexResync(p);
         5974  +  }
  5950   5975   
  5951   5976     /* This is the only way out of this procedure.  We have to
  5952   5977     ** release the mutexes on btrees that were acquired at the
  5953   5978     ** top. */
  5954   5979   vdbe_return:
  5955         -  sqlite3BtreeMutexArrayLeave(&p->aMutex);
         5980  +  sqlite3VdbeLeave(p);
  5956   5981     return rc;
  5957   5982   
  5958   5983     /* Jump to here if a string or blob larger than SQLITE_MAX_LENGTH
  5959   5984     ** is encountered.
  5960   5985     */
  5961   5986   too_big:
  5962   5987     sqlite3SetString(&p->zErrMsg, db, "string or blob too big");

Changes to src/vdbeInt.h.

   299    299     u8 minWriteFileFormat;  /* Minimum file format for writable database files */
   300    300     u8 inVtabMethod;        /* See comments above */
   301    301     u8 usesStmtJournal;     /* True if uses a statement journal */
   302    302     u8 readOnly;            /* True for read-only statements */
   303    303     u8 isPrepareV2;         /* True if prepared with prepare_v2() */
   304    304     int nChange;            /* Number of db changes made since last reset */
   305    305     yDbMask btreeMask;      /* Bitmask of db->aDb[] entries referenced */
          306  +  u32 iMutexCounter;      /* Mutex counter upon sqlite3VdbeEnter() */
   306    307     int iStatement;         /* Statement number (or 0 if has not opened stmt) */
   307    308     int aCounter[3];        /* Counters used by sqlite3_stmt_status() */
   308         -  BtreeMutexArray aMutex; /* An array of Btree used here and needing locks */
   309    309   #ifndef SQLITE_OMIT_TRACE
   310    310     i64 startTime;          /* Time when query started - used for profiling */
   311    311   #endif
   312    312     i64 nFkConstraint;      /* Number of imm. FK constraints this VM */
   313    313     i64 nStmtDefCons;       /* Number of def. constraints when stmt started */
   314    314     char *zSql;             /* Text of the SQL statement that generated this */
   315    315     void *pFree;            /* Free this when deleting the vdbe */
................................................................................
   383    383   int sqlite3VdbeMemFinalize(Mem*, FuncDef*);
   384    384   const char *sqlite3OpcodeName(int);
   385    385   int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve);
   386    386   int sqlite3VdbeCloseStatement(Vdbe *, int);
   387    387   void sqlite3VdbeFrameDelete(VdbeFrame*);
   388    388   int sqlite3VdbeFrameRestore(VdbeFrame *);
   389    389   void sqlite3VdbeMemStoreType(Mem *pMem);
          390  +void sqlite3VdbeEnter(Vdbe*);
          391  +void sqlite3VdbeLeave(Vdbe*);
          392  +void sqlite3VdbeMutexResync(Vdbe*);
   390    393   
   391    394   #ifdef SQLITE_DEBUG
   392    395   void sqlite3VdbeMemPrepareToChange(Vdbe*,Mem*);
   393    396   #endif
   394    397   
   395    398   #ifndef SQLITE_OMIT_FOREIGN_KEY
   396    399   int sqlite3VdbeCheckFk(Vdbe *, int);
   397    400   #else
   398    401   # define sqlite3VdbeCheckFk(p,i) 0
   399    402   #endif
   400    403   
   401         -#ifndef SQLITE_OMIT_SHARED_CACHE
   402         -void sqlite3VdbeMutexArrayEnter(Vdbe *p);
   403         -#else
   404         -# define sqlite3VdbeMutexArrayEnter(p)
   405         -#endif
   406         -
   407    404   int sqlite3VdbeMemTranslate(Mem*, u8);
   408    405   #ifdef SQLITE_DEBUG
   409    406     void sqlite3VdbePrintSql(Vdbe*);
   410    407     void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf);
   411    408   #endif
   412    409   int sqlite3VdbeMemHandleBom(Mem *pMem);
   413    410   

Changes to src/vdbeaux.c.

   153    153     pOp->p5 = 0;
   154    154     pOp->p1 = p1;
   155    155     pOp->p2 = p2;
   156    156     pOp->p3 = p3;
   157    157     pOp->p4.p = 0;
   158    158     pOp->p4type = P4_NOTUSED;
   159    159     p->expired = 0;
          160  +  if( op==OP_ParseSchema ){
          161  +    /* Any program that uses the OP_ParseSchema opcode needs to lock
          162  +    ** all btrees. */
          163  +    p->btreeMask = ~(yDbMask)0;
          164  +  }
   160    165   #ifdef SQLITE_DEBUG
   161    166     pOp->zComment = 0;
   162    167     if( sqlite3VdbeAddopTrace ) sqlite3VdbePrintOp(0, i, &p->aOp[i]);
   163    168   #endif
   164    169   #ifdef VDBE_PROFILE
   165    170     pOp->cycles = 0;
   166    171     pOp->cnt = 0;
................................................................................
   453    458   ** returned program.
   454    459   */
   455    460   VdbeOp *sqlite3VdbeTakeOpArray(Vdbe *p, int *pnOp, int *pnMaxArg){
   456    461     VdbeOp *aOp = p->aOp;
   457    462     assert( aOp && !p->db->mallocFailed );
   458    463   
   459    464     /* Check that sqlite3VdbeUsesBtree() was not called on this VM */
   460         -  assert( p->aMutex.nMutex==0 );
          465  +  assert( p->btreeMask==0 );
   461    466   
   462    467     resolveP2Values(p, pnMaxArg);
   463    468     *pnOp = p->nOp;
   464    469     p->aOp = 0;
   465    470     return aOp;
   466    471   }
   467    472   
................................................................................
   941    946     return zP4;
   942    947   }
   943    948   #endif
   944    949   
   945    950   /*
   946    951   ** Declare to the Vdbe that the BTree object at db->aDb[i] is used.
   947    952   **
   948         -** The prepared statement has to know in advance which Btree objects
   949         -** will be used so that it can acquire mutexes on them all in sorted
   950         -** order (via sqlite3VdbeMutexArrayEnter().  Mutexes are acquired
   951         -** in order (and released in reverse order) to avoid deadlocks.
          953  +** The prepared statements need to know in advance the complete set of
          954  +** attached databases that they will be using.  A mask of these databases
          955  +** is maintained in p->btreeMask and is used for locking and other purposes.
   952    956   */
   953    957   void sqlite3VdbeUsesBtree(Vdbe *p, int i){
   954         -  yDbMask mask;
   955    958     assert( i>=0 && i<p->db->nDb && i<sizeof(yDbMask)*8 );
   956    959     assert( i<(int)sizeof(p->btreeMask)*8 );
   957         -  mask = ((yDbMask)1)<<i;
   958         -  if( (p->btreeMask & mask)==0 ){
   959         -    p->btreeMask |= mask;
   960         -    sqlite3BtreeMutexArrayInsert(&p->aMutex, p->db->aDb[i].pBt);
   961         -  }
   962         -}
   963         -
          960  +  p->btreeMask |= ((yDbMask)1)<<i;
          961  +}
          962  +
          963  +/*
          964  +** Compute the sum of all mutex counters for all btrees in the
          965  +** given prepared statement.
          966  +*/
          967  +#ifndef SQLITE_OMIT_SHARED_CACHE
          968  +static u32 mutexCounterSum(Vdbe *p){
          969  +  u32 cntSum = 0;
          970  +#ifdef SQLITE_DEBUG
          971  +  int i;
          972  +  yDbMask mask;
          973  +  sqlite3 *db = p->db;
          974  +  Db *aDb = db->aDb;
          975  +  int nDb = db->nDb;
          976  +  for(i=0, mask=1; i<nDb; i++, mask += mask){
          977  +    if( i!=1 && (mask & p->btreeMask)!=0 && ALWAYS(aDb[i].pBt!=0) ){
          978  +      cntSum += sqlite3BtreeMutexCounter(aDb[i].pBt);
          979  +    }
          980  +  }
          981  +#endif
          982  +  return cntSum;
          983  +}
          984  +#endif
          985  +
          986  +/*
          987  +** If SQLite is compiled to support shared-cache mode and to be threadsafe,
          988  +** this routine obtains the mutex associated with each BtShared structure
          989  +** that may be accessed by the VM passed as an argument. In doing so it also
          990  +** sets the BtShared.db member of each of the BtShared structures, ensuring
          991  +** that the correct busy-handler callback is invoked if required.
          992  +**
          993  +** If SQLite is not threadsafe but does support shared-cache mode, then
          994  +** sqlite3BtreeEnter() is invoked to set the BtShared.db variables
          995  +** of all of BtShared structures accessible via the database handle 
          996  +** associated with the VM.
          997  +**
          998  +** If SQLite is not threadsafe and does not support shared-cache mode, this
          999  +** function is a no-op.
         1000  +**
         1001  +** The p->btreeMask field is a bitmask of all btrees that the prepared 
         1002  +** statement p will ever use.  Let N be the number of bits in p->btreeMask
         1003  +** corresponding to btrees that use shared cache.  Then the runtime of
         1004  +** this routine is N*N.  But as N is rarely more than 1, this should not
         1005  +** be a problem.
         1006  +*/
         1007  +void sqlite3VdbeEnter(Vdbe *p){
         1008  +#ifndef SQLITE_OMIT_SHARED_CACHE
         1009  +  int i;
         1010  +  yDbMask mask;
         1011  +  sqlite3 *db = p->db;
         1012  +  Db *aDb = db->aDb;
         1013  +  int nDb = db->nDb;
         1014  +  for(i=0, mask=1; i<nDb; i++, mask += mask){
         1015  +    if( i!=1 && (mask & p->btreeMask)!=0 && ALWAYS(aDb[i].pBt!=0) ){
         1016  +      sqlite3BtreeEnter(aDb[i].pBt);
         1017  +    }
         1018  +  }
         1019  +  p->iMutexCounter = mutexCounterSum(p);
         1020  +#endif
         1021  +}
         1022  +
         1023  +/*
         1024  +** Unlock all of the btrees previously locked by a call to sqlite3VdbeEnter().
         1025  +*/
         1026  +void sqlite3VdbeLeave(Vdbe *p){
         1027  +#ifndef SQLITE_OMIT_SHARED_CACHE
         1028  +  int i;
         1029  +  yDbMask mask;
         1030  +  sqlite3 *db = p->db;
         1031  +  Db *aDb = db->aDb;
         1032  +  int nDb = db->nDb;
         1033  +
         1034  +  /* Assert that the all mutexes have been held continously since
         1035  +  ** the most recent sqlite3VdbeEnter() or sqlite3VdbeMutexResync().
         1036  +  */
         1037  +  assert( mutexCounterSum(p) == p->iMutexCounter );
         1038  +
         1039  +  for(i=0, mask=1; i<nDb; i++, mask += mask){
         1040  +    if( i!=1 && (mask & p->btreeMask)!=0 && ALWAYS(aDb[i].pBt!=0) ){
         1041  +      sqlite3BtreeLeave(aDb[i].pBt);
         1042  +    }
         1043  +  }
         1044  +#endif
         1045  +}
         1046  +
         1047  +/*
         1048  +** Recompute the sum of the mutex counters on all btrees used by the
         1049  +** prepared statement p.
         1050  +**
         1051  +** Call this routine while holding a sqlite3VdbeEnter() after doing something
         1052  +** that might cause one or more of the individual mutexes held by the
         1053  +** prepared statement to be released.  Calling sqlite3BtreeEnter() on 
         1054  +** any BtShared mutex which is not used by the prepared statement is one
         1055  +** way to cause one or more of the mutexes in the prepared statement
         1056  +** to be temporarily released.  The anti-deadlocking logic in
         1057  +** sqlite3BtreeEnter() can cause mutexes to be released temporarily then
         1058  +** reacquired.
         1059  +**
         1060  +** Calling this routine is an acknowledgement that some of the individual
         1061  +** mutexes in the prepared statement might have been released and reacquired.
         1062  +** So checks to verify that mutex-protected content did not change
         1063  +** unexpectedly should accompany any call to this routine.
         1064  +*/
         1065  +void sqlite3VdbeMutexResync(Vdbe *p){
         1066  +#if !defined(SQLITE_OMIT_SHARED_CACHE) && defined(SQLITE_DEBUG)
         1067  +  p->iMutexCounter = mutexCounterSum(p);
         1068  +#endif
         1069  +}
   964   1070   
   965   1071   #if defined(VDBE_PROFILE) || defined(SQLITE_DEBUG)
   966   1072   /*
   967   1073   ** Print a single opcode.  This routine is used for debugging only.
   968   1074   */
   969   1075   void sqlite3VdbePrintOp(FILE *pOut, int pc, Op *pOp){
   970   1076     char *zP4;
................................................................................
  1955   2061       if( eOp==SAVEPOINT_ROLLBACK ){
  1956   2062         db->nDeferredCons = p->nStmtDefCons;
  1957   2063       }
  1958   2064     }
  1959   2065     return rc;
  1960   2066   }
  1961   2067   
  1962         -/*
  1963         -** If SQLite is compiled to support shared-cache mode and to be threadsafe,
  1964         -** this routine obtains the mutex associated with each BtShared structure
  1965         -** that may be accessed by the VM passed as an argument. In doing so it
  1966         -** sets the BtShared.db member of each of the BtShared structures, ensuring
  1967         -** that the correct busy-handler callback is invoked if required.
  1968         -**
  1969         -** If SQLite is not threadsafe but does support shared-cache mode, then
  1970         -** sqlite3BtreeEnterAll() is invoked to set the BtShared.db variables
  1971         -** of all of BtShared structures accessible via the database handle 
  1972         -** associated with the VM. Of course only a subset of these structures
  1973         -** will be accessed by the VM, and we could use Vdbe.btreeMask to figure
  1974         -** that subset out, but there is no advantage to doing so.
  1975         -**
  1976         -** If SQLite is not threadsafe and does not support shared-cache mode, this
  1977         -** function is a no-op.
  1978         -*/
  1979         -#ifndef SQLITE_OMIT_SHARED_CACHE
  1980         -void sqlite3VdbeMutexArrayEnter(Vdbe *p){
  1981         -#if SQLITE_THREADSAFE
  1982         -  sqlite3BtreeMutexArrayEnter(&p->aMutex);
  1983         -#else
  1984         -  sqlite3BtreeEnterAll(p->db);
  1985         -#endif
  1986         -}
  1987         -#endif
  1988         -
  1989   2068   /*
  1990   2069   ** This function is called when a transaction opened by the database 
  1991   2070   ** handle associated with the VM passed as an argument is about to be 
  1992   2071   ** committed. If there are outstanding deferred foreign key constraint
  1993   2072   ** violations, return SQLITE_ERROR. Otherwise, SQLITE_OK.
  1994   2073   **
  1995   2074   ** If there are outstanding FK violations and this function returns 
................................................................................
  2054   2133     /* No commit or rollback needed if the program never started */
  2055   2134     if( p->pc>=0 ){
  2056   2135       int mrc;   /* Primary error code from p->rc */
  2057   2136       int eStatementOp = 0;
  2058   2137       int isSpecialError;            /* Set to true if a 'special' error */
  2059   2138   
  2060   2139       /* Lock all btrees used by the statement */
  2061         -    sqlite3VdbeMutexArrayEnter(p);
         2140  +    sqlite3VdbeEnter(p);
  2062   2141   
  2063   2142       /* Check for one of the special errors */
  2064   2143       mrc = p->rc & 0xff;
  2065   2144       assert( p->rc!=SQLITE_IOERR_BLOCKED );  /* This error no longer exists */
  2066   2145       isSpecialError = mrc==SQLITE_NOMEM || mrc==SQLITE_IOERR
  2067   2146                        || mrc==SQLITE_INTERRUPT || mrc==SQLITE_FULL;
  2068   2147       if( isSpecialError ){
................................................................................
  2108   2187        && db->autoCommit 
  2109   2188        && db->writeVdbeCnt==(p->readOnly==0) 
  2110   2189       ){
  2111   2190         if( p->rc==SQLITE_OK || (p->errorAction==OE_Fail && !isSpecialError) ){
  2112   2191           rc = sqlite3VdbeCheckFk(p, 1);
  2113   2192           if( rc!=SQLITE_OK ){
  2114   2193             if( NEVER(p->readOnly) ){
  2115         -            sqlite3BtreeMutexArrayLeave(&p->aMutex);
         2194  +            sqlite3VdbeLeave(p);
  2116   2195               return SQLITE_ERROR;
  2117   2196             }
  2118   2197             rc = SQLITE_CONSTRAINT;
  2119   2198           }else{ 
  2120   2199             /* The auto-commit flag is true, the vdbe program was successful 
  2121   2200             ** or hit an 'OR FAIL' constraint and there are no deferred foreign
  2122   2201             ** key constraints to hold up the transaction. This means a commit 
  2123   2202             ** is required. */
  2124   2203             rc = vdbeCommit(db, p);
  2125   2204           }
  2126   2205           if( rc==SQLITE_BUSY && p->readOnly ){
  2127         -          sqlite3BtreeMutexArrayLeave(&p->aMutex);
         2206  +          sqlite3VdbeLeave(p);
  2128   2207             return SQLITE_BUSY;
  2129   2208           }else if( rc!=SQLITE_OK ){
  2130   2209             p->rc = rc;
  2131   2210             sqlite3RollbackAll(db);
  2132   2211           }else{
  2133   2212             db->nDeferredCons = 0;
  2134   2213             sqlite3CommitInternalChanges(db);
................................................................................
  2192   2271       /* Rollback or commit any schema changes that occurred. */
  2193   2272       if( p->rc!=SQLITE_OK && db->flags&SQLITE_InternChanges ){
  2194   2273         sqlite3ResetInternalSchema(db, 0);
  2195   2274         db->flags = (db->flags | SQLITE_InternChanges);
  2196   2275       }
  2197   2276   
  2198   2277       /* Release the locks */
  2199         -    sqlite3BtreeMutexArrayLeave(&p->aMutex);
         2278  +    sqlite3VdbeMutexResync(p);
         2279  +    sqlite3VdbeLeave(p);
  2200   2280     }
  2201   2281   
  2202   2282     /* We have successfully halted and closed the VM.  Record this fact. */
  2203   2283     if( p->pc>=0 ){
  2204   2284       db->activeVdbeCnt--;
  2205   2285       if( !p->readOnly ){
  2206   2286         db->writeVdbeCnt--;