SQLite

** The consequence of this all is that the hash table for the lockInfo
** structure has to include the process id as part of its key because
** locks in different threads are treated as distinct.  But the 
** openCnt structure should not include the process id in its
** key because close() clears lock on all threads, not just the current
** thread.  Were it not for this goofiness in linux threads, we could
** combine the lockInfo and openCnt structures into a single structure.
**
** 2004-Jun-28:
** On some versions of linux, threads can override each others locks.
** On others not.  Sometimes you can change the behavior on the same
** system by setting the LD_ASSUME_KERNEL environment variable.  The
** POSIX standard is silent as to which behavior is correct, as far
** as I can tell, so other versions of unix might show the same
** inconsistency.  There is no little doubt in my mind that posix
** advisory locks and linux threads are profoundly broken.
**
** To work around the inconsistencies, we have to test at runtime 
** whether or not threads can override each others locks.  This test
** is run once, the first time any lock is attempted.  A static 
** variable is set to record the results of this test for future
** use.
*/

/*
** An instance of the following structure serves as the key used
** to locate a particular lockInfo structure given its inode.


**
** If threads cannot override each others locks, then we set the
** lockKey.tid field to the thread ID.  If threads can override
** each others locks then tid is always set to zero.  tid is also
** set to zero if we compile without threading support.
*/
struct lockKey {
  dev_t dev;       /* Device number */
  ino_t ino;       /* Inode number */
#ifdef SQLITE_UNIX_THREADS
  pthread_t tid;   /* Thread ID or zero if threads cannot override each other */
#endif
};

/*
** An instance of the following structure is allocated for each open
** inode on each thread with a different process ID.  (Threads have
** different process IDs on linux, but not on most other unixes.)
**

  int locktype;        /* One of SHARED_LOCK, RESERVED_LOCK etc. */
  int nRef;            /* Number of pointers to this structure */
};

/*
** An instance of the following structure serves as the key used
** to locate a particular openCnt structure given its inode.  This
** is the same as the lockKey except that the thread ID is omitted.
*/
struct openKey {
  dev_t dev;   /* Device number */
  ino_t ino;   /* Inode number */
};

/*

/* 
** These hash table maps inodes and process IDs into lockInfo and openCnt
** structures.  Access to these hash tables must be protected by a mutex.
*/
static Hash lockHash = { SQLITE_HASH_BINARY, 0, 0, 0, 0, 0 };
static Hash openHash = { SQLITE_HASH_BINARY, 0, 0, 0, 0, 0 };


#ifdef SQLITE_UNIX_THREADS
/*
** This variable records whether or not threads can override each others
** locks.
**
**    0:  No.  Threads cannot override each others locks.
**    1:  Yes.  Threads can override each others locks.
**   -1:  We don't know yet.
*/
static int threadsOverrideEachOthersLocks = -1;

/*
** This structure holds information passed into individual test
** threads by the testThreadLockingBehavior() routine.
*/
struct threadTestData {
  int fd;                /* File to be locked */
  struct flock lock;     /* The locking operation */
  int result;            /* Result of the locking operation */
};

/*
** The testThreadLockingBehavior() routine launches two separate
** threads on this routine.  This routine attempts to lock a file
** descriptor then returns.  The success or failure of that attempt
** allows the testThreadLockingBehavior() procedure to determine
** whether or not threads can override each others locks.
*/
static void *threadLockingTest(void *pArg){
  struct threadTestData *pData = (struct threadTestData*)pArg;
  pData->result = fcntl(pData->fd, F_SETLK, &pData->lock);
  return pArg;
}

/*
** This procedure attempts to determine whether or not threads
** can override each others locks then sets the 
** threadsOverrideEachOthersLocks variable appropriately.
*/
static void testThreadLockingBehavior(fd_orig){
  int fd;
  struct threadTestData d[2];
  pthread_t t[2];

  fd = dup(fd_orig);
  if( fd<0 ) return;
  memset(d, 0, sizeof(d));
  d[0].fd = fd;
  d[0].lock.l_type = F_RDLCK;
  d[0].lock.l_len = 1;
  d[0].lock.l_start = 0;
  d[0].lock.l_whence = SEEK_SET;
  d[1] = d[0];
  d[1].lock.l_type = F_WRLCK;
  pthread_create(&t[0], 0, threadLockingTest, &d[0]);
  pthread_create(&t[1], 0, threadLockingTest, &d[1]);
  pthread_join(t[0], 0);
  pthread_join(t[1], 0);
  close(fd);
  threadsOverrideEachOthersLocks =  d[0].result==0 && d[1].result==0;
}
#endif /* SQLITE_UNIX_THREADS */

/*
** Release a lockInfo structure previously allocated by findLockInfo().
*/
static void releaseLockInfo(struct lockInfo *pLock){
  pLock->nRef--;
  if( pLock->nRef==0 ){
    sqlite3HashInsert(&lockHash, &pLock->key, sizeof(pLock->key), 0);

** return values might be unset if an error occurs.
**
** Return the number of errors.
*/
static int findLockInfo(
  int fd,                      /* The file descriptor used in the key */
  struct lockInfo **ppLock,    /* Return the lockInfo structure here */
  struct openCnt **ppOpen      /* Return the openCnt structure here */
){
  int rc;
  struct lockKey key1;
  struct openKey key2;
  struct stat statbuf;
  struct lockInfo *pLock;
  struct openCnt *pOpen;
  rc = fstat(fd, &statbuf);
  if( rc!=0 ) return 1;
  memset(&key1, 0, sizeof(key1));
  key1.dev = statbuf.st_dev;
  key1.ino = statbuf.st_ino;
#ifdef SQLITE_UNIX_THREADS
  if( threadsOverrideEachOthersLocks<0 ){
    testThreadLockingBehavior(fd);
  }
  key1.tid = threadsOverrideEachOthersLocks ? 0 : pthread_self();
#endif
  memset(&key2, 0, sizeof(key2));
  key2.dev = statbuf.st_dev;
  key2.ino = statbuf.st_ino;
  pLock = (struct lockInfo*)sqlite3HashFind(&lockHash, &key1, sizeof(key1));
  if( pLock==0 ){
    struct lockInfo *pOld;
    pLock = sqliteMallocRaw( sizeof(*pLock) );

/*
** Come here to die.
*/
static void Exit(int rc){
  exit(rc);
}

extern char *sqlite3_mprintf(const char *zFormat, ...);
extern char *sqlite3_vmprintf(const char *zFormat, va_list);

/*
** When a lock occurs, yield.
*/
static int db_is_locked(void *NotUsed, int iCount){
  /* sched_yield(); */
  if( verbose ) printf("BUSY %s #%d\n", (char*)NotUsed, iCount);
  usleep(100);
  return iCount<25;
}

/*
** Used to accumulate query results by db_query()
*/
struct QueryResult {
  const char *zFile;  /* Filename - used for error reporting */

      fprintf(stdout,"%s: malloc failed\n", pResult->zFile);
      return 1;
    }
  }
  if( azArg==0 ) return 0;
  for(i=0; i<nArg; i++){
    pResult->azElem[pResult->nElem++] =
        sqlite3_mprintf("%s",azArg[i] ? azArg[i] : ""); 
  }
  return 0;
}

/*
** Execute a query against the database.  NULL values are returned
** as an empty string.  The list is terminated by a single NULL pointer.
*/
char **db_query(sqlite *db, const char *zFile, const char *zFormat, ...){
  char *zSql;
  int rc;
  char *zErrMsg = 0;
  va_list ap;
  struct QueryResult sResult;
  va_start(ap, zFormat);
  zSql = sqlite3_vmprintf(zFormat, ap);
  va_end(ap);
  memset(&sResult, 0, sizeof(sResult));
  sResult.zFile = zFile;
  if( verbose ) printf("QUERY %s: %s\n", zFile, zSql);
  rc = sqlite3_exec(db, zSql, db_query_callback, &sResult, &zErrMsg);
  if( rc==SQLITE_SCHEMA ){
    if( zErrMsg ) free(zErrMsg);
    rc = sqlite3_exec(db, zSql, db_query_callback, &sResult, &zErrMsg);
  }
  if( verbose ) printf("DONE %s %s\n", zFile, zSql);
  if( zErrMsg ){
    fprintf(stdout,"%s: query failed: %s - %s\n", zFile, zSql, zErrMsg);
    free(zErrMsg);
    free(zSql);
    Exit(1);
  }
  sqlite3_free(zSql);
  if( sResult.azElem==0 ){
    db_query_callback(&sResult, 0, 0, 0);
  }
  sResult.azElem[sResult.nElem] = 0;
  return sResult.azElem;
}

/*
** Execute an SQL statement.
*/
void db_execute(sqlite *db, const char *zFile, const char *zFormat, ...){
  char *zSql;
  int rc;
  char *zErrMsg = 0;
  va_list ap;
  va_start(ap, zFormat);
  zSql = sqlite3_vmprintf(zFormat, ap);
  va_end(ap);
  if( verbose ) printf("EXEC %s: %s\n", zFile, zSql);
  do{
    rc = sqlite3_exec(db, zSql, 0, 0, &zErrMsg);
  }while( rc==SQLITE_BUSY );



  if( verbose ) printf("DONE %s: %s\n", zFile, zSql);
  if( zErrMsg ){
    fprintf(stdout,"%s: command failed: %s - %s\n", zFile, zSql, zErrMsg);
    free(zErrMsg);
    sqlite3_free(zSql);
    Exit(1);
  }
  sqlite3_free(zSql);
}

/*
** Free the results of a db_query() call.
*/
void db_query_free(char **az){
  int i;
  for(i=0; az[i]; i++){
    sqlite3_free(az[i]);
  }
  free(az);
}

/*
** Check results
*/

  pthread_mutex_lock(&lock);
  thread_cnt++;
  pthread_mutex_unlock(&lock);
  printf("%s: START\n", zFilename);
  fflush(stdout);
  for(cnt=0; cnt<10; cnt++){
    sqlite3_open(&zFilename[2], &db);
    if( db==0 ){
      fprintf(stdout,"%s: can't open\n", zFilename);
      Exit(1);
    }
    sqlite3_busy_handler(db, db_is_locked, zFilename);
    db_execute(db, zFilename, "CREATE TABLE t%d(a,b,c);", t);
    for(i=1; i<=100; i++){
      db_execute(db, zFilename, "INSERT INTO t%d VALUES(%d,%d,%d);",
         t, i, i*2, i*i);
    }
    az = db_query(db, zFilename, "SELECT count(*) FROM t%d", t);
    db_check(zFilename, "tX size", az, "100", 0);  
    az = db_query(db, zFilename, "SELECT avg(b) FROM t%d", t);
    db_check(zFilename, "tX avg", az, "101", 0);  
    db_execute(db, zFilename, "DELETE FROM t%d WHERE a>50", t);
    az = db_query(db, zFilename, "SELECT avg(b) FROM t%d", t);
    db_check(zFilename, "tX avg2", az, "51", 0);
    for(i=1; i<=50; i++){
      char z1[30], z2[30];
      az = db_query(db, zFilename, "SELECT b, c FROM t%d WHERE a=%d", t, i);
      sprintf(z1, "%d", i*2);
      sprintf(z2, "%d", i*i);
      db_check(zFilename, "readback", az, z1, z2, 0);
    }
    db_execute(db, zFilename, "DROP TABLE t%d;", t);
    sqlite3_close(db);
  }
  printf("%s: END\n", zFilename);
  /* unlink(zFilename); */
  fflush(stdout);
  pthread_mutex_lock(&lock);
  thread_cnt--;
  if( thread_cnt<=0 ){

  if( argc<2 || (n=atoi(argv[1]))<1 ) n = 10;
  for(i=0; i<n; i++){
    char zBuf[200];
    sprintf(zBuf, "testdb-%d", (i+1)/2);
    unlink(zBuf);
  }
  for(i=0; i<n; i++){
    zFile = sqlite3_mprintf("%d.testdb-%d", i%2+1, (i+2)/2);
    unlink(zFile);
    pthread_create(&id, 0, worker_bee, (void*)zFile);
    pthread_detach(id);
  }
  pthread_mutex_lock(&lock);
  while( thread_cnt>0 ){
    pthread_cond_wait(&sig, &lock);

** Do an integrity check on the database.  If the first integrity check
** fails, try it a second time.
*/
int integrity_check(sqlite *db){
  int rc;
  if( all_stop ) return 0;
  /* fprintf(stderr,"pid=%d: CHECK\n", getpid()); */
  rc = sqlite3_exec(db, "pragma integrity_check", check_callback, 0, 0);
  if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){
    fprintf(stderr,"pid=%d, Integrity check returns %d\n", getpid(), rc);
  }
  if( all_stop ){
    sqlite3_exec(db, "pragma integrity_check", check_callback, 0, 0);
  }
  return 0;
}

/*
** This is the worker thread
*/
void *worker(void *notUsed){
  sqlite *db;
  int rc;
  int cnt = 0;
  while( !all_stop && cnt++<10000 ){
    if( cnt%1000==0 ) printf("pid=%d: %d\n", getpid(), cnt);
    while( (sqlite3_open(DB_FILE, &db))!=SQLITE_OK ) sched_yield();
    sqlite3_exec(db, "PRAGMA synchronous=OFF", 0, 0, 0);
    integrity_check(db);
    if( all_stop ){ sqlite3_close(db); break; }
    /* fprintf(stderr, "pid=%d: BEGIN\n", getpid()); */
    rc = sqlite3_exec(db, "INSERT INTO t1 VALUES('bogus data')", 0, 0, 0);
    /* fprintf(stderr, "pid=%d: END rc=%d\n", getpid(), rc); */
    sqlite3_close(db);
  }
  return 0;
}

/*
** Initialize the database and start the threads
*/
int main(int argc, char **argv){
  sqlite *db;
  int i, rc;
  pthread_t aThread[5];

  if( strcmp(DB_FILE,":memory:") ) unlink(DB_FILE);
  sqlite3_open(DB_FILE, &db);
  if( db==0 ){
    fprintf(stderr,"unable to initialize database\n");
    exit(1);
  }
  rc = sqlite3_exec(db, "CREATE TABLE t1(x);", 0,0,0);
  if( rc ){
    fprintf(stderr,"cannot create table t1: %d\n", rc);
    exit(1);
  }
  sqlite3_close(db);
  for(i=0; i<sizeof(aThread)/sizeof(aThread[0]); i++){
    pthread_create(&aThread[i], 0, worker, 0);
  }
  for(i=0; i<sizeof(aThread)/sizeof(aThread[i]); i++){
    pthread_join(aThread[i], 0);
  }
  if( !all_stop ){