SQLite

Check-in [cb9302cca4]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Defer opening and writing statement journals until the size reaches a threshold (currently 64KiB).
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: cb9302cca423de41305719a49208daa392ec09da
User & Date: drh 2016-03-04 14:43:44.681
Context
2016-03-04
21:18
Fix an assert() in sqlite3VarintLen(), even though it is impossible to hit in SQLite due to the way sqlite3VarintLen() is used. (check-in: 251424c586 user: drh tags: trunk)
18:45
Merge changes from trunk. (check-in: 5294c977d9 user: drh tags: analyze-worst-case)
16:42
Merge recent enhancements from trunk. Default page size is 4096. Writes to statement journals are avoided. (check-in: 456df3365e user: drh tags: sessions)
14:57
Merge recent enhancements from trunk, and especially the changes that reduce the heap-memory footprint of schemas, and defer opening and writing to statement journals. (check-in: 2f0c195ccc user: drh tags: apple-osx)
14:43
Defer opening and writing statement journals until the size reaches a threshold (currently 64KiB). (check-in: cb9302cca4 user: drh tags: trunk)
14:23
Update test cases to taken deferred statement-journal opening into account. (Closed-Leaf check-in: 5b2fe5219a user: drh tags: memjournal-exp)
04:01
Change the default cache_size to -2000 (which means 2000*1024 bytes independent of page_size). (check-in: 2682e8e413 user: drh tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/journal.c.
20
21
22
23
24
25
26

27
28
29
30
31
32
33
** be used to service read() and write() requests. The actual file
** on disk is not created or populated until either:
**
**   1) The in-memory representation grows too large for the allocated 
**      buffer, or
**   2) The sqlite3JournalCreate() function is called.
*/

#ifdef SQLITE_ENABLE_ATOMIC_WRITE
#include "sqliteInt.h"


/*
** A JournalFile object is a subclass of sqlite3_file used by
** as an open file handle for journal files.







>







20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
** be used to service read() and write() requests. The actual file
** on disk is not created or populated until either:
**
**   1) The in-memory representation grows too large for the allocated 
**      buffer, or
**   2) The sqlite3JournalCreate() function is called.
*/
#if 0 
#ifdef SQLITE_ENABLE_ATOMIC_WRITE
#include "sqliteInt.h"


/*
** A JournalFile object is a subclass of sqlite3_file used by
** as an open file handle for journal files.
249
250
251
252
253
254
255
256

/* 
** Return the number of bytes required to store a JournalFile that uses vfs
** pVfs to create the underlying on-disk files.
*/
int sqlite3JournalSize(sqlite3_vfs *pVfs){
  return (pVfs->szOsFile+sizeof(JournalFile));
}
#endif









>
250
251
252
253
254
255
256
257
258
/* 
** Return the number of bytes required to store a JournalFile that uses vfs
** pVfs to create the underlying on-disk files.
*/
int sqlite3JournalSize(sqlite3_vfs *pVfs){
  return (pVfs->szOsFile+sizeof(JournalFile));
}
#endif
#endif
Changes to src/memjournal.c.
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35



36
37
38
39
40
41











42
43
44
45
46
47
48
49
50
51
52
53
54
55
56




57
58
59





60
61
62
63
64
65
66
67
68
69
70
71
72





73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103

104
105
106




















































107
108
109
110
111
112
113
114
115
116
117
118
119
120
















121
122
123



124
125
126


127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152



153
154
155
156
157
158
159




160
161
162



163
164
165
166
167
168
169
170

171
172
173
174
175
176
177
178
179

180

181
182
183
184
185
186
187

188
189
190
191
192
193


194

195
196
197
198
199
200
201
202



203
204
205
206
207
208
209
#include "sqliteInt.h"

/* Forward references to internal structures */
typedef struct MemJournal MemJournal;
typedef struct FilePoint FilePoint;
typedef struct FileChunk FileChunk;

/* Space to hold the rollback journal is allocated in increments of
** this many bytes.
**
** The size chosen is a little less than a power of two.  That way,
** the FileChunk object will have a size that almost exactly fills
** a power-of-two allocation.  This minimizes wasted space in power-of-two
** memory allocators.
*/
#define JOURNAL_CHUNKSIZE ((int)(1024-sizeof(FileChunk*)))

/*
** The rollback journal is composed of a linked list of these structures.



*/
struct FileChunk {
  FileChunk *pNext;               /* Next chunk in the journal */
  u8 zChunk[JOURNAL_CHUNKSIZE];   /* Content of this chunk */
};












/*
** An instance of this object serves as a cursor into the rollback journal.
** The cursor can be either for reading or writing.
*/
struct FilePoint {
  sqlite3_int64 iOffset;          /* Offset from the beginning of the file */
  FileChunk *pChunk;              /* Specific chunk into which cursor points */
};

/*
** This subclass is a subclass of sqlite3_file.  Each open memory-journal
** is an instance of this class.
*/
struct MemJournal {
  sqlite3_io_methods *pMethod;    /* Parent class. MUST BE FIRST */




  FileChunk *pFirst;              /* Head of in-memory chunk-list */
  FilePoint endpoint;             /* Pointer to the end of the file */
  FilePoint readpoint;            /* Pointer to the end of the last xRead() */





};

/*
** Read data from the in-memory journal file.  This is the implementation
** of the sqlite3_vfs.xRead method.
*/
static int memjrnlRead(
  sqlite3_file *pJfd,    /* The journal file from which to read */
  void *zBuf,            /* Put the results here */
  int iAmt,              /* Number of bytes to read */
  sqlite_int64 iOfst     /* Begin reading at this offset */
){
  MemJournal *p = (MemJournal *)pJfd;





  u8 *zOut = zBuf;
  int nRead = iAmt;
  int iChunkOffset;
  FileChunk *pChunk;

  /* SQLite never tries to read past the end of a rollback journal file */
  assert( iOfst+iAmt<=p->endpoint.iOffset );

  if( p->readpoint.iOffset!=iOfst || iOfst==0 ){
    sqlite3_int64 iOff = 0;
    for(pChunk=p->pFirst; 
        ALWAYS(pChunk) && (iOff+JOURNAL_CHUNKSIZE)<=iOfst;
        pChunk=pChunk->pNext
    ){
      iOff += JOURNAL_CHUNKSIZE;
    }
  }else{
    pChunk = p->readpoint.pChunk;
  }

  iChunkOffset = (int)(iOfst%JOURNAL_CHUNKSIZE);
  do {
    int iSpace = JOURNAL_CHUNKSIZE - iChunkOffset;
    int nCopy = MIN(nRead, (JOURNAL_CHUNKSIZE - iChunkOffset));
    memcpy(zOut, &pChunk->zChunk[iChunkOffset], nCopy);
    zOut += nCopy;
    nRead -= iSpace;
    iChunkOffset = 0;
  } while( nRead>=0 && (pChunk=pChunk->pNext)!=0 && nRead>0 );
  p->readpoint.iOffset = iOfst+iAmt;
  p->readpoint.pChunk = pChunk;


  return SQLITE_OK;
}





















































/*
** Write data to the file.
*/
static int memjrnlWrite(
  sqlite3_file *pJfd,    /* The journal file into which to write */
  const void *zBuf,      /* Take data to be written from here */
  int iAmt,              /* Number of bytes to write */
  sqlite_int64 iOfst     /* Begin writing at this offset into the file */
){
  MemJournal *p = (MemJournal *)pJfd;
  int nWrite = iAmt;
  u8 *zWrite = (u8 *)zBuf;

















  /* An in-memory journal file should only ever be appended to. Random
  ** access writes are not required by sqlite.
  */



  assert( iOfst==p->endpoint.iOffset );
  UNUSED_PARAMETER(iOfst);



  while( nWrite>0 ){
    FileChunk *pChunk = p->endpoint.pChunk;
    int iChunkOffset = (int)(p->endpoint.iOffset%JOURNAL_CHUNKSIZE);
    int iSpace = MIN(nWrite, JOURNAL_CHUNKSIZE - iChunkOffset);

    if( iChunkOffset==0 ){
      /* New chunk is required to extend the file. */
      FileChunk *pNew = sqlite3_malloc(sizeof(FileChunk));
      if( !pNew ){
        return SQLITE_IOERR_NOMEM_BKPT;
      }
      pNew->pNext = 0;
      if( pChunk ){
        assert( p->pFirst );
        pChunk->pNext = pNew;
      }else{
        assert( !p->pFirst );
        p->pFirst = pNew;
      }
      p->endpoint.pChunk = pNew;
    }

    memcpy(&p->endpoint.pChunk->zChunk[iChunkOffset], zWrite, iSpace);
    zWrite += iSpace;
    nWrite -= iSpace;
    p->endpoint.iOffset += iSpace;



  }

  return SQLITE_OK;
}

/*
** Truncate the file.




*/
static int memjrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){
  MemJournal *p = (MemJournal *)pJfd;



  FileChunk *pChunk;
  assert(size==0);
  UNUSED_PARAMETER(size);
  pChunk = p->pFirst;
  while( pChunk ){
    FileChunk *pTmp = pChunk;
    pChunk = pChunk->pNext;
    sqlite3_free(pTmp);

  }
  sqlite3MemJournalOpen(pJfd);
  return SQLITE_OK;
}

/*
** Close the file.
*/
static int memjrnlClose(sqlite3_file *pJfd){

  memjrnlTruncate(pJfd, 0);

  return SQLITE_OK;
}


/*
** Sync the file.
**

** Syncing an in-memory journal is a no-op.  And, in fact, this routine
** is never called in a working implementation.  This implementation
** exists purely as a contingency, in case some malfunction in some other
** part of SQLite causes Sync to be called by mistake.
*/
static int memjrnlSync(sqlite3_file *NotUsed, int NotUsed2){


  UNUSED_PARAMETER2(NotUsed, NotUsed2);

  return SQLITE_OK;
}

/*
** Query the size of the file in bytes.
*/
static int memjrnlFileSize(sqlite3_file *pJfd, sqlite_int64 *pSize){
  MemJournal *p = (MemJournal *)pJfd;



  *pSize = (sqlite_int64) p->endpoint.iOffset;
  return SQLITE_OK;
}

/*
** Table of methods for MemJournal sqlite3_file object.
*/







<
<
<
<
<
<
<
<
<
<


>
>
>



|


>
>
>
>
>
>
>
>
>
>
>










|



|
>
>
>
>



>
>
>
>
>













>
>
>
>
>
|
|
|
|

<
<
<
|
|
|
|
|
|
|
|
|
|
|

|
|
|
|
|
|
|
|
|
|
|
>



>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>














>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
|
<
>
>
>
|
|
|
>
>
|
|
|
|

|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|

|
|
|
|
>
>
>







>
>
>
>



>
>
>
|
|
<
|
<
|
|
<
>

<







>
|
>


<




>
|
<
<
<

|
>
>
|
>








>
>
>







17
18
19
20
21
22
23










24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95



96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206

207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262

263

264
265

266
267

268
269
270
271
272
273
274
275
276
277
278
279

280
281
282
283
284
285



286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
#include "sqliteInt.h"

/* Forward references to internal structures */
typedef struct MemJournal MemJournal;
typedef struct FilePoint FilePoint;
typedef struct FileChunk FileChunk;











/*
** The rollback journal is composed of a linked list of these structures.
**
** The zChunk array is always at least 8 bytes in size - usually much more.
** Its actual size is stored in the MemJournal.nChunkSize variable.
*/
struct FileChunk {
  FileChunk *pNext;               /* Next chunk in the journal */
  u8 zChunk[8];                   /* Content of this chunk */
};

/*
** By default, allocate this many bytes of memory for each FileChunk object.
*/
#define MEMJOURNAL_DFLT_FILECHUNKSIZE 1024

/*
** For chunk size nChunkSize, return the number of bytes that should
** be allocated for each FileChunk structure.
*/
#define fileChunkSize(nChunkSize) (sizeof(FileChunk) + ((nChunkSize)-8))

/*
** An instance of this object serves as a cursor into the rollback journal.
** The cursor can be either for reading or writing.
*/
struct FilePoint {
  sqlite3_int64 iOffset;          /* Offset from the beginning of the file */
  FileChunk *pChunk;              /* Specific chunk into which cursor points */
};

/*
** This structure is a subclass of sqlite3_file. Each open memory-journal
** is an instance of this class.
*/
struct MemJournal {
  const sqlite3_io_methods *pMethod; /* Parent class. MUST BE FIRST */
  int nChunkSize;                 /* In-memory chunk-size */

  int nBuf;                       /* Bytes of data before flushing */
  int nSize;                      /* Bytes of data currently in memory */
  FileChunk *pFirst;              /* Head of in-memory chunk-list */
  FilePoint endpoint;             /* Pointer to the end of the file */
  FilePoint readpoint;            /* Pointer to the end of the last xRead() */

  int flags;                      /* xOpen flags */
  sqlite3_vfs *pVfs;              /* The "real" underlying VFS */
  const char *zJournal;           /* Name of the journal file */
  sqlite3_file *pReal;            /* The "real" underlying file descriptor */
};

/*
** Read data from the in-memory journal file.  This is the implementation
** of the sqlite3_vfs.xRead method.
*/
static int memjrnlRead(
  sqlite3_file *pJfd,    /* The journal file from which to read */
  void *zBuf,            /* Put the results here */
  int iAmt,              /* Number of bytes to read */
  sqlite_int64 iOfst     /* Begin reading at this offset */
){
  MemJournal *p = (MemJournal *)pJfd;
  if( p->pReal ){
    return sqlite3OsRead(p->pReal, zBuf, iAmt, iOfst);
  }else if( (iAmt+iOfst)>p->endpoint.iOffset ){
    return SQLITE_IOERR_SHORT_READ;
  }else{
    u8 *zOut = zBuf;
    int nRead = iAmt;
    int iChunkOffset;
    FileChunk *pChunk;




    if( p->readpoint.iOffset!=iOfst || iOfst==0 ){
      sqlite3_int64 iOff = 0;
      for(pChunk=p->pFirst; 
          ALWAYS(pChunk) && (iOff+p->nChunkSize)<=iOfst;
          pChunk=pChunk->pNext
      ){
        iOff += p->nChunkSize;
      }
    }else{
      pChunk = p->readpoint.pChunk;
    }

    iChunkOffset = (int)(iOfst%p->nChunkSize);
    do {
      int iSpace = p->nChunkSize - iChunkOffset;
      int nCopy = MIN(nRead, (p->nChunkSize - iChunkOffset));
      memcpy(zOut, &pChunk->zChunk[iChunkOffset], nCopy);
      zOut += nCopy;
      nRead -= iSpace;
      iChunkOffset = 0;
    } while( nRead>=0 && (pChunk=pChunk->pNext)!=0 && nRead>0 );
    p->readpoint.iOffset = iOfst+iAmt;
    p->readpoint.pChunk = pChunk;
  }

  return SQLITE_OK;
}

/*
** Free the list of FileChunk structures headed at MemJournal.pFirst.
*/
static void memjrnlFreeChunks(MemJournal *p){
  FileChunk *pIter;
  FileChunk *pNext;
  for(pIter=p->pFirst; pIter; pIter=pNext){
    pNext = pIter->pNext;
    sqlite3_free(pIter);
  } 
  p->pFirst = 0;
}

/*
** Flush the contents of memory to a real file on disk.
*/
static int createFile(MemJournal *p){
  int rc = SQLITE_OK;
  if( !p->pReal ){
    sqlite3_file *pReal = (sqlite3_file *)&p[1];
    rc = sqlite3OsOpen(p->pVfs, p->zJournal, pReal, p->flags, 0);
    if( rc==SQLITE_OK ){
      int nChunk = p->nChunkSize;
      i64 iOff = 0;
      FileChunk *pIter;
      p->pReal = pReal;
      for(pIter=p->pFirst; pIter && rc==SQLITE_OK; pIter=pIter->pNext){
        int nWrite = nChunk;
        if( pIter==p->endpoint.pChunk ){
          nWrite = p->endpoint.iOffset % p->nChunkSize;
          if( nWrite==0 ) nWrite = p->nChunkSize;
        }
        rc = sqlite3OsWrite(pReal, pIter->zChunk, nWrite, iOff);
        iOff += nWrite;
      }
      if( rc!=SQLITE_OK ){
        /* If an error occurred while writing to the file, close it before
        ** returning. This way, SQLite uses the in-memory journal data to 
        ** roll back changes made to the internal page-cache before this
        ** function was called.  */
        sqlite3OsClose(pReal);
        p->pReal = 0;
      }else{
        /* No error has occurred. Free the in-memory buffers. */
        memjrnlFreeChunks(p);
      }
    }
  }
  return rc;
}


/*
** Write data to the file.
*/
static int memjrnlWrite(
  sqlite3_file *pJfd,    /* The journal file into which to write */
  const void *zBuf,      /* Take data to be written from here */
  int iAmt,              /* Number of bytes to write */
  sqlite_int64 iOfst     /* Begin writing at this offset into the file */
){
  MemJournal *p = (MemJournal *)pJfd;
  int nWrite = iAmt;
  u8 *zWrite = (u8 *)zBuf;

  /* If the file has already been created on disk. */
  if( p->pReal ){
    return sqlite3OsWrite(p->pReal, zBuf, iAmt, iOfst);
  }

  /* If the file should be created now. */
  else if( p->nBuf>0 && (iAmt+iOfst)>p->nBuf ){
    int rc = createFile(p);
    if( rc==SQLITE_OK ){
      rc = memjrnlWrite(pJfd, zBuf, iAmt, iOfst);
    }
    return rc;
  }

  /* If the contents of this write should be stored in memory */
  else{
    /* An in-memory journal file should only ever be appended to. Random
    ** access writes are not required. The only exception to this is when

    ** the in-memory journal is being used by a connection using the
    ** atomic-write optimization. In this case the first 28 bytes of the
    ** journal file may be written as part of committing the transaction. */ 
    assert( iOfst==p->endpoint.iOffset || iOfst==0 );
    if( iOfst==0 && p->pFirst ){
      assert( p->nChunkSize>iAmt );
      memcpy(p->pFirst->zChunk, zBuf, iAmt);
    }else{
      while( nWrite>0 ){
        FileChunk *pChunk = p->endpoint.pChunk;
        int iChunkOffset = (int)(p->endpoint.iOffset%p->nChunkSize);
        int iSpace = MIN(nWrite, p->nChunkSize - iChunkOffset);

        if( iChunkOffset==0 ){
          /* New chunk is required to extend the file. */
          FileChunk *pNew = sqlite3_malloc(fileChunkSize(p->nChunkSize));
          if( !pNew ){
            return SQLITE_IOERR_NOMEM_BKPT;
          }
          pNew->pNext = 0;
          if( pChunk ){
            assert( p->pFirst );
            pChunk->pNext = pNew;
          }else{
            assert( !p->pFirst );
            p->pFirst = pNew;
          }
          p->endpoint.pChunk = pNew;
        }

        memcpy(&p->endpoint.pChunk->zChunk[iChunkOffset], zWrite, iSpace);
        zWrite += iSpace;
        nWrite -= iSpace;
        p->endpoint.iOffset += iSpace;
      }
      p->nSize = iAmt + iOfst;
    }
  }

  return SQLITE_OK;
}

/*
** Truncate the file.
**
** If the journal file is already on disk, truncate it there. Or, if it
** is still in main memory but is being truncated to zero bytes in size,
** ignore 
*/
static int memjrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){
  MemJournal *p = (MemJournal *)pJfd;
  if( p->pReal ){
    return sqlite3OsTruncate(p->pReal, size);
  }else if( size==0 ){
    memjrnlFreeChunks(p);
    p->nSize = 0;

    p->endpoint.pChunk = 0;

    p->endpoint.iOffset = 0;
    p->readpoint.pChunk = 0;

    p->readpoint.iOffset = 0;
  }

  return SQLITE_OK;
}

/*
** Close the file.
*/
static int memjrnlClose(sqlite3_file *pJfd){
  MemJournal *p = (MemJournal *)pJfd;
  memjrnlFreeChunks(p);
  if( p->pReal ) sqlite3OsClose(p->pReal);
  return SQLITE_OK;
}


/*
** Sync the file.
**
** If the real file has been created, call its xSync method. Otherwise, 
** syncing an in-memory journal is a no-op. 



*/
static int memjrnlSync(sqlite3_file *pJfd, int flags){
  MemJournal *p = (MemJournal *)pJfd;
  if( p->pReal ){
    return sqlite3OsSync(p->pReal, flags);
  }
  return SQLITE_OK;
}

/*
** Query the size of the file in bytes.
*/
static int memjrnlFileSize(sqlite3_file *pJfd, sqlite_int64 *pSize){
  MemJournal *p = (MemJournal *)pJfd;
  if( p->pReal ){
    return sqlite3OsFileSize(p->pReal, pSize);
  }
  *pSize = (sqlite_int64) p->endpoint.iOffset;
  return SQLITE_OK;
}

/*
** Table of methods for MemJournal sqlite3_file object.
*/
226
227
228
229
230
231
232
233









234
235






236
237




238



239





240







241

242
243
244
245
246












247

248

249


250









251

252
253
254
255
  0,                /* xShmBarrier */
  0,                /* xShmUnmap */
  0,                /* xFetch */
  0                 /* xUnfetch */
};

/* 
** Open a journal file.









*/
void sqlite3MemJournalOpen(sqlite3_file *pJfd){






  MemJournal *p = (MemJournal *)pJfd;
  assert( EIGHT_BYTE_ALIGNMENT(p) );




  memset(p, 0, sqlite3MemJournalSize());



  p->pMethod = (sqlite3_io_methods*)&MemJournalMethods;





}









/*
** Return true if the file-handle passed as an argument is 
** an in-memory journal 
*/
int sqlite3IsMemJournal(sqlite3_file *pJfd){












  return pJfd->pMethods==&MemJournalMethods;

}




/* 









** Return the number of bytes required to store a MemJournal file descriptor.

*/
int sqlite3MemJournalSize(void){
  return sizeof(MemJournal);
}







|
>
>
>
>
>
>
>
>
>

|
>
>
>
>
>
>
|
|
>
>
>
>
|
>
>
>
|
>
>
>
>
>
|
>
>
>
>
>
>
>
|
>

<
|

|
>
>
>
>
>
>
>
>
>
>
>
>
|
>
|
>
|
>
>
|
>
>
>
>
>
>
>
>
>
|
>

|
|

326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377

378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
  0,                /* xShmBarrier */
  0,                /* xShmUnmap */
  0,                /* xFetch */
  0                 /* xUnfetch */
};

/* 
** Open a journal file. 
**
** The behaviour of the journal file depends on the value of parameter 
** nBuf. If nBuf is 0, then the journal file is always create and 
** accessed using the underlying VFS. If nBuf is less than zero, then
** all content is always stored in main-memory. Finally, if nBuf is a
** positive value, then the journal file is initially created in-memory
** but may be flushed to disk later on. In this case the journal file is
** flushed to disk either when it grows larger than nBuf bytes in size,
** or when sqlite3JournalCreate() is called.
*/
int sqlite3JournalOpen(
  sqlite3_vfs *pVfs,         /* The VFS to use for actual file I/O */
  const char *zName,         /* Name of the journal file */
  sqlite3_file *pJfd,        /* Preallocated, blank file handle */
  int flags,                 /* Opening flags */
  int nBuf                   /* Bytes buffered before opening the file */
){
  MemJournal *p = (MemJournal*)pJfd;

  /* Zero the file-handle object. If nBuf was passed zero, initialize
  ** it using the sqlite3OsOpen() function of the underlying VFS. In this
  ** case none of the code in this module is executed as a result of calls
  ** made on the journal file-handle.  */
  memset(p, 0, sizeof(MemJournal) + (pVfs ? pVfs->szOsFile : 0));
  if( nBuf==0 ){
    return sqlite3OsOpen(pVfs, zName, pJfd, flags, 0);
  }

  if( nBuf>0 ){
    p->nChunkSize = nBuf;
  }else{
    p->nChunkSize = 8 + MEMJOURNAL_DFLT_FILECHUNKSIZE - sizeof(FileChunk);
    assert( MEMJOURNAL_DFLT_FILECHUNKSIZE==fileChunkSize(p->nChunkSize) );
  }

  p->pMethod = (const sqlite3_io_methods*)&MemJournalMethods;
  p->nBuf = nBuf;
  p->flags = flags;
  p->zJournal = zName;
  p->pVfs = pVfs;
  return SQLITE_OK;
}

/*

** Open an in-memory journal file.
*/
void sqlite3MemJournalOpen(sqlite3_file *pJfd){
  sqlite3JournalOpen(0, 0, pJfd, 0, -1);
}

#ifdef SQLITE_ENABLE_ATOMIC_WRITE
/*
** If the argument p points to a MemJournal structure that is not an 
** in-memory-only journal file (i.e. is one that was opened with a +ve
** nBuf parameter), and the underlying file has not yet been created, 
** create it now.
*/
int sqlite3JournalCreate(sqlite3_file *p){
  int rc = SQLITE_OK;
  if( p->pMethods==&MemJournalMethods && ((MemJournal*)p)->nBuf>0 ){
    rc = createFile((MemJournal*)p);
  }
  return rc;
}
#endif

/*
** The file-handle passed as the only argument is open on a journal file.
** Return true if this "journal file" is currently stored in heap memory,
** or false otherwise.
*/
int sqlite3JournalIsInMemory(sqlite3_file *p){
  return p->pMethods==&MemJournalMethods && ((MemJournal*)p)->pReal==0;
}

/* 
** Return the number of bytes required to store a JournalFile that uses vfs
** pVfs to create the underlying on-disk files.
*/
int sqlite3JournalSize(sqlite3_vfs *pVfs){
  return pVfs->szOsFile + sizeof(MemJournal);
}
Changes to src/os.c.
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
**     sqlite3OsAccess()
**     sqlite3OsFullPathname()
**
*/
#if defined(SQLITE_TEST)
int sqlite3_memdebug_vfs_oom_test = 1;
  #define DO_OS_MALLOC_TEST(x)                                       \
  if (sqlite3_memdebug_vfs_oom_test && (!x || !sqlite3IsMemJournal(x))) {  \
    void *pTstAlloc = sqlite3Malloc(10);                             \
    if (!pTstAlloc) return SQLITE_IOERR_NOMEM_BKPT;                  \
    sqlite3_free(pTstAlloc);                                         \
  }
#else
  #define DO_OS_MALLOC_TEST(x)
#endif







|







62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
**     sqlite3OsAccess()
**     sqlite3OsFullPathname()
**
*/
#if defined(SQLITE_TEST)
int sqlite3_memdebug_vfs_oom_test = 1;
  #define DO_OS_MALLOC_TEST(x)                                       \
  if (sqlite3_memdebug_vfs_oom_test && (!x || !sqlite3JournalIsInMemory(x))) { \
    void *pTstAlloc = sqlite3Malloc(10);                             \
    if (!pTstAlloc) return SQLITE_IOERR_NOMEM_BKPT;                  \
    sqlite3_free(pTstAlloc);                                         \
  }
#else
  #define DO_OS_MALLOC_TEST(x)
#endif
Changes to src/pager.c.
1339
1340
1341
1342
1343
1344
1345

1346
1347
1348
1349
1350
1351
1352
**
** If an IO error occurs, abandon processing and return the IO error code.
** Otherwise, return SQLITE_OK.
*/
static int zeroJournalHdr(Pager *pPager, int doTruncate){
  int rc = SQLITE_OK;                               /* Return code */
  assert( isOpen(pPager->jfd) );

  if( pPager->journalOff ){
    const i64 iLimit = pPager->journalSizeLimit;    /* Local cache of jsl */

    IOTRACE(("JZEROHDR %p\n", pPager))
    if( doTruncate || iLimit==0 ){
      rc = sqlite3OsTruncate(pPager->jfd, 0);
    }else{







>







1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
**
** If an IO error occurs, abandon processing and return the IO error code.
** Otherwise, return SQLITE_OK.
*/
static int zeroJournalHdr(Pager *pPager, int doTruncate){
  int rc = SQLITE_OK;                               /* Return code */
  assert( isOpen(pPager->jfd) );
  assert( !sqlite3JournalIsInMemory(pPager->jfd) );
  if( pPager->journalOff ){
    const i64 iLimit = pPager->journalSizeLimit;    /* Local cache of jsl */

    IOTRACE(("JZEROHDR %p\n", pPager))
    if( doTruncate || iLimit==0 ){
      rc = sqlite3OsTruncate(pPager->jfd, 0);
    }else{
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
** if it is open and the pager is not in exclusive mode.
*/
static void releaseAllSavepoints(Pager *pPager){
  int ii;               /* Iterator for looping through Pager.aSavepoint */
  for(ii=0; ii<pPager->nSavepoint; ii++){
    sqlite3BitvecDestroy(pPager->aSavepoint[ii].pInSavepoint);
  }
  if( !pPager->exclusiveMode || sqlite3IsMemJournal(pPager->sjfd) ){
    sqlite3OsClose(pPager->sjfd);
  }
  sqlite3_free(pPager->aSavepoint);
  pPager->aSavepoint = 0;
  pPager->nSavepoint = 0;
  pPager->nSubRec = 0;
}







|







1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
** if it is open and the pager is not in exclusive mode.
*/
static void releaseAllSavepoints(Pager *pPager){
  int ii;               /* Iterator for looping through Pager.aSavepoint */
  for(ii=0; ii<pPager->nSavepoint; ii++){
    sqlite3BitvecDestroy(pPager->aSavepoint[ii].pInSavepoint);
  }
  if( !pPager->exclusiveMode || sqlite3JournalIsInMemory(pPager->sjfd) ){
    sqlite3OsClose(pPager->sjfd);
  }
  sqlite3_free(pPager->aSavepoint);
  pPager->aSavepoint = 0;
  pPager->nSavepoint = 0;
  pPager->nSubRec = 0;
}
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973

  releaseAllSavepoints(pPager);
  assert( isOpen(pPager->jfd) || pPager->pInJournal==0 );
  if( isOpen(pPager->jfd) ){
    assert( !pagerUseWal(pPager) );

    /* Finalize the journal file. */
    if( sqlite3IsMemJournal(pPager->jfd) ){
      assert( pPager->journalMode==PAGER_JOURNALMODE_MEMORY );
      sqlite3OsClose(pPager->jfd);
    }else if( pPager->journalMode==PAGER_JOURNALMODE_TRUNCATE ){
      if( pPager->journalOff==0 ){
        rc = SQLITE_OK;
      }else{
        rc = sqlite3OsTruncate(pPager->jfd, 0);
        if( rc==SQLITE_OK && pPager->fullSync ){







|
|







1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974

  releaseAllSavepoints(pPager);
  assert( isOpen(pPager->jfd) || pPager->pInJournal==0 );
  if( isOpen(pPager->jfd) ){
    assert( !pagerUseWal(pPager) );

    /* Finalize the journal file. */
    if( sqlite3JournalIsInMemory(pPager->jfd) ){
      /* assert( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ); */
      sqlite3OsClose(pPager->jfd);
    }else if( pPager->journalMode==PAGER_JOURNALMODE_TRUNCATE ){
      if( pPager->journalOff==0 ){
        rc = SQLITE_OK;
      }else{
        rc = sqlite3OsTruncate(pPager->jfd, 0);
        if( rc==SQLITE_OK && pPager->fullSync ){
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994

1995
1996
1997
1998
1999
2000
2001
    ){
      rc = zeroJournalHdr(pPager, hasMaster);
      pPager->journalOff = 0;
    }else{
      /* This branch may be executed with Pager.journalMode==MEMORY if
      ** a hot-journal was just rolled back. In this case the journal
      ** file should be closed and deleted. If this connection writes to
      ** the database file, it will do so using an in-memory journal. 
      */
      int bDelete = (!pPager->tempFile && sqlite3JournalExists(pPager->jfd));

      assert( pPager->journalMode==PAGER_JOURNALMODE_DELETE 
           || pPager->journalMode==PAGER_JOURNALMODE_MEMORY 
           || pPager->journalMode==PAGER_JOURNALMODE_WAL 
      );
      sqlite3OsClose(pPager->jfd);
      if( bDelete ){
        rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, pPager->extraSync);







|

|
>







1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
    ){
      rc = zeroJournalHdr(pPager, hasMaster);
      pPager->journalOff = 0;
    }else{
      /* This branch may be executed with Pager.journalMode==MEMORY if
      ** a hot-journal was just rolled back. In this case the journal
      ** file should be closed and deleted. If this connection writes to
      ** the database file, it will do so using an in-memory journal.
      */
      int bDelete = !pPager->tempFile;
      assert( sqlite3JournalIsInMemory(pPager->jfd)==0 );
      assert( pPager->journalMode==PAGER_JOURNALMODE_DELETE 
           || pPager->journalMode==PAGER_JOURNALMODE_MEMORY 
           || pPager->journalMode==PAGER_JOURNALMODE_WAL 
      );
      sqlite3OsClose(pPager->jfd);
      if( bDelete ){
        rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, pPager->extraSync);
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
  ** If a master journal file name is specified, but the file is not
  ** present on disk, then the journal is not hot and does not need to be
  ** played back.
  **
  ** TODO: Technically the following is an error because it assumes that
  ** buffer Pager.pTmpSpace is (mxPathname+1) bytes or larger. i.e. that
  ** (pPager->pageSize >= pPager->pVfs->mxPathname+1). Using os_unix.c,
  **  mxPathname is 512, which is the same as the minimum allowable value
  ** for pageSize.
  */
  zMaster = pPager->pTmpSpace;
  rc = readMasterJournal(pPager->jfd, zMaster, pPager->pVfs->mxPathname+1);
  if( rc==SQLITE_OK && zMaster[0] ){
    rc = sqlite3OsAccess(pVfs, zMaster, SQLITE_ACCESS_EXISTS, &res);
  }







|







2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
  ** If a master journal file name is specified, but the file is not
  ** present on disk, then the journal is not hot and does not need to be
  ** played back.
  **
  ** TODO: Technically the following is an error because it assumes that
  ** buffer Pager.pTmpSpace is (mxPathname+1) bytes or larger. i.e. that
  ** (pPager->pageSize >= pPager->pVfs->mxPathname+1). Using os_unix.c,
  ** mxPathname is 512, which is the same as the minimum allowable value
  ** for pageSize.
  */
  zMaster = pPager->pTmpSpace;
  rc = readMasterJournal(pPager->jfd, zMaster, pPager->pVfs->mxPathname+1);
  if( rc==SQLITE_OK && zMaster[0] ){
    rc = sqlite3OsAccess(pVfs, zMaster, SQLITE_ACCESS_EXISTS, &res);
  }
4349
4350
4351
4352
4353
4354
4355




4356
4357
4358
4359

4360

4361
4362
4363
4364
4365
4366
4367
** SQLITE_OK is returned if everything goes according to plan. An 
** SQLITE_IOERR_XXX error code is returned if a call to sqlite3OsOpen() 
** fails.
*/
static int openSubJournal(Pager *pPager){
  int rc = SQLITE_OK;
  if( !isOpen(pPager->sjfd) ){




    if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->subjInMemory ){
      sqlite3MemJournalOpen(pPager->sjfd);
    }else{
      rc = pagerOpentemp(pPager, pPager->sjfd, SQLITE_OPEN_SUBJOURNAL);

    }

  }
  return rc;
}

/*
** Append a record of the current state of page pPg to the sub-journal. 
**







>
>
>
>

<
<
<
>

>







4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362



4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
** SQLITE_OK is returned if everything goes according to plan. An 
** SQLITE_IOERR_XXX error code is returned if a call to sqlite3OsOpen() 
** fails.
*/
static int openSubJournal(Pager *pPager){
  int rc = SQLITE_OK;
  if( !isOpen(pPager->sjfd) ){
    const int flags =  SQLITE_OPEN_SUBJOURNAL | SQLITE_OPEN_READWRITE 
      | SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE 
      | SQLITE_OPEN_DELETEONCLOSE;
    int nBuf = 64*1024;
    if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->subjInMemory ){



      nBuf = -1;
    }
    rc = sqlite3JournalOpen(pPager->pVfs, 0, pPager->sjfd, flags, nBuf);
  }
  return rc;
}

/*
** Append a record of the current state of page pPg to the sub-journal. 
**
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
  int useJournal = (flags & PAGER_OMIT_JOURNAL)==0; /* False to omit journal */
  int pcacheSize = sqlite3PcacheSize();       /* Bytes to allocate for PCache */
  u32 szPageDflt = SQLITE_DEFAULT_PAGE_SIZE;  /* Default page size */
  const char *zUri = 0;    /* URI args to copy */
  int nUri = 0;            /* Number of bytes of URI args at *zUri */

  /* Figure out how much space is required for each journal file-handle
  ** (there are two of them, the main journal and the sub-journal). This
  ** is the maximum space required for an in-memory journal file handle 
  ** and a regular journal file-handle. Note that a "regular journal-handle"
  ** may be a wrapper capable of caching the first portion of the journal
  ** file in memory to implement the atomic-write optimization (see 
  ** source file journal.c).
  */
  if( sqlite3JournalSize(pVfs)>sqlite3MemJournalSize() ){
    journalFileSize = ROUND8(sqlite3JournalSize(pVfs));
  }else{
    journalFileSize = ROUND8(sqlite3MemJournalSize());
  }

  /* Set the output variable to NULL in case an error occurs. */
  *ppPager = 0;

#ifndef SQLITE_OMIT_MEMORYDB
  if( flags & PAGER_MEMORY ){
    memDb = 1;







|
<
<
<
<
<
<
<
|
<
<
<







4579
4580
4581
4582
4583
4584
4585
4586







4587



4588
4589
4590
4591
4592
4593
4594
  int useJournal = (flags & PAGER_OMIT_JOURNAL)==0; /* False to omit journal */
  int pcacheSize = sqlite3PcacheSize();       /* Bytes to allocate for PCache */
  u32 szPageDflt = SQLITE_DEFAULT_PAGE_SIZE;  /* Default page size */
  const char *zUri = 0;    /* URI args to copy */
  int nUri = 0;            /* Number of bytes of URI args at *zUri */

  /* Figure out how much space is required for each journal file-handle
  ** (there are two of them, the main journal and the sub-journal).  */







  journalFileSize = ROUND8(sqlite3JournalSize(pVfs));




  /* Set the output variable to NULL in case an error occurs. */
  *ppPager = 0;

#ifndef SQLITE_OMIT_MEMORYDB
  if( flags & PAGER_MEMORY ){
    memDb = 1;
6663
6664
6665
6666
6667
6668
6669
6670
6671
6672
6673
6674
6675
6676
6677
    pPager->nSavepoint = nNew;

    /* If this is a release of the outermost savepoint, truncate 
    ** the sub-journal to zero bytes in size. */
    if( op==SAVEPOINT_RELEASE ){
      if( nNew==0 && isOpen(pPager->sjfd) ){
        /* Only truncate if it is an in-memory sub-journal. */
        if( sqlite3IsMemJournal(pPager->sjfd) ){
          rc = sqlite3OsTruncate(pPager->sjfd, 0);
          assert( rc==SQLITE_OK );
        }
        pPager->nSubRec = 0;
      }
    }
    /* Else this is a rollback operation, playback the specified savepoint.







|







6658
6659
6660
6661
6662
6663
6664
6665
6666
6667
6668
6669
6670
6671
6672
    pPager->nSavepoint = nNew;

    /* If this is a release of the outermost savepoint, truncate 
    ** the sub-journal to zero bytes in size. */
    if( op==SAVEPOINT_RELEASE ){
      if( nNew==0 && isOpen(pPager->sjfd) ){
        /* Only truncate if it is an in-memory sub-journal. */
        if( sqlite3JournalIsInMemory(pPager->sjfd) ){
          rc = sqlite3OsTruncate(pPager->sjfd, 0);
          assert( rc==SQLITE_OK );
        }
        pPager->nSubRec = 0;
      }
    }
    /* Else this is a rollback operation, playback the specified savepoint.
Changes to src/sqliteInt.h.
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003

4004
4005
4006
4007
4008
4009
4010

4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
** Allowed flags for the 3rd parameter to sqlite3FindInIndex().
*/
#define IN_INDEX_NOOP_OK     0x0001  /* OK to return IN_INDEX_NOOP */
#define IN_INDEX_MEMBERSHIP  0x0002  /* IN operator used for membership test */
#define IN_INDEX_LOOP        0x0004  /* IN operator used as a loop */
int sqlite3FindInIndex(Parse *, Expr *, u32, int*);

#ifdef SQLITE_ENABLE_ATOMIC_WRITE
  int sqlite3JournalOpen(sqlite3_vfs *, const char *, sqlite3_file *, int, int);
  int sqlite3JournalSize(sqlite3_vfs *);

  int sqlite3JournalCreate(sqlite3_file *);
  int sqlite3JournalExists(sqlite3_file *p);
#else
  #define sqlite3JournalSize(pVfs) ((pVfs)->szOsFile)
  #define sqlite3JournalExists(p) 1
#endif


void sqlite3MemJournalOpen(sqlite3_file *);
int sqlite3MemJournalSize(void);
int sqlite3IsMemJournal(sqlite3_file *);

void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p);
#if SQLITE_MAX_EXPR_DEPTH>0
  int sqlite3SelectExprHeight(Select *);
  int sqlite3ExprCheckHeight(Parse*, int);
#else
  #define sqlite3SelectExprHeight(x) 0







<
|
|
>

<
<
<
<


>

<
<







3994
3995
3996
3997
3998
3999
4000

4001
4002
4003
4004




4005
4006
4007
4008


4009
4010
4011
4012
4013
4014
4015
** Allowed flags for the 3rd parameter to sqlite3FindInIndex().
*/
#define IN_INDEX_NOOP_OK     0x0001  /* OK to return IN_INDEX_NOOP */
#define IN_INDEX_MEMBERSHIP  0x0002  /* IN operator used for membership test */
#define IN_INDEX_LOOP        0x0004  /* IN operator used as a loop */
int sqlite3FindInIndex(Parse *, Expr *, u32, int*);


int sqlite3JournalOpen(sqlite3_vfs *, const char *, sqlite3_file *, int, int);
int sqlite3JournalSize(sqlite3_vfs *);
#ifdef SQLITE_ENABLE_ATOMIC_WRITE
  int sqlite3JournalCreate(sqlite3_file *);




#endif

int sqlite3JournalIsInMemory(sqlite3_file *p);
void sqlite3MemJournalOpen(sqlite3_file *);



void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p);
#if SQLITE_MAX_EXPR_DEPTH>0
  int sqlite3SelectExprHeight(Select *);
  int sqlite3ExprCheckHeight(Parse*, int);
#else
  #define sqlite3SelectExprHeight(x) 0
Changes to test/conflict.test.
281
282
283
284
285
286
287
288
289
290


291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
#   cmd    An UPDATE command to execute against table t1
#   t0     True if there is an error from $cmd
#   t1     Content of "b" column of t1 assuming no error in $cmd
#   t2     Content of "x" column of t3
#   t3     Number of temporary files for tables
#   t4     Number of temporary files for statement journals
#
# Update: Since temporary table files are now opened lazily, and none
# of the following tests use large quantities of data, t3 is always 0.
#


foreach {i conf1 cmd t0 t1 t2 t3 t4} {
  1 {}       UPDATE                  1 {6 7 8 9}  1 0 1
  2 REPLACE  UPDATE                  0 {7 6 9}    1 0 0
  3 IGNORE   UPDATE                  0 {6 7 3 9}  1 0 0
  4 FAIL     UPDATE                  1 {6 7 3 4}  1 0 0
  5 ABORT    UPDATE                  1 {1 2 3 4}  1 0 1
  6 ROLLBACK UPDATE                  1 {1 2 3 4}  0 0 0
  7 REPLACE  {UPDATE OR IGNORE}      0 {6 7 3 9}  1 0 0
  8 IGNORE   {UPDATE OR REPLACE}     0 {7 6 9}    1 0 0
  9 FAIL     {UPDATE OR IGNORE}      0 {6 7 3 9}  1 0 0
 10 ABORT    {UPDATE OR REPLACE}     0 {7 6 9}    1 0 0
 11 ROLLBACK {UPDATE OR IGNORE}      0 {6 7 3 9}  1 0 0
 12 {}       {UPDATE OR IGNORE}      0 {6 7 3 9}  1 0 0
 13 {}       {UPDATE OR REPLACE}     0 {7 6 9}    1 0 0
 14 {}       {UPDATE OR FAIL}        1 {6 7 3 4}  1 0 0
 15 {}       {UPDATE OR ABORT}       1 {1 2 3 4}  1 0 1
 16 {}       {UPDATE OR ROLLBACK}    1 {1 2 3 4}  0 0 0
} {
  if {$t0} {set t1 {UNIQUE constraint failed: t1.a}}
  if {[info exists TEMP_STORE] && $TEMP_STORE==3} {
    set t3 0
  } else {
    set t3 [expr {$t3+$t4}]







|
|

>
>

|



|









|







281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
#   cmd    An UPDATE command to execute against table t1
#   t0     True if there is an error from $cmd
#   t1     Content of "b" column of t1 assuming no error in $cmd
#   t2     Content of "x" column of t3
#   t3     Number of temporary files for tables
#   t4     Number of temporary files for statement journals
#
# Update (2007-08-21): Since temporary table files are now opened lazily, 
# and none of the following tests use large quantities of data, t3 is always 0.
#
# Update (2016-03-04): Subjournals now also open lazily, so t4 is also always 0.
#
foreach {i conf1 cmd t0 t1 t2 t3 t4} {
  1 {}       UPDATE                  1 {6 7 8 9}  1 0 0
  2 REPLACE  UPDATE                  0 {7 6 9}    1 0 0
  3 IGNORE   UPDATE                  0 {6 7 3 9}  1 0 0
  4 FAIL     UPDATE                  1 {6 7 3 4}  1 0 0
  5 ABORT    UPDATE                  1 {1 2 3 4}  1 0 0
  6 ROLLBACK UPDATE                  1 {1 2 3 4}  0 0 0
  7 REPLACE  {UPDATE OR IGNORE}      0 {6 7 3 9}  1 0 0
  8 IGNORE   {UPDATE OR REPLACE}     0 {7 6 9}    1 0 0
  9 FAIL     {UPDATE OR IGNORE}      0 {6 7 3 9}  1 0 0
 10 ABORT    {UPDATE OR REPLACE}     0 {7 6 9}    1 0 0
 11 ROLLBACK {UPDATE OR IGNORE}      0 {6 7 3 9}  1 0 0
 12 {}       {UPDATE OR IGNORE}      0 {6 7 3 9}  1 0 0
 13 {}       {UPDATE OR REPLACE}     0 {7 6 9}    1 0 0
 14 {}       {UPDATE OR FAIL}        1 {6 7 3 4}  1 0 0
 15 {}       {UPDATE OR ABORT}       1 {1 2 3 4}  1 0 0
 16 {}       {UPDATE OR ROLLBACK}    1 {1 2 3 4}  0 0 0
} {
  if {$t0} {set t1 {UNIQUE constraint failed: t1.a}}
  if {[info exists TEMP_STORE] && $TEMP_STORE==3} {
    set t3 0
  } else {
    set t3 [expr {$t3+$t4}]
Changes to test/conflict2.test.
282
283
284
285
286
287
288
289



290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
#   t1     Content of "b" column of t1 assuming no error in $cmd
#   t2     Content of "x" column of t3
#   t3     Number of temporary files for tables
#   t4     Number of temporary files for statement journals
#
# Update: Since temporary table files are now opened lazily, and none
# of the following tests use large quantities of data, t3 is always 0.
#



foreach {i conf1 cmd t0 t1 t2 t3 t4} {
  1 {}       UPDATE                  1 {6 7 8 9}  1 0 1
  2 REPLACE  UPDATE                  0 {7 6 9}    1 0 1
  3 IGNORE   UPDATE                  0 {6 7 3 9}  1 0 1
  4 FAIL     UPDATE                  1 {6 7 3 4}  1 0 1
  5 ABORT    UPDATE                  1 {1 2 3 4}  1 0 1
  6 ROLLBACK UPDATE                  1 {1 2 3 4}  0 0 1
  7 REPLACE  {UPDATE OR IGNORE}      0 {6 7 3 9}  1 0 0
  8 IGNORE   {UPDATE OR REPLACE}     0 {7 6 9}    1 0 1
  9 FAIL     {UPDATE OR IGNORE}      0 {6 7 3 9}  1 0 0
 10 ABORT    {UPDATE OR REPLACE}     0 {7 6 9}    1 0 1
 11 ROLLBACK {UPDATE OR IGNORE}      0 {6 7 3 9}  1 0 0
 12 {}       {UPDATE OR IGNORE}      0 {6 7 3 9}  1 0 0
 13 {}       {UPDATE OR REPLACE}     0 {7 6 9}    1 0 1
 14 {}       {UPDATE OR FAIL}        1 {6 7 3 4}  1 0 0
 15 {}       {UPDATE OR ABORT}       1 {1 2 3 4}  1 0 1
 16 {}       {UPDATE OR ROLLBACK}    1 {1 2 3 4}  0 0 0
} {

  # When using in-memory journals, no temporary files are required for
  # statement journals.
  if {[permutation] == "inmemory_journal"} { set t4 0 }









>
>
>

|
|
|
|
|
|

|

|


|

|







282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
#   t1     Content of "b" column of t1 assuming no error in $cmd
#   t2     Content of "x" column of t3
#   t3     Number of temporary files for tables
#   t4     Number of temporary files for statement journals
#
# Update: Since temporary table files are now opened lazily, and none
# of the following tests use large quantities of data, t3 is always 0.
#
# Update (2016-03-04): Subjournals now only open when their size
# exceeds 64KB.
#
foreach {i conf1 cmd t0 t1 t2 t3 t4} {
  1 {}       UPDATE                  1 {6 7 8 9}  1 0 0
  2 REPLACE  UPDATE                  0 {7 6 9}    1 0 0
  3 IGNORE   UPDATE                  0 {6 7 3 9}  1 0 0
  4 FAIL     UPDATE                  1 {6 7 3 4}  1 0 0
  5 ABORT    UPDATE                  1 {1 2 3 4}  1 0 0
  6 ROLLBACK UPDATE                  1 {1 2 3 4}  0 0 0
  7 REPLACE  {UPDATE OR IGNORE}      0 {6 7 3 9}  1 0 0
  8 IGNORE   {UPDATE OR REPLACE}     0 {7 6 9}    1 0 0
  9 FAIL     {UPDATE OR IGNORE}      0 {6 7 3 9}  1 0 0
 10 ABORT    {UPDATE OR REPLACE}     0 {7 6 9}    1 0 0
 11 ROLLBACK {UPDATE OR IGNORE}      0 {6 7 3 9}  1 0 0
 12 {}       {UPDATE OR IGNORE}      0 {6 7 3 9}  1 0 0
 13 {}       {UPDATE OR REPLACE}     0 {7 6 9}    1 0 0
 14 {}       {UPDATE OR FAIL}        1 {6 7 3 4}  1 0 0
 15 {}       {UPDATE OR ABORT}       1 {1 2 3 4}  1 0 0
 16 {}       {UPDATE OR ROLLBACK}    1 {1 2 3 4}  0 0 0
} {

  # When using in-memory journals, no temporary files are required for
  # statement journals.
  if {[permutation] == "inmemory_journal"} { set t4 0 }

Changes to test/exclusive.test.
416
417
418
419
420
421
422

423
424
425
426
427
428
429
430
431
432
    BEGIN;
    INSERT INTO abc VALUES(1, 2, 3);
    INSERT INTO abc SELECT a+1, b+1, c+1 FROM abc;
  }
} {}
do_test exclusive-5.1 {
  # Three files are open: The db, journal and statement-journal.

  set sqlite_open_file_count
  expr $sqlite_open_file_count-$extrafds
} [expr 3 - ($TEMP_STORE>=2)]
do_test exclusive-5.2 {
  execsql {
    COMMIT;
  }
  # One file open: the db.
  set sqlite_open_file_count
  expr $sqlite_open_file_count-$extrafds







>


|







416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
    BEGIN;
    INSERT INTO abc VALUES(1, 2, 3);
    INSERT INTO abc SELECT a+1, b+1, c+1 FROM abc;
  }
} {}
do_test exclusive-5.1 {
  # Three files are open: The db, journal and statement-journal.
  # (2016-03-04) The statement-journal is now opened lazily
  set sqlite_open_file_count
  expr $sqlite_open_file_count-$extrafds
} [expr 2 - ($TEMP_STORE>=2)]
do_test exclusive-5.2 {
  execsql {
    COMMIT;
  }
  # One file open: the db.
  set sqlite_open_file_count
  expr $sqlite_open_file_count-$extrafds
442
443
444
445
446
447
448

449
450
451
452
453
454
455
456

457
458
459
460
461
462
463
464
465
466
  expr $sqlite_open_file_count-$extrafds
} {2}
do_test exclusive-5.4 {
  execsql {
    INSERT INTO abc SELECT a+10, b+10, c+10 FROM abc;
  }
  # Three files are open: The db, journal and statement-journal.

  set sqlite_open_file_count
  expr $sqlite_open_file_count-$extrafds
} [expr 3 - ($TEMP_STORE>=2)]
do_test exclusive-5.5 {
  execsql {
    COMMIT;
  }
  # Three files are still open: The db, journal and statement-journal.

  set sqlite_open_file_count
  expr $sqlite_open_file_count-$extrafds
} [expr 3 - ($TEMP_STORE>=2)]
do_test exclusive-5.6 {
  execsql {
    PRAGMA locking_mode = normal;
    SELECT * FROM abc;
  }
} {normal 1 2 3 2 3 4 5 6 7 11 12 13 12 13 14 15 16 17}
do_test exclusive-5.7 {







>


|





>


|







443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
  expr $sqlite_open_file_count-$extrafds
} {2}
do_test exclusive-5.4 {
  execsql {
    INSERT INTO abc SELECT a+10, b+10, c+10 FROM abc;
  }
  # Three files are open: The db, journal and statement-journal.
  # 2016-03-04: The statement-journal open is deferred
  set sqlite_open_file_count
  expr $sqlite_open_file_count-$extrafds
} [expr 2 - ($TEMP_STORE>=2)]
do_test exclusive-5.5 {
  execsql {
    COMMIT;
  }
  # Three files are still open: The db, journal and statement-journal.
  # 2016-03-04: The statement-journal open is deferred
  set sqlite_open_file_count
  expr $sqlite_open_file_count-$extrafds
} [expr 2 - ($TEMP_STORE>=2)]
do_test exclusive-5.6 {
  execsql {
    PRAGMA locking_mode = normal;
    SELECT * FROM abc;
  }
} {normal 1 2 3 2 3 4 5 6 7 11 12 13 12 13 14 15 16 17}
do_test exclusive-5.7 {
Changes to test/stmt.test.
42
43
44
45
46
47
48

49
50
51
52
53
54
55
56
57
58
59
60
61
62
63

64
65
66
67
68
69
70
71
  set sqlite_open_file_count
} {2}
do_test stmt-1.4 {
  execsql {
    INSERT INTO t1 SELECT a+1, b+1 FROM t1;
  }
  set sqlite_open_file_count

} {3}
do_test stmt-1.5 {
  execsql COMMIT
  set sqlite_open_file_count
} {1}
do_test stmt-1.6.1 {
  execsql {
    BEGIN;
      INSERT INTO t1 SELECT a+2, b+2 FROM t1;
  }
  set sqlite_open_file_count
} {2}
do_test stmt-1.6.2 {
  execsql { INSERT INTO t1 SELECT a+4, b+4 FROM t1 }
  set sqlite_open_file_count

} {3}
do_test stmt-1.7 {
  execsql COMMIT
  set sqlite_open_file_count
} {1}


proc filecount {testname sql expected} {







>
|














>
|







42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
  set sqlite_open_file_count
} {2}
do_test stmt-1.4 {
  execsql {
    INSERT INTO t1 SELECT a+1, b+1 FROM t1;
  }
  set sqlite_open_file_count
  # 2016-03-04: statement-journal open deferred
} {2}
do_test stmt-1.5 {
  execsql COMMIT
  set sqlite_open_file_count
} {1}
do_test stmt-1.6.1 {
  execsql {
    BEGIN;
      INSERT INTO t1 SELECT a+2, b+2 FROM t1;
  }
  set sqlite_open_file_count
} {2}
do_test stmt-1.6.2 {
  execsql { INSERT INTO t1 SELECT a+4, b+4 FROM t1 }
  set sqlite_open_file_count
  # 2016-03-04: statement-journal open deferred
} {2}
do_test stmt-1.7 {
  execsql COMMIT
  set sqlite_open_file_count
} {1}


proc filecount {testname sql expected} {
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97

filecount stmt-2.1 { INSERT INTO t1 VALUES(9, 9)  } 2
filecount stmt-2.2 { REPLACE INTO t1 VALUES(9, 9) } 2
filecount stmt-2.3 { INSERT INTO t1 SELECT 9, 9   } 2
filecount stmt-2.4 { 
    INSERT INTO t1 SELECT 9, 9;
    INSERT INTO t1 SELECT 10, 10;
} 3

do_test stmt-2.5 {
  execsql { CREATE INDEX i1 ON t1(b) }
} {}
filecount stmt-2.6 { 
  REPLACE INTO t1 VALUES(5, 5);
  REPLACE INTO t1 VALUES(5, 5); 
} 3

finish_test







|







|


82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99

filecount stmt-2.1 { INSERT INTO t1 VALUES(9, 9)  } 2
filecount stmt-2.2 { REPLACE INTO t1 VALUES(9, 9) } 2
filecount stmt-2.3 { INSERT INTO t1 SELECT 9, 9   } 2
filecount stmt-2.4 { 
    INSERT INTO t1 SELECT 9, 9;
    INSERT INTO t1 SELECT 10, 10;
} 2

do_test stmt-2.5 {
  execsql { CREATE INDEX i1 ON t1(b) }
} {}
filecount stmt-2.6 { 
  REPLACE INTO t1 VALUES(5, 5);
  REPLACE INTO t1 VALUES(5, 5); 
} 2

finish_test
Changes to test/tempdb.test.
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
      INSERT INTO t1 VALUES(1, 2, 3);
      INSERT INTO t1 VALUES(4, 5, 6);
      INSERT INTO t2 VALUES(7, 8, 9);
      INSERT INTO t2 SELECT * FROM t1;
  }
  catchsql { INSERT INTO t1 SELECT * FROM t2 }
  set sqlite_open_file_count
} [expr 1 + (0==$jrnl_in_memory) + (0==$subj_in_memory)]
do_test tempdb-2.3 {
  execsql {
    PRAGMA temp_store = 'memory';
    ROLLBACK;
    BEGIN;
      INSERT INTO t1 VALUES(1, 2, 3);
      INSERT INTO t1 VALUES(4, 5, 6);







|







72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
      INSERT INTO t1 VALUES(1, 2, 3);
      INSERT INTO t1 VALUES(4, 5, 6);
      INSERT INTO t2 VALUES(7, 8, 9);
      INSERT INTO t2 SELECT * FROM t1;
  }
  catchsql { INSERT INTO t1 SELECT * FROM t2 }
  set sqlite_open_file_count
} [expr 1 + (0==$jrnl_in_memory)]
do_test tempdb-2.3 {
  execsql {
    PRAGMA temp_store = 'memory';
    ROLLBACK;
    BEGIN;
      INSERT INTO t1 VALUES(1, 2, 3);
      INSERT INTO t1 VALUES(4, 5, 6);