SQLite

Check-in [43a1264088]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add a hard limit to the number of chunks a multiplexed database may consist of if ENABLE_8_3_NAMES is defined.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | nx-devkit
Files: files | file ages | folders
SHA1: 43a1264088c57bf598787b7a9f5d7a2536603d67
User & Date: dan 2011-12-13 19:03:34.498
Context
2011-12-14
01:38
Improvements to comments. No code changes. (check-in: 08bbbd8e38 user: drh tags: nx-devkit)
2011-12-13
19:03
Add a hard limit to the number of chunks a multiplexed database may consist of if ENABLE_8_3_NAMES is defined. (check-in: 43a1264088 user: dan tags: nx-devkit)
18:22
Change the SQLITE_EXTRA_INIT routine to take a single argument which is a pointer to a string. Call SQLITE_EXTRA_INIT with a NULL argument. Fixes to multiplexor to treat the VFS properly in corner cases. Fix the initialization of multiplex3.test. (check-in: 8e65b91325 user: drh tags: nx-devkit)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/test_multiplex.c.
︙︙
77
78
79
80
81
82
83


84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107

#define sqlite3_mutex_enter(X)
#define sqlite3_mutex_try(X)      SQLITE_OK
#define sqlite3_mutex_leave(X)
#define sqlite3_mutex_held(X)     ((void)(X),1)
#define sqlite3_mutex_notheld(X)  ((void)(X),1)
#endif /* SQLITE_THREADSAFE==0 */




/************************ Shim Definitions ******************************/

#ifndef SQLITE_MULTIPLEX_VFS_NAME
# define SQLITE_MULTIPLEX_VFS_NAME "multiplex"
#endif

/* This is the limit on the chunk size.  It may be changed by calling
** the xFileControl() interface.  It will be rounded up to a 
** multiple of MAX_PAGE_SIZE.  We default it here to 2GiB less 64KiB.
*/
#ifndef SQLITE_MULTIPLEX_CHUNK_SIZE
# define SQLITE_MULTIPLEX_CHUNK_SIZE 2147418112
#endif

/* This used to be the default limit on number of chunks, but
** it is no longer enforced.  There is currently no limit to the
** number of chunks.
**
** May be changed by calling the xFileControl() interface.
*/
#ifndef SQLITE_MULTIPLEX_MAX_CHUNKS
# define SQLITE_MULTIPLEX_MAX_CHUNKS 12
#endif








>
>
















|








77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109

#define sqlite3_mutex_enter(X)
#define sqlite3_mutex_try(X)      SQLITE_OK
#define sqlite3_mutex_leave(X)
#define sqlite3_mutex_held(X)     ((void)(X),1)
#define sqlite3_mutex_notheld(X)  ((void)(X),1)
#endif /* SQLITE_THREADSAFE==0 */

#define SQLITE_MULTIPLEX_JOURNAL_8_3_OFFSET 400


/************************ Shim Definitions ******************************/

#ifndef SQLITE_MULTIPLEX_VFS_NAME
# define SQLITE_MULTIPLEX_VFS_NAME "multiplex"
#endif

/* This is the limit on the chunk size.  It may be changed by calling
** the xFileControl() interface.  It will be rounded up to a 
** multiple of MAX_PAGE_SIZE.  We default it here to 2GiB less 64KiB.
*/
#ifndef SQLITE_MULTIPLEX_CHUNK_SIZE
# define SQLITE_MULTIPLEX_CHUNK_SIZE 2147418112
#endif

/* This used to be the default limit on number of chunks, but
** it is no longer enforced. There is currently no limit to the
** number of chunks.
**
** May be changed by calling the xFileControl() interface.
*/
#ifndef SQLITE_MULTIPLEX_MAX_CHUNKS
# define SQLITE_MULTIPLEX_MAX_CHUNKS 12
#endif
︙︙
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266












267
268
269
270
271
272
273

      for(i=n-1; i>0 && i>=n-4 && z[i]!='.'; i--){}
      if( i>=n-4 ) n = i+1;
      if( pGroup->flags & (SQLITE_OPEN_MAIN_JOURNAL|SQLITE_OPEN_TEMP_JOURNAL) ){
        /* The extensions on overflow files for main databases are 001, 002,
        ** 003 and so forth.  To avoid name collisions, add 100 to the 
        ** extensions of journal files so that they are 101, 102, 103, ....
        */
        iChunk += 100;
      }
#endif
      sqlite3_snprintf(4,&z[n],"%03d",iChunk);
    }
  }
  return SQLITE_OK;
}

/* Translate an sqlite3_file* that is really a multiplexGroup* into
** the sqlite3_file* for the underlying original VFS.
*/
static sqlite3_file *multiplexSubOpen(
  multiplexGroup *pGroup,
  int iChunk,
  int *rc,
  int *pOutFlags
){
  sqlite3_file *pSubOpen = 0;
  sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs;        /* Real VFS */












  *rc = multiplexSubFilename(pGroup, iChunk);
  if( (*rc)==SQLITE_OK && (pSubOpen = pGroup->aReal[iChunk].p)==0 ){
    pSubOpen = sqlite3_malloc( pOrigVfs->szOsFile );
    if( pSubOpen==0 ){
      *rc = SQLITE_NOMEM;
      return 0;
    }








|



















>
>
>
>
>
>
>
>
>
>
>
>








242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287

      for(i=n-1; i>0 && i>=n-4 && z[i]!='.'; i--){}
      if( i>=n-4 ) n = i+1;
      if( pGroup->flags & (SQLITE_OPEN_MAIN_JOURNAL|SQLITE_OPEN_TEMP_JOURNAL) ){
        /* The extensions on overflow files for main databases are 001, 002,
        ** 003 and so forth.  To avoid name collisions, add 100 to the 
        ** extensions of journal files so that they are 101, 102, 103, ....
        */
        iChunk += SQLITE_MULTIPLEX_JOURNAL_8_3_OFFSET;
      }
#endif
      sqlite3_snprintf(4,&z[n],"%03d",iChunk);
    }
  }
  return SQLITE_OK;
}

/* Translate an sqlite3_file* that is really a multiplexGroup* into
** the sqlite3_file* for the underlying original VFS.
*/
static sqlite3_file *multiplexSubOpen(
  multiplexGroup *pGroup,
  int iChunk,
  int *rc,
  int *pOutFlags
){
  sqlite3_file *pSubOpen = 0;
  sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs;        /* Real VFS */

#ifdef SQLITE_ENABLE_8_3_NAMES
  /* If JOURNAL_8_3_OFFSET is set to (say) 500, then any overflow files are 
  ** part of a database journal are named db.501, db.502, and so on. A 
  ** database may therefore not grow to larger than 500 chunks. Attempting
  ** to open chunk 501 indicates the database is full. */
  if( iChunk>=SQLITE_MULTIPLEX_JOURNAL_8_3_OFFSET ){
    *rc = SQLITE_FULL;
    return 0;
  }
#endif

  *rc = multiplexSubFilename(pGroup, iChunk);
  if( (*rc)==SQLITE_OK && (pSubOpen = pGroup->aReal[iChunk].p)==0 ){
    pSubOpen = sqlite3_malloc( pOrigVfs->szOsFile );
    if( pSubOpen==0 ){
      *rc = SQLITE_NOMEM;
      return 0;
    }
︙︙
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659

    sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL);
    if( pSubOpen==0 ){
      rc = SQLITE_IOERR_WRITE;
    }else{
      rc = pSubOpen->pMethods->xWrite(pSubOpen, pBuf, iAmt, iOfst);
    }
  }else{
    while( iAmt > 0 ){
      int i = (int)(iOfst / pGroup->szChunk);
      sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL);
      if( pSubOpen ){
        int extra = ((int)(iOfst % pGroup->szChunk) + iAmt) -
                    pGroup->szChunk;
        if( extra<0 ) extra = 0;
        iAmt -= extra;
        rc = pSubOpen->pMethods->xWrite(pSubOpen, pBuf, iAmt,
                                        iOfst % pGroup->szChunk);
        if( rc!=SQLITE_OK ) break;
        pBuf = (char *)pBuf + iAmt;
        iOfst += iAmt;
        iAmt = extra;
      }else{
        rc = SQLITE_IOERR_WRITE;
        break;
      }
    }
  }
  multiplexLeave();
  return rc;
}









|









<



<
<
<








643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659

660
661
662



663
664
665
666
667
668
669

    sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL);
    if( pSubOpen==0 ){
      rc = SQLITE_IOERR_WRITE;
    }else{
      rc = pSubOpen->pMethods->xWrite(pSubOpen, pBuf, iAmt, iOfst);
    }
  }else{
    while( rc==SQLITE_OK && iAmt>0 ){
      int i = (int)(iOfst / pGroup->szChunk);
      sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL);
      if( pSubOpen ){
        int extra = ((int)(iOfst % pGroup->szChunk) + iAmt) -
                    pGroup->szChunk;
        if( extra<0 ) extra = 0;
        iAmt -= extra;
        rc = pSubOpen->pMethods->xWrite(pSubOpen, pBuf, iAmt,
                                        iOfst % pGroup->szChunk);

        pBuf = (char *)pBuf + iAmt;
        iOfst += iAmt;
        iAmt = extra;



      }
    }
  }
  multiplexLeave();
  return rc;
}
︙︙
Changes to test/multiplex2.test.
︙︙
10
11
12
13
14
15
16

17
18
19
20
21
22
23

#***********************************************************************
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/malloc_common.tcl
source $testdir/lock_common.tcl

db close

do_multiclient_test tn {
  foreach f [glob -nocomplain test.*] { forcedelete $f }

  code1 { catch { sqlite3_multiplex_initialize "" 0 } }
  code2 { catch { sqlite3_multiplex_initialize "" 0 } }








>








10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

#***********************************************************************
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/malloc_common.tcl
source $testdir/lock_common.tcl
set testprefix multiplex2
db close

do_multiclient_test tn {
  foreach f [glob -nocomplain test.*] { forcedelete $f }

  code1 { catch { sqlite3_multiplex_initialize "" 0 } }
  code2 { catch { sqlite3_multiplex_initialize "" 0 } }
︙︙
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70














































71
72

    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --   64
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --  128
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --  256
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --  512
    SELECT count(*) FROM t1;
  } 

  do_test multiplex-1.$tn.1 { sql1 { SELECT count(*) FROM t1 } } 512
  do_test multiplex-1.$tn.2 { sql2 { SELECT count(*) FROM t1 } } 512
  sql2 { DELETE FROM t1 ; VACUUM }
  do_test multiplex-1.$tn.3 { sql1 { SELECT count(*) FROM t1 } } 0

  sql1 {
    INSERT INTO t1 VALUES(randomblob(10), randomblob(4000));          --    1
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --    2
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --    4
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --    8
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --   16
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --   32
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --   64
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --  128
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --  256
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --  512
    SELECT count(*) FROM t1;
  }

  do_test multiplex-1.$tn.4 { sql2 { SELECT count(*) FROM t1 } } 512
}















































catch { sqlite3_multiplex_shutdown }
finish_test








|
|

|















|


>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>



43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119

    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --   64
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --  128
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --  256
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --  512
    SELECT count(*) FROM t1;
  } 

  do_test 1.$tn.1 { sql1 { SELECT count(*) FROM t1 } } 512
  do_test 1.$tn.2 { sql2 { SELECT count(*) FROM t1 } } 512
  sql2 { DELETE FROM t1 ; VACUUM }
  do_test 1.$tn.3 { sql1 { SELECT count(*) FROM t1 } } 0

  sql1 {
    INSERT INTO t1 VALUES(randomblob(10), randomblob(4000));          --    1
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --    2
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --    4
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --    8
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --   16
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --   32
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --   64
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --  128
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --  256
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --  512
    SELECT count(*) FROM t1;
  }

  do_test 1.$tn.4 { sql2 { SELECT count(*) FROM t1 } } 512
}

catch {db close}
foreach f [glob -nocomplain test.*] { forcedelete $f }

ifcapable 8_3_names {
  sqlite3 db test.db -vfs multiplex
  sqlite3_multiplex_control db main chunk_size [expr 256*1024]

  # Insert 512 * 256K (128MB) of data. If each row is around 4K, this means
  # we need 32768 rows.
  do_catchsql_test 2.1 {
    CREATE TABLE t1(a, b);
    INSERT INTO t1 VALUES(randomblob(10), randomblob(4000));          --    1
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --    2
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --    4
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --    8
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --   16
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --   32
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --   64
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --  128
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --  256
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --  512

    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --   1K
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --   2K
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --   4K
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --   8K
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --  16K
    INSERT INTO t1 SELECT randomblob(10), randomblob(4000) FROM t1;   --  32K
  
  } {1 {database or disk is full}}

  do_execsql_test 2.2 {
    UPDATE t1 SET a=randomblob(9), b=randomblob(3900);
    PRAGMA integrity_check;
  } ok

  db close
  sqlite3 db test.db -vfs multiplex
  sqlite3_multiplex_control db main chunk_size [expr 256*1024]

  do_execsql_test 2.3 {
    PRAGMA integrity_check;
  } ok
}

catch { db close }
catch { sqlite3_multiplex_shutdown }
finish_test