Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | In-progress changes - do not use; Removed prefix support; Added file control interface to enable/disable and adjust chunk size; added app-def function for same; |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | multiplex-enhancements |
Files: | files | file ages | folders |
SHA1: |
bc02d0c193225bd49a8d8a3295aeac75 |
User & Date: | shaneh 2011-03-29 05:06:46.821 |
Context
2011-03-30
| ||
21:03 | Updates to multiplex.test script; misc. bug fixes; (check-in: c41ff2358e user: shaneh tags: multiplex-enhancements) | |
2011-03-29
| ||
05:06 | In-progress changes - do not use; Removed prefix support; Added file control interface to enable/disable and adjust chunk size; added app-def function for same; (check-in: bc02d0c193 user: shaneh tags: multiplex-enhancements) | |
2011-03-15
| ||
04:45 | Allow multiplex file names to be preceeded by prefix of the form ":multiplex:chunksize:maxchunks:" Still work to be done, though it compiles and prefixes are ignored. (check-in: cfa4a2f7ea user: shaneh tags: multiplex-enhancements) | |
Changes
Changes to src/test_multiplex.c.
︙ | ︙ | |||
19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 | ** file size of the underlying file system. ** */ #include "sqlite3.h" #include <string.h> #include <assert.h> #include "sqliteInt.h" /* ** For a build without mutexes, no-op the mutex calls. */ #if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE==0 #define sqlite3_mutex_alloc(X) ((sqlite3_mutex*)8) #define sqlite3_mutex_free(X) #define sqlite3_mutex_enter(X) #define sqlite3_mutex_try(X) SQLITE_OK #define sqlite3_mutex_leave(X) #define sqlite3_mutex_held(X) ((void)(X),1) #define sqlite3_mutex_notheld(X) ((void)(X),1) #endif /* SQLITE_THREADSAFE==0 */ /************************ Shim Definitions ******************************/ #define SQLITE_MULTIPLEX_VFS_NAME "multiplex" /* This is the limit on the chunk size. It may be changed by calling | > > > > | | | 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 | ** file size of the underlying file system. ** */ #include "sqlite3.h" #include <string.h> #include <assert.h> #include "sqliteInt.h" #include "test_multiplex.h" #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 /* ** For a build without mutexes, no-op the mutex calls. */ #if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE==0 #define sqlite3_mutex_alloc(X) ((sqlite3_mutex*)8) #define sqlite3_mutex_free(X) #define sqlite3_mutex_enter(X) #define sqlite3_mutex_try(X) SQLITE_OK #define sqlite3_mutex_leave(X) #define sqlite3_mutex_held(X) ((void)(X),1) #define sqlite3_mutex_notheld(X) ((void)(X),1) #endif /* SQLITE_THREADSAFE==0 */ /************************ Shim Definitions ******************************/ #define SQLITE_MULTIPLEX_VFS_NAME "multiplex" /* This is the limit on the chunk size. It may be changed by calling ** the xFileControl() interface. */ #define SQLITE_MULTIPLEX_CHUNK_SIZE 0x40000000 /* Default limit on number of chunks. Care should be taken ** so that values for chunks numbers fit in the SQLITE_MULTIPLEX_EXT_FMT ** format specifier. It may be changed by calling ** the xFileControl() interface. */ #define SQLITE_MULTIPLEX_MAX_CHUNKS 32 /* If SQLITE_MULTIPLEX_EXT_OVWR is defined, the ** last SQLITE_MULTIPLEX_EXT_SZ characters of the ** filename will be overwritten, otherwise, the ** multiplex extension is simply appended to the filename. |
︙ | ︙ | |||
80 81 82 83 84 85 86 87 88 89 90 91 92 93 | */ struct multiplexGroup { sqlite3_file **pReal; /* Handles to each chunk */ char *bOpen; /* array of bools - 0 if chunk not opened */ char *zName; /* Base filename of this group */ int nName; /* Length of base filename */ int flags; /* Flags used for original opening */ multiplexGroup *pNext, *pPrev; /* Doubly linked list of all group objects */ }; /* ** An instance of the following object represents each open connection ** to a file that is multiplex'ed. This object is a ** subclass of sqlite3_file. The sqlite3_file object for the underlying | > > > | 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 | */ struct multiplexGroup { sqlite3_file **pReal; /* Handles to each chunk */ char *bOpen; /* array of bools - 0 if chunk not opened */ char *zName; /* Base filename of this group */ int nName; /* Length of base filename */ int flags; /* Flags used for original opening */ int nChunkSize; /* Chunk size used for this group */ int nMaxChunks; /* Max number of chunks for this group */ int bEnabled; /* TRUE to use Multiplex VFS for this file */ multiplexGroup *pNext, *pPrev; /* Doubly linked list of all group objects */ }; /* ** An instance of the following object represents each open connection ** to a file that is multiplex'ed. This object is a ** subclass of sqlite3_file. The sqlite3_file object for the underlying |
︙ | ︙ | |||
138 139 140 141 142 143 144 | */ sqlite3_mutex *pMutex; /* List of multiplexGroup objects. */ multiplexGroup *pGroups; | < < < < < | 145 146 147 148 149 150 151 152 153 154 155 156 157 158 | */ sqlite3_mutex *pMutex; /* List of multiplexGroup objects. */ multiplexGroup *pGroups; /* Storage for temp file names. Allocated during ** initialization to the max pathname of the underlying VFS. */ char *zName; } gMultiplex; |
︙ | ︙ | |||
164 165 166 167 168 169 170 | /* Translate an sqlite3_file* that is really a multiplexGroup* into ** the sqlite3_file* for the underlying original VFS. */ static sqlite3_file *multiplexSubOpen(multiplexConn *pConn, int iChunk, int *rc, int *pOutFlags){ multiplexGroup *pGroup = pConn->pGroup; sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ | | | 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 | /* Translate an sqlite3_file* that is really a multiplexGroup* into ** the sqlite3_file* for the underlying original VFS. */ static sqlite3_file *multiplexSubOpen(multiplexConn *pConn, int iChunk, int *rc, int *pOutFlags){ multiplexGroup *pGroup = pConn->pGroup; sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ if( iChunk<pGroup->nMaxChunks ){ sqlite3_file *pSubOpen = pGroup->pReal[iChunk]; /* Real file descriptor */ if( !pGroup->bOpen[iChunk] ){ memcpy(gMultiplex.zName, pGroup->zName, pGroup->nName+1); if( iChunk ){ #ifdef SQLITE_MULTIPLEX_EXT_OVWR sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, gMultiplex.zName+pGroup->nName-SQLITE_MULTIPLEX_EXT_SZ, SQLITE_MULTIPLEX_EXT_FMT, iChunk); #else |
︙ | ︙ | |||
189 190 191 192 193 194 195 | *rc = SQLITE_OK; return pSubOpen; } *rc = SQLITE_FULL; return NULL; } | < | < < < < < > | | < > | < | > | | < > | | < | > | < < < < < < < < | < < < < < < < < < > | > > > > > | < | | 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 | *rc = SQLITE_OK; return pSubOpen; } *rc = SQLITE_FULL; return NULL; } static void multiplexControlFunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ extern const char *sqlite3TestErrorName(int); extern int multiplexFileControl(sqlite3_file *, int, void *); sqlite3_file *db = (sqlite3_file *)sqlite3_user_data(context); int op = sqlite3_value_int(argv[0]); int iVal = sqlite3_value_int(argv[1]); int rc = multiplexFileControl(db, op, &iVal); if( rc== 0 ){ sqlite3_result_text(context, (char *)sqlite3TestErrorName(rc), -1, SQLITE_TRANSIENT); } sqlite3_result_text(context, (char *)sqlite3TestErrorName(rc), -1, SQLITE_TRANSIENT); } /* ** This is the entry point to register the extension for the multiplex_control() function. */ static int multiplexFuncInit( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi ){ sqlite3_create_function(db, "multiplex_control", 2, SQLITE_ANY, db, multiplexControlFunc, 0, 0); return 0; } /************************* VFS Method Wrappers *****************************/ /* ** This is the xOpen method used for the "multiplex" VFS. ** |
︙ | ︙ | |||
270 271 272 273 274 275 276 | /* We need to create a group structure and manage ** access to this group of files. */ multiplexEnter(); pMultiplexOpen = (multiplexConn*)pConn; /* allocate space for group */ sz = sizeof(multiplexGroup) /* multiplexGroup */ | | | | > > | | > | | 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 | /* We need to create a group structure and manage ** access to this group of files. */ multiplexEnter(); pMultiplexOpen = (multiplexConn*)pConn; /* allocate space for group */ sz = sizeof(multiplexGroup) /* multiplexGroup */ + (sizeof(sqlite3_file *)*SQLITE_MULTIPLEX_MAX_CHUNKS) /* pReal[] */ + (pOrigVfs->szOsFile*SQLITE_MULTIPLEX_MAX_CHUNKS) /* *pReal */ + SQLITE_MULTIPLEX_MAX_CHUNKS /* bOpen[] */ + nName + 1; /* zName */ #ifndef SQLITE_MULTIPLEX_EXT_OVWR sz += SQLITE_MULTIPLEX_EXT_SZ; assert(nName+SQLITE_MULTIPLEX_EXT_SZ < pOrigVfs->mxPathname); #else assert(nName >= SQLITE_MULTIPLEX_EXT_SZ); assert(nName < pOrigVfs->mxPathname); #endif pGroup = sqlite3_malloc( sz ); if( pGroup==0 ){ rc=SQLITE_NOMEM; }else{ /* assign pointers to extra space allocated */ char *p = (char *)&pGroup[1]; pMultiplexOpen->pGroup = pGroup; memset(pGroup, 0, sz); pGroup->nChunkSize = SQLITE_MULTIPLEX_CHUNK_SIZE; pGroup->nMaxChunks = SQLITE_MULTIPLEX_MAX_CHUNKS; pGroup->pReal = (sqlite3_file **)p; p += (sizeof(sqlite3_file *)*pGroup->nMaxChunks); for(i=0; i<pGroup->nMaxChunks; i++){ pGroup->pReal[i] = (sqlite3_file *)p; p += pOrigVfs->szOsFile; } /* bOpen[] vals should all be zero from memset above */ pGroup->bOpen = p; p += pGroup->nMaxChunks; pGroup->zName = p; /* save off base filename, name length, and original open flags */ memcpy(pGroup->zName, zName, nName+1); pGroup->nName = nName; pGroup->flags = flags; pSubOpen = multiplexSubOpen(pMultiplexOpen, 0, &rc, pOutFlags); if( pSubOpen ){ |
︙ | ︙ | |||
340 341 342 343 344 345 346 | int nName = sqlite3Strlen30(zName); int i; UNUSED_PARAMETER(pVfs); multiplexEnter(); memcpy(gMultiplex.zName, zName, nName+1); | | | | | | | < < < < | < < < < | 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 | int nName = sqlite3Strlen30(zName); int i; UNUSED_PARAMETER(pVfs); multiplexEnter(); memcpy(gMultiplex.zName, zName, nName+1); for(i=0; i<SQLITE_MULTIPLEX_MAX_CHUNKS; i++){ int rc2; int exists = 0; if( i ){ #ifdef SQLITE_MULTIPLEX_EXT_OVWR sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, gMultiplex.zName+nName-SQLITE_MULTIPLEX_EXT_SZ, SQLITE_MULTIPLEX_EXT_FMT, i); #else sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, gMultiplex.zName+nName, SQLITE_MULTIPLEX_EXT_FMT, i); #endif } rc2 = pOrigVfs->xAccess(pOrigVfs, gMultiplex.zName, SQLITE_ACCESS_EXISTS, &exists); if( rc2==SQLITE_OK && exists){ /* if it exists, delete it */ rc2 = pOrigVfs->xDelete(pOrigVfs, gMultiplex.zName, syncDir); if( rc2!=SQLITE_OK ) rc = rc2; }else{ /* stop at first "gap" */ break; } } multiplexLeave(); return rc; } static int multiplexAccess(sqlite3_vfs *a, const char *b, int c, int *d){ return gMultiplex.pOrigVfs->xAccess(gMultiplex.pOrigVfs, b, c, d); } static int multiplexFullPathname(sqlite3_vfs *a, const char *b, int c, char *d){ return gMultiplex.pOrigVfs->xFullPathname(gMultiplex.pOrigVfs, b, c, d); } static void *multiplexDlOpen(sqlite3_vfs *a, const char *b){ return gMultiplex.pOrigVfs->xDlOpen(gMultiplex.pOrigVfs, b); } static void multiplexDlError(sqlite3_vfs *a, int b, char *c){ gMultiplex.pOrigVfs->xDlError(gMultiplex.pOrigVfs, b, c); } |
︙ | ︙ | |||
420 421 422 423 424 425 426 | static int multiplexClose(sqlite3_file *pConn){ multiplexConn *p = (multiplexConn*)pConn; multiplexGroup *pGroup = p->pGroup; int rc = SQLITE_OK; int i; multiplexEnter(); /* close any open handles */ | | | 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 | static int multiplexClose(sqlite3_file *pConn){ multiplexConn *p = (multiplexConn*)pConn; multiplexGroup *pGroup = p->pGroup; int rc = SQLITE_OK; int i; multiplexEnter(); /* close any open handles */ for(i=0; i<pGroup->nMaxChunks; i++){ if( pGroup->bOpen[i] ){ sqlite3_file *pSubOpen = pGroup->pReal[i]; int rc2 = pSubOpen->pMethods->xClose(pSubOpen); if( rc2!=SQLITE_OK ) rc = rc2; pGroup->bOpen[i] = 0; } } |
︙ | ︙ | |||
451 452 453 454 455 456 457 458 459 460 | static int multiplexRead( sqlite3_file *pConn, void *pBuf, int iAmt, sqlite3_int64 iOfst ){ multiplexConn *p = (multiplexConn*)pConn; int rc = SQLITE_OK; multiplexEnter(); while( iAmt > 0 ){ | > | | | | 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 | static int multiplexRead( sqlite3_file *pConn, void *pBuf, int iAmt, sqlite3_int64 iOfst ){ multiplexConn *p = (multiplexConn*)pConn; multiplexGroup *pGroup = p->pGroup; int rc = SQLITE_OK; multiplexEnter(); while( iAmt > 0 ){ int i = (int)(iOfst / pGroup->nChunkSize); sqlite3_file *pSubOpen = multiplexSubOpen(p, i, &rc, NULL); if( pSubOpen ){ int extra = ((int)(iOfst % pGroup->nChunkSize) + iAmt) - pGroup->nChunkSize; if( extra<0 ) extra = 0; iAmt -= extra; rc = pSubOpen->pMethods->xRead(pSubOpen, pBuf, iAmt, iOfst % pGroup->nChunkSize); if( rc!=SQLITE_OK ) break; pBuf = (char *)pBuf + iAmt; iOfst += iAmt; iAmt = extra; }else{ rc = SQLITE_IOERR_READ; break; |
︙ | ︙ | |||
485 486 487 488 489 490 491 492 493 494 | static int multiplexWrite( sqlite3_file *pConn, const void *pBuf, int iAmt, sqlite3_int64 iOfst ){ multiplexConn *p = (multiplexConn*)pConn; int rc = SQLITE_OK; multiplexEnter(); while( iAmt > 0 ){ | > | | | | 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 | static int multiplexWrite( sqlite3_file *pConn, const void *pBuf, int iAmt, sqlite3_int64 iOfst ){ multiplexConn *p = (multiplexConn*)pConn; multiplexGroup *pGroup = p->pGroup; int rc = SQLITE_OK; multiplexEnter(); while( iAmt > 0 ){ int i = (int)(iOfst / pGroup->nChunkSize); sqlite3_file *pSubOpen = multiplexSubOpen(p, i, &rc, NULL); if( pSubOpen ){ int extra = ((int)(iOfst % pGroup->nChunkSize) + iAmt) - pGroup->nChunkSize; if( extra<0 ) extra = 0; iAmt -= extra; rc = pSubOpen->pMethods->xWrite(pSubOpen, pBuf, iAmt, iOfst % pGroup->nChunkSize); if( rc!=SQLITE_OK ) break; pBuf = (char *)pBuf + iAmt; iOfst += iAmt; iAmt = extra; }else{ rc = SQLITE_IOERR_WRITE; break; |
︙ | ︙ | |||
523 524 525 526 527 528 529 | int rc2; int i; sqlite3_file *pSubOpen; sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ multiplexEnter(); memcpy(gMultiplex.zName, pGroup->zName, pGroup->nName+1); /* delete the chunks above the truncate limit */ | | | | | | | 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 | int rc2; int i; sqlite3_file *pSubOpen; sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ multiplexEnter(); memcpy(gMultiplex.zName, pGroup->zName, pGroup->nName+1); /* delete the chunks above the truncate limit */ for(i=(int)(size / pGroup->nChunkSize)+1; i<pGroup->nMaxChunks; i++){ /* close any open chunks before deleting them */ if( pGroup->bOpen[i] ){ pSubOpen = pGroup->pReal[i]; rc2 = pSubOpen->pMethods->xClose(pSubOpen); if( rc2!=SQLITE_OK ) rc = SQLITE_IOERR_TRUNCATE; pGroup->bOpen[i] = 0; } #ifdef SQLITE_MULTIPLEX_EXT_OVWR sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, gMultiplex.zName+pGroup->nName-SQLITE_MULTIPLEX_EXT_SZ, SQLITE_MULTIPLEX_EXT_FMT, i); #else sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, gMultiplex.zName+pGroup->nName, SQLITE_MULTIPLEX_EXT_FMT, i); #endif rc2 = pOrigVfs->xDelete(pOrigVfs, gMultiplex.zName, 0); if( rc2!=SQLITE_OK ) rc = SQLITE_IOERR_TRUNCATE; } pSubOpen = multiplexSubOpen(p, (int)(size / pGroup->nChunkSize), &rc2, NULL); if( pSubOpen ){ rc2 = pSubOpen->pMethods->xTruncate(pSubOpen, size % pGroup->nChunkSize); if( rc2!=SQLITE_OK ) rc = rc2; }else{ rc = SQLITE_IOERR_TRUNCATE; } multiplexLeave(); return rc; } /* Pass xSync requests through to the original VFS without change */ static int multiplexSync(sqlite3_file *pConn, int flags){ multiplexConn *p = (multiplexConn*)pConn; multiplexGroup *pGroup = p->pGroup; int rc = SQLITE_OK; int i; multiplexEnter(); for(i=0; i<pGroup->nMaxChunks; i++){ /* if we don't have it open, we don't need to sync it */ if( pGroup->bOpen[i] ){ sqlite3_file *pSubOpen = pGroup->pReal[i]; int rc2 = pSubOpen->pMethods->xSync(pSubOpen, flags); if( rc2!=SQLITE_OK ) rc = rc2; } } |
︙ | ︙ | |||
581 582 583 584 585 586 587 | multiplexConn *p = (multiplexConn*)pConn; multiplexGroup *pGroup = p->pGroup; int rc = SQLITE_OK; int rc2; int i; multiplexEnter(); *pSize = 0; | | | | | 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 | multiplexConn *p = (multiplexConn*)pConn; multiplexGroup *pGroup = p->pGroup; int rc = SQLITE_OK; int rc2; int i; multiplexEnter(); *pSize = 0; for(i=0; i<pGroup->nMaxChunks; i++){ sqlite3_file *pSubOpen = NULL; /* if not opened already, check to see if the chunk exists */ if( pGroup->bOpen[i] ){ pSubOpen = pGroup->pReal[i]; }else{ sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ int exists = 0; memcpy(gMultiplex.zName, pGroup->zName, pGroup->nName+1); if( i ){ #ifdef SQLITE_MULTIPLEX_EXT_OVWR sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, gMultiplex.zName+pGroup->nName-SQLITE_MULTIPLEX_EXT_SZ, SQLITE_MULTIPLEX_EXT_FMT, i); #else sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, gMultiplex.zName+pGroup->nName, SQLITE_MULTIPLEX_EXT_FMT, i); #endif } rc2 = pOrigVfs->xAccess(pOrigVfs, gMultiplex.zName, SQLITE_ACCESS_EXISTS, &exists); if( rc2==SQLITE_OK && exists){ /* if it exists, open it */ pSubOpen = multiplexSubOpen(p, i, &rc, NULL); }else{ /* stop at first "gap" */ break; } } if( pSubOpen ){ sqlite3_int64 sz; rc2 = pSubOpen->pMethods->xFileSize(pSubOpen, &sz); if( rc2!=SQLITE_OK ){ rc = rc2; }else{ if( sz>pGroup->nChunkSize ){ rc = SQLITE_IOERR_FSTAT; } *pSize += sz; } }else{ break; } |
︙ | ︙ | |||
665 666 667 668 669 670 671 | return SQLITE_IOERR_CHECKRESERVEDLOCK; } /* Pass xFileControl requests through to the original VFS unchanged. */ static int multiplexFileControl(sqlite3_file *pConn, int op, void *pArg){ multiplexConn *p = (multiplexConn*)pConn; | > | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | > > | | 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 | return SQLITE_IOERR_CHECKRESERVEDLOCK; } /* Pass xFileControl requests through to the original VFS unchanged. */ static int multiplexFileControl(sqlite3_file *pConn, int op, void *pArg){ multiplexConn *p = (multiplexConn*)pConn; multiplexGroup *pGroup = p->pGroup; int rc = SQLITE_ERROR; sqlite3_file *pSubOpen; if( !gMultiplex.isInitialized ) return SQLITE_MISUSE; switch( op ){ case MULTIPLEX_CTRL_ENABLE: if( pArg ) { int bEnabled = *(int *)pArg; pGroup->bEnabled = bEnabled; rc = SQLITE_OK; } break; case MULTIPLEX_CTRL_SET_CHUNK_SIZE: if( pArg ) { int nChunkSize = *(int *)pArg; if( nChunkSize<32 ){ rc = SQLITE_MISUSE; }else{ pGroup->nChunkSize = nChunkSize; rc = SQLITE_OK; } } break; case MULTIPLEX_CTRL_SET_MAX_CHUNKS: if( pArg ) { int nMaxChunks = *(int *)pArg; if(( nMaxChunks<1 ) || ( nMaxChunks>99 )){ rc = SQLITE_MISUSE; }else{ pGroup->nMaxChunks = nMaxChunks; rc = SQLITE_OK; } } break; default: pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); if( pSubOpen ){ rc = pSubOpen->pMethods->xFileControl(pSubOpen, op, pArg); } break; } return rc; } /* Pass xSectorSize requests through to the original VFS unchanged. */ static int multiplexSectorSize(sqlite3_file *pConn){ multiplexConn *p = (multiplexConn*)pConn; int rc; |
︙ | ︙ | |||
784 785 786 787 788 789 790 | return SQLITE_NOMEM; } gMultiplex.zName = sqlite3_malloc(pOrigVfs->mxPathname); if( !gMultiplex.zName ){ sqlite3_mutex_free(gMultiplex.pMutex); return SQLITE_NOMEM; } | < < | 801 802 803 804 805 806 807 808 809 810 811 812 813 814 | return SQLITE_NOMEM; } gMultiplex.zName = sqlite3_malloc(pOrigVfs->mxPathname); if( !gMultiplex.zName ){ sqlite3_mutex_free(gMultiplex.pMutex); return SQLITE_NOMEM; } gMultiplex.pGroups = NULL; gMultiplex.isInitialized = 1; gMultiplex.pOrigVfs = pOrigVfs; gMultiplex.sThisVfs = *pOrigVfs; gMultiplex.sThisVfs.szOsFile += sizeof(multiplexConn); gMultiplex.sThisVfs.zName = SQLITE_MULTIPLEX_VFS_NAME; gMultiplex.sThisVfs.xOpen = multiplexOpen; |
︙ | ︙ | |||
826 827 828 829 830 831 832 833 834 835 836 837 838 839 | gMultiplex.sIoMethodsV2 = gMultiplex.sIoMethodsV1; gMultiplex.sIoMethodsV2.iVersion = 2; gMultiplex.sIoMethodsV2.xShmMap = multiplexShmMap; gMultiplex.sIoMethodsV2.xShmLock = multiplexShmLock; gMultiplex.sIoMethodsV2.xShmBarrier = multiplexShmBarrier; gMultiplex.sIoMethodsV2.xShmUnmap = multiplexShmUnmap; sqlite3_vfs_register(&gMultiplex.sThisVfs, makeDefault); return SQLITE_OK; } /* ** Shutdown the multiplex system. ** ** All SQLite database connections must be closed before calling this | > > > | 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 | gMultiplex.sIoMethodsV2 = gMultiplex.sIoMethodsV1; gMultiplex.sIoMethodsV2.iVersion = 2; gMultiplex.sIoMethodsV2.xShmMap = multiplexShmMap; gMultiplex.sIoMethodsV2.xShmLock = multiplexShmLock; gMultiplex.sIoMethodsV2.xShmBarrier = multiplexShmBarrier; gMultiplex.sIoMethodsV2.xShmUnmap = multiplexShmUnmap; sqlite3_vfs_register(&gMultiplex.sThisVfs, makeDefault); sqlite3_auto_extension((void*)multiplexFuncInit); return SQLITE_OK; } /* ** Shutdown the multiplex system. ** ** All SQLite database connections must be closed before calling this |
︙ | ︙ | |||
849 850 851 852 853 854 855 | sqlite3_free(gMultiplex.zName); sqlite3_mutex_free(gMultiplex.pMutex); sqlite3_vfs_unregister(&gMultiplex.sThisVfs); memset(&gMultiplex, 0, sizeof(gMultiplex)); return SQLITE_OK; } | < < < < < < < < < < < < < < < < < < < < < < < | 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 | sqlite3_free(gMultiplex.zName); sqlite3_mutex_free(gMultiplex.pMutex); sqlite3_vfs_unregister(&gMultiplex.sThisVfs); memset(&gMultiplex, 0, sizeof(gMultiplex)); return SQLITE_OK; } /***************************** Test Code ***********************************/ #ifdef SQLITE_TEST #include <tcl.h> /* ** tclcmd: sqlite3_multiplex_initialize NAME MAKEDEFAULT */ static int test_multiplex_initialize( void * clientData, Tcl_Interp *interp, |
︙ | ︙ | |||
933 934 935 936 937 938 939 | /* Call sqlite3_multiplex_shutdown() */ rc = sqlite3_multiplex_shutdown(); Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); return TCL_OK; } | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | 928 929 930 931 932 933 934 935 936 937 938 939 940 941 | /* Call sqlite3_multiplex_shutdown() */ rc = sqlite3_multiplex_shutdown(); Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); return TCL_OK; } /* ** tclcmd: sqlite3_multiplex_dump */ static int test_multiplex_dump( void * clientData, Tcl_Interp *interp, int objc, |
︙ | ︙ | |||
996 997 998 999 1000 1001 1002 | Tcl_NewStringObj(pGroup->zName, -1)); Tcl_ListObjAppendElement(interp, pGroupTerm, Tcl_NewIntObj(pGroup->nName)); Tcl_ListObjAppendElement(interp, pGroupTerm, Tcl_NewIntObj(pGroup->flags)); /* count number of chunks with open handles */ | | | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > < > | 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 | Tcl_NewStringObj(pGroup->zName, -1)); Tcl_ListObjAppendElement(interp, pGroupTerm, Tcl_NewIntObj(pGroup->nName)); Tcl_ListObjAppendElement(interp, pGroupTerm, Tcl_NewIntObj(pGroup->flags)); /* count number of chunks with open handles */ for(i=0; i<pGroup->nMaxChunks; i++){ if( pGroup->bOpen[i] ) nChunks++; } Tcl_ListObjAppendElement(interp, pGroupTerm, Tcl_NewIntObj(nChunks)); Tcl_ListObjAppendElement(interp, pGroupTerm, Tcl_NewIntObj(pGroup->nChunkSize)); Tcl_ListObjAppendElement(interp, pGroupTerm, Tcl_NewIntObj(pGroup->nMaxChunks)); Tcl_ListObjAppendElement(interp, pResult, pGroupTerm); } multiplexLeave(); Tcl_SetObjResult(interp, pResult); return TCL_OK; } /* ** Tclcmd: test_multiplex_control HANDLE DBNAME SUB-COMMAND ?INT-VALUE? */ static int test_multiplex_control( ClientData cd, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[] ){ int rc; /* Return code from file_control() */ int idx; /* Index in aSub[] */ Tcl_CmdInfo cmdInfo; /* Command info structure for HANDLE */ sqlite3 *db; /* Underlying db handle for HANDLE */ int iValue = 0; void *pArg = 0; struct SubCommand { const char *zName; int op; int argtype; } aSub[] = { { "enable", MULTIPLEX_CTRL_ENABLE, 1 }, { "chunk_size", MULTIPLEX_CTRL_SET_CHUNK_SIZE, 1 }, { "max_chunks", MULTIPLEX_CTRL_SET_MAX_CHUNKS, 1 }, { 0, 0, 0 } }; if( objc!=4 && objc!=5 ){ Tcl_WrongNumArgs(interp, 1, objv, "HANDLE DBNAME SUB-COMMAND ?INT-VALUE?"); return TCL_ERROR; } if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(objv[1]), &cmdInfo) ){ Tcl_AppendResult(interp, "expected database handle, got \"", 0); Tcl_AppendResult(interp, Tcl_GetString(objv[1]), "\"", 0); return TCL_ERROR; }else{ db = *(sqlite3 **)cmdInfo.objClientData; } rc = Tcl_GetIndexFromObjStruct( interp, objv[3], aSub, sizeof(aSub[0]), "sub-command", 0, &idx ); if( rc!=TCL_OK ) return rc; switch( aSub[idx].argtype ){ case 1: if( objc!=5 ){ Tcl_WrongNumArgs(interp, 4, objv, "INT-VALUE"); return TCL_ERROR; } if( Tcl_GetIntFromObj(interp, objv[4], &iValue) ){ return TCL_ERROR; } pArg = (void *)&iValue; break; default: Tcl_WrongNumArgs(interp, 4, objv, "SUB-COMMAND"); return TCL_ERROR; } rc = sqlite3_file_control(db, Tcl_GetString(objv[2]), aSub[idx].op, pArg); Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); return (rc==SQLITE_OK) ? TCL_OK : TCL_ERROR; } /* ** This routine registers the custom TCL commands defined in this ** module. This should be the only procedure visible from outside ** of this module. */ int Sqlitemultiplex_Init(Tcl_Interp *interp){ static struct { char *zName; Tcl_ObjCmdProc *xProc; } aCmd[] = { { "sqlite3_multiplex_initialize", test_multiplex_initialize }, { "sqlite3_multiplex_shutdown", test_multiplex_shutdown }, { "sqlite3_multiplex_dump", test_multiplex_dump }, { "sqlite3_multiplex_control", test_multiplex_control }, }; int i; for(i=0; i<sizeof(aCmd)/sizeof(aCmd[0]); i++){ Tcl_CreateObjCommand(interp, aCmd[i].zName, aCmd[i].xProc, 0, 0); } return TCL_OK; } #endif |
Added src/test_multiplex.h.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 | /* ** 2011 March 18 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ************************************************************************* ** ** This file contains a VFS "shim" - a layer that sits in between the ** pager and the real VFS. ** ** This particular shim enforces a multiplex system on DB files. ** This shim shards/partitions a single DB file into smaller ** "chunks" such that the total DB file size may exceed the maximum ** file size of the underlying file system. ** */ #ifndef _TEST_MULTIPLEX_H #define _TEST_MULTIPLEX_H /* ** CAPI: File-control Operations Supported by Multiplex VFS ** ** Values interpreted by the xFileControl method of a Multiplex VFS db file-handle. ** ** MULTIPLEX_CTRL_ENABLE: ** This file control is used to enable or disable the multiplex ** shim. ** ** MULTIPLEX_CTRL_SET_CHUNK_SIZE: ** This file control is used to set the maximum allowed chunk ** size for a multiplex file set. ** ** MULTIPLEX_CTRL_SET_MAX_CHUNKS: ** This file control is used to set the maximum number of chunks ** allowed to be used for a mutliplex file set. */ #define MULTIPLEX_CTRL_ENABLE 214014 #define MULTIPLEX_CTRL_SET_CHUNK_SIZE 214015 #define MULTIPLEX_CTRL_SET_MAX_CHUNKS 214016 #endif |
Changes to test/multiplex.test.
︙ | ︙ | |||
28 29 30 31 32 33 34 | ifcapable {multiplex_ext_overwrite} { set name [string range $name 0 [expr [string length $name]-2-1]] } return $name$num } # This saves off the parameters and calls the | | | | > > > > | 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 | ifcapable {multiplex_ext_overwrite} { set name [string range $name 0 [expr [string length $name]-2-1]] } return $name$num } # This saves off the parameters and calls the # underlying sqlite3_multiplex_control() API. proc multiplex_set {db name chunk_size max_chunks} { global g_chunk_size global g_max_chunks set g_chunk_size $chunk_size set g_max_chunks $max_chunks set rc [catch {sqlite3_multiplex_control $db $name chunk_size $chunk_size} msg] if { $rc==0 } { set rc [catch {sqlite3_multiplex_control $db $name max_chunks $max_chunks} msg] } list $msg } # This attempts to delete the base file and # and files with the chunk extension. proc multiplex_delete {name} { global g_max_chunks for {set i 0} {$i<$g_max_chunks} {incr i} { |
︙ | ︙ | |||
67 68 69 70 71 72 73 | do_test multiplex-1.5 { sqlite3_multiplex_initialize "" 0 } {SQLITE_OK} do_test multiplex-1.6 { sqlite3_multiplex_shutdown } {SQLITE_OK} do_test multiplex-1.7 { sqlite3_multiplex_initialize "" 1 } {SQLITE_OK} do_test multiplex-1.8 { sqlite3_multiplex_shutdown } {SQLITE_OK} do_test multiplex-1.9 { sqlite3_multiplex_initialize "" 1 } {SQLITE_OK} | > | | | | | > < | | | 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 | do_test multiplex-1.5 { sqlite3_multiplex_initialize "" 0 } {SQLITE_OK} do_test multiplex-1.6 { sqlite3_multiplex_shutdown } {SQLITE_OK} do_test multiplex-1.7 { sqlite3_multiplex_initialize "" 1 } {SQLITE_OK} do_test multiplex-1.8 { sqlite3_multiplex_shutdown } {SQLITE_OK} do_test multiplex-1.9 { sqlite3_multiplex_initialize "" 1 } {SQLITE_OK} sqlite3 db test.db do_test multiplex-1.10.1 { multiplex_set db main 32768 16 } {SQLITE_OK} do_test multiplex-1.10.2 { multiplex_set db main 32768 -1 } {SQLITE_MISUSE} do_test multiplex-1.10.3 { multiplex_set db main -1 16 } {SQLITE_MISUSE} do_test multiplex-1.10.4 { multiplex_set db main 31 16 } {SQLITE_MISUSE} do_test multiplex-1.10.5 { multiplex_set db main 32768 100 } {SQLITE_MISUSE} db close do_test multiplex-1.11 { sqlite3_multiplex_shutdown } {SQLITE_OK} #------------------------------------------------------------------------- # Some simple warm-body tests with a single database file in rollback # mode: # # multiplex-2.1.*: Test simple writing to a multiplex file. # # multiplex-2.2.*: More writing. # # multiplex-2.3.*: Open and close a second db. # # multiplex-2.4.*: Try to shutdown the multiplex system befor e closing the db # file. Check that this fails and the multiplex system still works # afterwards. Then close the database and successfully shut # down the multiplex system. # # multiplex-2.5.*: More reading/writing. # # multiplex-2.6.*: More reading/writing with varying small chunk sizes, as # well as varying journal mode. sqlite3_multiplex_initialize "" 1 multiplex_set db main 32768 16 do_test multiplex-2.1.2 { sqlite3 db test.db execsql { PRAGMA page_size=1024; PRAGMA auto_vacuum=OFF; PRAGMA journal_mode=DELETE; |
︙ | ︙ | |||
125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 | } {} do_test multiplex-2.2.3 { file size [multiplex_name test.db 0] } {6144} do_test multiplex-2.3.1 { sqlite3 db2 test2.db db2 close } {} do_test multiplex-2.4.1 { sqlite3_multiplex_shutdown } {SQLITE_MISUSE} do_test multiplex-2.4.2 { execsql { INSERT INTO t1 VALUES(3, randomblob(1100)) } } {} do_test multiplex-2.4.4 { file size [multiplex_name test.db 0] } {7168} do_test multiplex-2.4.99 { db close sqlite3_multiplex_shutdown } {SQLITE_OK} do_test multiplex-2.5.1 { multiplex_delete test.db sqlite3_multiplex_initialize "" 1 | > > | < | 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 | } {} do_test multiplex-2.2.3 { file size [multiplex_name test.db 0] } {6144} do_test multiplex-2.3.1 { sqlite3 db2 test2.db db2 close } {} do_test multiplex-2.4.1 { sqlite3_multiplex_shutdown } {SQLITE_MISUSE} do_test multiplex-2.4.2 { execsql { INSERT INTO t1 VALUES(3, randomblob(1100)) } } {} do_test multiplex-2.4.4 { file size [multiplex_name test.db 0] } {7168} do_test multiplex-2.4.99 { db close sqlite3_multiplex_shutdown } {SQLITE_OK} do_test multiplex-2.5.1 { multiplex_delete test.db sqlite3_multiplex_initialize "" 1 sqlite3 db test.db multiplex_set db main 4096 16 } {SQLITE_OK} do_test multiplex-2.5.2 { execsql { PRAGMA page_size = 1024; PRAGMA journal_mode = delete; PRAGMA auto_vacuum = off; CREATE TABLE t1(a PRIMARY KEY, b); } } {delete} |
︙ | ︙ | |||
193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 | do_test multiplex-2.5.10 { file size [multiplex_name test.db 1] } [list $g_chunk_size] do_test multiplex-2.5.99 { db close sqlite3_multiplex_shutdown } {SQLITE_OK} set all_journal_modes {delete persist truncate memory off} foreach jmode $all_journal_modes { for {set sz 151} {$sz<8000} {set sz [expr $sz+419]} { do_test multiplex-2.6.1.$sz.$jmode { multiplex_delete test.db sqlite3_multiplex_initialize "" 1 | > > > | | 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 | do_test multiplex-2.5.10 { file size [multiplex_name test.db 1] } [list $g_chunk_size] do_test multiplex-2.5.99 { db close sqlite3_multiplex_shutdown } {SQLITE_OK} return # TBD fix the below set all_journal_modes {delete persist truncate memory off} foreach jmode $all_journal_modes { for {set sz 151} {$sz<8000} {set sz [expr $sz+419]} { do_test multiplex-2.6.1.$sz.$jmode { multiplex_delete test.db sqlite3_multiplex_initialize "" 1 multiplex_set db main $sz 32 } {SQLITE_OK} do_test multiplex-2.6.2.$sz.$jmode { sqlite3 db test.db db eval { PRAGMA page_size = 1024; PRAGMA auto_vacuum = off; |
︙ | ︙ | |||
251 252 253 254 255 256 257 | # # multiplex-3.2.*: Two connections to each of several database files (that # are in the same multiplex group). # do_test multiplex-3.1.1 { multiplex_delete test.db sqlite3_multiplex_initialize "" 1 | | | 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 | # # multiplex-3.2.*: Two connections to each of several database files (that # are in the same multiplex group). # do_test multiplex-3.1.1 { multiplex_delete test.db sqlite3_multiplex_initialize "" 1 multiplex_set db main 32768 16 } {SQLITE_OK} do_test multiplex-3.1.2 { sqlite3 db test.db execsql { PRAGMA page_size = 1024; PRAGMA journal_mode = delete; PRAGMA auto_vacuum = off; |
︙ | ︙ | |||
337 338 339 340 341 342 343 | foreach db {db1a db2a db2b db1b} { catch { $db close } } } {} #------------------------------------------------------------------------- # sqlite3_multiplex_initialize "" 1 | | | 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 | foreach db {db1a db2a db2b db1b} { catch { $db close } } } {} #------------------------------------------------------------------------- # sqlite3_multiplex_initialize "" 1 multiplex_set db main 32768 16 # Return a list of all currently defined multiplexs. proc multiplex_list {} { set allq {} foreach q [sqlite3_multiplex_dump] { lappend allq [lindex $q 0] } |
︙ | ︙ | |||
399 400 401 402 403 404 405 | #------------------------------------------------------------------------- # The following tests test that the multiplex VFS handles malloc and IO # errors. # sqlite3_multiplex_initialize "" 1 | | | 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 | #------------------------------------------------------------------------- # The following tests test that the multiplex VFS handles malloc and IO # errors. # sqlite3_multiplex_initialize "" 1 multiplex_set db main 32768 16 do_faultsim_test multiplex-5.1 -prep { catch {db close} } -body { sqlite3 db test2.db } do_faultsim_test multiplex-5.2 -prep { |
︙ | ︙ | |||
444 445 446 447 448 449 450 | } {1 {unable to open database file}} catch { file delete test.db } do_faultsim_test multiplex-5.5 -prep { catch { sqlite3_multiplex_shutdown } } -body { sqlite3_multiplex_initialize "" 1 | | | 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 | } {1 {unable to open database file}} catch { file delete test.db } do_faultsim_test multiplex-5.5 -prep { catch { sqlite3_multiplex_shutdown } } -body { sqlite3_multiplex_initialize "" 1 multiplex_set db main 32768 16 } # test that mismatch filesize is detected # # Do not run this test if $::G(perm:presql) is set. If it is set, then the # expected IO error will occur within the Tcl [sqlite3] wrapper, not within # the first SQL statement executed below. This breaks the test case. |
︙ | ︙ | |||
477 478 479 480 481 482 483 | INSERT INTO t1 VALUES(2, randomblob(1100)); INSERT INTO t1 VALUES(3, randomblob(1100)); INSERT INTO t1 VALUES(4, randomblob(1100)); INSERT INTO t1 VALUES(5, randomblob(1100)); } db close sqlite3_multiplex_initialize "" 1 | | | 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 | INSERT INTO t1 VALUES(2, randomblob(1100)); INSERT INTO t1 VALUES(3, randomblob(1100)); INSERT INTO t1 VALUES(4, randomblob(1100)); INSERT INTO t1 VALUES(5, randomblob(1100)); } db close sqlite3_multiplex_initialize "" 1 multiplex_set db main 4096 16 sqlite3 db test.db } {} do_test multiplex-5.6.3.$jmode { catchsql { INSERT INTO t1 VALUES(6, randomblob(1100)); } } {1 {disk I/O error}} |
︙ | ︙ |