/ Check-in [dcc40797]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Make the freelist format a separate feature from the page-level locking. Freelist format is now configure using "PRAGMA freelist_format".
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | server-process-edition
Files: files | file ages | folders
SHA3-256: dcc407972aa213d28f95cbaf4da5ab7d89ddbad2740f41cfb44b2c2ce20a3132
User & Date: dan 2017-07-13 21:06:51
Context
2017-07-14
08:15
Add simple tests for "PRAGMA freelist_format". check-in: 98a36f4c user: dan tags: server-process-edition
2017-07-13
21:06
Make the freelist format a separate feature from the page-level locking. Freelist format is now configure using "PRAGMA freelist_format". check-in: dcc40797 user: dan tags: server-process-edition
2017-07-11
18:38
Add SQL function usleep() to test program tserver.c. check-in: 8cbe8f2b user: dan tags: server-process-edition
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/btree.c.

   234    234         return 1;
   235    235       }
   236    236     }
   237    237     return 0;
   238    238   }
   239    239   #endif    /* #ifdef SQLITE_DEBUG */
   240    240   
          241  +#ifdef SQLITE_SERVER_EDITION
          242  +/*
          243  +** Return true if the b-tree uses free-list format 2. Or false otherwise.
          244  +*/
          245  +static int btreeFreelistFormat2(BtShared *pBt){
          246  +  return (pBt->pPage1->aData[18] > 2);
          247  +}
          248  +#endif
          249  +
   241    250   /*
   242    251   ** Query to see if Btree handle p may obtain a lock of type eLock 
   243    252   ** (READ_LOCK or WRITE_LOCK) on the table with root-page iTab. Return
   244    253   ** SQLITE_OK if the lock may be obtained (by calling
   245    254   ** setSharedCacheTableLock()), or SQLITE_LOCKED if not.
   246    255   */
   247    256   static int querySharedCacheTableLock(Btree *p, Pgno iTab, u8 eLock){
................................................................................
  2923   2932     if( nPage==0 || memcmp(24+(u8*)pPage1->aData, 92+(u8*)pPage1->aData,4)!=0 ){
  2924   2933       nPage = nPageFile;
  2925   2934     }
  2926   2935     if( nPage>0 ){
  2927   2936       u32 pageSize;
  2928   2937       u32 usableSize;
  2929   2938       u8 *page1 = pPage1->aData;
         2939  +    u8 i18 = page1[18];
         2940  +    u8 i19 = page1[19];
         2941  +#ifdef SQLITE_SERVER_EDITION
         2942  +    if( i18==i19 && i18>2 ){
         2943  +      i18 -= 2;
         2944  +      i19 -= 2;
         2945  +    }
         2946  +#endif
         2947  +
  2930   2948       rc = SQLITE_NOTADB;
  2931   2949       /* EVIDENCE-OF: R-43737-39999 Every valid SQLite database file begins
  2932   2950       ** with the following 16 bytes (in hex): 53 51 4c 69 74 65 20 66 6f 72 6d
  2933   2951       ** 61 74 20 33 00. */
  2934   2952       if( memcmp(page1, zMagicHeader, 16)!=0 ){
  2935   2953         goto page1_init_failed;
  2936   2954       }
  2937   2955   
  2938   2956   #ifdef SQLITE_OMIT_WAL
  2939         -    if( page1[18]>1 ){
         2957  +    if( i18>1 ){
  2940   2958         pBt->btsFlags |= BTS_READ_ONLY;
  2941   2959       }
  2942         -    if( page1[19]>1 ){
         2960  +    if( i19>1 ){
  2943   2961         goto page1_init_failed;
  2944   2962       }
  2945   2963   #else
  2946         -    if( page1[18]>2 ){
         2964  +    if( i18>2 ){
  2947   2965         pBt->btsFlags |= BTS_READ_ONLY;
  2948   2966       }
  2949         -    if( page1[19]>2 ){
         2967  +    if( i19>2 ){
  2950   2968         goto page1_init_failed;
  2951   2969       }
  2952   2970   
  2953   2971       /* If the write version is set to 2, this database should be accessed
  2954   2972       ** in WAL mode. If the log is not already open, open it now. Then 
  2955   2973       ** return SQLITE_OK and return without populating BtShared.pPage1.
  2956   2974       ** The caller detects this and calls this function again. This is
  2957   2975       ** required as the version of page 1 currently in the page1 buffer
  2958   2976       ** may not be the latest version - there may be a newer one in the log
  2959   2977       ** file.
  2960   2978       */
  2961         -    if( page1[19]==2 && (pBt->btsFlags & BTS_NO_WAL)==0 ){
         2979  +    if( i19==2 && (pBt->btsFlags & BTS_NO_WAL)==0 ){
  2962   2980         int isOpen = 0;
  2963   2981         rc = sqlite3PagerOpenWal(pBt->pPager, &isOpen);
  2964   2982         if( rc!=SQLITE_OK ){
  2965   2983           goto page1_init_failed;
  2966   2984         }else{
  2967   2985           setDefaultSyncFlag(pBt, SQLITE_DEFAULT_WAL_SYNCHRONOUS+1);
  2968   2986           if( isOpen==0 ){
................................................................................
  5896   5914     int rc;
  5897   5915     u32 n;     /* Number of pages on the freelist */
  5898   5916     u32 k;     /* Number of leaves on the trunk of the freelist */
  5899   5917     MemPage *pTrunk = 0;
  5900   5918     MemPage *pPrevTrunk = 0;
  5901   5919     Pgno mxPage;     /* Total size of the database file */
  5902   5920   
  5903         -  if( sqlite3PagerIsServer(pBt->pPager) ){
         5921  +  if( btreeFreelistFormat2(pBt) ){
  5904   5922       return allocateServerPage(pBt, ppPage, pPgno, nearby, eMode); 
  5905   5923     }
  5906   5924   
  5907   5925     assert( sqlite3_mutex_held(pBt->mutex) );
  5908   5926     assert( eMode==BTALLOC_ANY || (nearby>0 && IfNotOmitAV(pBt->autoVacuum)) );
  5909   5927     pPage1 = pBt->pPage1;
  5910   5928     mxPage = btreePagecount(pBt);
................................................................................
  6239   6257        ||            ((rc = sqlite3PagerWrite(pPage->pDbPage))!=0)
  6240   6258       ){
  6241   6259         goto freepage_out;
  6242   6260       }
  6243   6261       memset(pPage->aData, 0, pPage->pBt->pageSize);
  6244   6262     }
  6245   6263     
  6246         -  if( sqlite3PagerIsServer(pBt->pPager) ){
         6264  +  if( btreeFreelistFormat2(pBt) ){
  6247   6265       rc = freeServerPage2(pBt, pPage, iPage);
  6248   6266       goto freepage_out;
  6249   6267     }
  6250   6268   
  6251   6269     /* Increment the free page count on pPage1 */
  6252   6270     rc = sqlite3PagerWrite(pPage1->pDbPage);
  6253   6271     if( rc ) goto freepage_out;
................................................................................
  9812   9830     i = PENDING_BYTE_PAGE(pBt);
  9813   9831     if( i<=sCheck.nPage ) setPageReferenced(&sCheck, i);
  9814   9832   
  9815   9833     /* Check the integrity of the freelist
  9816   9834     */
  9817   9835     sCheck.zPfx = "Main freelist: ";
  9818   9836   #ifdef SQLITE_SERVER_EDITION
  9819         -  if( sqlite3PagerIsServer(pBt->pPager) ){
         9837  +  if( btreeFreelistFormat2(pBt) ){
  9820   9838       checkServerList(&sCheck);
  9821   9839     }else
  9822   9840   #endif
  9823   9841     {
  9824   9842       checkList(&sCheck, 1, get4byte(&pBt->pPage1->aData[32]),
  9825   9843           get4byte(&pBt->pPage1->aData[36]));
  9826   9844     }
................................................................................
 10087  10105   ** Mark this cursor as an incremental blob cursor.
 10088  10106   */
 10089  10107   void sqlite3BtreeIncrblobCursor(BtCursor *pCur){
 10090  10108     pCur->curFlags |= BTCF_Incrblob;
 10091  10109     pCur->pBtree->hasIncrblobCur = 1;
 10092  10110   }
 10093  10111   #endif
        10112  +
        10113  +int btreeSetVersion(Btree *pBtree, int iVersion, int iFreelistFmt){
        10114  +  BtShared *pBt = pBtree->pBt;
        10115  +  int rc = sqlite3BtreeBeginTrans(pBtree, 0);
        10116  +  if( rc==SQLITE_OK ){
        10117  +    u8 iVal;
        10118  +    u8 *aData = pBt->pPage1->aData;
        10119  +
        10120  +    assert( (iVersion==0 && (iFreelistFmt==1 || iFreelistFmt==2))
        10121  +         || (iFreelistFmt==0 && (iVersion==1 || iVersion==2))
        10122  +    );
        10123  +    if( iVersion==0 ){
        10124  +      iVal = ((aData[18] & 0x01) ? 1 : 2) + (u8)(iFreelistFmt==2 ? 2 : 0);
        10125  +    }else{
        10126  +      iVal = (u8)iVersion + (u8)(aData[18]>2 ? 2 : 0);
        10127  +    }
        10128  +
        10129  +    if( aData[18]!=iVal || aData[19]!=iVal ){
        10130  +      rc = sqlite3BtreeBeginTrans(pBtree, 2);
        10131  +      if( rc==SQLITE_OK ){
        10132  +        rc = sqlite3PagerWrite(pBt->pPage1->pDbPage);
        10133  +        if( rc==SQLITE_OK ){
        10134  +          aData[18] = iVal;
        10135  +          aData[19] = iVal;
        10136  +        }
        10137  +      }
        10138  +    }
        10139  +  }
        10140  +
        10141  +  return rc;
        10142  +}
 10094  10143   
 10095  10144   /*
 10096  10145   ** Set both the "read version" (single byte at byte offset 18) and 
 10097  10146   ** "write version" (single byte at byte offset 19) fields in the database
 10098  10147   ** header to iVersion.
 10099  10148   */
 10100  10149   int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){
................................................................................
 10104  10153     assert( iVersion==1 || iVersion==2 );
 10105  10154   
 10106  10155     /* If setting the version fields to 1, do not automatically open the
 10107  10156     ** WAL connection, even if the version fields are currently set to 2.
 10108  10157     */
 10109  10158     pBt->btsFlags &= ~BTS_NO_WAL;
 10110  10159     if( iVersion==1 ) pBt->btsFlags |= BTS_NO_WAL;
        10160  +  rc = btreeSetVersion(pBtree, iVersion, 0); 
        10161  +  pBt->btsFlags &= ~BTS_NO_WAL;
        10162  +  return rc;
        10163  +}
 10111  10164   
 10112         -  rc = sqlite3BtreeBeginTrans(pBtree, 0);
 10113         -  if( rc==SQLITE_OK ){
 10114         -    u8 *aData = pBt->pPage1->aData;
 10115         -    if( aData[18]!=(u8)iVersion || aData[19]!=(u8)iVersion ){
 10116         -      rc = sqlite3BtreeBeginTrans(pBtree, 2);
 10117         -      if( rc==SQLITE_OK ){
 10118         -        rc = sqlite3PagerWrite(pBt->pPage1->pDbPage);
 10119         -        if( rc==SQLITE_OK ){
 10120         -          aData[18] = (u8)iVersion;
 10121         -          aData[19] = (u8)iVersion;
 10122         -        }
 10123         -      }
        10165  +int sqlite3BtreeFreelistFormat(Btree *p, int eParam, int *peFmt){
        10166  +  int rc = SQLITE_OK;
        10167  +  sqlite3BtreeEnter(p);
        10168  +  if( eParam ){
        10169  +    u8 *aData = p->pBt->pPage1->aData;
        10170  +    if( 0==get4byte(&aData[32]) ){
        10171  +      rc = btreeSetVersion(p, 0, eParam);
 10124  10172       }
 10125  10173     }
 10126         -
 10127         -  pBt->btsFlags &= ~BTS_NO_WAL;
        10174  +  *peFmt = (btreeFreelistFormat2(p->pBt) ? 2 : 1);
        10175  +  sqlite3BtreeLeave(p);
 10128  10176     return rc;
 10129  10177   }
 10130  10178   
 10131  10179   /*
 10132  10180   ** Return true if the cursor has a hint specified.  This routine is
 10133  10181   ** only used from within assert() statements
 10134  10182   */

Changes to src/btree.h.

   363    363   # define sqlite3BtreeLeaveAll(X)
   364    364   
   365    365   # define sqlite3BtreeHoldsMutex(X) 1
   366    366   # define sqlite3BtreeHoldsAllMutexes(X) 1
   367    367   # define sqlite3SchemaMutexHeld(X,Y,Z) 1
   368    368   #endif
   369    369   
          370  +#ifdef SQLITE_SERVER_EDITION
          371  +int sqlite3BtreeFreelistFormat(Btree *p, int eParam, int *peFmt);
          372  +#endif
   370    373   
   371    374   #endif /* SQLITE_BTREE_H */

Changes to src/pragma.c.

   651    651           sqlite3VdbeUsesBtree(v, ii);
   652    652           sqlite3VdbeAddOp3(v, OP_JournalMode, ii, 1, eMode);
   653    653         }
   654    654       }
   655    655       sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1);
   656    656       break;
   657    657     }
          658  +
          659  +#ifdef SQLITE_SERVER_EDITION
          660  +  /*
          661  +  **  PRAGMA [schema.]freelist_format
          662  +  **  PRAGMA [schema.]freelist_format = (1|2)
          663  +  */
          664  +  case PragTyp_FREELIST_FORMAT: {
          665  +    sqlite3VdbeUsesBtree(v, iDb);
          666  +    static const VdbeOpList freelist[] = {
          667  +      { OP_Transaction,    0,  0,  0},    /* 0 */
          668  +      { OP_FreelistFmt,    0,  1,  0},    /* 1 */
          669  +      { OP_ResultRow,      1,  1,  0}     /* 2 */
          670  +    };
          671  +    VdbeOp *aOp;
          672  +    sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(freelist));
          673  +    aOp = sqlite3VdbeAddOpList(v, ArraySize(freelist), freelist,0);
          674  +    aOp[0].p1 = iDb;
          675  +    aOp[1].p1 = iDb;
          676  +
          677  +    if( zRight && (zRight[0]=='1' || zRight[0]=='2') && zRight[1]=='\0' ){
          678  +      aOp[0].p2 = 1;              /* Open a write transaction */
          679  +      aOp[1].p3 = (int)(zRight[0] - '0');
          680  +    }
          681  +
          682  +    break;
          683  +  }
          684  +#endif
   658    685   
   659    686     /*
   660    687     **  PRAGMA [schema.]journal_size_limit
   661    688     **  PRAGMA [schema.]journal_size_limit=N
   662    689     **
   663    690     ** Get or set the size limit on rollback journal files.
   664    691     */

Changes to src/pragma.h.

    16     16   #define PragTyp_COMPILE_OPTIONS                8
    17     17   #define PragTyp_DATA_STORE_DIRECTORY           9
    18     18   #define PragTyp_DATABASE_LIST                 10
    19     19   #define PragTyp_DEFAULT_CACHE_SIZE            11
    20     20   #define PragTyp_ENCODING                      12
    21     21   #define PragTyp_FOREIGN_KEY_CHECK             13
    22     22   #define PragTyp_FOREIGN_KEY_LIST              14
    23         -#define PragTyp_INCREMENTAL_VACUUM            15
    24         -#define PragTyp_INDEX_INFO                    16
    25         -#define PragTyp_INDEX_LIST                    17
    26         -#define PragTyp_INTEGRITY_CHECK               18
    27         -#define PragTyp_JOURNAL_MODE                  19
    28         -#define PragTyp_JOURNAL_SIZE_LIMIT            20
    29         -#define PragTyp_LOCK_PROXY_FILE               21
    30         -#define PragTyp_LOCKING_MODE                  22
    31         -#define PragTyp_PAGE_COUNT                    23
    32         -#define PragTyp_MMAP_SIZE                     24
    33         -#define PragTyp_OPTIMIZE                      25
    34         -#define PragTyp_PAGE_SIZE                     26
    35         -#define PragTyp_SECURE_DELETE                 27
    36         -#define PragTyp_SHRINK_MEMORY                 28
    37         -#define PragTyp_SOFT_HEAP_LIMIT               29
    38         -#define PragTyp_SYNCHRONOUS                   30
    39         -#define PragTyp_TABLE_INFO                    31
    40         -#define PragTyp_TEMP_STORE                    32
    41         -#define PragTyp_TEMP_STORE_DIRECTORY          33
    42         -#define PragTyp_THREADS                       34
    43         -#define PragTyp_WAL_AUTOCHECKPOINT            35
    44         -#define PragTyp_WAL_CHECKPOINT                36
    45         -#define PragTyp_ACTIVATE_EXTENSIONS           37
    46         -#define PragTyp_HEXKEY                        38
    47         -#define PragTyp_KEY                           39
    48         -#define PragTyp_REKEY                         40
    49         -#define PragTyp_LOCK_STATUS                   41
    50         -#define PragTyp_PARSER_TRACE                  42
    51         -#define PragTyp_STATS                         43
           23  +#define PragTyp_FREELIST_FORMAT               15
           24  +#define PragTyp_INCREMENTAL_VACUUM            16
           25  +#define PragTyp_INDEX_INFO                    17
           26  +#define PragTyp_INDEX_LIST                    18
           27  +#define PragTyp_INTEGRITY_CHECK               19
           28  +#define PragTyp_JOURNAL_MODE                  20
           29  +#define PragTyp_JOURNAL_SIZE_LIMIT            21
           30  +#define PragTyp_LOCK_PROXY_FILE               22
           31  +#define PragTyp_LOCKING_MODE                  23
           32  +#define PragTyp_PAGE_COUNT                    24
           33  +#define PragTyp_MMAP_SIZE                     25
           34  +#define PragTyp_OPTIMIZE                      26
           35  +#define PragTyp_PAGE_SIZE                     27
           36  +#define PragTyp_SECURE_DELETE                 28
           37  +#define PragTyp_SHRINK_MEMORY                 29
           38  +#define PragTyp_SOFT_HEAP_LIMIT               30
           39  +#define PragTyp_SYNCHRONOUS                   31
           40  +#define PragTyp_TABLE_INFO                    32
           41  +#define PragTyp_TEMP_STORE                    33
           42  +#define PragTyp_TEMP_STORE_DIRECTORY          34
           43  +#define PragTyp_THREADS                       35
           44  +#define PragTyp_WAL_AUTOCHECKPOINT            36
           45  +#define PragTyp_WAL_CHECKPOINT                37
           46  +#define PragTyp_ACTIVATE_EXTENSIONS           38
           47  +#define PragTyp_HEXKEY                        39
           48  +#define PragTyp_KEY                           40
           49  +#define PragTyp_REKEY                         41
           50  +#define PragTyp_LOCK_STATUS                   42
           51  +#define PragTyp_PARSER_TRACE                  43
           52  +#define PragTyp_STATS                         44
    52     53   
    53     54   /* Property flags associated with various pragma. */
    54     55   #define PragFlg_NeedSchema 0x01 /* Force schema load before running */
    55     56   #define PragFlg_NoColumns  0x02 /* OP_ResultRow called with zero columns */
    56     57   #define PragFlg_NoColumns1 0x04 /* zero columns if RHS argument is present */
    57     58   #define PragFlg_ReadOnly   0x08 /* Read-only HEADER_VALUE */
    58     59   #define PragFlg_Result0    0x10 /* Acts as query when no argument */
................................................................................
   289    290   #endif
   290    291   #if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS)
   291    292    {/* zName:     */ "freelist_count",
   292    293     /* ePragTyp:  */ PragTyp_HEADER_VALUE,
   293    294     /* ePragFlg:  */ PragFlg_ReadOnly|PragFlg_Result0,
   294    295     /* ColNames:  */ 0, 0,
   295    296     /* iArg:      */ BTREE_FREE_PAGE_COUNT },
          297  +#endif
          298  +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && defined(SQLITE_SERVER_EDITION)
          299  + {/* zName:     */ "freelist_format",
          300  +  /* ePragTyp:  */ PragTyp_FREELIST_FORMAT,
          301  +  /* ePragFlg:  */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq,
          302  +  /* ColNames:  */ 0, 0,
          303  +  /* iArg:      */ 0 },
   296    304   #endif
   297    305   #if !defined(SQLITE_OMIT_FLAG_PRAGMAS)
   298    306    {/* zName:     */ "full_column_names",
   299    307     /* ePragTyp:  */ PragTyp_FLAG,
   300    308     /* ePragFlg:  */ PragFlg_Result0|PragFlg_NoColumns1,
   301    309     /* ColNames:  */ 0, 0,
   302    310     /* iArg:      */ SQLITE_FullColNames },
................................................................................
   609    617    {/* zName:     */ "writable_schema",
   610    618     /* ePragTyp:  */ PragTyp_FLAG,
   611    619     /* ePragFlg:  */ PragFlg_Result0|PragFlg_NoColumns1,
   612    620     /* ColNames:  */ 0, 0,
   613    621     /* iArg:      */ SQLITE_WriteSchema },
   614    622   #endif
   615    623   };
   616         -/* Number of pragmas: 60 on by default, 74 total. */
          624  +/* Number of pragmas: 60 on by default, 75 total. */

Changes to src/vdbe.c.

  3372   3372       ** schema is changed.  Ticket #1644 */
  3373   3373       sqlite3ExpirePreparedStatements(db);
  3374   3374       p->expired = 0;
  3375   3375     }
  3376   3376     if( rc ) goto abort_due_to_error;
  3377   3377     break;
  3378   3378   }
         3379  +
         3380  +#ifdef SQLITE_SERVER_EDITION
         3381  +/* Opcode: FreelistFmt P1 P2 P3 * *
         3382  +**
         3383  +** Parameter P3 must be 0, 1 or 2. If it is not 0, attempt to set the
         3384  +** freelist format of database P1 to format 1 or format 2. Before
         3385  +** returning, store the final freelist format (either 1 or 2) of 
         3386  +** database P1 into register P2.
         3387  +*/
         3388  +case OP_FreelistFmt: {  /* out2 */
         3389  +  Db *pDb;
         3390  +  int iVal;
         3391  +
         3392  +  assert( pOp->p1>=0 && pOp->p1<db->nDb );
         3393  +  assert( DbMaskTest(p->btreeMask, pOp->p1) );
         3394  +  assert( sqlite3SchemaMutexHeld(db, pOp->p1, 0) );
         3395  +  pDb = &db->aDb[pOp->p1];
         3396  +  assert( pDb->pBt!=0 );
         3397  +
         3398  +  pOut = out2Prerelease(p, pOp);
         3399  +  rc = sqlite3BtreeFreelistFormat(pDb->pBt, pOp->p3, &iVal);
         3400  +  if( rc ) goto abort_due_to_error;
         3401  +  pOut->u.i = iVal;
         3402  +
         3403  +  break;
         3404  +}
         3405  +#endif
  3379   3406   
  3380   3407   /* Opcode: OpenRead P1 P2 P3 P4 P5
  3381   3408   ** Synopsis: root=P2 iDb=P3
  3382   3409   **
  3383   3410   ** Open a read-only cursor for the database table whose root page is
  3384   3411   ** P2 in a database file.  The database file is determined by P3. 
  3385   3412   ** P3==0 means the main database, P3==1 means the database used for 

Changes to tool/mkpragmatab.tcl.

   360    360     FLAG: Result0
   361    361   
   362    362     NAME: threads
   363    363     FLAG: Result0
   364    364   
   365    365     NAME: optimize
   366    366     FLAG: Result1 NeedSchema
          367  +
          368  +  NAME: freelist_format
          369  +  FLAG: NeedSchema Result0 SchemaReq
          370  +  IF:   !defined(SQLITE_OMIT_PAGER_PRAGMAS) && defined(SQLITE_SERVER_EDITION)
   367    371   }
   368    372   
   369    373   # Open the output file
   370    374   #
   371    375   set destfile "[file dir [file dir [file normal $argv0]]]/src/pragma.h"
   372    376   puts "Overwriting $destfile with new pragma table..."
   373    377   set fd [open $destfile wb]