/ Check-in [cb3aa778]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge enhancements from trunk, and especially the cell-overwrite optimization.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | apple-osx
Files: files | file ages | folders
SHA3-256: cb3aa77802d45e9515e9c631af86f492fcd962c3302c79ba6f69213745fb0bb5
User & Date: drh 2018-05-07 13:01:46
Context
2018-05-08
13:32
Merge changes from trunk, especially the activation of the cell-overwrite optimization for indexes and WITHOUT ROWID tables. check-in: a016144b user: drh tags: apple-osx
2018-05-07
13:01
Merge enhancements from trunk, and especially the cell-overwrite optimization. check-in: cb3aa778 user: drh tags: apple-osx
11:48
On an UPDATE, try to overwrite an existing btree cell with the modified content, if the old and new cell are the same size. Use memcmp() first to avoid dirtying pages that are unchanged. check-in: 5887d8be user: drh tags: trunk
2018-05-04
20:00
Bring the code that changes the owner of WAL and rollback files when running as root into alignment with trunk. check-in: 180516bf user: drh tags: apple-osx
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/btree.c.

  6228   6228     u32 ovflPageSize;
  6229   6229   
  6230   6230     assert( sqlite3_mutex_held(pPage->pBt->mutex) );
  6231   6231     pPage->xParseCell(pPage, pCell, pInfo);
  6232   6232     if( pInfo->nLocal==pInfo->nPayload ){
  6233   6233       return SQLITE_OK;  /* No overflow pages. Return without doing anything */
  6234   6234     }
  6235         -  if( pCell+pInfo->nSize-1 > pPage->aData+pPage->maskPage ){
         6235  +  testcase( pCell + pInfo->nSize == pPage->aDataEnd );
         6236  +  testcase( pCell + (pInfo->nSize-1) == pPage->aDataEnd );
         6237  +  if( pCell + pInfo->nSize > pPage->aDataEnd ){
  6236   6238       /* Cell extends past end of page */
  6237   6239       return SQLITE_CORRUPT_PAGE(pPage);
  6238   6240     }
  6239   6241     ovflPgno = get4byte(pCell + pInfo->nSize - 4);
  6240   6242     pBt = pPage->pBt;
  6241   6243     assert( pBt->usableSize > 4 );
  6242   6244     ovflPageSize = pBt->usableSize - 4;
................................................................................
  8154   8156   
  8155   8157     if( pFree ){
  8156   8158       sqlite3PageFree(pFree);
  8157   8159     }
  8158   8160     return rc;
  8159   8161   }
  8160   8162   
         8163  +/* Overwrite content from pX into pDest.  Only do the write if the
         8164  +** content is different from what is already there.
         8165  +*/
         8166  +static int btreeOverwriteContent(
         8167  +  MemPage *pPage,           /* MemPage on which writing will occur */
         8168  +  u8 *pDest,                /* Pointer to the place to start writing */
         8169  +  const BtreePayload *pX,   /* Source of data to write */
         8170  +  int iOffset,              /* Offset of first byte to write */
         8171  +  int iAmt                  /* Number of bytes to be written */
         8172  +){
         8173  +  int nData = pX->nData - iOffset;
         8174  +  if( nData<=0 ){
         8175  +    /* Overwritting with zeros */
         8176  +    int i;
         8177  +    for(i=0; i<iAmt && pDest[i]==0; i++){}
         8178  +    if( i<iAmt ){
         8179  +      int rc = sqlite3PagerWrite(pPage->pDbPage);
         8180  +      if( rc ) return rc;
         8181  +      memset(pDest + i, 0, iAmt - i);
         8182  +    }
         8183  +  }else{
         8184  +    if( nData<iAmt ){
         8185  +      /* Mixed read data and zeros at the end.  Make a recursive call
         8186  +      ** to write the zeros then fall through to write the real data */
         8187  +      int rc = btreeOverwriteContent(pPage, pDest+nData, pX, iOffset+nData,
         8188  +                                 iAmt-nData);
         8189  +      if( rc ) return rc;
         8190  +      iAmt = nData;
         8191  +    }
         8192  +    if( memcmp(pDest, ((u8*)pX->pData) + iOffset, iAmt)!=0 ){
         8193  +      int rc = sqlite3PagerWrite(pPage->pDbPage);
         8194  +      if( rc ) return rc;
         8195  +      memcpy(pDest, ((u8*)pX->pData) + iOffset, iAmt);
         8196  +    }
         8197  +  }
         8198  +  return SQLITE_OK;
         8199  +}
         8200  +
         8201  +/*
         8202  +** Overwrite the cell that cursor pCur is pointing to with fresh content
         8203  +** contained in pX.
         8204  +*/
         8205  +static int btreeOverwriteCell(BtCursor *pCur, const BtreePayload *pX){
         8206  +  int iOffset;                        /* Next byte of pX->pData to write */
         8207  +  int nTotal = pX->nData + pX->nZero; /* Total bytes of to write */
         8208  +  int rc;                             /* Return code */
         8209  +  MemPage *pPage = pCur->pPage;       /* Page being written */
         8210  +  BtShared *pBt;                      /* Btree */
         8211  +  Pgno ovflPgno;                      /* Next overflow page to write */
         8212  +  u32 ovflPageSize;                   /* Size to write on overflow page */
         8213  +
         8214  +  if( pCur->info.pPayload + pCur->info.nLocal > pPage->aDataEnd ){
         8215  +    return SQLITE_CORRUPT_BKPT;
         8216  +  }
         8217  +  /* Overwrite the local portion first */
         8218  +  rc = btreeOverwriteContent(pPage, pCur->info.pPayload, pX,
         8219  +                             0, pCur->info.nLocal);
         8220  +  if( rc ) return rc;
         8221  +  if( pCur->info.nLocal==nTotal ) return SQLITE_OK;
         8222  +
         8223  +  /* Now overwrite the overflow pages */
         8224  +  iOffset = pCur->info.nLocal;
         8225  +  assert( nTotal>=0 );
         8226  +  assert( iOffset>=0 );
         8227  +  ovflPgno = get4byte(pCur->info.pPayload + iOffset);
         8228  +  pBt = pPage->pBt;
         8229  +  ovflPageSize = pBt->usableSize - 4;
         8230  +  do{
         8231  +    rc = btreeGetPage(pBt, ovflPgno, &pPage, 0);
         8232  +    if( rc ) return rc;
         8233  +    if( sqlite3PagerPageRefcount(pPage->pDbPage)!=1 ){
         8234  +      rc = SQLITE_CORRUPT_BKPT;
         8235  +    }else{
         8236  +      if( iOffset+ovflPageSize<(u32)nTotal ){
         8237  +        ovflPgno = get4byte(pPage->aData);
         8238  +      }else{
         8239  +        ovflPageSize = nTotal - iOffset;
         8240  +      }
         8241  +      rc = btreeOverwriteContent(pPage, pPage->aData+4, pX,
         8242  +                                 iOffset, ovflPageSize);
         8243  +    }
         8244  +    sqlite3PagerUnref(pPage->pDbPage);
         8245  +    if( rc ) return rc;
         8246  +    iOffset += ovflPageSize;
         8247  +  }while( iOffset<nTotal );
         8248  +  return SQLITE_OK;    
         8249  +}
         8250  +
  8161   8251   
  8162   8252   /*
  8163   8253   ** Insert a new record into the BTree.  The content of the new record
  8164   8254   ** is described by the pX object.  The pCur cursor is used only to
  8165   8255   ** define what table the record should be inserted into, and is left
  8166   8256   ** pointing at a random location.
  8167   8257   **
................................................................................
  8252   8342       assert( (flags & BTREE_SAVEPOSITION)==0 || 
  8253   8343               ((pCur->curFlags&BTCF_ValidNKey)!=0 && pX->nKey==pCur->info.nKey) );
  8254   8344   
  8255   8345       /* If the cursor is currently on the last row and we are appending a
  8256   8346       ** new row onto the end, set the "loc" to avoid an unnecessary
  8257   8347       ** btreeMoveto() call */
  8258   8348       if( (pCur->curFlags&BTCF_ValidNKey)!=0 && pX->nKey==pCur->info.nKey ){
         8349  +      /* The current is currently pointing to the entry that is to be
         8350  +      ** overwritten */
         8351  +      assert( pX->nData>=0 && pX->nZero>=0 );
         8352  +      if( pCur->info.nSize!=0
         8353  +       && pCur->info.nPayload==(u32)pX->nData+pX->nZero
         8354  +      ){
         8355  +        return btreeOverwriteCell(pCur, pX);
         8356  +      }
  8259   8357         loc = 0;
  8260   8358       }else if( loc==0 ){
  8261   8359         rc = sqlite3BtreeMovetoUnpacked(pCur, 0, pX->nKey, flags!=0, &loc);
  8262   8360         if( rc ) return rc;
  8263   8361       }
  8264   8362     }else if( loc==0 && (flags & BTREE_SAVEPOSITION)==0 ){
  8265   8363       if( pX->nMem ){

Changes to src/loadext.c.

   496    496   
   497    497     zEntry = zProc ? zProc : "sqlite3_extension_init";
   498    498   
   499    499     handle = sqlite3OsDlOpen(pVfs, zFile);
   500    500   #if SQLITE_OS_UNIX || SQLITE_OS_WIN
   501    501     for(ii=0; ii<ArraySize(azEndings) && handle==0; ii++){
   502    502       char *zAltFile = sqlite3_mprintf("%s.%s", zFile, azEndings[ii]);
   503         -    int bExists = 0;
   504    503       if( zAltFile==0 ) return SQLITE_NOMEM_BKPT;
   505         -    sqlite3OsAccess(pVfs, zAltFile, SQLITE_ACCESS_EXISTS, &bExists);
   506         -    if( bExists )  handle = sqlite3OsDlOpen(pVfs, zAltFile);
          504  +    handle = sqlite3OsDlOpen(pVfs, zAltFile);
   507    505       sqlite3_free(zAltFile);
   508    506     }
   509    507   #endif
   510    508     if( handle==0 ){
   511    509       if( pzErrMsg ){
   512    510         *pzErrMsg = zErrmsg = sqlite3_malloc64(nMsg);
   513    511         if( zErrmsg ){

Changes to src/select.c.

   623    623     **   (1) The data to be sorted has already been packed into a Record
   624    624     **       by a prior OP_MakeRecord.  In this case nData==1 and regData
   625    625     **       will be completely unrelated to regOrigData.
   626    626     **   (2) All output columns are included in the sort record.  In that
   627    627     **       case regData==regOrigData.
   628    628     **   (3) Some output columns are omitted from the sort record due to
   629    629     **       the SQLITE_ENABLE_SORTER_REFERENCE optimization, or due to the
   630         -  **       SQLITE_ECEL_OMITREF optimization.  In that case, regOrigData==0
   631         -  **       to prevent this routine from trying to copy values that might
   632         -  **       not exist.
          630  +  **       SQLITE_ECEL_OMITREF optimization, or due to the 
          631  +  **       SortCtx.pDeferredRowLoad optimiation.  In any of these cases
          632  +  **       regOrigData is 0 to prevent this routine from trying to copy
          633  +  **       values that might not yet exist.
   633    634     */
   634    635     assert( nData==1 || regData==regOrigData || regOrigData==0 );
   635    636   
   636    637     if( nPrefixReg ){
   637    638       assert( nPrefixReg==nExpr+bSeq );
   638    639       regBase = regData - nPrefixReg;
   639    640     }else{
................................................................................
  1006   1007       if( p->iLimit
  1007   1008        && (ecelFlags & SQLITE_ECEL_OMITREF)!=0 
  1008   1009        && nPrefixReg>0
  1009   1010       ){
  1010   1011         assert( pSort!=0 );
  1011   1012         assert( hasDistinct==0 );
  1012   1013         pSort->pDeferredRowLoad = &sRowLoadInfo;
         1014  +      regOrig = 0;
  1013   1015       }else{
  1014   1016         innerLoopLoadRow(pParse, p, &sRowLoadInfo);
  1015   1017       }
  1016   1018     }
  1017   1019   
  1018   1020     /* If the DISTINCT keyword was present on the SELECT statement
  1019   1021     ** and this row has been seen before, then do not make this row

Changes to src/shell.c.in.

  1718   1718     EQPGraphRow *pRow, *pNext;
  1719   1719     int n = strlen30(p->sGraph.zPrefix);
  1720   1720     char *z;
  1721   1721     for(pRow = eqp_next_row(p, iEqpId, 0); pRow; pRow = pNext){
  1722   1722       pNext = eqp_next_row(p, iEqpId, pRow);
  1723   1723       z = pRow->zText;
  1724   1724       utf8_printf(p->out, "%s%s%s\n", p->sGraph.zPrefix, pNext ? "|--" : "`--", z);
  1725         -    if( n<sizeof(p->sGraph.zPrefix)-7 ){
         1725  +    if( n<(int)sizeof(p->sGraph.zPrefix)-7 ){
  1726   1726         memcpy(&p->sGraph.zPrefix[n], pNext ? "|  " : "   ", 4);
  1727   1727         eqp_render_level(p, pRow->iEqpId);
  1728   1728         p->sGraph.zPrefix[n] = 0;
  1729   1729       }
  1730   1730     }
  1731   1731   }
  1732   1732   

Changes to src/where.c.

  4984   4984       if( (pLevel->pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 ){
  4985   4985         constructAutomaticIndex(pParse, &pWInfo->sWC,
  4986   4986                   &pTabList->a[pLevel->iFrom], notReady, pLevel);
  4987   4987         if( db->mallocFailed ) goto whereBeginError;
  4988   4988       }
  4989   4989   #endif
  4990   4990       addrExplain = sqlite3WhereExplainOneScan(
  4991         -        pParse, pTabList, pLevel, ii, pLevel->iFrom, wctrlFlags
         4991  +        pParse, pTabList, pLevel, wctrlFlags
  4992   4992       );
  4993   4993       pLevel->addrBody = sqlite3VdbeCurrentAddr(v);
  4994   4994       notReady = sqlite3WhereCodeOneLoopStart(pWInfo, ii, notReady);
  4995   4995       pWInfo->iContinue = pLevel->addrCont;
  4996   4996       if( (wsFlags&WHERE_MULTI_OR)==0 && (wctrlFlags&WHERE_OR_SUBCLAUSE)==0 ){
  4997   4997         sqlite3WhereAddScanStatus(v, pTabList, pLevel, addrExplain);
  4998   4998       }

Changes to src/whereInt.h.

   463    463   
   464    464   /* wherecode.c: */
   465    465   #ifndef SQLITE_OMIT_EXPLAIN
   466    466   int sqlite3WhereExplainOneScan(
   467    467     Parse *pParse,                  /* Parse context */
   468    468     SrcList *pTabList,              /* Table list this loop refers to */
   469    469     WhereLevel *pLevel,             /* Scan to write OP_Explain opcode for */
   470         -  int iLevel,                     /* Value for "level" column of output */
   471         -  int iFrom,                      /* Value for "from" column of output */
   472    470     u16 wctrlFlags                  /* Flags passed to sqlite3WhereBegin() */
   473    471   );
   474    472   #else
   475         -# define sqlite3WhereExplainOneScan(u,v,w,x,y,z) 0
          473  +# define sqlite3WhereExplainOneScan(u,v,w,x) 0
   476    474   #endif /* SQLITE_OMIT_EXPLAIN */
   477    475   #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
   478    476   void sqlite3WhereAddScanStatus(
   479    477     Vdbe *v,                        /* Vdbe to add scanstatus entry to */
   480    478     SrcList *pSrclist,              /* FROM clause pLvl reads data from */
   481    479     WhereLevel *pLvl,               /* Level to add scanstatus() entry for */
   482    480     int addrExplain                 /* Address of OP_Explain (or 0) */

Changes to src/wherecode.c.

   118    118   ** If an OP_Explain opcode is added to the VM, its address is returned.
   119    119   ** Otherwise, if no OP_Explain is coded, zero is returned.
   120    120   */
   121    121   int sqlite3WhereExplainOneScan(
   122    122     Parse *pParse,                  /* Parse context */
   123    123     SrcList *pTabList,              /* Table list this loop refers to */
   124    124     WhereLevel *pLevel,             /* Scan to write OP_Explain opcode for */
   125         -  int iLevel,                     /* Value for "level" column of output */
   126         -  int iFrom,                      /* Value for "from" column of output */
   127    125     u16 wctrlFlags                  /* Flags passed to sqlite3WhereBegin() */
   128    126   ){
   129    127     int ret = 0;
   130    128   #if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS)
   131    129     if( sqlite3ParseToplevel(pParse)->explain==2 )
   132    130   #endif
   133    131     {
................................................................................
  1952   1950           WHERETRACE(0xffff, ("Subplan for OR-clause:\n"));
  1953   1951           pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0,
  1954   1952                                         wctrlFlags, iCovCur);
  1955   1953           assert( pSubWInfo || pParse->nErr || db->mallocFailed );
  1956   1954           if( pSubWInfo ){
  1957   1955             WhereLoop *pSubLoop;
  1958   1956             int addrExplain = sqlite3WhereExplainOneScan(
  1959         -              pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0
         1957  +              pParse, pOrTab, &pSubWInfo->a[0], 0
  1960   1958             );
  1961   1959             sqlite3WhereAddScanStatus(v, pOrTab, &pSubWInfo->a[0], addrExplain);
  1962   1960   
  1963   1961             /* This is the sub-WHERE clause body.  First skip over
  1964   1962             ** duplicate rows from prior sub-WHERE clauses, and record the
  1965   1963             ** rowid (or PRIMARY KEY) for the current row so that the same
  1966   1964             ** row will be skipped in subsequent sub-WHERE clauses.

Changes to test/ctime.test.

   217    217       set ans1 [catch {db one {
   218    218         SELECT sqlite_compileoption_get($N);
   219    219       }} msg]
   220    220       lappend ans1 $msg
   221    221       set ans2 [ catchsql {
   222    222         SELECT sqlite_compileoption_used($opt);
   223    223       } ]
   224         -    list [ lindex $ans1 0 ] [ expr { [lindex [lindex $ans1 1] 0]==$opt } ] \
          224  +    list [ lindex $ans1 0 ] [ expr { [lindex $ans1 1]==$opt } ] \
   225    225            [ expr { $ans2 } ]
   226    226     } {0 1 {0 1}}
   227    227     incr tc 1
   228    228   }
   229    229   # test 1 past array bounds
   230    230   do_test ctime-2.5.$tc {
   231    231     set N [ expr {$tc-1} ]

Changes to test/orderby1.test.

   543    543     CREATE TABLE t1(x INTEGER PRIMARY KEY);
   544    544     INSERT INTO t1 VALUES(1),(2);
   545    545     DROP TABLE IF EXISTS t2;
   546    546     CREATE TABLE t2(y);
   547    547     INSERT INTO t2 VALUES(9),(8),(3),(4);
   548    548     SELECT (SELECT x||y FROM t2, t1 ORDER BY x, y);
   549    549   } {13}
          550  +
          551  +# Problem found by OSSFuzz on 2018-05-05.  This was caused by a new
          552  +# optimization that had not been previously released.
          553  +#
          554  +do_execsql_test 10.0 {
          555  +  CREATE TABLE t10(a,b);
          556  +  INSERT INTO t10 VALUES(1,2),(8,9),(3,4),(5,4),(0,7);
          557  +  CREATE INDEX t10b ON t10(b);
          558  +  SELECT b, rowid, '^' FROM t10 ORDER BY b, a LIMIT 4;
          559  +} {2 1 ^ 4 3 ^ 4 4 ^ 7 5 ^}
   550    560   
   551    561   
   552    562   finish_test

Changes to test/pager1.test.

  1148   1148   do_test pager1-5.5.1 {
  1149   1149     sqlite3 db test.db
  1150   1150     execsql { 
  1151   1151       ATTACH 'test.db2' AS aux;
  1152   1152       PRAGMA journal_mode = PERSIST;
  1153   1153       CREATE TABLE t3(a, b);
  1154   1154       INSERT INTO t3 SELECT randomblob(1500), randomblob(1500) FROM t1;
  1155         -    UPDATE t3 SET b = randomblob(1500);
         1155  +    UPDATE t3 SET b = randomblob(1501);
  1156   1156     }
  1157   1157     expr [file size test.db-journal] > 15000
  1158   1158   } {1}
  1159   1159   do_test pager1-5.5.2 {
  1160   1160     execsql {
  1161   1161       PRAGMA synchronous = full;
  1162   1162       BEGIN;