/ Check-in [10aaf3b1]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge latest enhancements, including the SQLITE_ENABLE_API_ARMOR patch, from trunk.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | apple-osx
Files: files | file ages | folders
SHA1:10aaf3b148404df7912d9604ccb6921173e7ac80
User & Date: drh 2014-10-27 18:42:03
Context
2014-10-31
15:20
Merge all recent trunk enhancements into the apple-osx branch. check-in: fef8430f user: drh tags: apple-osx
2014-10-27
18:42
Merge latest enhancements, including the SQLITE_ENABLE_API_ARMOR patch, from trunk. check-in: 10aaf3b1 user: drh tags: apple-osx
18:34
Add the SQLITE_ENABLE_API_ARMOR compile-time option for extra API parameter validation. Enhance sqlite3_stricmp(), sqlite3_strnicmp(), and sqlite3_uri_parameter() for improved NULL parameter handling. check-in: ffb9d814 user: drh tags: trunk
2014-10-17
12:11
Merge the 3.8.7 changes into the apple-osx branch. check-in: aa7bbed1 user: drh tags: apple-osx
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/rtree/rtree6.test.

    98     98     0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:C0E1} 
    99     99     0 1 1 {SEARCH TABLE t2 USING AUTOMATIC COVERING INDEX (v=?)}
   100    100   }
   101    101   do_eqp_test rtree6.2.4.2 {
   102    102     SELECT * FROM t1,t2 WHERE v=10 and x1<10 and x2>10
   103    103   } {
   104    104     0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:C0E1} 
   105         -  0 1 1 {SEARCH TABLE t2 USING AUTOMATIC COVERING INDEX (v=?)}
          105  +  0 1 1 {SEARCH TABLE t2 USING AUTOMATIC PARTIAL COVERING INDEX (v=?)}
   106    106   }
   107    107   
   108    108   do_eqp_test rtree6.2.5 {
   109    109     SELECT * FROM t1,t2 WHERE k=ii AND x1<v
   110    110   } {
   111    111     0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:} 
   112    112     0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?)}

Changes to src/analyze.c.

  1595   1595           nRow = pFinal->anLt[iCol];
  1596   1596           nDist100 = (i64)100 * pFinal->anDLt[iCol];
  1597   1597           nSample--;
  1598   1598         }else{
  1599   1599           nRow = pIdx->aiRowEst[0];
  1600   1600           nDist100 = ((i64)100 * pIdx->aiRowEst[0]) / pIdx->aiRowEst[iCol+1];
  1601   1601         }
         1602  +      pIdx->nRowEst0 = nRow;
  1602   1603   
  1603   1604         /* Set nSum to the number of distinct (iCol+1) field prefixes that
  1604   1605         ** occur in the stat4 table for this index. Set sumEq to the sum of 
  1605   1606         ** the nEq values for column iCol for the same set (adding the value 
  1606   1607         ** only once where there exist duplicate prefixes).  */
  1607   1608         for(i=0; i<nSample; i++){
  1608   1609           if( i==(pIdx->nSample-1)
................................................................................
  1856   1857       rc = sqlite3_exec(db, zSql, analysisLoader, &sInfo, 0);
  1857   1858       sqlite3DbFree(db, zSql);
  1858   1859     }
  1859   1860   
  1860   1861   
  1861   1862     /* Load the statistics from the sqlite_stat4 table. */
  1862   1863   #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
  1863         -  if( rc==SQLITE_OK ){
         1864  +  if( rc==SQLITE_OK && OptimizationEnabled(db, SQLITE_Stat34) ){
  1864   1865       int lookasideEnabled = db->lookaside.bEnabled;
  1865   1866       db->lookaside.bEnabled = 0;
  1866   1867       rc = loadStat4(db, sInfo.zDatabase);
  1867   1868       db->lookaside.bEnabled = lookasideEnabled;
  1868   1869     }
  1869   1870     for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){
  1870   1871       Index *pIdx = sqliteHashData(i);

Changes to src/auth.c.

    68     68   ** setting of the auth function is NULL.
    69     69   */
    70     70   int sqlite3_set_authorizer(
    71     71     sqlite3 *db,
    72     72     int (*xAuth)(void*,int,const char*,const char*,const char*,const char*),
    73     73     void *pArg
    74     74   ){
           75  +#ifdef SQLITE_ENABLE_API_ARMOR
           76  +  if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT;
           77  +#endif
    75     78     sqlite3_mutex_enter(db->mutex);
    76     79     db->xAuth = (sqlite3_xauth)xAuth;
    77     80     db->pAuthArg = pArg;
    78     81     sqlite3ExpirePreparedStatements(db);
    79     82     sqlite3_mutex_leave(db->mutex);
    80     83     return SQLITE_OK;
    81     84   }

Changes to src/backup.c.

   133    133   sqlite3_backup *sqlite3_backup_init(
   134    134     sqlite3* pDestDb,                     /* Database to write to */
   135    135     const char *zDestDb,                  /* Name of database within pDestDb */
   136    136     sqlite3* pSrcDb,                      /* Database connection to read from */
   137    137     const char *zSrcDb                    /* Name of database within pSrcDb */
   138    138   ){
   139    139     sqlite3_backup *p;                    /* Value to return */
          140  +
          141  +#ifdef SQLITE_ENABLE_API_ARMOR
          142  +  if( !sqlite3SafetyCheckOk(pSrcDb)||!sqlite3SafetyCheckOk(pDestDb) ){
          143  +    (void)SQLITE_MISUSE_BKPT;
          144  +    return 0;
          145  +  }
          146  +#endif
   140    147   
   141    148     /* Lock the source database handle. The destination database
   142    149     ** handle is not locked in this routine, but it is locked in
   143    150     ** sqlite3_backup_step(). The user is required to ensure that no
   144    151     ** other thread accesses the destination handle for the duration
   145    152     ** of the backup operation.  Any attempt to use the destination
   146    153     ** database connection while a backup is in progress may cause
................................................................................
   330    337   */
   331    338   int sqlite3_backup_step(sqlite3_backup *p, int nPage){
   332    339     int rc;
   333    340     int destMode;       /* Destination journal mode */
   334    341     int pgszSrc = 0;    /* Source page size */
   335    342     int pgszDest = 0;   /* Destination page size */
   336    343   
          344  +#ifdef SQLITE_ENABLE_API_ARMOR
          345  +  if( p==0 ) return SQLITE_MISUSE_BKPT;
          346  +#endif
   337    347     sqlite3_mutex_enter(p->pSrcDb->mutex);
   338    348     sqlite3BtreeEnter(p->pSrc);
   339    349     if( p->pDestDb ){
   340    350       sqlite3_mutex_enter(p->pDestDb->mutex);
   341    351     }
   342    352   
   343    353     rc = p->rc;
................................................................................
   619    629   }
   620    630   
   621    631   /*
   622    632   ** Return the number of pages still to be backed up as of the most recent
   623    633   ** call to sqlite3_backup_step().
   624    634   */
   625    635   int sqlite3_backup_remaining(sqlite3_backup *p){
          636  +#ifdef SQLITE_ENABLE_API_ARMOR
          637  +  if( p==0 ){
          638  +    (void)SQLITE_MISUSE_BKPT;
          639  +    return 0;
          640  +  }
          641  +#endif
   626    642     return p->nRemaining;
   627    643   }
   628    644   
   629    645   /*
   630    646   ** Return the total number of pages in the source database as of the most 
   631    647   ** recent call to sqlite3_backup_step().
   632    648   */
   633    649   int sqlite3_backup_pagecount(sqlite3_backup *p){
          650  +#ifdef SQLITE_ENABLE_API_ARMOR
          651  +  if( p==0 ){
          652  +    (void)SQLITE_MISUSE_BKPT;
          653  +    return 0;
          654  +  }
          655  +#endif
   634    656     return p->nPagecount;
   635    657   }
   636    658   
   637    659   /*
   638    660   ** This function is called after the contents of page iPage of the
   639    661   ** source database have been modified. If page iPage has already been 
   640    662   ** copied into the destination database, then the data written to the

Changes to src/btree.c.

  1153   1153     int size;                  /* Size of a cell */
  1154   1154     int usableSize;            /* Number of usable bytes on a page */
  1155   1155     int cellOffset;            /* Offset to the cell pointer array */
  1156   1156     int cbrk;                  /* Offset to the cell content area */
  1157   1157     int nCell;                 /* Number of cells on the page */
  1158   1158     unsigned char *data;       /* The page data */
  1159   1159     unsigned char *temp;       /* Temp area for cell content */
         1160  +  unsigned char *src;        /* Source of content */
  1160   1161     int iCellFirst;            /* First allowable cell index */
  1161   1162     int iCellLast;             /* Last possible cell index */
  1162   1163   
  1163   1164   
  1164   1165     assert( sqlite3PagerIswriteable(pPage->pDbPage) );
  1165   1166     assert( pPage->pBt!=0 );
  1166   1167     assert( pPage->pBt->usableSize <= SQLITE_MAX_PAGE_SIZE );
  1167   1168     assert( pPage->nOverflow==0 );
  1168   1169     assert( sqlite3_mutex_held(pPage->pBt->mutex) );
  1169         -  temp = sqlite3PagerTempSpace(pPage->pBt->pPager);
  1170         -  data = pPage->aData;
         1170  +  temp = 0;
         1171  +  src = data = pPage->aData;
  1171   1172     hdr = pPage->hdrOffset;
  1172   1173     cellOffset = pPage->cellOffset;
  1173   1174     nCell = pPage->nCell;
  1174   1175     assert( nCell==get2byte(&data[hdr+3]) );
  1175   1176     usableSize = pPage->pBt->usableSize;
  1176         -  cbrk = get2byte(&data[hdr+5]);
  1177         -  memcpy(&temp[cbrk], &data[cbrk], usableSize - cbrk);
  1178   1177     cbrk = usableSize;
  1179   1178     iCellFirst = cellOffset + 2*nCell;
  1180   1179     iCellLast = usableSize - 4;
  1181   1180     for(i=0; i<nCell; i++){
  1182   1181       u8 *pAddr;     /* The i-th cell pointer */
  1183   1182       pAddr = &data[cellOffset + i*2];
  1184   1183       pc = get2byte(pAddr);
................................................................................
  1189   1188       ** if SQLITE_ENABLE_OVERSIZE_CELL_CHECK is defined 
  1190   1189       */
  1191   1190       if( pc<iCellFirst || pc>iCellLast ){
  1192   1191         return SQLITE_CORRUPT_BKPT;
  1193   1192       }
  1194   1193   #endif
  1195   1194       assert( pc>=iCellFirst && pc<=iCellLast );
  1196         -    size = cellSizePtr(pPage, &temp[pc]);
         1195  +    size = cellSizePtr(pPage, &src[pc]);
  1197   1196       cbrk -= size;
  1198   1197   #if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK)
  1199   1198       if( cbrk<iCellFirst ){
  1200   1199         return SQLITE_CORRUPT_BKPT;
  1201   1200       }
  1202   1201   #else
  1203   1202       if( cbrk<iCellFirst || pc+size>usableSize ){
  1204   1203         return SQLITE_CORRUPT_BKPT;
  1205   1204       }
  1206   1205   #endif
  1207   1206       assert( cbrk+size<=usableSize && cbrk>=iCellFirst );
  1208   1207       testcase( cbrk+size==usableSize );
  1209   1208       testcase( pc+size==usableSize );
  1210         -    memcpy(&data[cbrk], &temp[pc], size);
  1211   1209       put2byte(pAddr, cbrk);
         1210  +    if( temp==0 ){
         1211  +      int x;
         1212  +      if( cbrk==pc ) continue;
         1213  +      temp = sqlite3PagerTempSpace(pPage->pBt->pPager);
         1214  +      x = get2byte(&data[hdr+5]);
         1215  +      memcpy(&temp[x], &data[x], (cbrk+size) - x);
         1216  +      src = temp;
         1217  +    }
         1218  +    memcpy(&data[cbrk], &src[pc], size);
  1212   1219     }
  1213   1220     assert( cbrk>=iCellFirst );
  1214   1221     put2byte(&data[hdr+5], cbrk);
  1215   1222     data[hdr+1] = 0;
  1216   1223     data[hdr+2] = 0;
  1217   1224     data[hdr+7] = 0;
  1218   1225     memset(&data[iCellFirst], 0, cbrk-iCellFirst);
  1219   1226     assert( sqlite3PagerIswriteable(pPage->pDbPage) );
  1220   1227     if( cbrk-iCellFirst!=pPage->nFree ){
  1221   1228       return SQLITE_CORRUPT_BKPT;
  1222   1229     }
  1223   1230     return SQLITE_OK;
  1224   1231   }
         1232  +
         1233  +/*
         1234  +** Search the free-list on page pPg for space to store a cell nByte bytes in
         1235  +** size. If one can be found, return a pointer to the space and remove it
         1236  +** from the free-list.
         1237  +**
         1238  +** If no suitable space can be found on the free-list, return NULL.
         1239  +**
         1240  +** This function may detect corruption within pPg. If it does and argument 
         1241  +** pRc is non-NULL, then *pRc is set to SQLITE_CORRUPT and NULL is returned.
         1242  +** Or, if corruption is detected and pRc is NULL, NULL is returned and the
         1243  +** corruption goes unreported.
         1244  +**
         1245  +** If a slot of at least nByte bytes is found but cannot be used because 
         1246  +** there are already at least 60 fragmented bytes on the page, return NULL.
         1247  +** In this case, if pbDefrag parameter is not NULL, set *pbDefrag to true.
         1248  +*/
         1249  +static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc, int *pbDefrag){
         1250  +  const int hdr = pPg->hdrOffset;
         1251  +  u8 * const aData = pPg->aData;
         1252  +  int iAddr;
         1253  +  int pc;
         1254  +  int usableSize = pPg->pBt->usableSize;
         1255  +
         1256  +  for(iAddr=hdr+1; (pc = get2byte(&aData[iAddr]))>0; iAddr=pc){
         1257  +    int size;            /* Size of the free slot */
         1258  +    if( pc>usableSize-4 || pc<iAddr+4 ){
         1259  +      if( pRc ) *pRc = SQLITE_CORRUPT_BKPT;
         1260  +      return 0;
         1261  +    }
         1262  +    size = get2byte(&aData[pc+2]);
         1263  +    if( size>=nByte ){
         1264  +      int x = size - nByte;
         1265  +      testcase( x==4 );
         1266  +      testcase( x==3 );
         1267  +      if( x<4 ){
         1268  +        if( aData[hdr+7]>=60 ){
         1269  +          if( pbDefrag ) *pbDefrag = 1;
         1270  +          return 0;
         1271  +        }
         1272  +        /* Remove the slot from the free-list. Update the number of
         1273  +        ** fragmented bytes within the page. */
         1274  +        memcpy(&aData[iAddr], &aData[pc], 2);
         1275  +        aData[hdr+7] += (u8)x;
         1276  +      }else if( size+pc > usableSize ){
         1277  +        if( pRc ) *pRc = SQLITE_CORRUPT_BKPT;
         1278  +        return 0;
         1279  +      }else{
         1280  +        /* The slot remains on the free-list. Reduce its size to account
         1281  +         ** for the portion used by the new allocation. */
         1282  +        put2byte(&aData[pc+2], x);
         1283  +      }
         1284  +      return &aData[pc + x];
         1285  +    }
         1286  +  }
         1287  +
         1288  +  return 0;
         1289  +}
  1225   1290   
  1226   1291   /*
  1227   1292   ** Allocate nByte bytes of space from within the B-Tree page passed
  1228   1293   ** as the first argument. Write into *pIdx the index into pPage->aData[]
  1229   1294   ** of the first byte of allocated space. Return either SQLITE_OK or
  1230   1295   ** an error code (usually SQLITE_CORRUPT).
  1231   1296   **
................................................................................
  1238   1303   */
  1239   1304   static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){
  1240   1305     const int hdr = pPage->hdrOffset;    /* Local cache of pPage->hdrOffset */
  1241   1306     u8 * const data = pPage->aData;      /* Local cache of pPage->aData */
  1242   1307     int top;                             /* First byte of cell content area */
  1243   1308     int gap;        /* First byte of gap between cell pointers and cell content */
  1244   1309     int rc;         /* Integer return code */
  1245         -  int usableSize; /* Usable size of the page */
  1246   1310     
  1247   1311     assert( sqlite3PagerIswriteable(pPage->pDbPage) );
  1248   1312     assert( pPage->pBt );
  1249   1313     assert( sqlite3_mutex_held(pPage->pBt->mutex) );
  1250   1314     assert( nByte>=0 );  /* Minimum cell size is 4 */
  1251   1315     assert( pPage->nFree>=nByte );
  1252   1316     assert( pPage->nOverflow==0 );
  1253         -  usableSize = pPage->pBt->usableSize;
  1254         -  assert( nByte < usableSize-8 );
         1317  +  assert( nByte < pPage->pBt->usableSize-8 );
  1255   1318   
  1256   1319     assert( pPage->cellOffset == hdr + 12 - 4*pPage->leaf );
  1257   1320     gap = pPage->cellOffset + 2*pPage->nCell;
  1258   1321     assert( gap<=65536 );
  1259   1322     top = get2byte(&data[hdr+5]);
  1260   1323     if( gap>top ){
  1261   1324       if( top==0 ){
................................................................................
  1269   1332     ** array entry offset, and if the freelist is not empty, then search the
  1270   1333     ** freelist looking for a free slot big enough to satisfy the request.
  1271   1334     */
  1272   1335     testcase( gap+2==top );
  1273   1336     testcase( gap+1==top );
  1274   1337     testcase( gap==top );
  1275   1338     if( gap+2<=top && (data[hdr+1] || data[hdr+2]) ){
  1276         -    int pc, addr;
  1277         -    for(addr=hdr+1; (pc = get2byte(&data[addr]))>0; addr=pc){
  1278         -      int size;            /* Size of the free slot */
  1279         -      if( pc>usableSize-4 || pc<addr+4 ){
  1280         -        return SQLITE_CORRUPT_BKPT;
  1281         -      }
  1282         -      size = get2byte(&data[pc+2]);
  1283         -      if( size>=nByte ){
  1284         -        int x = size - nByte;
  1285         -        testcase( x==4 );
  1286         -        testcase( x==3 );
  1287         -        if( x<4 ){
  1288         -          if( data[hdr+7]>=60 ) goto defragment_page;
  1289         -          /* Remove the slot from the free-list. Update the number of
  1290         -          ** fragmented bytes within the page. */
  1291         -          memcpy(&data[addr], &data[pc], 2);
  1292         -          data[hdr+7] += (u8)x;
  1293         -        }else if( size+pc > usableSize ){
  1294         -          return SQLITE_CORRUPT_BKPT;
  1295         -        }else{
  1296         -          /* The slot remains on the free-list. Reduce its size to account
  1297         -          ** for the portion used by the new allocation. */
  1298         -          put2byte(&data[pc+2], x);
  1299         -        }
  1300         -        *pIdx = pc + x;
  1301         -        return SQLITE_OK;
  1302         -      }
         1339  +    int rc = SQLITE_OK;
         1340  +    int bDefrag = 0;
         1341  +    u8 *pSpace = pageFindSlot(pPage, nByte, &rc, &bDefrag);
         1342  +    if( rc ) return rc;
         1343  +    if( bDefrag ) goto defragment_page;
         1344  +    if( pSpace ){
         1345  +      *pIdx = pSpace - data;
         1346  +      return SQLITE_OK;
  1303   1347       }
  1304   1348     }
  1305   1349   
  1306   1350     /* The request could not be fulfilled using a freelist slot.  Check
  1307   1351     ** to see if defragmentation is necessary.
  1308   1352     */
  1309   1353     testcase( gap+2+nByte==top );
  1310   1354     if( gap+2+nByte>top ){
  1311         -defragment_page:
         1355  + defragment_page:
  1312   1356       testcase( pPage->nCell==0 );
  1313   1357       rc = defragmentPage(pPage);
  1314   1358       if( rc ) return rc;
  1315   1359       top = get2byteNotZero(&data[hdr+5]);
  1316   1360       assert( gap+nByte<=top );
  1317   1361     }
  1318   1362   
................................................................................
  1352   1396     u32 iLast = pPage->pBt->usableSize-4; /* Largest possible freeblock offset */
  1353   1397     u32 iEnd = iStart + iSize;            /* First byte past the iStart buffer */
  1354   1398     unsigned char *data = pPage->aData;   /* Page content */
  1355   1399   
  1356   1400     assert( pPage->pBt!=0 );
  1357   1401     assert( sqlite3PagerIswriteable(pPage->pDbPage) );
  1358   1402     assert( iStart>=pPage->hdrOffset+6+pPage->childPtrSize );
  1359         -  assert( iEnd <= pPage->pBt->usableSize );
         1403  +  assert( CORRUPT_DB || iEnd <= pPage->pBt->usableSize );
  1360   1404     assert( sqlite3_mutex_held(pPage->pBt->mutex) );
  1361   1405     assert( iSize>=4 );   /* Minimum cell size is 4 */
  1362   1406     assert( iStart<=iLast );
  1363   1407   
  1364   1408     /* Overwrite deleted information with zeros when the secure_delete
  1365   1409     ** option is enabled */
  1366   1410     if( pPage->pBt->btsFlags & BTS_SECURE_DELETE ){
................................................................................
  5961   6005         ptrmapPutOvflPtr(pPage, pCell, pRC);
  5962   6006       }
  5963   6007   #endif
  5964   6008     }
  5965   6009   }
  5966   6010   
  5967   6011   /*
  5968         -** Add a list of cells to a page.  The page should be initially empty.
  5969         -** The cells are guaranteed to fit on the page.
  5970         -*/
  5971         -static void assemblePage(
  5972         -  MemPage *pPage,   /* The page to be assembled */
  5973         -  int nCell,        /* The number of cells to add to this page */
  5974         -  u8 **apCell,      /* Pointers to cell bodies */
  5975         -  u16 *aSize        /* Sizes of the cells */
         6012  +** Array apCell[] contains pointers to nCell b-tree page cells. The 
         6013  +** szCell[] array contains the size in bytes of each cell. This function
         6014  +** replaces the current contents of page pPg with the contents of the cell
         6015  +** array.
         6016  +**
         6017  +** Some of the cells in apCell[] may currently be stored in pPg. This
         6018  +** function works around problems caused by this by making a copy of any 
         6019  +** such cells before overwriting the page data.
         6020  +**
         6021  +** The MemPage.nFree field is invalidated by this function. It is the 
         6022  +** responsibility of the caller to set it correctly.
         6023  +*/
         6024  +static void rebuildPage(
         6025  +  MemPage *pPg,                   /* Edit this page */
         6026  +  int nCell,                      /* Final number of cells on page */
         6027  +  u8 **apCell,                    /* Array of cells */
         6028  +  u16 *szCell                     /* Array of cell sizes */
         6029  +){
         6030  +  const int hdr = pPg->hdrOffset;          /* Offset of header on pPg */
         6031  +  u8 * const aData = pPg->aData;           /* Pointer to data for pPg */
         6032  +  const int usableSize = pPg->pBt->usableSize;
         6033  +  u8 * const pEnd = &aData[usableSize];
         6034  +  int i;
         6035  +  u8 *pCellptr = pPg->aCellIdx;
         6036  +  u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager);
         6037  +  u8 *pData;
         6038  +
         6039  +  i = get2byte(&aData[hdr+5]);
         6040  +  memcpy(&pTmp[i], &aData[i], usableSize - i);
         6041  +
         6042  +  pData = pEnd;
         6043  +  for(i=0; i<nCell; i++){
         6044  +    u8 *pCell = apCell[i];
         6045  +    if( pCell>aData && pCell<pEnd ){
         6046  +      pCell = &pTmp[pCell - aData];
         6047  +    }
         6048  +    pData -= szCell[i];
         6049  +    memcpy(pData, pCell, szCell[i]);
         6050  +    put2byte(pCellptr, (pData - aData));
         6051  +    pCellptr += 2;
         6052  +    assert( szCell[i]==cellSizePtr(pPg, pCell) );
         6053  +  }
         6054  +
         6055  +  /* The pPg->nFree field is now set incorrectly. The caller will fix it. */
         6056  +  pPg->nCell = nCell;
         6057  +  pPg->nOverflow = 0;
         6058  +
         6059  +  put2byte(&aData[hdr+1], 0);
         6060  +  put2byte(&aData[hdr+3], pPg->nCell);
         6061  +  put2byte(&aData[hdr+5], pData - aData);
         6062  +  aData[hdr+7] = 0x00;
         6063  +}
         6064  +
         6065  +/*
         6066  +** Array apCell[] contains nCell pointers to b-tree cells. Array szCell
         6067  +** contains the size in bytes of each such cell. This function attempts to 
         6068  +** add the cells stored in the array to page pPg. If it cannot (because 
         6069  +** the page needs to be defragmented before the cells will fit), non-zero
         6070  +** is returned. Otherwise, if the cells are added successfully, zero is
         6071  +** returned.
         6072  +**
         6073  +** Argument pCellptr points to the first entry in the cell-pointer array
         6074  +** (part of page pPg) to populate. After cell apCell[0] is written to the
         6075  +** page body, a 16-bit offset is written to pCellptr. And so on, for each
         6076  +** cell in the array. It is the responsibility of the caller to ensure
         6077  +** that it is safe to overwrite this part of the cell-pointer array.
         6078  +**
         6079  +** When this function is called, *ppData points to the start of the 
         6080  +** content area on page pPg. If the size of the content area is extended,
         6081  +** *ppData is updated to point to the new start of the content area
         6082  +** before returning.
         6083  +**
         6084  +** Finally, argument pBegin points to the byte immediately following the
         6085  +** end of the space required by this page for the cell-pointer area (for
         6086  +** all cells - not just those inserted by the current call). If the content
         6087  +** area must be extended to before this point in order to accomodate all
         6088  +** cells in apCell[], then the cells do not fit and non-zero is returned.
         6089  +*/
         6090  +static int pageInsertArray(
         6091  +  MemPage *pPg,                   /* Page to add cells to */
         6092  +  u8 *pBegin,                     /* End of cell-pointer array */
         6093  +  u8 **ppData,                    /* IN/OUT: Page content -area pointer */
         6094  +  u8 *pCellptr,                   /* Pointer to cell-pointer area */
         6095  +  int nCell,                      /* Number of cells to add to pPg */
         6096  +  u8 **apCell,                    /* Array of cells */
         6097  +  u16 *szCell                     /* Array of cell sizes */
         6098  +){
         6099  +  int i;
         6100  +  u8 *aData = pPg->aData;
         6101  +  u8 *pData = *ppData;
         6102  +  const int bFreelist = aData[1] || aData[2];
         6103  +  assert( CORRUPT_DB || pPg->hdrOffset==0 );    /* Never called on page 1 */
         6104  +  for(i=0; i<nCell; i++){
         6105  +    int sz = szCell[i];
         6106  +    u8 *pSlot;
         6107  +    if( bFreelist==0 || (pSlot = pageFindSlot(pPg, sz, 0, 0))==0 ){
         6108  +      pData -= sz;
         6109  +      if( pData<pBegin ) return 1;
         6110  +      pSlot = pData;
         6111  +    }
         6112  +    memcpy(pSlot, apCell[i], sz);
         6113  +    put2byte(pCellptr, (pSlot - aData));
         6114  +    pCellptr += 2;
         6115  +  }
         6116  +  *ppData = pData;
         6117  +  return 0;
         6118  +}
         6119  +
         6120  +/*
         6121  +** Array apCell[] contains nCell pointers to b-tree cells. Array szCell 
         6122  +** contains the size in bytes of each such cell. This function adds the
         6123  +** space associated with each cell in the array that is currently stored 
         6124  +** within the body of pPg to the pPg free-list. The cell-pointers and other
         6125  +** fields of the page are not updated.
         6126  +**
         6127  +** This function returns the total number of cells added to the free-list.
         6128  +*/
         6129  +static int pageFreeArray(
         6130  +  MemPage *pPg,                   /* Page to edit */
         6131  +  int nCell,                      /* Cells to delete */
         6132  +  u8 **apCell,                    /* Array of cells */
         6133  +  u16 *szCell                     /* Array of cell sizes */
         6134  +){
         6135  +  u8 * const aData = pPg->aData;
         6136  +  u8 * const pEnd = &aData[pPg->pBt->usableSize];
         6137  +  u8 * const pStart = &aData[pPg->hdrOffset + 8 + pPg->childPtrSize];
         6138  +  int nRet = 0;
         6139  +  int i;
         6140  +  u8 *pFree = 0;
         6141  +  int szFree = 0;
         6142  +
         6143  +  for(i=0; i<nCell; i++){
         6144  +    u8 *pCell = apCell[i];
         6145  +    if( pCell>=pStart && pCell<pEnd ){
         6146  +      int sz = szCell[i];
         6147  +      if( pFree!=(pCell + sz) ){
         6148  +        if( pFree ) freeSpace(pPg, pFree - aData, szFree);
         6149  +        pFree = pCell;
         6150  +        szFree = sz;
         6151  +        if( pFree+sz>pEnd ) return 0;
         6152  +      }else{
         6153  +        pFree = pCell;
         6154  +        szFree += sz;
         6155  +      }
         6156  +      nRet++;
         6157  +    }
         6158  +  }
         6159  +  if( pFree ) freeSpace(pPg, pFree - aData, szFree);
         6160  +  return nRet;
         6161  +}
         6162  +
         6163  +/*
         6164  +** The pPg->nFree field is invalid when this function returns. It is the
         6165  +** responsibility of the caller to set it correctly.
         6166  +*/
         6167  +static void editPage(
         6168  +  MemPage *pPg,                   /* Edit this page */
         6169  +  int iOld,                       /* Index of first cell currently on page */
         6170  +  int iNew,                       /* Index of new first cell on page */
         6171  +  int nNew,                       /* Final number of cells on page */
         6172  +  u8 **apCell,                    /* Array of cells */
         6173  +  u16 *szCell                     /* Array of cell sizes */
  5976   6174   ){
  5977         -  int i;            /* Loop counter */
  5978         -  u8 *pCellptr;     /* Address of next cell pointer */
  5979         -  int cellbody;     /* Address of next cell body */
  5980         -  u8 * const data = pPage->aData;             /* Pointer to data for pPage */
  5981         -  const int hdr = pPage->hdrOffset;           /* Offset of header on pPage */
  5982         -  const int nUsable = pPage->pBt->usableSize; /* Usable size of page */
  5983         -
  5984         -  assert( pPage->nOverflow==0 );
  5985         -  assert( sqlite3_mutex_held(pPage->pBt->mutex) );
  5986         -  assert( nCell>=0 && nCell<=(int)MX_CELL(pPage->pBt)
  5987         -            && (int)MX_CELL(pPage->pBt)<=10921);
  5988         -  assert( sqlite3PagerIswriteable(pPage->pDbPage) );
  5989         -
  5990         -  /* Check that the page has just been zeroed by zeroPage() */
  5991         -  assert( pPage->nCell==0 );
  5992         -  assert( get2byteNotZero(&data[hdr+5])==nUsable );
  5993         -
  5994         -  pCellptr = &pPage->aCellIdx[nCell*2];
  5995         -  cellbody = nUsable;
  5996         -  for(i=nCell-1; i>=0; i--){
  5997         -    u16 sz = aSize[i];
  5998         -    pCellptr -= 2;
  5999         -    cellbody -= sz;
  6000         -    put2byte(pCellptr, cellbody);
  6001         -    memcpy(&data[cellbody], apCell[i], sz);
  6002         -  }
  6003         -  put2byte(&data[hdr+3], nCell);
  6004         -  put2byte(&data[hdr+5], cellbody);
  6005         -  pPage->nFree -= (nCell*2 + nUsable - cellbody);
  6006         -  pPage->nCell = (u16)nCell;
         6175  +  u8 * const aData = pPg->aData;
         6176  +  const int hdr = pPg->hdrOffset;
         6177  +  u8 *pBegin = &pPg->aCellIdx[nNew * 2];
         6178  +  int nCell = pPg->nCell;       /* Cells stored on pPg */
         6179  +  u8 *pData;
         6180  +  u8 *pCellptr;
         6181  +  int i;
         6182  +  int iOldEnd = iOld + pPg->nCell + pPg->nOverflow;
         6183  +  int iNewEnd = iNew + nNew;
         6184  +
         6185  +#ifdef SQLITE_DEBUG
         6186  +  u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager);
         6187  +  memcpy(pTmp, aData, pPg->pBt->usableSize);
         6188  +#endif
         6189  +
         6190  +  /* Remove cells from the start and end of the page */
         6191  +  if( iOld<iNew ){
         6192  +    int nShift = pageFreeArray(
         6193  +        pPg, iNew-iOld, &apCell[iOld], &szCell[iOld]
         6194  +    );
         6195  +    memmove(pPg->aCellIdx, &pPg->aCellIdx[nShift*2], nCell*2);
         6196  +    nCell -= nShift;
         6197  +  }
         6198  +  if( iNewEnd < iOldEnd ){
         6199  +    nCell -= pageFreeArray(
         6200  +        pPg, iOldEnd-iNewEnd, &apCell[iNewEnd], &szCell[iNewEnd]
         6201  +    );
         6202  +  }
         6203  +
         6204  +  pData = &aData[get2byte(&aData[hdr+5])];
         6205  +  if( pData<pBegin ) goto editpage_fail;
         6206  +
         6207  +  /* Add cells to the start of the page */
         6208  +  if( iNew<iOld ){
         6209  +    int nAdd = iOld-iNew;
         6210  +    pCellptr = pPg->aCellIdx;
         6211  +    memmove(&pCellptr[nAdd*2], pCellptr, nCell*2);
         6212  +    if( pageInsertArray(
         6213  +          pPg, pBegin, &pData, pCellptr,
         6214  +          nAdd, &apCell[iNew], &szCell[iNew]
         6215  +    ) ) goto editpage_fail;
         6216  +    nCell += nAdd;
         6217  +  }
         6218  +
         6219  +  /* Add any overflow cells */
         6220  +  for(i=0; i<pPg->nOverflow; i++){
         6221  +    int iCell = (iOld + pPg->aiOvfl[i]) - iNew;
         6222  +    if( iCell>=0 && iCell<nNew ){
         6223  +      u8 *pCellptr = &pPg->aCellIdx[iCell * 2];
         6224  +      memmove(&pCellptr[2], pCellptr, (nCell - iCell) * 2);
         6225  +      nCell++;
         6226  +      if( pageInsertArray(
         6227  +            pPg, pBegin, &pData, pCellptr,
         6228  +            1, &apCell[iCell + iNew], &szCell[iCell + iNew]
         6229  +      ) ) goto editpage_fail;
         6230  +    }
         6231  +  }
         6232  +
         6233  +  /* Append cells to the end of the page */
         6234  +  pCellptr = &pPg->aCellIdx[nCell*2];
         6235  +  if( pageInsertArray(
         6236  +        pPg, pBegin, &pData, pCellptr,
         6237  +        nNew-nCell, &apCell[iNew+nCell], &szCell[iNew+nCell]
         6238  +  ) ) goto editpage_fail;
         6239  +
         6240  +  pPg->nCell = nNew;
         6241  +  pPg->nOverflow = 0;
         6242  +
         6243  +  put2byte(&aData[hdr+3], pPg->nCell);
         6244  +  put2byte(&aData[hdr+5], pData - aData);
         6245  +
         6246  +#ifdef SQLITE_DEBUG
         6247  +  for(i=0; i<nNew && !CORRUPT_DB; i++){
         6248  +    u8 *pCell = apCell[i+iNew];
         6249  +    int iOff = get2byte(&pPg->aCellIdx[i*2]);
         6250  +    if( pCell>=aData && pCell<&aData[pPg->pBt->usableSize] ){
         6251  +      pCell = &pTmp[pCell - aData];
         6252  +    }
         6253  +    assert( 0==memcmp(pCell, &aData[iOff], szCell[i+iNew]) );
         6254  +  }
         6255  +#endif
         6256  +
         6257  +  return;
         6258  + editpage_fail:
         6259  +  /* Unable to edit this page. Rebuild it from scratch instead. */
         6260  +  rebuildPage(pPg, nNew, &apCell[iNew], &szCell[iNew]);
  6007   6261   }
  6008   6262   
  6009   6263   /*
  6010   6264   ** The following parameters determine how many adjacent pages get involved
  6011   6265   ** in a balancing operation.  NN is the number of neighbors on either side
  6012   6266   ** of the page that participate in the balancing operation.  NB is the
  6013   6267   ** total number of pages that participate, including the target page and
................................................................................
  6071   6325       u8 *pCell = pPage->apOvfl[0];
  6072   6326       u16 szCell = cellSizePtr(pPage, pCell);
  6073   6327       u8 *pStop;
  6074   6328   
  6075   6329       assert( sqlite3PagerIswriteable(pNew->pDbPage) );
  6076   6330       assert( pPage->aData[0]==(PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF) );
  6077   6331       zeroPage(pNew, PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF);
  6078         -    assemblePage(pNew, 1, &pCell, &szCell);
         6332  +    rebuildPage(pNew, 1, &pCell, &szCell);
         6333  +    pNew->nFree = pBt->usableSize - pNew->cellOffset - 2 - szCell;
  6079   6334   
  6080   6335       /* If this is an auto-vacuum database, update the pointer map
  6081   6336       ** with entries for the new page, and any pointer from the 
  6082   6337       ** cell on the page to an overflow page. If either of these
  6083   6338       ** operations fails, the return code is set, but the contents
  6084   6339       ** of the parent page are still manipulated by thh code below.
  6085   6340       ** That is Ok, at this point the parent page is guaranteed to
................................................................................
  6290   6545     int usableSpace;             /* Bytes in pPage beyond the header */
  6291   6546     int pageFlags;               /* Value of pPage->aData[0] */
  6292   6547     int subtotal;                /* Subtotal of bytes in cells on one page */
  6293   6548     int iSpace1 = 0;             /* First unused byte of aSpace1[] */
  6294   6549     int iOvflSpace = 0;          /* First unused byte of aOvflSpace[] */
  6295   6550     int szScratch;               /* Size of scratch memory requested */
  6296   6551     MemPage *apOld[NB];          /* pPage and up to two siblings */
  6297         -  MemPage *apCopy[NB];         /* Private copies of apOld[] pages */
  6298   6552     MemPage *apNew[NB+2];        /* pPage and up to NB siblings after balancing */
  6299   6553     u8 *pRight;                  /* Location in parent of right-sibling pointer */
  6300   6554     u8 *apDiv[NB-1];             /* Divider cells in pParent */
  6301   6555     int cntNew[NB+2];            /* Index in aCell[] of cell after i-th page */
         6556  +  int cntOld[NB+2];            /* Old index in aCell[] after i-th page */
  6302   6557     int szNew[NB+2];             /* Combined size of cells place on i-th page */
  6303   6558     u8 **apCell = 0;             /* All cells begin balanced */
  6304   6559     u16 *szCell;                 /* Local size of all cells in apCell[] */
  6305   6560     u8 *aSpace1;                 /* Space for copies of dividers cells */
  6306   6561     Pgno pgno;                   /* Temp var to store a page number in */
         6562  +  u8 abDone[NB+2];             /* True after i'th new page is populated */
         6563  +  Pgno aPgno[NB+2];            /* Page numbers of new pages before shuffling */
         6564  +  u16 aPgFlags[NB+2];          /* flags field of new pages before shuffling */
  6307   6565   
         6566  +  memset(abDone, 0, sizeof(abDone));
  6308   6567     pBt = pParent->pBt;
  6309   6568     assert( sqlite3_mutex_held(pBt->mutex) );
  6310   6569     assert( sqlite3PagerIswriteable(pParent->pDbPage) );
  6311   6570   
  6312   6571   #if 0
  6313   6572     TRACE(("BALANCE: begin page %d child of %d\n", pPage->pgno, pParent->pgno));
  6314   6573   #endif
................................................................................
  6409   6668     /* Make nMaxCells a multiple of 4 in order to preserve 8-byte
  6410   6669     ** alignment */
  6411   6670     nMaxCells = (nMaxCells + 3)&~3;
  6412   6671   
  6413   6672     /*
  6414   6673     ** Allocate space for memory structures
  6415   6674     */
  6416         -  k = pBt->pageSize + ROUND8(sizeof(MemPage));
  6417   6675     szScratch =
  6418   6676          nMaxCells*sizeof(u8*)                       /* apCell */
  6419   6677        + nMaxCells*sizeof(u16)                       /* szCell */
  6420         -     + pBt->pageSize                               /* aSpace1 */
  6421         -     + k*nOld;                                     /* Page copies (apCopy) */
         6678  +     + pBt->pageSize;                              /* aSpace1 */
  6422   6679     apCell = sqlite3ScratchMalloc( szScratch ); 
  6423   6680     if( apCell==0 ){
  6424   6681       rc = SQLITE_NOMEM;
  6425   6682       goto balance_cleanup;
  6426   6683     }
  6427   6684     szCell = (u16*)&apCell[nMaxCells];
  6428   6685     aSpace1 = (u8*)&szCell[nMaxCells];
  6429   6686     assert( EIGHT_BYTE_ALIGNMENT(aSpace1) );
  6430   6687   
  6431   6688     /*
  6432   6689     ** Load pointers to all cells on sibling pages and the divider cells
  6433   6690     ** into the local apCell[] array.  Make copies of the divider cells
  6434         -  ** into space obtained from aSpace1[] and remove the divider cells
  6435         -  ** from pParent.
         6691  +  ** into space obtained from aSpace1[]. The divider cells have already
         6692  +  ** been removed from pParent.
  6436   6693     **
  6437   6694     ** If the siblings are on leaf pages, then the child pointers of the
  6438   6695     ** divider cells are stripped from the cells before they are copied
  6439   6696     ** into aSpace1[].  In this way, all cells in apCell[] are without
  6440   6697     ** child pointers.  If siblings are not leaves, then all cell in
  6441   6698     ** apCell[] include child pointers.  Either way, all cells in apCell[]
  6442   6699     ** are alike.
................................................................................
  6444   6701     ** leafCorrection:  4 if pPage is a leaf.  0 if pPage is not a leaf.
  6445   6702     **       leafData:  1 if pPage holds key+data and pParent holds only keys.
  6446   6703     */
  6447   6704     leafCorrection = apOld[0]->leaf*4;
  6448   6705     leafData = apOld[0]->intKeyLeaf;
  6449   6706     for(i=0; i<nOld; i++){
  6450   6707       int limit;
  6451         -    
  6452         -    /* Before doing anything else, take a copy of the i'th original sibling
  6453         -    ** The rest of this function will use data from the copies rather
  6454         -    ** that the original pages since the original pages will be in the
  6455         -    ** process of being overwritten.  */
  6456         -    MemPage *pOld = apCopy[i] = (MemPage*)&aSpace1[pBt->pageSize + k*i];
  6457         -    memcpy(pOld, apOld[i], sizeof(MemPage));
  6458         -    pOld->aData = (void*)&pOld[1];
  6459         -    memcpy(pOld->aData, apOld[i]->aData, pBt->pageSize);
         6708  +    MemPage *pOld = apOld[i];
  6460   6709   
  6461   6710       limit = pOld->nCell+pOld->nOverflow;
  6462   6711       if( pOld->nOverflow>0 ){
  6463   6712         for(j=0; j<limit; j++){
  6464   6713           assert( nCell<nMaxCells );
  6465   6714           apCell[nCell] = findOverflowCell(pOld, j);
  6466   6715           szCell[nCell] = cellSizePtr(pOld, apCell[nCell]);
................................................................................
  6473   6722         for(j=0; j<limit; j++){
  6474   6723           assert( nCell<nMaxCells );
  6475   6724           apCell[nCell] = findCellv2(aData, maskPage, cellOffset, j);
  6476   6725           szCell[nCell] = cellSizePtr(pOld, apCell[nCell]);
  6477   6726           nCell++;
  6478   6727         }
  6479   6728       }       
         6729  +    cntOld[i] = nCell;
  6480   6730       if( i<nOld-1 && !leafData){
  6481   6731         u16 sz = (u16)szNew[i];
  6482   6732         u8 *pTemp;
  6483   6733         assert( nCell<nMaxCells );
  6484   6734         szCell[nCell] = sz;
  6485   6735         pTemp = &aSpace1[iSpace1];
  6486   6736         iSpace1 += sz;
................................................................................
  6524   6774     ** 
  6525   6775     */
  6526   6776     usableSpace = pBt->usableSize - 12 + leafCorrection;
  6527   6777     for(subtotal=k=i=0; i<nCell; i++){
  6528   6778       assert( i<nMaxCells );
  6529   6779       subtotal += szCell[i] + 2;
  6530   6780       if( subtotal > usableSpace ){
  6531         -      szNew[k] = subtotal - szCell[i];
         6781  +      szNew[k] = subtotal - szCell[i] - 2;
  6532   6782         cntNew[k] = i;
  6533   6783         if( leafData ){ i--; }
  6534   6784         subtotal = 0;
  6535   6785         k++;
  6536   6786         if( k>NB+1 ){ rc = SQLITE_CORRUPT_BKPT; goto balance_cleanup; }
  6537   6787       }
  6538   6788     }
................................................................................
  6581   6831     ** file is corrupt.  The corruption will be detected and reported later
  6582   6832     ** in this procedure so there is no need to act upon it now.
  6583   6833     */
  6584   6834   #if 0
  6585   6835     assert( cntNew[0]>0 || (pParent->pgno==1 && pParent->nCell==0) );
  6586   6836   #endif
  6587   6837   
  6588         -  TRACE(("BALANCE: old: %d %d %d  ",
  6589         -    apOld[0]->pgno, 
  6590         -    nOld>=2 ? apOld[1]->pgno : 0,
  6591         -    nOld>=3 ? apOld[2]->pgno : 0
         6838  +  TRACE(("BALANCE: old: %d(nc=%d) %d(nc=%d) %d(nc=%d)\n",
         6839  +    apOld[0]->pgno, apOld[0]->nCell,
         6840  +    nOld>=2 ? apOld[1]->pgno : 0, nOld>=2 ? apOld[1]->nCell : 0,
         6841  +    nOld>=3 ? apOld[2]->pgno : 0, nOld>=3 ? apOld[2]->nCell : 0
  6592   6842     ));
  6593   6843   
  6594   6844     /*
  6595   6845     ** Allocate k new pages.  Reuse old pages where possible.
  6596   6846     */
  6597   6847     if( apOld[0]->pgno<=1 ){
  6598   6848       rc = SQLITE_CORRUPT_BKPT;
................................................................................
  6607   6857         rc = sqlite3PagerWrite(pNew->pDbPage);
  6608   6858         nNew++;
  6609   6859         if( rc ) goto balance_cleanup;
  6610   6860       }else{
  6611   6861         assert( i>0 );
  6612   6862         rc = allocateBtreePage(pBt, &pNew, &pgno, (bBulk ? 1 : pgno), 0);
  6613   6863         if( rc ) goto balance_cleanup;
         6864  +      zeroPage(pNew, pageFlags);
  6614   6865         apNew[i] = pNew;
  6615   6866         nNew++;
         6867  +      cntOld[i] = nCell;
  6616   6868   
  6617   6869         /* Set the pointer-map entry for the new sibling page. */
  6618   6870         if( ISAUTOVACUUM ){
  6619   6871           ptrmapPut(pBt, pNew->pgno, PTRMAP_BTREE, pParent->pgno, &rc);
  6620   6872           if( rc!=SQLITE_OK ){
  6621   6873             goto balance_cleanup;
  6622   6874           }
  6623   6875         }
  6624   6876       }
  6625   6877     }
  6626   6878   
  6627         -  /* Free any old pages that were not reused as new pages.
  6628         -  */
  6629         -  while( i<nOld ){
  6630         -    freePage(apOld[i], &rc);
  6631         -    if( rc ) goto balance_cleanup;
  6632         -    releasePage(apOld[i]);
  6633         -    apOld[i] = 0;
  6634         -    i++;
  6635         -  }
  6636         -
  6637   6879     /*
  6638         -  ** Put the new pages in ascending order.  This helps to
  6639         -  ** keep entries in the disk file in order so that a scan
  6640         -  ** of the table is a linear scan through the file.  That
  6641         -  ** in turn helps the operating system to deliver pages
  6642         -  ** from the disk more rapidly.
  6643         -  **
  6644         -  ** An O(n^2) insertion sort algorithm is used, but since
  6645         -  ** n is never more than NB (a small constant), that should
  6646         -  ** not be a problem.
  6647         -  **
  6648         -  ** When NB==3, this one optimization makes the database
  6649         -  ** about 25% faster for large insertions and deletions.
  6650         -  */
  6651         -  for(i=0; i<k-1; i++){
  6652         -    int minV = apNew[i]->pgno;
  6653         -    int minI = i;
  6654         -    for(j=i+1; j<k; j++){
  6655         -      if( apNew[j]->pgno<(unsigned)minV ){
  6656         -        minI = j;
  6657         -        minV = apNew[j]->pgno;
  6658         -      }
  6659         -    }
  6660         -    if( minI>i ){
  6661         -      MemPage *pT;
  6662         -      pT = apNew[i];
  6663         -      apNew[i] = apNew[minI];
  6664         -      apNew[minI] = pT;
  6665         -    }
  6666         -  }
  6667         -  TRACE(("new: %d(%d) %d(%d) %d(%d) %d(%d) %d(%d)\n",
  6668         -    apNew[0]->pgno, szNew[0],
         6880  +  ** Reassign page numbers so that the new pages are in ascending order. 
         6881  +  ** This helps to keep entries in the disk file in order so that a scan
         6882  +  ** of the table is closer to a linear scan through the file. That in turn 
         6883  +  ** helps the operating system to deliver pages from the disk more rapidly.
         6884  +  **
         6885  +  ** An O(n^2) insertion sort algorithm is used, but since n is never more 
         6886  +  ** than (NB+2) (a small constant), that should not be a problem.
         6887  +  **
         6888  +  ** When NB==3, this one optimization makes the database about 25% faster 
         6889  +  ** for large insertions and deletions.
         6890  +  */
         6891  +  for(i=0; i<nNew; i++){
         6892  +    aPgno[i] = apNew[i]->pgno;
         6893  +    aPgFlags[i] = apNew[i]->pDbPage->flags;
         6894  +    for(j=0; j<i; j++){
         6895  +      if( aPgno[j]==aPgno[i] ){
         6896  +        /* This branch is taken if the set of sibling pages somehow contains
         6897  +        ** duplicate entries. This can happen if the database is corrupt. 
         6898  +        ** It would be simpler to detect this as part of the loop below, but
         6899  +        ** in order to avoid populating the pager cache with two separate
         6900  +        ** objects associated with the same page number.  */
         6901  +        assert( CORRUPT_DB );
         6902  +        rc = SQLITE_CORRUPT_BKPT;
         6903  +        goto balance_cleanup;
         6904  +      }
         6905  +    }
         6906  +  }
         6907  +  for(i=0; i<nNew; i++){
         6908  +    int iBest = 0;                /* aPgno[] index of page number to use */
         6909  +    Pgno pgno;                    /* Page number to use */
         6910  +    for(j=1; j<nNew; j++){
         6911  +      if( aPgno[j]<aPgno[iBest] ) iBest = j;
         6912  +    }
         6913  +    pgno = aPgno[iBest];
         6914  +    aPgno[iBest] = 0xffffffff;
         6915  +    if( iBest!=i ){
         6916  +      if( iBest>i ){
         6917  +        sqlite3PagerRekey(apNew[iBest]->pDbPage, pBt->nPage+iBest+1, 0);
         6918  +      }
         6919  +      sqlite3PagerRekey(apNew[i]->pDbPage, pgno, aPgFlags[iBest]);
         6920  +      apNew[i]->pgno = pgno;
         6921  +    }
         6922  +  }
         6923  +
         6924  +  TRACE(("BALANCE: new: %d(%d nc=%d) %d(%d nc=%d) %d(%d nc=%d) "
         6925  +         "%d(%d nc=%d) %d(%d nc=%d)\n",
         6926  +    apNew[0]->pgno, szNew[0], cntNew[0],
  6669   6927       nNew>=2 ? apNew[1]->pgno : 0, nNew>=2 ? szNew[1] : 0,
         6928  +    nNew>=2 ? cntNew[1] - cntNew[0] - !leafData : 0,
  6670   6929       nNew>=3 ? apNew[2]->pgno : 0, nNew>=3 ? szNew[2] : 0,
         6930  +    nNew>=3 ? cntNew[2] - cntNew[1] - !leafData : 0,
  6671   6931       nNew>=4 ? apNew[3]->pgno : 0, nNew>=4 ? szNew[3] : 0,
  6672         -    nNew>=5 ? apNew[4]->pgno : 0, nNew>=5 ? szNew[4] : 0));
         6932  +    nNew>=4 ? cntNew[3] - cntNew[2] - !leafData : 0,
         6933  +    nNew>=5 ? apNew[4]->pgno : 0, nNew>=5 ? szNew[4] : 0,
         6934  +    nNew>=5 ? cntNew[4] - cntNew[3] - !leafData : 0
         6935  +  ));
  6673   6936   
  6674   6937     assert( sqlite3PagerIswriteable(pParent->pDbPage) );
  6675   6938     put4byte(pRight, apNew[nNew-1]->pgno);
  6676   6939   
  6677         -  /*
  6678         -  ** Evenly distribute the data in apCell[] across the new pages.
  6679         -  ** Insert divider cells into pParent as necessary.
  6680         -  */
  6681         -  j = 0;
  6682         -  for(i=0; i<nNew; i++){
  6683         -    /* Assemble the new sibling page. */
  6684         -    MemPage *pNew = apNew[i];
  6685         -    assert( j<nMaxCells );
  6686         -    zeroPage(pNew, pageFlags);
  6687         -    assemblePage(pNew, cntNew[i]-j, &apCell[j], &szCell[j]);
  6688         -    assert( pNew->nCell>0 || (nNew==1 && cntNew[0]==0) );
  6689         -    assert( pNew->nOverflow==0 );
  6690         -
         6940  +  /* If the sibling pages are not leaves, ensure that the right-child pointer
         6941  +  ** of the right-most new sibling page is set to the value that was 
         6942  +  ** originally in the same field of the right-most old sibling page. */
         6943  +  if( (pageFlags & PTF_LEAF)==0 && nOld!=nNew ){
         6944  +    MemPage *pOld = (nNew>nOld ? apNew : apOld)[nOld-1];
         6945  +    memcpy(&apNew[nNew-1]->aData[8], &pOld->aData[8], 4);
         6946  +  }
         6947  +
         6948  +  /* Make any required updates to pointer map entries associated with 
         6949  +  ** cells stored on sibling pages following the balance operation. Pointer
         6950  +  ** map entries associated with divider cells are set by the insertCell()
         6951  +  ** routine. The associated pointer map entries are:
         6952  +  **
         6953  +  **   a) if the cell contains a reference to an overflow chain, the
         6954  +  **      entry associated with the first page in the overflow chain, and
         6955  +  **
         6956  +  **   b) if the sibling pages are not leaves, the child page associated
         6957  +  **      with the cell.
         6958  +  **
         6959  +  ** If the sibling pages are not leaves, then the pointer map entry 
         6960  +  ** associated with the right-child of each sibling may also need to be 
         6961  +  ** updated. This happens below, after the sibling pages have been 
         6962  +  ** populated, not here.
         6963  +  */
         6964  +  if( ISAUTOVACUUM ){
         6965  +    MemPage *pNew = apNew[0];
         6966  +    u8 *aOld = pNew->aData;
         6967  +    int cntOldNext = pNew->nCell + pNew->nOverflow;
         6968  +    int usableSize = pBt->usableSize;
         6969  +    int iNew = 0;
         6970  +    int iOld = 0;
         6971  +
         6972  +    for(i=0; i<nCell; i++){
         6973  +      u8 *pCell = apCell[i];
         6974  +      if( i==cntOldNext ){
         6975  +        MemPage *pOld = (++iOld)<nNew ? apNew[iOld] : apOld[iOld];
         6976  +        cntOldNext += pOld->nCell + pOld->nOverflow + !leafData;
         6977  +        aOld = pOld->aData;
         6978  +      }
         6979  +      if( i==cntNew[iNew] ){
         6980  +        pNew = apNew[++iNew];
         6981  +        if( !leafData ) continue;
         6982  +      }
         6983  +
         6984  +      /* Cell pCell is destined for new sibling page pNew. Originally, it
         6985  +      ** was either part of sibling page iOld (possibly an overflow page), 
         6986  +      ** or else the divider cell to the left of sibling page iOld. So,
         6987  +      ** if sibling page iOld had the same page number as pNew, and if
         6988  +      ** pCell really was a part of sibling page iOld (not a divider or
         6989  +      ** overflow cell), we can skip updating the pointer map entries.  */
         6990  +      if( pNew->pgno!=aPgno[iOld] || pCell<aOld || pCell>=&aOld[usableSize] ){
         6991  +        if( !leafCorrection ){
         6992  +          ptrmapPut(pBt, get4byte(pCell), PTRMAP_BTREE, pNew->pgno, &rc);
         6993  +        }
         6994  +        if( szCell[i]>pNew->minLocal ){
         6995  +          ptrmapPutOvflPtr(pNew, pCell, &rc);
         6996  +        }
         6997  +      }
         6998  +    }
         6999  +  }
         7000  +
         7001  +  /* Insert new divider cells into pParent. */
         7002  +  for(i=0; i<nNew-1; i++){
         7003  +    u8 *pCell;
         7004  +    u8 *pTemp;
         7005  +    int sz;
         7006  +    MemPage *pNew = apNew[i];
  6691   7007       j = cntNew[i];
  6692   7008   
  6693         -    /* If the sibling page assembled above was not the right-most sibling,
  6694         -    ** insert a divider cell into the parent page.
  6695         -    */
  6696         -    assert( i<nNew-1 || j==nCell );
  6697         -    if( j<nCell ){
  6698         -      u8 *pCell;
  6699         -      u8 *pTemp;
  6700         -      int sz;
  6701         -
  6702         -      assert( j<nMaxCells );
  6703         -      pCell = apCell[j];
  6704         -      sz = szCell[j] + leafCorrection;
  6705         -      pTemp = &aOvflSpace[iOvflSpace];
  6706         -      if( !pNew->leaf ){
  6707         -        memcpy(&pNew->aData[8], pCell, 4);
  6708         -      }else if( leafData ){
  6709         -        /* If the tree is a leaf-data tree, and the siblings are leaves, 
  6710         -        ** then there is no divider cell in apCell[]. Instead, the divider 
  6711         -        ** cell consists of the integer key for the right-most cell of 
  6712         -        ** the sibling-page assembled above only.
  6713         -        */
  6714         -        CellInfo info;
  6715         -        j--;
  6716         -        btreeParseCellPtr(pNew, apCell[j], &info);
  6717         -        pCell = pTemp;
  6718         -        sz = 4 + putVarint(&pCell[4], info.nKey);
  6719         -        pTemp = 0;
         7009  +    assert( j<nMaxCells );
         7010  +    pCell = apCell[j];
         7011  +    sz = szCell[j] + leafCorrection;
         7012  +    pTemp = &aOvflSpace[iOvflSpace];
         7013  +    if( !pNew->leaf ){
         7014  +      memcpy(&pNew->aData[8], pCell, 4);
         7015  +    }else if( leafData ){
         7016  +      /* If the tree is a leaf-data tree, and the siblings are leaves, 
         7017  +      ** then there is no divider cell in apCell[]. Instead, the divider 
         7018  +      ** cell consists of the integer key for the right-most cell of 
         7019  +      ** the sibling-page assembled above only.
         7020  +      */
         7021  +      CellInfo info;
         7022  +      j--;
         7023  +      btreeParseCellPtr(pNew, apCell[j], &info);
         7024  +      pCell = pTemp;
         7025  +      sz = 4 + putVarint(&pCell[4], info.nKey);
         7026  +      pTemp = 0;
         7027  +    }else{
         7028  +      pCell -= 4;
         7029  +      /* Obscure case for non-leaf-data trees: If the cell at pCell was
         7030  +      ** previously stored on a leaf node, and its reported size was 4
         7031  +      ** bytes, then it may actually be smaller than this 
         7032  +      ** (see btreeParseCellPtr(), 4 bytes is the minimum size of
         7033  +      ** any cell). But it is important to pass the correct size to 
         7034  +      ** insertCell(), so reparse the cell now.
         7035  +      **
         7036  +      ** Note that this can never happen in an SQLite data file, as all
         7037  +      ** cells are at least 4 bytes. It only happens in b-trees used
         7038  +      ** to evaluate "IN (SELECT ...)" and similar clauses.
         7039  +      */
         7040  +      if( szCell[j]==4 ){
         7041  +        assert(leafCorrection==4);
         7042  +        sz = cellSizePtr(pParent, pCell);
         7043  +      }
         7044  +    }
         7045  +    iOvflSpace += sz;
         7046  +    assert( sz<=pBt->maxLocal+23 );
         7047  +    assert( iOvflSpace <= (int)pBt->pageSize );
         7048  +    insertCell(pParent, nxDiv+i, pCell, sz, pTemp, pNew->pgno, &rc);
         7049  +    if( rc!=SQLITE_OK ) goto balance_cleanup;
         7050  +    assert( sqlite3PagerIswriteable(pParent->pDbPage) );
         7051  +  }
         7052  +
         7053  +  /* Now update the actual sibling pages. The order in which they are updated
         7054  +  ** is important, as this code needs to avoid disrupting any page from which
         7055  +  ** cells may still to be read. In practice, this means:
         7056  +  **
         7057  +  **   1) If cells are to be removed from the start of the page and shifted
         7058  +  **      to the left-hand sibling, it is not safe to update the page until 
         7059  +  **      the left-hand sibling (apNew[i-1]) has already been updated.
         7060  +  **
         7061  +  **   2) If cells are to be removed from the end of the page and shifted
         7062  +  **      to the right-hand sibling, it is not safe to update the page until 
         7063  +  **      the right-hand sibling (apNew[i+1]) has already been updated.
         7064  +  **
         7065  +  ** If neither of the above apply, the page is safe to update.
         7066  +  */
         7067  +  for(i=0; i<nNew*2; i++){
         7068  +    int iPg = (i>=nNew ? i-nNew : nNew-1-i);
         7069  +    if( abDone[iPg]==0 
         7070  +     && (iPg==0 || cntOld[iPg-1]>=cntNew[iPg-1] || abDone[iPg-1])
         7071  +     && (cntNew[iPg]>=cntOld[iPg] || abDone[iPg+1])
         7072  +    ){
         7073  +      int iNew;
         7074  +      int iOld;
         7075  +      int nNewCell;
         7076  +
         7077  +      if( iPg==0 ){
         7078  +        iNew = iOld = 0;
         7079  +        nNewCell = cntNew[0];
  6720   7080         }else{
  6721         -        pCell -= 4;
  6722         -        /* Obscure case for non-leaf-data trees: If the cell at pCell was
  6723         -        ** previously stored on a leaf node, and its reported size was 4
  6724         -        ** bytes, then it may actually be smaller than this 
  6725         -        ** (see btreeParseCellPtr(), 4 bytes is the minimum size of
  6726         -        ** any cell). But it is important to pass the correct size to 
  6727         -        ** insertCell(), so reparse the cell now.
  6728         -        **
  6729         -        ** Note that this can never happen in an SQLite data file, as all
  6730         -        ** cells are at least 4 bytes. It only happens in b-trees used
  6731         -        ** to evaluate "IN (SELECT ...)" and similar clauses.
  6732         -        */
  6733         -        if( szCell[j]==4 ){
  6734         -          assert(leafCorrection==4);
  6735         -          sz = cellSizePtr(pParent, pCell);
  6736         -        }
  6737         -      }
  6738         -      iOvflSpace += sz;
  6739         -      assert( sz<=pBt->maxLocal+23 );
  6740         -      assert( iOvflSpace <= (int)pBt->pageSize );
  6741         -      insertCell(pParent, nxDiv, pCell, sz, pTemp, pNew->pgno, &rc);
  6742         -      if( rc!=SQLITE_OK ) goto balance_cleanup;
  6743         -      assert( sqlite3PagerIswriteable(pParent->pDbPage) );
  6744         -
  6745         -      j++;
  6746         -      nxDiv++;
  6747         -    }
  6748         -  }
  6749         -  assert( j==nCell );
         7081  +        iOld = iPg<nOld ? (cntOld[iPg-1] + !leafData) : nCell;
         7082  +        iNew = cntNew[iPg-1] + !leafData;
         7083  +        nNewCell = cntNew[iPg] - iNew;
         7084  +      }
         7085  +
         7086  +      editPage(apNew[iPg], iOld, iNew, nNewCell, apCell, szCell);
         7087  +      abDone[iPg] = 1;
         7088  +      apNew[iPg]->nFree = usableSpace-szNew[iPg];
         7089  +      assert( apNew[iPg]->nOverflow==0 );
         7090  +      assert( apNew[iPg]->nCell==nNewCell );
         7091  +    }
         7092  +  }
         7093  +  assert( memcmp(abDone, "\01\01\01\01\01", nNew)==0 );
         7094  +
  6750   7095     assert( nOld>0 );
  6751   7096     assert( nNew>0 );
  6752         -  if( (pageFlags & PTF_LEAF)==0 ){
  6753         -    u8 *zChild = &apCopy[nOld-1]->aData[8];
  6754         -    memcpy(&apNew[nNew-1]->aData[8], zChild, 4);
  6755         -  }
  6756   7097   
  6757   7098     if( isRoot && pParent->nCell==0 && pParent->hdrOffset<=apNew[0]->nFree ){
  6758   7099       /* The root page of the b-tree now contains no cells. The only sibling
  6759   7100       ** page is the right-child of the parent. Copy the contents of the
  6760   7101       ** child page into the parent, decreasing the overall height of the
  6761   7102       ** b-tree structure by one. This is described as the "balance-shallower"
  6762   7103       ** sub-algorithm in some documentation.
................................................................................
  6766   7107       ** for which the pointer is stored within the content being copied.
  6767   7108       **
  6768   7109       ** The second assert below verifies that the child page is defragmented
  6769   7110       ** (it must be, as it was just reconstructed using assemblePage()). This
  6770   7111       ** is important if the parent page happens to be page 1 of the database
  6771   7112       ** image.  */
  6772   7113       assert( nNew==1 );
  6773         -    assert( apNew[0]->nFree == 
  6774         -        (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) 
  6775         -    );
  6776         -    copyNodeContent(apNew[0], pParent, &rc);
  6777         -    freePage(apNew[0], &rc);
  6778         -  }else if( ISAUTOVACUUM ){
  6779         -    /* Fix the pointer-map entries for all the cells that were shifted around. 
  6780         -    ** There are several different types of pointer-map entries that need to
  6781         -    ** be dealt with by this routine. Some of these have been set already, but
  6782         -    ** many have not. The following is a summary:
  6783         -    **
  6784         -    **   1) The entries associated with new sibling pages that were not
  6785         -    **      siblings when this function was called. These have already
  6786         -    **      been set. We don't need to worry about old siblings that were
  6787         -    **      moved to the free-list - the freePage() code has taken care
  6788         -    **      of those.
  6789         -    **
  6790         -    **   2) The pointer-map entries associated with the first overflow
  6791         -    **      page in any overflow chains used by new divider cells. These 
  6792         -    **      have also already been taken care of by the insertCell() code.
  6793         -    **
  6794         -    **   3) If the sibling pages are not leaves, then the child pages of
  6795         -    **      cells stored on the sibling pages may need to be updated.
  6796         -    **
  6797         -    **   4) If the sibling pages are not internal intkey nodes, then any
  6798         -    **      overflow pages used by these cells may need to be updated
  6799         -    **      (internal intkey nodes never contain pointers to overflow pages).
  6800         -    **
  6801         -    **   5) If the sibling pages are not leaves, then the pointer-map
  6802         -    **      entries for the right-child pages of each sibling may need
  6803         -    **      to be updated.
  6804         -    **
  6805         -    ** Cases 1 and 2 are dealt with above by other code. The next
  6806         -    ** block deals with cases 3 and 4 and the one after that, case 5. Since
  6807         -    ** setting a pointer map entry is a relatively expensive operation, this
  6808         -    ** code only sets pointer map entries for child or overflow pages that have
  6809         -    ** actually moved between pages.  */
  6810         -    MemPage *pNew = apNew[0];
  6811         -    MemPage *pOld = apCopy[0];
  6812         -    int nOverflow = pOld->nOverflow;
  6813         -    int iNextOld = pOld->nCell + nOverflow;
  6814         -    int iOverflow = (nOverflow ? pOld->aiOvfl[0] : -1);
  6815         -    j = 0;                             /* Current 'old' sibling page */
  6816         -    k = 0;                             /* Current 'new' sibling page */
  6817         -    for(i=0; i<nCell; i++){
  6818         -      int isDivider = 0;
  6819         -      while( i==iNextOld ){
  6820         -        /* Cell i is the cell immediately following the last cell on old
  6821         -        ** sibling page j. If the siblings are not leaf pages of an
  6822         -        ** intkey b-tree, then cell i was a divider cell. */
  6823         -        assert( j+1 < ArraySize(apCopy) );
  6824         -        assert( j+1 < nOld );
  6825         -        pOld = apCopy[++j];
  6826         -        iNextOld = i + !leafData + pOld->nCell + pOld->nOverflow;
  6827         -        if( pOld->nOverflow ){
  6828         -          nOverflow = pOld->nOverflow;
  6829         -          iOverflow = i + !leafData + pOld->aiOvfl[0];
  6830         -        }
  6831         -        isDivider = !leafData;  
  6832         -      }
  6833         -
  6834         -      assert(nOverflow>0 || iOverflow<i );
  6835         -      assert(nOverflow<2 || pOld->aiOvfl[0]==pOld->aiOvfl[1]-1);
  6836         -      assert(nOverflow<3 || pOld->aiOvfl[1]==pOld->aiOvfl[2]-1);
  6837         -      if( i==iOverflow ){
  6838         -        isDivider = 1;
  6839         -        if( (--nOverflow)>0 ){
  6840         -          iOverflow++;
  6841         -        }
  6842         -      }
  6843         -
  6844         -      if( i==cntNew[k] ){
  6845         -        /* Cell i is the cell immediately following the last cell on new
  6846         -        ** sibling page k. If the siblings are not leaf pages of an
  6847         -        ** intkey b-tree, then cell i is a divider cell.  */
  6848         -        pNew = apNew[++k];
  6849         -        if( !leafData ) continue;
  6850         -      }
  6851         -      assert( j<nOld );
  6852         -      assert( k<nNew );
  6853         -
  6854         -      /* If the cell was originally divider cell (and is not now) or
  6855         -      ** an overflow cell, or if the cell was located on a different sibling
  6856         -      ** page before the balancing, then the pointer map entries associated
  6857         -      ** with any child or overflow pages need to be updated.  */
  6858         -      if( isDivider || pOld->pgno!=pNew->pgno ){
  6859         -        if( !leafCorrection ){
  6860         -          ptrmapPut(pBt, get4byte(apCell[i]), PTRMAP_BTREE, pNew->pgno, &rc);
  6861         -        }
  6862         -        if( szCell[i]>pNew->minLocal ){
  6863         -          ptrmapPutOvflPtr(pNew, apCell[i], &rc);
  6864         -        }
  6865         -      }
  6866         -    }
  6867         -
  6868         -    if( !leafCorrection ){
  6869         -      for(i=0; i<nNew; i++){
  6870         -        u32 key = get4byte(&apNew[i]->aData[8]);
  6871         -        ptrmapPut(pBt, key, PTRMAP_BTREE, apNew[i]->pgno, &rc);
  6872         -      }
  6873         -    }
         7114  +    rc = defragmentPage(apNew[0]);
         7115  +    if( rc==SQLITE_OK ){
         7116  +      assert( apNew[0]->nFree == 
         7117  +          (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2)
         7118  +      );
         7119  +      copyNodeContent(apNew[0], pParent, &rc);
         7120  +      freePage(apNew[0], &rc);
         7121  +    }
         7122  +  }else if( ISAUTOVACUUM && !leafCorrection ){
         7123  +    /* Fix the pointer map entries associated with the right-child of each
         7124  +    ** sibling page. All other pointer map entries have already been taken
         7125  +    ** care of.  */
         7126  +    for(i=0; i<nNew; i++){
         7127  +      u32 key = get4byte(&apNew[i]->aData[8]);
         7128  +      ptrmapPut(pBt, key, PTRMAP_BTREE, apNew[i]->pgno, &rc);
         7129  +    }
         7130  +  }
         7131  +
         7132  +  assert( pParent->isInit );
         7133  +  TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n",
         7134  +          nOld, nNew, nCell));
         7135  +
         7136  +  /* Free any old pages that were not reused as new pages.
         7137  +  */
         7138  +  for(i=nNew; i<nOld; i++){
         7139  +    freePage(apOld[i], &rc);
         7140  +  }
  6874   7141   
  6875   7142   #if 0
         7143  +  if( ISAUTOVACUUM && rc==SQLITE_OK && apNew[0]->isInit ){
  6876   7144       /* The ptrmapCheckPages() contains assert() statements that verify that
  6877   7145       ** all pointer map pages are set correctly. This is helpful while 
  6878   7146       ** debugging. This is usually disabled because a corrupt database may
  6879   7147       ** cause an assert() statement to fail.  */
  6880   7148       ptrmapCheckPages(apNew, nNew);
  6881   7149       ptrmapCheckPages(&pParent, 1);
         7150  +  }
  6882   7151   #endif
  6883         -  }
  6884         -
  6885         -  assert( pParent->isInit );
  6886         -  TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n",
  6887         -          nOld, nNew, nCell));
  6888   7152   
  6889   7153     /*
  6890   7154     ** Cleanup before returning.
  6891   7155     */
  6892   7156   balance_cleanup:
  6893   7157     sqlite3ScratchFree(apCell);
  6894   7158     for(i=0; i<nOld; i++){

Changes to src/build.c.

   303    303   ** auxiliary databases added using the ATTACH command.
   304    304   **
   305    305   ** See also sqlite3LocateTable().
   306    306   */
   307    307   Table *sqlite3FindTable(sqlite3 *db, const char *zName, const char *zDatabase){
   308    308     Table *p = 0;
   309    309     int i;
   310         -  assert( zName!=0 );
          310  +
          311  +#ifdef SQLITE_ENABLE_API_ARMOR
          312  +  if( !sqlite3SafetyCheckOk(db) || zName==0 ) return 0;
          313  +#endif
          314  +
   311    315     /* All mutexes are required for schema access.  Make sure we hold them. */
   312    316     assert( zDatabase!=0 || sqlite3BtreeHoldsAllMutexes(db) );
   313    317   #if SQLITE_USER_AUTHENTICATION
   314    318     /* Only the admin user is allowed to know that the sqlite_user table
   315    319     ** exists */
   316    320     if( db->auth.authLevel<UAUTH_Admin && sqlite3UserAuthTable(zName)!=0 ){
   317    321       return 0;

Changes to src/complete.c.

   100    100   ** If we compile with SQLITE_OMIT_TRIGGER, all of the computation needed
   101    101   ** to recognize the end of a trigger can be omitted.  All we have to do
   102    102   ** is look for a semicolon that is not part of an string or comment.
   103    103   */
   104    104   int sqlite3_complete(const char *zSql){
   105    105     u8 state = 0;   /* Current state, using numbers defined in header comment */
   106    106     u8 token;       /* Value of the next token */
          107  +
          108  +#ifdef SQLITE_ENABLE_API_ARMOR
          109  +  if( zSql==0 ){
          110  +    (void)SQLITE_MISUSE_BKPT;
          111  +    return 0;
          112  +  }
          113  +#endif
   107    114   
   108    115   #ifndef SQLITE_OMIT_TRIGGER
   109    116     /* A complex statement machine used to detect the end of a CREATE TRIGGER
   110    117     ** statement.  This is the normal case.
   111    118     */
   112    119     static const u8 trans[8][8] = {
   113    120                        /* Token:                                                */

Changes to src/ctime.c.

    58     58     "DEFAULT_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_MMAP_SIZE),
    59     59   #endif
    60     60   #ifdef SQLITE_DISABLE_DIRSYNC
    61     61     "DISABLE_DIRSYNC",
    62     62   #endif
    63     63   #ifdef SQLITE_DISABLE_LFS
    64     64     "DISABLE_LFS",
           65  +#endif
           66  +#ifdef SQLITE_ENABLE_API_ARMOR
           67  +  "ENABLE_API_ARMOR",
    65     68   #endif
    66     69   #ifdef SQLITE_ENABLE_ATOMIC_WRITE
    67     70     "ENABLE_ATOMIC_WRITE",
    68     71   #endif
    69     72   #ifdef SQLITE_ENABLE_CEROD
    70     73     "ENABLE_CEROD",
    71     74   #endif
................................................................................
   384    387   ** was used and false if not.
   385    388   **
   386    389   ** The name can optionally begin with "SQLITE_" but the "SQLITE_" prefix
   387    390   ** is not required for a match.
   388    391   */
   389    392   int sqlite3_compileoption_used(const char *zOptName){
   390    393     int i, n;
          394  +
          395  +#ifdef SQLITE_ENABLE_API_ARMORE
          396  +  if( zOptName==0 ){
          397  +    (void)SQLITE_MISUSE_BKPT;
          398  +    return 0;
          399  +  }
          400  +#endif
   391    401     if( sqlite3StrNICmp(zOptName, "SQLITE_", 7)==0 ) zOptName += 7;
   392    402     n = sqlite3Strlen30(zOptName);
   393    403   
   394    404     /* Since ArraySize(azCompileOpt) is normally in single digits, a
   395    405     ** linear search is adequate.  No need for a binary search. */
   396    406     for(i=0; i<ArraySize(azCompileOpt); i++){
   397    407       if( sqlite3StrNICmp(zOptName, azCompileOpt[i], n)==0

Changes to src/expr.c.

  1206   1206       sqlite3DbFree(db, pItem->zSpan);
  1207   1207     }
  1208   1208     sqlite3DbFree(db, pList->a);
  1209   1209     sqlite3DbFree(db, pList);
  1210   1210   }
  1211   1211   
  1212   1212   /*
  1213         -** These routines are Walker callbacks.  Walker.u.pi is a pointer
  1214         -** to an integer.  These routines are checking an expression to see
  1215         -** if it is a constant.  Set *Walker.u.i to 0 if the expression is
  1216         -** not constant.
         1213  +** These routines are Walker callbacks used to check expressions to
         1214  +** see if they are "constant" for some definition of constant.  The
         1215  +** Walker.eCode value determines the type of "constant" we are looking
         1216  +** for.
  1217   1217   **
  1218   1218   ** These callback routines are used to implement the following:
  1219   1219   **
  1220         -**     sqlite3ExprIsConstant()                  pWalker->u.i==1
  1221         -**     sqlite3ExprIsConstantNotJoin()           pWalker->u.i==2
  1222         -**     sqlite3ExprIsConstantOrFunction()        pWalker->u.i==3 or 4
         1220  +**     sqlite3ExprIsConstant()                  pWalker->eCode==1
         1221  +**     sqlite3ExprIsConstantNotJoin()           pWalker->eCode==2
         1222  +**     sqlite3ExprRefOneTableOnly()             pWalker->eCode==3
         1223  +**     sqlite3ExprIsConstantOrFunction()        pWalker->eCode==4 or 5
         1224  +**
         1225  +** In all cases, the callbacks set Walker.eCode=0 and abort if the expression
         1226  +** is found to not be a constant.
  1223   1227   **
  1224   1228   ** The sqlite3ExprIsConstantOrFunction() is used for evaluating expressions
  1225         -** in a CREATE TABLE statement.  The Walker.u.i value is 4 when parsing
  1226         -** an existing schema and 3 when processing a new statement.  A bound
         1229  +** in a CREATE TABLE statement.  The Walker.eCode value is 5 when parsing
         1230  +** an existing schema and 4 when processing a new statement.  A bound
  1227   1231   ** parameter raises an error for new statements, but is silently converted
  1228   1232   ** to NULL for existing schemas.  This allows sqlite_master tables that 
  1229   1233   ** contain a bound parameter because they were generated by older versions
  1230   1234   ** of SQLite to be parsed by newer versions of SQLite without raising a
  1231   1235   ** malformed schema error.
  1232   1236   */
  1233   1237   static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){
  1234   1238   
  1235         -  /* If pWalker->u.i is 2 then any term of the expression that comes from
  1236         -  ** the ON or USING clauses of a join disqualifies the expression
         1239  +  /* If pWalker->eCode is 2 then any term of the expression that comes from
         1240  +  ** the ON or USING clauses of a left join disqualifies the expression
  1237   1241     ** from being considered constant. */
  1238         -  if( pWalker->u.i==2 && ExprHasProperty(pExpr, EP_FromJoin) ){
  1239         -    pWalker->u.i = 0;
         1242  +  if( pWalker->eCode==2 && ExprHasProperty(pExpr, EP_FromJoin) ){
         1243  +    pWalker->eCode = 0;
  1240   1244       return WRC_Abort;
  1241   1245     }
  1242   1246   
  1243   1247     switch( pExpr->op ){
  1244   1248       /* Consider functions to be constant if all their arguments are constant
  1245         -    ** and either pWalker->u.i==3 or 4 or the function as the SQLITE_FUNC_CONST
  1246         -    ** flag. */
         1249  +    ** and either pWalker->eCode==4 or 5 or the function has the
         1250  +    ** SQLITE_FUNC_CONST flag. */
  1247   1251       case TK_FUNCTION:
  1248         -      if( pWalker->u.i>=3 || ExprHasProperty(pExpr,EP_Constant) ){
         1252  +      if( pWalker->eCode>=4 || ExprHasProperty(pExpr,EP_Constant) ){
  1249   1253           return WRC_Continue;
         1254  +      }else{
         1255  +        pWalker->eCode = 0;
         1256  +        return WRC_Abort;
  1250   1257         }
  1251         -      /* Fall through */
  1252   1258       case TK_ID:
  1253   1259       case TK_COLUMN:
  1254   1260       case TK_AGG_FUNCTION:
  1255   1261       case TK_AGG_COLUMN:
  1256   1262         testcase( pExpr->op==TK_ID );
  1257   1263         testcase( pExpr->op==TK_COLUMN );
  1258   1264         testcase( pExpr->op==TK_AGG_FUNCTION );
  1259   1265         testcase( pExpr->op==TK_AGG_COLUMN );
  1260         -      pWalker->u.i = 0;
  1261         -      return WRC_Abort;
         1266  +      if( pWalker->eCode==3 && pExpr->iTable==pWalker->u.iCur ){
         1267  +        return WRC_Continue;
         1268  +      }else{
         1269  +        pWalker->eCode = 0;
         1270  +        return WRC_Abort;
         1271  +      }
  1262   1272       case TK_VARIABLE:
  1263         -      if( pWalker->u.i==4 ){
         1273  +      if( pWalker->eCode==5 ){
  1264   1274           /* Silently convert bound parameters that appear inside of CREATE
  1265   1275           ** statements into a NULL when parsing the CREATE statement text out
  1266   1276           ** of the sqlite_master table */
  1267   1277           pExpr->op = TK_NULL;
  1268         -      }else if( pWalker->u.i==3 ){
         1278  +      }else if( pWalker->eCode==4 ){
  1269   1279           /* A bound parameter in a CREATE statement that originates from
  1270   1280           ** sqlite3_prepare() causes an error */
  1271         -        pWalker->u.i = 0;
         1281  +        pWalker->eCode = 0;
  1272   1282           return WRC_Abort;
  1273   1283         }
  1274   1284         /* Fall through */
  1275   1285       default:
  1276   1286         testcase( pExpr->op==TK_SELECT ); /* selectNodeIsConstant will disallow */
  1277   1287         testcase( pExpr->op==TK_EXISTS ); /* selectNodeIsConstant will disallow */
  1278   1288         return WRC_Continue;
  1279   1289     }
  1280   1290   }
  1281   1291   static int selectNodeIsConstant(Walker *pWalker, Select *NotUsed){
  1282   1292     UNUSED_PARAMETER(NotUsed);
  1283         -  pWalker->u.i = 0;
         1293  +  pWalker->eCode = 0;
  1284   1294     return WRC_Abort;
  1285   1295   }
  1286         -static int exprIsConst(Expr *p, int initFlag){
         1296  +static int exprIsConst(Expr *p, int initFlag, int iCur){
  1287   1297     Walker w;
  1288   1298     memset(&w, 0, sizeof(w));
  1289         -  w.u.i = initFlag;
         1299  +  w.eCode = initFlag;
  1290   1300     w.xExprCallback = exprNodeIsConstant;
  1291   1301     w.xSelectCallback = selectNodeIsConstant;
         1302  +  w.u.iCur = iCur;
  1292   1303     sqlite3WalkExpr(&w, p);
  1293         -  return w.u.i;
         1304  +  return w.eCode;
  1294   1305   }
  1295   1306   
  1296   1307   /*
  1297         -** Walk an expression tree.  Return 1 if the expression is constant
         1308  +** Walk an expression tree.  Return non-zero if the expression is constant
  1298   1309   ** and 0 if it involves variables or function calls.
  1299   1310   **
  1300   1311   ** For the purposes of this function, a double-quoted string (ex: "abc")
  1301   1312   ** is considered a variable but a single-quoted string (ex: 'abc') is
  1302   1313   ** a constant.
  1303   1314   */
  1304   1315   int sqlite3ExprIsConstant(Expr *p){
  1305         -  return exprIsConst(p, 1);
         1316  +  return exprIsConst(p, 1, 0);
  1306   1317   }
  1307   1318   
  1308   1319   /*
  1309         -** Walk an expression tree.  Return 1 if the expression is constant
         1320  +** Walk an expression tree.  Return non-zero if the expression is constant
  1310   1321   ** that does no originate from the ON or USING clauses of a join.
  1311   1322   ** Return 0 if it involves variables or function calls or terms from
  1312   1323   ** an ON or USING clause.
  1313   1324   */
  1314   1325   int sqlite3ExprIsConstantNotJoin(Expr *p){
  1315         -  return exprIsConst(p, 2);
         1326  +  return exprIsConst(p, 2, 0);
  1316   1327   }
  1317   1328   
  1318   1329   /*
  1319         -** Walk an expression tree.  Return 1 if the expression is constant
         1330  +** Walk an expression tree.  Return non-zero if the expression constant
         1331  +** for any single row of the table with cursor iCur.  In other words, the
         1332  +** expression must not refer to any non-deterministic function nor any
         1333  +** table other than iCur.
         1334  +*/
         1335  +int sqlite3ExprIsTableConstant(Expr *p, int iCur){
         1336  +  return exprIsConst(p, 3, iCur);
         1337  +}
         1338  +
         1339  +/*
         1340  +** Walk an expression tree.  Return non-zero if the expression is constant
  1320   1341   ** or a function call with constant arguments.  Return and 0 if there
  1321   1342   ** are any variables.
  1322   1343   **
  1323   1344   ** For the purposes of this function, a double-quoted string (ex: "abc")
  1324   1345   ** is considered a variable but a single-quoted string (ex: 'abc') is
  1325   1346   ** a constant.
  1326   1347   */
  1327   1348   int sqlite3ExprIsConstantOrFunction(Expr *p, u8 isInit){
  1328   1349     assert( isInit==0 || isInit==1 );
  1329         -  return exprIsConst(p, 3+isInit);
         1350  +  return exprIsConst(p, 4+isInit, 0);
  1330   1351   }
  1331   1352   
  1332   1353   /*
  1333   1354   ** If the expression p codes a constant integer that is small enough
  1334   1355   ** to fit in a 32-bit integer, return 1 and put the value of the integer
  1335   1356   ** in *pValue.  If the expression is not an integer or if it is too big
  1336   1357   ** to fit in a signed 32-bit integer, return 0 and leave *pValue unchanged.

Changes to src/main.c.

   597    597     return SQLITE_OK;
   598    598   }
   599    599   
   600    600   /*
   601    601   ** Return the mutex associated with a database connection.
   602    602   */
   603    603   sqlite3_mutex *sqlite3_db_mutex(sqlite3 *db){
          604  +#ifdef SQLITE_ENABLE_API_ARMOR
          605  +  if( !sqlite3SafetyCheckOk(db) ){
          606  +    (void)SQLITE_MISUSE_BKPT;
          607  +    return 0;
          608  +  }
          609  +#endif
   604    610     return db->mutex;
   605    611   }
   606    612   
   607    613   /*
   608    614   ** Free up as much memory as we can from the given database
   609    615   ** connection.
   610    616   */
   611    617   int sqlite3_db_release_memory(sqlite3 *db){
   612    618     int i;
          619  +
          620  +#ifdef SQLITE_ENABLE_API_ARMOR
          621  +  if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT;
          622  +#endif
   613    623     sqlite3_mutex_enter(db->mutex);
   614    624     sqlite3BtreeEnterAll(db);
   615    625     for(i=0; i<db->nDb; i++){
   616    626       Btree *pBt = db->aDb[i].pBt;
   617    627       if( pBt ){
   618    628         Pager *pPager = sqlite3BtreePager(pBt);
   619    629         sqlite3PagerShrink(pPager);
................................................................................
   736    746     return r;
   737    747   }
   738    748   
   739    749   /*
   740    750   ** Return the ROWID of the most recent insert
   741    751   */
   742    752   sqlite_int64 sqlite3_last_insert_rowid(sqlite3 *db){
          753  +#ifdef SQLITE_ENABLE_API_ARMOR
          754  +  if( !sqlite3SafetyCheckOk(db) ){
          755  +    (void)SQLITE_MISUSE_BKPT;
          756  +    return 0;
          757  +  }
          758  +#endif
   743    759     return db->lastRowid;
   744    760   }
   745    761   
   746    762   /*
   747    763   ** Return the number of changes in the most recent call to sqlite3_exec().
   748    764   */
   749    765   int sqlite3_changes(sqlite3 *db){
          766  +#ifdef SQLITE_ENABLE_API_ARMOR
          767  +  if( !sqlite3SafetyCheckOk(db) ){
          768  +    (void)SQLITE_MISUSE_BKPT;
          769  +    return 0;
          770  +  }
          771  +#endif
   750    772     return db->nChange;
   751    773   }
   752    774   
   753    775   /*
   754    776   ** Return the number of changes since the database handle was opened.
   755    777   */
   756    778   int sqlite3_total_changes(sqlite3 *db){
          779  +#ifdef SQLITE_ENABLE_API_ARMOR
          780  +  if( !sqlite3SafetyCheckOk(db) ){
          781  +    (void)SQLITE_MISUSE_BKPT;
          782  +    return 0;
          783  +  }
          784  +#endif
   757    785     return db->nTotalChange;
   758    786   }
   759    787   
   760    788   /*
   761    789   ** Close all open savepoints. This function only manipulates fields of the
   762    790   ** database handle object, it does not close any savepoints that may be open
   763    791   ** at the b-tree/pager level.
................................................................................
  1299   1327   ** given callback function with the given argument.
  1300   1328   */
  1301   1329   int sqlite3_busy_handler(
  1302   1330     sqlite3 *db,
  1303   1331     int (*xBusy)(void*,int),
  1304   1332     void *pArg
  1305   1333   ){
         1334  +#ifdef SQLITE_ENABLE_API_ARMOR
         1335  +  if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE;
         1336  +#endif
  1306   1337     sqlite3_mutex_enter(db->mutex);
  1307   1338     db->busyHandler.xFunc = xBusy;
  1308   1339     db->busyHandler.pArg = pArg;
  1309   1340     db->busyHandler.nBusy = 0;
  1310   1341     db->busyTimeout = 0;
  1311   1342     sqlite3_mutex_leave(db->mutex);
  1312   1343     return SQLITE_OK;
................................................................................
  1320   1351   */
  1321   1352   void sqlite3_progress_handler(
  1322   1353     sqlite3 *db, 
  1323   1354     int nOps,
  1324   1355     int (*xProgress)(void*), 
  1325   1356     void *pArg
  1326   1357   ){
         1358  +#ifdef SQLITE_ENABLE_API_ARMOR
         1359  +  if( !sqlite3SafetyCheckOk(db) ){
         1360  +    (void)SQLITE_MISUSE_BKPT;
         1361  +    return;
         1362  +  }
         1363  +#endif
  1327   1364     sqlite3_mutex_enter(db->mutex);
  1328   1365     if( nOps>0 ){
  1329   1366       db->xProgress = xProgress;
  1330   1367       db->nProgressOps = (unsigned)nOps;
  1331   1368       db->pProgressArg = pArg;
  1332   1369     }else{
  1333   1370       db->xProgress = 0;
................................................................................
  1340   1377   
  1341   1378   
  1342   1379   /*
  1343   1380   ** This routine installs a default busy handler that waits for the
  1344   1381   ** specified number of milliseconds before returning 0.
  1345   1382   */
  1346   1383   int sqlite3_busy_timeout(sqlite3 *db, int ms){
         1384  +#ifdef SQLITE_ENABLE_API_ARMOR
         1385  +  if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT;
         1386  +#endif
  1347   1387     if( ms>0 ){
  1348   1388       sqlite3_busy_handler(db, sqliteDefaultBusyCallback, (void*)db);
  1349   1389       db->busyTimeout = ms;
  1350   1390     }else{
  1351   1391       sqlite3_busy_handler(db, 0, 0);
  1352   1392     }
  1353   1393     return SQLITE_OK;
  1354   1394   }
  1355   1395   
  1356   1396   /*
  1357   1397   ** Cause any pending operation to stop at its earliest opportunity.
  1358   1398   */
  1359   1399   void sqlite3_interrupt(sqlite3 *db){
         1400  +#ifdef SQLITE_ENABLE_API_ARMOR
         1401  +  if( !sqlite3SafetyCheckOk(db) ){
         1402  +    (void)SQLITE_MISUSE_BKPT;
         1403  +    return;
         1404  +  }
         1405  +#endif
  1360   1406     db->u1.isInterrupted = 1;
  1361   1407   }
  1362   1408   
  1363   1409   
  1364   1410   /*
  1365   1411   ** This function is exactly the same as sqlite3_create_function(), except
  1366   1412   ** that it is designed to be called by internal code. The difference is
................................................................................
  1490   1536     void (*xFunc)(sqlite3_context*,int,sqlite3_value **),
  1491   1537     void (*xStep)(sqlite3_context*,int,sqlite3_value **),
  1492   1538     void (*xFinal)(sqlite3_context*),
  1493   1539     void (*xDestroy)(void *)
  1494   1540   ){
  1495   1541     int rc = SQLITE_ERROR;
  1496   1542     FuncDestructor *pArg = 0;
         1543  +
         1544  +#ifdef SQLITE_ENABLE_API_ARMOR
         1545  +  if( !sqlite3SafetyCheckOk(db) ){
         1546  +    return SQLITE_MISUSE_BKPT;
         1547  +  }
         1548  +#endif
  1497   1549     sqlite3_mutex_enter(db->mutex);
  1498   1550     if( xDestroy ){
  1499   1551       pArg = (FuncDestructor *)sqlite3DbMallocZero(db, sizeof(FuncDestructor));
  1500   1552       if( !pArg ){
  1501   1553         xDestroy(p);
  1502   1554         goto out;
  1503   1555       }
................................................................................
  1526   1578     void *p,
  1527   1579     void (*xFunc)(sqlite3_context*,int,sqlite3_value**),
  1528   1580     void (*xStep)(sqlite3_context*,int,sqlite3_value**),
  1529   1581     void (*xFinal)(sqlite3_context*)
  1530   1582   ){
  1531   1583     int rc;
  1532   1584     char *zFunc8;
         1585  +
         1586  +#ifdef SQLITE_ENABLE_API_ARMOR
         1587  +  if( !sqlite3SafetyCheckOk(db) || zFunctionName==0 ) return SQLITE_MISUSE_BKPT;
         1588  +#endif
  1533   1589     sqlite3_mutex_enter(db->mutex);
  1534   1590     assert( !db->mallocFailed );
  1535   1591     zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1, SQLITE_UTF16NATIVE);
  1536   1592     rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xFunc, xStep, xFinal,0);
  1537   1593     sqlite3DbFree(db, zFunc8);
  1538   1594     rc = sqlite3ApiExit(db, rc);
  1539   1595     sqlite3_mutex_leave(db->mutex);
................................................................................
  1557   1613   int sqlite3_overload_function(
  1558   1614     sqlite3 *db,
  1559   1615     const char *zName,
  1560   1616     int nArg
  1561   1617   ){
  1562   1618     int nName = sqlite3Strlen30(zName);
  1563   1619     int rc = SQLITE_OK;
         1620  +
         1621  +#ifdef SQLITE_ENABLE_API_ARMOR
         1622  +  if( !sqlite3SafetyCheckOk(db) || zName==0 || nArg<-2 ){
         1623  +    return SQLITE_MISUSE_BKPT;
         1624  +  }
         1625  +#endif
  1564   1626     sqlite3_mutex_enter(db->mutex);
  1565   1627     if( sqlite3FindFunction(db, zName, nName, nArg, SQLITE_UTF8, 0)==0 ){
  1566   1628       rc = sqlite3CreateFunc(db, zName, nArg, SQLITE_UTF8,
  1567   1629                              0, sqlite3InvalidFunction, 0, 0, 0);
  1568   1630     }
  1569   1631     rc = sqlite3ApiExit(db, rc);
  1570   1632     sqlite3_mutex_leave(db->mutex);
................................................................................
  1578   1640   **
  1579   1641   ** A NULL trace function means that no tracing is executes.  A non-NULL
  1580   1642   ** trace is a pointer to a function that is invoked at the start of each
  1581   1643   ** SQL statement.
  1582   1644   */
  1583   1645   void *sqlite3_trace(sqlite3 *db, void (*xTrace)(void*,const char*), void *pArg){
  1584   1646     void *pOld;
         1647  +
         1648  +#ifdef SQLITE_ENABLE_API_ARMOR
         1649  +  if( !sqlite3SafetyCheckOk(db) ){
         1650  +    (void)SQLITE_MISUSE_BKPT;
         1651  +    return 0;
         1652  +  }
         1653  +#endif
  1585   1654     sqlite3_mutex_enter(db->mutex);
  1586   1655     pOld = db->pTraceArg;
  1587   1656     db->xTrace = xTrace;
  1588   1657     db->pTraceArg = pArg;
  1589   1658     sqlite3_mutex_leave(db->mutex);
  1590   1659     return pOld;
  1591   1660   }
................................................................................
  1599   1668   */
  1600   1669   void *sqlite3_profile(
  1601   1670     sqlite3 *db,
  1602   1671     void (*xProfile)(void*,const char*,sqlite_uint64),
  1603   1672     void *pArg
  1604   1673   ){
  1605   1674     void *pOld;
         1675  +
         1676  +#ifdef SQLITE_ENABLE_API_ARMOR
         1677  +  if( !sqlite3SafetyCheckOk(db) ){
         1678  +    (void)SQLITE_MISUSE_BKPT;
         1679  +    return 0;
         1680  +  }
         1681  +#endif
  1606   1682     sqlite3_mutex_enter(db->mutex);
  1607   1683     pOld = db->pProfileArg;
  1608   1684     db->xProfile = xProfile;
  1609   1685     db->pProfileArg = pArg;
  1610   1686     sqlite3_mutex_leave(db->mutex);
  1611   1687     return pOld;
  1612   1688   }
................................................................................
  1619   1695   */
  1620   1696   void *sqlite3_commit_hook(
  1621   1697     sqlite3 *db,              /* Attach the hook to this database */
  1622   1698     int (*xCallback)(void*),  /* Function to invoke on each commit */
  1623   1699     void *pArg                /* Argument to the function */
  1624   1700   ){
  1625   1701     void *pOld;
         1702  +
         1703  +#ifdef SQLITE_ENABLE_API_ARMOR
         1704  +  if( !sqlite3SafetyCheckOk(db) ){
         1705  +    (void)SQLITE_MISUSE_BKPT;
         1706  +    return 0;
         1707  +  }
         1708  +#endif
  1626   1709     sqlite3_mutex_enter(db->mutex);
  1627   1710     pOld = db->pCommitArg;
  1628   1711     db->xCommitCallback = xCallback;
  1629   1712     db->pCommitArg = pArg;
  1630   1713     sqlite3_mutex_leave(db->mutex);
  1631   1714     return pOld;
  1632   1715   }
................................................................................
  1637   1720   */
  1638   1721   void *sqlite3_update_hook(
  1639   1722     sqlite3 *db,              /* Attach the hook to this database */
  1640   1723     void (*xCallback)(void*,int,char const *,char const *,sqlite_int64),
  1641   1724     void *pArg                /* Argument to the function */
  1642   1725   ){
  1643   1726     void *pRet;
         1727  +
         1728  +#ifdef SQLITE_ENABLE_API_ARMOR
         1729  +  if( !sqlite3SafetyCheckOk(db) ){
         1730  +    (void)SQLITE_MISUSE_BKPT;
         1731  +    return 0;
         1732  +  }
         1733  +#endif
  1644   1734     sqlite3_mutex_enter(db->mutex);
  1645   1735     pRet = db->pUpdateArg;
  1646   1736     db->xUpdateCallback = xCallback;
  1647   1737     db->pUpdateArg = pArg;
  1648   1738     sqlite3_mutex_leave(db->mutex);
  1649   1739     return pRet;
  1650   1740   }
................................................................................
  1655   1745   */
  1656   1746   void *sqlite3_rollback_hook(
  1657   1747     sqlite3 *db,              /* Attach the hook to this database */
  1658   1748     void (*xCallback)(void*), /* Callback function */
  1659   1749     void *pArg                /* Argument to the function */
  1660   1750   ){
  1661   1751     void *pRet;
         1752  +
         1753  +#ifdef SQLITE_ENABLE_API_ARMOR
         1754  +  if( !sqlite3SafetyCheckOk(db) ){
         1755  +    (void)SQLITE_MISUSE_BKPT;
         1756  +    return 0;
         1757  +  }
         1758  +#endif
  1662   1759     sqlite3_mutex_enter(db->mutex);
  1663   1760     pRet = db->pRollbackArg;
  1664   1761     db->xRollbackCallback = xCallback;
  1665   1762     db->pRollbackArg = pArg;
  1666   1763     sqlite3_mutex_leave(db->mutex);
  1667   1764     return pRet;
  1668   1765   }
................................................................................
  1701   1798   ** configured by this function.
  1702   1799   */
  1703   1800   int sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame){
  1704   1801   #ifdef SQLITE_OMIT_WAL
  1705   1802     UNUSED_PARAMETER(db);
  1706   1803     UNUSED_PARAMETER(nFrame);
  1707   1804   #else
         1805  +#ifdef SQLITE_ENABLE_API_ARMOR
         1806  +  if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT;
         1807  +#endif
  1708   1808     if( nFrame>0 ){
  1709   1809       sqlite3_wal_hook(db, sqlite3WalDefaultHook, SQLITE_INT_TO_PTR(nFrame));
  1710   1810     }else{
  1711   1811       sqlite3_wal_hook(db, 0, 0);
  1712   1812     }
  1713   1813   #endif
  1714   1814     return SQLITE_OK;
................................................................................
  1721   1821   void *sqlite3_wal_hook(
  1722   1822     sqlite3 *db,                    /* Attach the hook to this db handle */
  1723   1823     int(*xCallback)(void *, sqlite3*, const char*, int),
  1724   1824     void *pArg                      /* First argument passed to xCallback() */
  1725   1825   ){
  1726   1826   #ifndef SQLITE_OMIT_WAL
  1727   1827     void *pRet;
         1828  +#ifdef SQLITE_ENABLE_API_ARMOR
         1829  +  if( !sqlite3SafetyCheckOk(db) ){
         1830  +    (void)SQLITE_MISUSE_BKPT;
         1831  +    return 0;
         1832  +  }
         1833  +#endif
  1728   1834     sqlite3_mutex_enter(db->mutex);
  1729   1835     pRet = db->pWalArg;
  1730   1836     db->xWalCallback = xCallback;
  1731   1837     db->pWalArg = pArg;
  1732   1838     sqlite3_mutex_leave(db->mutex);
  1733   1839     return pRet;
  1734   1840   #else
................................................................................
  1747   1853     int *pnCkpt                     /* OUT: Total number of frames checkpointed */
  1748   1854   ){
  1749   1855   #ifdef SQLITE_OMIT_WAL
  1750   1856     return SQLITE_OK;
  1751   1857   #else
  1752   1858     int rc;                         /* Return code */
  1753   1859     int iDb = SQLITE_MAX_ATTACHED;  /* sqlite3.aDb[] index of db to checkpoint */
         1860  +
         1861  +#ifdef SQLITE_ENABLE_API_ARMOR
         1862  +  if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT;
         1863  +#endif
  1754   1864   
  1755   1865     /* Initialize the output variables to -1 in case an error occurs. */
  1756   1866     if( pnLog ) *pnLog = -1;
  1757   1867     if( pnCkpt ) *pnCkpt = -1;
  1758   1868   
  1759   1869     assert( SQLITE_CHECKPOINT_FULL>SQLITE_CHECKPOINT_PASSIVE );
  1760   1870     assert( SQLITE_CHECKPOINT_FULL<SQLITE_CHECKPOINT_RESTART );
................................................................................
  2144   2254   ** A new lower limit does not shrink existing constructs.
  2145   2255   ** It merely prevents new constructs that exceed the limit
  2146   2256   ** from forming.
  2147   2257   */
  2148   2258   int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){
  2149   2259     int oldLimit;
  2150   2260   
         2261  +#ifdef SQLITE_ENABLE_API_ARMOR
         2262  +  if( !sqlite3SafetyCheckOk(db) ){
         2263  +    (void)SQLITE_MISUSE_BKPT;
         2264  +    return -1;
         2265  +  }
         2266  +#endif
  2151   2267   
  2152   2268     /* EVIDENCE-OF: R-30189-54097 For each limit category SQLITE_LIMIT_NAME
  2153   2269     ** there is a hard upper bound set at compile-time by a C preprocessor
  2154   2270     ** macro called SQLITE_MAX_NAME. (The "_LIMIT_" in the name is changed to
  2155   2271     ** "_MAX_".)
  2156   2272     */
  2157   2273     assert( aHardLimit[SQLITE_LIMIT_LENGTH]==SQLITE_MAX_LENGTH );
................................................................................
  2518   2634   ){
  2519   2635     sqlite3 *db;                    /* Store allocated handle here */
  2520   2636     int rc;                         /* Return code */
  2521   2637     int isThreadsafe;               /* True for threadsafe connections */
  2522   2638     char *zOpen = 0;                /* Filename argument to pass to BtreeOpen() */
  2523   2639     char *zErrMsg = 0;              /* Error message from sqlite3ParseUri() */
  2524   2640   
         2641  +#ifdef SQLITE_ENABLE_API_ARMOR
         2642  +  if( ppDb==0 ) return SQLITE_MISUSE_BKPT;
         2643  +#endif
  2525   2644     *ppDb = 0;
  2526   2645   #ifndef SQLITE_OMIT_AUTOINIT
  2527   2646     rc = sqlite3_initialize();
  2528   2647     if( rc ) return rc;
  2529   2648   #endif
  2530   2649   
  2531   2650     /* Only allow sensible combinations of bits in the flags argument.  
................................................................................
  2833   2952     const void *zFilename, 
  2834   2953     sqlite3 **ppDb
  2835   2954   ){
  2836   2955     char const *zFilename8;   /* zFilename encoded in UTF-8 instead of UTF-16 */
  2837   2956     sqlite3_value *pVal;
  2838   2957     int rc;
  2839   2958   
  2840         -  assert( zFilename );
  2841         -  assert( ppDb );
         2959  +#ifdef SQLITE_ENABLE_API_ARMOR
         2960  +  if( ppDb==0 ) return SQLITE_MISUSE_BKPT;
         2961  +#endif
  2842   2962     *ppDb = 0;
  2843   2963   #ifndef SQLITE_OMIT_AUTOINIT
  2844   2964     rc = sqlite3_initialize();
  2845   2965     if( rc ) return rc;
  2846   2966   #endif
         2967  +  if( zFilename==0 ) zFilename = "\000\000";
  2847   2968     pVal = sqlite3ValueNew(0);
  2848   2969     sqlite3ValueSetStr(pVal, -1, zFilename, SQLITE_UTF16NATIVE, SQLITE_STATIC);
  2849   2970     zFilename8 = sqlite3ValueText(pVal, SQLITE_UTF8);
  2850   2971     if( zFilename8 ){
  2851   2972       rc = openDatabase(zFilename8, ppDb,
  2852   2973                         SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0);
  2853   2974       assert( *ppDb || rc==SQLITE_NOMEM );
................................................................................
  2869   2990   int sqlite3_create_collation(
  2870   2991     sqlite3* db, 
  2871   2992     const char *zName, 
  2872   2993     int enc, 
  2873   2994     void* pCtx,
  2874   2995     int(*xCompare)(void*,int,const void*,int,const void*)
  2875   2996   ){
  2876         -  int rc;
  2877         -  sqlite3_mutex_enter(db->mutex);
  2878         -  assert( !db->mallocFailed );
  2879         -  rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, 0);
  2880         -  rc = sqlite3ApiExit(db, rc);
  2881         -  sqlite3_mutex_leave(db->mutex);
  2882         -  return rc;
         2997  +  return sqlite3_create_collation_v2(db, zName, enc, pCtx, xCompare, 0);
  2883   2998   }
  2884   2999   
  2885   3000   /*
  2886   3001   ** Register a new collation sequence with the database handle db.
  2887   3002   */
  2888   3003   int sqlite3_create_collation_v2(
  2889   3004     sqlite3* db, 
................................................................................
  2890   3005     const char *zName, 
  2891   3006     int enc, 
  2892   3007     void* pCtx,
  2893   3008     int(*xCompare)(void*,int,const void*,int,const void*),
  2894   3009     void(*xDel)(void*)
  2895   3010   ){
  2896   3011     int rc;
         3012  +
         3013  +#ifdef SQLITE_ENABLE_API_ARMOR
         3014  +  if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT;
         3015  +#endif
  2897   3016     sqlite3_mutex_enter(db->mutex);
  2898   3017     assert( !db->mallocFailed );
  2899   3018     rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, xDel);
  2900   3019     rc = sqlite3ApiExit(db, rc);
  2901   3020     sqlite3_mutex_leave(db->mutex);
  2902   3021     return rc;
  2903   3022   }
................................................................................
  2911   3030     const void *zName,
  2912   3031     int enc, 
  2913   3032     void* pCtx,
  2914   3033     int(*xCompare)(void*,int,const void*,int,const void*)
  2915   3034   ){
  2916   3035     int rc = SQLITE_OK;
  2917   3036     char *zName8;
         3037  +
         3038  +#ifdef SQLITE_ENABLE_API_ARMOR
         3039  +  if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT;
         3040  +#endif
  2918   3041     sqlite3_mutex_enter(db->mutex);
  2919   3042     assert( !db->mallocFailed );
  2920   3043     zName8 = sqlite3Utf16to8(db, zName, -1, SQLITE_UTF16NATIVE);
  2921   3044     if( zName8 ){
  2922   3045       rc = createCollation(db, zName8, (u8)enc, pCtx, xCompare, 0);
  2923   3046       sqlite3DbFree(db, zName8);
  2924   3047     }
................................................................................
  2933   3056   ** db. Replace any previously installed collation sequence factory.
  2934   3057   */
  2935   3058   int sqlite3_collation_needed(
  2936   3059     sqlite3 *db, 
  2937   3060     void *pCollNeededArg, 
  2938   3061     void(*xCollNeeded)(void*,sqlite3*,int eTextRep,const char*)
  2939   3062   ){
         3063  +#ifdef SQLITE_ENABLE_API_ARMOR
         3064  +  if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT;
         3065  +#endif
  2940   3066     sqlite3_mutex_enter(db->mutex);
  2941   3067     db->xCollNeeded = xCollNeeded;
  2942   3068     db->xCollNeeded16 = 0;
  2943   3069     db->pCollNeededArg = pCollNeededArg;
  2944   3070     sqlite3_mutex_leave(db->mutex);
  2945   3071     return SQLITE_OK;
  2946   3072   }
................................................................................
  2951   3077   ** db. Replace any previously installed collation sequence factory.
  2952   3078   */
  2953   3079   int sqlite3_collation_needed16(
  2954   3080     sqlite3 *db, 
  2955   3081     void *pCollNeededArg, 
  2956   3082     void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*)
  2957   3083   ){
         3084  +#ifdef SQLITE_ENABLE_API_ARMOR
         3085  +  if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT;
         3086  +#endif
  2958   3087     sqlite3_mutex_enter(db->mutex);
  2959   3088     db->xCollNeeded = 0;
  2960   3089     db->xCollNeeded16 = xCollNeeded16;
  2961   3090     db->pCollNeededArg = pCollNeededArg;
  2962   3091     sqlite3_mutex_leave(db->mutex);
  2963   3092     return SQLITE_OK;
  2964   3093   }
................................................................................
  2977   3106   /*
  2978   3107   ** Test to see whether or not the database connection is in autocommit
  2979   3108   ** mode.  Return TRUE if it is and FALSE if not.  Autocommit mode is on
  2980   3109   ** by default.  Autocommit is disabled by a BEGIN statement and reenabled
  2981   3110   ** by the next COMMIT or ROLLBACK.
  2982   3111   */
  2983   3112   int sqlite3_get_autocommit(sqlite3 *db){
         3113  +#ifdef SQLITE_ENABLE_API_ARMOR
         3114  +  if( !sqlite3SafetyCheckOk(db) ){
         3115  +    (void)SQLITE_MISUSE_BKPT;
         3116  +    return 0;
         3117  +  }
         3118  +#endif
  2984   3119     return db->autoCommit;
  2985   3120   }
  2986   3121   
  2987   3122   /*
  2988   3123   ** The following routines are substitutes for constants SQLITE_CORRUPT,
  2989   3124   ** SQLITE_MISUSE, SQLITE_CANTOPEN, SQLITE_IOERR and possibly other error
  2990   3125   ** constants.  They serve two purposes:
................................................................................
  3159   3294     return rc;
  3160   3295   }
  3161   3296   
  3162   3297   /*
  3163   3298   ** Enable or disable the extended result codes.
  3164   3299   */
  3165   3300   int sqlite3_extended_result_codes(sqlite3 *db, int onoff){
         3301  +#ifdef SQLITE_ENABLE_API_ARMOR
         3302  +  if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT;
         3303  +#endif
  3166   3304     sqlite3_mutex_enter(db->mutex);
  3167   3305     db->errMask = onoff ? 0xffffffff : 0xff;
  3168   3306     sqlite3_mutex_leave(db->mutex);
  3169   3307     return SQLITE_OK;
  3170   3308   }
  3171   3309   
  3172   3310   /*
  3173   3311   ** Invoke the xFileControl method on a particular database.
  3174   3312   */
  3175   3313   int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){
  3176   3314     int rc = SQLITE_ERROR;
  3177   3315     Btree *pBtree;
  3178   3316   
         3317  +#ifdef SQLITE_ENABLE_API_ARMOR
         3318  +  if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT;
         3319  +#endif
  3179   3320     sqlite3_mutex_enter(db->mutex);
  3180   3321     pBtree = sqlite3DbNameToBtree(db, zDbName);
  3181   3322     if( pBtree ){
  3182   3323       Pager *pPager;
  3183   3324       sqlite3_file *fd;
  3184   3325       sqlite3BtreeEnter(pBtree);
  3185   3326       pPager = sqlite3BtreePager(pBtree);
................................................................................
  3523   3664   ** The zFilename argument is the filename pointer passed into the xOpen()
  3524   3665   ** method of a VFS implementation.  The zParam argument is the name of the
  3525   3666   ** query parameter we seek.  This routine returns the value of the zParam
  3526   3667   ** parameter if it exists.  If the parameter does not exist, this routine
  3527   3668   ** returns a NULL pointer.
  3528   3669   */
  3529   3670   const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam){
  3530         -  if( zFilename==0 ) return 0;
         3671  +  if( zFilename==0 || zParam==0 ) return 0;
  3531   3672     zFilename += sqlite3Strlen30(zFilename) + 1;
  3532   3673     while( zFilename[0] ){
  3533   3674       int x = strcmp(zFilename, zParam);
  3534   3675       zFilename += sqlite3Strlen30(zFilename) + 1;
  3535   3676       if( x==0 ) return zFilename;
  3536   3677       zFilename += sqlite3Strlen30(zFilename) + 1;
  3537   3678     }
................................................................................
  3579   3720   }
  3580   3721   
  3581   3722   /*
  3582   3723   ** Return the filename of the database associated with a database
  3583   3724   ** connection.
  3584   3725   */
  3585   3726   const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName){
         3727  +#ifdef SQLITE_ENABLE_API_ARMOR
         3728  +  if( !sqlite3SafetyCheckOk(db) ){
         3729  +    (void)SQLITE_MISUSE_BKPT;
         3730  +    return 0;
         3731  +  }
         3732  +#endif
  3586   3733     Btree *pBt = sqlite3DbNameToBtree(db, zDbName);
  3587   3734     return pBt ? sqlite3BtreeGetFilename(pBt) : 0;
  3588   3735   }
  3589   3736   
  3590   3737   /*
  3591   3738   ** Return 1 if database is read-only or 0 if read/write.  Return -1 if
  3592   3739   ** no such database exists.
  3593   3740   */
  3594   3741   int sqlite3_db_readonly(sqlite3 *db, const char *zDbName){
         3742  +#ifdef SQLITE_ENABLE_API_ARMOR
         3743  +  if( !sqlite3SafetyCheckOk(db) ){
         3744  +    (void)SQLITE_MISUSE_BKPT;
         3745  +    return -1;
         3746  +  }
         3747  +#endif
  3595   3748     Btree *pBt = sqlite3DbNameToBtree(db, zDbName);
  3596   3749     return pBt ? sqlite3BtreeIsReadonly(pBt) : -1;
  3597   3750   }
  3598   3751   
  3599   3752   #if (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__)
  3600   3753   
  3601   3754   #include "sqlite3_private.h"

Changes to src/mutex_unix.c.

   171    171           p->id = iType;
   172    172   #endif
   173    173           pthread_mutex_init(&p->mutex, 0);
   174    174         }
   175    175         break;
   176    176       }
   177    177       default: {
   178         -      assert( iType-2 >= 0 );
   179         -      assert( iType-2 < ArraySize(staticMutexes) );
          178  +#ifdef SQLITE_ENABLE_API_ARMOR
          179  +      if( iType-2<0 || iType-2>=ArraySize(staticMutexes) ){
          180  +        (void)SQLITE_MISUSE_BKPT;
          181  +        return 0;
          182  +      }
          183  +#endif
   180    184         p = &staticMutexes[iType-2];
   181    185   #if SQLITE_MUTEX_NREF
   182    186         p->id = iType;
   183    187   #endif
   184    188         break;
   185    189       }
   186    190     }

Changes to src/os.c.

   363    363   */
   364    364   int sqlite3_vfs_register(sqlite3_vfs *pVfs, int makeDflt){
   365    365     MUTEX_LOGIC(sqlite3_mutex *mutex;)
   366    366   #ifndef SQLITE_OMIT_AUTOINIT
   367    367     int rc = sqlite3_initialize();
   368    368     if( rc ) return rc;
   369    369   #endif
          370  +#ifdef SQLITE_ENABLE_API_ARMOR
          371  +  if( pVfs==0 ) return SQLITE_MISUSE_BKPT;
          372  +#endif
          373  +
   370    374     MUTEX_LOGIC( mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); )
   371    375     sqlite3_mutex_enter(mutex);
   372    376     vfsUnlink(pVfs);
   373    377     if( makeDflt || vfsList==0 ){
   374    378       pVfs->pNext = vfsList;
   375    379       vfsList = pVfs;
   376    380     }else{

Changes to src/pager.c.

  1937   1937         assert( pPager->journalMode==PAGER_JOURNALMODE_MEMORY );
  1938   1938         sqlite3OsClose(pPager->jfd);
  1939   1939       }else if( pPager->journalMode==PAGER_JOURNALMODE_TRUNCATE ){
  1940   1940         if( pPager->journalOff==0 ){
  1941   1941           rc = SQLITE_OK;
  1942   1942         }else{
  1943   1943           rc = sqlite3OsTruncate(pPager->jfd, 0);
         1944  +        if( rc==SQLITE_OK && pPager->fullSync ){
         1945  +          /* Make sure the new file size is written into the inode right away.
         1946  +          ** Otherwise the journal might resurrect following a power loss and
         1947  +          ** cause the last transaction to roll back.  See
         1948  +          ** https://bugzilla.mozilla.org/show_bug.cgi?id=1072773
         1949  +          */
         1950  +          rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags);
         1951  +        }
  1944   1952         }
  1945   1953         pPager->journalOff = 0;
  1946   1954       }else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST
  1947   1955         || (pPager->exclusiveMode && pPager->journalMode!=PAGER_JOURNALMODE_WAL)
  1948   1956       ){
  1949   1957         rc = zeroJournalHdr(pPager, hasMaster);
  1950   1958         pPager->journalOff = 0;
................................................................................
  6869   6877       sqlite3PcacheMakeDirty(pPgHdr);
  6870   6878       sqlite3PagerUnrefNotNull(pPgHdr);
  6871   6879     }
  6872   6880   
  6873   6881     return SQLITE_OK;
  6874   6882   }
  6875   6883   #endif
         6884  +
         6885  +/*
         6886  +** The page handle passed as the first argument refers to a dirty page 
         6887  +** with a page number other than iNew. This function changes the page's 
         6888  +** page number to iNew and sets the value of the PgHdr.flags field to 
         6889  +** the value passed as the third parameter.
         6890  +*/
         6891  +void sqlite3PagerRekey(DbPage *pPg, Pgno iNew, u16 flags){
         6892  +  assert( pPg->pgno!=iNew );
         6893  +  pPg->flags = flags;
         6894  +  sqlite3PcacheMove(pPg, iNew);
         6895  +}
  6876   6896   
  6877   6897   /*
  6878   6898   ** Return a pointer to the data for the specified page.
  6879   6899   */
  6880   6900   void *sqlite3PagerGetData(DbPage *pPg){
  6881   6901     assert( pPg->nRef>0 || pPg->pPager->memDb );
  6882   6902     return pPg->pData;
................................................................................
  7278   7298   ** is empty, return 0.
  7279   7299   */
  7280   7300   int sqlite3PagerWalFramesize(Pager *pPager){
  7281   7301     assert( pPager->eState>=PAGER_READER );
  7282   7302     return sqlite3WalFramesize(pPager->pWal);
  7283   7303   }
  7284   7304   #endif
         7305  +
  7285   7306   
  7286   7307   #endif /* SQLITE_OMIT_DISKIO */

Changes to src/pager.h.

   184    184   int sqlite3PagerIsMemdb(Pager*);
   185    185   void sqlite3PagerCacheStat(Pager *, int, int, int *);
   186    186   void sqlite3PagerClearCache(Pager *);
   187    187   int sqlite3SectorSize(sqlite3_file *);
   188    188   
   189    189   /* Functions used to truncate the database file. */
   190    190   void sqlite3PagerTruncateImage(Pager*,Pgno);
          191  +
          192  +void sqlite3PagerRekey(DbPage*, Pgno, u16);
   191    193   
   192    194   #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL)
   193    195   void *sqlite3PagerCodec(DbPage *);
   194    196   #endif
   195    197   
   196    198   /* Functions to support testing and debugging. */
   197    199   #if !defined(NDEBUG) || defined(SQLITE_TEST)

Changes to src/prepare.c.

   708    708     int nBytes,               /* Length of zSql in bytes. */
   709    709     int saveSqlFlag,          /* True to copy SQL text into the sqlite3_stmt */
   710    710     Vdbe *pOld,               /* VM being reprepared */
   711    711     sqlite3_stmt **ppStmt,    /* OUT: A pointer to the prepared statement */
   712    712     const char **pzTail       /* OUT: End of parsed string */
   713    713   ){
   714    714     int rc;
   715         -  assert( ppStmt!=0 );
          715  +
          716  +#ifdef SQLITE_ENABLE_API_ARMOR
          717  +  if( ppStmt==0 ) return SQLITE_MISUSE_BKPT;
          718  +#endif
   716    719     *ppStmt = 0;
   717         -  if( !sqlite3SafetyCheckOk(db) ){
          720  +  if( !sqlite3SafetyCheckOk(db)||zSql==0 ){
   718    721       return SQLITE_MISUSE_BKPT;
   719    722     }
   720    723     sqlite3_mutex_enter(db->mutex);
   721    724     sqlite3BtreeEnterAll(db);
   722    725     rc = sqlite3Prepare(db, zSql, nBytes, saveSqlFlag, pOld, ppStmt, pzTail);
   723    726     if( rc==SQLITE_SCHEMA ){
   724    727       sqlite3_finalize(*ppStmt);
................................................................................
   823    826     ** encoded string to UTF-8, then invoking sqlite3_prepare(). The
   824    827     ** tricky bit is figuring out the pointer to return in *pzTail.
   825    828     */
   826    829     char *zSql8;
   827    830     const char *zTail8 = 0;
   828    831     int rc = SQLITE_OK;
   829    832   
   830         -  assert( ppStmt );
          833  +#ifdef SQLITE_ENABLE_API_ARMOR
          834  +  if( ppStmt==0 ) return SQLITE_MISUSE_BKPT;
          835  +#endif
   831    836     *ppStmt = 0;
   832         -  if( !sqlite3SafetyCheckOk(db) ){
          837  +  if( !sqlite3SafetyCheckOk(db)||zSql==0 ){
   833    838       return SQLITE_MISUSE_BKPT;
   834    839     }
   835    840     if( nBytes>=0 ){
   836    841       int sz;
   837    842       const char *z = (const char*)zSql;
   838    843       for(sz=0; sz<nBytes && (z[sz]!=0 || z[sz+1]!=0); sz += 2){}
   839    844       nBytes = sz;

Changes to src/printf.c.

    17     17   /*
    18     18   ** If the strchrnul() library function is available, then set
    19     19   ** HAVE_STRCHRNUL.  If that routine is not available, this module
    20     20   ** will supply its own.  The built-in version is slower than
    21     21   ** the glibc version so the glibc version is definitely preferred.
    22     22   */
    23     23   #if !defined(HAVE_STRCHRNUL)
    24         -# if defined(linux)
    25         -#  define HAVE_STRCHRNUL 1
    26         -# else
    27         -#  define HAVE_STRCHRNUL 0
    28         -# endif
           24  +# define HAVE_STRCHRNUL 0
    29     25   #endif
    30     26   
    31     27   
    32     28   /*
    33     29   ** Conversion types fall into various categories as defined by the
    34     30   ** following enumeration.
    35     31   */
................................................................................
   223    219     double rounder;            /* Used for rounding floating point values */
   224    220     etByte flag_dp;            /* True if decimal point should be shown */
   225    221     etByte flag_rtz;           /* True if trailing zeros should be removed */
   226    222   #endif
   227    223     PrintfArguments *pArgList = 0; /* Arguments for SQLITE_PRINTF_SQLFUNC */
   228    224     char buf[etBUFSIZE];       /* Conversion buffer */
   229    225   
          226  +#ifdef SQLITE_ENABLE_API_ARMOR
          227  +  if( ap==0 ){
          228  +    (void)SQLITE_MISUSE_BKPT;
          229  +    sqlite3StrAccumReset(pAccum);
          230  +    return;
          231  +  }
          232  +#endif
   230    233     bufpt = 0;
   231    234     if( bFlags ){
   232    235       if( (bArgList = (bFlags & SQLITE_PRINTF_SQLFUNC))!=0 ){
   233    236         pArgList = va_arg(ap, PrintfArguments*);
   234    237       }
   235    238       useIntern = bFlags & SQLITE_PRINTF_INTERNAL;
   236    239     }else{
................................................................................
   943    946   ** Print into memory obtained from sqlite3_malloc().  Omit the internal
   944    947   ** %-conversion extensions.
   945    948   */
   946    949   char *sqlite3_vmprintf(const char *zFormat, va_list ap){
   947    950     char *z;
   948    951     char zBase[SQLITE_PRINT_BUF_SIZE];
   949    952     StrAccum acc;
          953  +
          954  +#ifdef SQLITE_ENABLE_API_ARMOR  
          955  +  if( zFormat==0 ){
          956  +    (void)SQLITE_MISUSE_BKPT;
          957  +    return 0;
          958  +  }
          959  +#endif
   950    960   #ifndef SQLITE_OMIT_AUTOINIT
   951    961     if( sqlite3_initialize() ) return 0;
   952    962   #endif
   953    963     sqlite3StrAccumInit(&acc, zBase, sizeof(zBase), SQLITE_MAX_LENGTH);
   954    964     acc.useMalloc = 2;
   955    965     sqlite3VXPrintf(&acc, 0, zFormat, ap);
   956    966     z = sqlite3StrAccumFinish(&acc);
................................................................................
   985    995   ** mistake.
   986    996   **
   987    997   ** sqlite3_vsnprintf() is the varargs version.
   988    998   */
   989    999   char *sqlite3_vsnprintf(int n, char *zBuf, const char *zFormat, va_list ap){
   990   1000     StrAccum acc;
   991   1001     if( n<=0 ) return zBuf;
         1002  +#ifdef SQLITE_ENABLE_API_ARMOR
         1003  +  if( zBuf==0 || zFormat==0 ) {
         1004  +    (void)SQLITE_MISUSE_BKPT;
         1005  +    if( zBuf && n>0 ) zBuf[0] = 0;
         1006  +    return zBuf;
         1007  +  }
         1008  +#endif
   992   1009     sqlite3StrAccumInit(&acc, zBuf, n, 0);
   993   1010     acc.useMalloc = 0;
   994   1011     sqlite3VXPrintf(&acc, 0, zFormat, ap);
   995   1012     return sqlite3StrAccumFinish(&acc);
   996   1013   }
   997   1014   char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){
   998   1015     char *z;

Changes to src/random.c.

    29     29   
    30     30   /*
    31     31   ** Return N random bytes.
    32     32   */
    33     33   void sqlite3_randomness(int N, void *pBuf){
    34     34     unsigned char t;
    35     35     unsigned char *zBuf = pBuf;
           36  +
           37  +#ifndef SQLITE_OMIT_AUTOINIT
           38  +  if( sqlite3_initialize() ) return;
           39  +#endif
    36     40   
    37     41     /* The "wsdPrng" macro will resolve to the pseudo-random number generator
    38     42     ** state vector.  If writable static data is unsupported on the target,
    39     43     ** we have to locate the state vector at run-time.  In the more common
    40     44     ** case where writable static data is supported, wsdPrng can refer directly
    41     45     ** to the "sqlite3Prng" state vector declared above.
    42     46     */
................................................................................
    48     52   #endif
    49     53   
    50     54   #if SQLITE_THREADSAFE
    51     55     sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_PRNG);
    52     56     sqlite3_mutex_enter(mutex);
    53     57   #endif
    54     58   
    55         -  if( N<=0 ){
           59  +  if( N<=0 || pBuf==0 ){
    56     60       wsdPrng.isInit = 0;
    57     61       sqlite3_mutex_leave(mutex);
    58     62       return;
    59     63     }
    60     64   
    61     65     /* Initialize the state of the random number generator once,
    62     66     ** the first time this routine is called.  The seed value does

Changes to src/resolve.c.

    24     24   ** This needs to occur when copying a TK_AGG_FUNCTION node from an
    25     25   ** outer query into an inner subquery.
    26     26   **
    27     27   ** incrAggFunctionDepth(pExpr,n) is the main routine.  incrAggDepth(..)
    28     28   ** is a helper function - a callback for the tree walker.
    29     29   */
    30     30   static int incrAggDepth(Walker *pWalker, Expr *pExpr){
    31         -  if( pExpr->op==TK_AGG_FUNCTION ) pExpr->op2 += pWalker->u.i;
           31  +  if( pExpr->op==TK_AGG_FUNCTION ) pExpr->op2 += pWalker->u.n;
    32     32     return WRC_Continue;
    33     33   }
    34     34   static void incrAggFunctionDepth(Expr *pExpr, int N){
    35     35     if( N>0 ){
    36     36       Walker w;
    37     37       memset(&w, 0, sizeof(w));
    38     38       w.xExprCallback = incrAggDepth;
    39         -    w.u.i = N;
           39  +    w.u.n = N;
    40     40       sqlite3WalkExpr(&w, pExpr);
    41     41     }
    42     42   }
    43     43   
    44     44   /*
    45     45   ** Turn the pExpr expression into an alias for the iCol-th column of the
    46     46   ** result set in pEList.
................................................................................
   580    580   */
   581    581   static int exprProbability(Expr *p){
   582    582     double r = -1.0;
   583    583     if( p->op!=TK_FLOAT ) return -1;
   584    584     sqlite3AtoF(p->u.zToken, &r, sqlite3Strlen30(p->u.zToken), SQLITE_UTF8);
   585    585     assert( r>=0.0 );
   586    586     if( r>1.0 ) return -1;
   587         -  return (int)(r*1000.0);
          587  +  return (int)(r*134217728.0);
   588    588   }
   589    589   
   590    590   /*
   591    591   ** This routine is callback for sqlite3WalkExpr().
   592    592   **
   593    593   ** Resolve symbolic names into TK_COLUMN operators for the current
   594    594   ** node in the expression tree.  Return 0 to continue the search down
................................................................................
   712    712               ** EVIDENCE-OF: R-01283-11636 The unlikely(X) function is short-hand for
   713    713               ** likelihood(X,0.0625).
   714    714               ** EVIDENCE-OF: R-36850-34127 The likely(X) function is short-hand for
   715    715               ** likelihood(X,0.9375).
   716    716               ** EVIDENCE-OF: R-53436-40973 The likely(X) function is equivalent to
   717    717               ** likelihood(X,0.9375). */
   718    718               /* TUNING: unlikely() probability is 0.0625.  likely() is 0.9375 */
   719         -            pExpr->iTable = pDef->zName[0]=='u' ? 62 : 938;
          719  +            pExpr->iTable = pDef->zName[0]=='u' ? 8388608 : 125829120;
   720    720             }             
   721    721           }
   722    722   #ifndef SQLITE_OMIT_AUTHORIZATION
   723    723           auth = sqlite3AuthCheck(pParse, SQLITE_FUNCTION, 0, pDef->zName, 0);
   724    724           if( auth!=SQLITE_OK ){
   725    725             if( auth==SQLITE_DENY ){
   726    726               sqlite3ErrorMsg(pParse, "not authorized to use function: %s",

Changes to src/shell.c.

   878    878   #endif
   879    879         if( p->cnt++==0 && p->showHeader ){
   880    880           for(i=0; i<nArg; i++){
   881    881             output_csv(p, azCol[i] ? azCol[i] : "", i<nArg-1);
   882    882           }
   883    883           fprintf(p->out,"%s",p->newline);
   884    884         }
   885         -      if( azArg>0 ){
          885  +      if( nArg>0 ){
   886    886           for(i=0; i<nArg; i++){
   887    887             output_csv(p, azArg[i], i<nArg-1);
   888    888           }
   889    889           fprintf(p->out,"%s",p->newline);
   890    890         }
   891    891   #if defined(WIN32) || defined(_WIN32)
   892    892         fflush(p->out);

Changes to src/sqliteInt.h.

  1220   1220   /*                not used    0x0010   // Was: SQLITE_IdxRealAsInt */
  1221   1221   #define SQLITE_DistinctOpt    0x0020   /* DISTINCT using indexes */
  1222   1222   #define SQLITE_CoverIdxScan   0x0040   /* Covering index scans */
  1223   1223   #define SQLITE_OrderByIdxJoin 0x0080   /* ORDER BY of joins via index */
  1224   1224   #define SQLITE_SubqCoroutine  0x0100   /* Evaluate subqueries as coroutines */
  1225   1225   #define SQLITE_Transitive     0x0200   /* Transitive constraints */
  1226   1226   #define SQLITE_OmitNoopJoin   0x0400   /* Omit unused tables in joins */
  1227         -#define SQLITE_Stat3          0x0800   /* Use the SQLITE_STAT3 table */
         1227  +#define SQLITE_Stat34         0x0800   /* Use STAT3 or STAT4 data */
  1228   1228   #define SQLITE_AllOpts        0xffff   /* All optimizations */
  1229   1229   
  1230   1230   /*
  1231   1231   ** Macros for testing whether or not optimizations are enabled or disabled.
  1232   1232   */
  1233   1233   #ifndef SQLITE_OMIT_BUILTIN_TEST
  1234   1234   #define OptimizationDisabled(db, mask)  (((db)->dbOptFlags&(mask))!=0)
................................................................................
  1807   1807     unsigned isResized:1;    /* True if resizeIndexObject() has been called */
  1808   1808     unsigned isCovering:1;   /* True if this is a covering index */
  1809   1809   #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
  1810   1810     int nSample;             /* Number of elements in aSample[] */
  1811   1811     int nSampleCol;          /* Size of IndexSample.anEq[] and so on */
  1812   1812     tRowcnt *aAvgEq;         /* Average nEq values for keys not in aSample */
  1813   1813     IndexSample *aSample;    /* Samples of the left-most key */
  1814         -  tRowcnt *aiRowEst;       /* Non-logarithmic stat1 data for this table */
         1814  +  tRowcnt *aiRowEst;       /* Non-logarithmic stat1 data for this index */
         1815  +  tRowcnt nRowEst0;        /* Non-logarithmic number of rows in the index */
  1815   1816   #endif
  1816   1817   };
  1817   1818   
  1818   1819   /*
  1819   1820   ** Allowed values for Index.idxType
  1820   1821   */
  1821   1822   #define SQLITE_IDXTYPE_APPDEF      0   /* Created using CREATE INDEX */
................................................................................
  2005   2006   
  2006   2007   #if SQLITE_MAX_EXPR_DEPTH>0
  2007   2008     int nHeight;           /* Height of the tree headed by this node */
  2008   2009   #endif
  2009   2010     int iTable;            /* TK_COLUMN: cursor number of table holding column
  2010   2011                            ** TK_REGISTER: register number
  2011   2012                            ** TK_TRIGGER: 1 -> new, 0 -> old
  2012         -                         ** EP_Unlikely:  1000 times likelihood */
         2013  +                         ** EP_Unlikely:  134217728 times likelihood */
  2013   2014     ynVar iColumn;         /* TK_COLUMN: column index.  -1 for rowid.
  2014   2015                            ** TK_VARIABLE: variable number (always >= 1). */
  2015   2016     i16 iAgg;              /* Which entry in pAggInfo->aCol[] or ->aFunc[] */
  2016   2017     i16 iRightJoinTable;   /* If EP_FromJoin, the right table of the join */
  2017   2018     u8 op2;                /* TK_REGISTER: original value of Expr.op
  2018   2019                            ** TK_COLUMN: the value of p5 for OP_Column
  2019   2020                            ** TK_AGG_FUNCTION: nesting depth */
................................................................................
  2897   2898   */
  2898   2899   struct Walker {
  2899   2900     int (*xExprCallback)(Walker*, Expr*);     /* Callback for expressions */
  2900   2901     int (*xSelectCallback)(Walker*,Select*);  /* Callback for SELECTs */
  2901   2902     void (*xSelectCallback2)(Walker*,Select*);/* Second callback for SELECTs */
  2902   2903     Parse *pParse;                            /* Parser context.  */
  2903   2904     int walkerDepth;                          /* Number of subqueries */
         2905  +  u8 eCode;                                 /* A small processing code */
  2904   2906     union {                                   /* Extra data for callback */
  2905   2907       NameContext *pNC;                          /* Naming context */
  2906         -    int i;                                     /* Integer value */
         2908  +    int n;                                     /* A counter */
         2909  +    int iCur;                                  /* A cursor number */
  2907   2910       SrcList *pSrcList;                         /* FROM clause */
  2908   2911       struct SrcCount *pSrcCount;                /* Counting column references */
  2909   2912     } u;
  2910   2913   };
  2911   2914   
  2912   2915   /* Forward declarations */
  2913   2916   int sqlite3WalkExpr(Walker*, Expr*);
................................................................................
  3300   3303   void sqlite3RollbackTransaction(Parse*);
  3301   3304   void sqlite3Savepoint(Parse*, int, Token*);
  3302   3305   void sqlite3CloseSavepoints(sqlite3 *);
  3303   3306   void sqlite3LeaveMutexAndCloseZombie(sqlite3*);
  3304   3307   int sqlite3ExprIsConstant(Expr*);
  3305   3308   int sqlite3ExprIsConstantNotJoin(Expr*);
  3306   3309   int sqlite3ExprIsConstantOrFunction(Expr*, u8);
         3310  +int sqlite3ExprIsTableConstant(Expr*,int);
  3307   3311   int sqlite3ExprIsInteger(Expr*, int*);
  3308   3312   int sqlite3ExprCanBeNull(const Expr*);
  3309   3313   int sqlite3ExprNeedsNoAffinityChange(const Expr*, char);
  3310   3314   int sqlite3IsRowid(const char*);
  3311   3315   void sqlite3GenerateRowDelete(Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8);
  3312   3316   void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*);
  3313   3317   int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int);

Changes to src/status.c.

    82     82   ** then this routine is not threadsafe.
    83     83   */
    84     84   int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag){
    85     85     wsdStatInit;
    86     86     if( op<0 || op>=ArraySize(wsdStat.nowValue) ){
    87     87       return SQLITE_MISUSE_BKPT;
    88     88     }
           89  +#ifdef SQLITE_ENABLE_API_ARMOR
           90  +  if( pCurrent==0 || pHighwater==0 ) return SQLITE_MISUSE_BKPT;
           91  +#endif
    89     92     *pCurrent = wsdStat.nowValue[op];
    90     93     *pHighwater = wsdStat.mxValue[op];
    91     94     if( resetFlag ){
    92     95       wsdStat.mxValue[op] = wsdStat.nowValue[op];
    93     96     }
    94     97     return SQLITE_OK;
    95     98   }
................................................................................
   101    104     sqlite3 *db,          /* The database connection whose status is desired */
   102    105     int op,               /* Status verb */
   103    106     int *pCurrent,        /* Write current value here */
   104    107     int *pHighwater,      /* Write high-water mark here */
   105    108     int resetFlag         /* Reset high-water mark if true */
   106    109   ){
   107    110     int rc = SQLITE_OK;   /* Return code */
          111  +#ifdef SQLITE_ENABLE_API_ARMOR
          112  +  if( !sqlite3SafetyCheckOk(db) || pCurrent==0|| pHighwater==0 ){
          113  +    return SQLITE_MISUSE_BKPT;
          114  +  }
          115  +#endif
   108    116     sqlite3_mutex_enter(db->mutex);
   109    117     switch( op ){
   110    118       case SQLITE_DBSTATUS_LOOKASIDE_USED: {
   111    119         *pCurrent = db->lookaside.nOut;
   112    120         *pHighwater = db->lookaside.mxOut;
   113    121         if( resetFlag ){
   114    122           db->lookaside.mxOut = db->lookaside.nOut;

Changes to src/table.c.

   122    122     int *pnRow,                 /* Write the number of rows in the result here */
   123    123     int *pnColumn,              /* Write the number of columns of result here */
   124    124     char **pzErrMsg             /* Write error messages here */
   125    125   ){
   126    126     int rc;
   127    127     TabResult res;
   128    128   
          129  +#ifdef SQLITE_ENABLE_API_ARMOR
          130  +  if( pazResult==0 ) return SQLITE_MISUSE_BKPT;
          131  +#endif
   129    132     *pazResult = 0;
   130    133     if( pnColumn ) *pnColumn = 0;
   131    134     if( pnRow ) *pnRow = 0;
   132    135     if( pzErrMsg ) *pzErrMsg = 0;
   133    136     res.zErrMsg = 0;
   134    137     res.nRow = 0;
   135    138     res.nColumn = 0;

Changes to src/test1.c.

  6481   6481       { "factor-constants",    SQLITE_FactorOutConst },
  6482   6482       { "distinct-opt",        SQLITE_DistinctOpt    },
  6483   6483       { "cover-idx-scan",      SQLITE_CoverIdxScan   },
  6484   6484       { "order-by-idx-join",   SQLITE_OrderByIdxJoin },
  6485   6485       { "transitive",          SQLITE_Transitive     },
  6486   6486       { "subquery-coroutine",  SQLITE_SubqCoroutine  },
  6487   6487       { "omit-noop-join",      SQLITE_OmitNoopJoin   },
  6488         -    { "stat3",               SQLITE_Stat3          },
         6488  +    { "stat3",               SQLITE_Stat34         },
         6489  +    { "stat4",               SQLITE_Stat34         },
  6489   6490     };
  6490   6491   
  6491   6492     if( objc!=4 ){
  6492   6493       Tcl_WrongNumArgs(interp, 1, objv, "DB OPT BOOLEAN");
  6493   6494       return TCL_ERROR;
  6494   6495     }
  6495   6496     if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;

Changes to src/util.c.

   247    247   ** sqlite3_strnicmp() APIs allow applications and extensions to compare
   248    248   ** the contents of two buffers containing UTF-8 strings in a
   249    249   ** case-independent fashion, using the same definition of "case
   250    250   ** independence" that SQLite uses internally when comparing identifiers.
   251    251   */
   252    252   int sqlite3_stricmp(const char *zLeft, const char *zRight){
   253    253     register unsigned char *a, *b;
          254  +  if( zLeft==0 ){
          255  +    return zRight ? -1 : 0;
          256  +  }else if( zRight==0 ){
          257  +    return 1;
          258  +  }
   254    259     a = (unsigned char *)zLeft;
   255    260     b = (unsigned char *)zRight;
   256    261     while( *a!=0 && UpperToLower[*a]==UpperToLower[*b]){ a++; b++; }
   257    262     return UpperToLower[*a] - UpperToLower[*b];
   258    263   }
   259    264   int sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){
   260    265     register unsigned char *a, *b;
          266  +  if( zLeft==0 ){
          267  +    return zRight ? -1 : 0;
          268  +  }else if( zRight==0 ){
          269  +    return 1;
          270  +  }
   261    271     a = (unsigned char *)zLeft;
   262    272     b = (unsigned char *)zRight;
   263    273     while( N-- > 0 && *a!=0 && UpperToLower[*a]==UpperToLower[*b]){ a++; b++; }
   264    274     return N<0 ? 0 : UpperToLower[*a] - UpperToLower[*b];
   265    275   }
   266    276   
   267    277   /*

Changes to src/vdbeapi.c.

   978    978   */
   979    979   static const void *columnName(
   980    980     sqlite3_stmt *pStmt,
   981    981     int N,
   982    982     const void *(*xFunc)(Mem*),
   983    983     int useType
   984    984   ){
   985         -  const void *ret = 0;
   986         -  Vdbe *p = (Vdbe *)pStmt;
          985  +  const void *ret;
          986  +  Vdbe *p;
   987    987     int n;
   988         -  sqlite3 *db = p->db;
   989         -  
          988  +  sqlite3 *db;
          989  +#ifdef SQLITE_ENABLE_API_ARMOR
          990  +  if( pStmt==0 ){
          991  +    (void)SQLITE_MISUSE_BKPT;
          992  +    return 0;
          993  +  }
          994  +#endif
          995  +  ret = 0;
          996  +  p = (Vdbe *)pStmt;
          997  +  db = p->db;
   990    998     assert( db!=0 );
   991    999     n = sqlite3_column_count(pStmt);
   992   1000     if( N<n && N>=0 ){
   993   1001       N += useType*n;
   994   1002       sqlite3_mutex_enter(db->mutex);
   995   1003       assert( db->mallocFailed==0 );
   996   1004       ret = xFunc(&p->aColName[N]);
................................................................................
  1474   1482   ** Return a pointer to the next prepared statement after pStmt associated
  1475   1483   ** with database connection pDb.  If pStmt is NULL, return the first
  1476   1484   ** prepared statement for the database connection.  Return NULL if there
  1477   1485   ** are no more.
  1478   1486   */
  1479   1487   sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt){
  1480   1488     sqlite3_stmt *pNext;
         1489  +#ifdef SQLITE_ENABLE_API_ARMOR
         1490  +  if( !sqlite3SafetyCheckOk(pDb) ){
         1491  +    (void)SQLITE_MISUSE_BKPT;
         1492  +    return 0;
         1493  +  }
         1494  +#endif
  1481   1495     sqlite3_mutex_enter(pDb->mutex);
  1482   1496     if( pStmt==0 ){
  1483   1497       pNext = (sqlite3_stmt*)pDb->pVdbe;
  1484   1498     }else{
  1485   1499       pNext = (sqlite3_stmt*)((Vdbe*)pStmt)->pNext;
  1486   1500     }
  1487   1501     sqlite3_mutex_leave(pDb->mutex);
................................................................................
  1489   1503   }
  1490   1504   
  1491   1505   /*
  1492   1506   ** Return the value of a status counter for a prepared statement
  1493   1507   */
  1494   1508   int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){
  1495   1509     Vdbe *pVdbe = (Vdbe*)pStmt;
  1496         -  u32 v = pVdbe->aCounter[op];
         1510  +  u32 v;
         1511  +#ifdef SQLITE_ENABLE_API_ARMOR
         1512  +  if( !pStmt ){
         1513  +    (void)SQLITE_MISUSE_BKPT;
         1514  +    return 0;
         1515  +  }
         1516  +#endif
         1517  +  v = pVdbe->aCounter[op];
  1497   1518     if( resetFlag ) pVdbe->aCounter[op] = 0;
  1498   1519     return (int)v;
  1499   1520   }

Changes to src/vdbeblob.c.

   149    149   
   150    150     int rc = SQLITE_OK;
   151    151     char *zErr = 0;
   152    152     Table *pTab;
   153    153     Parse *pParse = 0;
   154    154     Incrblob *pBlob = 0;
   155    155   
          156  +#ifdef SQLITE_ENABLE_API_ARMOR
          157  +  if( !sqlite3SafetyCheckOk(db) || ppBlob==0 || zTable==0 ){
          158  +    return SQLITE_MISUSE_BKPT;
          159  +  }
          160  +#endif
   156    161     flags = !!flags;                /* flags = (flags ? 1 : 0); */
   157    162     *ppBlob = 0;
   158    163   
   159    164     sqlite3_mutex_enter(db->mutex);
   160    165   
   161    166     pBlob = (Incrblob *)sqlite3DbMallocZero(db, sizeof(Incrblob));
   162    167     if( !pBlob ) goto blob_open_out;

Changes to src/vtab.c.

    77     77   */
    78     78   int sqlite3_create_module(
    79     79     sqlite3 *db,                    /* Database in which module is registered */
    80     80     const char *zName,              /* Name assigned to this module */
    81     81     const sqlite3_module *pModule,  /* The definition of the module */
    82     82     void *pAux                      /* Context pointer for xCreate/xConnect */
    83     83   ){
           84  +#ifdef SQLITE_ENABLE_API_ARMOR
           85  +  if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT;
           86  +#endif
    84     87     return createModule(db, zName, pModule, pAux, 0);
    85     88   }
    86     89   
    87     90   /*
    88     91   ** External API function used to create a new virtual-table module.
    89     92   */
    90     93   int sqlite3_create_module_v2(
    91     94     sqlite3 *db,                    /* Database in which module is registered */
    92     95     const char *zName,              /* Name assigned to this module */
    93     96     const sqlite3_module *pModule,  /* The definition of the module */
    94     97     void *pAux,                     /* Context pointer for xCreate/xConnect */
    95     98     void (*xDestroy)(void *)        /* Module destructor function */
    96     99   ){
          100  +#ifdef SQLITE_ENABLE_API_ARMOR
          101  +  if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT;
          102  +#endif
    97    103     return createModule(db, zName, pModule, pAux, xDestroy);
    98    104   }
    99    105   
   100    106   /*
   101    107   ** Lock the virtual table so that it cannot be disconnected.
   102    108   ** Locks nest.  Every lock should have a corresponding unlock.
   103    109   ** If an unlock is omitted, resources leaks will occur.  
................................................................................
   694    700   int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){
   695    701     Parse *pParse;
   696    702   
   697    703     int rc = SQLITE_OK;
   698    704     Table *pTab;
   699    705     char *zErr = 0;
   700    706   
          707  +#ifdef SQLITE_ENABLE_API_ARMOR
          708  +  if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT;
          709  +#endif
   701    710     sqlite3_mutex_enter(db->mutex);
   702    711     if( !db->pVtabCtx || !(pTab = db->pVtabCtx->pTab) ){
   703    712       sqlite3Error(db, SQLITE_MISUSE);
   704    713       sqlite3_mutex_leave(db->mutex);
   705    714       return SQLITE_MISUSE_BKPT;
   706    715     }
   707    716     assert( (pTab->tabFlags & TF_Virtual)!=0 );
................................................................................
  1050   1059   ** The results of this routine are undefined unless it is called from
  1051   1060   ** within an xUpdate method.
  1052   1061   */
  1053   1062   int sqlite3_vtab_on_conflict(sqlite3 *db){
  1054   1063     static const unsigned char aMap[] = { 
  1055   1064       SQLITE_ROLLBACK, SQLITE_ABORT, SQLITE_FAIL, SQLITE_IGNORE, SQLITE_REPLACE 
  1056   1065     };
         1066  +#ifdef SQLITE_ENABLE_API_ARMOR
         1067  +  if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT;
         1068  +#endif
  1057   1069     assert( OE_Rollback==1 && OE_Abort==2 && OE_Fail==3 );
  1058   1070     assert( OE_Ignore==4 && OE_Replace==5 );
  1059   1071     assert( db->vtabOnConflict>=1 && db->vtabOnConflict<=5 );
  1060   1072     return (int)aMap[db->vtabOnConflict-1];
  1061   1073   }
  1062   1074   
  1063   1075   /*
................................................................................
  1065   1077   ** the SQLite core with additional information about the behavior
  1066   1078   ** of the virtual table being implemented.
  1067   1079   */
  1068   1080   int sqlite3_vtab_config(sqlite3 *db, int op, ...){
  1069   1081     va_list ap;
  1070   1082     int rc = SQLITE_OK;
  1071   1083   
         1084  +#ifdef SQLITE_ENABLE_API_ARMOR
         1085  +  if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT;
         1086  +#endif
  1072   1087     sqlite3_mutex_enter(db->mutex);
  1073         -
  1074   1088     va_start(ap, op);
  1075   1089     switch( op ){
  1076   1090       case SQLITE_VTAB_CONSTRAINT_SUPPORT: {
  1077   1091         VtabCtx *p = db->pVtabCtx;
  1078   1092         if( !p ){
  1079   1093           rc = SQLITE_MISUSE_BKPT;
  1080   1094         }else{

Changes to src/where.c.

   221    221       if( pOld!=pWC->aStatic ){
   222    222         sqlite3DbFree(db, pOld);
   223    223       }
   224    224       pWC->nSlot = sqlite3DbMallocSize(db, pWC->a)/sizeof(pWC->a[0]);
   225    225     }
   226    226     pTerm = &pWC->a[idx = pWC->nTerm++];
   227    227     if( p && ExprHasProperty(p, EP_Unlikely) ){
   228         -    pTerm->truthProb = sqlite3LogEst(p->iTable) - 99;
          228  +    pTerm->truthProb = sqlite3LogEst(p->iTable) - 270;
   229    229     }else{
   230    230       pTerm->truthProb = 1;
   231    231     }
   232    232     pTerm->pExpr = sqlite3ExprSkipCollate(p);
   233    233     pTerm->wtFlags = wtFlags;
   234    234     pTerm->pWC = pWC;
   235    235     pTerm->iParent = -1;
................................................................................
   751    751   */
   752    752   static void transferJoinMarkings(Expr *pDerived, Expr *pBase){
   753    753     if( pDerived ){
   754    754       pDerived->flags |= pBase->flags & EP_FromJoin;
   755    755       pDerived->iRightJoinTable = pBase->iRightJoinTable;
   756    756     }
   757    757   }
          758  +
          759  +/*
          760  +** Mark term iChild as being a child of term iParent
          761  +*/
          762  +static void markTermAsChild(WhereClause *pWC, int iChild, int iParent){
          763  +  pWC->a[iChild].iParent = iParent;
          764  +  pWC->a[iChild].truthProb = pWC->a[iParent].truthProb;
          765  +  pWC->a[iParent].nChild++;
          766  +}
   758    767   
   759    768   #if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY)
   760    769   /*
   761    770   ** Analyze a term that consists of two or more OR-connected
   762    771   ** subterms.  So in:
   763    772   **
   764    773   **     ... WHERE  (a=5) AND (b=7 OR c=9 OR d=13) AND (d=13)
................................................................................
  1049   1058           transferJoinMarkings(pNew, pExpr);
  1050   1059           assert( !ExprHasProperty(pNew, EP_xIsSelect) );
  1051   1060           pNew->x.pList = pList;
  1052   1061           idxNew = whereClauseInsert(pWC, pNew, TERM_VIRTUAL|TERM_DYNAMIC);
  1053   1062           testcase( idxNew==0 );
  1054   1063           exprAnalyze(pSrc, pWC, idxNew);
  1055   1064           pTerm = &pWC->a[idxTerm];
  1056         -        pWC->a[idxNew].iParent = idxTerm;
  1057         -        pTerm->nChild = 1;
         1065  +        markTermAsChild(pWC, idxNew, idxTerm);
  1058   1066         }else{
  1059   1067           sqlite3ExprListDelete(db, pList);
  1060   1068         }
  1061   1069         pTerm->eOperator = WO_NOOP;  /* case 1 trumps case 2 */
  1062   1070       }
  1063   1071     }
  1064   1072   }
................................................................................
  1152   1160           if( db->mallocFailed ){
  1153   1161             sqlite3ExprDelete(db, pDup);
  1154   1162             return;
  1155   1163           }
  1156   1164           idxNew = whereClauseInsert(pWC, pDup, TERM_VIRTUAL|TERM_DYNAMIC);
  1157   1165           if( idxNew==0 ) return;
  1158   1166           pNew = &pWC->a[idxNew];
  1159         -        pNew->iParent = idxTerm;
         1167  +        markTermAsChild(pWC, idxNew, idxTerm);
  1160   1168           pTerm = &pWC->a[idxTerm];
  1161         -        pTerm->nChild = 1;
  1162   1169           pTerm->wtFlags |= TERM_COPIED;
  1163   1170           if( pExpr->op==TK_EQ
  1164   1171            && !ExprHasProperty(pExpr, EP_FromJoin)
  1165   1172            && OptimizationEnabled(db, SQLITE_Transitive)
  1166   1173           ){
  1167   1174             pTerm->eOperator |= WO_EQUIV;
  1168   1175             eExtraOp = WO_EQUIV;
................................................................................
  1211   1218                                sqlite3ExprDup(db, pExpr->pLeft, 0),
  1212   1219                                sqlite3ExprDup(db, pList->a[i].pExpr, 0), 0);
  1213   1220         transferJoinMarkings(pNewExpr, pExpr);
  1214   1221         idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC);
  1215   1222         testcase( idxNew==0 );
  1216   1223         exprAnalyze(pSrc, pWC, idxNew);
  1217   1224         pTerm = &pWC->a[idxTerm];
  1218         -      pWC->a[idxNew].iParent = idxTerm;
         1225  +      markTermAsChild(pWC, idxNew, idxTerm);
  1219   1226       }
  1220         -    pTerm->nChild = 2;
  1221   1227     }
  1222   1228   #endif /* SQLITE_OMIT_BETWEEN_OPTIMIZATION */
  1223   1229   
  1224   1230   #if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY)
  1225   1231     /* Analyze a term that is composed of two or more subterms connected by
  1226   1232     ** an OR operator.
  1227   1233     */
................................................................................
  1288   1294              pStr2, 0);
  1289   1295       transferJoinMarkings(pNewExpr2, pExpr);
  1290   1296       idxNew2 = whereClauseInsert(pWC, pNewExpr2, TERM_VIRTUAL|TERM_DYNAMIC);
  1291   1297       testcase( idxNew2==0 );
  1292   1298       exprAnalyze(pSrc, pWC, idxNew2);
  1293   1299       pTerm = &pWC->a[idxTerm];
  1294   1300       if( isComplete ){
  1295         -      pWC->a[idxNew1].iParent = idxTerm;
  1296         -      pWC->a[idxNew2].iParent = idxTerm;
  1297         -      pTerm->nChild = 2;
         1301  +      markTermAsChild(pWC, idxNew1, idxTerm);
         1302  +      markTermAsChild(pWC, idxNew2, idxTerm);
  1298   1303       }
  1299   1304     }
  1300   1305   #endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */
  1301   1306   
  1302   1307   #ifndef SQLITE_OMIT_VIRTUALTABLE
  1303   1308     /* Add a WO_MATCH auxiliary term to the constraint set if the
  1304   1309     ** current expression is of the form:  column MATCH expr.
................................................................................
  1323   1328         idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC);
  1324   1329         testcase( idxNew==0 );
  1325   1330         pNewTerm = &pWC->a[idxNew];
  1326   1331         pNewTerm->prereqRight = prereqExpr;
  1327   1332         pNewTerm->leftCursor = pLeft->iTable;
  1328   1333         pNewTerm->u.leftColumn = pLeft->iColumn;
  1329   1334         pNewTerm->eOperator = WO_MATCH;
  1330         -      pNewTerm->iParent = idxTerm;
         1335  +      markTermAsChild(pWC, idxNew, idxTerm);
  1331   1336         pTerm = &pWC->a[idxTerm];
  1332         -      pTerm->nChild = 1;
  1333   1337         pTerm->wtFlags |= TERM_COPIED;
  1334   1338         pNewTerm->prereqAll = pTerm->prereqAll;
  1335   1339       }
  1336   1340     }
  1337   1341   #endif /* SQLITE_OMIT_VIRTUALTABLE */
  1338   1342   
  1339   1343   #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
................................................................................
  1346   1350     ** TERM_VNULL tag will suppress the not-null check at the beginning
  1347   1351     ** of the loop.  Without the TERM_VNULL flag, the not-null check at
  1348   1352     ** the start of the loop will prevent any results from being returned.
  1349   1353     */
  1350   1354     if( pExpr->op==TK_NOTNULL
  1351   1355      && pExpr->pLeft->op==TK_COLUMN
  1352   1356      && pExpr->pLeft->iColumn>=0
  1353         -   && OptimizationEnabled(db, SQLITE_Stat3)
         1357  +   && OptimizationEnabled(db, SQLITE_Stat34)
  1354   1358     ){
  1355   1359       Expr *pNewExpr;
  1356   1360       Expr *pLeft = pExpr->pLeft;
  1357   1361       int idxNew;
  1358   1362       WhereTerm *pNewTerm;
  1359   1363   
  1360   1364       pNewExpr = sqlite3PExpr(pParse, TK_GT,
................................................................................
  1365   1369                                 TERM_VIRTUAL|TERM_DYNAMIC|TERM_VNULL);
  1366   1370       if( idxNew ){
  1367   1371         pNewTerm = &pWC->a[idxNew];
  1368   1372         pNewTerm->prereqRight = 0;
  1369   1373         pNewTerm->leftCursor = pLeft->iTable;
  1370   1374         pNewTerm->u.leftColumn = pLeft->iColumn;
  1371   1375         pNewTerm->eOperator = WO_GT;
  1372         -      pNewTerm->iParent = idxTerm;
         1376  +      markTermAsChild(pWC, idxNew, idxTerm);
  1373   1377         pTerm = &pWC->a[idxTerm];
  1374         -      pTerm->nChild = 1;
  1375   1378         pTerm->wtFlags |= TERM_COPIED;
  1376   1379         pNewTerm->prereqAll = pTerm->prereqAll;
  1377   1380       }
  1378   1381     }
  1379   1382   #endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */
  1380   1383   
  1381   1384     /* Prevent ON clause terms of a LEFT JOIN from being used to drive
................................................................................
  1587   1590     int mxBitCol;               /* Maximum column in pSrc->colUsed */
  1588   1591     CollSeq *pColl;             /* Collating sequence to on a column */
  1589   1592     WhereLoop *pLoop;           /* The Loop object */
  1590   1593     char *zNotUsed;             /* Extra space on the end of pIdx */
  1591   1594     Bitmask idxCols;            /* Bitmap of columns used for indexing */
  1592   1595     Bitmask extraCols;          /* Bitmap of additional columns */
  1593   1596     u8 sentWarning = 0;         /* True if a warnning has been issued */
         1597  +  Expr *pPartial = 0;         /* Partial Index Expression */
         1598  +  int iContinue = 0;          /* Jump here to skip excluded rows */
  1594   1599   
  1595   1600     /* Generate code to skip over the creation and initialization of the
  1596   1601     ** transient index on 2nd and subsequent iterations of the loop. */
  1597   1602     v = pParse->pVdbe;
  1598   1603     assert( v!=0 );
  1599   1604     addrInit = sqlite3CodeOnce(pParse); VdbeCoverage(v);
  1600   1605   
................................................................................
  1602   1607     ** and used to match WHERE clause constraints */
  1603   1608     nKeyCol = 0;
  1604   1609     pTable = pSrc->pTab;
  1605   1610     pWCEnd = &pWC->a[pWC->nTerm];
  1606   1611     pLoop = pLevel->pWLoop;
  1607   1612     idxCols = 0;
  1608   1613     for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){
         1614  +    if( pLoop->prereq==0
         1615  +     && (pTerm->wtFlags & TERM_VIRTUAL)==0
         1616  +     && sqlite3ExprIsTableConstant(pTerm->pExpr, pSrc->iCursor) ){
         1617  +      pPartial = sqlite3ExprAnd(pParse->db, pPartial,
         1618  +                                sqlite3ExprDup(pParse->db, pTerm->pExpr, 0));
         1619  +    }
  1609   1620       if( termCanDriveIndex(pTerm, pSrc, notReady) ){
  1610   1621         int iCol = pTerm->u.leftColumn;
  1611   1622         Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol);
  1612   1623         testcase( iCol==BMS );
  1613   1624         testcase( iCol==BMS-1 );
  1614   1625         if( !sentWarning ){
  1615   1626           sqlite3_log(SQLITE_WARNING_AUTOINDEX,
  1616   1627               "automatic index on %s(%s)", pTable->zName,
  1617   1628               pTable->aCol[iCol].zName);
  1618   1629           sentWarning = 1;
  1619   1630         }
  1620   1631         if( (idxCols & cMask)==0 ){
  1621         -        if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ) return;
         1632  +        if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ){
         1633  +          goto end_auto_index_create;
         1634  +        }
  1622   1635           pLoop->aLTerm[nKeyCol++] = pTerm;
  1623   1636           idxCols |= cMask;
  1624   1637         }
  1625   1638       }
  1626   1639     }
  1627   1640     assert( nKeyCol>0 );
  1628   1641     pLoop->u.btree.nEq = pLoop->nLTerm = nKeyCol;
................................................................................
  1647   1660     if( pSrc->colUsed & MASKBIT(BMS-1) ){
  1648   1661       nKeyCol += pTable->nCol - BMS + 1;
  1649   1662     }
  1650   1663     pLoop->wsFlags |= WHERE_COLUMN_EQ | WHERE_IDX_ONLY;
  1651   1664   
  1652   1665     /* Construct the Index object to describe this index */
  1653   1666     pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+1, 0, &zNotUsed);
  1654         -  if( pIdx==0 ) return;
         1667  +  if( pIdx==0 ) goto end_auto_index_create;
  1655   1668     pLoop->u.btree.pIndex = pIdx;
  1656   1669     pIdx->zName = "auto-index";
  1657   1670     pIdx->pTable = pTable;
  1658   1671     n = 0;
  1659   1672     idxCols = 0;
  1660   1673     for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){
  1661   1674       if( termCanDriveIndex(pTerm, pSrc, notReady) ){
................................................................................
  1699   1712     assert( pLevel->iIdxCur>=0 );
  1700   1713     pLevel->iIdxCur = pParse->nTab++;
  1701   1714     sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1);
  1702   1715     sqlite3VdbeSetP4KeyInfo(pParse, pIdx);
  1703   1716     VdbeComment((v, "for %s", pTable->zName));
  1704   1717   
  1705   1718     /* Fill the automatic index with content */
         1719  +  sqlite3ExprCachePush(pParse);
  1706   1720     addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v);
         1721  +  if( pPartial ){
         1722  +    iContinue = sqlite3VdbeMakeLabel(v);
         1723  +    sqlite3ExprIfFalse(pParse, pPartial, iContinue, SQLITE_JUMPIFNULL);
         1724  +    pLoop->wsFlags |= WHERE_PARTIALIDX;
         1725  +  }
  1707   1726     regRecord = sqlite3GetTempReg(pParse);
  1708   1727     sqlite3GenerateIndexKey(pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0);
  1709   1728     sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord);
  1710   1729     sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
         1730  +  if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue);
  1711   1731     sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v);
  1712   1732     sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX);
  1713   1733     sqlite3VdbeJumpHere(v, addrTop);
  1714   1734     sqlite3ReleaseTempReg(pParse, regRecord);
         1735  +  sqlite3ExprCachePop(pParse);
  1715   1736     
  1716   1737     /* Jump here when skipping the initialization */
  1717   1738     sqlite3VdbeJumpHere(v, addrInit);
         1739  +
         1740  +end_auto_index_create:
         1741  +  sqlite3ExprDelete(pParse->db, pPartial);
  1718   1742   }
  1719   1743   #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */
  1720   1744   
  1721   1745   #ifndef SQLITE_OMIT_VIRTUALTABLE
  1722   1746   /*
  1723   1747   ** Allocate and populate an sqlite3_index_info structure. It is the 
  1724   1748   ** responsibility of the caller to eventually release the structure
................................................................................
  2155   2179   
  2156   2180   #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
  2157   2181     Index *p = pLoop->u.btree.pIndex;
  2158   2182     int nEq = pLoop->u.btree.nEq;
  2159   2183   
  2160   2184     if( p->nSample>0
  2161   2185      && nEq<p->nSampleCol
  2162         -   && OptimizationEnabled(pParse->db, SQLITE_Stat3) 
  2163   2186     ){
  2164   2187       if( nEq==pBuilder->nRecValid ){
  2165   2188         UnpackedRecord *pRec = pBuilder->pRec;
  2166   2189         tRowcnt a[2];
  2167   2190         u8 aff;
  2168   2191   
  2169   2192         /* Variable iLower will be set to the estimate of the number of rows in 
................................................................................
  2194   2217           aff = SQLITE_AFF_INTEGER;
  2195   2218         }else{
  2196   2219           aff = p->pTable->aCol[p->aiColumn[nEq]].affinity;
  2197   2220         }
  2198   2221         /* Determine iLower and iUpper using ($P) only. */
  2199   2222         if( nEq==0 ){
  2200   2223           iLower = 0;
  2201         -        iUpper = sqlite3LogEstToInt(p->aiRowLogEst[0]);
         2224  +        iUpper = p->nRowEst0;
  2202   2225         }else{
  2203   2226           /* Note: this call could be optimized away - since the same values must 
  2204   2227           ** have been requested when testing key $P in whereEqualScanEst().  */
  2205   2228           whereKeyStats(pParse, p, pRec, 0, a);
  2206   2229           iLower = a[0];
  2207   2230           iUpper = a[0] + a[1];
  2208   2231         }
................................................................................
  2269   2292     UNUSED_PARAMETER(pBuilder);
  2270   2293     assert( pLower || pUpper );
  2271   2294   #endif
  2272   2295     assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 );
  2273   2296     nNew = whereRangeAdjust(pLower, nOut);
  2274   2297     nNew = whereRangeAdjust(pUpper, nNew);
  2275   2298   
  2276         -  /* TUNING: If there is both an upper and lower limit, assume the range is
         2299  +  /* TUNING: If there is both an upper and lower limit and neither limit
         2300  +  ** has an application-defined likelihood(), assume the range is
  2277   2301     ** reduced by an additional 75%. This means that, by default, an open-ended
  2278   2302     ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the
  2279   2303     ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to
  2280   2304     ** match 1/64 of the index. */ 
  2281         -  if( pLower && pUpper ) nNew -= 20;
         2305  +  if( pLower && pLower->truthProb>0 && pUpper && pUpper->truthProb>0 ){
         2306  +    nNew -= 20;
         2307  +  }
  2282   2308   
  2283   2309     nOut -= (pLower!=0) + (pUpper!=0);
  2284   2310     if( nNew<10 ) nNew = 10;
  2285   2311     if( nNew<nOut ) nOut = nNew;
  2286   2312   #if defined(WHERETRACE_ENABLED)
  2287   2313     if( pLoop->nOut>nOut ){
  2288   2314       WHERETRACE(0x10,("Range scan lowers nOut from %d to %d\n",
................................................................................
  2634   2660     int nReg;                     /* Number of registers to allocate */
  2635   2661     char *zAff;                   /* Affinity string to return */
  2636   2662   
  2637   2663     /* This module is only called on query plans that use an index. */
  2638   2664     pLoop = pLevel->pWLoop;
  2639   2665     assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 );
  2640   2666     nEq = pLoop->u.btree.nEq;
  2641         -  nSkip = pLoop->u.btree.nSkip;
         2667  +  nSkip = pLoop->nSkip;
  2642   2668     pIdx = pLoop->u.btree.pIndex;
  2643   2669     assert( pIdx!=0 );
  2644   2670   
  2645   2671     /* Figure out how many memory cells we will need then allocate them.
  2646   2672     */
  2647   2673     regBase = pParse->nMem + 1;
  2648   2674     nReg = pLoop->u.btree.nEq + nExtraReg;
................................................................................
  2748   2774   ** string similar to:
  2749   2775   **
  2750   2776   **   "a=? AND b>?"
  2751   2777   */
  2752   2778   static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop, Table *pTab){
  2753   2779     Index *pIndex = pLoop->u.btree.pIndex;
  2754   2780     u16 nEq = pLoop->u.btree.nEq;
  2755         -  u16 nSkip = pLoop->u.btree.nSkip;
         2781  +  u16 nSkip = pLoop->nSkip;
  2756   2782     int i, j;
  2757   2783     Column *aCol = pTab->aCol;
  2758   2784     i16 *aiColumn = pIndex->aiColumn;
  2759   2785   
  2760   2786     if( nEq==0 && (pLoop->wsFlags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ) return;
  2761   2787     sqlite3StrAccumAppend(pStr, " (", 2);
  2762   2788     for(i=0; i<nEq; i++){
................................................................................
  2837   2863         assert( pLoop->u.btree.pIndex!=0 );
  2838   2864         pIdx = pLoop->u.btree.pIndex;
  2839   2865         assert( !(flags&WHERE_AUTO_INDEX) || (flags&WHERE_IDX_ONLY) );
  2840   2866         if( !HasRowid(pItem->pTab) && IsPrimaryKeyIndex(pIdx) ){
  2841   2867           if( isSearch ){
  2842   2868             zFmt = "PRIMARY KEY";
  2843   2869           }
         2870  +      }else if( flags & WHERE_PARTIALIDX ){
         2871  +        zFmt = "AUTOMATIC PARTIAL COVERING INDEX";
  2844   2872         }else if( flags & WHERE_AUTO_INDEX ){
  2845   2873           zFmt = "AUTOMATIC COVERING INDEX";
  2846   2874         }else if( flags & WHERE_IDX_ONLY ){
  2847   2875           zFmt = "COVERING INDEX %s";
  2848   2876         }else{
  2849   2877           zFmt = "INDEX %s";
  2850   2878         }
................................................................................
  3185   3213       char *zStartAff;             /* Affinity for start of range constraint */
  3186   3214       char cEndAff = 0;            /* Affinity for end of range constraint */
  3187   3215       u8 bSeekPastNull = 0;        /* True to seek past initial nulls */
  3188   3216       u8 bStopAtNull = 0;          /* Add condition to terminate at NULLs */
  3189   3217   
  3190   3218       pIdx = pLoop->u.btree.pIndex;
  3191   3219       iIdxCur = pLevel->iIdxCur;
  3192         -    assert( nEq>=pLoop->u.btree.nSkip );
         3220  +    assert( nEq>=pLoop->nSkip );
  3193   3221   
  3194   3222       /* If this loop satisfies a sort order (pOrderBy) request that 
  3195   3223       ** was passed to this function to implement a "SELECT min(x) ..." 
  3196   3224       ** query, then the caller will only allow the loop to run for
  3197   3225       ** a single iteration. This means that the first row returned
  3198   3226       ** should not have a NULL value stored in 'x'. If column 'x' is
  3199   3227       ** the first one after the nEq equality constraints in the index,
................................................................................
  3202   3230       assert( pWInfo->pOrderBy==0
  3203   3231            || pWInfo->pOrderBy->nExpr==1
  3204   3232            || (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 );
  3205   3233       if( (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)!=0
  3206   3234        && pWInfo->nOBSat>0
  3207   3235        && (pIdx->nKeyCol>nEq)
  3208   3236       ){
  3209         -      assert( pLoop->u.btree.nSkip==0 );
         3237  +      assert( pLoop->nSkip==0 );
  3210   3238         bSeekPastNull = 1;
  3211   3239         nExtraReg = 1;
  3212   3240       }
  3213   3241   
  3214   3242       /* Find any inequality constraint terms for the start and end 
  3215   3243       ** of the range. 
  3216   3244       */
................................................................................
  3823   3851       }else{
  3824   3852         z = sqlite3_mprintf("(%d,%x)", p->u.vtab.idxNum, p->u.vtab.omitMask);
  3825   3853       }
  3826   3854       sqlite3DebugPrintf(" %-19s", z);
  3827   3855       sqlite3_free(z);
  3828   3856     }
  3829   3857     if( p->wsFlags & WHERE_SKIPSCAN ){
  3830         -    sqlite3DebugPrintf(" f %05x %d-%d", p->wsFlags, p->nLTerm,p->u.btree.nSkip);
         3858  +    sqlite3DebugPrintf(" f %05x %d-%d", p->wsFlags, p->nLTerm,p->nSkip);
  3831   3859     }else{
  3832   3860       sqlite3DebugPrintf(" f %05x N %d", p->wsFlags, p->nLTerm);
  3833   3861     }
  3834   3862     sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut);
  3835   3863     if( p->nLTerm && (sqlite3WhereTrace & 0x100)!=0 ){
  3836   3864       int i;
  3837   3865       for(i=0; i<p->nLTerm; i++){
................................................................................
  3952   3980   ** relationship is inverted and needs to be adjusted.
  3953   3981   */
  3954   3982   static int whereLoopCheaperProperSubset(
  3955   3983     const WhereLoop *pX,       /* First WhereLoop to compare */
  3956   3984     const WhereLoop *pY        /* Compare against this WhereLoop */
  3957   3985   ){
  3958   3986     int i, j;
  3959         -  if( pX->nLTerm >= pY->nLTerm ) return 0; /* X is not a subset of Y */
         3987  +  if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){
         3988  +    return 0; /* X is not a subset of Y */
         3989  +  }
  3960   3990     if( pX->rRun >= pY->rRun ){
  3961   3991       if( pX->rRun > pY->rRun ) return 0;    /* X costs more than Y */
  3962   3992       if( pX->nOut > pY->nOut ) return 0;    /* X costs more than Y */
  3963   3993     }
  3964   3994     for(i=pX->nLTerm-1; i>=0; i--){
         3995  +    if( pX->aLTerm[i]==0 ) continue;
  3965   3996       for(j=pY->nLTerm-1; j>=0; j--){
  3966   3997         if( pY->aLTerm[j]==pX->aLTerm[i] ) break;
  3967   3998       }
  3968   3999       if( j<0 ) return 0;  /* X not a subset of Y since term X[i] not used by Y */
  3969   4000     }
  3970   4001     return 1;  /* All conditions meet */
  3971   4002   }
................................................................................
  3979   4010   **
  3980   4011   **   (2) pTemplate costs more than any other WhereLoops for which pTemplate
  3981   4012   **       is a proper subset.
  3982   4013   **
  3983   4014   ** To say "WhereLoop X is a proper subset of Y" means that X uses fewer
  3984   4015   ** WHERE clause terms than Y and that every WHERE clause term used by X is
  3985   4016   ** also used by Y.
  3986         -**
  3987         -** This adjustment is omitted for SKIPSCAN loops.  In a SKIPSCAN loop, the
  3988         -** WhereLoop.nLTerm field is not an accurate measure of the number of WHERE
  3989         -** clause terms covered, since some of the first nLTerm entries in aLTerm[]
  3990         -** will be NULL (because they are skipped).  That makes it more difficult
  3991         -** to compare the loops.  We could add extra code to do the comparison, and
  3992         -** perhaps we will someday.  But SKIPSCAN is sufficiently uncommon, and this
  3993         -** adjustment is sufficient minor, that it is very difficult to construct
  3994         -** a test case where the extra code would improve the query plan.  Better
  3995         -** to avoid the added complexity and just omit cost adjustments to SKIPSCAN
  3996         -** loops.
  3997   4017   */
  3998   4018   static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){
  3999   4019     if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return;
  4000         -  if( (pTemplate->wsFlags & WHERE_SKIPSCAN)!=0 ) return;
  4001   4020     for(; p; p=p->pNextLoop){
  4002   4021       if( p->iTab!=pTemplate->iTab ) continue;
  4003   4022       if( (p->wsFlags & WHERE_INDEXED)==0 ) continue;
  4004         -    if( (p->wsFlags & WHERE_SKIPSCAN)!=0 ) continue;
  4005   4023       if( whereLoopCheaperProperSubset(p, pTemplate) ){
  4006   4024         /* Adjust pTemplate cost downward so that it is cheaper than its 
  4007         -      ** subset p */
         4025  +      ** subset p.  Except, do not adjust the cost estimate downward for
         4026  +      ** a loop that skips more columns. */
         4027  +      if( pTemplate->nSkip>p->nSkip ) continue;
         4028  +      WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n",
         4029  +                       pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut-1));
  4008   4030         pTemplate->rRun = p->rRun;
  4009   4031         pTemplate->nOut = p->nOut - 1;
  4010   4032       }else if( whereLoopCheaperProperSubset(pTemplate, p) ){
  4011   4033         /* Adjust pTemplate cost upward so that it is costlier than p since
  4012   4034         ** pTemplate is a proper subset of p */
         4035  +      WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n",
         4036  +                       pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut+1));
  4013   4037         pTemplate->rRun = p->rRun;
  4014   4038         pTemplate->nOut = p->nOut + 1;
  4015   4039       }
  4016   4040     }
  4017   4041   }
  4018   4042   
  4019   4043   /*
................................................................................
  4291   4315     WhereLoop *pNew;                /* Template WhereLoop under construction */
  4292   4316     WhereTerm *pTerm;               /* A WhereTerm under consideration */
  4293   4317     int opMask;                     /* Valid operators for constraints */
  4294   4318     WhereScan scan;                 /* Iterator for WHERE terms */
  4295   4319     Bitmask saved_prereq;           /* Original value of pNew->prereq */
  4296   4320     u16 saved_nLTerm;               /* Original value of pNew->nLTerm */
  4297   4321     u16 saved_nEq;                  /* Original value of pNew->u.btree.nEq */
  4298         -  u16 saved_nSkip;                /* Original value of pNew->u.btree.nSkip */
         4322  +  u16 saved_nSkip;                /* Original value of pNew->nSkip */
  4299   4323     u32 saved_wsFlags;              /* Original value of pNew->wsFlags */
  4300   4324     LogEst saved_nOut;              /* Original value of pNew->nOut */
  4301   4325     int iCol;                       /* Index of the column in the table */
  4302   4326     int rc = SQLITE_OK;             /* Return code */
  4303   4327     LogEst rSize;                   /* Number of rows in the table */
  4304   4328     LogEst rLogSize;                /* Logarithm of table size */
  4305   4329     WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */
................................................................................
  4320   4344   
  4321   4345     assert( pNew->u.btree.nEq<pProbe->nColumn );
  4322   4346     iCol = pProbe->aiColumn[pNew->u.btree.nEq];
  4323   4347   
  4324   4348     pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol,
  4325   4349                           opMask, pProbe);
  4326   4350     saved_nEq = pNew->u.btree.nEq;
  4327         -  saved_nSkip = pNew->u.btree.nSkip;
         4351  +  saved_nSkip = pNew->nSkip;
  4328   4352     saved_nLTerm = pNew->nLTerm;
  4329   4353     saved_wsFlags = pNew->wsFlags;
  4330   4354     saved_prereq = pNew->prereq;
  4331   4355     saved_nOut = pNew->nOut;
  4332   4356     pNew->rSetup = 0;
  4333   4357     rSize = pProbe->aiRowLogEst[0];
  4334   4358     rLogSize = estLog(rSize);
  4335         -
  4336         -  /* Consider using a skip-scan if there are no WHERE clause constraints
  4337         -  ** available for the left-most terms of the index, and if the average
  4338         -  ** number of repeats in the left-most terms is at least 18. 
  4339         -  **
  4340         -  ** The magic number 18 is selected on the basis that scanning 17 rows
  4341         -  ** is almost always quicker than an index seek (even though if the index
  4342         -  ** contains fewer than 2^17 rows we assume otherwise in other parts of
  4343         -  ** the code). And, even if it is not, it should not be too much slower. 
  4344         -  ** On the other hand, the extra seeks could end up being significantly
  4345         -  ** more expensive.  */
  4346         -  assert( 42==sqlite3LogEst(18) );
  4347         -  if( saved_nEq==saved_nSkip
  4348         -   && saved_nEq+1<pProbe->nKeyCol
  4349         -   && pProbe->aiRowLogEst[saved_nEq+1]>=42  /* TUNING: Minimum for skip-scan */
  4350         -   && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK
  4351         -  ){
  4352         -    LogEst nIter;
  4353         -    pNew->u.btree.nEq++;
  4354         -    pNew->u.btree.nSkip++;
  4355         -    pNew->aLTerm[pNew->nLTerm++] = 0;
  4356         -    pNew->wsFlags |= WHERE_SKIPSCAN;
  4357         -    nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1];
  4358         -    if( pTerm ){
  4359         -      /* TUNING:  When estimating skip-scan for a term that is also indexable,
  4360         -      ** multiply the cost of the skip-scan by 2.0, to make it a little less
  4361         -      ** desirable than the regular index lookup. */
  4362         -      nIter += 10;  assert( 10==sqlite3LogEst(2) );
  4363         -    }
  4364         -    pNew->nOut -= nIter;
  4365         -    /* TUNING:  Because uncertainties in the estimates for skip-scan queries,
  4366         -    ** add a 1.375 fudge factor to make skip-scan slightly less likely. */
  4367         -    nIter += 5;
  4368         -    whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul);
  4369         -    pNew->nOut = saved_nOut;
  4370         -    pNew->u.btree.nEq = saved_nEq;
  4371         -    pNew->u.btree.nSkip = saved_nSkip;
  4372         -  }
  4373   4359     for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){
  4374   4360       u16 eOp = pTerm->eOperator;   /* Shorthand for pTerm->eOperator */
  4375   4361       LogEst rCostIdx;
  4376   4362       LogEst nOutUnadjusted;        /* nOut before IN() and WHERE adjustments */
  4377   4363       int nIn = 0;
  4378   4364   #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
  4379   4365       int nRecValid = pBuilder->nRecValid;
................................................................................
  4460   4446           pNew->nOut -= nIn;
  4461   4447         }else{
  4462   4448   #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
  4463   4449           tRowcnt nOut = 0;
  4464   4450           if( nInMul==0 
  4465   4451            && pProbe->nSample 
  4466   4452            && pNew->u.btree.nEq<=pProbe->nSampleCol
  4467         -         && OptimizationEnabled(db, SQLITE_Stat3) 
  4468   4453            && ((eOp & WO_IN)==0 || !ExprHasProperty(pTerm->pExpr, EP_xIsSelect))
  4469   4454           ){
  4470   4455             Expr *pExpr = pTerm->pExpr;
  4471   4456             if( (eOp & (WO_EQ|WO_ISNULL))!=0 ){
  4472   4457               testcase( eOp & WO_EQ );
  4473   4458               testcase( eOp & WO_ISNULL );
  4474   4459               rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut);
................................................................................
  4528   4513       pNew->nOut = saved_nOut;
  4529   4514   #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
  4530   4515       pBuilder->nRecValid = nRecValid;
  4531   4516   #endif
  4532   4517     }
  4533   4518     pNew->prereq = saved_prereq;
  4534   4519     pNew->u.btree.nEq = saved_nEq;
  4535         -  pNew->u.btree.nSkip = saved_nSkip;
         4520  +  pNew->nSkip = saved_nSkip;
  4536   4521     pNew->wsFlags = saved_wsFlags;
  4537   4522     pNew->nOut = saved_nOut;
  4538   4523     pNew->nLTerm = saved_nLTerm;
         4524  +
         4525  +  /* Consider using a skip-scan if there are no WHERE clause constraints
         4526  +  ** available for the left-most terms of the index, and if the average
         4527  +  ** number of repeats in the left-most terms is at least 18. 
         4528  +  **
         4529  +  ** The magic number 18 is selected on the basis that scanning 17 rows
         4530  +  ** is almost always quicker than an index seek (even though if the index
         4531  +  ** contains fewer than 2^17 rows we assume otherwise in other parts of
         4532  +  ** the code). And, even if it is not, it should not be too much slower. 
         4533  +  ** On the other hand, the extra seeks could end up being significantly
         4534  +  ** more expensive.  */
         4535  +  assert( 42==sqlite3LogEst(18) );
         4536  +  if( saved_nEq==saved_nSkip
         4537  +   && saved_nEq+1<pProbe->nKeyCol
         4538  +   && pProbe->aiRowLogEst[saved_nEq+1]>=42  /* TUNING: Minimum for skip-scan */
         4539  +   && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK
         4540  +  ){
         4541  +    LogEst nIter;
         4542  +    pNew->u.btree.nEq++;
         4543  +    pNew->nSkip++;
         4544  +    pNew->aLTerm[pNew->nLTerm++] = 0;
         4545  +    pNew->wsFlags |= WHERE_SKIPSCAN;
         4546  +    nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1];
         4547  +    pNew->nOut -= nIter;
         4548  +    /* TUNING:  Because uncertainties in the estimates for skip-scan queries,
         4549  +    ** add a 1.375 fudge factor to make skip-scan slightly less likely. */
         4550  +    nIter += 5;
         4551  +    whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul);
         4552  +    pNew->nOut = saved_nOut;
         4553  +    pNew->u.btree.nEq = saved_nEq;
         4554  +    pNew->nSkip = saved_nSkip;
         4555  +    pNew->wsFlags = saved_wsFlags;
         4556  +  }
         4557  +
  4539   4558     return rc;
  4540   4559   }
  4541   4560   
  4542   4561   /*
  4543   4562   ** Return True if it is possible that pIndex might be useful in
  4544   4563   ** implementing the ORDER BY clause in pBuilder.
  4545   4564   **
................................................................................
  4710   4729       /* Generate auto-index WhereLoops */
  4711   4730       WhereTerm *pTerm;
  4712   4731       WhereTerm *pWCEnd = pWC->a + pWC->nTerm;
  4713   4732       for(pTerm=pWC->a; rc==SQLITE_OK && pTerm<pWCEnd; pTerm++){
  4714   4733         if( pTerm->prereqRight & pNew->maskSelf ) continue;
  4715   4734         if( termCanDriveIndex(pTerm, pSrc, 0) ){
  4716   4735           pNew->u.btree.nEq = 1;
  4717         -        pNew->u.btree.nSkip = 0;
         4736  +        pNew->nSkip = 0;
  4718   4737           pNew->u.btree.pIndex = 0;
  4719   4738           pNew->nLTerm = 1;
  4720   4739           pNew->aLTerm[0] = pTerm;
  4721   4740           /* TUNING: One-time cost for computing the automatic index is
  4722   4741           ** estimated to be X*N*log2(N) where N is the number of rows in
  4723   4742           ** the table being indexed and where X is 7 (LogEst=28) for normal
  4724   4743           ** tables or 1.375 (LogEst=4) for views and subqueries.  The value
................................................................................
  4751   4770       if( pProbe->pPartIdxWhere!=0
  4752   4771        && !whereUsablePartialIndex(pSrc->iCursor, pWC, pProbe->pPartIdxWhere) ){
  4753   4772         testcase( pNew->iTab!=pSrc->iCursor );  /* See ticket [98d973b8f5] */
  4754   4773         continue;  /* Partial index inappropriate for this query */
  4755   4774       }
  4756   4775       rSize = pProbe->aiRowLogEst[0];
  4757   4776       pNew->u.btree.nEq = 0;
  4758         -    pNew->u.btree.nSkip = 0;
         4777  +    pNew->nSkip = 0;
  4759   4778       pNew->nLTerm = 0;
  4760   4779       pNew->iSortIdx = 0;
  4761   4780       pNew->rSetup = 0;
  4762   4781       pNew->prereq = mExtra;
  4763   4782       pNew->nOut = rSize;
  4764   4783       pNew->u.btree.pIndex = pProbe;
  4765   4784       b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor);
................................................................................
  5301   5320         rev = revSet = 0;
  5302   5321         distinctColumns = 0;
  5303   5322         for(j=0; j<nColumn; j++){
  5304   5323           u8 bOnce;   /* True to run the ORDER BY search loop */
  5305   5324   
  5306   5325           /* Skip over == and IS NULL terms */
  5307   5326           if( j<pLoop->u.btree.nEq
  5308         -         && pLoop->u.btree.nSkip==0
         5327  +         && pLoop->nSkip==0
  5309   5328            && ((i = pLoop->aLTerm[j]->eOperator) & (WO_EQ|WO_ISNULL))!=0
  5310   5329           ){
  5311   5330             if( i & WO_ISNULL ){
  5312   5331               testcase( isOrderDistinct );
  5313   5332               isOrderDistinct = 0;
  5314   5333             }
  5315   5334             continue;  
................................................................................
  5755   5774               }
  5756   5775             }
  5757   5776           }
  5758   5777         }
  5759   5778       }
  5760   5779   
  5761   5780   #ifdef WHERETRACE_ENABLED  /* >=2 */
  5762         -    if( sqlite3WhereTrace>=2 ){
         5781  +    if( sqlite3WhereTrace & 0x02 ){
  5763   5782         sqlite3DebugPrintf("---- after round %d ----\n", iLoop);
  5764   5783         for(ii=0, pTo=aTo; ii<nTo; ii++, pTo++){
  5765   5784           sqlite3DebugPrintf(" %s cost=%-3d nrow=%-3d order=%c",
  5766   5785              wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
  5767   5786              pTo->isOrdered>=0 ? (pTo->isOrdered+'0') : '?');
  5768   5787           if( pTo->isOrdered>0 ){
  5769   5788             sqlite3DebugPrintf(" rev=0x%llx\n", pTo->revLoop);
................................................................................
  5874   5893     pTab = pItem->pTab;
  5875   5894     if( IsVirtual(pTab) ) return 0;
  5876   5895     if( pItem->zIndex ) return 0;
  5877   5896     iCur = pItem->iCursor;
  5878   5897     pWC = &pWInfo->sWC;
  5879   5898     pLoop = pBuilder->pNew;
  5880   5899     pLoop->wsFlags = 0;
  5881         -  pLoop->u.btree.nSkip = 0;
         5900  +  pLoop->nSkip = 0;
  5882   5901     pTerm = findTerm(pWC, iCur, -1, 0, WO_EQ, 0);
  5883   5902     if( pTerm ){
  5884   5903       pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_IPK|WHERE_ONEROW;
  5885   5904       pLoop->aLTerm[0] = pTerm;
  5886   5905       pLoop->nLTerm = 1;
  5887   5906       pLoop->u.btree.nEq = 1;
  5888   5907       /* TUNING: Cost of a rowid lookup is 10 */
  5889   5908       pLoop->rRun = 33;  /* 33==sqlite3LogEst(10) */
  5890   5909     }else{
  5891   5910       for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
  5892   5911         assert( pLoop->aLTermSpace==pLoop->aLTerm );
  5893         -      assert( ArraySize(pLoop->aLTermSpace)==4 );
  5894   5912         if( !IsUniqueIndex(pIdx)
  5895   5913          || pIdx->pPartIdxWhere!=0 
  5896   5914          || pIdx->nKeyCol>ArraySize(pLoop->aLTermSpace) 
  5897   5915         ) continue;
  5898   5916         for(j=0; j<pIdx->nKeyCol; j++){
  5899   5917           pTerm = findTerm(pWC, iCur, pIdx->aiColumn[j], 0, WO_EQ, pIdx);
  5900   5918           if( pTerm==0 ) break;

Changes to src/whereInt.h.

   111    111     u8 iSortIdx;          /* Sorting index number.  0==None */
   112    112     LogEst rSetup;        /* One-time setup cost (ex: create transient index) */
   113    113     LogEst rRun;          /* Cost of running each loop */
   114    114     LogEst nOut;          /* Estimated number of output rows */
   115    115     union {
   116    116       struct {               /* Information for internal btree tables */
   117    117         u16 nEq;               /* Number of equality constraints */
   118         -      u16 nSkip;             /* Number of initial index columns to skip */
   119    118         Index *pIndex;         /* Index used, or NULL */
   120    119       } btree;
   121    120       struct {               /* Information for virtual tables */
   122    121         int idxNum;            /* Index number */
   123    122         u8 needFree;           /* True if sqlite3_free(idxStr) is needed */
   124    123         i8 isOrdered;          /* True if satisfies ORDER BY */
   125    124         u16 omitMask;          /* Terms that may be omitted */
   126    125         char *idxStr;          /* Index identifier string */
   127    126       } vtab;
   128    127     } u;
   129    128     u32 wsFlags;          /* WHERE_* flags describing the plan */
   130    129     u16 nLTerm;           /* Number of entries in aLTerm[] */
          130  +  u16 nSkip;            /* Number of NULL aLTerm[] entries */
   131    131     /**** whereLoopXfer() copies fields above ***********************/
   132    132   # define WHERE_LOOP_XFER_SZ offsetof(WhereLoop,nLSlot)
   133    133     u16 nLSlot;           /* Number of slots allocated for aLTerm[] */
   134    134     WhereTerm **aLTerm;   /* WhereTerms used */
   135    135     WhereLoop *pNextLoop; /* Next WhereLoop object in the WhereClause */
   136         -  WhereTerm *aLTermSpace[4];  /* Initial aLTerm[] space */
          136  +  WhereTerm *aLTermSpace[3];  /* Initial aLTerm[] space */
   137    137   };
   138    138   
   139    139   /* This object holds the prerequisites and the cost of running a
   140    140   ** subquery on one operand of an OR operator in the WHERE clause.
   141    141   ** See WhereOrSet for additional information 
   142    142   */
   143    143   struct WhereOrCost {
................................................................................
   455    455   #define WHERE_VIRTUALTABLE 0x00000400  /* WhereLoop.u.vtab is valid */
   456    456   #define WHERE_IN_ABLE      0x00000800  /* Able to support an IN operator */
   457    457   #define WHERE_ONEROW       0x00001000  /* Selects no more than one row */
   458    458   #define WHERE_MULTI_OR     0x00002000  /* OR using multiple indices */
   459    459   #define WHERE_AUTO_INDEX   0x00004000  /* Uses an ephemeral index */
   460    460   #define WHERE_SKIPSCAN     0x00008000  /* Uses the skip-scan algorithm */
   461    461   #define WHERE_UNQ_WANTED   0x00010000  /* WHERE_ONEROW would have been helpful*/
          462  +#define WHERE_PARTIALIDX   0x00020000  /* The automatic index is partial */

Added test/autoindex4.test.

            1  +# 2014-10-24
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +#
           12  +# This file implements regression tests for SQLite library.  The
           13  +# focus of this script is testing automatic index creation logic,
           14  +# and specifically creation of automatic partial indexes.
           15  +#
           16  +
           17  +set testdir [file dirname $argv0]
           18  +source $testdir/tester.tcl
           19  +
           20  +do_execsql_test autoindex4-1.0 {
           21  +  CREATE TABLE t1(a,b);
           22  +  INSERT INTO t1 VALUES(123,'abc'),(234,'def'),(234,'ghi'),(345,'jkl');
           23  +  CREATE TABLE t2(x,y);
           24  +  INSERT INTO t2 VALUES(987,'zyx'),(654,'wvu'),(987,'rqp');
           25  +
           26  +  SELECT *, '|' FROM t1, t2 WHERE a=234 AND x=987 ORDER BY +b;
           27  +} {234 def 987 rqp | 234 def 987 zyx | 234 ghi 987 rqp | 234 ghi 987 zyx |}
           28  +do_execsql_test autoindex4-1.1 {
           29  +  SELECT *, '|' FROM t1, t2 WHERE a=234 AND x=555;
           30  +} {}
           31  +
           32  +do_execsql_test autoindex4-1.2 {
           33  +  SELECT *, '|' FROM t1 LEFT JOIN t2 ON a=234 AND x=555;
           34  +} {123 abc {} {} | 234 def {} {} | 234 ghi {} {} | 345 jkl {} {} |}
           35  +do_execsql_test autoindex4-1.3 {
           36  +  SELECT *, '|' FROM t1 LEFT JOIN t2 ON x=555 WHERE a=234;
           37  +} {234 def {} {} | 234 ghi {} {} |}
           38  +do_execsql_test autoindex4-1.4 {
           39  +  SELECT *, '|' FROM t1 LEFT JOIN t2 WHERE a=234 AND x=555;
           40  +} {}
           41  +
           42  +
           43  +do_execsql_test autoindex4-2.0 {
           44  +  CREATE TABLE t3(e,f);
           45  +  INSERT INTO t3 VALUES(123,654),(555,444),(234,987);
           46  +
           47  +  SELECT (SELECT count(*) FROM t1, t2 WHERE a=e AND x=f), e, f, '|'
           48  +    FROM t3
           49  +   ORDER BY rowid;
           50  +} {1 123 654 | 0 555 444 | 4 234 987 |}
           51  +
           52  +finish_test

Added test/e_wal.test.

            1  +# 2011 May 06
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +
           13  +set testdir [file dirname $argv0]
           14  +source $testdir/tester.tcl
           15  +set testprefix e_wal
           16  +
           17  +db close
           18  +testvfs oldvfs -iversion 1
           19  +
           20  +
           21  +# EVIDENCE-OF: R-58297-14483 WAL databases can be created, read, and
           22  +# written even if shared memory is unavailable as long as the
           23  +# locking_mode is set to EXCLUSIVE before the first attempted access.
           24  +#
           25  +# EVIDENCE-OF: R-00449-33772 This feature allows WAL databases to be
           26  +# created, read, and written by legacy VFSes that lack the "version 2"
           27  +# shared-memory methods xShmMap, xShmLock, xShmBarrier, and xShmUnmap on
           28  +# the sqlite3_io_methods object.
           29  +#
           30  +# 1.1: "create" tests.
           31  +# 1.2: "read" tests.
           32  +# 1.3: "write" tests.
           33  +#
           34  +# All three done with VFS "oldvfs", which has iVersion==1 and so does
           35  +# not support shared memory.
           36  +# 
           37  +sqlite3 db test.db -vfs oldvfs
           38  +do_execsql_test 1.1.1 {
           39  +  PRAGMA journal_mode = WAL;
           40  +} {delete}
           41  +do_execsql_test 1.1.2 {
           42  +  PRAGMA locking_mode = EXCLUSIVE;
           43  +  PRAGMA journal_mode = WAL;
           44  +} {exclusive wal}
           45  +do_execsql_test 1.1.3 {
           46  +  CREATE TABLE t1(x, y);
           47  +  INSERT INTO t1 VALUES(1, 2);
           48  +} {}
           49  +do_test 1.1.4 {
           50  +  list [file exists test.db-shm] [file exists test.db-wal]
           51  +} {0 1}
           52  +
           53  +do_test 1.2.1 {
           54  +  db close
           55  +  sqlite3 db test.db -vfs oldvfs
           56  +  catchsql { SELECT * FROM t1 }
           57  +} {1 {unable to open database file}}
           58  +do_test 1.2.2 {
           59  +  execsql { PRAGMA locking_mode = EXCLUSIVE }
           60  +  execsql { SELECT * FROM t1 }
           61  +} {1 2}
           62  +do_test 1.2.3 {
           63  +  list [file exists test.db-shm] [file exists test.db-wal]
           64  +} {0 1}
           65  +
           66  +do_test 1.3.1 {
           67  +  db close
           68  +  sqlite3 db test.db -vfs oldvfs
           69  +  catchsql { INSERT INTO t1 VALUES(3, 4) }
           70  +} {1 {unable to open database file}}
           71  +do_test 1.3.2 {
           72  +  execsql { PRAGMA locking_mode = EXCLUSIVE }
           73  +  execsql { INSERT INTO t1 VALUES(3, 4) }
           74  +  execsql { SELECT * FROM t1 }
           75  +} {1 2 3 4}
           76  +do_test 1.3.3 {
           77  +  list [file exists test.db-shm] [file exists test.db-wal]
           78  +} {0 1}
           79  +
           80  +# EVIDENCE-OF: R-31969-57825 If EXCLUSIVE locking mode is set prior to
           81  +# the first WAL-mode database access, then SQLite never attempts to call
           82  +# any of the shared-memory methods and hence no shared-memory wal-index
           83  +# is ever created.
           84  +#
           85  +db close
           86  +sqlite3 db test.db
           87  +do_execsql_test 2.1.1 {
           88  +  PRAGMA locking_mode = EXCLUSIVE;
           89  +  SELECT * FROM t1;
           90  +} {exclusive 1 2 3 4}
           91  +do_test 2.1.2 {
           92  +  list [file exists test.db-shm] [file exists test.db-wal]
           93  +} {0 1}
           94  +
           95  +# EVIDENCE-OF: R-36328-16367 In that case, the database connection
           96  +# remains in EXCLUSIVE mode as long as the journal mode is WAL; attempts
           97  +# to change the locking mode using "PRAGMA locking_mode=NORMAL;" are
           98  +# no-ops.
           99  +#
          100  +do_execsql_test 2.2.1 {
          101  +  PRAGMA locking_mode = NORMAL;
          102  +  SELECT * FROM t1;
          103  +} {exclusive 1 2 3 4}
          104  +do_test 2.2.2 {
          105  +  sqlite3 db2 test.db
          106  +  catchsql {SELECT * FROM t1} db2
          107  +} {1 {database is locked}}
          108  +db2 close
          109  +
          110  +# EVIDENCE-OF: R-63522-46088 The only way to change out of EXCLUSIVE
          111  +# locking mode is to first change out of WAL journal mode.
          112  +#
          113  +do_execsql_test 2.3.1 {
          114  +  PRAGMA journal_mode = DELETE;
          115  +  SELECT * FROM t1;
          116  +} {delete 1 2 3 4}
          117  +do_test 2.3.2 {
          118  +  sqlite3 db2 test.db
          119  +  catchsql {SELECT * FROM t1} db2
          120  +} {1 {database is locked}}
          121  +do_execsql_test 2.3.3 {
          122  +  PRAGMA locking_mode = NORMAL;
          123  +  SELECT * FROM t1;
          124  +} {normal 1 2 3 4}
          125  +do_test 2.3.4 {
          126  +  sqlite3 db2 test.db
          127  +  catchsql {SELECT * FROM t1} db2
          128  +} {0 {1 2 3 4}}
          129  +db2 close
          130  +db close
          131  +
          132  +
          133  +# EVIDENCE-OF: R-57239-11845 If NORMAL locking mode is in effect for the
          134  +# first WAL-mode database access, then the shared-memory wal-index is
          135  +# created.
          136  +#
          137  +do_test 3.0 {
          138  +  sqlite3 db test.db
          139  +  execsql { PRAGMA journal_mode = WAL }
          140  +  db close
          141  +} {}
          142  +do_test 3.1 {
          143  +  sqlite3 db test.db
          144  +  execsql { SELECT * FROM t1 }
          145  +  list [file exists test.db-shm] [file exists test.db-wal]
          146  +} {1 1}
          147  +
          148  +# EVIDENCE-OF: R-13779-07711 As long as exactly one connection is using
          149  +# a shared-memory wal-index, the locking mode can be changed freely
          150  +# between NORMAL and EXCLUSIVE.
          151  +#
          152  +do_execsql_test 3.2.1 {
          153  +  PRAGMA locking_mode = EXCLUSIVE;
          154  +  PRAGMA locking_mode = NORMAL;
          155  +  PRAGMA locking_mode = EXCLUSIVE;
          156  +  INSERT INTO t1 VALUES(5, 6);
          157  +} {exclusive normal exclusive}
          158  +do_test 3.2.2 {
          159  +  sqlite3 db2 test.db
          160  +  catchsql { SELECT * FROM t1 } db2
          161  +} {1 {database is locked}}
          162  +
          163  +# EVIDENCE-OF: R-10993-11647 It is only when the shared-memory wal-index
          164  +# is omitted, when the locking mode is EXCLUSIVE prior to the first
          165  +# WAL-mode database access, that the locking mode is stuck in EXCLUSIVE.
          166  +#
          167  +do_execsql_test 3.2.3 {
          168  +  PRAGMA locking_mode = NORMAL;
          169  +  SELECT * FROM t1;
          170  +} {normal 1 2 3 4 5 6}
          171  +do_test 3.2.4 {
          172  +  catchsql { SELECT * FROM t1 } db2
          173  +} {0 {1 2 3 4 5 6}}
          174  +
          175  +do_catchsql_test 3.2.5 {
          176  +  PRAGMA locking_mode = EXCLUSIVE;
          177  +  INSERT INTO t1 VALUES(7, 8);
          178  +} {1 {database is locked}}
          179  +
          180  +db2 close
          181  +
          182  +# EVIDENCE-OF: R-46197-42811 This means that the underlying VFS must
          183  +# support the "version 2" shared-memory.
          184  +#
          185  +# EVIDENCE-OF: R-55316-21772 If the VFS does not support shared-memory
          186  +# methods, then the attempt to open a database that is already in WAL
          187  +# mode, or the attempt convert a database into WAL mode, will fail.
          188  +#
          189  +db close
          190  +do_test 3.4.1 {
          191  +  sqlite3 db test.db -vfs oldvfs
          192  +  catchsql { SELECT * FROM t1 }
          193  +} {1 {unable to open database file}}
          194  +db close
          195  +do_test 3.4.2 {
          196  +  forcedelete test.db2
          197  +  sqlite3 db test.db2 -vfs oldvfs
          198  +  catchsql { PRAGMA journal_mode = WAL }
          199  +} {0 delete}
          200  +db close
          201  +
          202  +
          203  +# EVIDENCE-OF: R-22428-28959 To prevent older versions of SQLite from
          204  +# trying to recover a WAL-mode database (and making matters worse) the
          205  +# database file format version numbers (bytes 18 and 19 in the database
          206  +# header) are increased from 1 to 2 in WAL mode.
          207  +#
          208  +reset_db
          209  +do_execsql_test 4.1.1 { CREATE TABLE t1(x, y) }
          210  +do_test 4.1.2 { hexio_read test.db 18 2 } {0101}
          211  +do_execsql_test 4.1.3 { PRAGMA journal_mode = wAL } {wal}
          212  +do_test 4.1.4 { hexio_read test.db 18 2 } {0202}
          213  +
          214  +
          215  +# EVIDENCE-OF: R-02535-05811 One can explicitly change out of WAL mode
          216  +# using a pragma such as this: PRAGMA journal_mode=DELETE;
          217  +#
          218  +do_execsql_test 4.2.1 { INSERT INTO t1 VALUES(1, 1); } {}
          219  +do_test 4.2.2 { file exists test.db-wal } {1}
          220  +do_execsql_test 4.2.3 { PRAGMA journal_mode = delete } {delete}
          221  +do_test 4.2.4 { file exists test.db-wal } {0}
          222  +
          223  +# EVIDENCE-OF: R-60175-02388 Deliberately changing out of WAL mode
          224  +# changes the database file format version numbers back to 1 so that
          225  +# older versions of SQLite can once again access the database file.
          226  +#
          227  +do_test 4.3 { hexio_read test.db 18 2 } {0101}
          228  +
          229  +finish_test

Added test/skipscan6.test.

            1  +# 2014-10-21
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +# This file implements tests of the "skip-scan" query strategy. In 
           13  +# particular, this file verifies that use of all columns of an index
           14  +# is always preferred over the use of a skip-scan on some columns of
           15  +# the same index.  Because of difficulties in scoring a skip-scan,
           16  +# the skip-scan can sometimes come out with a lower raw score when
           17  +# using STAT4.  But the query planner should detect this and use the
           18  +# full index rather than the skip-scan.
           19  +#
           20  +
           21  +set testdir [file dirname $argv0]
           22  +source $testdir/tester.tcl
           23  +set testprefix skipscan6
           24  +
           25  +ifcapable !stat4 {
           26  +  finish_test
           27  +  return
           28  +}
           29  +
           30  +do_execsql_test 1.1 {
           31  +  CREATE TABLE t1(
           32  +    aa int,
           33  +    bb int,
           34  +    cc int,
           35  +    dd int,
           36  +    ee int
           37  +  );
           38  +  CREATE INDEX ix on t1(aa, bb, cc,  dd DESC);
           39  +  ANALYZE sqlite_master;
           40  +  INSERT INTO sqlite_stat1 VALUES('t1','ix','2695116 1347558 264 18 2');
           41  +  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 196859 196859 32 1','0 15043 15043 92468 92499','0 19 286 81846 92499',X'0609010804031552977BD725BD28');
           42  +  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 14687 161 1 1','0 289067 299306 299457 299457','0 199 6772 273984 299457',X'060902020403013406314D67456415B819');
           43  +  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 19313 19308 22 1','0 325815 325815 343725 343746','0 261 9545 315009 343746',X'060902080403018A49B0A3AD1ED931');
           44  +  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 25047 9051 15 1','0 350443 350443 356590 356604','0 266 9795 325519 356604',X'06090208040301914C2DD2E91F93CF');
           45  +  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 42327 9906 7 1','0 376381 376381 380291 380297','0 268 10100 344232 380297',X'06090208040301934BF672511F7ED3');
           46  +  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 24513 2237 1 1','0 455150 467779 470015 470015','0 286 10880 425401 470015',X'06090202040301A703464A28F2611EF1EE');
           47  +  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 18730 18724 15 1','0 479663 479663 498271 498285','0 287 10998 450793 498285',X'06090208040301A8494AF3A41EC50C');
           48  +  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 119603 47125 1 1','0 572425 572425 598915 598915','0 404 14230 546497 598915',X'06090208040302474FD1929A03194F');
           49  +  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 1454 1454 1 1','0 898346 898346 898373 898373','0 952 31165 827562 898373',X'06090208040304FD53F6A2A2097F64');
           50  +  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 57138 7069 1 1','0 1122389 1122389 1129457 1129457','0 1967 46801 1045943 1129457',X'06090208040309884BC4C52F1F6EB7');
           51  +  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 285 11 1 1','0 1197683 1197824 1197831 1197831','0 2033 50990 1112280 1197831',X'06090202040309D80346503FE2A9038E4F');
           52  +  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 25365 9773 1 1','0 1301013 1301013 1310785 1310785','0 2561 58806 1217877 1310785',X'0609020804030C5F4C8F88AB0AF2A2');
           53  +  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 45180 7222 1 1','0 1326378 1326378 1333599 1333599','0 2562 59921 1240187 1333599',X'0609020804030C604CAB75490B0351');
           54  +  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 8537 41 1 1','0 1496959 1497288 1497289 1497289','0 3050 68246 1394126 1497289',X'0609020204030EA0057F527459B0257C4B');
           55  +  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 26139 26131 17 1','0 1507977 1507977 1520578 1520594','0 3074 69188 1416111 1520594',X'0609020804030EB95169453423D4EA');
           56  +  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 102894 29678 1 1','0 1537421 1550467 1564894 1564894','0 3109 69669 1459820 1564894',X'0609020204030EE3183652A6ED3006EBCB');
           57  +  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 319 3 1 1','0 1796728 1796746 1796747 1796747','0 3650 86468 1682243 1796747',X'0609020204031163033550D0C41018C28D');
           58  +  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 127 127 1 1','0 2096194 2096194 2096205 2096205','0 5145 106437 1951535 2096205',X'060902080403180F53BB1AF727EE50');
           59  +  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 66574 5252 1 1','0 2230524 2265961 2271212 2271212','0 5899 114976 2085829 2271212',X'0609020204031B8A05195009976D223B90');
           60  +  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 19440 19440 1 1','0 2391680 2391680 2395663 2395663','0 6718 123714 2184781 2395663',X'0609020804031F7452E00A7B07431A');
           61  +  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 18321 2177 1 1','0 2522928 2523231 2525407 2525407','0 7838 139084 2299958 2525407',X'06090201040324A7475231103B1AA7B8');
           62  +  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 22384 1361 1 1','0 2541249 2544834 2546194 2546194','0 7839 139428 2308416 2546194',X'06090202040324A8011652323D4B1AA9EB');
           63  +  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 18699 855 1 1','0 2563633 2578178 2579032 2579032','0 7840 139947 2321671 2579032',X'06090202040324A9077452323D7D1052C5');
           64  +  INSERT INTO sqlite_stat4 VALUES('t1','ix','17965 1579 1579 1 1','2677151 2690666 2690666 2692244 2692244','1 9870 153959 2418294 2692244',X'060102080403021B8A4FE1AB84032B35');
           65  +  ANALYZE sqlite_master;
           66  +} {}
           67  +do_execsql_test 1.2 {
           68  +  EXPLAIN QUERY PLAN
           69  +  SELECT COUNT(*)
           70  +    FROM t1
           71  +   WHERE bb=21
           72  +     AND aa=1
           73  +     AND dd BETWEEN 1413833728 and 1413837331;
           74  +} {/INDEX ix .aa=. AND bb=../}
           75  +
           76  +do_execsql_test 2.1 {
           77  +  DROP INDEX ix;
           78  +  CREATE INDEX good on t1(bb, aa, dd DESC);
           79  +  CREATE INDEX bad on t1(aa, bb, cc,  dd DESC);
           80  +  DELETE FROM sqlite_stat1;
           81  +  DELETE FROM sqlite_stat4;
           82  +  INSERT INTO sqlite_stat1 VALUES('t1','good','2695116 299 264 2');
           83  +  INSERT INTO sqlite_stat1 VALUES('t1','bad','2695116 1347558 264 18 2');
           84  +  INSERT INTO sqlite_stat4 VALUES('t1','good','197030 196859 32 1','15086 15086 92511 92536','19 25 81644 92536',X'05010904031552977BD725BD22');
           85  +  INSERT INTO sqlite_stat4 VALUES('t1','good','14972 14687 1 1','289878 289878 299457 299457','199 244 267460 299457',X'050209040301344F7E569402C419');
           86  +  INSERT INTO sqlite_stat4 VALUES('t1','good','19600 19313 22 1','327127 327127 346222 346243','261 319 306884 346243',X'0502090403018A49503BC01EC577');
           87  +  INSERT INTO sqlite_stat4 VALUES('t1','good','25666 25047 15 1','352087 352087 372692 372706','266 327 325601 372706',X'050209040301914C2DD2E91F93CF');
           88  +  INSERT INTO sqlite_stat4 VALUES('t1','good','42392 42327 26 1','378657 378657 382547 382572','268 331 333529 382572',X'05020904030193533B2FE326ED48');
           89  +  INSERT INTO sqlite_stat4 VALUES('t1','good','24619 24513 11 1','457872 457872 461748 461758','286 358 399322 461758',X'050209040301A752B1557825EA7C');
           90  +  INSERT INTO sqlite_stat4 VALUES('t1','good','18969 18730 15 1','482491 482491 501105 501119','287 360 433605 501119',X'050209040301A8494AF3A41EC50C');
           91  +  INSERT INTO sqlite_stat4 VALUES('t1','good','119710 119603 1 1','576500 576500 598915 598915','404 505 519877 598915',X'05020904030247539A7A7912F617');
           92  +  INSERT INTO sqlite_stat4 VALUES('t1','good','11955 11946 1 1','889796 889796 898373 898373','938 1123 794694 898373',X'050209040304EF4DF9C4150BBB28');
           93  +  INSERT INTO sqlite_stat4 VALUES('t1','good','57197 57138 24 1','1129865 1129865 1151492 1151515','1967 2273 1027048 1151515',X'05020904030988533510BC26E20A');
           94  +  INSERT INTO sqlite_stat4 VALUES('t1','good','3609 3543 1 1','1196265 1196265 1197831 1197831','2002 2313 1070108 1197831',X'050209040309B050E95CD718D94D');
           95  +  INSERT INTO sqlite_stat4 VALUES('t1','good','25391 25365 13 1','1309378 1309378 1315567 1315579','2561 2936 1178358 1315579',X'05020904030C5F53DF9E13283570');
           96  +  INSERT INTO sqlite_stat4 VALUES('t1','good','45232 45180 17 1','1334769 1334769 1337946 1337962','2562 2938 1198998 1337962',X'05020904030C60541CACEE28BCAC');
           97  +  INSERT INTO sqlite_stat4 VALUES('t1','good','5496 5493 1 1','1495882 1495882 1497289 1497289','3043 3479 1348695 1497289',X'05020904030E99515C62AD0F0B34');
           98  +  INSERT INTO sqlite_stat4 VALUES('t1','good','26348 26139 17 1','1517381 1517381 1529990 1530006','3074 3519 1378320 1530006',X'05020904030EB95169453423D4EA');
           99  +  INSERT INTO sqlite_stat4 VALUES('t1','good','102927 102894 10 1','1547088 1547088 1649950 1649959','3109 3559 1494260 1649959',X'05020904030EE34D309F671FFA47');
          100  +  INSERT INTO sqlite_stat4 VALUES('t1','good','3602 3576 1 1','1793873 1793873 1796747 1796747','3601 4128 1630783 1796747',X'050209040311294FE88B432219B9');
          101  +  INSERT INTO sqlite_stat4 VALUES('t1','good','154 154 1 1','2096059 2096059 2096205 2096205','5037 5779 1893039 2096205',X'050209040317994EFF05A016DCED');
          102  +  INSERT INTO sqlite_stat4 VALUES('t1','good','68153 66574 60 1','2244039 2244039 2268892 2268951','5899 6749 2027553 2268951',X'05020904031B8A532DBC5A26D2BA');
          103  +  INSERT INTO sqlite_stat4 VALUES('t1','good','321 321 1 1','2395618 2395618 2395663 2395663','6609 7528 2118435 2395663',X'05020904031EFA54078EEE1E2D65');
          104  +  INSERT INTO sqlite_stat4 VALUES('t1','good','19449 19440 22 1','2407769 2407769 2426049 2426070','6718 7651 2146904 2426070',X'05020904031F7450E6118C2336BD');
          105  +  INSERT INTO sqlite_stat4 VALUES('t1','good','18383 18321 56 1','2539949 2539949 2551080 2551135','7838 8897 2245459 2551135',X'050209040324A752EA2E1E2642B2');
          106  +  INSERT INTO sqlite_stat4 VALUES('t1','good','22479 22384 60 1','2558332 2558332 2565233 2565292','7839 8899 2251202 2565292',X'050209040324A853926538279A5F');
          107  +  INSERT INTO sqlite_stat4 VALUES('t1','good','18771 18699 63 1','2580811 2580811 2596914 2596976','7840 8901 2263572 2596976',X'050209040324A9526C1DE9256E72');
          108  +  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 196859 196859 32 1','0 15043 15043 92468 92499','0 19 286 81846 92499',X'0609010804031552977BD725BD28');
          109  +  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 14687 161 1 1','0 289067 299306 299457 299457','0 199 6772 273984 299457',X'060902020403013406314D67456415B819');
          110  +  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 19313 19308 22 1','0 325815 325815 343725 343746','0 261 9545 315009 343746',X'060902080403018A49B0A3AD1ED931');
          111  +  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 25047 9051 15 1','0 350443 350443 356590 356604','0 266 9795 325519 356604',X'06090208040301914C2DD2E91F93CF');
          112  +  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 42327 9906 7 1','0 376381 376381 380291 380297','0 268 10100 344232 380297',X'06090208040301934BF672511F7ED3');
          113  +  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 24513 2237 1 1','0 455150 467779 470015 470015','0 286 10880 425401 470015',X'06090202040301A703464A28F2611EF1EE');
          114  +  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 18730 18724 15 1','0 479663 479663 498271 498285','0 287 10998 450793 498285',X'06090208040301A8494AF3A41EC50C');
          115  +  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 119603 47125 1 1','0 572425 572425 598915 598915','0 404 14230 546497 598915',X'06090208040302474FD1929A03194F');
          116  +  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 1454 1454 1 1','0 898346 898346 898373 898373','0 952 31165 827562 898373',X'06090208040304FD53F6A2A2097F64');
          117  +  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 57138 7069 1 1','0 1122389 1122389 1129457 1129457','0 1967 46801 1045943 1129457',X'06090208040309884BC4C52F1F6EB7');
          118  +  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 285 11 1 1','0 1197683 1197824 1197831 1197831','0 2033 50990 1112280 1197831',X'06090202040309D80346503FE2A9038E4F');
          119  +  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 25365 9773 1 1','0 1301013 1301013 1310785 1310785','0 2561 58806 1217877 1310785',X'0609020804030C5F4C8F88AB0AF2A2');
          120  +  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 45180 7222 1 1','0 1326378 1326378 1333599 1333599','0 2562 59921 1240187 1333599',X'0609020804030C604CAB75490B0351');
          121  +  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 8537 41 1 1','0 1496959 1497288 1497289 1497289','0 3050 68246 1394126 1497289',X'0609020204030EA0057F527459B0257C4B');
          122  +  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 26139 26131 17 1','0 1507977 1507977 1520578 1520594','0 3074 69188 1416111 1520594',X'0609020804030EB95169453423D4EA');
          123  +  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 102894 29678 1 1','0 1537421 1550467 1564894 1564894','0 3109 69669 1459820 1564894',X'0609020204030EE3183652A6ED3006EBCB');
          124  +  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 319 3 1 1','0 1796728 1796746 1796747 1796747','0 3650 86468 1682243 1796747',X'0609020204031163033550D0C41018C28D');
          125  +  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 127 127 1 1','0 2096194 2096194 2096205 2096205','0 5145 106437 1951535 2096205',X'060902080403180F53BB1AF727EE50');
          126  +  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 66574 5252 1 1','0 2230524 2265961 2271212 2271212','0 5899 114976 2085829 2271212',X'0609020204031B8A05195009976D223B90');
          127  +  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 19440 19440 1 1','0 2391680 2391680 2395663 2395663','0 6718 123714 2184781 2395663',X'0609020804031F7452E00A7B07431A');
          128  +  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 18321 2177 1 1','0 2522928 2523231 2525407 2525407','0 7838 139084 2299958 2525407',X'06090201040324A7475231103B1AA7B8');
          129  +  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 22384 1361 1 1','0 2541249 2544834 2546194 2546194','0 7839 139428 2308416 2546194',X'06090202040324A8011652323D4B1AA9EB');
          130  +  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 18699 855 1 1','0 2563633 2578178 2579032 2579032','0 7840 139947 2321671 2579032',X'06090202040324A9077452323D7D1052C5');
          131  +  INSERT INTO sqlite_stat4 VALUES('t1','bad','17965 1579 1579 1 1','2677151 2690666 2690666 2692244 2692244','1 9870 153959 2418294 2692244',X'060102080403021B8A4FE1AB84032B35');
          132  +  ANALYZE sqlite_master;
          133  +} {}
          134  +do_execsql_test 2.2 {
          135  +  EXPLAIN QUERY PLAN
          136  +  SELECT COUNT(*)
          137  +    FROM t1
          138  +   WHERE bb=21
          139  +     AND aa=1
          140  +     AND dd BETWEEN 1413833728 and 1413837331;
          141  +} {/INDEX good .bb=. AND aa=. AND dd>. AND dd<../}
          142  +
          143  +
          144  +
          145  +finish_test