/ Check-in [880df756]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge from trunk recent micro-optimizations and the fix for the DESC index GROUP BY ORDER BY bug.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | apple-osx
Files: files | file ages | folders
SHA1:880df7568add1b5825f713688dcd7a03cd895389
User & Date: drh 2014-10-14 14:17:15
Context
2014-10-17
12:11
Merge the 3.8.7 changes into the apple-osx branch. check-in: aa7bbed1 user: drh tags: apple-osx
2014-10-14
14:17
Merge from trunk recent micro-optimizations and the fix for the DESC index GROUP BY ORDER BY bug. check-in: 880df756 user: drh tags: apple-osx
2014-10-13
23:39
Optimize a database corruption test inside of the OP_Column opcode. check-in: 005e5b38 user: drh tags: trunk
2014-10-09
11:40
Record the errno on fstat() failures. check-in: 2a6a0820 user: drh tags: apple-osx
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

  3112   3112     if( idxNum & FTS3_HAVE_DOCID_GE ) pDocidGe = apVal[iIdx++];
  3113   3113     if( idxNum & FTS3_HAVE_DOCID_LE ) pDocidLe = apVal[iIdx++];
  3114   3114     assert( iIdx==nVal );
  3115   3115   
  3116   3116     /* In case the cursor has been used before, clear it now. */
  3117   3117     sqlite3_finalize(pCsr->pStmt);
  3118   3118     sqlite3_free(pCsr->aDoclist);
         3119  +  sqlite3_free(pCsr->aMatchinfo);
  3119   3120     sqlite3Fts3ExprFree(pCsr->pExpr);
  3120   3121     memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
  3121   3122   
  3122   3123     /* Set the lower and upper bounds on docids to return */
  3123   3124     pCsr->iMinDocid = fts3DocidRange(pDocidGe, SMALLEST_INT64);
  3124   3125     pCsr->iMaxDocid = fts3DocidRange(pDocidLe, LARGEST_INT64);
  3125   3126   

Changes to ext/fts3/fts3_expr.c.

   186    186     sqlite3_tokenizer_cursor *pCursor;
   187    187     Fts3Expr *pRet = 0;
   188    188     int i = 0;
   189    189   
   190    190     /* Set variable i to the maximum number of bytes of input to tokenize. */
   191    191     for(i=0; i<n; i++){
   192    192       if( sqlite3_fts3_enable_parentheses && (z[i]=='(' || z[i]==')') ) break;
   193         -    if( z[i]=='*' || z[i]=='"' ) break;
          193  +    if( z[i]=='"' ) break;
   194    194     }
   195    195   
   196    196     *pnConsumed = i;
   197    197     rc = sqlite3Fts3OpenTokenizer(pTokenizer, pParse->iLangid, z, i, &pCursor);
   198    198     if( rc==SQLITE_OK ){
   199    199       const char *zToken;
   200    200       int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0;

Changes to src/btree.c.

   778    778   **
   779    779   ** Calling this routine with a NULL cursor pointer returns false.
   780    780   **
   781    781   ** Use the separate sqlite3BtreeCursorRestore() routine to restore a cursor
   782    782   ** back to where it ought to be if this routine returns true.
   783    783   */
   784    784   int sqlite3BtreeCursorHasMoved(BtCursor *pCur){
   785         -  return pCur && pCur->eState!=CURSOR_VALID;
          785  +  return pCur->eState!=CURSOR_VALID;
   786    786   }
   787    787   
   788    788   /*
   789    789   ** This routine restores a cursor back to its original position after it
   790    790   ** has been moved by some outside activity (such as a btree rebalance or
   791    791   ** a row having been deleted out from under the cursor).  
   792    792   **
................................................................................
  5868   5868   ** If the cell content will fit on the page, then put it there.  If it
  5869   5869   ** will not fit, then make a copy of the cell content into pTemp if
  5870   5870   ** pTemp is not null.  Regardless of pTemp, allocate a new entry
  5871   5871   ** in pPage->apOvfl[] and make it point to the cell content (either
  5872   5872   ** in pTemp or the original pCell) and also record its index. 
  5873   5873   ** Allocating a new entry in pPage->aCell[] implies that 
  5874   5874   ** pPage->nOverflow is incremented.
  5875         -**
  5876         -** If nSkip is non-zero, then do not copy the first nSkip bytes of the
  5877         -** cell. The caller will overwrite them after this function returns. If
  5878         -** nSkip is non-zero, then pCell may not point to an invalid memory location 
  5879         -** (but pCell+nSkip is always valid).
  5880   5875   */
  5881   5876   static void insertCell(
  5882   5877     MemPage *pPage,   /* Page into which we are copying */
  5883   5878     int i,            /* New cell becomes the i-th cell of the page */
  5884   5879     u8 *pCell,        /* Content of the new cell */
  5885   5880     int sz,           /* Bytes of content in pCell */
  5886   5881     u8 *pTemp,        /* Temp storage space for pCell, if needed */
................................................................................
  5889   5884   ){
  5890   5885     int idx = 0;      /* Where to write new cell content in data[] */
  5891   5886     int j;            /* Loop counter */
  5892   5887     int end;          /* First byte past the last cell pointer in data[] */
  5893   5888     int ins;          /* Index in data[] where new cell pointer is inserted */
  5894   5889     int cellOffset;   /* Address of first cell pointer in data[] */
  5895   5890     u8 *data;         /* The content of the whole page */
  5896         -  int nSkip = (iChild ? 4 : 0);
  5897   5891   
  5898   5892     if( *pRC ) return;
  5899   5893   
  5900   5894     assert( i>=0 && i<=pPage->nCell+pPage->nOverflow );
  5901   5895     assert( MX_CELL(pPage->pBt)<=10921 );
  5902   5896     assert( pPage->nCell<=MX_CELL(pPage->pBt) || CORRUPT_DB );
  5903   5897     assert( pPage->nOverflow<=ArraySize(pPage->apOvfl) );
................................................................................
  5907   5901     ** malformed cell from a leaf page to an interior page, if the cell size
  5908   5902     ** wanted to be less than 4 but got rounded up to 4 on the leaf, then size
  5909   5903     ** might be less than 8 (leaf-size + pointer) on the interior node.  Hence
  5910   5904     ** the term after the || in the following assert(). */
  5911   5905     assert( sz==cellSizePtr(pPage, pCell) || (sz==8 && iChild>0) );
  5912   5906     if( pPage->nOverflow || sz+2>pPage->nFree ){
  5913   5907       if( pTemp ){
  5914         -      memcpy(pTemp+nSkip, pCell+nSkip, sz-nSkip);
         5908  +      memcpy(pTemp, pCell, sz);
  5915   5909         pCell = pTemp;
  5916   5910       }
  5917   5911       if( iChild ){
  5918   5912         put4byte(pCell, iChild);
  5919   5913       }
  5920   5914       j = pPage->nOverflow++;
  5921   5915       assert( j<(int)(sizeof(pPage->apOvfl)/sizeof(pPage->apOvfl[0])) );
................................................................................
  5936   5930       if( rc ){ *pRC = rc; return; }
  5937   5931       /* The allocateSpace() routine guarantees the following two properties
  5938   5932       ** if it returns success */
  5939   5933       assert( idx >= end+2 );
  5940   5934       assert( idx+sz <= (int)pPage->pBt->usableSize );
  5941   5935       pPage->nCell++;
  5942   5936       pPage->nFree -= (u16)(2 + sz);
  5943         -    memcpy(&data[idx+nSkip], pCell+nSkip, sz-nSkip);
         5937  +    memcpy(&data[idx], pCell, sz);
  5944   5938       if( iChild ){
  5945   5939         put4byte(&data[idx], iChild);
  5946   5940       }
  5947   5941       memmove(&data[ins+2], &data[ins], end-ins);
  5948   5942       put2byte(&data[ins], idx);
  5949   5943       put2byte(&data[pPage->hdrOffset+3], pPage->nCell);
  5950   5944   #ifndef SQLITE_OMIT_AUTOVACUUM

Changes to src/build.c.

  2743   2743       addr2 = sqlite3VdbeCurrentAddr(v);
  2744   2744       sqlite3VdbeAddOp4Int(v, OP_SorterCompare, iSorter, j2, regRecord,
  2745   2745                            pIndex->nKeyCol); VdbeCoverage(v);
  2746   2746       sqlite3UniqueConstraint(pParse, OE_Abort, pIndex);
  2747   2747     }else{
  2748   2748       addr2 = sqlite3VdbeCurrentAddr(v);
  2749   2749     }
  2750         -  sqlite3VdbeAddOp2(v, OP_SorterData, iSorter, regRecord);
         2750  +  sqlite3VdbeAddOp3(v, OP_SorterData, iSorter, regRecord, iIdx);
  2751   2751     sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 1);
  2752   2752     sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
  2753   2753     sqlite3ReleaseTempReg(pParse, regRecord);
  2754   2754     sqlite3VdbeAddOp2(v, OP_SorterNext, iSorter, addr2); VdbeCoverage(v);
  2755   2755     sqlite3VdbeJumpHere(v, addr1);
  2756   2756   
  2757   2757     sqlite3VdbeAddOp1(v, OP_Close, iTab);

Changes to src/pcache1.c.

   684    684     assert( pCache->nPage >= pCache->nRecyclable );
   685    685     nPinned = pCache->nPage - pCache->nRecyclable;
   686    686     assert( pGroup->mxPinned == pGroup->nMaxPage + 10 - pGroup->nMinPage );
   687    687     assert( pCache->n90pct == pCache->nMax*9/10 );
   688    688     if( createFlag==1 && (
   689    689           nPinned>=pGroup->mxPinned
   690    690        || nPinned>=pCache->n90pct
   691         -     || pcache1UnderMemoryPressure(pCache)
          691  +     || (pcache1UnderMemoryPressure(pCache) && pCache->nRecyclable<nPinned)
   692    692     )){
   693    693       return 0;
   694    694     }
   695    695   
   696    696     if( pCache->nPage>=pCache->nHash ) pcache1ResizeHash(pCache);
   697    697     assert( pCache->nHash>0 && pCache->apHash );
   698    698   

Changes to src/select.c.

  1177   1177     int eDest = pDest->eDest;
  1178   1178     int iParm = pDest->iSDParm;
  1179   1179     int regRow;
  1180   1180     int regRowid;
  1181   1181     int nKey;
  1182   1182     int iSortTab;                   /* Sorter cursor to read from */
  1183   1183     int nSortData;                  /* Trailing values to read from sorter */
  1184         -  u8 p5;                          /* p5 parameter for 1st OP_Column */
  1185   1184     int i;
  1186   1185     int bSeq;                       /* True if sorter record includes seq. no. */
  1187   1186   #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS
  1188   1187     struct ExprList_item *aOutEx = p->pEList->a;
  1189   1188   #endif
  1190   1189   
  1191   1190     if( pSort->labelBkOut ){
................................................................................
  1211   1210         addrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v);
  1212   1211       }
  1213   1212       sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut, nKey+1+nSortData);
  1214   1213       if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce);
  1215   1214       addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak);
  1216   1215       VdbeCoverage(v);
  1217   1216       codeOffset(v, p->iOffset, addrContinue);
  1218         -    sqlite3VdbeAddOp2(v, OP_SorterData, iTab, regSortOut);
  1219         -    p5 = OPFLAG_CLEARCACHE;
         1217  +    sqlite3VdbeAddOp3(v, OP_SorterData, iTab, regSortOut, iSortTab);
  1220   1218       bSeq = 0;
  1221   1219     }else{
  1222   1220       addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); VdbeCoverage(v);
  1223   1221       codeOffset(v, p->iOffset, addrContinue);
  1224   1222       iSortTab = iTab;
  1225         -    p5 = 0;
  1226   1223       bSeq = 1;
  1227   1224     }
  1228   1225     for(i=0; i<nSortData; i++){
  1229   1226       sqlite3VdbeAddOp3(v, OP_Column, iSortTab, nKey+bSeq+i, regRow+i);
  1230         -    if( i==0 ) sqlite3VdbeChangeP5(v, p5);
  1231   1227       VdbeComment((v, "%s", aOutEx[i].zName ? aOutEx[i].zName : aOutEx[i].zSpan));
  1232   1228     }
  1233   1229     switch( eDest ){
  1234   1230       case SRT_Table:
  1235   1231       case SRT_EphemTab: {
  1236   1232         testcase( eDest==SRT_Table );
  1237   1233         testcase( eDest==SRT_EphemTab );
................................................................................
  5152   5148         ** (b0 is memory location iBMem+0, b1 is iBMem+1, and so forth)
  5153   5149         ** Then compare the current GROUP BY terms against the GROUP BY terms
  5154   5150         ** from the previous row currently stored in a0, a1, a2...
  5155   5151         */
  5156   5152         addrTopOfLoop = sqlite3VdbeCurrentAddr(v);
  5157   5153         sqlite3ExprCacheClear(pParse);
  5158   5154         if( groupBySort ){
  5159         -        sqlite3VdbeAddOp2(v, OP_SorterData, sAggInfo.sortingIdx, sortOut);
         5155  +        sqlite3VdbeAddOp3(v, OP_SorterData, sAggInfo.sortingIdx, sortOut,sortPTab);
  5160   5156         }
  5161   5157         for(j=0; j<pGroupBy->nExpr; j++){
  5162   5158           if( groupBySort ){
  5163   5159             sqlite3VdbeAddOp3(v, OP_Column, sortPTab, j, iBMem+j);
  5164         -          if( j==0 ) sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE);
  5165   5160           }else{
  5166   5161             sAggInfo.directMode = 1;
  5167   5162             sqlite3ExprCode(pParse, pGroupBy->a[j].pExpr, iBMem+j);
  5168   5163           }
  5169   5164         }
  5170   5165         sqlite3VdbeAddOp4(v, OP_Compare, iAMem, iBMem, pGroupBy->nExpr,
  5171   5166                             (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO);

Changes to src/shell.c.

  3721   3721         if( p->echoOn ) printf("%s\n", zSql);
  3722   3722         nSql = 0;
  3723   3723       }
  3724   3724     }
  3725   3725     if( nSql ){
  3726   3726       if( !_all_whitespace(zSql) ){
  3727   3727         fprintf(stderr, "Error: incomplete SQL: %s\n", zSql);
         3728  +      errCnt++;
  3728   3729       }
  3729   3730       free(zSql);
  3730   3731     }
  3731   3732     free(zLine);
  3732   3733     return errCnt>0;
  3733   3734   }
  3734   3735   

Changes to src/sqliteInt.h.

   155    155   
   156    156   /*
   157    157   ** A macro to hint to the compiler that a function should not be
   158    158   ** inlined.
   159    159   */
   160    160   #if defined(__GNUC__)
   161    161   #  define SQLITE_NOINLINE  __attribute__((noinline))
   162         -#elif defined(_MSC_VER)
          162  +#elif defined(_MSC_VER) && _MSC_VER>=1310
   163    163   #  define SQLITE_NOINLINE  __declspec(noinline)
   164    164   #else
   165    165   #  define SQLITE_NOINLINE
   166    166   #endif
   167    167   
   168    168   /*
   169    169   ** The SQLITE_THREADSAFE macro must be defined as 0, 1, or 2.
................................................................................
  2672   2672   */
  2673   2673   #define OPFLAG_NCHANGE       0x01    /* Set to update db->nChange */
  2674   2674   #define OPFLAG_EPHEM         0x01    /* OP_Column: Ephemeral output is ok */
  2675   2675   #define OPFLAG_LASTROWID     0x02    /* Set to update db->lastRowid */
  2676   2676   #define OPFLAG_ISUPDATE      0x04    /* This OP_Insert is an sql UPDATE */
  2677   2677   #define OPFLAG_APPEND        0x08    /* This is likely to be an append */
  2678   2678   #define OPFLAG_USESEEKRESULT 0x10    /* Try to avoid a seek in BtreeInsert() */
  2679         -#define OPFLAG_CLEARCACHE    0x20    /* Clear pseudo-table cache in OP_Column */
  2680   2679   #define OPFLAG_LENGTHARG     0x40    /* OP_Column only used for length() */
  2681   2680   #define OPFLAG_TYPEOFARG     0x80    /* OP_Column only used for typeof() */
  2682   2681   #define OPFLAG_BULKCSR       0x01    /* OP_Open** used to open bulk cursor */
  2683   2682   #define OPFLAG_P2ISREG       0x02    /* P2 to OP_Open** is a register number */
  2684   2683   #define OPFLAG_PERMUTE       0x01    /* OP_Compare: use the permutation */
  2685   2684   
  2686   2685   /*

Changes to src/threads.c.

   101    101   #if SQLITE_OS_WIN && !SQLITE_OS_WINRT && SQLITE_THREADSAFE>0
   102    102   
   103    103   #define SQLITE_THREADS_IMPLEMENTED 1  /* Prevent the single-thread code below */
   104    104   #include <process.h>
   105    105   
   106    106   /* A running thread */
   107    107   struct SQLiteThread {
   108         -  uintptr_t tid;           /* The thread handle */
          108  +  void *tid;               /* The thread handle */
   109    109     unsigned id;             /* The thread identifier */
   110    110     void *(*xTask)(void*);   /* The routine to run as a thread */
   111    111     void *pIn;               /* Argument to xTask */
   112    112     void *pResult;           /* Result of xTask */
   113    113   };
   114    114   
   115    115   /* Thread procedure Win32 compatibility shim */
................................................................................
   149    149     p = sqlite3Malloc(sizeof(*p));
   150    150     if( p==0 ) return SQLITE_NOMEM;
   151    151     if( sqlite3GlobalConfig.bCoreMutex==0 ){
   152    152       memset(p, 0, sizeof(*p));
   153    153     }else{
   154    154       p->xTask = xTask;
   155    155       p->pIn = pIn;
   156         -    p->tid = _beginthreadex(0, 0, sqlite3ThreadProc, p, 0, &p->id);
          156  +    p->tid = (void*)_beginthreadex(0, 0, sqlite3ThreadProc, p, 0, &p->id);
   157    157       if( p->tid==0 ){
   158    158         memset(p, 0, sizeof(*p));
   159    159       }
   160    160     }
   161    161     if( p->xTask==0 ){
   162    162       p->id = GetCurrentThreadId();
   163    163       p->pResult = xTask(pIn);

Changes to src/vdbe.c.

   210    210       p->apCsr[iCur] = 0;
   211    211     }
   212    212     if( SQLITE_OK==sqlite3VdbeMemClearAndResize(pMem, nByte) ){
   213    213       p->apCsr[iCur] = pCx = (VdbeCursor*)pMem->z;
   214    214       memset(pCx, 0, sizeof(VdbeCursor));
   215    215       pCx->iDb = iDb;
   216    216       pCx->nField = nField;
          217  +    pCx->aOffset = &pCx->aType[nField];
   217    218       if( isBtreeCursor ){
   218    219         pCx->pCursor = (BtCursor*)
   219    220             &pMem->z[ROUND8(sizeof(VdbeCursor))+2*sizeof(u32)*nField];
   220    221         sqlite3BtreeCursorZero(pCx->pCursor);
   221    222       }
   222    223     }
   223    224     return pCx;
................................................................................
  2272   2273     assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) );
  2273   2274     pDest = &aMem[pOp->p3];
  2274   2275     memAboutToChange(p, pDest);
  2275   2276     assert( pOp->p1>=0 && pOp->p1<p->nCursor );
  2276   2277     pC = p->apCsr[pOp->p1];
  2277   2278     assert( pC!=0 );
  2278   2279     assert( p2<pC->nField );
  2279         -  aOffset = pC->aType + pC->nField;
         2280  +  aOffset = pC->aOffset;
  2280   2281   #ifndef SQLITE_OMIT_VIRTUALTABLE
  2281   2282     assert( pC->pVtabCursor==0 ); /* OP_Column never called on virtual table */
  2282   2283   #endif
  2283   2284     pCrsr = pC->pCursor;
  2284   2285     assert( pCrsr!=0 || pC->pseudoTableReg>0 ); /* pCrsr NULL on PseudoTables */
  2285   2286     assert( pCrsr!=0 || pC->nullRow );          /* pC->nullRow on PseudoTables */
  2286   2287   
  2287   2288     /* If the cursor cache is stale, bring it up-to-date */
  2288   2289     rc = sqlite3VdbeCursorMoveto(pC);
  2289   2290     if( rc ) goto abort_due_to_error;
  2290         -  if( pC->cacheStatus!=p->cacheCtr || (pOp->p5&OPFLAG_CLEARCACHE)!=0 ){
         2291  +  if( pC->cacheStatus!=p->cacheCtr ){
  2291   2292       if( pC->nullRow ){
  2292   2293         if( pCrsr==0 ){
  2293   2294           assert( pC->pseudoTableReg>0 );
  2294   2295           pReg = &aMem[pC->pseudoTableReg];
  2295   2296           assert( pReg->flags & MEM_Blob );
  2296   2297           assert( memIsValid(pReg) );
  2297   2298           pC->payloadSize = pC->szRow = avail = pReg->n;
................................................................................
  2328   2329           goto too_big;
  2329   2330         }
  2330   2331       }
  2331   2332       pC->cacheStatus = p->cacheCtr;
  2332   2333       pC->iHdrOffset = getVarint32(pC->aRow, offset);
  2333   2334       pC->nHdrParsed = 0;
  2334   2335       aOffset[0] = offset;
  2335         -    if( avail<offset ){
  2336         -      /* pC->aRow does not have to hold the entire row, but it does at least
  2337         -      ** need to cover the header of the record.  If pC->aRow does not contain
  2338         -      ** the complete header, then set it to zero, forcing the header to be
  2339         -      ** dynamically allocated. */
  2340         -      pC->aRow = 0;
  2341         -      pC->szRow = 0;
  2342         -    }
  2343   2336   
  2344   2337       /* Make sure a corrupt database has not given us an oversize header.
  2345   2338       ** Do this now to avoid an oversize memory allocation.
  2346   2339       **
  2347   2340       ** Type entries can be between 1 and 5 bytes each.  But 4 and 5 byte
  2348   2341       ** types use so much data space that there can only be 4096 and 32 of
  2349   2342       ** them, respectively.  So the maximum header length results from a
................................................................................
  2350   2343       ** 3-byte type for each of the maximum of 32768 columns plus three
  2351   2344       ** extra bytes for the header length itself.  32768*3 + 3 = 98307.
  2352   2345       */
  2353   2346       if( offset > 98307 || offset > pC->payloadSize ){
  2354   2347         rc = SQLITE_CORRUPT_BKPT;
  2355   2348         goto op_column_error;
  2356   2349       }
         2350  +
         2351  +    if( avail<offset ){
         2352  +      /* pC->aRow does not have to hold the entire row, but it does at least
         2353  +      ** need to cover the header of the record.  If pC->aRow does not contain
         2354  +      ** the complete header, then set it to zero, forcing the header to be
         2355  +      ** dynamically allocated. */
         2356  +      pC->aRow = 0;
         2357  +      pC->szRow = 0;
         2358  +    }
         2359  +
         2360  +    /* The following goto is an optimization.  It can be omitted and
         2361  +    ** everything will still work.  But OP_Column is measurably faster
         2362  +    ** by skipping the subsequent conditional, which is always true.
         2363  +    */
         2364  +    assert( pC->nHdrParsed<=p2 );         /* Conditional skipped */
         2365  +    goto op_column_read_header;
  2357   2366     }
  2358   2367   
  2359   2368     /* Make sure at least the first p2+1 entries of the header have been
  2360   2369     ** parsed and valid information is in aOffset[] and pC->aType[].
  2361   2370     */
  2362   2371     if( pC->nHdrParsed<=p2 ){
  2363   2372       /* If there is more header available for parsing in the record, try
  2364   2373       ** to extract additional fields up through the p2+1-th field 
  2365   2374       */
         2375  +    op_column_read_header:
  2366   2376       if( pC->iHdrOffset<aOffset[0] ){
  2367   2377         /* Make sure zData points to enough of the record to cover the header. */
  2368   2378         if( pC->aRow==0 ){
  2369   2379           memset(&sMem, 0, sizeof(sMem));
  2370   2380           rc = sqlite3VdbeMemFromBtree(pCrsr, 0, aOffset[0], 
  2371   2381                                        !pC->isTable, &sMem);
  2372   2382           if( rc!=SQLITE_OK ){
................................................................................
  2403   2413         pC->nHdrParsed = i;
  2404   2414         pC->iHdrOffset = (u32)(zHdr - zData);
  2405   2415         if( pC->aRow==0 ){
  2406   2416           sqlite3VdbeMemRelease(&sMem);
  2407   2417           sMem.flags = MEM_Null;
  2408   2418         }
  2409   2419     
  2410         -      /* If we have read more header data than was contained in the header,
  2411         -      ** or if the end of the last field appears to be past the end of the
  2412         -      ** record, or if the end of the last field appears to be before the end
  2413         -      ** of the record (when all fields present), then we must be dealing 
  2414         -      ** with a corrupt database.
         2420  +      /* The record is corrupt if any of the following are true:
         2421  +      ** (1) the bytes of the header extend past the declared header size
         2422  +      **          (zHdr>zEndHdr)
         2423  +      ** (2) the entire header was used but not all data was used
         2424  +      **          (zHdr==zEndHdr && offset!=pC->payloadSize)
         2425  +      ** (3) the end of the data extends beyond the end of the record.
         2426  +      **          (offset > pC->payloadSize)
  2415   2427         */
  2416         -      if( (zHdr > zEndHdr)
         2428  +      if( (zHdr>=zEndHdr && (zHdr>zEndHdr || offset!=pC->payloadSize))
  2417   2429          || (offset > pC->payloadSize)
  2418         -       || (zHdr==zEndHdr && offset!=pC->payloadSize)
  2419   2430         ){
  2420   2431           rc = SQLITE_CORRUPT_BKPT;
  2421   2432           goto op_column_error;
  2422   2433         }
  2423   2434       }
  2424   2435   
  2425   2436       /* If after trying to extra new entries from the header, nHdrParsed is
................................................................................
  2602   2613   
  2603   2614     /* Loop through the elements that will make up the record to figure
  2604   2615     ** out how much space is required for the new record.
  2605   2616     */
  2606   2617     pRec = pLast;
  2607   2618     do{
  2608   2619       assert( memIsValid(pRec) );
  2609         -    serial_type = sqlite3VdbeSerialType(pRec, file_format);
         2620  +    pRec->uTemp = serial_type = sqlite3VdbeSerialType(pRec, file_format);
  2610   2621       len = sqlite3VdbeSerialTypeLen(serial_type);
  2611   2622       if( pRec->flags & MEM_Zero ){
  2612   2623         if( nData ){
  2613   2624           sqlite3VdbeMemExpandBlob(pRec);
  2614   2625         }else{
  2615   2626           nZero += pRec->u.nZero;
  2616   2627           len -= pRec->u.nZero;
................................................................................
  2651   2662   
  2652   2663     /* Write the record */
  2653   2664     i = putVarint32(zNewRecord, nHdr);
  2654   2665     j = nHdr;
  2655   2666     assert( pData0<=pLast );
  2656   2667     pRec = pData0;
  2657   2668     do{
  2658         -    serial_type = sqlite3VdbeSerialType(pRec, file_format);
         2669  +    serial_type = pRec->uTemp;
  2659   2670       i += putVarint32(&zNewRecord[i], serial_type);            /* serial type */
  2660   2671       j += sqlite3VdbeSerialPut(&zNewRecord[j], pRec, serial_type); /* content */
  2661   2672     }while( (++pRec)<=pLast );
  2662   2673     assert( i==nHdr );
  2663   2674     assert( j==nByte );
  2664   2675   
  2665   2676     assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) );
................................................................................
  3550   3561       ** blob, or NULL.  But it needs to be an integer before we can do
  3551   3562       ** the seek, so convert it. */
  3552   3563       pIn3 = &aMem[pOp->p3];
  3553   3564       if( (pIn3->flags & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){
  3554   3565         applyNumericAffinity(pIn3, 0);
  3555   3566       }
  3556   3567       iKey = sqlite3VdbeIntValue(pIn3);
  3557         -    pC->rowidIsValid = 0;
  3558   3568   
  3559   3569       /* If the P3 value could not be converted into an integer without
  3560   3570       ** loss of information, then special processing is required... */
  3561   3571       if( (pIn3->flags & MEM_Int)==0 ){
  3562   3572         if( (pIn3->flags & MEM_Real)==0 ){
  3563   3573           /* If the P3 value cannot be converted into any kind of a number,
  3564   3574           ** then the seek is not possible, so jump to P2 */
................................................................................
  3586   3596           assert( OP_SeekLE==(OP_SeekLT+1) );
  3587   3597           assert( OP_SeekGT==(OP_SeekGE+1) );
  3588   3598           assert( (OP_SeekLT & 0x0001)==(OP_SeekGE & 0x0001) );
  3589   3599           if( (oc & 0x0001)==(OP_SeekLT & 0x0001) ) oc++;
  3590   3600         }
  3591   3601       } 
  3592   3602       rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, 0, (u64)iKey, 0, &res);
         3603  +    pC->movetoTarget = iKey;  /* Used by OP_Delete */
  3593   3604       if( rc!=SQLITE_OK ){
  3594   3605         goto abort_due_to_error;
  3595   3606       }
  3596         -    if( res==0 ){
  3597         -      pC->rowidIsValid = 1;
  3598         -      pC->lastRowid = iKey;
  3599         -    }
  3600   3607     }else{
  3601   3608       nField = pOp->p4.i;
  3602   3609       assert( pOp->p4type==P4_INT32 );
  3603   3610       assert( nField>0 );
  3604   3611       r.pKeyInfo = pC->pKeyInfo;
  3605   3612       r.nField = (u16)nField;
  3606   3613   
................................................................................
  3622   3629       { int i; for(i=0; i<r.nField; i++) assert( memIsValid(&r.aMem[i]) ); }
  3623   3630   #endif
  3624   3631       ExpandBlob(r.aMem);
  3625   3632       rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, &r, 0, 0, &res);
  3626   3633       if( rc!=SQLITE_OK ){
  3627   3634         goto abort_due_to_error;
  3628   3635       }
  3629         -    pC->rowidIsValid = 0;
  3630   3636     }
  3631   3637     pC->deferredMoveto = 0;
  3632   3638     pC->cacheStatus = CACHE_STALE;
  3633   3639   #ifdef SQLITE_TEST
  3634   3640     sqlite3_search_count++;
  3635   3641   #endif
  3636   3642     if( oc>=OP_SeekGE ){  assert( oc==OP_SeekGE || oc==OP_SeekGT );
  3637   3643       if( res<0 || (res==0 && oc==OP_SeekGT) ){
  3638   3644         res = 0;
  3639   3645         rc = sqlite3BtreeNext(pC->pCursor, &res);
  3640   3646         if( rc!=SQLITE_OK ) goto abort_due_to_error;
  3641         -      pC->rowidIsValid = 0;
  3642   3647       }else{
  3643   3648         res = 0;
  3644   3649       }
  3645   3650     }else{
  3646   3651       assert( oc==OP_SeekLT || oc==OP_SeekLE );
  3647   3652       if( res>0 || (res==0 && oc==OP_SeekLT) ){
  3648   3653         res = 0;
  3649   3654         rc = sqlite3BtreePrevious(pC->pCursor, &res);
  3650   3655         if( rc!=SQLITE_OK ) goto abort_due_to_error;
  3651         -      pC->rowidIsValid = 0;
  3652   3656       }else{
  3653   3657         /* res might be negative because the table is empty.  Check to
  3654   3658         ** see if this is the case.
  3655   3659         */
  3656   3660         res = sqlite3BtreeEof(pC->pCursor);
  3657   3661       }
  3658   3662     }
................................................................................
  3681   3685     pC = p->apCsr[pOp->p1];
  3682   3686     assert( pC!=0 );
  3683   3687     assert( pC->pCursor!=0 );
  3684   3688     assert( pC->isTable );
  3685   3689     pC->nullRow = 0;
  3686   3690     pIn2 = &aMem[pOp->p2];
  3687   3691     pC->movetoTarget = sqlite3VdbeIntValue(pIn2);
  3688         -  pC->rowidIsValid = 0;
  3689   3692     pC->deferredMoveto = 1;
  3690   3693     break;
  3691   3694   }
  3692   3695     
  3693   3696   
  3694   3697   /* Opcode: Found P1 P2 P3 P4 *
  3695   3698   ** Synopsis: key=r[P3@P4]
................................................................................
  3867   3870     assert( pC->isTable );
  3868   3871     assert( pC->pseudoTableReg==0 );
  3869   3872     pCrsr = pC->pCursor;
  3870   3873     assert( pCrsr!=0 );
  3871   3874     res = 0;
  3872   3875     iKey = pIn3->u.i;
  3873   3876     rc = sqlite3BtreeMovetoUnpacked(pCrsr, 0, iKey, 0, &res);
  3874         -  pC->lastRowid = pIn3->u.i;
  3875         -  pC->rowidIsValid = res==0 ?1:0;
         3877  +  pC->movetoTarget = iKey;  /* Used by OP_Delete */
  3876   3878     pC->nullRow = 0;
  3877   3879     pC->cacheStatus = CACHE_STALE;
  3878   3880     pC->deferredMoveto = 0;
  3879   3881     VdbeBranchTaken(res!=0,2);
  3880   3882     if( res!=0 ){
  3881   3883       pc = pOp->p2 - 1;
  3882         -    assert( pC->rowidIsValid==0 );
  3883   3884     }
  3884   3885     pC->seekResult = res;
  3885   3886     break;
  3886   3887   }
  3887   3888   
  3888   3889   /* Opcode: Sequence P1 P2 * * *
  3889   3890   ** Synopsis: r[P2]=cursor[P1].ctr++
................................................................................
  4023   4024               && (++cnt<100));
  4024   4025         if( rc==SQLITE_OK && res==0 ){
  4025   4026           rc = SQLITE_FULL;   /* IMP: R-38219-53002 */
  4026   4027           goto abort_due_to_error;
  4027   4028         }
  4028   4029         assert( v>0 );  /* EV: R-40812-03570 */
  4029   4030       }
  4030         -    pC->rowidIsValid = 0;
  4031   4031       pC->deferredMoveto = 0;
  4032   4032       pC->cacheStatus = CACHE_STALE;
  4033   4033     }
  4034   4034     pOut->u.i = v;
  4035   4035     break;
  4036   4036   }
  4037   4037   
................................................................................
  4128   4128     }else{
  4129   4129       nZero = 0;
  4130   4130     }
  4131   4131     rc = sqlite3BtreeInsert(pC->pCursor, 0, iKey,
  4132   4132                             pData->z, pData->n, nZero,
  4133   4133                             (pOp->p5 & OPFLAG_APPEND)!=0, seekResult
  4134   4134     );
  4135         -  pC->rowidIsValid = 0;
  4136   4135     pC->deferredMoveto = 0;
  4137   4136     pC->cacheStatus = CACHE_STALE;
  4138   4137   
  4139   4138     /* Invoke the update-hook if required. */
  4140   4139     if( rc==SQLITE_OK && db->xUpdateCallback && pOp->p4.z ){
  4141   4140       zDb = db->aDb[pC->iDb].zName;
  4142   4141       zTbl = pOp->p4.z;
................................................................................
  4165   4164   **
  4166   4165   ** If P4 is not NULL, then it is the name of the table that P1 is
  4167   4166   ** pointing to.  The update hook will be invoked, if it exists.
  4168   4167   ** If P4 is not NULL then the P1 cursor must have been positioned
  4169   4168   ** using OP_NotFound prior to invoking this opcode.
  4170   4169   */
  4171   4170   case OP_Delete: {
  4172         -  i64 iKey;
  4173   4171     VdbeCursor *pC;
  4174   4172   
  4175   4173     assert( pOp->p1>=0 && pOp->p1<p->nCursor );
  4176   4174     pC = p->apCsr[pOp->p1];
  4177   4175     assert( pC!=0 );
  4178   4176     assert( pC->pCursor!=0 );  /* Only valid for real tables, no pseudotables */
  4179         -  iKey = pC->lastRowid;      /* Only used for the update hook */
  4180         -
  4181         -  /* The OP_Delete opcode always follows an OP_NotExists or OP_Last or
  4182         -  ** OP_Column on the same table without any intervening operations that
  4183         -  ** might move or invalidate the cursor.  Hence cursor pC is always pointing
  4184         -  ** to the row to be deleted and the sqlite3VdbeCursorMoveto() operation
  4185         -  ** below is always a no-op and cannot fail.  We will run it anyhow, though,
  4186         -  ** to guard against future changes to the code generator.
  4187         -  **/
  4188   4177     assert( pC->deferredMoveto==0 );
  4189         -  rc = sqlite3VdbeCursorMoveto(pC);
  4190         -  if( NEVER(rc!=SQLITE_OK) ) goto abort_due_to_error;
  4191   4178   
         4179  +#ifdef SQLITE_DEBUG
         4180  +  /* The seek operation that positioned the cursor prior to OP_Delete will
         4181  +  ** have also set the pC->movetoTarget field to the rowid of the row that
         4182  +  ** is being deleted */
         4183  +  if( pOp->p4.z && pC->isTable ){
         4184  +    i64 iKey = 0;
         4185  +    sqlite3BtreeKeySize(pC->pCursor, &iKey);
         4186  +    assert( pC->movetoTarget==iKey ); 
         4187  +  }
         4188  +#endif
         4189  + 
  4192   4190     rc = sqlite3BtreeDelete(pC->pCursor);
  4193   4191     pC->cacheStatus = CACHE_STALE;
  4194   4192   
  4195   4193     /* Invoke the update-hook if required. */
  4196   4194     if( rc==SQLITE_OK && db->xUpdateCallback && pOp->p4.z && pC->isTable ){
  4197   4195       db->xUpdateCallback(db->pUpdateArg, SQLITE_DELETE,
  4198         -                        db->aDb[pC->iDb].zName, pOp->p4.z, iKey);
         4196  +                        db->aDb[pC->iDb].zName, pOp->p4.z, pC->movetoTarget);
  4199   4197       assert( pC->iDb>=0 );
  4200   4198     }
  4201   4199     if( pOp->p2 & OPFLAG_NCHANGE ) p->nChange++;
  4202   4200     break;
  4203   4201   }
  4204   4202   /* Opcode: ResetCount * * * * *
  4205   4203   **
................................................................................
  4244   4242     VdbeBranchTaken(res!=0,2);
  4245   4243     if( res ){
  4246   4244       pc = pOp->p2-1;
  4247   4245     }
  4248   4246     break;
  4249   4247   };
  4250   4248   
  4251         -/* Opcode: SorterData P1 P2 * * *
         4249  +/* Opcode: SorterData P1 P2 P3 * *
  4252   4250   ** Synopsis: r[P2]=data
  4253   4251   **
  4254   4252   ** Write into register P2 the current sorter data for sorter cursor P1.
         4253  +** Then clear the column header cache on cursor P3.
         4254  +**
         4255  +** This opcode is normally use to move a record out of the sorter and into
         4256  +** a register that is the source for a pseudo-table cursor created using
         4257  +** OpenPseudo.  That pseudo-table cursor is the one that is identified by
         4258  +** parameter P3.  Clearing the P3 column cache as part of this opcode saves
         4259  +** us from having to issue a separate NullRow instruction to clear that cache.
  4255   4260   */
  4256   4261   case OP_SorterData: {
  4257   4262     VdbeCursor *pC;
  4258   4263   
  4259   4264     pOut = &aMem[pOp->p2];
  4260   4265     pC = p->apCsr[pOp->p1];
  4261   4266     assert( isSorter(pC) );
  4262   4267     rc = sqlite3VdbeSorterRowkey(pC, pOut);
  4263   4268     assert( rc!=SQLITE_OK || (pOut->flags & MEM_Blob) );
         4269  +  assert( pOp->p1>=0 && pOp->p1<p->nCursor );
         4270  +  p->apCsr[pOp->p3]->cacheStatus = CACHE_STALE;
  4264   4271     break;
  4265   4272   }
  4266   4273   
  4267   4274   /* Opcode: RowData P1 P2 * * *
  4268   4275   ** Synopsis: r[P2]=data
  4269   4276   **
  4270   4277   ** Write into register P2 the complete row data for cursor P1.
................................................................................
  4303   4310     assert( pC->isTable || pOp->opcode!=OP_RowData );
  4304   4311     assert( pC->isTable==0 || pOp->opcode==OP_RowData );
  4305   4312     assert( pC!=0 );
  4306   4313     assert( pC->nullRow==0 );
  4307   4314     assert( pC->pseudoTableReg==0 );
  4308   4315     assert( pC->pCursor!=0 );
  4309   4316     pCrsr = pC->pCursor;
  4310         -  assert( sqlite3BtreeCursorIsValid(pCrsr) );
  4311   4317   
  4312   4318     /* The OP_RowKey and OP_RowData opcodes always follow OP_NotExists or
  4313   4319     ** OP_Rewind/Op_Next with no intervening instructions that might invalidate
  4314         -  ** the cursor.  Hence the following sqlite3VdbeCursorMoveto() call is always
  4315         -  ** a no-op and can never fail.  But we leave it in place as a safety.
         4320  +  ** the cursor.  If this where not the case, on of the following assert()s
         4321  +  ** would fail.  Should this ever change (because of changes in the code
         4322  +  ** generator) then the fix would be to insert a call to
         4323  +  ** sqlite3VdbeCursorMoveto().
  4316   4324     */
  4317   4325     assert( pC->deferredMoveto==0 );
         4326  +  assert( sqlite3BtreeCursorIsValid(pCrsr) );
         4327  +#if 0  /* Not required due to the previous to assert() statements */
  4318   4328     rc = sqlite3VdbeCursorMoveto(pC);
  4319         -  if( NEVER(rc!=SQLITE_OK) ) goto abort_due_to_error;
         4329  +  if( rc!=SQLITE_OK ) goto abort_due_to_error;
         4330  +#endif
  4320   4331   
  4321   4332     if( pC->isTable==0 ){
  4322   4333       assert( !pC->isTable );
  4323   4334       VVA_ONLY(rc =) sqlite3BtreeKeySize(pCrsr, &n64);
  4324   4335       assert( rc==SQLITE_OK );    /* True because of CursorMoveto() call above */
  4325   4336       if( n64>db->aLimit[SQLITE_LIMIT_LENGTH] ){
  4326   4337         goto too_big;
................................................................................
  4381   4392       pModule = pVtab->pModule;
  4382   4393       assert( pModule->xRowid );
  4383   4394       rc = pModule->xRowid(pC->pVtabCursor, &v);
  4384   4395       sqlite3VtabImportErrmsg(p, pVtab);
  4385   4396   #endif /* SQLITE_OMIT_VIRTUALTABLE */
  4386   4397     }else{
  4387   4398       assert( pC->pCursor!=0 );
  4388         -    rc = sqlite3VdbeCursorMoveto(pC);
         4399  +    rc = sqlite3VdbeCursorRestore(pC);
  4389   4400       if( rc ) goto abort_due_to_error;
  4390         -    if( pC->rowidIsValid ){
  4391         -      v = pC->lastRowid;
  4392         -    }else{
  4393         -      rc = sqlite3BtreeKeySize(pC->pCursor, &v);
  4394         -      assert( rc==SQLITE_OK );  /* Always so because of CursorMoveto() above */
  4395         -    }
         4401  +    rc = sqlite3BtreeKeySize(pC->pCursor, &v);
         4402  +    assert( rc==SQLITE_OK );  /* Always so because of CursorRestore() above */
  4396   4403     }
  4397   4404     pOut->u.i = v;
  4398   4405     break;
  4399   4406   }
  4400   4407   
  4401   4408   /* Opcode: NullRow P1 * * * *
  4402   4409   **
................................................................................
  4407   4414   case OP_NullRow: {
  4408   4415     VdbeCursor *pC;
  4409   4416   
  4410   4417     assert( pOp->p1>=0 && pOp->p1<p->nCursor );
  4411   4418     pC = p->apCsr[pOp->p1];
  4412   4419     assert( pC!=0 );
  4413   4420     pC->nullRow = 1;
  4414         -  pC->rowidIsValid = 0;
  4415   4421     pC->cacheStatus = CACHE_STALE;
  4416   4422     if( pC->pCursor ){
  4417   4423       sqlite3BtreeClearCursor(pC->pCursor);
  4418   4424     }
  4419   4425     break;
  4420   4426   }
  4421   4427   
................................................................................
  4441   4447     assert( pC!=0 );
  4442   4448     pCrsr = pC->pCursor;
  4443   4449     res = 0;
  4444   4450     assert( pCrsr!=0 );
  4445   4451     rc = sqlite3BtreeLast(pCrsr, &res);
  4446   4452     pC->nullRow = (u8)res;
  4447   4453     pC->deferredMoveto = 0;
  4448         -  pC->rowidIsValid = 0;
  4449   4454     pC->cacheStatus = CACHE_STALE;
  4450   4455   #ifdef SQLITE_DEBUG
  4451   4456     pC->seekOp = OP_Last;
  4452   4457   #endif
  4453   4458     if( pOp->p2>0 ){
  4454   4459       VdbeBranchTaken(res!=0,2);
  4455   4460       if( res ) pc = pOp->p2 - 1;
................................................................................
  4508   4513       rc = sqlite3VdbeSorterRewind(pC, &res);
  4509   4514     }else{
  4510   4515       pCrsr = pC->pCursor;
  4511   4516       assert( pCrsr );
  4512   4517       rc = sqlite3BtreeFirst(pCrsr, &res);
  4513   4518       pC->deferredMoveto = 0;
  4514   4519       pC->cacheStatus = CACHE_STALE;
  4515         -    pC->rowidIsValid = 0;
  4516   4520     }
  4517   4521     pC->nullRow = (u8)res;
  4518   4522     assert( pOp->p2>0 && pOp->p2<p->nOp );
  4519   4523     VdbeBranchTaken(res!=0,2);
  4520   4524     if( res ){
  4521   4525       pc = pOp->p2 - 1;
  4522   4526     }
................................................................................
  4634   4638       p->aCounter[pOp->p5]++;
  4635   4639   #ifdef SQLITE_TEST
  4636   4640       sqlite3_search_count++;
  4637   4641   #endif
  4638   4642     }else{
  4639   4643       pC->nullRow = 1;
  4640   4644     }
  4641         -  pC->rowidIsValid = 0;
  4642   4645     goto check_for_interrupt;
  4643   4646   }
  4644   4647   
  4645   4648   /* Opcode: IdxInsert P1 P2 P3 * P5
  4646   4649   ** Synopsis: key=r[P2]
  4647   4650   **
  4648   4651   ** Register P2 holds an SQL index key made using the
................................................................................
  4750   4753   
  4751   4754     assert( pOp->p1>=0 && pOp->p1<p->nCursor );
  4752   4755     pC = p->apCsr[pOp->p1];
  4753   4756     assert( pC!=0 );
  4754   4757     pCrsr = pC->pCursor;
  4755   4758     assert( pCrsr!=0 );
  4756   4759     pOut->flags = MEM_Null;
  4757         -  rc = sqlite3VdbeCursorMoveto(pC);
  4758         -  if( NEVER(rc) ) goto abort_due_to_error;
         4760  +  assert( pC->isTable==0 );
  4759   4761     assert( pC->deferredMoveto==0 );
  4760         -  assert( pC->isTable==0 );
         4762  +
         4763  +  /* sqlite3VbeCursorRestore() can only fail if the record has been deleted
         4764  +  ** out from under the cursor.  That will never happend for an IdxRowid
         4765  +  ** opcode, hence the NEVER() arround the check of the return value.
         4766  +  */
         4767  +  rc = sqlite3VdbeCursorRestore(pC);
         4768  +  if( NEVER(rc!=SQLITE_OK) ) goto abort_due_to_error;
         4769  +
  4761   4770     if( !pC->nullRow ){
  4762   4771       rowid = 0;  /* Not needed.  Only used to silence a warning. */
  4763   4772       rc = sqlite3VdbeIdxRowid(db, pCrsr, &rowid);
  4764   4773       if( rc!=SQLITE_OK ){
  4765   4774         goto abort_due_to_error;
  4766   4775       }
  4767   4776       pOut->u.i = rowid;

Changes to src/vdbeInt.h.

    69     69     i16 nField;           /* Number of fields in the header */
    70     70     u16 nHdrParsed;       /* Number of header fields parsed so far */
    71     71   #ifdef SQLITE_DEBUG
    72     72     u8 seekOp;            /* Most recent seek operation on this cursor */
    73     73   #endif
    74     74     i8 iDb;               /* Index of cursor database in db->aDb[] (or -1) */
    75     75     u8 nullRow;           /* True if pointing to a row with no data */
    76         -  u8 rowidIsValid;      /* True if lastRowid is valid */
    77     76     u8 deferredMoveto;    /* A call to sqlite3BtreeMoveto() is needed */
    78     77     Bool isEphemeral:1;   /* True for an ephemeral table */
    79     78     Bool useRandomRowid:1;/* Generate new record numbers semi-randomly */
    80     79     Bool isTable:1;       /* True if a table requiring integer keys */
    81     80     Bool isOrdered:1;     /* True if the underlying table is BTREE_UNORDERED */
    82     81     Pgno pgnoRoot;        /* Root page of the open btree cursor */
    83     82     sqlite3_vtab_cursor *pVtabCursor;  /* The cursor for a virtual table */
    84     83     i64 seqCount;         /* Sequence counter */
    85     84     i64 movetoTarget;     /* Argument to the deferred sqlite3BtreeMoveto() */
    86         -  i64 lastRowid;        /* Rowid being deleted by OP_Delete */
    87     85     VdbeSorter *pSorter;  /* Sorter object for OP_SorterOpen cursors */
    88     86   
    89     87     /* Cached information about the header for the data record that the
    90     88     ** cursor is currently pointing to.  Only valid if cacheStatus matches
    91     89     ** Vdbe.cacheCtr.  Vdbe.cacheCtr will never take on the value of
    92     90     ** CACHE_STALE and so setting cacheStatus=CACHE_STALE guarantees that
    93     91     ** the cache is out of date.
................................................................................
    96     94     ** be NULL.
    97     95     */
    98     96     u32 cacheStatus;      /* Cache is valid if this matches Vdbe.cacheCtr */
    99     97     u32 payloadSize;      /* Total number of bytes in the record */
   100     98     u32 szRow;            /* Byte available in aRow */
   101     99     u32 iHdrOffset;       /* Offset to next unparsed byte of the header */
   102    100     const u8 *aRow;       /* Data for the current row, if all on one page */
          101  +  u32 *aOffset;         /* Pointer to aType[nField] */
   103    102     u32 aType[1];         /* Type values for all entries in the record */
   104    103     /* 2*nField extra array elements allocated for aType[], beyond the one
   105    104     ** static element declared in the structure.  nField total array slots for
   106    105     ** aType[] and nField+1 array slots for aOffset[] */
   107    106   };
   108    107   typedef struct VdbeCursor VdbeCursor;
   109    108   
................................................................................
   172    171     u16 flags;          /* Some combination of MEM_Null, MEM_Str, MEM_Dyn, etc. */
   173    172     u8  enc;            /* SQLITE_UTF8, SQLITE_UTF16BE, SQLITE_UTF16LE */
   174    173     int n;              /* Number of characters in string value, excluding '\0' */
   175    174     char *z;            /* String or BLOB value */
   176    175     /* ShallowCopy only needs to copy the information above */
   177    176     char *zMalloc;      /* Space to hold MEM_Str or MEM_Blob if szMalloc>0 */
   178    177     int szMalloc;       /* Size of the zMalloc allocation */
   179         -  int iPadding1;      /* Padding for 8-byte alignment */
          178  +  u32 uTemp;          /* Transient storage for serial_type in OP_MakeRecord */
   180    179     sqlite3 *db;        /* The associated database connection */
   181    180     void (*xDel)(void*);/* Destructor for Mem.z - only valid if MEM_Dyn */
   182    181   #ifdef SQLITE_DEBUG
   183    182     Mem *pScopyFrom;    /* This Mem is a shallow copy of pScopyFrom */
   184    183     void *pFiller;      /* So that sizeof(Mem) is a multiple of 8 */
   185    184   #endif
   186    185   };
................................................................................
   380    379   
   381    380   /*
   382    381   ** Function prototypes
   383    382   */
   384    383   void sqlite3VdbeFreeCursor(Vdbe *, VdbeCursor*);
   385    384   void sqliteVdbePopStack(Vdbe*,int);
   386    385   int sqlite3VdbeCursorMoveto(VdbeCursor*);
          386  +int sqlite3VdbeCursorRestore(VdbeCursor*);
   387    387   #if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE)
   388    388   void sqlite3VdbePrintOp(FILE*, int, Op*);
   389    389   #endif
   390    390   u32 sqlite3VdbeSerialTypeLen(u32);
   391    391   u32 sqlite3VdbeSerialType(Mem*, int);
   392    392   u32 sqlite3VdbeSerialPut(unsigned char*, Mem*, u32);
   393    393   u32 sqlite3VdbeSerialGet(const unsigned char*, u32, Mem*);

Changes to src/vdbeaux.c.

  1743   1743       sqlite3BtreeClose(pCx->pBt);
  1744   1744       /* The pCx->pCursor will be close automatically, if it exists, by
  1745   1745       ** the call above. */
  1746   1746     }else if( pCx->pCursor ){
  1747   1747       sqlite3BtreeCloseCursor(pCx->pCursor);
  1748   1748     }
  1749   1749   #ifndef SQLITE_OMIT_VIRTUALTABLE
  1750         -  if( pCx->pVtabCursor ){
         1750  +  else if( pCx->pVtabCursor ){
  1751   1751       sqlite3_vtab_cursor *pVtabCursor = pCx->pVtabCursor;
  1752   1752       const sqlite3_module *pModule = pVtabCursor->pVtab->pModule;
  1753   1753       p->inVtabMethod = 1;
  1754   1754       pModule->xClose(pVtabCursor);
  1755   1755       p->inVtabMethod = 0;
  1756   1756     }
  1757   1757   #endif
................................................................................
  1786   1786   ** open cursors.
  1787   1787   */
  1788   1788   static void closeAllCursors(Vdbe *p){
  1789   1789     if( p->pFrame ){
  1790   1790       VdbeFrame *pFrame;
  1791   1791       for(pFrame=p->pFrame; pFrame->pParent; pFrame=pFrame->pParent);
  1792   1792       sqlite3VdbeFrameRestore(pFrame);
         1793  +    p->pFrame = 0;
         1794  +    p->nFrame = 0;
  1793   1795     }
  1794         -  p->pFrame = 0;
  1795         -  p->nFrame = 0;
         1796  +  assert( p->nFrame==0 );
  1796   1797   
  1797   1798     if( p->apCsr ){
  1798   1799       int i;
  1799   1800       for(i=0; i<p->nCursor; i++){
  1800   1801         VdbeCursor *pC = p->apCsr[i];
  1801   1802         if( pC ){
  1802   1803           sqlite3VdbeFreeCursor(p, pC);
................................................................................
  1810   1811     while( p->pDelFrame ){
  1811   1812       VdbeFrame *pDel = p->pDelFrame;
  1812   1813       p->pDelFrame = pDel->pParent;
  1813   1814       sqlite3VdbeFrameDelete(pDel);
  1814   1815     }
  1815   1816   
  1816   1817     /* Delete any auxdata allocations made by the VM */
  1817         -  sqlite3VdbeDeleteAuxData(p, -1, 0);
         1818  +  if( p->pAuxData ) sqlite3VdbeDeleteAuxData(p, -1, 0);
  1818   1819     assert( p->pAuxData==0 );
  1819   1820   }
  1820   1821   
  1821   1822   /*
  1822   1823   ** Clean up the VM after a single run.
  1823   1824   */
  1824   1825   static void Cleanup(Vdbe *p){
................................................................................
  2716   2717   #ifdef SQLITE_TEST
  2717   2718     extern int sqlite3_search_count;
  2718   2719   #endif
  2719   2720     assert( p->deferredMoveto );
  2720   2721     assert( p->isTable );
  2721   2722     rc = sqlite3BtreeMovetoUnpacked(p->pCursor, 0, p->movetoTarget, 0, &res);
  2722   2723     if( rc ) return rc;
  2723         -  p->lastRowid = p->movetoTarget;
  2724   2724     if( res!=0 ) return SQLITE_CORRUPT_BKPT;
  2725         -  p->rowidIsValid = 1;
  2726   2725   #ifdef SQLITE_TEST
  2727   2726     sqlite3_search_count++;
  2728   2727   #endif
  2729   2728     p->deferredMoveto = 0;
  2730   2729     p->cacheStatus = CACHE_STALE;
  2731   2730     return SQLITE_OK;
  2732   2731   }
................................................................................
  2743   2742     assert( p->pCursor!=0 );
  2744   2743     assert( sqlite3BtreeCursorHasMoved(p->pCursor) );
  2745   2744     rc = sqlite3BtreeCursorRestore(p->pCursor, &isDifferentRow);
  2746   2745     p->cacheStatus = CACHE_STALE;
  2747   2746     if( isDifferentRow ) p->nullRow = 1;
  2748   2747     return rc;
  2749   2748   }
         2749  +
         2750  +/*
         2751  +** Check to ensure that the cursor is valid.  Restore the cursor
         2752  +** if need be.  Return any I/O error from the restore operation.
         2753  +*/
         2754  +int sqlite3VdbeCursorRestore(VdbeCursor *p){
         2755  +  if( sqlite3BtreeCursorHasMoved(p->pCursor) ){
         2756  +    return handleMovedCursor(p);
         2757  +  }
         2758  +  return SQLITE_OK;
         2759  +}
  2750   2760   
  2751   2761   /*
  2752   2762   ** Make sure the cursor p is ready to read or write the row to which it
  2753   2763   ** was last positioned.  Return an error code if an OOM fault or I/O error
  2754   2764   ** prevents us from positioning the cursor to its correct position.
  2755   2765   **
  2756   2766   ** If a MoveTo operation is pending on the given cursor, then do that
................................................................................
  2761   2771   ** If the cursor is already pointing to the correct row and that row has
  2762   2772   ** not been deleted out from under the cursor, then this routine is a no-op.
  2763   2773   */
  2764   2774   int sqlite3VdbeCursorMoveto(VdbeCursor *p){
  2765   2775     if( p->deferredMoveto ){
  2766   2776       return handleDeferredMoveto(p);
  2767   2777     }
  2768         -  if( sqlite3BtreeCursorHasMoved(p->pCursor) ){
         2778  +  if( p->pCursor && sqlite3BtreeCursorHasMoved(p->pCursor) ){
  2769   2779       return handleMovedCursor(p);
  2770   2780     }
  2771   2781     return SQLITE_OK;
  2772   2782   }
  2773   2783   
  2774   2784   /*
  2775   2785   ** The following functions:

Changes to src/where.c.

  2205   2205           whereKeyStats(pParse, p, pRec, 0, a);
  2206   2206           iLower = a[0];
  2207   2207           iUpper = a[0] + a[1];
  2208   2208         }
  2209   2209   
  2210   2210         assert( pLower==0 || (pLower->eOperator & (WO_GT|WO_GE))!=0 );
  2211   2211         assert( pUpper==0 || (pUpper->eOperator & (WO_LT|WO_LE))!=0 );
  2212         -      assert( p->pKeyInfo!=0 && p->pKeyInfo->aSortOrder!=0 );
  2213         -      if( p->pKeyInfo->aSortOrder[nEq] ){
         2212  +      assert( p->aSortOrder!=0 );
         2213  +      if( p->aSortOrder[nEq] ){
  2214   2214           /* The roles of pLower and pUpper are swapped for a DESC index */
  2215   2215           SWAP(WhereTerm*, pLower, pUpper);
  2216   2216         }
  2217   2217   
  2218   2218         /* If possible, improve on the iLower estimate using ($P:$L). */
  2219   2219         if( pLower ){
  2220   2220           int bOk;                    /* True if value is extracted from pExpr */
................................................................................
  2733   2733     sqlite3StrAccumAppendAll(pStr, zColumn);
  2734   2734     sqlite3StrAccumAppend(pStr, zOp, 1);
  2735   2735     sqlite3StrAccumAppend(pStr, "?", 1);
  2736   2736   }
  2737   2737   
  2738   2738   /*
  2739   2739   ** Argument pLevel describes a strategy for scanning table pTab. This 
  2740         -** function returns a pointer to a string buffer containing a description
  2741         -** of the subset of table rows scanned by the strategy in the form of an
  2742         -** SQL expression. Or, if all rows are scanned, NULL is returned.
         2740  +** function appends text to pStr that describes the subset of table
         2741  +** rows scanned by the strategy in the form of an SQL expression.
  2743   2742   **
  2744   2743   ** For example, if the query:
  2745   2744   **
  2746   2745   **   SELECT * FROM t1 WHERE a=1 AND b>2;
  2747   2746   **
  2748   2747   ** is run and there is an index on (a, b), then this function returns a
  2749   2748   ** string similar to:
  2750   2749   **
  2751   2750   **   "a=? AND b>?"
  2752         -**
  2753         -** The returned pointer points to memory obtained from sqlite3DbMalloc().
  2754         -** It is the responsibility of the caller to free the buffer when it is
  2755         -** no longer required.
  2756   2751   */
  2757         -static char *explainIndexRange(sqlite3 *db, WhereLoop *pLoop, Table *pTab){
         2752  +static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop, Table *pTab){
  2758   2753     Index *pIndex = pLoop->u.btree.pIndex;
  2759   2754     u16 nEq = pLoop->u.btree.nEq;
  2760   2755     u16 nSkip = pLoop->u.btree.nSkip;
  2761   2756     int i, j;
  2762   2757     Column *aCol = pTab->aCol;
  2763   2758     i16 *aiColumn = pIndex->aiColumn;
  2764         -  StrAccum txt;
  2765   2759   
  2766         -  if( nEq==0 && (pLoop->wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ){
  2767         -    return 0;
  2768         -  }
  2769         -  sqlite3StrAccumInit(&txt, 0, 0, SQLITE_MAX_LENGTH);
  2770         -  txt.db = db;
  2771         -  sqlite3StrAccumAppend(&txt, " (", 2);
         2760  +  if( nEq==0 && (pLoop->wsFlags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ) return;
         2761  +  sqlite3StrAccumAppend(pStr, " (", 2);
  2772   2762     for(i=0; i<nEq; i++){
  2773   2763       char *z = aiColumn[i] < 0 ? "rowid" : aCol[aiColumn[i]].zName;
  2774   2764       if( i>=nSkip ){
  2775         -      explainAppendTerm(&txt, i, z, "=");
         2765  +      explainAppendTerm(pStr, i, z, "=");
  2776   2766       }else{
  2777         -      if( i ) sqlite3StrAccumAppend(&txt, " AND ", 5);
  2778         -      sqlite3StrAccumAppend(&txt, "ANY(", 4);
  2779         -      sqlite3StrAccumAppendAll(&txt, z);
  2780         -      sqlite3StrAccumAppend(&txt, ")", 1);
         2767  +      if( i ) sqlite3StrAccumAppend(pStr, " AND ", 5);
         2768  +      sqlite3XPrintf(pStr, 0, "ANY(%s)", z);
  2781   2769       }
  2782   2770     }
  2783   2771   
  2784   2772     j = i;
  2785   2773     if( pLoop->wsFlags&WHERE_BTM_LIMIT ){
  2786   2774       char *z = aiColumn[j] < 0 ? "rowid" : aCol[aiColumn[j]].zName;
  2787         -    explainAppendTerm(&txt, i++, z, ">");
         2775  +    explainAppendTerm(pStr, i++, z, ">");
  2788   2776     }
  2789   2777     if( pLoop->wsFlags&WHERE_TOP_LIMIT ){
  2790   2778       char *z = aiColumn[j] < 0 ? "rowid" : aCol[aiColumn[j]].zName;
  2791         -    explainAppendTerm(&txt, i, z, "<");
         2779  +    explainAppendTerm(pStr, i, z, "<");
  2792   2780     }
  2793         -  sqlite3StrAccumAppend(&txt, ")", 1);
  2794         -  return sqlite3StrAccumFinish(&txt);
         2781  +  sqlite3StrAccumAppend(pStr, ")", 1);
  2795   2782   }
  2796   2783   
  2797   2784   /*
  2798   2785   ** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN
  2799   2786   ** command. If the query being compiled is an EXPLAIN QUERY PLAN, a single
  2800   2787   ** record is added to the output to describe the table scan strategy in 
  2801   2788   ** pLevel.
................................................................................
  2811   2798   #ifndef SQLITE_DEBUG
  2812   2799     if( pParse->explain==2 )
  2813   2800   #endif
  2814   2801     {
  2815   2802       struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom];
  2816   2803       Vdbe *v = pParse->pVdbe;      /* VM being constructed */
  2817   2804       sqlite3 *db = pParse->db;     /* Database handle */
  2818         -    char *zMsg;                   /* Text to add to EQP output */
  2819   2805       int iId = pParse->iSelectId;  /* Select id (left-most output column) */
  2820   2806       int isSearch;                 /* True for a SEARCH. False for SCAN. */
  2821   2807       WhereLoop *pLoop;             /* The controlling WhereLoop object */
  2822   2808       u32 flags;                    /* Flags that describe this loop */
         2809  +    char *zMsg;                   /* Text to add to EQP output */
         2810  +    StrAccum str;                 /* EQP output string */
         2811  +    char zBuf[100];               /* Initial space for EQP output string */
  2823   2812   
  2824   2813       pLoop = pLevel->pWLoop;
  2825   2814       flags = pLoop->wsFlags;
  2826   2815       if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return;
  2827   2816   
  2828   2817       isSearch = (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0
  2829   2818               || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0))
  2830   2819               || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX));
  2831   2820   
  2832         -    zMsg = sqlite3MPrintf(db, "%s", isSearch?"SEARCH":"SCAN");
         2821  +    sqlite3StrAccumInit(&str, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH);
         2822  +    str.db = db;
         2823  +    sqlite3StrAccumAppendAll(&str, isSearch ? "SEARCH" : "SCAN");
  2833   2824       if( pItem->pSelect ){
  2834         -      zMsg = sqlite3MAppendf(db, zMsg, "%s SUBQUERY %d", zMsg,pItem->iSelectId);
         2825  +      sqlite3XPrintf(&str, 0, " SUBQUERY %d", pItem->iSelectId);
  2835   2826       }else{
  2836         -      zMsg = sqlite3MAppendf(db, zMsg, "%s TABLE %s", zMsg, pItem->zName);
         2827  +      sqlite3XPrintf(&str, 0, " TABLE %s", pItem->zName);
  2837   2828       }
  2838   2829   
  2839   2830       if( pItem->zAlias ){
  2840         -      zMsg = sqlite3MAppendf(db, zMsg, "%s AS %s", zMsg, pItem->zAlias);
         2831  +      sqlite3XPrintf(&str, 0, " AS %s", pItem->zAlias);
  2841   2832       }
  2842         -    if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0
  2843         -     && ALWAYS(pLoop->u.btree.pIndex!=0)
  2844         -    ){
  2845         -      const char *zFmt;
  2846         -      Index *pIdx = pLoop->u.btree.pIndex;
  2847         -      char *zWhere = explainIndexRange(db, pLoop, pItem->pTab);
         2833  +    if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){
         2834  +      const char *zFmt = 0;
         2835  +      Index *pIdx;
         2836  +
         2837  +      assert( pLoop->u.btree.pIndex!=0 );
         2838  +      pIdx = pLoop->u.btree.pIndex;
  2848   2839         assert( !(flags&WHERE_AUTO_INDEX) || (flags&WHERE_IDX_ONLY) );
  2849   2840         if( !HasRowid(pItem->pTab) && IsPrimaryKeyIndex(pIdx) ){
  2850         -        zFmt = zWhere ? "%s USING PRIMARY KEY%.0s%s" : "%s%.0s%s";
         2841  +        if( isSearch ){
         2842  +          zFmt = "PRIMARY KEY";
         2843  +        }
  2851   2844         }else if( flags & WHERE_AUTO_INDEX ){
  2852         -        zFmt = "%s USING AUTOMATIC COVERING INDEX%.0s%s";
         2845  +        zFmt = "AUTOMATIC COVERING INDEX";
  2853   2846         }else if( flags & WHERE_IDX_ONLY ){
  2854         -        zFmt = "%s USING COVERING INDEX %s%s";
         2847  +        zFmt = "COVERING INDEX %s";
  2855   2848         }else{
  2856         -        zFmt = "%s USING INDEX %s%s";
         2849  +        zFmt = "INDEX %s";
  2857   2850         }
  2858         -      zMsg = sqlite3MAppendf(db, zMsg, zFmt, zMsg, pIdx->zName, zWhere);
  2859         -      sqlite3DbFree(db, zWhere);
         2851  +      if( zFmt ){
         2852  +        sqlite3StrAccumAppend(&str, " USING ", 7);
         2853  +        sqlite3XPrintf(&str, 0, zFmt, pIdx->zName);
         2854  +        explainIndexRange(&str, pLoop, pItem->pTab);
         2855  +      }
  2860   2856       }else if( (flags & WHERE_IPK)!=0 && (flags & WHERE_CONSTRAINT)!=0 ){
  2861         -      zMsg = sqlite3MAppendf(db, zMsg, "%s USING INTEGER PRIMARY KEY", zMsg);
  2862         -
         2857  +      const char *zRange;
  2863   2858         if( flags&(WHERE_COLUMN_EQ|WHERE_COLUMN_IN) ){
  2864         -        zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid=?)", zMsg);
         2859  +        zRange = "(rowid=?)";
  2865   2860         }else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){
  2866         -        zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid>? AND rowid<?)", zMsg);
         2861  +        zRange = "(rowid>? AND rowid<?)";
  2867   2862         }else if( flags&WHERE_BTM_LIMIT ){
  2868         -        zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid>?)", zMsg);
  2869         -      }else if( ALWAYS(flags&WHERE_TOP_LIMIT) ){
  2870         -        zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid<?)", zMsg);
         2863  +        zRange = "(rowid>?)";
         2864  +      }else{
         2865  +        assert( flags&WHERE_TOP_LIMIT);
         2866  +        zRange = "(rowid<?)";
  2871   2867         }
         2868  +      sqlite3StrAccumAppendAll(&str, " USING INTEGER PRIMARY KEY ");
         2869  +      sqlite3StrAccumAppendAll(&str, zRange);
  2872   2870       }
  2873   2871   #ifndef SQLITE_OMIT_VIRTUALTABLE
  2874   2872       else if( (flags & WHERE_VIRTUALTABLE)!=0 ){
  2875         -      zMsg = sqlite3MAppendf(db, zMsg, "%s VIRTUAL TABLE INDEX %d:%s", zMsg,
         2873  +      sqlite3XPrintf(&str, 0, " VIRTUAL TABLE INDEX %d:%s",
  2876   2874                     pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr);
  2877   2875       }
  2878   2876   #endif
  2879         -    zMsg = sqlite3MAppendf(db, zMsg, "%s", zMsg);
         2877  +#ifdef SQLITE_EXPLAIN_ESTIMATED_ROWS
         2878  +    if( pLoop->nOut>=10 ){
         2879  +      sqlite3XPrintf(&str, 0, " (~%llu rows)", sqlite3LogEstToInt(pLoop->nOut));
         2880  +    }else{
         2881  +      sqlite3StrAccumAppend(&str, " (~1 row)", 9);
         2882  +    }
         2883  +#endif
         2884  +    zMsg = sqlite3StrAccumFinish(&str);
  2880   2885       sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg, P4_DYNAMIC);
  2881   2886     }
  2882   2887   }
  2883   2888   #else
  2884   2889   # define explainOneScan(u,v,w,x,y,z)
  2885   2890   #endif /* SQLITE_OMIT_EXPLAIN */
  2886   2891   
................................................................................
  5351   5356               pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr);
  5352   5357               if( !pColl ) pColl = db->pDfltColl;
  5353   5358               if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) continue;
  5354   5359             }
  5355   5360             isMatch = 1;
  5356   5361             break;
  5357   5362           }
  5358         -        if( isMatch && (pWInfo->wctrlFlags & WHERE_GROUPBY)==0 ){
         5363  +        if( isMatch && (wctrlFlags & WHERE_GROUPBY)==0 ){
  5359   5364             /* Make sure the sort order is compatible in an ORDER BY clause.
  5360   5365             ** Sort order is irrelevant for a GROUP BY clause. */
  5361   5366             if( revSet ){
  5362   5367               if( (rev ^ revIdx)!=pOrderBy->a[i].sortOrder ) isMatch = 0;
  5363   5368             }else{
  5364   5369               rev = revIdx ^ pOrderBy->a[i].sortOrder;
  5365   5370               if( rev ) *pRevMask |= MASKBIT(iLoop);
................................................................................
  5816   5821         pWInfo->nOBSat = pFrom->isOrdered;
  5817   5822         if( pWInfo->nOBSat<0 ) pWInfo->nOBSat = 0;
  5818   5823         pWInfo->revMask = pFrom->revLoop;
  5819   5824       }
  5820   5825       if( (pWInfo->wctrlFlags & WHERE_SORTBYGROUP)
  5821   5826           && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr
  5822   5827       ){
  5823         -      Bitmask notUsed = 0;
         5828  +      Bitmask revMask = 0;
  5824   5829         int nOrder = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, 
  5825         -          pFrom, 0, nLoop-1, pFrom->aLoop[nLoop-1], &notUsed
         5830  +          pFrom, 0, nLoop-1, pFrom->aLoop[nLoop-1], &revMask
  5826   5831         );
  5827   5832         assert( pWInfo->sorted==0 );
  5828         -      pWInfo->sorted = (nOrder==pWInfo->pOrderBy->nExpr);
         5833  +      if( nOrder==pWInfo->pOrderBy->nExpr ){
         5834  +        pWInfo->sorted = 1;
         5835  +        pWInfo->revMask = revMask;
         5836  +      }
  5829   5837       }
  5830   5838     }
  5831   5839   
  5832   5840   
  5833   5841     pWInfo->nRowOut = pFrom->nRow;
  5834   5842   
  5835   5843     /* Free temporary memory and return success */

Changes to test/fts3expr4.test.

    20     20   ifcapable !fts3||!icu {
    21     21     finish_test
    22     22     return
    23     23   }
    24     24   
    25     25   set sqlite_fts3_enable_parentheses 1
    26     26   
    27         -proc test_icu_fts3expr {expr} {
    28         -  db one {SELECT fts3_exprtest('icu', $expr, 'a', 'b', 'c')}
           27  +proc test_fts3expr {tokenizer expr} {
           28  +  db one {SELECT fts3_exprtest($tokenizer, $expr, 'a', 'b', 'c')}
    29     29   }
    30     30   
    31     31   proc do_icu_expr_test {tn expr res} {
    32         -  uplevel [list do_test $tn [list test_icu_fts3expr $expr] $res]
           32  +  uplevel [list do_test $tn [list test_fts3expr icu $expr] [list {*}$res]]
           33  +}
           34  +
           35  +proc do_simple_expr_test {tn expr res} {
           36  +  uplevel [list do_test $tn [list test_fts3expr simple $expr] [list {*}$res]]
    33     37   }
    34     38   
    35     39   #-------------------------------------------------------------------------
    36     40   #
    37     41   do_icu_expr_test 1.1 "abcd"    {PHRASE 3 0 abcd}
    38     42   do_icu_expr_test 1.2 " tag "   {PHRASE 3 0 tag}
    39     43   do_icu_expr_test 1.3 {"x y z"} {PHRASE 3 0 x y z}
................................................................................
    48     52   do_icu_expr_test 1.8 {d:word} {PHRASE 3 0 d:word}
    49     53   
    50     54   set sqlite_fts3_enable_parentheses 0
    51     55   
    52     56   do_icu_expr_test 2.1 {
    53     57     f (e NEAR/2 a)
    54     58   } {AND {AND {AND {PHRASE 3 0 f} {PHRASE 3 0 (}} {NEAR/2 {PHRASE 3 0 e} {PHRASE 3 0 a}}} {PHRASE 3 0 )}}
           59  +
           60  +#-------------------------------------------------------------------------
           61  +#
           62  +do_simple_expr_test 3.1 {*lOl* *h4h*} {
           63  +  AND {PHRASE 3 0 lol+} {PHRASE 3 0 h4h+}
           64  +}
           65  +
           66  +do_icu_expr_test 3.2 {*lOl* *h4h*} {
           67  +  AND {AND {AND {PHRASE 3 0 *} {PHRASE 3 0 lol+}} {PHRASE 3 0 *}} {PHRASE 3 0 h4h+}
           68  +}
           69  +
           70  +do_simple_expr_test 3.3 { * }    { }
           71  +do_simple_expr_test 3.4 { *a }   { PHRASE 3 0 a }
           72  +do_simple_expr_test 3.5 { a*b }  { AND {PHRASE 3 0 a+} {PHRASE 3 0 b} }
           73  +do_simple_expr_test 3.6 { *a*b } { AND {PHRASE 3 0 a+} {PHRASE 3 0 b} }
           74  +do_simple_expr_test 3.7 { *"abc" } { PHRASE 3 0 abc }
           75  +do_simple_expr_test 3.8 { "abc"* } { PHRASE 3 0 abc }
           76  +do_simple_expr_test 3.8 { "ab*c" } { PHRASE 3 0 ab+ c }
           77  +
           78  +do_icu_expr_test    3.9 { "ab*c" } { PHRASE 3 0 ab+ * c }
           79  +do_icu_expr_test    3.10 { ab*c } { AND {PHRASE 3 0 ab+} {PHRASE 3 0 c}}
    55     80   
    56     81   finish_test
    57     82   

Changes to test/fts3matchinfo.test.

   428    428   
   429    429   do_execsql_test 9.1 {
   430    430     CREATE VIRTUAL TABLE ft2 USING fts4;
   431    431     INSERT INTO ft2 VALUES('a b c d e');
   432    432     INSERT INTO ft2 VALUES('f a b c d');
   433    433     SELECT snippet(ft2, '[', ']', '', -1, 1) FROM ft2 WHERE ft2 MATCH 'c';
   434    434   } {{[c]} {[c]}}
          435  +
          436  +#---------------------------------------------------------------------------
          437  +# Test for a memory leak
          438  +#
          439  +do_execsql_test 10.1 {
          440  +  DROP TABLE t10;
          441  +  CREATE VIRTUAL TABLE t10 USING fts4(idx, value);
          442  +  INSERT INTO t10 values (1, 'one'),(2, 'two'),(3, 'three');
          443  +  SELECT docId, t10.*
          444  +    FROM t10
          445  +    JOIN (SELECT 1 AS idx UNION SELECT 2 UNION SELECT 3) AS x
          446  +   WHERE t10 MATCH x.idx
          447  +     AND matchinfo(t10) not null
          448  +   GROUP BY docId
          449  +   ORDER BY 1;
          450  +} {1 1 one 2 2 two 3 3 three}
          451  +  
   435    452   
   436    453   finish_test

Changes to test/index5.test.

    12     12   
    13     13   
    14     14   set testdir [file dirname $argv0]
    15     15   source $testdir/tester.tcl
    16     16   set ::testprefix index5
    17     17   
    18     18   do_test 1.1 {
           19  +  if {[permutation]=="memsubsys1"} {
           20  +    execsql { PRAGMA auto_vacuum = 0; }
           21  +  }
    19     22     execsql {
    20     23       PRAGMA page_size = 1024;
    21     24       CREATE TABLE t1(x);
    22     25       BEGIN;
    23     26     }
    24     27     for {set i 0} {$i < 100000} {incr i} {
    25     28       execsql { INSERT INTO t1 VALUES(randstr(100,100)) }
................................................................................
    34     37   
    35     38   db close
    36     39   testvfs tvfs
    37     40   tvfs filter xWrite
    38     41   tvfs script write_cb
    39     42   proc write_cb {xCall file handle iOfst args} {
    40     43     if {[file tail $file]=="test.db"} {
    41         -    lappend ::write_list [expr $iOfst/1024]
           44  +    lappend ::write_list [expr $iOfst/1024 + 1]
    42     45     }
    43     46   }
    44     47   
    45     48   do_test 1.2 {
    46     49     sqlite3 db test.db -vfs tvfs
    47     50     set ::write_list [list]
    48     51     execsql { CREATE INDEX i1 ON t1(x) }

Changes to test/releasetest.tcl.

     9      9   This Tcl script is used to test the various configurations required
    10     10   before releasing a new version. Supported command line options (all 
    11     11   optional) are:
    12     12   
    13     13       -makefile PATH-TO-MAKEFILE           (default "releasetest.mk")
    14     14       -platform PLATFORM                   (see below)
    15     15       -quick    BOOLEAN                    (default "0")
           16  +    -config   CONFIGNAME                 (Run only CONFIGNAME)
    16     17   
    17     18   The default value for -makefile is "./releasetest.mk".
    18     19   
    19     20   The script determines the default value for -platform using the
    20     21   $tcl_platform(os) and $tcl_platform(machine) variables. Supported 
    21     22   platforms are "Linux-x86", "Linux-x86_64" and "Darwin-i386".
    22     23   
................................................................................
   288    289   # Currently the only option supported is "-makefile", default
   289    290   # "releasetest.mk". Set the ::MAKEFILE variable to the value of this
   290    291   # option.
   291    292   #
   292    293   proc process_options {argv} {
   293    294     set ::MAKEFILE releasetest.mk                       ;# Default value
   294    295     set ::QUICK    0                                    ;# Default value
          296  +  set config {}
   295    297     set platform $::tcl_platform(os)-$::tcl_platform(machine)
   296    298   
   297    299     for {set i 0} {$i < [llength $argv]} {incr i} {
   298    300       switch -- [lindex $argv $i] {
   299    301         -makefile {
   300    302           incr i
   301    303           set ::MAKEFILE [lindex $argv $i]
................................................................................
   306    308           set platform [lindex $argv $i]
   307    309         }
   308    310   
   309    311         -quick {
   310    312           incr i
   311    313           set ::QUICK [lindex $argv $i]
   312    314         }
          315  +
          316  +      -config {
          317  +        incr i
          318  +        set config [lindex $argv $i]
          319  +      }
   313    320     
   314    321         default {
   315    322           puts stderr ""
   316    323           puts stderr [string trim $::USAGE_MESSAGE]
   317    324           exit -1
   318    325         }
   319    326       }
................................................................................
   329    336         lappend print "\"$p\""
   330    337       }
   331    338       lset print end "or [lindex $print end]"
   332    339       puts "[join $print {, }]."
   333    340       exit
   334    341     }
   335    342   
   336         -  set ::CONFIGLIST $::Platforms($platform)
          343  +  if {$config!=""} {
          344  +    if {[llength $config]==1} {lappend config fulltest}
          345  +    set ::CONFIGLIST $config
          346  +  } else {
          347  +    set ::CONFIGLIST $::Platforms($platform)
          348  +  }
   337    349     puts "Running the following configurations for $platform:"
   338    350     puts "    [string trim $::CONFIGLIST]"
   339    351   }
   340    352   
   341    353   # Main routine.
   342    354   #
   343    355   proc main {argv} {

Changes to test/speedtest1.c.

   930    930       nElem, nElem
   931    931     );
   932    932     speedtest1_run();
   933    933     speedtest1_end_test();
   934    934   
   935    935   }
   936    936   
          937  +#ifdef SQLITE_ENABLE_RTREE
   937    938   /* Generate two numbers between 1 and mx.  The first number is less than
   938    939   ** the second.  Usually the numbers are near each other but can sometimes
   939    940   ** be far apart.
   940    941   */
   941    942   static void twoCoords(
   942    943     int p1, int p2,                   /* Parameters adjusting sizes */
   943    944     unsigned mx,                      /* Range of 1..mx */
................................................................................
   950    951     if( speedtest1_random()%p2==0 ) span = mx/2;
   951    952     d = speedtest1_random()%span + 1;
   952    953     x0 = speedtest1_random()%(mx-d) + 1;
   953    954     x1 = x0 + d;
   954    955     *pX0 = x0;
   955    956     *pX1 = x1;
   956    957   }
          958  +#endif
   957    959   
          960  +#ifdef SQLITE_ENABLE_RTREE
   958    961   /* The following routine is an R-Tree geometry callback.  It returns
   959    962   ** true if the object overlaps a slice on the Y coordinate between the
   960    963   ** two values given as arguments.  In other words
   961    964   **
   962    965   **     SELECT count(*) FROM rt1 WHERE id MATCH xslice(10,20);
   963    966   **
   964    967   ** Is the same as saying:
................................................................................
   970    973     int nCoord,
   971    974     double *aCoord,
   972    975     int *pRes
   973    976   ){
   974    977     *pRes = aCoord[3]>=p->aParam[0] && aCoord[2]<=p->aParam[1];
   975    978     return SQLITE_OK;
   976    979   }
          980  +#endif /* SQLITE_ENABLE_RTREE */
   977    981   
          982  +#ifdef SQLITE_ENABLE_RTREE
   978    983   /*
   979    984   ** A testset for the R-Tree virtual table
   980    985   */
   981    986   void testset_rtree(int p1, int p2){
   982    987     unsigned i, n;
   983    988     unsigned mxCoord;
   984    989     unsigned x0, x1, y0, y1, z0, z1;
................................................................................
  1106   1111     speedtest1_prepare("SELECT * FROM rt1 WHERE id=?1");
  1107   1112     for(i=1; i<=n; i++){
  1108   1113       sqlite3_bind_int(g.pStmt, 1, i);
  1109   1114       speedtest1_run();
  1110   1115     }
  1111   1116     speedtest1_end_test();
  1112   1117   }
         1118  +#endif /* SQLITE_ENABLE_RTREE */
  1113   1119   
  1114   1120   /*
  1115   1121   ** A testset used for debugging speedtest1 itself.
  1116   1122   */
  1117   1123   void testset_debug1(void){
  1118   1124     unsigned i, n;
  1119   1125     unsigned x1, x2;
................................................................................
  1325   1331     if( strcmp(zTSet,"main")==0 ){
  1326   1332       testset_main();
  1327   1333     }else if( strcmp(zTSet,"debug1")==0 ){
  1328   1334       testset_debug1();
  1329   1335     }else if( strcmp(zTSet,"cte")==0 ){
  1330   1336       testset_cte();
  1331   1337     }else if( strcmp(zTSet,"rtree")==0 ){
         1338  +#ifdef SQLITE_ENABLE_RTREE
  1332   1339       testset_rtree(6, 147);
         1340  +#else
         1341  +    fatal_error("compile with -DSQLITE_ENABLE_RTREE to enable "
         1342  +                "the R-Tree tests\n");
         1343  +#endif
  1333   1344     }else{
  1334   1345       fatal_error("unknown testset: \"%s\"\nChoices: main debug1 cte rtree\n",
  1335   1346                    zTSet);
  1336   1347     }
  1337   1348     speedtest1_final();
  1338   1349   
  1339   1350     /* Database connection statistics printed after both prepared statements

Added test/tkt-ba7cbfaedc.test.

            1  +# 2014-10-11
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +#
           12  +# Test that ticket [ba7cbfaedc] has been fixed.
           13  +#
           14  +
           15  +set testdir [file dirname $argv0]
           16  +source $testdir/tester.tcl
           17  +set testprefix tkt-ba7cbfaedc
           18  +
           19  +do_execsql_test 1 {
           20  +  CREATE TABLE t1 (x, y);
           21  +  INSERT INTO t1 VALUES (3, 'a');
           22  +  INSERT INTO t1 VALUES (1, 'a'); 
           23  +  INSERT INTO t1 VALUES (2, 'b');
           24  +  INSERT INTO t1 VALUES (2, 'a');
           25  +  INSERT INTO t1 VALUES (3, 'b');
           26  +  INSERT INTO t1 VALUES (1, 'b'); 
           27  +}
           28  +
           29  +do_execsql_test 1.1 {
           30  +  CREATE INDEX i1 ON t1(x, y);
           31  +}
           32  +
           33  +foreach {n idx} {
           34  +  1 { CREATE INDEX i1 ON t1(x, y) }
           35  +  2 { CREATE INDEX i1 ON t1(x DESC, y) }
           36  +  3 { CREATE INDEX i1 ON t1(x, y DESC) }
           37  +  4 { CREATE INDEX i1 ON t1(x DESC, y DESC) }
           38  +} {
           39  +  catchsql { DROP INDEX i1 }
           40  +  execsql $idx
           41  +  foreach {tn q res} {
           42  +    1 "GROUP BY x, y ORDER BY x, y"            {1 a 1 b   2 a 2 b   3 a 3 b}
           43  +    2 "GROUP BY x, y ORDER BY x DESC, y"       {3 a 3 b   2 a 2 b   1 a 1 b}
           44  +    3 "GROUP BY x, y ORDER BY x, y DESC"       {1 b 1 a   2 b 2 a   3 b 3 a}
           45  +    4 "GROUP BY x, y ORDER BY x DESC, y DESC"  {3 b 3 a   2 b 2 a   1 b 1 a}
           46  +  } {
           47  +    do_execsql_test 1.$n.$tn "SELECT * FROM t1 $q" $res
           48  +  }
           49  +}
           50  +
           51  +do_execsql_test 2.0 {
           52  +  drop table if exists t1;
           53  +  create table t1(id int);
           54  +  insert into t1(id) values(1),(2),(3),(4),(5);
           55  +  create index t1_idx_id on t1(id asc);
           56  +  select * from t1 group by id order by id;
           57  +  select * from t1 group by id order by id asc;
           58  +  select * from t1 group by id order by id desc;
           59  +} {
           60  +  1 2 3 4 5   1 2 3 4 5   5 4 3 2 1
           61  +}
           62  +
           63  +finish_test
           64  +
           65  +