Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Refactoring the OP_Column opcode for improved performance and maintainability. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | OP_Column-refactor |
Files: | files | file ages | folders |
SHA1: |
7c914e3997d2b28164a2fa7eb4398262 |
User & Date: | drh 2013-11-20 17:25:55.912 |
Context
2013-11-20
| ||
19:28 | Further performance tweaks to OP_Column. (check-in: 0e3f5df695 user: drh tags: OP_Column-refactor) | |
17:25 | Refactoring the OP_Column opcode for improved performance and maintainability. (check-in: 7c914e3997 user: drh tags: OP_Column-refactor) | |
02:53 | Simplifications to the VdbeCursor object. (check-in: 5562cd343d user: drh tags: trunk) | |
Changes
Changes to src/vdbe.c.
︙ | ︙ | |||
210 211 212 213 214 215 216 | Mem *pMem = &p->aMem[p->nMem-iCur]; int nByte; VdbeCursor *pCx = 0; nByte = ROUND8(sizeof(VdbeCursor)) + (isBtreeCursor?sqlite3BtreeCursorSize():0) + | | > | | 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 | Mem *pMem = &p->aMem[p->nMem-iCur]; int nByte; VdbeCursor *pCx = 0; nByte = ROUND8(sizeof(VdbeCursor)) + (isBtreeCursor?sqlite3BtreeCursorSize():0) + (2*nField+2)*sizeof(u32); assert( iCur<p->nCursor ); if( p->apCsr[iCur] ){ sqlite3VdbeFreeCursor(p, p->apCsr[iCur]); p->apCsr[iCur] = 0; } if( SQLITE_OK==sqlite3VdbeMemGrow(pMem, nByte, 0) ){ p->apCsr[iCur] = pCx = (VdbeCursor*)pMem->z; memset(pCx, 0, sizeof(VdbeCursor)); pCx->iDb = iDb; pCx->nField = nField; if( nField ){ pCx->aType = (u32 *)&pMem->z[ROUND8(sizeof(VdbeCursor))]; pCx->aOffset = pCx->aType + nField; } if( isBtreeCursor ){ pCx->pCursor = (BtCursor*) &pMem->z[ROUND8(sizeof(VdbeCursor))+(2*nField+2)*sizeof(u32)]; sqlite3BtreeCursorZero(pCx->pCursor); } } return pCx; } /* |
︙ | ︙ | |||
2249 2250 2251 2252 2253 2254 2255 | ** ** If the OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG bits are set on P5 when ** the result is guaranteed to only be used as the argument of a length() ** or typeof() function, respectively. The loading of large blobs can be ** skipped for length() and all content loading can be skipped for typeof(). */ case OP_Column: { | < < < | > | < < > | < < < < < < < < < < < < < < < < > > > > | | > | | < < | < < | < < < < < < < < < < < < < < | | | | | | | | | | | < < < | < < < | | < < < < < < < < < < < < < < < < < < < < < < < < < > > > > > > > | > > > > | < < < < < | | < | | > > | > | < < > | > > > > > > | | < < < < < < < < < < < < < < < < | < < < > | > > | < | | > > | < < | | | < < < | | > | | | | | | | | | | | | | < < < < < < < | | < > > > | | > > | | > | | | | > | | | 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 | ** ** If the OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG bits are set on P5 when ** the result is guaranteed to only be used as the argument of a length() ** or typeof() function, respectively. The loading of large blobs can be ** skipped for length() and all content loading can be skipped for typeof(). */ case OP_Column: { i64 payloadSize64; /* Number of bytes in the record */ int p1; /* P1 value of the opcode */ int p2; /* column number to retrieve */ VdbeCursor *pC; /* The VDBE cursor */ BtCursor *pCrsr; /* The BTree cursor */ u32 *aType; /* aType[i] holds the numeric type of the i-th column */ u32 *aOffset; /* aOffset[i] is offset to start of data for i-th column */ int nField; /* number of fields in the record */ int len; /* The length of the serialized data for the column */ int i; /* Loop counter */ Mem *pDest; /* Where to write the extracted value */ Mem sMem; /* For storing the record being decoded */ const u8 *zData; /* Part of the record being decoded */ const u8 *zHdr; /* Next unparsed byte of the header */ const u8 *zEndHdr; /* Pointer to first byte after the header */ u32 offset; /* Offset into the data */ u32 szField; /* Number of bytes in the content of a field */ int avail; /* Number of bytes of available data */ u32 t; /* A type code from the record header */ Mem *pReg; /* PseudoTable input register */ p1 = pOp->p1; assert( p1<p->nCursor ); p2 = pOp->p2; sMem.zMalloc = 0; assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); pDest = &aMem[pOp->p3]; memAboutToChange(p, pDest); pC = p->apCsr[p1]; assert( pC!=0 ); nField = pC->nField; assert( p2<nField ); aType = pC->aType; aOffset = pC->aOffset; #ifndef SQLITE_OMIT_VIRTUALTABLE assert( pC->pVtabCursor==0 ); #endif pCrsr = pC->pCursor; assert( pCrsr!=0 || pC->pseudoTableReg>0 ); /* If the cursor cache is stale, bring it up-to-date */ rc = sqlite3VdbeCursorMoveto(pC); if( rc ) goto abort_due_to_error; if( pC->cacheStatus!=p->cacheCtr || (pOp->p5&OPFLAG_CLEARCACHE)!=0 ){ if( pCrsr==0 ){ assert( pC->pseudoTableReg>0 ); pReg = &aMem[pC->pseudoTableReg]; if( pC->multiPseudo ){ sqlite3VdbeMemShallowCopy(pDest, pReg+p2, MEM_Ephem); Deephemeralize(pDest); goto op_column_out; } assert( pReg->flags & MEM_Blob ); assert( memIsValid(pReg) ); pC->payloadSize = pC->szRow = avail = pReg->n; pC->aRow = (u8*)pReg->z; }else if( pC->nullRow ){ MemSetTypeFlag(pDest, MEM_Null); goto op_column_out; }else{ if( pC->isIndex ){ assert( sqlite3BtreeCursorIsValid(pCrsr) ); VVA_ONLY(rc =) sqlite3BtreeKeySize(pCrsr, &payloadSize64); assert( rc==SQLITE_OK ); /* True because of CursorMoveto() call above */ /* sqlite3BtreeParseCellPtr() uses getVarint32() to extract the ** payload size, so it is impossible for payloadSize64 to be ** larger than 32 bits. */ assert( (payloadSize64 & SQLITE_MAX_U32)==(u64)payloadSize64 ); pC->aRow = sqlite3BtreeKeyFetch(pCrsr, &avail); pC->payloadSize = (u32)payloadSize64; }else{ assert( sqlite3BtreeCursorIsValid(pCrsr) ); VVA_ONLY(rc =) sqlite3BtreeDataSize(pCrsr, &pC->payloadSize); assert( rc==SQLITE_OK ); /* DataSize() cannot fail */ pC->aRow = sqlite3BtreeDataFetch(pCrsr, &avail); } assert( avail<=65536 ); /* Maximum page size is 64KiB */ if( pC->payloadSize <= (u32)avail ){ pC->szRow = pC->payloadSize; }else{ pC->szRow = avail; } if( pC->payloadSize > (u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){ goto too_big; } } pC->cacheStatus = p->cacheCtr; pC->iHdrOffset = getVarint32(pC->aRow, offset); pC->nHdrParsed = 0; aOffset[0] = offset; if( avail<offset ){ pC->aRow = 0; pC->szRow = 0; } /* Make sure a corrupt database has not given us an oversize header. ** Do this now to avoid an oversize memory allocation. ** ** Type entries can be between 1 and 5 bytes each. But 4 and 5 byte ** types use so much data space that there can only be 4096 and 32 of ** them, respectively. So the maximum header length results from a ** 3-byte type for each of the maximum of 32768 columns plus three ** extra bytes for the header length itself. 32768*3 + 3 = 98307. */ if( offset > 98307 || offset > pC->payloadSize ){ rc = SQLITE_CORRUPT_BKPT; goto op_column_out; } } /* Make sure at least the first p2+1 entries of the header have been ** parsed and valid information is in aOffset[] and aType[]. */ if( pC->nHdrParsed<=p2 && pC->iHdrOffset<aOffset[0] ){ /* Make sure zData points to enough of the record to cover the header. */ if( pC->aRow==0 ){ memset(&sMem, 0, sizeof(sMem)); rc = sqlite3VdbeMemFromBtree(pCrsr, 0, pC->aOffset[0], pC->isIndex,&sMem); if( rc!=SQLITE_OK ){ goto op_column_out; } zData = (u8*)sMem.z; }else{ zData = pC->aRow; } /* Fill in aType[i] and aOffset[i] values through the p2-th field. */ i = pC->nHdrParsed; offset = aOffset[i]; zHdr = zData + pC->iHdrOffset; zEndHdr = zData + pC->aOffset[0]; for(; i<=p2 && zHdr<zEndHdr; i++){ if( zHdr[0]<0x80 ){ t = zHdr[0]; zHdr++; }else{ zHdr += sqlite3GetVarint32(zHdr, &t); } aType[i] = t; szField = sqlite3VdbeSerialTypeLen(t); offset += szField; if( offset<szField ){ /* True if offset overflows */ zHdr = &zEndHdr[1]; /* Forces SQLITE_CORRUPT return below */ break; } aOffset[i+1] = offset; } pC->nHdrParsed = i; pC->iHdrOffset = (u32)(zHdr - zData); if( pC->aRow==0 ){ sqlite3VdbeMemRelease(&sMem); sMem.flags = MEM_Null; } /* If we have read more header data than was contained in the header, ** or if the end of the last field appears to be past the end of the ** record, or if the end of the last field appears to be before the end ** of the record (when all fields present), then we must be dealing ** with a corrupt database. */ if( (zHdr > zEndHdr) || (offset > pC->payloadSize) || (zHdr==zEndHdr && offset!=pC->payloadSize) ){ rc = SQLITE_CORRUPT_BKPT; goto op_column_out; } } /* Get the column information. If aOffset[p2] is non-zero, then ** deserialize the value from the record. If aOffset[p2] is zero, ** then there are not enough fields in the record to satisfy the ** request. In this case, set the value NULL or to P4 if P4 is ** a pointer to a Mem object. */ if( p2<pC->nHdrParsed ){ assert( rc==SQLITE_OK ); if( pC->szRow>=aOffset[p2+1] ){ /* This is the common case where the whole row fits on a single page */ VdbeMemRelease(pDest); sqlite3VdbeSerialGet(pC->aRow+aOffset[p2], aType[p2], pDest); }else{ /* This branch happens only when the row overflows onto multiple pages */ t = aType[p2]; if( (pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0 && ((t>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0) ){ /* Content is irrelevant for the typeof() function and for ** the length(X) function if X is a blob. So we might as well use ** bogus content rather than reading content from disk. NULL works ** for text and blob and whatever is in the payloadSize64 variable ** will work for everything else. */ zData = t<12 ? (u8*)&payloadSize64 : 0; }else{ len = sqlite3VdbeSerialTypeLen(t); memset(&sMem, 0, sizeof(sMem)); sqlite3VdbeMemMove(&sMem, pDest); rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len, pC->isIndex, &sMem); if( rc!=SQLITE_OK ){ goto op_column_out; } zData = (u8*)sMem.z; } sqlite3VdbeSerialGet(zData, t, pDest); } pDest->enc = encoding; }else{ if( pOp->p4type==P4_MEM ){ sqlite3VdbeMemShallowCopy(pDest, pOp->p4.pMem, MEM_Static); }else{ MemSetTypeFlag(pDest, MEM_Null); |
︙ | ︙ | |||
3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 | assert( pKeyInfo->enc==ENC(db) ); assert( pKeyInfo->db==db ); nField = pKeyInfo->nField+pKeyInfo->nXField; }else if( pOp->p4type==P4_INT32 ){ nField = pOp->p4.i; } assert( pOp->p1>=0 ); pCur = allocateCursor(p, pOp->p1, nField, iDb, 1); if( pCur==0 ) goto no_mem; pCur->nullRow = 1; pCur->isOrdered = 1; rc = sqlite3BtreeCursor(pX, p2, wrFlag, pKeyInfo, pCur->pCursor); pCur->pKeyInfo = pKeyInfo; assert( OPFLAG_BULKCSR==BTREE_BULKLOAD ); | > > | 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 | assert( pKeyInfo->enc==ENC(db) ); assert( pKeyInfo->db==db ); nField = pKeyInfo->nField+pKeyInfo->nXField; }else if( pOp->p4type==P4_INT32 ){ nField = pOp->p4.i; } assert( pOp->p1>=0 ); assert( nField>=0 ); testcase( nField==0 ); /* Table with INTEGER PRIMARY KEY and nothing else */ pCur = allocateCursor(p, pOp->p1, nField, iDb, 1); if( pCur==0 ) goto no_mem; pCur->nullRow = 1; pCur->isOrdered = 1; rc = sqlite3BtreeCursor(pX, p2, wrFlag, pKeyInfo, pCur->pCursor); pCur->pKeyInfo = pKeyInfo; assert( OPFLAG_BULKCSR==BTREE_BULKLOAD ); |
︙ | ︙ | |||
3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 | static const int vfsFlags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE | SQLITE_OPEN_TRANSIENT_DB; assert( pOp->p1>=0 ); pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); if( pCx==0 ) goto no_mem; pCx->nullRow = 1; rc = sqlite3BtreeOpen(db->pVfs, 0, db, &pCx->pBt, BTREE_OMIT_JOURNAL | BTREE_SINGLE | pOp->p5, vfsFlags); if( rc==SQLITE_OK ){ rc = sqlite3BtreeBeginTrans(pCx->pBt, 1); | > | 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 | static const int vfsFlags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE | SQLITE_OPEN_TRANSIENT_DB; assert( pOp->p1>=0 ); assert( pOp->p2>=0 ); pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); if( pCx==0 ) goto no_mem; pCx->nullRow = 1; rc = sqlite3BtreeOpen(db->pVfs, 0, db, &pCx->pBt, BTREE_OMIT_JOURNAL | BTREE_SINGLE | pOp->p5, vfsFlags); if( rc==SQLITE_OK ){ rc = sqlite3BtreeBeginTrans(pCx->pBt, 1); |
︙ | ︙ | |||
3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 | ** This opcode works like OP_OpenEphemeral except that it opens ** a transient index that is specifically designed to sort large ** tables using an external merge-sort algorithm. */ case OP_SorterOpen: { VdbeCursor *pCx; pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); if( pCx==0 ) goto no_mem; pCx->pKeyInfo = pOp->p4.pKeyInfo; assert( pCx->pKeyInfo->db==db ); assert( pCx->pKeyInfo->enc==ENC(db) ); pCx->isSorter = 1; rc = sqlite3VdbeSorterInit(db, pCx); | > > | 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 | ** This opcode works like OP_OpenEphemeral except that it opens ** a transient index that is specifically designed to sort large ** tables using an external merge-sort algorithm. */ case OP_SorterOpen: { VdbeCursor *pCx; assert( pOp->p1>=0 ); assert( pOp->p2>=0 ); pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); if( pCx==0 ) goto no_mem; pCx->pKeyInfo = pOp->p4.pKeyInfo; assert( pCx->pKeyInfo->db==db ); assert( pCx->pKeyInfo->enc==ENC(db) ); pCx->isSorter = 1; rc = sqlite3VdbeSorterInit(db, pCx); |
︙ | ︙ | |||
3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 | ** P3 is the number of fields in the records that will be stored by ** the pseudo-table. */ case OP_OpenPseudo: { VdbeCursor *pCx; assert( pOp->p1>=0 ); pCx = allocateCursor(p, pOp->p1, pOp->p3, -1, 0); if( pCx==0 ) goto no_mem; pCx->nullRow = 1; pCx->pseudoTableReg = pOp->p2; pCx->isTable = 1; pCx->isIndex = 0; pCx->multiPseudo = pOp->p5; | > | 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 | ** P3 is the number of fields in the records that will be stored by ** the pseudo-table. */ case OP_OpenPseudo: { VdbeCursor *pCx; assert( pOp->p1>=0 ); assert( pOp->p3>=0 ); pCx = allocateCursor(p, pOp->p1, pOp->p3, -1, 0); if( pCx==0 ) goto no_mem; pCx->nullRow = 1; pCx->pseudoTableReg = pOp->p2; pCx->isTable = 1; pCx->isIndex = 0; pCx->multiPseudo = pOp->p5; |
︙ | ︙ | |||
4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 | VdbeCursor *pC; assert( pOp->p1>=0 && pOp->p1<p->nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); pC->nullRow = 1; pC->rowidIsValid = 0; assert( pC->pCursor || pC->pVtabCursor ); if( pC->pCursor ){ sqlite3BtreeClearCursor(pC->pCursor); } break; } | > | 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 | VdbeCursor *pC; assert( pOp->p1>=0 && pOp->p1<p->nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); pC->nullRow = 1; pC->rowidIsValid = 0; pC->cacheStatus = CACHE_STALE; assert( pC->pCursor || pC->pVtabCursor ); if( pC->pCursor ){ sqlite3BtreeClearCursor(pC->pCursor); } break; } |
︙ | ︙ |
Changes to src/vdbeInt.h.
︙ | ︙ | |||
89 90 91 92 93 94 95 | ** CACHE_STALE and so setting cacheStatus=CACHE_STALE guarantees that ** the cache is out of date. ** ** aRow might point to (ephemeral) data for the current row, or it might ** be NULL. */ u32 cacheStatus; /* Cache is valid if this matches Vdbe.cacheCtr */ | | > > > > | | 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 | ** CACHE_STALE and so setting cacheStatus=CACHE_STALE guarantees that ** the cache is out of date. ** ** aRow might point to (ephemeral) data for the current row, or it might ** be NULL. */ u32 cacheStatus; /* Cache is valid if this matches Vdbe.cacheCtr */ u32 payloadSize; /* Total number of bytes in the record */ u16 nHdrParsed; /* Number of header fields parsed so far */ u16 nFieldPresent; /* Number of fields in the record */ u32 szRow; /* Byte available in aRow */ u32 iHdrOffset; /* Offset to next unparsed byte of the header */ u32 *aType; /* Type values for all entries in the record */ u32 *aOffset; /* Cached offsets to the start of each columns data */ const u8 *aRow; /* Data for the current row, if all on one page */ }; typedef struct VdbeCursor VdbeCursor; /* ** When a sub-program is executed (OP_Program), a structure of this type ** is allocated to store the current value of the program counter, as ** well as the current memory cell array and various other frame specific |
︙ | ︙ |
Changes to src/vdbeaux.c.
︙ | ︙ | |||
2654 2655 2656 2657 2658 2659 2660 | if( res!=0 ) return SQLITE_CORRUPT_BKPT; p->rowidIsValid = 1; #ifdef SQLITE_TEST sqlite3_search_count++; #endif p->deferredMoveto = 0; p->cacheStatus = CACHE_STALE; | | | 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 | if( res!=0 ) return SQLITE_CORRUPT_BKPT; p->rowidIsValid = 1; #ifdef SQLITE_TEST sqlite3_search_count++; #endif p->deferredMoveto = 0; p->cacheStatus = CACHE_STALE; }else if( p->pCursor ){ int hasMoved; int rc = sqlite3BtreeCursorHasMoved(p->pCursor, &hasMoved); if( rc ) return rc; if( hasMoved ){ p->cacheStatus = CACHE_STALE; p->nullRow = 1; } |
︙ | ︙ |