/ Check-in [0e3f5df6]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Further performance tweaks to OP_Column.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | OP_Column-refactor
Files: files | file ages | folders
SHA1: 0e3f5df695216a27602a53eed5d25231b055adc8
User & Date: drh 2013-11-20 19:28:03
Context
2013-11-20
20:58
Improved comments on the OP_Column changes. Optimize out loading of overflow pages for content with zero length. Add test cases for the latter. check-in: 0e05679d user: drh tags: OP_Column-refactor
19:28
Further performance tweaks to OP_Column. check-in: 0e3f5df6 user: drh tags: OP_Column-refactor
17:25
Refactoring the OP_Column opcode for improved performance and maintainability. check-in: 7c914e39 user: drh tags: OP_Column-refactor
Changes
Hide Diffs Side-by-Side Diffs Show Whitespace Changes Patch

Changes to src/vdbe.c.

  2251   2251   ** If the OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG bits are set on P5 when
  2252   2252   ** the result is guaranteed to only be used as the argument of a length()
  2253   2253   ** or typeof() function, respectively.  The loading of large blobs can be
  2254   2254   ** skipped for length() and all content loading can be skipped for typeof().
  2255   2255   */
  2256   2256   case OP_Column: {
  2257   2257     i64 payloadSize64; /* Number of bytes in the record */
  2258         -  int p1;            /* P1 value of the opcode */
  2259   2258     int p2;            /* column number to retrieve */
  2260   2259     VdbeCursor *pC;    /* The VDBE cursor */
  2261   2260     BtCursor *pCrsr;   /* The BTree cursor */
  2262   2261     u32 *aType;        /* aType[i] holds the numeric type of the i-th column */
  2263   2262     u32 *aOffset;      /* aOffset[i] is offset to start of data for i-th column */
  2264         -  int nField;        /* number of fields in the record */
  2265   2263     int len;           /* The length of the serialized data for the column */
  2266   2264     int i;             /* Loop counter */
  2267   2265     Mem *pDest;        /* Where to write the extracted value */
  2268   2266     Mem sMem;          /* For storing the record being decoded */
  2269   2267     const u8 *zData;   /* Part of the record being decoded */
  2270   2268     const u8 *zHdr;    /* Next unparsed byte of the header */
  2271   2269     const u8 *zEndHdr; /* Pointer to first byte after the header */
  2272   2270     u32 offset;        /* Offset into the data */
  2273   2271     u32 szField;       /* Number of bytes in the content of a field */
  2274   2272     int avail;         /* Number of bytes of available data */
  2275   2273     u32 t;             /* A type code from the record header */
  2276   2274     Mem *pReg;         /* PseudoTable input register */
  2277   2275   
  2278         -  p1 = pOp->p1;
  2279         -  assert( p1<p->nCursor );
  2280   2276     p2 = pOp->p2;
  2281         -  sMem.zMalloc = 0;
  2282   2277     assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) );
  2283   2278     pDest = &aMem[pOp->p3];
  2284   2279     memAboutToChange(p, pDest);
  2285         -  pC = p->apCsr[p1];
         2280  +  assert( pOp->p1>=0 && pOp->p1<p->nCursor );
         2281  +  pC = p->apCsr[pOp->p1];
  2286   2282     assert( pC!=0 );
  2287         -  nField = pC->nField;
  2288         -  assert( p2<nField );
         2283  +  assert( p2<pC->nField );
  2289   2284     aType = pC->aType;
  2290   2285     aOffset = pC->aOffset;
  2291   2286   #ifndef SQLITE_OMIT_VIRTUALTABLE
  2292   2287     assert( pC->pVtabCursor==0 );
  2293   2288   #endif
  2294   2289     pCrsr = pC->pCursor;
  2295   2290     assert( pCrsr!=0 || pC->pseudoTableReg>0 );
         2291  +  assert( pC->pseudoTableReg==0 || pC->nullRow );
  2296   2292   
  2297   2293     /* If the cursor cache is stale, bring it up-to-date */
  2298   2294     rc = sqlite3VdbeCursorMoveto(pC);
  2299   2295     if( rc ) goto abort_due_to_error;
  2300   2296     if( pC->cacheStatus!=p->cacheCtr || (pOp->p5&OPFLAG_CLEARCACHE)!=0 ){
         2297  +    if( pC->nullRow ){
  2301   2298       if( pCrsr==0 ){
  2302   2299         assert( pC->pseudoTableReg>0 );
  2303   2300         pReg = &aMem[pC->pseudoTableReg];
  2304   2301         if( pC->multiPseudo ){
  2305   2302           sqlite3VdbeMemShallowCopy(pDest, pReg+p2, MEM_Ephem);
  2306   2303           Deephemeralize(pDest);
  2307   2304           goto op_column_out;
  2308   2305         }
  2309   2306         assert( pReg->flags & MEM_Blob );
  2310   2307         assert( memIsValid(pReg) );
  2311   2308         pC->payloadSize = pC->szRow = avail = pReg->n;
  2312   2309         pC->aRow = (u8*)pReg->z;
  2313         -    }else if( pC->nullRow ){
         2310  +      }else{
  2314   2311         MemSetTypeFlag(pDest, MEM_Null);
  2315   2312         goto op_column_out;
         2313  +      }
  2316   2314       }else{
         2315  +      assert( pCrsr );
  2317   2316         if( pC->isIndex ){
  2318   2317           assert( sqlite3BtreeCursorIsValid(pCrsr) );
  2319   2318           VVA_ONLY(rc =) sqlite3BtreeKeySize(pCrsr, &payloadSize64);
  2320   2319           assert( rc==SQLITE_OK ); /* True because of CursorMoveto() call above */
  2321   2320           /* sqlite3BtreeParseCellPtr() uses getVarint32() to extract the
  2322   2321           ** payload size, so it is impossible for payloadSize64 to be
  2323   2322           ** larger than 32 bits. */
................................................................................
  2357   2356       ** types use so much data space that there can only be 4096 and 32 of
  2358   2357       ** them, respectively.  So the maximum header length results from a
  2359   2358       ** 3-byte type for each of the maximum of 32768 columns plus three
  2360   2359       ** extra bytes for the header length itself.  32768*3 + 3 = 98307.
  2361   2360       */
  2362   2361       if( offset > 98307 || offset > pC->payloadSize ){
  2363   2362         rc = SQLITE_CORRUPT_BKPT;
  2364         -      goto op_column_out;
         2363  +      goto op_column_error;
  2365   2364       }
  2366   2365     }
  2367   2366   
  2368   2367     /* Make sure at least the first p2+1 entries of the header have been
  2369   2368     ** parsed and valid information is in aOffset[] and aType[].
  2370   2369     */
  2371         -  if( pC->nHdrParsed<=p2 && pC->iHdrOffset<aOffset[0] ){
         2370  +  if( pC->nHdrParsed<=p2 ){
         2371  +    /* If there is more header available for parsing, try to extract 
         2372  +    ** additional fields up through the p2-th field 
         2373  +    */
         2374  +    if( pC->iHdrOffset<aOffset[0] ){
  2372   2375       /* Make sure zData points to enough of the record to cover the header. */
  2373   2376       if( pC->aRow==0 ){
  2374   2377         memset(&sMem, 0, sizeof(sMem));
  2375         -      rc = sqlite3VdbeMemFromBtree(pCrsr, 0, pC->aOffset[0], pC->isIndex,&sMem);
         2378  +        rc = sqlite3VdbeMemFromBtree(pCrsr, 0, pC->aOffset[0], pC->isIndex,
         2379  +                                     &sMem);
  2376   2380         if( rc!=SQLITE_OK ){
  2377         -        goto op_column_out;
         2381  +          goto op_column_error;
  2378   2382         }
  2379   2383         zData = (u8*)sMem.z;
  2380   2384       }else{
  2381   2385         zData = pC->aRow;
  2382   2386       }
  2383   2387   
  2384   2388       /* Fill in aType[i] and aOffset[i] values through the p2-th field. */
  2385   2389       i = pC->nHdrParsed;
  2386   2390       offset = aOffset[i];
  2387   2391       zHdr = zData + pC->iHdrOffset;
  2388         -    zEndHdr = zData + pC->aOffset[0];
  2389         -    for(; i<=p2 && zHdr<zEndHdr; i++){
         2392  +      zEndHdr = zData + aOffset[0];
         2393  +      assert( i<=p2 && zHdr<zEndHdr );
         2394  +      do{
  2390   2395         if( zHdr[0]<0x80 ){
  2391   2396           t = zHdr[0];
  2392   2397           zHdr++;
  2393   2398         }else{
  2394   2399           zHdr += sqlite3GetVarint32(zHdr, &t);
  2395   2400         }
  2396   2401         aType[i] = t;
  2397   2402         szField = sqlite3VdbeSerialTypeLen(t);
  2398   2403         offset += szField;
  2399   2404         if( offset<szField ){  /* True if offset overflows */
  2400   2405           zHdr = &zEndHdr[1];  /* Forces SQLITE_CORRUPT return below */
  2401   2406           break;
  2402   2407         }
  2403         -      aOffset[i+1] = offset;
  2404         -    }
         2408  +        i++;
         2409  +        aOffset[i] = offset;
         2410  +      }while( i<=p2 && zHdr<zEndHdr );
  2405   2411       pC->nHdrParsed = i;
  2406   2412       pC->iHdrOffset = (u32)(zHdr - zData);
  2407   2413       if( pC->aRow==0 ){
  2408   2414         sqlite3VdbeMemRelease(&sMem);
  2409   2415         sMem.flags = MEM_Null;
  2410   2416       }
  2411   2417   
................................................................................
  2416   2422       ** with a corrupt database.
  2417   2423       */
  2418   2424       if( (zHdr > zEndHdr)
  2419   2425        || (offset > pC->payloadSize)
  2420   2426        || (zHdr==zEndHdr && offset!=pC->payloadSize)
  2421   2427       ){
  2422   2428         rc = SQLITE_CORRUPT_BKPT;
         2429  +        goto op_column_error;
         2430  +      }
         2431  +    }
         2432  +
         2433  +    /* If after nHdrParsed is still not up to p2, that means that the record
         2434  +    ** has fewer than p2 columns.  So the result will be either the default
         2435  +    ** value or a NULL. */
         2436  +    if( pC->nHdrParsed<=p2 ){
         2437  +      if( pOp->p4type==P4_MEM ){
         2438  +        sqlite3VdbeMemShallowCopy(pDest, pOp->p4.pMem, MEM_Static);
         2439  +      }else{
         2440  +        MemSetTypeFlag(pDest, MEM_Null);
         2441  +      }
  2423   2442         goto op_column_out;
  2424   2443       }
  2425   2444     }
  2426   2445   
  2427   2446     /* Get the column information. If aOffset[p2] is non-zero, then 
  2428   2447     ** deserialize the value from the record. If aOffset[p2] is zero,
  2429   2448     ** then there are not enough fields in the record to satisfy the
  2430   2449     ** request.  In this case, set the value NULL or to P4 if P4 is
  2431   2450     ** a pointer to a Mem object.
  2432   2451     */
  2433         -  if( p2<pC->nHdrParsed ){
         2452  +  assert( p2<pC->nHdrParsed );
  2434   2453       assert( rc==SQLITE_OK );
  2435   2454       if( pC->szRow>=aOffset[p2+1] ){
  2436   2455         /* This is the common case where the whole row fits on a single page */
  2437   2456         VdbeMemRelease(pDest);
  2438   2457         sqlite3VdbeSerialGet(pC->aRow+aOffset[p2], aType[p2], pDest);
  2439   2458       }else{
  2440   2459         /* This branch happens only when the row overflows onto multiple pages */
................................................................................
  2444   2463         ){
  2445   2464           /* Content is irrelevant for the typeof() function and for
  2446   2465           ** the length(X) function if X is a blob.  So we might as well use
  2447   2466           ** bogus content rather than reading content from disk.  NULL works
  2448   2467           ** for text and blob and whatever is in the payloadSize64 variable
  2449   2468           ** will work for everything else. */
  2450   2469           zData = t<12 ? (u8*)&payloadSize64 : 0;
         2470  +      sMem.zMalloc = 0;
  2451   2471         }else{
  2452   2472           len = sqlite3VdbeSerialTypeLen(t);
  2453   2473           memset(&sMem, 0, sizeof(sMem));
  2454   2474           sqlite3VdbeMemMove(&sMem, pDest);
  2455   2475           rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len,  pC->isIndex,
  2456   2476                                        &sMem);
  2457   2477           if( rc!=SQLITE_OK ){
  2458         -          goto op_column_out;
         2478  +        goto op_column_error;
  2459   2479           }
  2460   2480           zData = (u8*)sMem.z;
  2461   2481         }
  2462   2482         sqlite3VdbeSerialGet(zData, t, pDest);
  2463         -    }
  2464         -    pDest->enc = encoding;
  2465         -  }else{
  2466         -    if( pOp->p4type==P4_MEM ){
  2467         -      sqlite3VdbeMemShallowCopy(pDest, pOp->p4.pMem, MEM_Static);
  2468         -    }else{
  2469         -      MemSetTypeFlag(pDest, MEM_Null);
  2470         -    }
  2471         -  }
  2472         -
  2473   2483     /* If we dynamically allocated space to hold the data (in the
  2474   2484     ** sqlite3VdbeMemFromBtree() call above) then transfer control of that
  2475   2485     ** dynamically allocated space over to the pDest structure.
  2476         -  ** This prevents a memory copy.
  2477         -  */
         2486  +    ** This prevents a memory copy. */
  2478   2487     if( sMem.zMalloc ){
  2479   2488       assert( sMem.z==sMem.zMalloc );
  2480   2489       assert( !(pDest->flags & MEM_Dyn) );
  2481   2490       assert( !(pDest->flags & (MEM_Blob|MEM_Str)) || pDest->z==sMem.z );
  2482   2491       pDest->flags &= ~(MEM_Ephem|MEM_Static);
  2483   2492       pDest->flags |= MEM_Term;
  2484   2493       pDest->z = sMem.z;
  2485   2494       pDest->zMalloc = sMem.zMalloc;
  2486   2495     }
  2487         -
  2488         -  rc = sqlite3VdbeMemMakeWriteable(pDest);
         2496  +  }
         2497  +  pDest->enc = encoding;
  2489   2498   
  2490   2499   op_column_out:
         2500  +  rc = sqlite3VdbeMemMakeWriteable(pDest);
         2501  +op_column_error:
  2491   2502     UPDATE_MAX_BLOBSIZE(pDest);
  2492   2503     REGISTER_TRACE(pOp->p3, pDest);
  2493   2504     break;
  2494   2505   }
  2495   2506   
  2496   2507   /* Opcode: Affinity P1 P2 * P4 *
  2497   2508   ** Synopsis: affinity(r[P1@P2])

Changes to test/analyze9.test.

   801    801     reset_db
   802    802     execsql {
   803    803       CREATE TABLE t1(a, UNIQUE(a));
   804    804       INSERT INTO t1 VALUES($two);
   805    805       ANALYZE;
   806    806     }
   807    807     set nByte2 [lindex [sqlite3_db_status db SCHEMA_USED 0] 1]
          808  +  puts -nonewline " (nByte=$nByte nByte2=$nByte2)"
   808    809   
   809         -  expr {$nByte2 > $nByte+900 && $nByte2 < $nByte+1050}
          810  +  expr {$nByte2 > $nByte+900 && $nByte2 < $nByte+1100}
   810    811   } {1}
   811    812   
   812    813   #-------------------------------------------------------------------------
   813    814   # Test that stat4 data may be used with partial indexes.
   814    815   #
   815    816   do_test 17.1 {
   816    817     reset_db