/ Check-in [0e05679d]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Improved comments on the OP_Column changes. Optimize out loading of overflow pages for content with zero length. Add test cases for the latter.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | OP_Column-refactor
Files: files | file ages | folders
SHA1: 0e05679db7aa302a49e087a81f85203844b98cbe
User & Date: drh 2013-11-20 20:58:00
Context
2013-11-20
21:51
Reduce the size of the VdbeCursor object from 144 to 120 bytes. check-in: 5f9d5068 user: drh tags: OP_Column-refactor
20:58
Improved comments on the OP_Column changes. Optimize out loading of overflow pages for content with zero length. Add test cases for the latter. check-in: 0e05679d user: drh tags: OP_Column-refactor
19:28
Further performance tweaks to OP_Column. check-in: 0e3f5df6 user: drh tags: OP_Column-refactor
Changes
Hide Diffs Unified Diffs Show Whitespace Changes Patch

Changes to src/vdbe.c.

2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
....
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347




2348
2349
2350
2351
2352
2353
2354
....
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
....
2426
2427
2428
2429
2430
2431
2432

2433
2434
2435

2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455

2456
2457
2458
2459
2460
2461
2462

2463
2464
2465
2466
2467
2468

2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
  assert( pOp->p1>=0 && pOp->p1<p->nCursor );
  pC = p->apCsr[pOp->p1];
  assert( pC!=0 );
  assert( p2<pC->nField );
  aType = pC->aType;
  aOffset = pC->aOffset;
#ifndef SQLITE_OMIT_VIRTUALTABLE
  assert( pC->pVtabCursor==0 );
#endif
  pCrsr = pC->pCursor;
  assert( pCrsr!=0 || pC->pseudoTableReg>0 );
  assert( pC->pseudoTableReg==0 || pC->nullRow );

  /* If the cursor cache is stale, bring it up-to-date */
  rc = sqlite3VdbeCursorMoveto(pC);
  if( rc ) goto abort_due_to_error;
  if( pC->cacheStatus!=p->cacheCtr || (pOp->p5&OPFLAG_CLEARCACHE)!=0 ){
    if( pC->nullRow ){
      if( pCrsr==0 ){
................................................................................
      }else{
        pC->szRow = avail;
      }
      if( pC->payloadSize > (u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){
        goto too_big;
      }
    }
      
    pC->cacheStatus = p->cacheCtr;
    pC->iHdrOffset = getVarint32(pC->aRow, offset);
    pC->nHdrParsed = 0;
    aOffset[0] = offset;
    if( avail<offset ){




      pC->aRow = 0;
      pC->szRow = 0;
    }

    /* Make sure a corrupt database has not given us an oversize header.
    ** Do this now to avoid an oversize memory allocation.
    **
................................................................................
    }
  }

  /* Make sure at least the first p2+1 entries of the header have been
  ** parsed and valid information is in aOffset[] and aType[].
  */
  if( pC->nHdrParsed<=p2 ){
    /* If there is more header available for parsing, try to extract 
    ** additional fields up through the p2-th field 
    */
    if( pC->iHdrOffset<aOffset[0] ){
      /* Make sure zData points to enough of the record to cover the header. */
      if( pC->aRow==0 ){
        memset(&sMem, 0, sizeof(sMem));
        rc = sqlite3VdbeMemFromBtree(pCrsr, 0, pC->aOffset[0], pC->isIndex,
                                     &sMem);
................................................................................
       || (zHdr==zEndHdr && offset!=pC->payloadSize)
      ){
        rc = SQLITE_CORRUPT_BKPT;
        goto op_column_error;
      }
    }


    /* If after nHdrParsed is still not up to p2, that means that the record
    ** has fewer than p2 columns.  So the result will be either the default
    ** value or a NULL. */

    if( pC->nHdrParsed<=p2 ){
      if( pOp->p4type==P4_MEM ){
        sqlite3VdbeMemShallowCopy(pDest, pOp->p4.pMem, MEM_Static);
      }else{
        MemSetTypeFlag(pDest, MEM_Null);
      }
      goto op_column_out;
    }
  }

  /* Get the column information. If aOffset[p2] is non-zero, then 
  ** deserialize the value from the record. If aOffset[p2] is zero,
  ** then there are not enough fields in the record to satisfy the
  ** request.  In this case, set the value NULL or to P4 if P4 is
  ** a pointer to a Mem object.
  */
  assert( p2<pC->nHdrParsed );
  assert( rc==SQLITE_OK );
  if( pC->szRow>=aOffset[p2+1] ){
    /* This is the common case where the whole row fits on a single page */

    VdbeMemRelease(pDest);
    sqlite3VdbeSerialGet(pC->aRow+aOffset[p2], aType[p2], pDest);
  }else{
    /* This branch happens only when the row overflows onto multiple pages */
    t = aType[p2];
    if( (pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0
     && ((t>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0)

    ){
      /* Content is irrelevant for the typeof() function and for
      ** the length(X) function if X is a blob.  So we might as well use
      ** bogus content rather than reading content from disk.  NULL works
      ** for text and blob and whatever is in the payloadSize64 variable
      ** will work for everything else. */

      zData = t<12 ? (u8*)&payloadSize64 : 0;
      sMem.zMalloc = 0;
    }else{
      len = sqlite3VdbeSerialTypeLen(t);
      memset(&sMem, 0, sizeof(sMem));
      sqlite3VdbeMemMove(&sMem, pDest);
      rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len,  pC->isIndex,
                                   &sMem);
      if( rc!=SQLITE_OK ){
        goto op_column_error;
      }







|


|
|







 







<





>
>
>
>







 







|
|







 







>
|
|
<
>










|
|
|
<
<




|
>



|

|
|
>





|
>
|


<







2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
....
2335
2336
2337
2338
2339
2340
2341

2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
....
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
....
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438

2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452


2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476

2477
2478
2479
2480
2481
2482
2483
  assert( pOp->p1>=0 && pOp->p1<p->nCursor );
  pC = p->apCsr[pOp->p1];
  assert( pC!=0 );
  assert( p2<pC->nField );
  aType = pC->aType;
  aOffset = pC->aOffset;
#ifndef SQLITE_OMIT_VIRTUALTABLE
  assert( pC->pVtabCursor==0 ); /* OP_Column never called on virtual table */
#endif
  pCrsr = pC->pCursor;
  assert( pCrsr!=0 || pC->pseudoTableReg>0 ); /* pCrsr NULL on PseudoTables */
  assert( pCrsr!=0 || pC->nullRow );          /* pC->nullRow on PseudoTables */

  /* If the cursor cache is stale, bring it up-to-date */
  rc = sqlite3VdbeCursorMoveto(pC);
  if( rc ) goto abort_due_to_error;
  if( pC->cacheStatus!=p->cacheCtr || (pOp->p5&OPFLAG_CLEARCACHE)!=0 ){
    if( pC->nullRow ){
      if( pCrsr==0 ){
................................................................................
      }else{
        pC->szRow = avail;
      }
      if( pC->payloadSize > (u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){
        goto too_big;
      }
    }

    pC->cacheStatus = p->cacheCtr;
    pC->iHdrOffset = getVarint32(pC->aRow, offset);
    pC->nHdrParsed = 0;
    aOffset[0] = offset;
    if( avail<offset ){
      /* pC->aRow does not have to hold the entire row, but it does at least
      ** need to cover the header of the record.  If pC->aRow does not contain
      ** the complete header, then set it to zero, forcing the header to be
      ** dynamically allocated. */
      pC->aRow = 0;
      pC->szRow = 0;
    }

    /* Make sure a corrupt database has not given us an oversize header.
    ** Do this now to avoid an oversize memory allocation.
    **
................................................................................
    }
  }

  /* Make sure at least the first p2+1 entries of the header have been
  ** parsed and valid information is in aOffset[] and aType[].
  */
  if( pC->nHdrParsed<=p2 ){
    /* If there is more header available for parsing in the record, try
    ** to extract additional fields up through the p2+1-th field 
    */
    if( pC->iHdrOffset<aOffset[0] ){
      /* Make sure zData points to enough of the record to cover the header. */
      if( pC->aRow==0 ){
        memset(&sMem, 0, sizeof(sMem));
        rc = sqlite3VdbeMemFromBtree(pCrsr, 0, pC->aOffset[0], pC->isIndex,
                                     &sMem);
................................................................................
       || (zHdr==zEndHdr && offset!=pC->payloadSize)
      ){
        rc = SQLITE_CORRUPT_BKPT;
        goto op_column_error;
      }
    }

    /* If after trying to extra new entries from the header, nHdrParsed is
    ** still not up to p2, that means that the record has fewer than p2
    ** columns.  So the result will be either the default value or a NULL.

    */
    if( pC->nHdrParsed<=p2 ){
      if( pOp->p4type==P4_MEM ){
        sqlite3VdbeMemShallowCopy(pDest, pOp->p4.pMem, MEM_Static);
      }else{
        MemSetTypeFlag(pDest, MEM_Null);
      }
      goto op_column_out;
    }
  }

  /* Extract the content for the p2+1-th column.  Control can only
  ** reach this point if aOffset[p2], aOffset[p2+1], and aType[p2] are
  ** all valid.


  */
  assert( p2<pC->nHdrParsed );
  assert( rc==SQLITE_OK );
  if( pC->szRow>=aOffset[p2+1] ){
    /* This is the common case where the desired content fits on the original
    ** page - where the content is not on an overflow page */
    VdbeMemRelease(pDest);
    sqlite3VdbeSerialGet(pC->aRow+aOffset[p2], aType[p2], pDest);
  }else{
    /* This branch happens only when content is on overflow pages */	
    t = aType[p2];
    if( ((pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0
          && ((t>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0))
     || (len = sqlite3VdbeSerialTypeLen(t))==0
    ){
      /* Content is irrelevant for the typeof() function and for
      ** the length(X) function if X is a blob.  So we might as well use
      ** bogus content rather than reading content from disk.  NULL works
      ** for text and blob and whatever is in the payloadSize64 variable
      ** will work for everything else.  Content is also irrelevant if
      ** the content length is 0. */
      zData = t<=13 ? (u8*)&payloadSize64 : 0;
      sMem.zMalloc = 0;
    }else{

      memset(&sMem, 0, sizeof(sMem));
      sqlite3VdbeMemMove(&sMem, pDest);
      rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len,  pC->isIndex,
                                   &sMem);
      if( rc!=SQLITE_OK ){
        goto op_column_error;
      }

Changes to test/func.test.

1314
1315
1316
1317
1318
1319
1320


















1321
1322
1323
1324
1325
1326
1327
  db eval {SELECT sum(length(x)) FROM t29}
} {1000009}
do_test func-29.6 {
  set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1]
  if {$x<5} {set x 1}
  set x
} {1}



















# EVIDENCE-OF: R-29701-50711 The unicode(X) function returns the numeric
# unicode code point corresponding to the first character of the string
# X.
#
# EVIDENCE-OF: R-55469-62130 The char(X1,X2,...,XN) function returns a
# string composed of characters having the unicode code point values of







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
  db eval {SELECT sum(length(x)) FROM t29}
} {1000009}
do_test func-29.6 {
  set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1]
  if {$x<5} {set x 1}
  set x
} {1}

# The OP_Column opcode has an optimization that avoids loading content
# for fields with content-length=0 when the content offset is on an overflow
# page.  Make sure the optimization works.
#
do_execsql_test func-29.10 {
  CREATE TABLE t29b(a,b,c,d,e,f,g,h,i);
  INSERT INTO t29b 
   VALUES(1, hex(randomblob(2000)), null, 0, 1, '', zeroblob(0),'x',x'01');
  SELECT typeof(c), typeof(d), typeof(e), typeof(f),
         typeof(g), typeof(h), typeof(i) FROM t29b;
} {null integer integer text blob text blob}
do_execsql_test func-29.11 {
  SELECT length(f), length(g), length(h), length(i) FROM t29b;
} {0 0 1 1}
do_execsql_test func-29.12 {
  SELECT quote(f), quote(g), quote(h), quote(i) FROM t29b;
} {'' X'' 'x' X'01'}

# EVIDENCE-OF: R-29701-50711 The unicode(X) function returns the numeric
# unicode code point corresponding to the first character of the string
# X.
#
# EVIDENCE-OF: R-55469-62130 The char(X1,X2,...,XN) function returns a
# string composed of characters having the unicode code point values of