/ Check-in [aeb8e9a9]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge sorter optimization into this branch.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | insert-select-opt
Files: files | file ages | folders
SHA1: aeb8e9a9f2092fda0c3e051f135e1d65cf77ff13
User & Date: dan 2015-03-26 12:38:20
Context
2015-03-30
15:45
Merge sorter optimizations with this branch. check-in: 9bf1cfb4 user: dan tags: insert-select-opt
2015-03-26
12:38
Merge sorter optimization into this branch. check-in: aeb8e9a9 user: dan tags: insert-select-opt
11:55
Optimize cases where all the sorter is sorting a set of records that all begin with integer values, or that all begin with text values to be compared using BINARY. check-in: ce5ad17c user: dan tags: sorter-opt
2015-03-24
19:43
Update this branch with latest trunk changes. check-in: 3ccd64ef user: dan tags: insert-select-opt
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to Makefile.msc.

   308    308   # also be noted here that building any target with these "stdcall" options
   309    309   # will most likely fail if the Tcl library is also required.  This is due
   310    310   # to how the Tcl library functions are declared and exported (i.e. without
   311    311   # an explicit calling convention, which results in "cdecl").
   312    312   #
   313    313   !IF $(USE_STDCALL)!=0
   314    314   !IF "$(PLATFORM)"=="x86"
   315         -CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl
   316         -SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl
          315  +CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
          316  +SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
   317    317   !ELSE
   318    318   !IFNDEF PLATFORM
   319         -CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl
   320         -SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl
          319  +CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
          320  +SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
   321    321   !ELSE
   322    322   CORE_CCONV_OPTS =
   323    323   SHELL_CCONV_OPTS =
   324    324   !ENDIF
   325    325   !ENDIF
   326    326   !ELSE
   327    327   CORE_CCONV_OPTS =

Changes to src/btree.c.

   596    596   **
   597    597   ** The caller must ensure that the cursor is valid (has eState==CURSOR_VALID)
   598    598   ** prior to calling this routine.  
   599    599   */
   600    600   static int saveCursorPosition(BtCursor *pCur){
   601    601     int rc;
   602    602   
   603         -  assert( CURSOR_VALID==pCur->eState );
          603  +  assert( CURSOR_VALID==pCur->eState || CURSOR_SKIPNEXT==pCur->eState );
   604    604     assert( 0==pCur->pKey );
   605    605     assert( cursorHoldsMutex(pCur) );
   606    606   
          607  +  if( pCur->eState==CURSOR_SKIPNEXT ){
          608  +    pCur->eState = CURSOR_VALID;
          609  +  }else{
          610  +    pCur->skipNext = 0;
          611  +  }
   607    612     rc = sqlite3BtreeKeySize(pCur, &pCur->nKey);
   608    613     assert( rc==SQLITE_OK );  /* KeySize() cannot fail */
   609    614   
   610    615     /* If this is an intKey table, then the above call to BtreeKeySize()
   611    616     ** stores the integer key in pCur->nKey. In this case this value is
   612    617     ** all that is required. Otherwise, if pCur is not open on an intKey
   613    618     ** table, then malloc space for and store the pCur->nKey bytes of key 
................................................................................
   670    675   static int SQLITE_NOINLINE saveCursorsOnList(
   671    676     BtCursor *p,         /* The first cursor that needs saving */
   672    677     Pgno iRoot,          /* Only save cursor with this iRoot. Save all if zero */
   673    678     BtCursor *pExcept    /* Do not save this cursor */
   674    679   ){
   675    680     do{
   676    681       if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) ){
   677         -      if( p->eState==CURSOR_VALID ){
          682  +      if( p->eState==CURSOR_VALID || p->eState==CURSOR_SKIPNEXT ){
   678    683           int rc = saveCursorPosition(p);
   679    684           if( SQLITE_OK!=rc ){
   680    685             return rc;
   681    686           }
   682    687         }else{
   683    688           testcase( p->iPage>0 );
   684    689           btreeReleaseAllCursorPages(p);
................................................................................
   742    747   ** when saveCursorPosition() was called. Note that this call deletes the 
   743    748   ** saved position info stored by saveCursorPosition(), so there can be
   744    749   ** at most one effective restoreCursorPosition() call after each 
   745    750   ** saveCursorPosition().
   746    751   */
   747    752   static int btreeRestoreCursorPosition(BtCursor *pCur){
   748    753     int rc;
          754  +  int skipNext;
   749    755     assert( cursorHoldsMutex(pCur) );
   750    756     assert( pCur->eState>=CURSOR_REQUIRESEEK );
   751    757     if( pCur->eState==CURSOR_FAULT ){
   752    758       return pCur->skipNext;
   753    759     }
   754    760     pCur->eState = CURSOR_INVALID;
   755         -  rc = btreeMoveto(pCur, pCur->pKey, pCur->nKey, 0, &pCur->skipNext);
          761  +  rc = btreeMoveto(pCur, pCur->pKey, pCur->nKey, 0, &skipNext);
   756    762     if( rc==SQLITE_OK ){
   757    763       sqlite3_free(pCur->pKey);
   758    764       pCur->pKey = 0;
   759    765       assert( pCur->eState==CURSOR_VALID || pCur->eState==CURSOR_INVALID );
          766  +    pCur->skipNext |= skipNext;
   760    767       if( pCur->skipNext && pCur->eState==CURSOR_VALID ){
   761    768         pCur->eState = CURSOR_SKIPNEXT;
   762    769       }
   763    770     }
   764    771     return rc;
   765    772   }
   766    773   
................................................................................
   804    811     assert( pCur!=0 );
   805    812     assert( pCur->eState!=CURSOR_VALID );
   806    813     rc = restoreCursorPosition(pCur);
   807    814     if( rc ){
   808    815       *pDifferentRow = 1;
   809    816       return rc;
   810    817     }
   811         -  if( pCur->eState!=CURSOR_VALID || NEVER(pCur->skipNext!=0) ){
          818  +  if( pCur->eState!=CURSOR_VALID ){
   812    819       *pDifferentRow = 1;
   813    820     }else{
          821  +    assert( pCur->skipNext==0 );
   814    822       *pDifferentRow = 0;
   815    823     }
   816    824     return SQLITE_OK;
   817    825   }
   818    826   
   819    827   #ifndef SQLITE_OMIT_AUTOVACUUM
   820    828   /*
................................................................................
  3621   3629   
  3622   3630     assert( (writeOnly==0 || writeOnly==1) && BTCF_WriteFlag==1 );
  3623   3631     if( pBtree ){
  3624   3632       sqlite3BtreeEnter(pBtree);
  3625   3633       for(p=pBtree->pBt->pCursor; p; p=p->pNext){
  3626   3634         int i;
  3627   3635         if( writeOnly && (p->curFlags & BTCF_WriteFlag)==0 ){
  3628         -        if( p->eState==CURSOR_VALID ){
         3636  +        if( p->eState==CURSOR_VALID || p->eState==CURSOR_SKIPNEXT ){
  3629   3637             rc = saveCursorPosition(p);
  3630   3638             if( rc!=SQLITE_OK ){
  3631   3639               (void)sqlite3BtreeTripAllCursors(pBtree, rc, 0);
  3632   3640               break;
  3633   3641             }
  3634   3642           }
  3635   3643         }else{
................................................................................
  4027   4035   ** Failure is not possible.  This function always returns SQLITE_OK.
  4028   4036   ** It might just as well be a procedure (returning void) but we continue
  4029   4037   ** to return an integer result code for historical reasons.
  4030   4038   */
  4031   4039   int sqlite3BtreeDataSize(BtCursor *pCur, u32 *pSize){
  4032   4040     assert( cursorHoldsMutex(pCur) );
  4033   4041     assert( pCur->eState==CURSOR_VALID );
         4042  +  assert( pCur->iPage>=0 );
         4043  +  assert( pCur->iPage<BTCURSOR_MAX_DEPTH );
  4034   4044     assert( pCur->apPage[pCur->iPage]->intKeyLeaf==1 );
  4035   4045     getCellInfo(pCur);
  4036   4046     *pSize = pCur->info.nPayload;
  4037   4047     return SQLITE_OK;
  4038   4048   }
  4039   4049   
  4040   4050   /*
................................................................................
  4505   4515     pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl);
  4506   4516     if( pNewPage->nCell<1 || pNewPage->intKey!=pCur->apPage[i]->intKey ){
  4507   4517       return SQLITE_CORRUPT_BKPT;
  4508   4518     }
  4509   4519     return SQLITE_OK;
  4510   4520   }
  4511   4521   
  4512         -#if 0
         4522  +#if SQLITE_DEBUG
  4513   4523   /*
  4514   4524   ** Page pParent is an internal (non-leaf) tree page. This function 
  4515   4525   ** asserts that page number iChild is the left-child if the iIdx'th
  4516   4526   ** cell in page pParent. Or, if iIdx is equal to the total number of
  4517   4527   ** cells in pParent, that page number iChild is the right-child of
  4518   4528   ** the page.
  4519   4529   */
  4520   4530   static void assertParentIndex(MemPage *pParent, int iIdx, Pgno iChild){
         4531  +  if( CORRUPT_DB ) return;  /* The conditions tested below might not be true
         4532  +                            ** in a corrupt database */
  4521   4533     assert( iIdx<=pParent->nCell );
  4522   4534     if( iIdx==pParent->nCell ){
  4523   4535       assert( get4byte(&pParent->aData[pParent->hdrOffset+8])==iChild );
  4524   4536     }else{
  4525   4537       assert( get4byte(findCell(pParent, iIdx))==iChild );
  4526   4538     }
  4527   4539   }
................................................................................
  4538   4550   ** the largest cell index.
  4539   4551   */
  4540   4552   static void moveToParent(BtCursor *pCur){
  4541   4553     assert( cursorHoldsMutex(pCur) );
  4542   4554     assert( pCur->eState==CURSOR_VALID );
  4543   4555     assert( pCur->iPage>0 );
  4544   4556     assert( pCur->apPage[pCur->iPage] );
  4545         -
  4546         -  /* UPDATE: It is actually possible for the condition tested by the assert
  4547         -  ** below to be untrue if the database file is corrupt. This can occur if
  4548         -  ** one cursor has modified page pParent while a reference to it is held 
  4549         -  ** by a second cursor. Which can only happen if a single page is linked
  4550         -  ** into more than one b-tree structure in a corrupt database.  */
  4551         -#if 0
  4552   4557     assertParentIndex(
  4553   4558       pCur->apPage[pCur->iPage-1], 
  4554   4559       pCur->aiIdx[pCur->iPage-1], 
  4555   4560       pCur->apPage[pCur->iPage]->pgno
  4556   4561     );
  4557         -#endif
  4558   4562     testcase( pCur->aiIdx[pCur->iPage-1] > pCur->apPage[pCur->iPage-1]->nCell );
  4559   4563   
  4560   4564     releasePage(pCur->apPage[pCur->iPage]);
  4561   4565     pCur->iPage--;
  4562   4566     pCur->info.nSize = 0;
  4563   4567     pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl);
  4564   4568   }
................................................................................
  7498   7502         }
  7499   7503   
  7500   7504         pPage->nOverflow = 0;
  7501   7505   
  7502   7506         /* The next iteration of the do-loop balances the parent page. */
  7503   7507         releasePage(pPage);
  7504   7508         pCur->iPage--;
         7509  +      assert( pCur->iPage>=0 );
  7505   7510       }
  7506   7511     }while( rc==SQLITE_OK );
  7507   7512   
  7508   7513     if( pFree ){
  7509   7514       sqlite3PageFree(pFree);
  7510   7515     }
  7511   7516     return rc;

Changes to src/sqlite.h.in.

    39     39   */
    40     40   #ifdef __cplusplus
    41     41   extern "C" {
    42     42   #endif
    43     43   
    44     44   
    45     45   /*
    46         -** Add the ability to override 'extern'
           46  +** Provide the ability to override linkage features of the interface.
    47     47   */
    48     48   #ifndef SQLITE_EXTERN
    49     49   # define SQLITE_EXTERN extern
    50     50   #endif
    51         -
    52         -/*
    53         -** Add the ability to override 'cdecl'
    54         -*/
           51  +#ifndef SQLITE_API
           52  +# define SQLITE_API
           53  +#endif
    55     54   #ifndef SQLITE_CDECL
    56     55   # define SQLITE_CDECL
    57     56   #endif
           57  +#ifndef SQLITE_STDCALL
           58  +# define SQLITE_STDCALL
           59  +#endif
    58     60   
    59     61   /*
    60     62   ** These no-op macros are used in front of interfaces to mark those
    61     63   ** interfaces as either deprecated or experimental.  New applications
    62     64   ** should not use deprecated interfaces - they are supported for backwards
    63     65   ** compatibility only.  Application writers should be aware that
    64     66   ** experimental interfaces are subject to change in point releases.

Changes to src/vdbesort.c.

   294    294   struct SortSubtask {
   295    295     SQLiteThread *pThread;          /* Background thread, if any */
   296    296     int bDone;                      /* Set if thread is finished but not joined */
   297    297     VdbeSorter *pSorter;            /* Sorter that owns this sub-task */
   298    298     UnpackedRecord *pUnpacked;      /* Space to unpack a record */
   299    299     SorterList list;                /* List for thread to write to a PMA */
   300    300     int nPMA;                       /* Number of PMAs currently in file */
          301  +  RecordCompare xCompare;         /* Compare function to use */
   301    302     SorterFile file;                /* Temp file for level-0 PMAs */
   302    303     SorterFile file2;               /* Space for other PMAs */
   303    304   };
   304    305   
   305    306   /*
   306    307   ** Main sorter structure. A single instance of this is allocated for each 
   307    308   ** sorter cursor created by the VDBE.
................................................................................
   324    325     SorterList list;                /* List of in-memory records */
   325    326     int iMemory;                    /* Offset of free space in list.aMemory */
   326    327     int nMemory;                    /* Size of list.aMemory allocation in bytes */
   327    328     u8 bUsePMA;                     /* True if one or more PMAs created */
   328    329     u8 bUseThreads;                 /* True to use background threads */
   329    330     u8 iPrev;                       /* Previous thread used to flush PMA */
   330    331     u8 nTask;                       /* Size of aTask[] array */
          332  +  u8 typeMask;
   331    333     SortSubtask aTask[1];           /* One or more subtasks */
   332    334   };
          335  +
          336  +#define SORTER_TYPE_INTEGER 0x01
          337  +#define SORTER_TYPE_TEXT    0x02
   333    338   
   334    339   /*
   335    340   ** An instance of the following object is used to read records out of a
   336    341   ** PMA, in sorted order.  The next key to be read is cached in nKey/aKey.
   337    342   ** aKey might point into aMap or into aBuffer.  If neither of those locations
   338    343   ** contain a contiguous representation of the key, then aAlloc is allocated
   339    344   ** and the key is copied into aAlloc and aKey is made to poitn to aAlloc.
................................................................................
   761    766     const void *pKey1, int nKey1,   /* Left side of comparison */
   762    767     const void *pKey2, int nKey2    /* Right side of comparison */
   763    768   ){
   764    769     UnpackedRecord *r2 = pTask->pUnpacked;
   765    770     if( pKey2 ){
   766    771       sqlite3VdbeRecordUnpack(pTask->pSorter->pKeyInfo, nKey2, pKey2, r2);
   767    772     }
   768         -  return sqlite3VdbeRecordCompare(nKey1, pKey1, r2);
          773  +  return pTask->xCompare(nKey1, pKey1, r2);
   769    774   }
   770    775   
   771    776   /*
   772    777   ** Initialize the temporary index cursor just opened as a sorter cursor.
   773    778   **
   774    779   ** Usually, the sorter module uses the value of (pCsr->pKeyInfo->nField)
   775    780   ** to determine the number of fields that should be compared from the
................................................................................
   831    836     pCsr->pSorter = pSorter;
   832    837     if( pSorter==0 ){
   833    838       rc = SQLITE_NOMEM;
   834    839     }else{
   835    840       pSorter->pKeyInfo = pKeyInfo = (KeyInfo*)((u8*)pSorter + sz);
   836    841       memcpy(pKeyInfo, pCsr->pKeyInfo, szKeyInfo);
   837    842       pKeyInfo->db = 0;
   838         -    if( nField && nWorker==0 ) pKeyInfo->nField = nField;
          843  +    if( nField && nWorker==0 ){
          844  +      pKeyInfo->nXField += (pKeyInfo->nField - nField);
          845  +      pKeyInfo->nField = nField;
          846  +    }
   839    847       pSorter->pgsz = pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt);
   840    848       pSorter->nTask = nWorker + 1;
          849  +    pSorter->iPrev = nWorker-1;
   841    850       pSorter->bUseThreads = (pSorter->nTask>1);
   842    851       pSorter->db = db;
   843    852       for(i=0; i<pSorter->nTask; i++){
   844    853         SortSubtask *pTask = &pSorter->aTask[i];
   845    854         pTask->pSorter = pSorter;
   846    855       }
   847    856   
................................................................................
   859    868         if( sqlite3GlobalConfig.pScratch==0 ){
   860    869           assert( pSorter->iMemory==0 );
   861    870           pSorter->nMemory = pgsz;
   862    871           pSorter->list.aMemory = (u8*)sqlite3Malloc(pgsz);
   863    872           if( !pSorter->list.aMemory ) rc = SQLITE_NOMEM;
   864    873         }
   865    874       }
          875  +
          876  +    if( (pKeyInfo->nField+pKeyInfo->nXField)<13 && pKeyInfo->aColl[0]==0 ){
          877  +      pSorter->typeMask = SORTER_TYPE_INTEGER | SORTER_TYPE_TEXT;
          878  +    }
   866    879     }
   867    880   
   868    881     return rc;
   869    882   }
   870    883   #undef nWorker   /* Defined at the top of this function */
   871    884   
   872    885   /*
................................................................................
  1178   1191       pTask->pUnpacked = sqlite3VdbeAllocUnpackedRecord(
  1179   1192           pTask->pSorter->pKeyInfo, 0, 0, &pFree
  1180   1193       );
  1181   1194       assert( pTask->pUnpacked==(UnpackedRecord*)pFree );
  1182   1195       if( pFree==0 ) return SQLITE_NOMEM;
  1183   1196       pTask->pUnpacked->nField = pTask->pSorter->pKeyInfo->nField;
  1184   1197       pTask->pUnpacked->errCode = 0;
         1198  +    if( pTask->pSorter->pKeyInfo->aSortOrder[0] ){
         1199  +      pTask->pUnpacked->r1 = 1;
         1200  +      pTask->pUnpacked->r2 = -1;
         1201  +    }else{
         1202  +      pTask->pUnpacked->r1 = -1;
         1203  +      pTask->pUnpacked->r2 = 1;
         1204  +    }
  1185   1205     }
  1186   1206     return SQLITE_OK;
  1187   1207   }
  1188   1208   
  1189   1209   
  1190   1210   /*
  1191   1211   ** Merge the two sorted lists p1 and p2 into a single list.
................................................................................
  1230   1250     int i;
  1231   1251     SorterRecord **aSlot;
  1232   1252     SorterRecord *p;
  1233   1253     int rc;
  1234   1254   
  1235   1255     rc = vdbeSortAllocUnpacked(pTask);
  1236   1256     if( rc!=SQLITE_OK ) return rc;
         1257  +
         1258  +  p = pList->pList;
         1259  +  if( pTask->pSorter->typeMask==0 ){
         1260  +    pTask->xCompare = sqlite3VdbeRecordCompare;
         1261  +  }else{
         1262  +    UnpackedRecord *pRec = pTask->pUnpacked;
         1263  +    sqlite3VdbeRecordUnpack(pTask->pSorter->pKeyInfo, p->nVal, SRVAL(p), pRec);
         1264  +    pTask->xCompare = sqlite3VdbeFindCompare(pRec);
         1265  +  }
  1237   1266   
  1238   1267     aSlot = (SorterRecord **)sqlite3MallocZero(64 * sizeof(SorterRecord *));
  1239   1268     if( !aSlot ){
  1240   1269       return SQLITE_NOMEM;
  1241   1270     }
  1242   1271   
  1243         -  p = pList->pList;
  1244   1272     while( p ){
  1245   1273       SorterRecord *pNext;
  1246   1274       if( pList->aMemory ){
  1247   1275         if( (u8*)p==pList->aMemory ){
  1248   1276           pNext = 0;
  1249   1277         }else{
  1250   1278           assert( p->u.iNext<sqlite3MallocSize(pList->aMemory) );
................................................................................
  1598   1626     VdbeSorter *pSorter = pCsr->pSorter;
  1599   1627     int rc = SQLITE_OK;             /* Return Code */
  1600   1628     SorterRecord *pNew;             /* New list element */
  1601   1629   
  1602   1630     int bFlush;                     /* True to flush contents of memory to PMA */
  1603   1631     int nReq;                       /* Bytes of memory required */
  1604   1632     int nPMA;                       /* Bytes of PMA space required */
         1633  +  int t;                          /* serial type of first record field */
         1634  +
         1635  +  getVarint32((const u8*)&pVal->z[1], t);
         1636  +  if( t>0 && t<10 && t!=7 ){
         1637  +    pSorter->typeMask &= SORTER_TYPE_INTEGER;
         1638  +  }else if( t & 0x01 ){
         1639  +    pSorter->typeMask &= SORTER_TYPE_TEXT;
         1640  +  }else{
         1641  +    pSorter->typeMask = 0;
         1642  +  }
  1605   1643   
  1606   1644     assert( pSorter );
  1607   1645   
  1608   1646     /* Figure out whether or not the current contents of memory should be
  1609   1647     ** flushed to a PMA before continuing. If so, do so.
  1610   1648     **
  1611   1649     ** If using the single large allocation mode (pSorter->aMemory!=0), then
................................................................................
  2284   2322   */
  2285   2323   static int vdbeSorterSetupMerge(VdbeSorter *pSorter){
  2286   2324     int rc;                         /* Return code */
  2287   2325     SortSubtask *pTask0 = &pSorter->aTask[0];
  2288   2326     MergeEngine *pMain = 0;
  2289   2327   #if SQLITE_MAX_WORKER_THREADS
  2290   2328     sqlite3 *db = pTask0->pSorter->db;
         2329  +  RecordCompare xCompare = (pSorter->typeMask==0 ? 
         2330  +      sqlite3VdbeRecordCompare : pSorter->aTask[0].xCompare
         2331  +  );
         2332  +  int i;
         2333  +  for(i=0; i<pSorter->nTask; i++){
         2334  +    pSorter->aTask[i].xCompare = xCompare;
         2335  +  }
  2291   2336   #endif
  2292   2337   
  2293   2338     rc = vdbeSorterMergeTreeBuild(pSorter, &pMain);
  2294   2339     if( rc==SQLITE_OK ){
  2295   2340   #if SQLITE_MAX_WORKER_THREADS
  2296   2341       assert( pSorter->bUseThreads==0 || pSorter->nTask>1 );
  2297   2342       if( pSorter->bUseThreads ){

Added test/btree02.test.

            1  +# 2015-03-25
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +# This file implements regression tests for SQLite library.
           12  +#
           13  +# The focus of this script is making multiple calls to saveCursorPosition()
           14  +# and restoreCursorPosition() when cursors have eState==CURSOR_SKIPNEXT
           15  +# 
           16  +
           17  +set testdir [file dirname $argv0]
           18  +source $testdir/tester.tcl
           19  +
           20  +load_static_extension db eval
           21  +do_execsql_test btree02-100 {
           22  +  CREATE TABLE t1(a TEXT, ax INTEGER, b INT, PRIMARY KEY(a,ax)) WITHOUT ROWID;
           23  +  WITH RECURSIVE c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<10)
           24  +    INSERT INTO t1(a,ax,b) SELECT printf('%02x',i), random(), i FROM c;
           25  +  CREATE INDEX t1a ON t1(a);
           26  +  CREATE TABLE t2(x,y);
           27  +  CREATE TABLE t3(cnt);
           28  +  WITH RECURSIVE c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<4)
           29  +    INSERT INTO t3(cnt) SELECT i FROM c;
           30  +  SELECT count(*) FROM t1;
           31  +} {10}
           32  +do_test btree02-110 {
           33  +  db eval BEGIN
           34  +  set i 0
           35  +  db eval {SELECT a, ax, b, cnt FROM t1 CROSS JOIN t3 WHERE b IS NOT NULL} {
           36  +    db eval {INSERT INTO t2(x,y) VALUES($b,$cnt)}
           37  +    # puts "a,b,cnt = ($a,$b,$cnt)"
           38  +    incr i
           39  +    if {$i%2==1} {
           40  +      set bx [expr {$b+1000}]
           41  +      # puts "INSERT ($a),$bx"
           42  +      db eval {INSERT INTO t1(a,ax,b) VALUES(printf('(%s)',$a),random(),$bx)}
           43  +    } else {
           44  +      # puts "DELETE a=$a"
           45  +      db eval {DELETE FROM t1 WHERE a=$a}
           46  +    }
           47  +    db eval {COMMIT; BEGIN}
           48  +  }  
           49  +  db one {COMMIT; SELECT count(*) FROM t1;}
           50  +} {20}
           51  +
           52  +finish_test

Changes to tool/mksqlite3c-noext.tcl.

    13     13   # For example, the "parse.c" and "parse.h" files to implement the
    14     14   # the parser are derived from "parse.y" using lemon.  And the 
    15     15   # "keywordhash.h" files is generated by a program named "mkkeywordhash".
    16     16   #
    17     17   # After the "tsrc" directory has been created and populated, run
    18     18   # this script:
    19     19   #
    20         -#      tclsh mksqlite3c.tcl
           20  +#      tclsh mksqlite3c-noext.tcl
    21     21   #
    22     22   # The amalgamated SQLite code will be written into sqlite3.c
    23     23   #
    24     24   
    25     25   # Begin by reading the "sqlite3.h" header file.  Extract the version number
    26         -# from in this file.  The versioon number is needed to generate the header
           26  +# from in this file.  The version number is needed to generate the header
    27     27   # comment of the amalgamation.
    28     28   #
    29     29   if {[lsearch $argv --nostatic]>=0} {
    30     30     set addstatic 0
    31     31   } else {
    32     32     set addstatic 1
    33     33   }
................................................................................
    76     76   */
    77     77   #define SQLITE_CORE 1
    78     78   #define SQLITE_AMALGAMATION 1}]
    79     79   if {$addstatic} {
    80     80     puts $out \
    81     81   {#ifndef SQLITE_PRIVATE
    82     82   # define SQLITE_PRIVATE static
    83         -#endif
    84         -#ifndef SQLITE_API
    85         -# define SQLITE_API
    86     83   #endif}
    87     84   }
    88     85   
    89     86   # These are the header files used by SQLite.  The first time any of these 
    90     87   # files are seen in a #include statement in the C code, include the complete
    91     88   # text of the file in-line.  The file only needs to be included once.
    92     89   #
................................................................................
   102     99      os_common.h
   103    100      os_setup.h
   104    101      os_win.h
   105    102      os.h
   106    103      pager.h
   107    104      parse.h
   108    105      pcache.h
          106  +   pragma.h
   109    107      sqlite3ext.h
   110    108      sqlite3.h
   111    109      sqliteicu.h
   112    110      sqliteInt.h
   113    111      sqliteLimit.h
   114    112      vdbe.h
   115    113      vdbeInt.h
          114  +   vxworks.h
   116    115      wal.h
          116  +   whereInt.h
   117    117   } {
   118    118     set available_hdr($hdr) 1
   119    119   }
   120    120   set available_hdr(sqliteInt.h) 0
          121  +
          122  +# These headers should be copied into the amalgamation without modifying any
          123  +# of their function declarations or definitions.
          124  +set varonly_hdr(sqlite3.h) 1
          125  +
          126  +# These are the functions that accept a variable number of arguments.  They
          127  +# always need to use the "cdecl" calling convention even when another calling
          128  +# convention (e.g. "stcall") is being used for the rest of the library.
          129  +set cdecllist {
          130  +  sqlite3_config
          131  +  sqlite3_db_config
          132  +  sqlite3_log
          133  +  sqlite3_mprintf
          134  +  sqlite3_snprintf
          135  +  sqlite3_test_control
          136  +  sqlite3_vtab_config
          137  +}
   121    138   
   122    139   # 78 stars used for comment formatting.
   123    140   set s78 \
   124    141   {*****************************************************************************}
   125    142   
   126    143   # Insert a comment into the code
   127    144   #
................................................................................
   131    148     set nstar [expr {60 - $n}]
   132    149     set stars [string range $s78 0 $nstar]
   133    150     puts $out "/************** $text $stars/"
   134    151   }
   135    152   
   136    153   # Read the source file named $filename and write it into the
   137    154   # sqlite3.c output file.  If any #include statements are seen,
   138         -# process them approprately.
          155  +# process them appropriately.
   139    156   #
   140    157   proc copy_file {filename} {
   141         -  global seen_hdr available_hdr out addstatic linemacros
          158  +  global seen_hdr available_hdr varonly_hdr cdecllist out addstatic linemacros
   142    159     set ln 0
   143    160     set tail [file tail $filename]
   144    161     section_comment "Begin file $tail"
   145    162     if {$linemacros} {puts $out "#line 1 \"$filename\""}
   146    163     set in [open $filename r]
   147    164     set varpattern {^[a-zA-Z][a-zA-Z_0-9 *]+(sqlite3[_a-zA-Z0-9]+)(\[|;| =)}
   148         -  set declpattern {[a-zA-Z][a-zA-Z_0-9 ]+ \**(sqlite3[_a-zA-Z0-9]+)\(}
          165  +  set declpattern {([a-zA-Z][a-zA-Z_0-9 ]+ \**)(sqlite3[_a-zA-Z0-9]+)(\(.*)}
   149    166     if {[file extension $filename]==".h"} {
   150    167       set declpattern " *$declpattern"
   151    168     }
   152         -  set declpattern ^$declpattern
          169  +  set declpattern ^$declpattern\$
   153    170     while {![eof $in]} {
   154    171       set line [gets $in]
   155    172       incr ln
   156    173       if {[regexp {^\s*#\s*include\s+["<]([^">]+)[">]} $line all hdr]} {
   157    174         if {[info exists available_hdr($hdr)]} {
   158    175           if {$available_hdr($hdr)} {
   159    176             if {$hdr!="os_common.h" && $hdr!="hwtime.h"} {
................................................................................
   161    178             }
   162    179             section_comment "Include $hdr in the middle of $tail"
   163    180             copy_file tsrc/$hdr
   164    181             section_comment "Continuing where we left off in $tail"
   165    182             if {$linemacros} {puts $out "#line [expr {$ln+1}] \"$filename\""}
   166    183           }
   167    184         } elseif {![info exists seen_hdr($hdr)]} {
   168         -        set seen_hdr($hdr) 1
          185  +        if {![regexp {/\*\s+amalgamator:\s+dontcache\s+\*/} $line]} {
          186  +          set seen_hdr($hdr) 1
          187  +        }
          188  +        puts $out $line
          189  +      } elseif {[regexp {/\*\s+amalgamator:\s+keep\s+\*/} $line]} {
          190  +        # This include file must be kept because there was a "keep"
          191  +        # directive inside of a line comment.
   169    192           puts $out $line
   170    193         } else {
   171         -        puts $out "/* $line */"
          194  +        # Comment out the entire line, replacing any nested comment
          195  +        # begin/end markers with the harmless substring "**".
          196  +        puts $out "/* [string map [list /* ** */ **] $line] */"
   172    197         }
   173    198       } elseif {[regexp {^#ifdef __cplusplus} $line]} {
   174    199         puts $out "#if 0"
   175    200       } elseif {!$linemacros && [regexp {^#line} $line]} {
   176    201         # Skip #line directives.
   177    202       } elseif {$addstatic && ![regexp {^(static|typedef)} $line]} {
   178         -      regsub {^SQLITE_API } $line {} line
   179         -      if {[regexp $declpattern $line all funcname]} {
          203  +      # Skip adding the SQLITE_PRIVATE or SQLITE_API keyword before
          204  +      # functions if this header file does not need it.
          205  +      if {![info exists varonly_hdr($tail)]
          206  +       && [regexp $declpattern $line all rettype funcname rest]} {
          207  +        regsub {^SQLITE_API } $line {} line
   180    208           # Add the SQLITE_PRIVATE or SQLITE_API keyword before functions.
   181    209           # so that linkage can be modified at compile-time.
   182    210           if {[regexp {^sqlite3_} $funcname]} {
   183         -          puts $out "SQLITE_API $line"
          211  +          set line SQLITE_API
          212  +          append line " " [string trim $rettype]
          213  +          if {[string index $rettype end] ne "*"} {
          214  +            append line " "
          215  +          }
          216  +          if {[lsearch -exact $cdecllist $funcname] >= 0} {
          217  +            append line SQLITE_CDECL
          218  +          } else {
          219  +            append line SQLITE_STDCALL
          220  +          }
          221  +          append line " " $funcname $rest
          222  +          puts $out $line
   184    223           } else {
   185    224             puts $out "SQLITE_PRIVATE $line"
   186    225           }
   187    226         } elseif {[regexp $varpattern $line all varname]} {
   188         -        # Add the SQLITE_PRIVATE before variable declarations or
   189         -        # definitions for internal use
   190         -        if {![regexp {^sqlite3_} $varname]} {
   191         -          regsub {^extern } $line {} line
   192         -          puts $out "SQLITE_PRIVATE $line"
   193         -        } else {
   194         -          if {[regexp {const char sqlite3_version\[\];} $line]} {
   195         -            set line {const char sqlite3_version[] = SQLITE_VERSION;}
          227  +          # Add the SQLITE_PRIVATE before variable declarations or
          228  +          # definitions for internal use
          229  +          regsub {^SQLITE_API } $line {} line
          230  +          if {![regexp {^sqlite3_} $varname]} {
          231  +            regsub {^extern } $line {} line
          232  +            puts $out "SQLITE_PRIVATE $line"
          233  +          } else {
          234  +            if {[regexp {const char sqlite3_version\[\];} $line]} {
          235  +              set line {const char sqlite3_version[] = SQLITE_VERSION;}
          236  +            }
          237  +            regsub {^SQLITE_EXTERN } $line {} line
          238  +            puts $out "SQLITE_API $line"
   196    239             }
   197         -          regsub {^SQLITE_EXTERN } $line {} line
   198         -          puts $out "SQLITE_API $line"
   199         -        }
   200    240         } elseif {[regexp {^(SQLITE_EXTERN )?void \(\*sqlite3IoTrace\)} $line]} {
          241  +        regsub {^SQLITE_API } $line {} line
   201    242           regsub {^SQLITE_EXTERN } $line {} line
   202         -        puts $out "SQLITE_PRIVATE $line"
          243  +        puts $out $line
   203    244         } elseif {[regexp {^void \(\*sqlite3Os} $line]} {
          245  +        regsub {^SQLITE_API } $line {} line
   204    246           puts $out "SQLITE_PRIVATE $line"
   205    247         } else {
   206    248           puts $out $line
   207    249         }
   208    250       } else {
   209    251         puts $out $line
   210    252       }

Changes to tool/mksqlite3c.tcl.

    76     76   */
    77     77   #define SQLITE_CORE 1
    78     78   #define SQLITE_AMALGAMATION 1}]
    79     79   if {$addstatic} {
    80     80     puts $out \
    81     81   {#ifndef SQLITE_PRIVATE
    82     82   # define SQLITE_PRIVATE static
    83         -#endif
    84         -#ifndef SQLITE_API
    85         -# define SQLITE_API
    86     83   #endif}
    87     84   }
    88     85   
    89     86   # These are the header files used by SQLite.  The first time any of these 
    90     87   # files are seen in a #include statement in the C code, include the complete
    91     88   # text of the file in-line.  The file only needs to be included once.
    92     89   #
................................................................................
   122    119      vxworks.h
   123    120      wal.h
   124    121      whereInt.h
   125    122   } {
   126    123     set available_hdr($hdr) 1
   127    124   }
   128    125   set available_hdr(sqliteInt.h) 0
          126  +
          127  +# These headers should be copied into the amalgamation without modifying any
          128  +# of their function declarations or definitions.
          129  +set varonly_hdr(sqlite3.h) 1
          130  +
          131  +# These are the functions that accept a variable number of arguments.  They
          132  +# always need to use the "cdecl" calling convention even when another calling
          133  +# convention (e.g. "stcall") is being used for the rest of the library.
          134  +set cdecllist {
          135  +  sqlite3_config
          136  +  sqlite3_db_config
          137  +  sqlite3_log
          138  +  sqlite3_mprintf
          139  +  sqlite3_snprintf
          140  +  sqlite3_test_control
          141  +  sqlite3_vtab_config
          142  +}
   129    143   
   130    144   # 78 stars used for comment formatting.
   131    145   set s78 \
   132    146   {*****************************************************************************}
   133    147   
   134    148   # Insert a comment into the code
   135    149   #
................................................................................
   142    156   }
   143    157   
   144    158   # Read the source file named $filename and write it into the
   145    159   # sqlite3.c output file.  If any #include statements are seen,
   146    160   # process them appropriately.
   147    161   #
   148    162   proc copy_file {filename} {
   149         -  global seen_hdr available_hdr out addstatic linemacros
          163  +  global seen_hdr available_hdr varonly_hdr cdecllist out addstatic linemacros
   150    164     set ln 0
   151    165     set tail [file tail $filename]
   152    166     section_comment "Begin file $tail"
   153    167     if {$linemacros} {puts $out "#line 1 \"$filename\""}
   154    168     set in [open $filename r]
   155    169     set varpattern {^[a-zA-Z][a-zA-Z_0-9 *]+(sqlite3[_a-zA-Z0-9]+)(\[|;| =)}
   156         -  set declpattern {[a-zA-Z][a-zA-Z_0-9 ]+ \**(sqlite3[_a-zA-Z0-9]+)\(}
          170  +  set declpattern {([a-zA-Z][a-zA-Z_0-9 ]+ \**)(sqlite3[_a-zA-Z0-9]+)(\(.*)}
   157    171     if {[file extension $filename]==".h"} {
   158    172       set declpattern " *$declpattern"
   159    173     }
   160         -  set declpattern ^$declpattern
          174  +  set declpattern ^$declpattern\$
   161    175     while {![eof $in]} {
   162    176       set line [gets $in]
   163    177       incr ln
   164    178       if {[regexp {^\s*#\s*include\s+["<]([^">]+)[">]} $line all hdr]} {
   165    179         if {[info exists available_hdr($hdr)]} {
   166    180           if {$available_hdr($hdr)} {
   167    181             if {$hdr!="os_common.h" && $hdr!="hwtime.h"} {
................................................................................
   187    201           puts $out "/* [string map [list /* ** */ **] $line] */"
   188    202         }
   189    203       } elseif {[regexp {^#ifdef __cplusplus} $line]} {
   190    204         puts $out "#if 0"
   191    205       } elseif {!$linemacros && [regexp {^#line} $line]} {
   192    206         # Skip #line directives.
   193    207       } elseif {$addstatic && ![regexp {^(static|typedef)} $line]} {
   194         -      regsub {^SQLITE_API } $line {} line
   195         -      if {[regexp $declpattern $line all funcname]} {
          208  +      # Skip adding the SQLITE_PRIVATE or SQLITE_API keyword before
          209  +      # functions if this header file does not need it.
          210  +      if {![info exists varonly_hdr($tail)]
          211  +       && [regexp $declpattern $line all rettype funcname rest]} {
          212  +        regsub {^SQLITE_API } $line {} line
   196    213           # Add the SQLITE_PRIVATE or SQLITE_API keyword before functions.
   197    214           # so that linkage can be modified at compile-time.
   198    215           if {[regexp {^sqlite3_} $funcname]} {
   199         -          puts $out "SQLITE_API $line"
          216  +          set line SQLITE_API
          217  +          append line " " [string trim $rettype]
          218  +          if {[string index $rettype end] ne "*"} {
          219  +            append line " "
          220  +          }
          221  +          if {[lsearch -exact $cdecllist $funcname] >= 0} {
          222  +            append line SQLITE_CDECL
          223  +          } else {
          224  +            append line SQLITE_STDCALL
          225  +          }
          226  +          append line " " $funcname $rest
          227  +          puts $out $line
   200    228           } else {
   201    229             puts $out "SQLITE_PRIVATE $line"
   202    230           }
   203    231         } elseif {[regexp $varpattern $line all varname]} {
   204         -        # Add the SQLITE_PRIVATE before variable declarations or
   205         -        # definitions for internal use
   206         -        if {![regexp {^sqlite3_} $varname]} {
   207         -          regsub {^extern } $line {} line
   208         -          puts $out "SQLITE_PRIVATE $line"
   209         -        } else {
   210         -          if {[regexp {const char sqlite3_version\[\];} $line]} {
   211         -            set line {const char sqlite3_version[] = SQLITE_VERSION;}
          232  +          # Add the SQLITE_PRIVATE before variable declarations or
          233  +          # definitions for internal use
          234  +          regsub {^SQLITE_API } $line {} line
          235  +          if {![regexp {^sqlite3_} $varname]} {
          236  +            regsub {^extern } $line {} line
          237  +            puts $out "SQLITE_PRIVATE $line"
          238  +          } else {
          239  +            if {[regexp {const char sqlite3_version\[\];} $line]} {
          240  +              set line {const char sqlite3_version[] = SQLITE_VERSION;}
          241  +            }
          242  +            regsub {^SQLITE_EXTERN } $line {} line
          243  +            puts $out "SQLITE_API $line"
   212    244             }
   213         -          regsub {^SQLITE_EXTERN } $line {} line
   214         -          puts $out "SQLITE_API $line"
   215         -        }
   216    245         } elseif {[regexp {^(SQLITE_EXTERN )?void \(\*sqlite3IoTrace\)} $line]} {
          246  +        regsub {^SQLITE_API } $line {} line
   217    247           regsub {^SQLITE_EXTERN } $line {} line
   218    248           puts $out $line
   219    249         } elseif {[regexp {^void \(\*sqlite3Os} $line]} {
          250  +        regsub {^SQLITE_API } $line {} line
   220    251           puts $out "SQLITE_PRIVATE $line"
   221    252         } else {
   222    253           puts $out $line
   223    254         }
   224    255       } else {
   225    256         puts $out $line
   226    257       }

Changes to tool/mksqlite3h.tcl.

    59     59     }
    60     60   }
    61     61   close $in
    62     62   
    63     63   # Set up patterns for recognizing API declarations.
    64     64   #
    65     65   set varpattern {^[a-zA-Z][a-zA-Z_0-9 *]+sqlite3_[_a-zA-Z0-9]+(\[|;| =)}
    66         -set declpattern {^ *[a-zA-Z][a-zA-Z_0-9 ]+ \**sqlite3_[_a-zA-Z0-9]+\(}
           66  +set declpattern {^ *([a-zA-Z][a-zA-Z_0-9 ]+ \**)(sqlite3_[_a-zA-Z0-9]+)(\(.*)$}
    67     67   
    68     68   # Force the output to use unix line endings, even on Windows.
    69     69   fconfigure stdout -translation lf
    70     70   
    71     71   set filelist [subst {
    72     72     $TOP/src/sqlite.h.in
    73     73     $TOP/ext/rtree/sqlite3rtree.h
    74     74   }]
           75  +
           76  +# These are the functions that accept a variable number of arguments.  They
           77  +# always need to use the "cdecl" calling convention even when another calling
           78  +# convention (e.g. "stcall") is being used for the rest of the library.
           79  +set cdecllist {
           80  +  sqlite3_config
           81  +  sqlite3_db_config
           82  +  sqlite3_log
           83  +  sqlite3_mprintf
           84  +  sqlite3_snprintf
           85  +  sqlite3_test_control
           86  +  sqlite3_vtab_config
           87  +}
    75     88   
    76     89   # Process the source files.
    77     90   #
    78     91   foreach file $filelist {
    79     92     set in [open $file]
    80     93     while {![eof $in]} {
    81     94     
................................................................................
    85     98       # line when copying sqlite3rtree.h into sqlite3.h.
    86     99       #
    87    100       if {[string match {*#include*<sqlite3.h>*} $line]} continue
    88    101     
    89    102       regsub -- --VERS--           $line $zVersion line
    90    103       regsub -- --VERSION-NUMBER-- $line $nVersion line
    91    104       regsub -- --SOURCE-ID--      $line "$zDate $zUuid" line
    92         -  
    93         -    if {[regexp {define SQLITE_EXTERN extern} $line]} {
    94         -      puts $line
    95         -      puts [gets $in]
    96         -      puts ""
    97         -      puts "#ifndef SQLITE_API"
    98         -      puts "# define SQLITE_API"
    99         -      puts "#endif"
   100         -      set line ""
   101         -    }
   102         -  
   103         -    if {([regexp $varpattern $line] && ![regexp {^ *typedef} $line])
   104         -     || ([regexp $declpattern $line])
   105         -    } {
          105  +
          106  +    if {[regexp $varpattern $line] && ![regexp {^ *typedef} $line]} {
   106    107         set line "SQLITE_API $line"
          108  +    } else {
          109  +      if {[regexp $declpattern $line all rettype funcname rest]} {
          110  +        set line SQLITE_API
          111  +        append line " " [string trim $rettype]
          112  +        if {[string index $rettype end] ne "*"} {
          113  +          append line " "
          114  +        }
          115  +        if {[lsearch -exact $cdecllist $funcname] >= 0} {
          116  +          append line SQLITE_CDECL
          117  +        } else {
          118  +          append line SQLITE_STDCALL
          119  +        }
          120  +        append line " " $funcname $rest
          121  +      }
   107    122       }
   108    123       puts $line
   109    124     }
   110    125     close $in
   111    126   }