/ Check-in [aeb8e9a9]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge sorter optimization into this branch.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | insert-select-opt
Files: files | file ages | folders
SHA1: aeb8e9a9f2092fda0c3e051f135e1d65cf77ff13
User & Date: dan 2015-03-26 12:38:20
Context
2015-03-30
15:45
Merge sorter optimizations with this branch. check-in: 9bf1cfb4 user: dan tags: insert-select-opt
2015-03-26
12:38
Merge sorter optimization into this branch. check-in: aeb8e9a9 user: dan tags: insert-select-opt
11:55
Optimize cases where all the sorter is sorting a set of records that all begin with integer values, or that all begin with text values to be compared using BINARY. check-in: ce5ad17c user: dan tags: sorter-opt
2015-03-24
19:43
Update this branch with latest trunk changes. check-in: 3ccd64ef user: dan tags: insert-select-opt
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to Makefile.msc.

308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
# also be noted here that building any target with these "stdcall" options
# will most likely fail if the Tcl library is also required.  This is due
# to how the Tcl library functions are declared and exported (i.e. without
# an explicit calling convention, which results in "cdecl").
#
!IF $(USE_STDCALL)!=0
!IF "$(PLATFORM)"=="x86"
CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl
SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl
!ELSE
!IFNDEF PLATFORM
CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl
SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl
!ELSE
CORE_CCONV_OPTS =
SHELL_CCONV_OPTS =
!ENDIF
!ENDIF
!ELSE
CORE_CCONV_OPTS =







|
|


|
|







308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
# also be noted here that building any target with these "stdcall" options
# will most likely fail if the Tcl library is also required.  This is due
# to how the Tcl library functions are declared and exported (i.e. without
# an explicit calling convention, which results in "cdecl").
#
!IF $(USE_STDCALL)!=0
!IF "$(PLATFORM)"=="x86"
CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
!ELSE
!IFNDEF PLATFORM
CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
!ELSE
CORE_CCONV_OPTS =
SHELL_CCONV_OPTS =
!ENDIF
!ENDIF
!ELSE
CORE_CCONV_OPTS =

Changes to src/btree.c.

596
597
598
599
600
601
602
603
604
605
606





607
608
609
610
611
612
613
...
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
...
742
743
744
745
746
747
748

749
750
751
752
753
754
755
756
757
758
759

760
761
762
763
764
765
766
...
804
805
806
807
808
809
810
811
812
813

814
815
816
817
818
819
820
....
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
....
4027
4028
4029
4030
4031
4032
4033


4034
4035
4036
4037
4038
4039
4040
....
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520


4521
4522
4523
4524
4525
4526
4527
....
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
....
7498
7499
7500
7501
7502
7503
7504

7505
7506
7507
7508
7509
7510
7511
**
** The caller must ensure that the cursor is valid (has eState==CURSOR_VALID)
** prior to calling this routine.  
*/
static int saveCursorPosition(BtCursor *pCur){
  int rc;

  assert( CURSOR_VALID==pCur->eState );
  assert( 0==pCur->pKey );
  assert( cursorHoldsMutex(pCur) );






  rc = sqlite3BtreeKeySize(pCur, &pCur->nKey);
  assert( rc==SQLITE_OK );  /* KeySize() cannot fail */

  /* If this is an intKey table, then the above call to BtreeKeySize()
  ** stores the integer key in pCur->nKey. In this case this value is
  ** all that is required. Otherwise, if pCur is not open on an intKey
  ** table, then malloc space for and store the pCur->nKey bytes of key 
................................................................................
static int SQLITE_NOINLINE saveCursorsOnList(
  BtCursor *p,         /* The first cursor that needs saving */
  Pgno iRoot,          /* Only save cursor with this iRoot. Save all if zero */
  BtCursor *pExcept    /* Do not save this cursor */
){
  do{
    if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) ){
      if( p->eState==CURSOR_VALID ){
        int rc = saveCursorPosition(p);
        if( SQLITE_OK!=rc ){
          return rc;
        }
      }else{
        testcase( p->iPage>0 );
        btreeReleaseAllCursorPages(p);
................................................................................
** when saveCursorPosition() was called. Note that this call deletes the 
** saved position info stored by saveCursorPosition(), so there can be
** at most one effective restoreCursorPosition() call after each 
** saveCursorPosition().
*/
static int btreeRestoreCursorPosition(BtCursor *pCur){
  int rc;

  assert( cursorHoldsMutex(pCur) );
  assert( pCur->eState>=CURSOR_REQUIRESEEK );
  if( pCur->eState==CURSOR_FAULT ){
    return pCur->skipNext;
  }
  pCur->eState = CURSOR_INVALID;
  rc = btreeMoveto(pCur, pCur->pKey, pCur->nKey, 0, &pCur->skipNext);
  if( rc==SQLITE_OK ){
    sqlite3_free(pCur->pKey);
    pCur->pKey = 0;
    assert( pCur->eState==CURSOR_VALID || pCur->eState==CURSOR_INVALID );

    if( pCur->skipNext && pCur->eState==CURSOR_VALID ){
      pCur->eState = CURSOR_SKIPNEXT;
    }
  }
  return rc;
}

................................................................................
  assert( pCur!=0 );
  assert( pCur->eState!=CURSOR_VALID );
  rc = restoreCursorPosition(pCur);
  if( rc ){
    *pDifferentRow = 1;
    return rc;
  }
  if( pCur->eState!=CURSOR_VALID || NEVER(pCur->skipNext!=0) ){
    *pDifferentRow = 1;
  }else{

    *pDifferentRow = 0;
  }
  return SQLITE_OK;
}

#ifndef SQLITE_OMIT_AUTOVACUUM
/*
................................................................................

  assert( (writeOnly==0 || writeOnly==1) && BTCF_WriteFlag==1 );
  if( pBtree ){
    sqlite3BtreeEnter(pBtree);
    for(p=pBtree->pBt->pCursor; p; p=p->pNext){
      int i;
      if( writeOnly && (p->curFlags & BTCF_WriteFlag)==0 ){
        if( p->eState==CURSOR_VALID ){
          rc = saveCursorPosition(p);
          if( rc!=SQLITE_OK ){
            (void)sqlite3BtreeTripAllCursors(pBtree, rc, 0);
            break;
          }
        }
      }else{
................................................................................
** Failure is not possible.  This function always returns SQLITE_OK.
** It might just as well be a procedure (returning void) but we continue
** to return an integer result code for historical reasons.
*/
int sqlite3BtreeDataSize(BtCursor *pCur, u32 *pSize){
  assert( cursorHoldsMutex(pCur) );
  assert( pCur->eState==CURSOR_VALID );


  assert( pCur->apPage[pCur->iPage]->intKeyLeaf==1 );
  getCellInfo(pCur);
  *pSize = pCur->info.nPayload;
  return SQLITE_OK;
}

/*
................................................................................
  pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl);
  if( pNewPage->nCell<1 || pNewPage->intKey!=pCur->apPage[i]->intKey ){
    return SQLITE_CORRUPT_BKPT;
  }
  return SQLITE_OK;
}

#if 0
/*
** Page pParent is an internal (non-leaf) tree page. This function 
** asserts that page number iChild is the left-child if the iIdx'th
** cell in page pParent. Or, if iIdx is equal to the total number of
** cells in pParent, that page number iChild is the right-child of
** the page.
*/
static void assertParentIndex(MemPage *pParent, int iIdx, Pgno iChild){


  assert( iIdx<=pParent->nCell );
  if( iIdx==pParent->nCell ){
    assert( get4byte(&pParent->aData[pParent->hdrOffset+8])==iChild );
  }else{
    assert( get4byte(findCell(pParent, iIdx))==iChild );
  }
}
................................................................................
** the largest cell index.
*/
static void moveToParent(BtCursor *pCur){
  assert( cursorHoldsMutex(pCur) );
  assert( pCur->eState==CURSOR_VALID );
  assert( pCur->iPage>0 );
  assert( pCur->apPage[pCur->iPage] );

  /* UPDATE: It is actually possible for the condition tested by the assert
  ** below to be untrue if the database file is corrupt. This can occur if
  ** one cursor has modified page pParent while a reference to it is held 
  ** by a second cursor. Which can only happen if a single page is linked
  ** into more than one b-tree structure in a corrupt database.  */
#if 0
  assertParentIndex(
    pCur->apPage[pCur->iPage-1], 
    pCur->aiIdx[pCur->iPage-1], 
    pCur->apPage[pCur->iPage]->pgno
  );
#endif
  testcase( pCur->aiIdx[pCur->iPage-1] > pCur->apPage[pCur->iPage-1]->nCell );

  releasePage(pCur->apPage[pCur->iPage]);
  pCur->iPage--;
  pCur->info.nSize = 0;
  pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl);
}
................................................................................
      }

      pPage->nOverflow = 0;

      /* The next iteration of the do-loop balances the parent page. */
      releasePage(pPage);
      pCur->iPage--;

    }
  }while( rc==SQLITE_OK );

  if( pFree ){
    sqlite3PageFree(pFree);
  }
  return rc;







|



>
>
>
>
>







 







|







 







>






|




>







 







|


>







 







|







 







>
>







 







|








>
>







 







<
<
<
<
<
<
<





<







 







>







596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
...
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
...
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
...
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
....
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
....
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
....
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
....
4550
4551
4552
4553
4554
4555
4556







4557
4558
4559
4560
4561

4562
4563
4564
4565
4566
4567
4568
....
7502
7503
7504
7505
7506
7507
7508
7509
7510
7511
7512
7513
7514
7515
7516
**
** The caller must ensure that the cursor is valid (has eState==CURSOR_VALID)
** prior to calling this routine.  
*/
static int saveCursorPosition(BtCursor *pCur){
  int rc;

  assert( CURSOR_VALID==pCur->eState || CURSOR_SKIPNEXT==pCur->eState );
  assert( 0==pCur->pKey );
  assert( cursorHoldsMutex(pCur) );

  if( pCur->eState==CURSOR_SKIPNEXT ){
    pCur->eState = CURSOR_VALID;
  }else{
    pCur->skipNext = 0;
  }
  rc = sqlite3BtreeKeySize(pCur, &pCur->nKey);
  assert( rc==SQLITE_OK );  /* KeySize() cannot fail */

  /* If this is an intKey table, then the above call to BtreeKeySize()
  ** stores the integer key in pCur->nKey. In this case this value is
  ** all that is required. Otherwise, if pCur is not open on an intKey
  ** table, then malloc space for and store the pCur->nKey bytes of key 
................................................................................
static int SQLITE_NOINLINE saveCursorsOnList(
  BtCursor *p,         /* The first cursor that needs saving */
  Pgno iRoot,          /* Only save cursor with this iRoot. Save all if zero */
  BtCursor *pExcept    /* Do not save this cursor */
){
  do{
    if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) ){
      if( p->eState==CURSOR_VALID || p->eState==CURSOR_SKIPNEXT ){
        int rc = saveCursorPosition(p);
        if( SQLITE_OK!=rc ){
          return rc;
        }
      }else{
        testcase( p->iPage>0 );
        btreeReleaseAllCursorPages(p);
................................................................................
** when saveCursorPosition() was called. Note that this call deletes the 
** saved position info stored by saveCursorPosition(), so there can be
** at most one effective restoreCursorPosition() call after each 
** saveCursorPosition().
*/
static int btreeRestoreCursorPosition(BtCursor *pCur){
  int rc;
  int skipNext;
  assert( cursorHoldsMutex(pCur) );
  assert( pCur->eState>=CURSOR_REQUIRESEEK );
  if( pCur->eState==CURSOR_FAULT ){
    return pCur->skipNext;
  }
  pCur->eState = CURSOR_INVALID;
  rc = btreeMoveto(pCur, pCur->pKey, pCur->nKey, 0, &skipNext);
  if( rc==SQLITE_OK ){
    sqlite3_free(pCur->pKey);
    pCur->pKey = 0;
    assert( pCur->eState==CURSOR_VALID || pCur->eState==CURSOR_INVALID );
    pCur->skipNext |= skipNext;
    if( pCur->skipNext && pCur->eState==CURSOR_VALID ){
      pCur->eState = CURSOR_SKIPNEXT;
    }
  }
  return rc;
}

................................................................................
  assert( pCur!=0 );
  assert( pCur->eState!=CURSOR_VALID );
  rc = restoreCursorPosition(pCur);
  if( rc ){
    *pDifferentRow = 1;
    return rc;
  }
  if( pCur->eState!=CURSOR_VALID ){
    *pDifferentRow = 1;
  }else{
    assert( pCur->skipNext==0 );
    *pDifferentRow = 0;
  }
  return SQLITE_OK;
}

#ifndef SQLITE_OMIT_AUTOVACUUM
/*
................................................................................

  assert( (writeOnly==0 || writeOnly==1) && BTCF_WriteFlag==1 );
  if( pBtree ){
    sqlite3BtreeEnter(pBtree);
    for(p=pBtree->pBt->pCursor; p; p=p->pNext){
      int i;
      if( writeOnly && (p->curFlags & BTCF_WriteFlag)==0 ){
        if( p->eState==CURSOR_VALID || p->eState==CURSOR_SKIPNEXT ){
          rc = saveCursorPosition(p);
          if( rc!=SQLITE_OK ){
            (void)sqlite3BtreeTripAllCursors(pBtree, rc, 0);
            break;
          }
        }
      }else{
................................................................................
** Failure is not possible.  This function always returns SQLITE_OK.
** It might just as well be a procedure (returning void) but we continue
** to return an integer result code for historical reasons.
*/
int sqlite3BtreeDataSize(BtCursor *pCur, u32 *pSize){
  assert( cursorHoldsMutex(pCur) );
  assert( pCur->eState==CURSOR_VALID );
  assert( pCur->iPage>=0 );
  assert( pCur->iPage<BTCURSOR_MAX_DEPTH );
  assert( pCur->apPage[pCur->iPage]->intKeyLeaf==1 );
  getCellInfo(pCur);
  *pSize = pCur->info.nPayload;
  return SQLITE_OK;
}

/*
................................................................................
  pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl);
  if( pNewPage->nCell<1 || pNewPage->intKey!=pCur->apPage[i]->intKey ){
    return SQLITE_CORRUPT_BKPT;
  }
  return SQLITE_OK;
}

#if SQLITE_DEBUG
/*
** Page pParent is an internal (non-leaf) tree page. This function 
** asserts that page number iChild is the left-child if the iIdx'th
** cell in page pParent. Or, if iIdx is equal to the total number of
** cells in pParent, that page number iChild is the right-child of
** the page.
*/
static void assertParentIndex(MemPage *pParent, int iIdx, Pgno iChild){
  if( CORRUPT_DB ) return;  /* The conditions tested below might not be true
                            ** in a corrupt database */
  assert( iIdx<=pParent->nCell );
  if( iIdx==pParent->nCell ){
    assert( get4byte(&pParent->aData[pParent->hdrOffset+8])==iChild );
  }else{
    assert( get4byte(findCell(pParent, iIdx))==iChild );
  }
}
................................................................................
** the largest cell index.
*/
static void moveToParent(BtCursor *pCur){
  assert( cursorHoldsMutex(pCur) );
  assert( pCur->eState==CURSOR_VALID );
  assert( pCur->iPage>0 );
  assert( pCur->apPage[pCur->iPage] );







  assertParentIndex(
    pCur->apPage[pCur->iPage-1], 
    pCur->aiIdx[pCur->iPage-1], 
    pCur->apPage[pCur->iPage]->pgno
  );

  testcase( pCur->aiIdx[pCur->iPage-1] > pCur->apPage[pCur->iPage-1]->nCell );

  releasePage(pCur->apPage[pCur->iPage]);
  pCur->iPage--;
  pCur->info.nSize = 0;
  pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl);
}
................................................................................
      }

      pPage->nOverflow = 0;

      /* The next iteration of the do-loop balances the parent page. */
      releasePage(pPage);
      pCur->iPage--;
      assert( pCur->iPage>=0 );
    }
  }while( rc==SQLITE_OK );

  if( pFree ){
    sqlite3PageFree(pFree);
  }
  return rc;

Changes to src/sqlite.h.in.

39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57



58
59
60
61
62
63
64
*/
#ifdef __cplusplus
extern "C" {
#endif


/*
** Add the ability to override 'extern'
*/
#ifndef SQLITE_EXTERN
# define SQLITE_EXTERN extern
#endif

/*
** Add the ability to override 'cdecl'
*/
#ifndef SQLITE_CDECL
# define SQLITE_CDECL
#endif




/*
** These no-op macros are used in front of interfaces to mark those
** interfaces as either deprecated or experimental.  New applications
** should not use deprecated interfaces - they are supported for backwards
** compatibility only.  Application writers should be aware that
** experimental interfaces are subject to change in point releases.







|




|
|
|
<



>
>
>







39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

54
55
56
57
58
59
60
61
62
63
64
65
66
*/
#ifdef __cplusplus
extern "C" {
#endif


/*
** Provide the ability to override linkage features of the interface.
*/
#ifndef SQLITE_EXTERN
# define SQLITE_EXTERN extern
#endif
#ifndef SQLITE_API
# define SQLITE_API
#endif

#ifndef SQLITE_CDECL
# define SQLITE_CDECL
#endif
#ifndef SQLITE_STDCALL
# define SQLITE_STDCALL
#endif

/*
** These no-op macros are used in front of interfaces to mark those
** interfaces as either deprecated or experimental.  New applications
** should not use deprecated interfaces - they are supported for backwards
** compatibility only.  Application writers should be aware that
** experimental interfaces are subject to change in point releases.

Changes to src/vdbesort.c.

294
295
296
297
298
299
300

301
302
303
304
305
306
307
...
324
325
326
327
328
329
330

331
332



333
334
335
336
337
338
339
...
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
...
831
832
833
834
835
836
837


838

839
840

841
842
843
844
845
846
847
...
859
860
861
862
863
864
865




866
867
868
869
870
871
872
....
1178
1179
1180
1181
1182
1183
1184







1185
1186
1187
1188
1189
1190
1191
....
1230
1231
1232
1233
1234
1235
1236









1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
....
1598
1599
1600
1601
1602
1603
1604










1605
1606
1607
1608
1609
1610
1611
....
2284
2285
2286
2287
2288
2289
2290







2291
2292
2293
2294
2295
2296
2297
struct SortSubtask {
  SQLiteThread *pThread;          /* Background thread, if any */
  int bDone;                      /* Set if thread is finished but not joined */
  VdbeSorter *pSorter;            /* Sorter that owns this sub-task */
  UnpackedRecord *pUnpacked;      /* Space to unpack a record */
  SorterList list;                /* List for thread to write to a PMA */
  int nPMA;                       /* Number of PMAs currently in file */

  SorterFile file;                /* Temp file for level-0 PMAs */
  SorterFile file2;               /* Space for other PMAs */
};

/*
** Main sorter structure. A single instance of this is allocated for each 
** sorter cursor created by the VDBE.
................................................................................
  SorterList list;                /* List of in-memory records */
  int iMemory;                    /* Offset of free space in list.aMemory */
  int nMemory;                    /* Size of list.aMemory allocation in bytes */
  u8 bUsePMA;                     /* True if one or more PMAs created */
  u8 bUseThreads;                 /* True to use background threads */
  u8 iPrev;                       /* Previous thread used to flush PMA */
  u8 nTask;                       /* Size of aTask[] array */

  SortSubtask aTask[1];           /* One or more subtasks */
};




/*
** An instance of the following object is used to read records out of a
** PMA, in sorted order.  The next key to be read is cached in nKey/aKey.
** aKey might point into aMap or into aBuffer.  If neither of those locations
** contain a contiguous representation of the key, then aAlloc is allocated
** and the key is copied into aAlloc and aKey is made to poitn to aAlloc.
................................................................................
  const void *pKey1, int nKey1,   /* Left side of comparison */
  const void *pKey2, int nKey2    /* Right side of comparison */
){
  UnpackedRecord *r2 = pTask->pUnpacked;
  if( pKey2 ){
    sqlite3VdbeRecordUnpack(pTask->pSorter->pKeyInfo, nKey2, pKey2, r2);
  }
  return sqlite3VdbeRecordCompare(nKey1, pKey1, r2);
}

/*
** Initialize the temporary index cursor just opened as a sorter cursor.
**
** Usually, the sorter module uses the value of (pCsr->pKeyInfo->nField)
** to determine the number of fields that should be compared from the
................................................................................
  pCsr->pSorter = pSorter;
  if( pSorter==0 ){
    rc = SQLITE_NOMEM;
  }else{
    pSorter->pKeyInfo = pKeyInfo = (KeyInfo*)((u8*)pSorter + sz);
    memcpy(pKeyInfo, pCsr->pKeyInfo, szKeyInfo);
    pKeyInfo->db = 0;


    if( nField && nWorker==0 ) pKeyInfo->nField = nField;

    pSorter->pgsz = pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt);
    pSorter->nTask = nWorker + 1;

    pSorter->bUseThreads = (pSorter->nTask>1);
    pSorter->db = db;
    for(i=0; i<pSorter->nTask; i++){
      SortSubtask *pTask = &pSorter->aTask[i];
      pTask->pSorter = pSorter;
    }

................................................................................
      if( sqlite3GlobalConfig.pScratch==0 ){
        assert( pSorter->iMemory==0 );
        pSorter->nMemory = pgsz;
        pSorter->list.aMemory = (u8*)sqlite3Malloc(pgsz);
        if( !pSorter->list.aMemory ) rc = SQLITE_NOMEM;
      }
    }




  }

  return rc;
}
#undef nWorker   /* Defined at the top of this function */

/*
................................................................................
    pTask->pUnpacked = sqlite3VdbeAllocUnpackedRecord(
        pTask->pSorter->pKeyInfo, 0, 0, &pFree
    );
    assert( pTask->pUnpacked==(UnpackedRecord*)pFree );
    if( pFree==0 ) return SQLITE_NOMEM;
    pTask->pUnpacked->nField = pTask->pSorter->pKeyInfo->nField;
    pTask->pUnpacked->errCode = 0;







  }
  return SQLITE_OK;
}


/*
** Merge the two sorted lists p1 and p2 into a single list.
................................................................................
  int i;
  SorterRecord **aSlot;
  SorterRecord *p;
  int rc;

  rc = vdbeSortAllocUnpacked(pTask);
  if( rc!=SQLITE_OK ) return rc;










  aSlot = (SorterRecord **)sqlite3MallocZero(64 * sizeof(SorterRecord *));
  if( !aSlot ){
    return SQLITE_NOMEM;
  }

  p = pList->pList;
  while( p ){
    SorterRecord *pNext;
    if( pList->aMemory ){
      if( (u8*)p==pList->aMemory ){
        pNext = 0;
      }else{
        assert( p->u.iNext<sqlite3MallocSize(pList->aMemory) );
................................................................................
  VdbeSorter *pSorter = pCsr->pSorter;
  int rc = SQLITE_OK;             /* Return Code */
  SorterRecord *pNew;             /* New list element */

  int bFlush;                     /* True to flush contents of memory to PMA */
  int nReq;                       /* Bytes of memory required */
  int nPMA;                       /* Bytes of PMA space required */











  assert( pSorter );

  /* Figure out whether or not the current contents of memory should be
  ** flushed to a PMA before continuing. If so, do so.
  **
  ** If using the single large allocation mode (pSorter->aMemory!=0), then
................................................................................
*/
static int vdbeSorterSetupMerge(VdbeSorter *pSorter){
  int rc;                         /* Return code */
  SortSubtask *pTask0 = &pSorter->aTask[0];
  MergeEngine *pMain = 0;
#if SQLITE_MAX_WORKER_THREADS
  sqlite3 *db = pTask0->pSorter->db;







#endif

  rc = vdbeSorterMergeTreeBuild(pSorter, &pMain);
  if( rc==SQLITE_OK ){
#if SQLITE_MAX_WORKER_THREADS
    assert( pSorter->bUseThreads==0 || pSorter->nTask>1 );
    if( pSorter->bUseThreads ){







>







 







>


>
>
>







 







|







 







>
>
|
>


>







 







>
>
>
>







 







>
>
>
>
>
>
>







 







>
>
>
>
>
>
>
>
>






<







 







>
>
>
>
>
>
>
>
>
>







 







>
>
>
>
>
>
>







294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
...
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
...
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
...
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
...
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
....
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
....
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271

1272
1273
1274
1275
1276
1277
1278
....
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
....
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
struct SortSubtask {
  SQLiteThread *pThread;          /* Background thread, if any */
  int bDone;                      /* Set if thread is finished but not joined */
  VdbeSorter *pSorter;            /* Sorter that owns this sub-task */
  UnpackedRecord *pUnpacked;      /* Space to unpack a record */
  SorterList list;                /* List for thread to write to a PMA */
  int nPMA;                       /* Number of PMAs currently in file */
  RecordCompare xCompare;         /* Compare function to use */
  SorterFile file;                /* Temp file for level-0 PMAs */
  SorterFile file2;               /* Space for other PMAs */
};

/*
** Main sorter structure. A single instance of this is allocated for each 
** sorter cursor created by the VDBE.
................................................................................
  SorterList list;                /* List of in-memory records */
  int iMemory;                    /* Offset of free space in list.aMemory */
  int nMemory;                    /* Size of list.aMemory allocation in bytes */
  u8 bUsePMA;                     /* True if one or more PMAs created */
  u8 bUseThreads;                 /* True to use background threads */
  u8 iPrev;                       /* Previous thread used to flush PMA */
  u8 nTask;                       /* Size of aTask[] array */
  u8 typeMask;
  SortSubtask aTask[1];           /* One or more subtasks */
};

#define SORTER_TYPE_INTEGER 0x01
#define SORTER_TYPE_TEXT    0x02

/*
** An instance of the following object is used to read records out of a
** PMA, in sorted order.  The next key to be read is cached in nKey/aKey.
** aKey might point into aMap or into aBuffer.  If neither of those locations
** contain a contiguous representation of the key, then aAlloc is allocated
** and the key is copied into aAlloc and aKey is made to poitn to aAlloc.
................................................................................
  const void *pKey1, int nKey1,   /* Left side of comparison */
  const void *pKey2, int nKey2    /* Right side of comparison */
){
  UnpackedRecord *r2 = pTask->pUnpacked;
  if( pKey2 ){
    sqlite3VdbeRecordUnpack(pTask->pSorter->pKeyInfo, nKey2, pKey2, r2);
  }
  return pTask->xCompare(nKey1, pKey1, r2);
}

/*
** Initialize the temporary index cursor just opened as a sorter cursor.
**
** Usually, the sorter module uses the value of (pCsr->pKeyInfo->nField)
** to determine the number of fields that should be compared from the
................................................................................
  pCsr->pSorter = pSorter;
  if( pSorter==0 ){
    rc = SQLITE_NOMEM;
  }else{
    pSorter->pKeyInfo = pKeyInfo = (KeyInfo*)((u8*)pSorter + sz);
    memcpy(pKeyInfo, pCsr->pKeyInfo, szKeyInfo);
    pKeyInfo->db = 0;
    if( nField && nWorker==0 ){
      pKeyInfo->nXField += (pKeyInfo->nField - nField);
      pKeyInfo->nField = nField;
    }
    pSorter->pgsz = pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt);
    pSorter->nTask = nWorker + 1;
    pSorter->iPrev = nWorker-1;
    pSorter->bUseThreads = (pSorter->nTask>1);
    pSorter->db = db;
    for(i=0; i<pSorter->nTask; i++){
      SortSubtask *pTask = &pSorter->aTask[i];
      pTask->pSorter = pSorter;
    }

................................................................................
      if( sqlite3GlobalConfig.pScratch==0 ){
        assert( pSorter->iMemory==0 );
        pSorter->nMemory = pgsz;
        pSorter->list.aMemory = (u8*)sqlite3Malloc(pgsz);
        if( !pSorter->list.aMemory ) rc = SQLITE_NOMEM;
      }
    }

    if( (pKeyInfo->nField+pKeyInfo->nXField)<13 && pKeyInfo->aColl[0]==0 ){
      pSorter->typeMask = SORTER_TYPE_INTEGER | SORTER_TYPE_TEXT;
    }
  }

  return rc;
}
#undef nWorker   /* Defined at the top of this function */

/*
................................................................................
    pTask->pUnpacked = sqlite3VdbeAllocUnpackedRecord(
        pTask->pSorter->pKeyInfo, 0, 0, &pFree
    );
    assert( pTask->pUnpacked==(UnpackedRecord*)pFree );
    if( pFree==0 ) return SQLITE_NOMEM;
    pTask->pUnpacked->nField = pTask->pSorter->pKeyInfo->nField;
    pTask->pUnpacked->errCode = 0;
    if( pTask->pSorter->pKeyInfo->aSortOrder[0] ){
      pTask->pUnpacked->r1 = 1;
      pTask->pUnpacked->r2 = -1;
    }else{
      pTask->pUnpacked->r1 = -1;
      pTask->pUnpacked->r2 = 1;
    }
  }
  return SQLITE_OK;
}


/*
** Merge the two sorted lists p1 and p2 into a single list.
................................................................................
  int i;
  SorterRecord **aSlot;
  SorterRecord *p;
  int rc;

  rc = vdbeSortAllocUnpacked(pTask);
  if( rc!=SQLITE_OK ) return rc;

  p = pList->pList;
  if( pTask->pSorter->typeMask==0 ){
    pTask->xCompare = sqlite3VdbeRecordCompare;
  }else{
    UnpackedRecord *pRec = pTask->pUnpacked;
    sqlite3VdbeRecordUnpack(pTask->pSorter->pKeyInfo, p->nVal, SRVAL(p), pRec);
    pTask->xCompare = sqlite3VdbeFindCompare(pRec);
  }

  aSlot = (SorterRecord **)sqlite3MallocZero(64 * sizeof(SorterRecord *));
  if( !aSlot ){
    return SQLITE_NOMEM;
  }


  while( p ){
    SorterRecord *pNext;
    if( pList->aMemory ){
      if( (u8*)p==pList->aMemory ){
        pNext = 0;
      }else{
        assert( p->u.iNext<sqlite3MallocSize(pList->aMemory) );
................................................................................
  VdbeSorter *pSorter = pCsr->pSorter;
  int rc = SQLITE_OK;             /* Return Code */
  SorterRecord *pNew;             /* New list element */

  int bFlush;                     /* True to flush contents of memory to PMA */
  int nReq;                       /* Bytes of memory required */
  int nPMA;                       /* Bytes of PMA space required */
  int t;                          /* serial type of first record field */

  getVarint32((const u8*)&pVal->z[1], t);
  if( t>0 && t<10 && t!=7 ){
    pSorter->typeMask &= SORTER_TYPE_INTEGER;
  }else if( t & 0x01 ){
    pSorter->typeMask &= SORTER_TYPE_TEXT;
  }else{
    pSorter->typeMask = 0;
  }

  assert( pSorter );

  /* Figure out whether or not the current contents of memory should be
  ** flushed to a PMA before continuing. If so, do so.
  **
  ** If using the single large allocation mode (pSorter->aMemory!=0), then
................................................................................
*/
static int vdbeSorterSetupMerge(VdbeSorter *pSorter){
  int rc;                         /* Return code */
  SortSubtask *pTask0 = &pSorter->aTask[0];
  MergeEngine *pMain = 0;
#if SQLITE_MAX_WORKER_THREADS
  sqlite3 *db = pTask0->pSorter->db;
  RecordCompare xCompare = (pSorter->typeMask==0 ? 
      sqlite3VdbeRecordCompare : pSorter->aTask[0].xCompare
  );
  int i;
  for(i=0; i<pSorter->nTask; i++){
    pSorter->aTask[i].xCompare = xCompare;
  }
#endif

  rc = vdbeSorterMergeTreeBuild(pSorter, &pMain);
  if( rc==SQLITE_OK ){
#if SQLITE_MAX_WORKER_THREADS
    assert( pSorter->bUseThreads==0 || pSorter->nTask>1 );
    if( pSorter->bUseThreads ){

Added test/btree02.test.









































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
# 2015-03-25
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.
#
# The focus of this script is making multiple calls to saveCursorPosition()
# and restoreCursorPosition() when cursors have eState==CURSOR_SKIPNEXT
# 

set testdir [file dirname $argv0]
source $testdir/tester.tcl

load_static_extension db eval
do_execsql_test btree02-100 {
  CREATE TABLE t1(a TEXT, ax INTEGER, b INT, PRIMARY KEY(a,ax)) WITHOUT ROWID;
  WITH RECURSIVE c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<10)
    INSERT INTO t1(a,ax,b) SELECT printf('%02x',i), random(), i FROM c;
  CREATE INDEX t1a ON t1(a);
  CREATE TABLE t2(x,y);
  CREATE TABLE t3(cnt);
  WITH RECURSIVE c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<4)
    INSERT INTO t3(cnt) SELECT i FROM c;
  SELECT count(*) FROM t1;
} {10}
do_test btree02-110 {
  db eval BEGIN
  set i 0
  db eval {SELECT a, ax, b, cnt FROM t1 CROSS JOIN t3 WHERE b IS NOT NULL} {
    db eval {INSERT INTO t2(x,y) VALUES($b,$cnt)}
    # puts "a,b,cnt = ($a,$b,$cnt)"
    incr i
    if {$i%2==1} {
      set bx [expr {$b+1000}]
      # puts "INSERT ($a),$bx"
      db eval {INSERT INTO t1(a,ax,b) VALUES(printf('(%s)',$a),random(),$bx)}
    } else {
      # puts "DELETE a=$a"
      db eval {DELETE FROM t1 WHERE a=$a}
    }
    db eval {COMMIT; BEGIN}
  }  
  db one {COMMIT; SELECT count(*) FROM t1;}
} {20}

finish_test

Changes to tool/mksqlite3c-noext.tcl.

13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
..
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
...
102
103
104
105
106
107
108

109
110
111
112
113
114
115

116

117
118
119
120

















121
122
123
124
125
126
127
...
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
...
161
162
163
164
165
166
167

168

169




170
171



172
173
174
175
176
177




178
179
180
181
182











183
184
185
186
187
188
189

190
191
192
193
194
195
196
197
198
199
200

201
202
203

204
205
206
207
208
209
210
# For example, the "parse.c" and "parse.h" files to implement the
# the parser are derived from "parse.y" using lemon.  And the 
# "keywordhash.h" files is generated by a program named "mkkeywordhash".
#
# After the "tsrc" directory has been created and populated, run
# this script:
#
#      tclsh mksqlite3c.tcl
#
# The amalgamated SQLite code will be written into sqlite3.c
#

# Begin by reading the "sqlite3.h" header file.  Extract the version number
# from in this file.  The versioon number is needed to generate the header
# comment of the amalgamation.
#
if {[lsearch $argv --nostatic]>=0} {
  set addstatic 0
} else {
  set addstatic 1
}
................................................................................
*/
#define SQLITE_CORE 1
#define SQLITE_AMALGAMATION 1}]
if {$addstatic} {
  puts $out \
{#ifndef SQLITE_PRIVATE
# define SQLITE_PRIVATE static
#endif
#ifndef SQLITE_API
# define SQLITE_API
#endif}
}

# These are the header files used by SQLite.  The first time any of these 
# files are seen in a #include statement in the C code, include the complete
# text of the file in-line.  The file only needs to be included once.
#
................................................................................
   os_common.h
   os_setup.h
   os_win.h
   os.h
   pager.h
   parse.h
   pcache.h

   sqlite3ext.h
   sqlite3.h
   sqliteicu.h
   sqliteInt.h
   sqliteLimit.h
   vdbe.h
   vdbeInt.h

   wal.h

} {
  set available_hdr($hdr) 1
}
set available_hdr(sqliteInt.h) 0


















# 78 stars used for comment formatting.
set s78 \
{*****************************************************************************}

# Insert a comment into the code
#
................................................................................
  set nstar [expr {60 - $n}]
  set stars [string range $s78 0 $nstar]
  puts $out "/************** $text $stars/"
}

# Read the source file named $filename and write it into the
# sqlite3.c output file.  If any #include statements are seen,
# process them approprately.
#
proc copy_file {filename} {
  global seen_hdr available_hdr out addstatic linemacros
  set ln 0
  set tail [file tail $filename]
  section_comment "Begin file $tail"
  if {$linemacros} {puts $out "#line 1 \"$filename\""}
  set in [open $filename r]
  set varpattern {^[a-zA-Z][a-zA-Z_0-9 *]+(sqlite3[_a-zA-Z0-9]+)(\[|;| =)}
  set declpattern {[a-zA-Z][a-zA-Z_0-9 ]+ \**(sqlite3[_a-zA-Z0-9]+)\(}
  if {[file extension $filename]==".h"} {
    set declpattern " *$declpattern"
  }
  set declpattern ^$declpattern
  while {![eof $in]} {
    set line [gets $in]
    incr ln
    if {[regexp {^\s*#\s*include\s+["<]([^">]+)[">]} $line all hdr]} {
      if {[info exists available_hdr($hdr)]} {
        if {$available_hdr($hdr)} {
          if {$hdr!="os_common.h" && $hdr!="hwtime.h"} {
................................................................................
          }
          section_comment "Include $hdr in the middle of $tail"
          copy_file tsrc/$hdr
          section_comment "Continuing where we left off in $tail"
          if {$linemacros} {puts $out "#line [expr {$ln+1}] \"$filename\""}
        }
      } elseif {![info exists seen_hdr($hdr)]} {

        set seen_hdr($hdr) 1

        puts $out $line




      } else {
        puts $out "/* $line */"



      }
    } elseif {[regexp {^#ifdef __cplusplus} $line]} {
      puts $out "#if 0"
    } elseif {!$linemacros && [regexp {^#line} $line]} {
      # Skip #line directives.
    } elseif {$addstatic && ![regexp {^(static|typedef)} $line]} {




      regsub {^SQLITE_API } $line {} line
      if {[regexp $declpattern $line all funcname]} {
        # Add the SQLITE_PRIVATE or SQLITE_API keyword before functions.
        # so that linkage can be modified at compile-time.
        if {[regexp {^sqlite3_} $funcname]} {











          puts $out "SQLITE_API $line"
        } else {
          puts $out "SQLITE_PRIVATE $line"
        }
      } elseif {[regexp $varpattern $line all varname]} {
        # Add the SQLITE_PRIVATE before variable declarations or
        # definitions for internal use

        if {![regexp {^sqlite3_} $varname]} {
          regsub {^extern } $line {} line
          puts $out "SQLITE_PRIVATE $line"
        } else {
          if {[regexp {const char sqlite3_version\[\];} $line]} {
            set line {const char sqlite3_version[] = SQLITE_VERSION;}
          }
          regsub {^SQLITE_EXTERN } $line {} line
          puts $out "SQLITE_API $line"
        }
      } elseif {[regexp {^(SQLITE_EXTERN )?void \(\*sqlite3IoTrace\)} $line]} {

        regsub {^SQLITE_EXTERN } $line {} line
        puts $out "SQLITE_PRIVATE $line"
      } elseif {[regexp {^void \(\*sqlite3Os} $line]} {

        puts $out "SQLITE_PRIVATE $line"
      } else {
        puts $out $line
      }
    } else {
      puts $out $line
    }







|





|







 







<
<
<







 







>







>

>




>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







|


|






|



|







 







>
|
>

>
>
>
>

<
>
>
>






>
>
>
>
|
<



>
>
>
>
>
>
>
>
>
>
>
|




|
|
>
|
|
|
|
|
|
|
|
|
|

>

|

>







13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
..
76
77
78
79
80
81
82



83
84
85
86
87
88
89
..
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
...
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
...
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193

194
195
196
197
198
199
200
201
202
203
204
205
206
207

208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
# For example, the "parse.c" and "parse.h" files to implement the
# the parser are derived from "parse.y" using lemon.  And the 
# "keywordhash.h" files is generated by a program named "mkkeywordhash".
#
# After the "tsrc" directory has been created and populated, run
# this script:
#
#      tclsh mksqlite3c-noext.tcl
#
# The amalgamated SQLite code will be written into sqlite3.c
#

# Begin by reading the "sqlite3.h" header file.  Extract the version number
# from in this file.  The version number is needed to generate the header
# comment of the amalgamation.
#
if {[lsearch $argv --nostatic]>=0} {
  set addstatic 0
} else {
  set addstatic 1
}
................................................................................
*/
#define SQLITE_CORE 1
#define SQLITE_AMALGAMATION 1}]
if {$addstatic} {
  puts $out \
{#ifndef SQLITE_PRIVATE
# define SQLITE_PRIVATE static



#endif}
}

# These are the header files used by SQLite.  The first time any of these 
# files are seen in a #include statement in the C code, include the complete
# text of the file in-line.  The file only needs to be included once.
#
................................................................................
   os_common.h
   os_setup.h
   os_win.h
   os.h
   pager.h
   parse.h
   pcache.h
   pragma.h
   sqlite3ext.h
   sqlite3.h
   sqliteicu.h
   sqliteInt.h
   sqliteLimit.h
   vdbe.h
   vdbeInt.h
   vxworks.h
   wal.h
   whereInt.h
} {
  set available_hdr($hdr) 1
}
set available_hdr(sqliteInt.h) 0

# These headers should be copied into the amalgamation without modifying any
# of their function declarations or definitions.
set varonly_hdr(sqlite3.h) 1

# These are the functions that accept a variable number of arguments.  They
# always need to use the "cdecl" calling convention even when another calling
# convention (e.g. "stcall") is being used for the rest of the library.
set cdecllist {
  sqlite3_config
  sqlite3_db_config
  sqlite3_log
  sqlite3_mprintf
  sqlite3_snprintf
  sqlite3_test_control
  sqlite3_vtab_config
}

# 78 stars used for comment formatting.
set s78 \
{*****************************************************************************}

# Insert a comment into the code
#
................................................................................
  set nstar [expr {60 - $n}]
  set stars [string range $s78 0 $nstar]
  puts $out "/************** $text $stars/"
}

# Read the source file named $filename and write it into the
# sqlite3.c output file.  If any #include statements are seen,
# process them appropriately.
#
proc copy_file {filename} {
  global seen_hdr available_hdr varonly_hdr cdecllist out addstatic linemacros
  set ln 0
  set tail [file tail $filename]
  section_comment "Begin file $tail"
  if {$linemacros} {puts $out "#line 1 \"$filename\""}
  set in [open $filename r]
  set varpattern {^[a-zA-Z][a-zA-Z_0-9 *]+(sqlite3[_a-zA-Z0-9]+)(\[|;| =)}
  set declpattern {([a-zA-Z][a-zA-Z_0-9 ]+ \**)(sqlite3[_a-zA-Z0-9]+)(\(.*)}
  if {[file extension $filename]==".h"} {
    set declpattern " *$declpattern"
  }
  set declpattern ^$declpattern\$
  while {![eof $in]} {
    set line [gets $in]
    incr ln
    if {[regexp {^\s*#\s*include\s+["<]([^">]+)[">]} $line all hdr]} {
      if {[info exists available_hdr($hdr)]} {
        if {$available_hdr($hdr)} {
          if {$hdr!="os_common.h" && $hdr!="hwtime.h"} {
................................................................................
          }
          section_comment "Include $hdr in the middle of $tail"
          copy_file tsrc/$hdr
          section_comment "Continuing where we left off in $tail"
          if {$linemacros} {puts $out "#line [expr {$ln+1}] \"$filename\""}
        }
      } elseif {![info exists seen_hdr($hdr)]} {
        if {![regexp {/\*\s+amalgamator:\s+dontcache\s+\*/} $line]} {
          set seen_hdr($hdr) 1
        }
        puts $out $line
      } elseif {[regexp {/\*\s+amalgamator:\s+keep\s+\*/} $line]} {
        # This include file must be kept because there was a "keep"
        # directive inside of a line comment.
        puts $out $line
      } else {

        # Comment out the entire line, replacing any nested comment
        # begin/end markers with the harmless substring "**".
        puts $out "/* [string map [list /* ** */ **] $line] */"
      }
    } elseif {[regexp {^#ifdef __cplusplus} $line]} {
      puts $out "#if 0"
    } elseif {!$linemacros && [regexp {^#line} $line]} {
      # Skip #line directives.
    } elseif {$addstatic && ![regexp {^(static|typedef)} $line]} {
      # Skip adding the SQLITE_PRIVATE or SQLITE_API keyword before
      # functions if this header file does not need it.
      if {![info exists varonly_hdr($tail)]
       && [regexp $declpattern $line all rettype funcname rest]} {
        regsub {^SQLITE_API } $line {} line

        # Add the SQLITE_PRIVATE or SQLITE_API keyword before functions.
        # so that linkage can be modified at compile-time.
        if {[regexp {^sqlite3_} $funcname]} {
          set line SQLITE_API
          append line " " [string trim $rettype]
          if {[string index $rettype end] ne "*"} {
            append line " "
          }
          if {[lsearch -exact $cdecllist $funcname] >= 0} {
            append line SQLITE_CDECL
          } else {
            append line SQLITE_STDCALL
          }
          append line " " $funcname $rest
          puts $out $line
        } else {
          puts $out "SQLITE_PRIVATE $line"
        }
      } elseif {[regexp $varpattern $line all varname]} {
          # Add the SQLITE_PRIVATE before variable declarations or
          # definitions for internal use
          regsub {^SQLITE_API } $line {} line
          if {![regexp {^sqlite3_} $varname]} {
            regsub {^extern } $line {} line
            puts $out "SQLITE_PRIVATE $line"
          } else {
            if {[regexp {const char sqlite3_version\[\];} $line]} {
              set line {const char sqlite3_version[] = SQLITE_VERSION;}
            }
            regsub {^SQLITE_EXTERN } $line {} line
            puts $out "SQLITE_API $line"
          }
      } elseif {[regexp {^(SQLITE_EXTERN )?void \(\*sqlite3IoTrace\)} $line]} {
        regsub {^SQLITE_API } $line {} line
        regsub {^SQLITE_EXTERN } $line {} line
        puts $out $line
      } elseif {[regexp {^void \(\*sqlite3Os} $line]} {
        regsub {^SQLITE_API } $line {} line
        puts $out "SQLITE_PRIVATE $line"
      } else {
        puts $out $line
      }
    } else {
      puts $out $line
    }

Changes to tool/mksqlite3c.tcl.

76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
...
122
123
124
125
126
127
128

















129
130
131
132
133
134
135
...
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
...
187
188
189
190
191
192
193




194
195
196
197
198











199
200
201
202
203
204
205

206
207
208
209
210
211
212
213
214
215
216

217
218
219

220
221
222
223
224
225
226
*/
#define SQLITE_CORE 1
#define SQLITE_AMALGAMATION 1}]
if {$addstatic} {
  puts $out \
{#ifndef SQLITE_PRIVATE
# define SQLITE_PRIVATE static
#endif
#ifndef SQLITE_API
# define SQLITE_API
#endif}
}

# These are the header files used by SQLite.  The first time any of these 
# files are seen in a #include statement in the C code, include the complete
# text of the file in-line.  The file only needs to be included once.
#
................................................................................
   vxworks.h
   wal.h
   whereInt.h
} {
  set available_hdr($hdr) 1
}
set available_hdr(sqliteInt.h) 0


















# 78 stars used for comment formatting.
set s78 \
{*****************************************************************************}

# Insert a comment into the code
#
................................................................................
}

# Read the source file named $filename and write it into the
# sqlite3.c output file.  If any #include statements are seen,
# process them appropriately.
#
proc copy_file {filename} {
  global seen_hdr available_hdr out addstatic linemacros
  set ln 0
  set tail [file tail $filename]
  section_comment "Begin file $tail"
  if {$linemacros} {puts $out "#line 1 \"$filename\""}
  set in [open $filename r]
  set varpattern {^[a-zA-Z][a-zA-Z_0-9 *]+(sqlite3[_a-zA-Z0-9]+)(\[|;| =)}
  set declpattern {[a-zA-Z][a-zA-Z_0-9 ]+ \**(sqlite3[_a-zA-Z0-9]+)\(}
  if {[file extension $filename]==".h"} {
    set declpattern " *$declpattern"
  }
  set declpattern ^$declpattern
  while {![eof $in]} {
    set line [gets $in]
    incr ln
    if {[regexp {^\s*#\s*include\s+["<]([^">]+)[">]} $line all hdr]} {
      if {[info exists available_hdr($hdr)]} {
        if {$available_hdr($hdr)} {
          if {$hdr!="os_common.h" && $hdr!="hwtime.h"} {
................................................................................
        puts $out "/* [string map [list /* ** */ **] $line] */"
      }
    } elseif {[regexp {^#ifdef __cplusplus} $line]} {
      puts $out "#if 0"
    } elseif {!$linemacros && [regexp {^#line} $line]} {
      # Skip #line directives.
    } elseif {$addstatic && ![regexp {^(static|typedef)} $line]} {




      regsub {^SQLITE_API } $line {} line
      if {[regexp $declpattern $line all funcname]} {
        # Add the SQLITE_PRIVATE or SQLITE_API keyword before functions.
        # so that linkage can be modified at compile-time.
        if {[regexp {^sqlite3_} $funcname]} {











          puts $out "SQLITE_API $line"
        } else {
          puts $out "SQLITE_PRIVATE $line"
        }
      } elseif {[regexp $varpattern $line all varname]} {
        # Add the SQLITE_PRIVATE before variable declarations or
        # definitions for internal use

        if {![regexp {^sqlite3_} $varname]} {
          regsub {^extern } $line {} line
          puts $out "SQLITE_PRIVATE $line"
        } else {
          if {[regexp {const char sqlite3_version\[\];} $line]} {
            set line {const char sqlite3_version[] = SQLITE_VERSION;}
          }
          regsub {^SQLITE_EXTERN } $line {} line
          puts $out "SQLITE_API $line"
        }
      } elseif {[regexp {^(SQLITE_EXTERN )?void \(\*sqlite3IoTrace\)} $line]} {

        regsub {^SQLITE_EXTERN } $line {} line
        puts $out $line
      } elseif {[regexp {^void \(\*sqlite3Os} $line]} {

        puts $out "SQLITE_PRIVATE $line"
      } else {
        puts $out $line
      }
    } else {
      puts $out $line
    }







<
<
<







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







|






|



|







 







>
>
>
>
|
<



>
>
>
>
>
>
>
>
>
>
>
|




|
|
>
|
|
|
|
|
|
|
|
|
|

>



>







76
77
78
79
80
81
82



83
84
85
86
87
88
89
...
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
...
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
...
201
202
203
204
205
206
207
208
209
210
211
212

213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
*/
#define SQLITE_CORE 1
#define SQLITE_AMALGAMATION 1}]
if {$addstatic} {
  puts $out \
{#ifndef SQLITE_PRIVATE
# define SQLITE_PRIVATE static



#endif}
}

# These are the header files used by SQLite.  The first time any of these 
# files are seen in a #include statement in the C code, include the complete
# text of the file in-line.  The file only needs to be included once.
#
................................................................................
   vxworks.h
   wal.h
   whereInt.h
} {
  set available_hdr($hdr) 1
}
set available_hdr(sqliteInt.h) 0

# These headers should be copied into the amalgamation without modifying any
# of their function declarations or definitions.
set varonly_hdr(sqlite3.h) 1

# These are the functions that accept a variable number of arguments.  They
# always need to use the "cdecl" calling convention even when another calling
# convention (e.g. "stcall") is being used for the rest of the library.
set cdecllist {
  sqlite3_config
  sqlite3_db_config
  sqlite3_log
  sqlite3_mprintf
  sqlite3_snprintf
  sqlite3_test_control
  sqlite3_vtab_config
}

# 78 stars used for comment formatting.
set s78 \
{*****************************************************************************}

# Insert a comment into the code
#
................................................................................
}

# Read the source file named $filename and write it into the
# sqlite3.c output file.  If any #include statements are seen,
# process them appropriately.
#
proc copy_file {filename} {
  global seen_hdr available_hdr varonly_hdr cdecllist out addstatic linemacros
  set ln 0
  set tail [file tail $filename]
  section_comment "Begin file $tail"
  if {$linemacros} {puts $out "#line 1 \"$filename\""}
  set in [open $filename r]
  set varpattern {^[a-zA-Z][a-zA-Z_0-9 *]+(sqlite3[_a-zA-Z0-9]+)(\[|;| =)}
  set declpattern {([a-zA-Z][a-zA-Z_0-9 ]+ \**)(sqlite3[_a-zA-Z0-9]+)(\(.*)}
  if {[file extension $filename]==".h"} {
    set declpattern " *$declpattern"
  }
  set declpattern ^$declpattern\$
  while {![eof $in]} {
    set line [gets $in]
    incr ln
    if {[regexp {^\s*#\s*include\s+["<]([^">]+)[">]} $line all hdr]} {
      if {[info exists available_hdr($hdr)]} {
        if {$available_hdr($hdr)} {
          if {$hdr!="os_common.h" && $hdr!="hwtime.h"} {
................................................................................
        puts $out "/* [string map [list /* ** */ **] $line] */"
      }
    } elseif {[regexp {^#ifdef __cplusplus} $line]} {
      puts $out "#if 0"
    } elseif {!$linemacros && [regexp {^#line} $line]} {
      # Skip #line directives.
    } elseif {$addstatic && ![regexp {^(static|typedef)} $line]} {
      # Skip adding the SQLITE_PRIVATE or SQLITE_API keyword before
      # functions if this header file does not need it.
      if {![info exists varonly_hdr($tail)]
       && [regexp $declpattern $line all rettype funcname rest]} {
        regsub {^SQLITE_API } $line {} line

        # Add the SQLITE_PRIVATE or SQLITE_API keyword before functions.
        # so that linkage can be modified at compile-time.
        if {[regexp {^sqlite3_} $funcname]} {
          set line SQLITE_API
          append line " " [string trim $rettype]
          if {[string index $rettype end] ne "*"} {
            append line " "
          }
          if {[lsearch -exact $cdecllist $funcname] >= 0} {
            append line SQLITE_CDECL
          } else {
            append line SQLITE_STDCALL
          }
          append line " " $funcname $rest
          puts $out $line
        } else {
          puts $out "SQLITE_PRIVATE $line"
        }
      } elseif {[regexp $varpattern $line all varname]} {
          # Add the SQLITE_PRIVATE before variable declarations or
          # definitions for internal use
          regsub {^SQLITE_API } $line {} line
          if {![regexp {^sqlite3_} $varname]} {
            regsub {^extern } $line {} line
            puts $out "SQLITE_PRIVATE $line"
          } else {
            if {[regexp {const char sqlite3_version\[\];} $line]} {
              set line {const char sqlite3_version[] = SQLITE_VERSION;}
            }
            regsub {^SQLITE_EXTERN } $line {} line
            puts $out "SQLITE_API $line"
          }
      } elseif {[regexp {^(SQLITE_EXTERN )?void \(\*sqlite3IoTrace\)} $line]} {
        regsub {^SQLITE_API } $line {} line
        regsub {^SQLITE_EXTERN } $line {} line
        puts $out $line
      } elseif {[regexp {^void \(\*sqlite3Os} $line]} {
        regsub {^SQLITE_API } $line {} line
        puts $out "SQLITE_PRIVATE $line"
      } else {
        puts $out $line
      }
    } else {
      puts $out $line
    }

Changes to tool/mksqlite3h.tcl.

59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74













75
76
77
78
79
80
81
..
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106


107
108
109
110
111
  }
}
close $in

# Set up patterns for recognizing API declarations.
#
set varpattern {^[a-zA-Z][a-zA-Z_0-9 *]+sqlite3_[_a-zA-Z0-9]+(\[|;| =)}
set declpattern {^ *[a-zA-Z][a-zA-Z_0-9 ]+ \**sqlite3_[_a-zA-Z0-9]+\(}

# Force the output to use unix line endings, even on Windows.
fconfigure stdout -translation lf

set filelist [subst {
  $TOP/src/sqlite.h.in
  $TOP/ext/rtree/sqlite3rtree.h
}]














# Process the source files.
#
foreach file $filelist {
  set in [open $file]
  while {![eof $in]} {
  
................................................................................
    # line when copying sqlite3rtree.h into sqlite3.h.
    #
    if {[string match {*#include*<sqlite3.h>*} $line]} continue
  
    regsub -- --VERS--           $line $zVersion line
    regsub -- --VERSION-NUMBER-- $line $nVersion line
    regsub -- --SOURCE-ID--      $line "$zDate $zUuid" line
  
    if {[regexp {define SQLITE_EXTERN extern} $line]} {
      puts $line
      puts [gets $in]
      puts ""
      puts "#ifndef SQLITE_API"
      puts "# define SQLITE_API"
      puts "#endif"
      set line ""
    }
  
    if {([regexp $varpattern $line] && ![regexp {^ *typedef} $line])
     || ([regexp $declpattern $line])
    } {
      set line "SQLITE_API $line"


    }
    puts $line
  }
  close $in
}







|








>
>
>
>
>
>
>
>
>
>
>
>
>







 







|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
>
>





59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
..
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
  }
}
close $in

# Set up patterns for recognizing API declarations.
#
set varpattern {^[a-zA-Z][a-zA-Z_0-9 *]+sqlite3_[_a-zA-Z0-9]+(\[|;| =)}
set declpattern {^ *([a-zA-Z][a-zA-Z_0-9 ]+ \**)(sqlite3_[_a-zA-Z0-9]+)(\(.*)$}

# Force the output to use unix line endings, even on Windows.
fconfigure stdout -translation lf

set filelist [subst {
  $TOP/src/sqlite.h.in
  $TOP/ext/rtree/sqlite3rtree.h
}]

# These are the functions that accept a variable number of arguments.  They
# always need to use the "cdecl" calling convention even when another calling
# convention (e.g. "stcall") is being used for the rest of the library.
set cdecllist {
  sqlite3_config
  sqlite3_db_config
  sqlite3_log
  sqlite3_mprintf
  sqlite3_snprintf
  sqlite3_test_control
  sqlite3_vtab_config
}

# Process the source files.
#
foreach file $filelist {
  set in [open $file]
  while {![eof $in]} {
  
................................................................................
    # line when copying sqlite3rtree.h into sqlite3.h.
    #
    if {[string match {*#include*<sqlite3.h>*} $line]} continue
  
    regsub -- --VERS--           $line $zVersion line
    regsub -- --VERSION-NUMBER-- $line $nVersion line
    regsub -- --SOURCE-ID--      $line "$zDate $zUuid" line

    if {[regexp $varpattern $line] && ![regexp {^ *typedef} $line]} {
      set line "SQLITE_API $line"
    } else {
      if {[regexp $declpattern $line all rettype funcname rest]} {
        set line SQLITE_API
        append line " " [string trim $rettype]
        if {[string index $rettype end] ne "*"} {
          append line " "
        }
        if {[lsearch -exact $cdecllist $funcname] >= 0} {
          append line SQLITE_CDECL
        } else {
          append line SQLITE_STDCALL
        }
        append line " " $funcname $rest
      }
    }
    puts $line
  }
  close $in
}