SQLite
Check-in [9d3351b8]
Not logged in
HomeTimelineBranchesTagsTicketsWikiLoginMore...
Overview
SHA1:9d3351b8d713232133dad149c73fb2a27c72abb1
Date: 2014-04-03 16:25:29
User: dan
Comment:Fix an integer overflow problem in the sorter.
References
2014-04-04
07:52
[dceed2c8] Add test file sort3.test, which should have been part of commit [9d3351b8d7]. (user: dan, tags: threads)
Tags And Properties
Context
2014-04-03
16:42
[a0910079] Merge all recent changes from trunk. (user: drh, tags: threads)
16:25
[9d3351b8] Fix an integer overflow problem in the sorter. (user: dan, tags: threads)
14:29
[0561272a] Fix minor errors causing compilation to fail with SQLITE_MAX_WORKER_THREADS set to a value greater than zero. (user: dan, tags: threads)
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/main.c. 

2500
2501
2502
2503
2504
2505
2506

2507
2508
2509
2510
2511
2512
2513
....
3325
3326
3327
3328
3329
3330
3331







3332
3333
3334
3335
3336
3337
3338


  assert( sizeof(db->aLimit)==sizeof(aHardLimit) );
  memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit));
  db->autoCommit = 1;
  db->nextAutovac = -1;
  db->szMmap = sqlite3GlobalConfig.szMmap;
  db->nextPagesize = 0;

  db->flags |= SQLITE_ShortColNames | SQLITE_EnableTrigger | SQLITE_CacheSpill
#if !defined(SQLITE_DEFAULT_AUTOMATIC_INDEX) || SQLITE_DEFAULT_AUTOMATIC_INDEX
                 | SQLITE_AutoIndex
#endif
#if SQLITE_DEFAULT_FILE_FORMAT<4
                 | SQLITE_LegacyFileFmt
#endif
................................................................................
#ifdef SQLITE_VDBE_COVERAGE
      typedef void (*branch_callback)(void*,int,u8,u8);
      sqlite3GlobalConfig.xVdbeBranch = va_arg(ap,branch_callback);
      sqlite3GlobalConfig.pVdbeBranchArg = va_arg(ap,void*);
#endif
      break;
    }








  }
  va_end(ap);
#endif /* SQLITE_OMIT_BUILTIN_TEST */
  return rc;
}









>







 







>
>
>
>
>
>
>








2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
....
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346


  assert( sizeof(db->aLimit)==sizeof(aHardLimit) );
  memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit));
  db->autoCommit = 1;
  db->nextAutovac = -1;
  db->szMmap = sqlite3GlobalConfig.szMmap;
  db->nextPagesize = 0;
  db->nMaxSorterMmap = 0x7FFFFFFF;
  db->flags |= SQLITE_ShortColNames | SQLITE_EnableTrigger | SQLITE_CacheSpill
#if !defined(SQLITE_DEFAULT_AUTOMATIC_INDEX) || SQLITE_DEFAULT_AUTOMATIC_INDEX
                 | SQLITE_AutoIndex
#endif
#if SQLITE_DEFAULT_FILE_FORMAT<4
                 | SQLITE_LegacyFileFmt
#endif
................................................................................
#ifdef SQLITE_VDBE_COVERAGE
      typedef void (*branch_callback)(void*,int,u8,u8);
      sqlite3GlobalConfig.xVdbeBranch = va_arg(ap,branch_callback);
      sqlite3GlobalConfig.pVdbeBranchArg = va_arg(ap,void*);
#endif
      break;
    }

    /*   sqlite3_test_control(SQLITE_TESTCTRL_SORTER_MMAP, db, nMax); */
    case SQLITE_TESTCTRL_SORTER_MMAP: {
      sqlite3 *db = va_arg(ap, sqlite3*);
      db->nMaxSorterMmap = va_arg(ap, int);
      break;
    }

  }
  va_end(ap);
#endif /* SQLITE_OMIT_BUILTIN_TEST */
  return rc;
}

Changes to src/sqlite.h.in. 

6125
6126
6127
6128
6129
6130
6131

6132
6133
6134
6135
6136
6137
6138
6139

#define SQLITE_TESTCTRL_OPTIMIZATIONS           15
#define SQLITE_TESTCTRL_ISKEYWORD               16
#define SQLITE_TESTCTRL_SCRATCHMALLOC           17
#define SQLITE_TESTCTRL_LOCALTIME_FAULT         18
#define SQLITE_TESTCTRL_EXPLAIN_STMT            19
#define SQLITE_TESTCTRL_NEVER_CORRUPT           20
#define SQLITE_TESTCTRL_VDBE_COVERAGE           21

#define SQLITE_TESTCTRL_LAST                    21

/*
** CAPI3REF: SQLite Runtime Status
**
** ^This interface is used to retrieve runtime status information
** about the performance of SQLite, and optionally to reset various
** highwater marks.  ^The first argument is an integer code for








>
|








6125
6126
6127
6128
6129
6130
6131
6132
6133
6134
6135
6136
6137
6138
6139
6140

#define SQLITE_TESTCTRL_OPTIMIZATIONS           15
#define SQLITE_TESTCTRL_ISKEYWORD               16
#define SQLITE_TESTCTRL_SCRATCHMALLOC           17
#define SQLITE_TESTCTRL_LOCALTIME_FAULT         18
#define SQLITE_TESTCTRL_EXPLAIN_STMT            19
#define SQLITE_TESTCTRL_NEVER_CORRUPT           20
#define SQLITE_TESTCTRL_VDBE_COVERAGE           21
#define SQLITE_TESTCTRL_SORTER_MMAP             22
#define SQLITE_TESTCTRL_LAST                    22

/*
** CAPI3REF: SQLite Runtime Status
**
** ^This interface is used to retrieve runtime status information
** about the performance of SQLite, and optionally to reset various
** highwater marks.  ^The first argument is an integer code for

Changes to src/sqliteInt.h. 

977
978
979
980
981
982
983

984
985
986
987
988
989
990

  u8 vtabOnConflict;            /* Value to return for s3_vtab_on_conflict() */
  u8 isTransactionSavepoint;    /* True if the outermost savepoint is a TS */
  int nextPagesize;             /* Pagesize after VACUUM if >0 */
  u32 magic;                    /* Magic number for detect library misuse */
  int nChange;                  /* Value returned by sqlite3_changes() */
  int nTotalChange;             /* Value returned by sqlite3_total_changes() */
  int aLimit[SQLITE_N_LIMIT];   /* Limits */

  struct sqlite3InitInfo {      /* Information used during initialization */
    int newTnum;                /* Rootpage of table being initialized */
    u8 iDb;                     /* Which db file is being initialized */
    u8 busy;                    /* TRUE if currently initializing */
    u8 orphanTrigger;           /* Last statement is orphaned TEMP trigger */
  } init;
  int nVdbeActive;              /* Number of VDBEs currently running */








>








977
978
979
980
981
982
983
984
985
986
987
988
989
990
991

  u8 vtabOnConflict;            /* Value to return for s3_vtab_on_conflict() */
  u8 isTransactionSavepoint;    /* True if the outermost savepoint is a TS */
  int nextPagesize;             /* Pagesize after VACUUM if >0 */
  u32 magic;                    /* Magic number for detect library misuse */
  int nChange;                  /* Value returned by sqlite3_changes() */
  int nTotalChange;             /* Value returned by sqlite3_total_changes() */
  int aLimit[SQLITE_N_LIMIT];   /* Limits */
  int nMaxSorterMmap;           /* Maximum size of regions mapped by sorter */
  struct sqlite3InitInfo {      /* Information used during initialization */
    int newTnum;                /* Rootpage of table being initialized */
    u8 iDb;                     /* Which db file is being initialized */
    u8 busy;                    /* TRUE if currently initializing */
    u8 orphanTrigger;           /* Last statement is orphaned TEMP trigger */
  } init;
  int nVdbeActive;              /* Number of VDBEs currently running */

Changes to src/test1.c. 

5880
5881
5882
5883
5884
5885
5886

5887
5888
5889
5890
5891
5892
5893
....
5907
5908
5909
5910
5911
5912
5913













5914
5915
5916
5917
5918
5919
5920

  Tcl_Obj *CONST objv[]
){
  struct Verb {
    const char *zName;
    int i;
  } aVerb[] = {
    { "SQLITE_TESTCTRL_LOCALTIME_FAULT", SQLITE_TESTCTRL_LOCALTIME_FAULT }, 

  };
  int iVerb;
  int iFlag;
  int rc;

  if( objc<2 ){
    Tcl_WrongNumArgs(interp, 1, objv, "VERB ARGS...");
................................................................................
        Tcl_WrongNumArgs(interp, 2, objv, "ONOFF");
        return TCL_ERROR;
      }
      if( Tcl_GetBooleanFromObj(interp, objv[2], &val) ) return TCL_ERROR;
      sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, val);
      break;
    }













  }

  Tcl_ResetResult(interp);
  return TCL_OK;
}

#if SQLITE_OS_UNIX








>







 







>
>
>
>
>
>
>
>
>
>
>
>
>








5880
5881
5882
5883
5884
5885
5886
5887
5888
5889
5890
5891
5892
5893
5894
....
5908
5909
5910
5911
5912
5913
5914
5915
5916
5917
5918
5919
5920
5921
5922
5923
5924
5925
5926
5927
5928
5929
5930
5931
5932
5933
5934

  Tcl_Obj *CONST objv[]
){
  struct Verb {
    const char *zName;
    int i;
  } aVerb[] = {
    { "SQLITE_TESTCTRL_LOCALTIME_FAULT", SQLITE_TESTCTRL_LOCALTIME_FAULT }, 
    { "SQLITE_TESTCTRL_SORTER_MMAP", SQLITE_TESTCTRL_SORTER_MMAP }, 
  };
  int iVerb;
  int iFlag;
  int rc;

  if( objc<2 ){
    Tcl_WrongNumArgs(interp, 1, objv, "VERB ARGS...");
................................................................................
        Tcl_WrongNumArgs(interp, 2, objv, "ONOFF");
        return TCL_ERROR;
      }
      if( Tcl_GetBooleanFromObj(interp, objv[2], &val) ) return TCL_ERROR;
      sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, val);
      break;
    }

    case SQLITE_TESTCTRL_SORTER_MMAP: {
      int val;
      sqlite3 *db;
      if( objc!=4 ){
        Tcl_WrongNumArgs(interp, 2, objv, "DB LIMIT");
        return TCL_ERROR;
      }
      if( getDbPointer(interp, Tcl_GetString(objv[2]), &db) ) return TCL_ERROR;
      if( Tcl_GetIntFromObj(interp, objv[3], &val) ) return TCL_ERROR;
      sqlite3_test_control(SQLITE_TESTCTRL_SORTER_MMAP, db, val);
      break;
    }
  }

  Tcl_ResetResult(interp);
  return TCL_OK;
}

#if SQLITE_OS_UNIX

Changes to src/vdbesort.c. 

140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
...
510
511
512
513
514
515
516

517

518
519
520
521
522
523
524
...
666
667
668
669
670
671
672
673
674

675
676
677
678
679
680
681
....
1011
1012
1013
1014
1015
1016
1017
1018


1019
1020
1021
1022
1023

1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
....
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
....
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218

**     Merge existing PMAs until SortSubtask.nConsolidate or fewer
**     remain in temp file SortSubtask.pTemp1.
*/
struct SortSubtask {
  SQLiteThread *pThread;          /* Thread handle, or NULL */
  int bDone;                      /* Set to true by pTask when finished */

  sqlite3_vfs *pVfs;              /* VFS used to open temporary files */
  KeyInfo *pKeyInfo;              /* How to compare records */
  UnpackedRecord *pUnpacked;      /* Space to unpack a record */
  int pgsz;                       /* Main database page size */

  u8 eWork;                       /* One of the SORT_SUBTASK_* constants */
  int nConsolidate;               /* For SORT_SUBTASK_CONS, max final PMAs */
  SorterRecord *pList;            /* List of records for pTask to sort */
................................................................................
  pIter->pFile = pTask->pTemp1;
  pIter->iReadOff = iStart;
  pIter->nAlloc = 128;
  pIter->aAlloc = (u8*)sqlite3Malloc(pIter->nAlloc);
  if( pIter->aAlloc ){
    /* Try to xFetch() a mapping of the entire temp file. If this is possible,
    ** the PMA will be read via the mapping. Otherwise, use xRead().  */

    rc = sqlite3OsFetch(pIter->pFile, 0, pTask->iTemp1Off, &pMap);

  }else{
    rc = SQLITE_NOMEM;
  }

  if( rc==SQLITE_OK ){
    if( pMap ){
      pIter->aMap = (u8*)pMap;
................................................................................
    if( nField && nWorker==0 ) pKeyInfo->nField = nField;
    pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt);

    pSorter->nTask = nWorker + 1;
    for(i=0; i<pSorter->nTask; i++){
      SortSubtask *pTask = &pSorter->aTask[i];
      pTask->pKeyInfo = pKeyInfo;
      pTask->pVfs = db->pVfs;
      pTask->pgsz = pgsz;

    }

    if( !sqlite3TempInMemory(db) ){
      pSorter->mnPmaSize = SORTER_MIN_WORKING * pgsz;
      mxCache = db->aDb[0].pSchema->cache_size;
      if( mxCache<SORTER_MIN_WORKING ) mxCache = SORTER_MIN_WORKING;
      pSorter->mxPmaSize = mxCache * pgsz;
................................................................................
** is guaranteed to be nByte bytes or smaller in size. This function
** attempts to extend the file to nByte bytes in size and to ensure that
** the VFS has memory mapped it.
**
** Whether or not the file does end up memory mapped of course depends on
** the specific VFS implementation.
*/
static int vdbeSorterExtendFile(sqlite3_file *pFile, i64 nByte){


  int rc = sqlite3OsTruncate(pFile, nByte);
  if( rc==SQLITE_OK ){
    void *p = 0;
    sqlite3OsFetch(pFile, 0, nByte, &p);
    sqlite3OsUnfetch(pFile, 0, p);

  }
  return rc;
}
#else
# define vdbeSorterExtendFile(x,y) SQLITE_OK
#endif


/*
** Write the current contents of the in-memory linked-list to a PMA. Return
** SQLITE_OK if successful, or an SQLite error code otherwise.
**
................................................................................
  PmaWriter writer;               /* Object used to write to the file */

  memset(&writer, 0, sizeof(PmaWriter));
  assert( pTask->nInMemory>0 );

  /* If the first temporary PMA file has not been opened, open it now. */
  if( pTask->pTemp1==0 ){
    rc = vdbeSorterOpenTempFile(pTask->pVfs, &pTask->pTemp1);
    assert( rc!=SQLITE_OK || pTask->pTemp1 );
    assert( pTask->iTemp1Off==0 );
    assert( pTask->nPMA==0 );
  }

  /* Try to get the file to memory map */
  if( rc==SQLITE_OK ){
    rc = vdbeSorterExtendFile(
        pTask->pTemp1, pTask->iTemp1Off + pTask->nInMemory + 9
    );
  }

  if( rc==SQLITE_OK ){
    SorterRecord *p;
    SorterRecord *pNext = 0;
................................................................................
      pMerger = vdbeMergeEngineNew(nIter);
      if( pMerger==0 ){
        rc = SQLITE_NOMEM;
        break;
      }

      /* Open a second temp file to write merged data to */
      rc = vdbeSorterOpenTempFile(pTask->pVfs, &pTemp2);
      if( rc==SQLITE_OK ){
        rc = vdbeSorterExtendFile(pTemp2, pTask->iTemp1Off);
      }
      if( rc!=SQLITE_OK ){
        vdbeMergeEngineFree(pMerger);
        break;
      }

      /* This loop runs once for each output PMA. Each output PMA is made








|







 







>
|
>







 







<

>







 







|
>
>
|
|
|
|
|
>




|







 







|







|







 







|

|








140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
...
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
...
668
669
670
671
672
673
674

675
676
677
678
679
680
681
682
683
....
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
....
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
....
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223

**     Merge existing PMAs until SortSubtask.nConsolidate or fewer
**     remain in temp file SortSubtask.pTemp1.
*/
struct SortSubtask {
  SQLiteThread *pThread;          /* Thread handle, or NULL */
  int bDone;                      /* Set to true by pTask when finished */

  sqlite3 *db;                    /* Database connection */
  KeyInfo *pKeyInfo;              /* How to compare records */
  UnpackedRecord *pUnpacked;      /* Space to unpack a record */
  int pgsz;                       /* Main database page size */

  u8 eWork;                       /* One of the SORT_SUBTASK_* constants */
  int nConsolidate;               /* For SORT_SUBTASK_CONS, max final PMAs */
  SorterRecord *pList;            /* List of records for pTask to sort */
................................................................................
  pIter->pFile = pTask->pTemp1;
  pIter->iReadOff = iStart;
  pIter->nAlloc = 128;
  pIter->aAlloc = (u8*)sqlite3Malloc(pIter->nAlloc);
  if( pIter->aAlloc ){
    /* Try to xFetch() a mapping of the entire temp file. If this is possible,
    ** the PMA will be read via the mapping. Otherwise, use xRead().  */
    if( pTask->iTemp1Off<=(i64)(pTask->db->nMaxSorterMmap) ){
      rc = sqlite3OsFetch(pIter->pFile, 0, pTask->iTemp1Off, &pMap);
    }
  }else{
    rc = SQLITE_NOMEM;
  }

  if( rc==SQLITE_OK ){
    if( pMap ){
      pIter->aMap = (u8*)pMap;
................................................................................
    if( nField && nWorker==0 ) pKeyInfo->nField = nField;
    pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt);

    pSorter->nTask = nWorker + 1;
    for(i=0; i<pSorter->nTask; i++){
      SortSubtask *pTask = &pSorter->aTask[i];
      pTask->pKeyInfo = pKeyInfo;

      pTask->pgsz = pgsz;
      pTask->db = db;
    }

    if( !sqlite3TempInMemory(db) ){
      pSorter->mnPmaSize = SORTER_MIN_WORKING * pgsz;
      mxCache = db->aDb[0].pSchema->cache_size;
      if( mxCache<SORTER_MIN_WORKING ) mxCache = SORTER_MIN_WORKING;
      pSorter->mxPmaSize = mxCache * pgsz;
................................................................................
** is guaranteed to be nByte bytes or smaller in size. This function
** attempts to extend the file to nByte bytes in size and to ensure that
** the VFS has memory mapped it.
**
** Whether or not the file does end up memory mapped of course depends on
** the specific VFS implementation.
*/
static int vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFile, i64 nByte){
  int rc = SQLITE_OK;
  if( nByte<=(i64)(db->nMaxSorterMmap) ){
    rc = sqlite3OsTruncate(pFile, nByte);
    if( rc==SQLITE_OK ){
      void *p = 0;
      sqlite3OsFetch(pFile, 0, nByte, &p);
      sqlite3OsUnfetch(pFile, 0, p);
    }
  }
  return rc;
}
#else
# define vdbeSorterExtendFile(x,y,z) SQLITE_OK
#endif


/*
** Write the current contents of the in-memory linked-list to a PMA. Return
** SQLITE_OK if successful, or an SQLite error code otherwise.
**
................................................................................
  PmaWriter writer;               /* Object used to write to the file */

  memset(&writer, 0, sizeof(PmaWriter));
  assert( pTask->nInMemory>0 );

  /* If the first temporary PMA file has not been opened, open it now. */
  if( pTask->pTemp1==0 ){
    rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pTask->pTemp1);
    assert( rc!=SQLITE_OK || pTask->pTemp1 );
    assert( pTask->iTemp1Off==0 );
    assert( pTask->nPMA==0 );
  }

  /* Try to get the file to memory map */
  if( rc==SQLITE_OK ){
    rc = vdbeSorterExtendFile(pTask->db, 
        pTask->pTemp1, pTask->iTemp1Off + pTask->nInMemory + 9
    );
  }

  if( rc==SQLITE_OK ){
    SorterRecord *p;
    SorterRecord *pNext = 0;
................................................................................
      pMerger = vdbeMergeEngineNew(nIter);
      if( pMerger==0 ){
        rc = SQLITE_NOMEM;
        break;
      }

      /* Open a second temp file to write merged data to */
      rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pTemp2);
      if( rc==SQLITE_OK ){
        rc = vdbeSorterExtendFile(pTask->db, pTemp2, pTask->iTemp1Off);
      }
      if( rc!=SQLITE_OK ){
        vdbeMergeEngineFree(pMerger);
        break;
      }

      /* This loop runs once for each output PMA. Each output PMA is made

Changes to test/permutations.test. 

108
109
110
111
112
113
114

115
116
117
118
119
120
121

  savepoint4.test savepoint6.test select9.test 
  speed1.test speed1p.test speed2.test speed3.test speed4.test 
  speed4p.test sqllimits1.test tkt2686.test thread001.test thread002.test
  thread003.test thread004.test thread005.test trans2.test vacuum3.test 
  incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test
  vtab_err.test walslow.test walcrash.test walcrash3.test
  walthread.test rtree3.test indexfault.test securedel2.test

}]
if {[info exists ::env(QUICKTEST_INCLUDE)]} {
  set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)]
}

#############################################################################
# Start of tests








>








108
109
110
111
112
113
114
115
116
117
118
119
120
121
122

  savepoint4.test savepoint6.test select9.test 
  speed1.test speed1p.test speed2.test speed3.test speed4.test 
  speed4p.test sqllimits1.test tkt2686.test thread001.test thread002.test
  thread003.test thread004.test thread005.test trans2.test vacuum3.test 
  incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test
  vtab_err.test walslow.test walcrash.test walcrash3.test
  walthread.test rtree3.test indexfault.test securedel2.test
  sort3.test
}]
if {[info exists ::env(QUICKTEST_INCLUDE)]} {
  set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)]
}

#############################################################################
# Start of tests

This page was generated in about 0.054s by Fossil version 2.4 [7ac19afae3] 2017-10-21 19:37:09