/ Check-in [9d3351b8]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix an integer overflow problem in the sorter.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | threads
Files: files | file ages | folders
SHA1: 9d3351b8d713232133dad149c73fb2a27c72abb1
User & Date: dan 2014-04-03 16:25:29
References
2014-04-04
07:52
Add test file sort3.test, which should have been part of commit [9d3351b8d7]. check-in: dceed2c8 user: dan tags: threads
Context
2014-04-03
16:42
Merge all recent changes from trunk. check-in: a0910079 user: drh tags: threads
16:25
Fix an integer overflow problem in the sorter. check-in: 9d3351b8 user: dan tags: threads
14:29
Fix minor errors causing compilation to fail with SQLITE_MAX_WORKER_THREADS set to a value greater than zero. check-in: 0561272a user: dan tags: threads
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/main.c.

  2500   2500   
  2501   2501     assert( sizeof(db->aLimit)==sizeof(aHardLimit) );
  2502   2502     memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit));
  2503   2503     db->autoCommit = 1;
  2504   2504     db->nextAutovac = -1;
  2505   2505     db->szMmap = sqlite3GlobalConfig.szMmap;
  2506   2506     db->nextPagesize = 0;
         2507  +  db->nMaxSorterMmap = 0x7FFFFFFF;
  2507   2508     db->flags |= SQLITE_ShortColNames | SQLITE_EnableTrigger | SQLITE_CacheSpill
  2508   2509   #if !defined(SQLITE_DEFAULT_AUTOMATIC_INDEX) || SQLITE_DEFAULT_AUTOMATIC_INDEX
  2509   2510                    | SQLITE_AutoIndex
  2510   2511   #endif
  2511   2512   #if SQLITE_DEFAULT_FILE_FORMAT<4
  2512   2513                    | SQLITE_LegacyFileFmt
  2513   2514   #endif
................................................................................
  3325   3326   #ifdef SQLITE_VDBE_COVERAGE
  3326   3327         typedef void (*branch_callback)(void*,int,u8,u8);
  3327   3328         sqlite3GlobalConfig.xVdbeBranch = va_arg(ap,branch_callback);
  3328   3329         sqlite3GlobalConfig.pVdbeBranchArg = va_arg(ap,void*);
  3329   3330   #endif
  3330   3331         break;
  3331   3332       }
         3333  +
         3334  +    /*   sqlite3_test_control(SQLITE_TESTCTRL_SORTER_MMAP, db, nMax); */
         3335  +    case SQLITE_TESTCTRL_SORTER_MMAP: {
         3336  +      sqlite3 *db = va_arg(ap, sqlite3*);
         3337  +      db->nMaxSorterMmap = va_arg(ap, int);
         3338  +      break;
         3339  +    }
  3332   3340   
  3333   3341     }
  3334   3342     va_end(ap);
  3335   3343   #endif /* SQLITE_OMIT_BUILTIN_TEST */
  3336   3344     return rc;
  3337   3345   }
  3338   3346   

Changes to src/sqlite.h.in.

  6125   6125   #define SQLITE_TESTCTRL_OPTIMIZATIONS           15
  6126   6126   #define SQLITE_TESTCTRL_ISKEYWORD               16
  6127   6127   #define SQLITE_TESTCTRL_SCRATCHMALLOC           17
  6128   6128   #define SQLITE_TESTCTRL_LOCALTIME_FAULT         18
  6129   6129   #define SQLITE_TESTCTRL_EXPLAIN_STMT            19
  6130   6130   #define SQLITE_TESTCTRL_NEVER_CORRUPT           20
  6131   6131   #define SQLITE_TESTCTRL_VDBE_COVERAGE           21
  6132         -#define SQLITE_TESTCTRL_LAST                    21
         6132  +#define SQLITE_TESTCTRL_SORTER_MMAP             22
         6133  +#define SQLITE_TESTCTRL_LAST                    22
  6133   6134   
  6134   6135   /*
  6135   6136   ** CAPI3REF: SQLite Runtime Status
  6136   6137   **
  6137   6138   ** ^This interface is used to retrieve runtime status information
  6138   6139   ** about the performance of SQLite, and optionally to reset various
  6139   6140   ** highwater marks.  ^The first argument is an integer code for

Changes to src/sqliteInt.h.

   977    977     u8 vtabOnConflict;            /* Value to return for s3_vtab_on_conflict() */
   978    978     u8 isTransactionSavepoint;    /* True if the outermost savepoint is a TS */
   979    979     int nextPagesize;             /* Pagesize after VACUUM if >0 */
   980    980     u32 magic;                    /* Magic number for detect library misuse */
   981    981     int nChange;                  /* Value returned by sqlite3_changes() */
   982    982     int nTotalChange;             /* Value returned by sqlite3_total_changes() */
   983    983     int aLimit[SQLITE_N_LIMIT];   /* Limits */
          984  +  int nMaxSorterMmap;           /* Maximum size of regions mapped by sorter */
   984    985     struct sqlite3InitInfo {      /* Information used during initialization */
   985    986       int newTnum;                /* Rootpage of table being initialized */
   986    987       u8 iDb;                     /* Which db file is being initialized */
   987    988       u8 busy;                    /* TRUE if currently initializing */
   988    989       u8 orphanTrigger;           /* Last statement is orphaned TEMP trigger */
   989    990     } init;
   990    991     int nVdbeActive;              /* Number of VDBEs currently running */

Changes to src/test1.c.

  5880   5880     Tcl_Obj *CONST objv[]
  5881   5881   ){
  5882   5882     struct Verb {
  5883   5883       const char *zName;
  5884   5884       int i;
  5885   5885     } aVerb[] = {
  5886   5886       { "SQLITE_TESTCTRL_LOCALTIME_FAULT", SQLITE_TESTCTRL_LOCALTIME_FAULT }, 
         5887  +    { "SQLITE_TESTCTRL_SORTER_MMAP", SQLITE_TESTCTRL_SORTER_MMAP }, 
  5887   5888     };
  5888   5889     int iVerb;
  5889   5890     int iFlag;
  5890   5891     int rc;
  5891   5892   
  5892   5893     if( objc<2 ){
  5893   5894       Tcl_WrongNumArgs(interp, 1, objv, "VERB ARGS...");
................................................................................
  5907   5908           Tcl_WrongNumArgs(interp, 2, objv, "ONOFF");
  5908   5909           return TCL_ERROR;
  5909   5910         }
  5910   5911         if( Tcl_GetBooleanFromObj(interp, objv[2], &val) ) return TCL_ERROR;
  5911   5912         sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, val);
  5912   5913         break;
  5913   5914       }
         5915  +
         5916  +    case SQLITE_TESTCTRL_SORTER_MMAP: {
         5917  +      int val;
         5918  +      sqlite3 *db;
         5919  +      if( objc!=4 ){
         5920  +        Tcl_WrongNumArgs(interp, 2, objv, "DB LIMIT");
         5921  +        return TCL_ERROR;
         5922  +      }
         5923  +      if( getDbPointer(interp, Tcl_GetString(objv[2]), &db) ) return TCL_ERROR;
         5924  +      if( Tcl_GetIntFromObj(interp, objv[3], &val) ) return TCL_ERROR;
         5925  +      sqlite3_test_control(SQLITE_TESTCTRL_SORTER_MMAP, db, val);
         5926  +      break;
         5927  +    }
  5914   5928     }
  5915   5929   
  5916   5930     Tcl_ResetResult(interp);
  5917   5931     return TCL_OK;
  5918   5932   }
  5919   5933   
  5920   5934   #if SQLITE_OS_UNIX

Changes to src/vdbesort.c.

   140    140   **     Merge existing PMAs until SortSubtask.nConsolidate or fewer
   141    141   **     remain in temp file SortSubtask.pTemp1.
   142    142   */
   143    143   struct SortSubtask {
   144    144     SQLiteThread *pThread;          /* Thread handle, or NULL */
   145    145     int bDone;                      /* Set to true by pTask when finished */
   146    146   
   147         -  sqlite3_vfs *pVfs;              /* VFS used to open temporary files */
          147  +  sqlite3 *db;                    /* Database connection */
   148    148     KeyInfo *pKeyInfo;              /* How to compare records */
   149    149     UnpackedRecord *pUnpacked;      /* Space to unpack a record */
   150    150     int pgsz;                       /* Main database page size */
   151    151   
   152    152     u8 eWork;                       /* One of the SORT_SUBTASK_* constants */
   153    153     int nConsolidate;               /* For SORT_SUBTASK_CONS, max final PMAs */
   154    154     SorterRecord *pList;            /* List of records for pTask to sort */
................................................................................
   510    510     pIter->pFile = pTask->pTemp1;
   511    511     pIter->iReadOff = iStart;
   512    512     pIter->nAlloc = 128;
   513    513     pIter->aAlloc = (u8*)sqlite3Malloc(pIter->nAlloc);
   514    514     if( pIter->aAlloc ){
   515    515       /* Try to xFetch() a mapping of the entire temp file. If this is possible,
   516    516       ** the PMA will be read via the mapping. Otherwise, use xRead().  */
   517         -    rc = sqlite3OsFetch(pIter->pFile, 0, pTask->iTemp1Off, &pMap);
          517  +    if( pTask->iTemp1Off<=(i64)(pTask->db->nMaxSorterMmap) ){
          518  +      rc = sqlite3OsFetch(pIter->pFile, 0, pTask->iTemp1Off, &pMap);
          519  +    }
   518    520     }else{
   519    521       rc = SQLITE_NOMEM;
   520    522     }
   521    523   
   522    524     if( rc==SQLITE_OK ){
   523    525       if( pMap ){
   524    526         pIter->aMap = (u8*)pMap;
................................................................................
   666    668       if( nField && nWorker==0 ) pKeyInfo->nField = nField;
   667    669       pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt);
   668    670   
   669    671       pSorter->nTask = nWorker + 1;
   670    672       for(i=0; i<pSorter->nTask; i++){
   671    673         SortSubtask *pTask = &pSorter->aTask[i];
   672    674         pTask->pKeyInfo = pKeyInfo;
   673         -      pTask->pVfs = db->pVfs;
   674    675         pTask->pgsz = pgsz;
          676  +      pTask->db = db;
   675    677       }
   676    678   
   677    679       if( !sqlite3TempInMemory(db) ){
   678    680         pSorter->mnPmaSize = SORTER_MIN_WORKING * pgsz;
   679    681         mxCache = db->aDb[0].pSchema->cache_size;
   680    682         if( mxCache<SORTER_MIN_WORKING ) mxCache = SORTER_MIN_WORKING;
   681    683         pSorter->mxPmaSize = mxCache * pgsz;
................................................................................
  1011   1013   ** is guaranteed to be nByte bytes or smaller in size. This function
  1012   1014   ** attempts to extend the file to nByte bytes in size and to ensure that
  1013   1015   ** the VFS has memory mapped it.
  1014   1016   **
  1015   1017   ** Whether or not the file does end up memory mapped of course depends on
  1016   1018   ** the specific VFS implementation.
  1017   1019   */
  1018         -static int vdbeSorterExtendFile(sqlite3_file *pFile, i64 nByte){
  1019         -  int rc = sqlite3OsTruncate(pFile, nByte);
  1020         -  if( rc==SQLITE_OK ){
  1021         -    void *p = 0;
  1022         -    sqlite3OsFetch(pFile, 0, nByte, &p);
  1023         -    sqlite3OsUnfetch(pFile, 0, p);
         1020  +static int vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFile, i64 nByte){
         1021  +  int rc = SQLITE_OK;
         1022  +  if( nByte<=(i64)(db->nMaxSorterMmap) ){
         1023  +    rc = sqlite3OsTruncate(pFile, nByte);
         1024  +    if( rc==SQLITE_OK ){
         1025  +      void *p = 0;
         1026  +      sqlite3OsFetch(pFile, 0, nByte, &p);
         1027  +      sqlite3OsUnfetch(pFile, 0, p);
         1028  +    }
  1024   1029     }
  1025   1030     return rc;
  1026   1031   }
  1027   1032   #else
  1028         -# define vdbeSorterExtendFile(x,y) SQLITE_OK
         1033  +# define vdbeSorterExtendFile(x,y,z) SQLITE_OK
  1029   1034   #endif
  1030   1035   
  1031   1036   
  1032   1037   /*
  1033   1038   ** Write the current contents of the in-memory linked-list to a PMA. Return
  1034   1039   ** SQLITE_OK if successful, or an SQLite error code otherwise.
  1035   1040   **
................................................................................
  1047   1052     PmaWriter writer;               /* Object used to write to the file */
  1048   1053   
  1049   1054     memset(&writer, 0, sizeof(PmaWriter));
  1050   1055     assert( pTask->nInMemory>0 );
  1051   1056   
  1052   1057     /* If the first temporary PMA file has not been opened, open it now. */
  1053   1058     if( pTask->pTemp1==0 ){
  1054         -    rc = vdbeSorterOpenTempFile(pTask->pVfs, &pTask->pTemp1);
         1059  +    rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pTask->pTemp1);
  1055   1060       assert( rc!=SQLITE_OK || pTask->pTemp1 );
  1056   1061       assert( pTask->iTemp1Off==0 );
  1057   1062       assert( pTask->nPMA==0 );
  1058   1063     }
  1059   1064   
  1060   1065     /* Try to get the file to memory map */
  1061   1066     if( rc==SQLITE_OK ){
  1062         -    rc = vdbeSorterExtendFile(
         1067  +    rc = vdbeSorterExtendFile(pTask->db, 
  1063   1068           pTask->pTemp1, pTask->iTemp1Off + pTask->nInMemory + 9
  1064   1069       );
  1065   1070     }
  1066   1071   
  1067   1072     if( rc==SQLITE_OK ){
  1068   1073       SorterRecord *p;
  1069   1074       SorterRecord *pNext = 0;
................................................................................
  1202   1207         pMerger = vdbeMergeEngineNew(nIter);
  1203   1208         if( pMerger==0 ){
  1204   1209           rc = SQLITE_NOMEM;
  1205   1210           break;
  1206   1211         }
  1207   1212   
  1208   1213         /* Open a second temp file to write merged data to */
  1209         -      rc = vdbeSorterOpenTempFile(pTask->pVfs, &pTemp2);
         1214  +      rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pTemp2);
  1210   1215         if( rc==SQLITE_OK ){
  1211         -        rc = vdbeSorterExtendFile(pTemp2, pTask->iTemp1Off);
         1216  +        rc = vdbeSorterExtendFile(pTask->db, pTemp2, pTask->iTemp1Off);
  1212   1217         }
  1213   1218         if( rc!=SQLITE_OK ){
  1214   1219           vdbeMergeEngineFree(pMerger);
  1215   1220           break;
  1216   1221         }
  1217   1222   
  1218   1223         /* This loop runs once for each output PMA. Each output PMA is made

Changes to test/permutations.test.

   108    108     savepoint4.test savepoint6.test select9.test 
   109    109     speed1.test speed1p.test speed2.test speed3.test speed4.test 
   110    110     speed4p.test sqllimits1.test tkt2686.test thread001.test thread002.test
   111    111     thread003.test thread004.test thread005.test trans2.test vacuum3.test 
   112    112     incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test
   113    113     vtab_err.test walslow.test walcrash.test walcrash3.test
   114    114     walthread.test rtree3.test indexfault.test securedel2.test
          115  +  sort3.test
   115    116   }]
   116    117   if {[info exists ::env(QUICKTEST_INCLUDE)]} {
   117    118     set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)]
   118    119   }
   119    120   
   120    121   #############################################################################
   121    122   # Start of tests