/ Check-in [d7e2b0d9]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:In the sorter, only use large memory allocations if scratch memory has not been configured. Add #ifdefs to disable unused code when SQLITE_MAX_WORKER_THREADS is zero. Other sorter changes in support of testability.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | threads
Files: files | file ages | folders
SHA1: d7e2b0d9cb099eda3341bc934bedff9facfe88bd
User & Date: drh 2014-05-12 15:30:00
Context
2014-05-15
16:56
Use #ifdef to omit code that is not used when SQLITE_MAX_WORKER_THREADS is zero. check-in: 2e8d287d user: drh tags: threads
2014-05-12
15:30
In the sorter, only use large memory allocations if scratch memory has not been configured. Add #ifdefs to disable unused code when SQLITE_MAX_WORKER_THREADS is zero. Other sorter changes in support of testability. check-in: d7e2b0d9 user: drh tags: threads
2014-05-09
15:00
Merge the latest trunk changes into the threads branch. check-in: 9ac8f1e7 user: dan tags: threads
Changes
Hide Diffs Side-by-Side Diffs Show Whitespace Changes Patch

Changes to src/main.c.

   512    512       case SQLITE_CONFIG_WIN32_HEAPSIZE: {
   513    513         sqlite3GlobalConfig.nHeap = va_arg(ap, int);
   514    514         break;
   515    515       }
   516    516   #endif
   517    517   
   518    518       case SQLITE_CONFIG_WORKER_THREADS: {
          519  +#if SQLITE_MAX_WORKER_THREADS>0
   519    520         int n = va_arg(ap, int);
   520    521         if( n>SQLITE_MAX_WORKER_THREADS ) n = SQLITE_MAX_WORKER_THREADS;
   521    522         if( n>=0 ) sqlite3GlobalConfig.nWorker = n;
          523  +#endif
   522    524         break;
   523    525       }
   524    526   
   525    527       default: {
   526    528         rc = SQLITE_ERROR;
   527    529         break;
   528    530       }

Changes to src/vdbesort.c.

   445    445   ** Free all memory belonging to the PmaReader object passed as the second
   446    446   ** argument. All structure fields are set to zero before returning.
   447    447   */
   448    448   static void vdbePmaReaderClear(PmaReader *pIter){
   449    449     sqlite3_free(pIter->aAlloc);
   450    450     sqlite3_free(pIter->aBuffer);
   451    451     if( pIter->aMap ) sqlite3OsUnfetch(pIter->pFile, 0, pIter->aMap);
   452         -  if( pIter->pIncr ) vdbeIncrFree(pIter->pIncr);
          452  +  vdbeIncrFree(pIter->pIncr);
   453    453     memset(pIter, 0, sizeof(PmaReader));
   454    454   }
   455    455   
   456    456   /*
   457    457   ** Read nByte bytes of data from the stream of data iterated by object p.
   458    458   ** If successful, set *ppOut to point to a buffer containing the data
   459    459   ** and return SQLITE_OK. Otherwise, if an error occurs, return an SQLite
................................................................................
   824    824     int i;                          /* Used to iterate through aTask[] */
   825    825     int mxCache;                    /* Cache size */
   826    826     VdbeSorter *pSorter;            /* The new sorter */
   827    827     KeyInfo *pKeyInfo;              /* Copy of pCsr->pKeyInfo with db==0 */
   828    828     int szKeyInfo;                  /* Size of pCsr->pKeyInfo in bytes */
   829    829     int sz;                         /* Size of pSorter in bytes */
   830    830     int rc = SQLITE_OK;
          831  +#if SQLITE_MAX_WORKER_THREADS==0
          832  +  const int nWorker = 0;
          833  +#else
   831    834     int nWorker = (sqlite3GlobalConfig.bCoreMutex?sqlite3GlobalConfig.nWorker:0);
          835  +#endif
   832    836   
   833    837     assert( pCsr->pKeyInfo && pCsr->pBt==0 );
   834    838     szKeyInfo = sizeof(KeyInfo) + (pCsr->pKeyInfo->nField-1)*sizeof(CollSeq*);
   835    839     sz = sizeof(VdbeSorter) + nWorker * sizeof(SortSubtask);
   836    840   
   837    841     pSorter = (VdbeSorter*)sqlite3DbMallocZero(db, sz + szKeyInfo);
   838    842     pCsr->pSorter = pSorter;
................................................................................
   854    858   
   855    859       if( !sqlite3TempInMemory(db) ){
   856    860         pSorter->mnPmaSize = SORTER_MIN_WORKING * pgsz;
   857    861         mxCache = db->aDb[0].pSchema->cache_size;
   858    862         if( mxCache<SORTER_MIN_WORKING ) mxCache = SORTER_MIN_WORKING;
   859    863         pSorter->mxPmaSize = mxCache * pgsz;
   860    864   
   861         -      /* If the application is using memsys3 or memsys5, use a separate 
   862         -      ** allocation for each sort-key in memory. Otherwise, use a single big
   863         -      ** allocation at pSorter->aMemory for all sort-keys.  */
   864         -      if( sqlite3GlobalConfig.pHeap==0 ){
          865  +      /* If the application has not configure scratch memory using
          866  +      ** SQLITE_CONFIG_SCRATCH then we assume it is OK to do large memory
          867  +      ** allocations.  If scratch memory has been configured, then assume
          868  +      ** large memory allocations should be avoided to prevent heap
          869  +      ** fragmentation.
          870  +      */
          871  +      if( sqlite3GlobalConfig.pScratch==0 ){
   865    872           assert( pSorter->iMemory==0 );
   866    873           pSorter->nMemory = pgsz;
   867    874           pSorter->list.aMemory = (u8*)sqlite3Malloc(pgsz);
   868    875           if( !pSorter->list.aMemory ) rc = SQLITE_NOMEM;
   869    876         }
   870    877       }
   871    878     }
................................................................................
  1917   1924       rc = vdbeIncrInitMerger(pTask, pIncr->pMerger, eMode);
  1918   1925   
  1919   1926       /* Set up the required files for pIncr. A multi-theaded IncrMerge object
  1920   1927       ** requires two temp files to itself, whereas a single-threaded object
  1921   1928       ** only requires a region of pTask->file2. */
  1922   1929       if( rc==SQLITE_OK ){
  1923   1930         int mxSz = pIncr->mxSz;
  1924         -      if( pIncr->bUseThread==0 ){
         1931  +#if SQLITE_MAX_WORKER_THREADS>0
         1932  +      if( pIncr->bUseThread ){
         1933  +        rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[0].pFd);
         1934  +        if( rc==SQLITE_OK ){
         1935  +          rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[1].pFd);
         1936  +        }
         1937  +      }else
         1938  +#endif
         1939  +      /*if( !pIncr->bUseThread )*/{
  1925   1940           if( pTask->file2.pFd==0 ){
  1926   1941             assert( pTask->file2.iEof>0 );
  1927   1942             rc = vdbeSorterOpenTempFile(db, pTask->file2.iEof, &pTask->file2.pFd);
  1928   1943             pTask->file2.iEof = 0;
  1929   1944           }
  1930   1945           if( rc==SQLITE_OK ){
  1931   1946             pIncr->aFile[1].pFd = pTask->file2.pFd;
  1932   1947             pIncr->iStartOff = pTask->file2.iEof;
  1933   1948             pTask->file2.iEof += mxSz;
  1934   1949           }
  1935         -      }else{
  1936         -        rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[0].pFd);
  1937         -        if( rc==SQLITE_OK ){
  1938         -          rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[1].pFd);
  1939         -        }
  1940   1950         }
  1941   1951       }
  1942   1952   
         1953  +#if SQLITE_MAX_WORKER_THREADS>0
  1943   1954       if( rc==SQLITE_OK && pIncr->bUseThread ){
  1944   1955         /* Use the current thread to populate aFile[1], even though this
  1945   1956         ** iterator is multi-threaded. The reason being that this function
  1946   1957         ** is already running in background thread pIncr->pTask->thread. */
  1947   1958         assert( eMode==INCRINIT_ROOT || eMode==INCRINIT_TASK );
  1948   1959         rc = vdbeIncrPopulate(pIncr);
  1949   1960       }
         1961  +#endif
  1950   1962   
  1951   1963       if( rc==SQLITE_OK && eMode!=INCRINIT_TASK ){
  1952   1964         rc = vdbePmaReaderNext(pIter);
  1953   1965       }
  1954   1966     }
  1955   1967     return rc;
  1956   1968   }
................................................................................
  2107   2119   ** of *ppOut is undefined.
  2108   2120   */
  2109   2121   static int vdbeSorterMergeTreeBuild(VdbeSorter *pSorter, MergeEngine **ppOut){
  2110   2122     MergeEngine *pMain = 0;
  2111   2123     int rc = SQLITE_OK;
  2112   2124     int iTask;
  2113   2125   
         2126  +#if SQLITE_MAX_WORKER_THREADS>0
  2114   2127     /* If the sorter uses more than one task, then create the top-level 
  2115   2128     ** MergeEngine here. This MergeEngine will read data from exactly 
  2116   2129     ** one PmaReader per sub-task.  */
  2117   2130     assert( pSorter->bUseThreads || pSorter->nTask==1 );
  2118   2131     if( pSorter->nTask>1 ){
  2119   2132       pMain = vdbeMergeEngineNew(pSorter->nTask);
  2120   2133       if( pMain==0 ) rc = SQLITE_NOMEM;
  2121   2134     }
         2135  +#endif
  2122   2136   
  2123   2137     for(iTask=0; iTask<pSorter->nTask && rc==SQLITE_OK; iTask++){
  2124   2138       SortSubtask *pTask = &pSorter->aTask[iTask];
  2125   2139       if( pTask->nPMA ){
  2126   2140         MergeEngine *pRoot = 0;     /* Root node of tree for this task */
  2127   2141         int nDepth = vdbeSorterTreeDepth(pTask->nPMA);
  2128   2142         i64 iReadOff = 0;
................................................................................
  2297   2311     int rc;                         /* Return code */
  2298   2312   
  2299   2313     assert( pSorter->bUsePMA || (pSorter->pReader==0 && pSorter->pMerger==0) );
  2300   2314     if( pSorter->bUsePMA ){
  2301   2315       assert( pSorter->pReader==0 || pSorter->pMerger==0 );
  2302   2316       assert( pSorter->bUseThreads==0 || pSorter->pReader );
  2303   2317       assert( pSorter->bUseThreads==1 || pSorter->pMerger );
         2318  +#if SQLITE_MAX_WORKER_THREADS>0
  2304   2319       if( pSorter->bUseThreads ){
  2305   2320         rc = vdbePmaReaderNext(pSorter->pReader);
  2306   2321         *pbEof = (pSorter->pReader->pFile==0);
  2307         -    }else{
         2322  +    }else
         2323  +#endif
         2324  +    /*if( !pSorter->bUseThreads )*/ {
  2308   2325         rc = vdbeSorterNext(&pSorter->aTask[0], pSorter->pMerger, pbEof);
  2309   2326       }
  2310   2327     }else{
  2311   2328       SorterRecord *pFree = pSorter->list.pList;
  2312   2329       pSorter->list.pList = pFree->u.pNext;
  2313   2330       pFree->u.pNext = 0;
  2314   2331       if( pSorter->list.aMemory==0 ) vdbeSorterRecordFree(db, pFree);
................................................................................
  2324   2341   */
  2325   2342   static void *vdbeSorterRowkey(
  2326   2343     const VdbeSorter *pSorter,      /* Sorter object */
  2327   2344     int *pnKey                      /* OUT: Size of current key in bytes */
  2328   2345   ){
  2329   2346     void *pKey;
  2330   2347     if( pSorter->bUsePMA ){
  2331         -    PmaReader *pReader = (pSorter->bUseThreads ?
  2332         -        pSorter->pReader : &pSorter->pMerger->aIter[pSorter->pMerger->aTree[1]]
  2333         -    );
         2348  +    PmaReader *pReader;
         2349  +#if SQLITE_MAX_WORKER_THREADS>0
         2350  +    if( pSorter->bUseThreads ){
         2351  +      pReader = pSorter->pReader;
         2352  +    }else
         2353  +#endif
         2354  +    /*if( !pSorter->bUseThreads )*/{
         2355  +      pReader = &pSorter->pMerger->aIter[pSorter->pMerger->aTree[1]];
         2356  +    }
  2334   2357       *pnKey = pReader->nKey;
  2335   2358       pKey = pReader->aKey;
  2336   2359     }else{
  2337   2360       *pnKey = pSorter->list.pList->nVal;
  2338   2361       pKey = SRVAL(pSorter->list.pList);
  2339   2362     }
  2340   2363     return pKey;