Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Fix build problems in vdbesort.c. Add further comments and changes to make things easier to understand. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | threads |
Files: | files | file ages | folders |
SHA1: |
12b190db1d20c34390c768614b40ff58 |
User & Date: | dan 2014-04-17 08:57:17.925 |
Context
2014-04-18
| ||
12:38 | Merge recent trunk changes into the threads branch. (check-in: 8729aa3e3e user: drh tags: threads) | |
2014-04-17
| ||
08:57 | Fix build problems in vdbesort.c. Add further comments and changes to make things easier to understand. (check-in: 12b190db1d user: dan tags: threads) | |
2014-04-16
| ||
19:04 | Clarify the purpose of the nField argument passed to sqlite3VdbeSorterInit(). (check-in: c0c8cff17b user: dan tags: threads) | |
Changes
Changes to src/vdbesort.c.
︙ | ︙ | |||
907 908 909 910 911 912 913 | } } #ifdef SQLITE_DEBUG_SORTER_THREADS static void vdbeSorterWorkDebug(SortSubtask *pTask, const char *zEvent){ i64 t; int iTask = (pTask - pTask->pSorter->aTask); | | | | | 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 | } } #ifdef SQLITE_DEBUG_SORTER_THREADS static void vdbeSorterWorkDebug(SortSubtask *pTask, const char *zEvent){ i64 t; int iTask = (pTask - pTask->pSorter->aTask); sqlite3OsCurrentTimeInt64(pTask->pSorter->db->pVfs, &t); fprintf(stderr, "%lld:%d %s\n", t, iTask, zEvent); } static void vdbeSorterRewindDebug(sqlite3 *db, const char *zEvent){ i64 t; sqlite3OsCurrentTimeInt64(db->pVfs, &t); fprintf(stderr, "%lld:X %s\n", t, zEvent); } static void vdbeSorterPopulateDebug( SortSubtask *pTask, const char *zEvent ){ i64 t; int iTask = (pTask - pTask->pSorter->aTask); sqlite3OsCurrentTimeInt64(pTask->pSorter->db->pVfs, &t); fprintf(stderr, "%lld:bg%d %s\n", t, iTask, zEvent); } static void vdbeSorterBlockDebug( SortSubtask *pTask, int bBlocked, const char *zEvent ){ if( bBlocked ){ i64 t; sqlite3OsCurrentTimeInt64(pTask->pSorter->db->pVfs, &t); fprintf(stderr, "%lld:main %s\n", t, zEvent); } } #else # define vdbeSorterWorkDebug(x,y) # define vdbeSorterRewindDebug(x,y) # define vdbeSorterPopulateDebug(x,y) |
︙ | ︙ | |||
1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 | sqlite3VdbeSorterReset(db, pSorter); sqlite3_free(pSorter->list.aMemory); sqlite3DbFree(db, pSorter); pCsr->pSorter = 0; } } /* ** Allocate space for a file-handle and open a temporary file. If successful, ** set *ppFile to point to the malloc'd file-handle and return SQLITE_OK. ** Otherwise, set *ppFile to 0 and return an SQLite error code. */ | > > > > > > > > > > > > > > > > > > > > > > > > | > > > > | | > > > | 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 | sqlite3VdbeSorterReset(db, pSorter); sqlite3_free(pSorter->list.aMemory); sqlite3DbFree(db, pSorter); pCsr->pSorter = 0; } } #if SQLITE_MAX_MMAP_SIZE>0 /* ** The first argument is a file-handle open on a temporary file. The file ** is guaranteed to be nByte bytes or smaller in size. This function ** attempts to extend the file to nByte bytes in size and to ensure that ** the VFS has memory mapped it. ** ** Whether or not the file does end up memory mapped of course depends on ** the specific VFS implementation. */ static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFile, i64 nByte){ if( nByte<=(i64)(db->nMaxSorterMmap) ){ int rc = sqlite3OsTruncate(pFile, nByte); if( rc==SQLITE_OK ){ void *p = 0; sqlite3OsFetch(pFile, 0, nByte, &p); sqlite3OsUnfetch(pFile, 0, p); } } } #else # define vdbeSorterExtendFile(x,y,z) SQLITE_OK #endif /* ** Allocate space for a file-handle and open a temporary file. If successful, ** set *ppFile to point to the malloc'd file-handle and return SQLITE_OK. ** Otherwise, set *ppFile to 0 and return an SQLite error code. */ static int vdbeSorterOpenTempFile( sqlite3 *db, /* Database handle doing sort */ i64 nExtend, /* Attempt to extend file to this size */ sqlite3_file **ppFile ){ int rc; rc = sqlite3OsOpenMalloc(db->pVfs, 0, ppFile, SQLITE_OPEN_TEMP_JOURNAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE, &rc ); if( rc==SQLITE_OK ){ i64 max = SQLITE_MAX_MMAP_SIZE; sqlite3OsFileControlHint(*ppFile, SQLITE_FCNTL_MMAP_SIZE, (void*)&max); if( nExtend>0 ){ vdbeSorterExtendFile(db, *ppFile, nExtend); } } return rc; } /* ** If it has not already been allocated, allocate the UnpackedRecord ** structure at pTask->pUnpacked. Return SQLITE_OK if successful (or |
︙ | ︙ | |||
1302 1303 1304 1305 1306 1307 1308 | */ static void vdbePmaWriteVarint(PmaWriter *p, u64 iVal){ int nByte; u8 aByte[10]; nByte = sqlite3PutVarint(aByte, iVal); vdbePmaWriteBlob(p, aByte, nByte); } | < < < < < < < < < < < < < < < < < < < < < < < < < | 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 | */ static void vdbePmaWriteVarint(PmaWriter *p, u64 iVal){ int nByte; u8 aByte[10]; nByte = sqlite3PutVarint(aByte, iVal); vdbePmaWriteBlob(p, aByte, nByte); } /* ** Write the current contents of in-memory linked-list pList to a level-0 ** PMA in the temp file belonging to sub-task pTask. Return SQLITE_OK if ** successful, or an SQLite error code otherwise. ** ** The format of a PMA is: |
︙ | ︙ | |||
1359 1360 1361 1362 1363 1364 1365 | vdbeSorterWorkDebug(pTask, "enter"); memset(&writer, 0, sizeof(PmaWriter)); assert( pList->szPMA>0 ); /* If the first temporary PMA file has not been opened, open it now. */ if( pTask->file.pFd==0 ){ | | | 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 | vdbeSorterWorkDebug(pTask, "enter"); memset(&writer, 0, sizeof(PmaWriter)); assert( pList->szPMA>0 ); /* If the first temporary PMA file has not been opened, open it now. */ if( pTask->file.pFd==0 ){ rc = vdbeSorterOpenTempFile(db, 0, &pTask->file.pFd); assert( rc!=SQLITE_OK || pTask->file.pFd ); assert( pTask->file.iEof==0 ); assert( pTask->nPMA==0 ); } /* Try to get the file to memory map */ if( rc==SQLITE_OK ){ |
︙ | ︙ | |||
1798 1799 1800 1801 1802 1803 1804 | static void vdbeIncrSetThreads(IncrMerger *pIncr, int bUseThread){ if( bUseThread ){ pIncr->bUseThread = 1; pIncr->pTask->file2.iEof -= pIncr->mxSz; } } | | | | | | | | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | > > > < < | | < | | | | > > | | | | | 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 | static void vdbeIncrSetThreads(IncrMerger *pIncr, int bUseThread){ if( bUseThread ){ pIncr->bUseThread = 1; pIncr->pTask->file2.iEof -= pIncr->mxSz; } } #define INCRINIT_NORMAL 0 #define INCRINIT_TASK 1 #define INCRINIT_ROOT 2 static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode); /* ** Initialize the merger argument passed as the second argument. Once this ** function returns, the first key of merged data may be read from the merger ** object in the usual fashion. ** ** If argument eMode is INCRINIT_ROOT, then it is assumed that any IncrMerge ** objects attached to the PmaReader objects that the merger reads from have ** already been populated, but that they have not yet populated aFile[0] and ** set the PmaReader objects up to read from it. In this case all that is ** required is to call vdbePmaReaderNext() on each iterator to point it at ** its first key. ** ** Otherwise, if eMode is any value other than INCRINIT_ROOT, then use ** vdbePmaReaderIncrInit() to initialize each PmaReader that feeds data ** to pMerger. ** ** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ static int vdbeIncrInitMerger( SortSubtask *pTask, MergeEngine *pMerger, int eMode /* One of the INCRINIT_XXX constants */ ){ int rc = SQLITE_OK; /* Return code */ int i; /* For iterating through PmaReader objects */ for(i=0; rc==SQLITE_OK && i<pMerger->nTree; i++){ if( eMode==INCRINIT_ROOT ){ rc = vdbePmaReaderNext(&pMerger->aIter[i]); }else{ rc = vdbePmaReaderIncrInit(&pMerger->aIter[i], INCRINIT_NORMAL); } } for(i=pMerger->nTree-1; rc==SQLITE_OK && i>0; i--){ rc = vdbeSorterDoCompare(pTask, pMerger, i); } return rc; } /* ** If the PmaReader passed as the first argument is not an incremental-reader ** (if pIter->pIncr==0), then this function is a no-op. Otherwise, it serves ** to open and/or initialize the temp file related fields of the IncrMerge ** object at (pIter->pIncr). ** ** If argument eMode is set to INCRINIT_NORMAL, then PmaReader iterators ** in the sub-tree headed by pIter are also initialized. Data is then loaded ** into the buffers belonging to this iterator, pIter, and it is set to ** point to the first key in its range. ** ** If argument eMode is set to INCRINIT_TASK, then PmaReader is guaranteed ** to be a multi-threaded iterator and this function is being called in a ** background thread. In this case all iterators in the sub-tree are ** initialized as for INCRINIT_NORMAL and the aFile[1] buffer belonging to ** pIter is populated. However, the iterator itself is not set up to point ** to its first key. A call to vdbePmaReaderNext() is still required to do ** that. ** ** The reason this function does not call vdbePmaReaderNext() immediately ** in the INCRINIT_TASK case is that vdbePmaReaderNext() assumes that has ** to block on thread (pTask->thread) before accessing aFile[1]. But, since ** this entire function is being run by thread (pTask->thread), that will ** lead to the current background thread attempting to join itself. ** ** Finally, if argument eMode is set to INCRINIT_ROOT, it may be assumed ** that pIter->pIncr is a multi-threaded IncrMerge objects, and that all ** child-trees have already been initialized using IncrInit(INCRINIT_TASK). ** In this case vdbePmaReaderNext() is called on all child iterators and ** the current iterator set to point to the first key in its range. ** ** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode){ int rc = SQLITE_OK; IncrMerger *pIncr = pIter->pIncr; if( pIncr ){ SortSubtask *pTask = pIncr->pTask; sqlite3 *db = pTask->pSorter->db; assert( eMode==INCRINIT_NORMAL || pIncr->bUseThread==1 ); rc = vdbeIncrInitMerger(pTask, pIncr->pMerger, eMode); /* Set up the required files for pIncr. A multi-theaded IncrMerge object ** requires two temp files to itself, whereas a single-threaded object ** only requires a region of pTask->file2. */ if( rc==SQLITE_OK ){ int mxSz = pIncr->mxSz; if( pIncr->bUseThread==0 ){ if( pTask->file2.pFd==0 ){ assert( pTask->file2.iEof>0 ); rc = vdbeSorterOpenTempFile(db, pTask->file2.iEof, &pTask->file2.pFd); pTask->file2.iEof = 0; } if( rc==SQLITE_OK ){ pIncr->aFile[1].pFd = pTask->file2.pFd; pIncr->iStartOff = pTask->file2.iEof; pTask->file2.iEof += mxSz; } }else{ rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[0].pFd); if( rc==SQLITE_OK ){ rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[1].pFd); } } } if( rc==SQLITE_OK && pIncr->bUseThread ){ /* Use the current thread to populate aFile[1], even though this ** iterator is multi-threaded. The reason being that this function ** is already running in background thread pIncr->pTask->thread. */ assert( eMode==INCRINIT_ROOT || eMode==INCRINIT_TASK ); rc = vdbeIncrPopulate(pIncr); } if( rc==SQLITE_OK && eMode!=INCRINIT_TASK ){ rc = vdbePmaReaderNext(pIter); } } return rc; } #if SQLITE_MAX_WORKER_THREADS>0 /* ** The main routine for vdbePmaReaderIncrInit() operations run in ** background threads. */ static void *vdbePmaReaderBgInit(void *pCtx){ PmaReader *pReader = (PmaReader*)pCtx; void *pRet = SQLITE_INT_TO_PTR(vdbePmaReaderIncrInit(pReader,INCRINIT_TASK)); pReader->pIncr->pTask->bDone = 1; return pRet; } /* ** Use a background thread to invoke vdbePmaReaderIncrInit(INCRINIT_TASK) ** on the the PmaReader object passed as the first argument. ** ** This call will initialize the various fields of the pIter->pIncr ** structure and, if it is a multi-threaded IncrMerger, launch a ** background thread to populate aFile[1]. */ static int vdbePmaReaderBgIncrInit(PmaReader *pIter){ |
︙ | ︙ | |||
2149 2150 2151 2152 2153 2154 2155 | if( p->pIncr ){ rc = vdbePmaReaderBgIncrInit(p); } } } } pMain = 0; } if( rc==SQLITE_OK ){ | | | | 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 | if( p->pIncr ){ rc = vdbePmaReaderBgIncrInit(p); } } } } pMain = 0; } if( rc==SQLITE_OK ){ int eMode = (pSorter->nTask>1 ? INCRINIT_ROOT : INCRINIT_NORMAL); rc = vdbePmaReaderIncrInit(pIter, eMode); } }else #endif { rc = vdbeIncrInitMerger(pTask0, pMain, INCRINIT_NORMAL); pSorter->pMerger = pMain; pMain = 0; } } if( rc!=SQLITE_OK ){ vdbeMergeEngineFree(pMain); |
︙ | ︙ |