Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Merge trunk changes, including fixes for compiler warnings in fts5 code, with this branch. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | fts5-btree-index |
Files: | files | file ages | folders |
SHA1: |
7190d79ba452ceb1af77ce1375278b09 |
User & Date: | dan 2015-07-16 20:24:42.461 |
Context
2015-07-27
| ||
10:46 | Merge latest trunk changes with this branch. (check-in: 5ec933c257 user: dan tags: fts5-btree-index) | |
2015-07-16
| ||
20:24 | Merge trunk changes, including fixes for compiler warnings in fts5 code, with this branch. (check-in: 7190d79ba4 user: dan tags: fts5-btree-index) | |
20:17 | Fix compiler warnings in fts5 code. (check-in: e9bf275cd9 user: dan tags: trunk) | |
2015-07-15
| ||
19:46 | Use a WITHOUT ROWID table to index fts5 btree leaves. This is faster to query and only slightly larger than storing btree nodes within an intkey table. (check-in: 862418e350 user: dan tags: fts5-btree-index) | |
Changes
Changes to ext/fts5/fts5_expr.c.
︙ | ︙ | |||
79 80 81 82 83 84 85 | ** A phrase. One or more terms that must appear in a contiguous sequence ** within a document for it to match. */ struct Fts5ExprPhrase { Fts5ExprNode *pNode; /* FTS5_STRING node this phrase is part of */ Fts5Buffer poslist; /* Current position list */ int nTerm; /* Number of entries in aTerm[] */ | | | | 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 | ** A phrase. One or more terms that must appear in a contiguous sequence ** within a document for it to match. */ struct Fts5ExprPhrase { Fts5ExprNode *pNode; /* FTS5_STRING node this phrase is part of */ Fts5Buffer poslist; /* Current position list */ int nTerm; /* Number of entries in aTerm[] */ Fts5ExprTerm aTerm[1]; /* Terms that make up this phrase */ }; /* ** If a NEAR() clump may only match a specific set of columns, then ** Fts5ExprNearset.pColset points to an object of the following type. ** Each entry in the aiCol[] array */ struct Fts5ExprColset { int nCol; int aiCol[1]; }; /* ** One or more phrases that must appear within a certain token distance of ** each other within each matching document. */ struct Fts5ExprNearset { int nNear; /* NEAR parameter */ Fts5ExprColset *pColset; /* Columns to search (NULL -> all columns) */ int nPhrase; /* Number of entries in aPhrase[] array */ Fts5ExprPhrase *apPhrase[1]; /* Array of phrase pointers */ }; /* ** Parse context. */ struct Fts5Parse { |
︙ | ︙ |
Changes to ext/fts5/fts5_hash.c.
︙ | ︙ | |||
62 63 64 65 66 67 68 | int iSzPoslist; /* Offset of space for 4-byte poslist size */ int nData; /* Total bytes of data (incl. structure) */ u8 bDel; /* Set delete-flag @ iSzPoslist */ int iCol; /* Column of last value written */ int iPos; /* Position of last value written */ i64 iRowid; /* Rowid of last value written */ | | > > > > > > > | 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 | int iSzPoslist; /* Offset of space for 4-byte poslist size */ int nData; /* Total bytes of data (incl. structure) */ u8 bDel; /* Set delete-flag @ iSzPoslist */ int iCol; /* Column of last value written */ int iPos; /* Position of last value written */ i64 iRowid; /* Rowid of last value written */ char zKey[8]; /* Nul-terminated entry key */ }; /* ** Size of Fts5HashEntry without the zKey[] array. */ #define FTS5_HASHENTRYSIZE (sizeof(Fts5HashEntry)-8) /* ** Allocate a new hash table. */ int sqlite3Fts5HashNew(Fts5Hash **ppNew, int *pnByte){ int rc = SQLITE_OK; Fts5Hash *pNew; |
︙ | ︙ | |||
216 217 218 219 220 221 222 | ){ break; } } /* If an existing hash entry cannot be found, create a new one. */ if( p==0 ){ | | | | | 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 | ){ break; } } /* If an existing hash entry cannot be found, create a new one. */ if( p==0 ){ int nByte = FTS5_HASHENTRYSIZE + (nToken+1) + 1 + 64; if( nByte<128 ) nByte = 128; if( (pHash->nEntry*2)>=pHash->nSlot ){ int rc = fts5HashResize(pHash); if( rc!=SQLITE_OK ) return rc; iHash = fts5HashKey2(pHash->nSlot, bByte, pToken, nToken); } p = (Fts5HashEntry*)sqlite3_malloc(nByte); if( !p ) return SQLITE_NOMEM; memset(p, 0, FTS5_HASHENTRYSIZE); p->nAlloc = nByte; p->zKey[0] = bByte; memcpy(&p->zKey[1], pToken, nToken); assert( iHash==fts5HashKey(pHash->nSlot, p->zKey, nToken+1) ); p->zKey[nToken+1] = '\0'; p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE; p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid); p->iSzPoslist = p->nData; p->nData += 1; p->iRowid = iRowid; p->pHashNext = pHash->aSlot[iHash]; pHash->aSlot[iHash] = p; pHash->nEntry++; |
︙ | ︙ | |||
413 414 415 416 417 418 419 | for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ if( memcmp(p->zKey, pTerm, nTerm)==0 && p->zKey[nTerm]==0 ) break; } if( p ){ fts5HashAddPoslistSize(p); *ppDoclist = (const u8*)&p->zKey[nTerm+1]; | | | 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 | for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ if( memcmp(p->zKey, pTerm, nTerm)==0 && p->zKey[nTerm]==0 ) break; } if( p ){ fts5HashAddPoslistSize(p); *ppDoclist = (const u8*)&p->zKey[nTerm+1]; *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1); }else{ *ppDoclist = 0; *pnDoclist = 0; } return SQLITE_OK; } |
︙ | ︙ | |||
450 451 452 453 454 455 456 | ){ Fts5HashEntry *p; if( (p = pHash->pScan) ){ int nTerm = strlen(p->zKey); fts5HashAddPoslistSize(p); *pzTerm = p->zKey; *ppDoclist = (const u8*)&p->zKey[nTerm+1]; | | | 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 | ){ Fts5HashEntry *p; if( (p = pHash->pScan) ){ int nTerm = strlen(p->zKey); fts5HashAddPoslistSize(p); *pzTerm = p->zKey; *ppDoclist = (const u8*)&p->zKey[nTerm+1]; *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1); }else{ *pzTerm = 0; *ppDoclist = 0; *pnDoclist = 0; } } |
Changes to ext/fts5/fts5_index.c.
︙ | ︙ | |||
366 367 368 369 370 371 372 | Fts5StructureSegment *aSeg; /* Array of segments. aSeg[0] is oldest. */ }; struct Fts5Structure { int nRef; /* Object reference count */ u64 nWriteCounter; /* Total leaves written to level 0 */ int nSegment; /* Total segments in this structure */ int nLevel; /* Number of levels in this index */ | | | 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 | Fts5StructureSegment *aSeg; /* Array of segments. aSeg[0] is oldest. */ }; struct Fts5Structure { int nRef; /* Object reference count */ u64 nWriteCounter; /* Total leaves written to level 0 */ int nSegment; /* Total segments in this structure */ int nLevel; /* Number of levels in this index */ Fts5StructureLevel aLevel[1]; /* Array of nLevel level objects */ }; /* ** An object of type Fts5SegWriter is used to write to segments. */ struct Fts5PageWriter { int pgno; /* Page number for this page */ |
︙ | ︙ | |||
917 918 919 920 921 922 923 | /* Read the total number of levels and segments from the start of the ** structure record. */ i += fts5GetVarint32(&pData[i], nLevel); i += fts5GetVarint32(&pData[i], nSegment); nByte = ( sizeof(Fts5Structure) + /* Main structure */ | | | 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 | /* Read the total number of levels and segments from the start of the ** structure record. */ i += fts5GetVarint32(&pData[i], nLevel); i += fts5GetVarint32(&pData[i], nSegment); nByte = ( sizeof(Fts5Structure) + /* Main structure */ sizeof(Fts5StructureLevel) * (nLevel-1) /* aLevel[] array */ ); pRet = (Fts5Structure*)sqlite3Fts5MallocZero(&rc, nByte); if( pRet ){ pRet->nRef = 1; pRet->nLevel = nLevel; pRet->nSegment = nSegment; |
︙ | ︙ | |||
3068 3069 3070 3071 3072 3073 3074 | ** any currently existing segment. If a free segment id cannot be found, ** SQLITE_FULL is returned. ** ** If an error has already occurred, this function is a no-op. 0 is ** returned in this case. */ static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){ | | | < | | 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 | ** any currently existing segment. If a free segment id cannot be found, ** SQLITE_FULL is returned. ** ** If an error has already occurred, this function is a no-op. 0 is ** returned in this case. */ static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){ int iSegid = 0; if( p->rc==SQLITE_OK ){ if( pStruct->nSegment>=FTS5_MAX_SEGMENT ){ p->rc = SQLITE_FULL; }else{ while( iSegid==0 ){ int iLvl, iSeg; sqlite3_randomness(sizeof(u32), (void*)&iSegid); iSegid = iSegid & ((1 << FTS5_DATA_ID_B)-1); for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){ for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){ if( iSegid==pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ){ iSegid = 0; } } } } } } return iSegid; } /* ** Discard all data currently cached in the hash-tables. */ static void fts5IndexDiscardData(Fts5Index *p){ assert( p->pHash || p->nPendingData==0 ); |
︙ | ︙ |
Changes to ext/fts5/fts5_main.c.
︙ | ︙ | |||
141 142 143 144 145 146 147 | ** byte of the position list for the corresponding phrase. */ struct Fts5Sorter { sqlite3_stmt *pStmt; i64 iRowid; /* Current rowid */ const u8 *aPoslist; /* Position lists for current row */ int nIdx; /* Number of entries in aIdx[] */ | | | 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 | ** byte of the position list for the corresponding phrase. */ struct Fts5Sorter { sqlite3_stmt *pStmt; i64 iRowid; /* Current rowid */ const u8 *aPoslist; /* Position lists for current row */ int nIdx; /* Number of entries in aIdx[] */ int aIdx[1]; /* Offsets into aPoslist for current row */ }; /* ** Virtual-table cursor object. ** ** iSpecial: |
︙ | ︙ | |||
804 805 806 807 808 809 810 | int nByte; int rc = SQLITE_OK; char *zSql; const char *zRank = pCsr->zRank; const char *zRankArgs = pCsr->zRankArgs; nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); | | | 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 | int nByte; int rc = SQLITE_OK; char *zSql; const char *zRank = pCsr->zRank; const char *zRankArgs = pCsr->zRankArgs; nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); nByte = sizeof(Fts5Sorter) + sizeof(int) * (nPhrase-1); pSorter = (Fts5Sorter*)sqlite3_malloc(nByte); if( pSorter==0 ) return SQLITE_NOMEM; memset(pSorter, 0, nByte); pSorter->nIdx = nPhrase; /* TODO: It would be better to have some system for reusing statement ** handles here, rather than preparing a new one for each query. But that |
︙ | ︙ |
Changes to src/pcache1.c.
︙ | ︙ | |||
187 188 189 190 191 192 193 194 195 196 197 198 199 200 | /* Variables related to SQLITE_CONFIG_PAGECACHE settings. The ** szSlot, nSlot, pStart, pEnd, nReserve, and isInit values are all ** fixed at sqlite3_initialize() time and do not require mutex protection. ** The nFreeSlot and pFree values do require mutex protection. */ int isInit; /* True if initialized */ int separateCache; /* Use a new PGroup for each PCache */ int szSlot; /* Size of each free slot */ int nSlot; /* The number of pcache slots */ int nReserve; /* Try to keep nFreeSlot above this */ void *pStart, *pEnd; /* Bounds of global page cache memory */ /* Above requires no mutex. Use mutex below for variable that follow. */ sqlite3_mutex *mutex; /* Mutex for accessing the following: */ PgFreeslot *pFree; /* Free page blocks */ | > | 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 | /* Variables related to SQLITE_CONFIG_PAGECACHE settings. The ** szSlot, nSlot, pStart, pEnd, nReserve, and isInit values are all ** fixed at sqlite3_initialize() time and do not require mutex protection. ** The nFreeSlot and pFree values do require mutex protection. */ int isInit; /* True if initialized */ int separateCache; /* Use a new PGroup for each PCache */ int nInitPage; /* Initial bulk allocation size */ int szSlot; /* Size of each free slot */ int nSlot; /* The number of pcache slots */ int nReserve; /* Try to keep nFreeSlot above this */ void *pStart, *pEnd; /* Bounds of global page cache memory */ /* Above requires no mutex. Use mutex below for variable that follow. */ sqlite3_mutex *mutex; /* Mutex for accessing the following: */ PgFreeslot *pFree; /* Free page blocks */ |
︙ | ︙ | |||
254 255 256 257 258 259 260 261 262 263 264 265 266 267 | p->pNext = pcache1.pFree; pcache1.pFree = p; pBuf = (void*)&((char*)pBuf)[sz]; } pcache1.pEnd = pBuf; } } /* ** Malloc function used within this file to allocate space from the buffer ** configured using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no ** such buffer exists or there is no space left in it, this function falls ** back to sqlite3Malloc(). ** | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 | p->pNext = pcache1.pFree; pcache1.pFree = p; pBuf = (void*)&((char*)pBuf)[sz]; } pcache1.pEnd = pBuf; } } /* ** Try to initialize the pCache->pFree and pCache->pBulk fields. Return ** true if pCache->pFree ends up containing one or more free pages. */ static int pcache1InitBulk(PCache1 *pCache){ i64 szBulk; char *zBulk; if( pcache1.nInitPage==0 ) return 0; /* Do not bother with a bulk allocation if the cache size very small */ if( pCache->nMax<3 ) return 0; sqlite3BeginBenignMalloc(); if( pcache1.nInitPage>0 ){ szBulk = pCache->szAlloc * (i64)pcache1.nInitPage; }else{ szBulk = -1024 * (i64)pcache1.nInitPage; } if( szBulk > pCache->szAlloc*(i64)pCache->nMax ){ szBulk = pCache->szAlloc*pCache->nMax; } zBulk = pCache->pBulk = sqlite3Malloc( szBulk ); sqlite3EndBenignMalloc(); if( zBulk ){ int nBulk = sqlite3MallocSize(zBulk)/pCache->szAlloc; int i; for(i=0; i<nBulk; i++){ PgHdr1 *pX = (PgHdr1*)&zBulk[pCache->szPage]; pX->page.pBuf = zBulk; pX->page.pExtra = &pX[1]; pX->isBulkLocal = 1; pX->pNext = pCache->pFree; pCache->pFree = pX; zBulk += pCache->szAlloc; } } return pCache->pFree!=0; } /* ** Malloc function used within this file to allocate space from the buffer ** configured using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no ** such buffer exists or there is no space left in it, this function falls ** back to sqlite3Malloc(). ** |
︙ | ︙ | |||
355 356 357 358 359 360 361 | ** Allocate a new page object initially associated with cache pCache. */ static PgHdr1 *pcache1AllocPage(PCache1 *pCache){ PgHdr1 *p = 0; void *pPg; assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); | | | 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 | ** Allocate a new page object initially associated with cache pCache. */ static PgHdr1 *pcache1AllocPage(PCache1 *pCache){ PgHdr1 *p = 0; void *pPg; assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); if( pCache->pFree || (pCache->nPage==0 && pcache1InitBulk(pCache)) ){ p = pCache->pFree; pCache->pFree = p->pNext; p->pNext = 0; }else{ #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT /* The group mutex must be released before pcache1Alloc() is called. This ** is because it might call sqlite3_release_memory(), which assumes that |
︙ | ︙ | |||
559 560 561 562 563 564 565 | if( freeFlag ) pcache1FreePage(pPage); } /* ** If there are currently more than nMaxPage pages allocated, try ** to recycle pages to reduce the number allocated to nMaxPage. */ | | > > > > > | 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 | if( freeFlag ) pcache1FreePage(pPage); } /* ** If there are currently more than nMaxPage pages allocated, try ** to recycle pages to reduce the number allocated to nMaxPage. */ static void pcache1EnforceMaxPage(PCache1 *pCache){ PGroup *pGroup = pCache->pGroup; assert( sqlite3_mutex_held(pGroup->mutex) ); while( pGroup->nCurrentPage>pGroup->nMaxPage && pGroup->pLruTail ){ PgHdr1 *p = pGroup->pLruTail; assert( p->pCache->pGroup==pGroup ); assert( p->isPinned==0 ); pcache1PinPage(p); pcache1RemoveFromHash(p, 1); } if( pCache->nPage==0 && pCache->pBulk ){ sqlite3_free(pCache->pBulk); pCache->pBulk = pCache->pFree = 0; } } /* ** Discard all pages from cache pCache with a page number (key value) ** greater than or equal to iLimit. Any pinned pages that meet this ** criteria are unpinned before they are discarded. |
︙ | ︙ | |||
643 644 645 646 647 648 649 650 651 652 653 654 655 656 | #if SQLITE_THREADSAFE if( sqlite3GlobalConfig.bCoreMutex ){ pcache1.grp.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU); pcache1.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_PMEM); } #endif pcache1.grp.mxPinned = 10; pcache1.isInit = 1; return SQLITE_OK; } /* ** Implementation of the sqlite3_pcache.xShutdown method. | > > > > > > > > | 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 | #if SQLITE_THREADSAFE if( sqlite3GlobalConfig.bCoreMutex ){ pcache1.grp.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU); pcache1.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_PMEM); } #endif if( pcache1.separateCache && sqlite3GlobalConfig.nPage!=0 && sqlite3GlobalConfig.pPage==0 ){ pcache1.nInitPage = sqlite3GlobalConfig.nPage; }else{ pcache1.nInitPage = 0; } pcache1.grp.mxPinned = 10; pcache1.isInit = 1; return SQLITE_OK; } /* ** Implementation of the sqlite3_pcache.xShutdown method. |
︙ | ︙ | |||
697 698 699 700 701 702 703 | pcache1ResizeHash(pCache); if( bPurgeable ){ pCache->nMin = 10; pGroup->nMinPage += pCache->nMin; pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; } pcache1LeaveMutex(pGroup); | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | 748 749 750 751 752 753 754 755 756 757 758 759 760 761 | pcache1ResizeHash(pCache); if( bPurgeable ){ pCache->nMin = 10; pGroup->nMinPage += pCache->nMin; pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; } pcache1LeaveMutex(pGroup); if( pCache->nHash==0 ){ pcache1Destroy((sqlite3_pcache*)pCache); pCache = 0; } } return (sqlite3_pcache *)pCache; } |
︙ | ︙ | |||
749 750 751 752 753 754 755 | if( pCache->bPurgeable ){ PGroup *pGroup = pCache->pGroup; pcache1EnterMutex(pGroup); pGroup->nMaxPage += (nMax - pCache->nMax); pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; pCache->nMax = nMax; pCache->n90pct = pCache->nMax*9/10; | | | | 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 | if( pCache->bPurgeable ){ PGroup *pGroup = pCache->pGroup; pcache1EnterMutex(pGroup); pGroup->nMaxPage += (nMax - pCache->nMax); pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; pCache->nMax = nMax; pCache->n90pct = pCache->nMax*9/10; pcache1EnforceMaxPage(pCache); pcache1LeaveMutex(pGroup); } } /* ** Implementation of the sqlite3_pcache.xShrink method. ** ** Free up as much memory as possible. */ static void pcache1Shrink(sqlite3_pcache *p){ PCache1 *pCache = (PCache1*)p; if( pCache->bPurgeable ){ PGroup *pGroup = pCache->pGroup; int savedMaxPage; pcache1EnterMutex(pGroup); savedMaxPage = pGroup->nMaxPage; pGroup->nMaxPage = 0; pcache1EnforceMaxPage(pCache); pGroup->nMaxPage = savedMaxPage; pcache1LeaveMutex(pGroup); } } /* ** Implementation of the sqlite3_pcache.xPagecount method. |
︙ | ︙ | |||
1104 1105 1106 1107 1108 1109 1110 | pcache1EnterMutex(pGroup); pcache1TruncateUnsafe(pCache, 0); assert( pGroup->nMaxPage >= pCache->nMax ); pGroup->nMaxPage -= pCache->nMax; assert( pGroup->nMinPage >= pCache->nMin ); pGroup->nMinPage -= pCache->nMin; pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; | | | 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 | pcache1EnterMutex(pGroup); pcache1TruncateUnsafe(pCache, 0); assert( pGroup->nMaxPage >= pCache->nMax ); pGroup->nMaxPage -= pCache->nMax; assert( pGroup->nMinPage >= pCache->nMin ); pGroup->nMinPage -= pCache->nMin; pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; pcache1EnforceMaxPage(pCache); pcache1LeaveMutex(pGroup); sqlite3_free(pCache->pBulk); sqlite3_free(pCache->apHash); sqlite3_free(pCache); } /* |
︙ | ︙ |
Changes to src/pragma.c.
︙ | ︙ | |||
717 718 719 720 721 722 723 724 725 726 727 728 729 730 | ** number of pages in the cache. If N is negative, then the ** number of pages is adjusted so that the cache uses -N kibibytes ** of memory. */ case PragTyp_CACHE_SIZE: { assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( !zRight ){ returnSingleInt(pParse, "cache_size", pDb->pSchema->cache_size); }else{ int size = sqlite3Atoi(zRight); pDb->pSchema->cache_size = size; sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); } break; | > | 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 | ** number of pages in the cache. If N is negative, then the ** number of pages is adjusted so that the cache uses -N kibibytes ** of memory. */ case PragTyp_CACHE_SIZE: { assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( !zRight ){ if( sqlite3ReadSchema(pParse) ) goto pragma_out; returnSingleInt(pParse, "cache_size", pDb->pSchema->cache_size); }else{ int size = sqlite3Atoi(zRight); pDb->pSchema->cache_size = size; sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); } break; |
︙ | ︙ |
Changes to src/pragma.h.
︙ | ︙ | |||
82 83 84 85 86 87 88 | { /* zName: */ "busy_timeout", /* ePragTyp: */ PragTyp_BUSY_TIMEOUT, /* ePragFlag: */ 0, /* iArg: */ 0 }, #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) { /* zName: */ "cache_size", /* ePragTyp: */ PragTyp_CACHE_SIZE, | | | 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 | { /* zName: */ "busy_timeout", /* ePragTyp: */ PragTyp_BUSY_TIMEOUT, /* ePragFlag: */ 0, /* iArg: */ 0 }, #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) { /* zName: */ "cache_size", /* ePragTyp: */ PragTyp_CACHE_SIZE, /* ePragFlag: */ 0, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) { /* zName: */ "cache_spill", /* ePragTyp: */ PragTyp_FLAG, /* ePragFlag: */ 0, /* iArg: */ SQLITE_CacheSpill }, |
︙ | ︙ |
Changes to src/select.c.
︙ | ︙ | |||
1078 1079 1080 1081 1082 1083 1084 | pInfo->aColl[i-iStart] = pColl; pInfo->aSortOrder[i-iStart] = pItem->sortOrder; } } return pInfo; } | < < | 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 | pInfo->aColl[i-iStart] = pColl; pInfo->aSortOrder[i-iStart] = pItem->sortOrder; } } return pInfo; } /* ** Name of the connection operator, used for error messages. */ static const char *selectOpName(int id){ char *z; switch( id ){ case TK_ALL: z = "UNION ALL"; break; case TK_INTERSECT: z = "INTERSECT"; break; case TK_EXCEPT: z = "EXCEPT"; break; default: z = "UNION"; break; } return z; } #ifndef SQLITE_OMIT_EXPLAIN /* ** Unless an "EXPLAIN QUERY PLAN" command is being processed, this function ** is a no-op. Otherwise, it adds a single row of output to the EQP result, ** where the caption is of the form: ** |
︙ | ︙ | |||
2095 2096 2097 2098 2099 2100 2101 | /* Forward references */ static int multiSelectOrderBy( Parse *pParse, /* Parsing context */ Select *p, /* The right-most of SELECTs to be coded */ SelectDest *pDest /* What to do with query results */ ); | < < < < < < < < < < < < < | 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 | /* Forward references */ static int multiSelectOrderBy( Parse *pParse, /* Parsing context */ Select *p, /* The right-most of SELECTs to be coded */ SelectDest *pDest /* What to do with query results */ ); /* ** Handle the special case of a compound-select that originates from a ** VALUES clause. By handling this as a special case, we avoid deep ** recursion, and thus do not need to enforce the SQLITE_LIMIT_COMPOUND_SELECT ** on a VALUES clause. ** ** Because the Select object originates from a VALUES clause: |
︙ | ︙ | |||
2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 | multi_select_end: pDest->iSdst = dest.iSdst; pDest->nSdst = dest.nSdst; sqlite3SelectDelete(db, pDelete); return rc; } #endif /* SQLITE_OMIT_COMPOUND_SELECT */ /* ** Code an output subroutine for a coroutine implementation of a ** SELECT statment. ** ** The data to be output is contained in pIn->iSdst. There are ** pIn->nSdst columns to be output. pDest is where the output should | > > > > > > > > > > > > > | 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 | multi_select_end: pDest->iSdst = dest.iSdst; pDest->nSdst = dest.nSdst; sqlite3SelectDelete(db, pDelete); return rc; } #endif /* SQLITE_OMIT_COMPOUND_SELECT */ /* ** Error message for when two or more terms of a compound select have different ** size result sets. */ void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p){ if( p->selFlags & SF_Values ){ sqlite3ErrorMsg(pParse, "all VALUES must have the same number of terms"); }else{ sqlite3ErrorMsg(pParse, "SELECTs to the left and right of %s" " do not have the same number of result columns", selectOpName(p->op)); } } /* ** Code an output subroutine for a coroutine implementation of a ** SELECT statment. ** ** The data to be output is contained in pIn->iSdst. There are ** pIn->nSdst columns to be output. pDest is where the output should |
︙ | ︙ |
Changes to src/sqliteInt.h.
︙ | ︙ | |||
750 751 752 753 754 755 756 | # define SQLITE_MAX_MMAP_SIZE 0 # endif #endif #ifndef SQLITE_MAX_MMAP_SIZE # if defined(__linux__) \ || defined(_WIN32) \ || (defined(__APPLE__) && defined(__MACH__)) \ | | > > | 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 | # define SQLITE_MAX_MMAP_SIZE 0 # endif #endif #ifndef SQLITE_MAX_MMAP_SIZE # if defined(__linux__) \ || defined(_WIN32) \ || (defined(__APPLE__) && defined(__MACH__)) \ || defined(__sun) \ || defined(__FreeBSD__) \ || defined(__DragonFly__) # define SQLITE_MAX_MMAP_SIZE 0x7fff0000 /* 2147418112 */ # else # define SQLITE_MAX_MMAP_SIZE 0 # endif # define SQLITE_MAX_MMAP_SIZE_xc 1 /* exclude from ctime.c */ #endif |
︙ | ︙ |
Changes to test/malloc5.test.
︙ | ︙ | |||
37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 | ifcapable !memorymanage { finish_test return } sqlite3_soft_heap_limit 0 sqlite3 db test.db do_test malloc5-1.1 { # Simplest possible test. Call sqlite3_release_memory when there is exactly # one unused page in a single pager cache. The page cannot be freed, as # it is dirty. So sqlite3_release_memory() returns 0. # execsql { PRAGMA auto_vacuum=OFF; BEGIN; CREATE TABLE abc(a, b, c); } sqlite3_release_memory } {0} do_test malloc5-1.2 { # Test that the transaction started in the above test is still active. # The lock on the database file should not have been upgraded (this was # not the case before version 3.6.2). # sqlite3 db2 test.db | > | < | | 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 | ifcapable !memorymanage { finish_test return } sqlite3_soft_heap_limit 0 sqlite3 db test.db db eval {PRAGMA cache_size=1} do_test malloc5-1.1 { # Simplest possible test. Call sqlite3_release_memory when there is exactly # one unused page in a single pager cache. The page cannot be freed, as # it is dirty. So sqlite3_release_memory() returns 0. # execsql { PRAGMA auto_vacuum=OFF; BEGIN; CREATE TABLE abc(a, b, c); } sqlite3_release_memory } {0} do_test malloc5-1.2 { # Test that the transaction started in the above test is still active. # The lock on the database file should not have been upgraded (this was # not the case before version 3.6.2). # sqlite3 db2 test.db execsql {PRAGMA cache_size=2; SELECT * FROM sqlite_master } db2 } {} do_test malloc5-1.3 { # Call [sqlite3_release_memory] when there is exactly one unused page # in the cache belonging to db2. # set ::pgalloc [sqlite3_release_memory] } {0} # The sizes of memory allocations from system malloc() might vary, # depending on the memory allocator algorithms used. The following # routine is designed to support answers that fall within a range # of values while also supplying easy-to-understand "expected" values # when errors occur. # |
︙ | ︙ | |||
227 228 229 230 231 232 233 234 235 236 237 238 239 240 | sqlite3_memory_highwater 1 execsql {SELECT * FROM abc} set nMaxBytes [sqlite3_memory_highwater 1] puts -nonewline " (Highwater mark: $nMaxBytes) " expr $nMaxBytes > 1000000 } {1} do_test malloc5-4.2 { db cache flush sqlite3_release_memory sqlite3_soft_heap_limit 100000 sqlite3_memory_highwater 1 execsql {SELECT * FROM abc} set nMaxBytes [sqlite3_memory_highwater 1] puts -nonewline " (Highwater mark: $nMaxBytes) " | > | 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 | sqlite3_memory_highwater 1 execsql {SELECT * FROM abc} set nMaxBytes [sqlite3_memory_highwater 1] puts -nonewline " (Highwater mark: $nMaxBytes) " expr $nMaxBytes > 1000000 } {1} do_test malloc5-4.2 { db eval {PRAGMA cache_size=1} db cache flush sqlite3_release_memory sqlite3_soft_heap_limit 100000 sqlite3_memory_highwater 1 execsql {SELECT * FROM abc} set nMaxBytes [sqlite3_memory_highwater 1] puts -nonewline " (Highwater mark: $nMaxBytes) " |
︙ | ︙ | |||
298 299 300 301 302 303 304 | # This block of test-cases (malloc5-6.1.*) prepares two database files # for the subsequent tests. do_test malloc5-6.1.1 { sqlite3 db test.db execsql { PRAGMA page_size=1024; | | | 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 | # This block of test-cases (malloc5-6.1.*) prepares two database files # for the subsequent tests. do_test malloc5-6.1.1 { sqlite3 db test.db execsql { PRAGMA page_size=1024; PRAGMA default_cache_size=2; } execsql { PRAGMA temp_store = memory; BEGIN; CREATE TABLE abc(a PRIMARY KEY, b, c); INSERT INTO abc VALUES(randstr(50,50), randstr(75,75), randstr(100,100)); INSERT INTO abc |
︙ | ︙ | |||
321 322 323 324 325 326 327 328 329 330 331 332 | SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc; INSERT INTO abc SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc; COMMIT; } forcecopy test.db test2.db sqlite3 db2 test2.db list \ [expr ([file size test.db]/1024)>20] [expr ([file size test2.db]/1024)>20] } {1 1} do_test malloc5-6.1.2 { list [execsql {PRAGMA cache_size}] [execsql {PRAGMA cache_size} db2] | > | | | | | | | | | | | 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 | SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc; INSERT INTO abc SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc; COMMIT; } forcecopy test.db test2.db sqlite3 db2 test2.db db2 eval {PRAGMA cache_size=2} list \ [expr ([file size test.db]/1024)>20] [expr ([file size test2.db]/1024)>20] } {1 1} do_test malloc5-6.1.2 { list [execsql {PRAGMA cache_size}] [execsql {PRAGMA cache_size} db2] } {2 2} do_test malloc5-6.2.1 { execsql {SELECT * FROM abc} db2 execsql {SELECT * FROM abc} db expr [nPage db] + [nPage db2] } {4} do_test malloc5-6.2.2 { # If we now try to reclaim some memory, it should come from the db2 cache. sqlite3_release_memory 3000 expr [nPage db] + [nPage db2] } {4} do_test malloc5-6.2.3 { # Access the db2 cache again, so that all the db2 pages have been used # more recently than all the db pages. Then try to reclaim 3000 bytes. # This time, 3 pages should be pulled from the db cache. execsql { SELECT * FROM abc } db2 sqlite3_release_memory 3000 expr [nPage db] + [nPage db2] } {4} do_test malloc5-6.3.1 { # Now open a transaction and update 2 pages in the db2 cache. Then # do a SELECT on the db cache so that all the db pages are more recently # used than the db2 pages. When we try to free memory, SQLite should # free the non-dirty db2 pages, then the db pages, then finally use # sync() to free up the dirty db2 pages. The only page that cannot be # freed is page1 of db2. Because there is an open transaction, the # btree layer holds a reference to page 1 in the db2 cache. execsql { BEGIN; UPDATE abc SET c = randstr(100,100) WHERE rowid = 1 OR rowid = (SELECT max(rowid) FROM abc); } db2 execsql { SELECT * FROM abc } db expr [nPage db] + [nPage db2] } {4} do_test malloc5-6.3.2 { # Try to release 7700 bytes. This should release all the # non-dirty pages held by db2. sqlite3_release_memory [expr 7*1132] list [nPage db] [nPage db2] } {1 3} do_test malloc5-6.3.3 { # Try to release another 1000 bytes. This should come fromt the db # cache, since all three pages held by db2 are either in-use or diry. sqlite3_release_memory 1000 list [nPage db] [nPage db2] } {1 3} do_test malloc5-6.3.4 { # Now release 9900 more (about 9 pages worth). This should expunge # the rest of the db cache. But the db2 cache remains intact, because # SQLite tries to avoid calling sync(). if {$::tcl_platform(wordSize)==8} { sqlite3_release_memory 10500 } else { sqlite3_release_memory 9900 } list [nPage db] [nPage db2] } {1 3} do_test malloc5-6.3.5 { # But if we are really insistent, SQLite will consent to call sync() # if there is no other option. UPDATE: As of 3.6.2, SQLite will not # call sync() in this scenario. So no further memory can be reclaimed. sqlite3_release_memory 1000 list [nPage db] [nPage db2] } {1 3} do_test malloc5-6.3.6 { # The referenced page (page 1 of the db2 cache) will not be freed no # matter how much memory we ask for: sqlite3_release_memory 31459 list [nPage db] [nPage db2] } {1 3} db2 close sqlite3_soft_heap_limit $::soft_limit finish_test catch {db close} |
Changes to test/pcache.test.
︙ | ︙ | |||
69 70 71 72 73 74 75 | CREATE TABLE t9(a, b, c); } pcache_stats } {current 10 max 12 min 10 recyclable 0} do_test pcache-1.5 { sqlite3 db2 test.db | | | 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 | CREATE TABLE t9(a, b, c); } pcache_stats } {current 10 max 12 min 10 recyclable 0} do_test pcache-1.5 { sqlite3 db2 test.db execsql "PRAGMA cache_size; PRAGMA cache_size=10" db2 pcache_stats } {current 11 max 22 min 20 recyclable 1} do_test pcache-1.6.1 { execsql { BEGIN; SELECT * FROM sqlite_master; |
︙ | ︙ |
Changes to test/pcache2.test.
︙ | ︙ | |||
32 33 34 35 36 37 38 | } {0 0 0} # Open up two database connections to separate files. # do_test pcache2-1.2 { forcedelete test.db test.db-journal sqlite3 db test.db | | | | 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 | } {0 0 0} # Open up two database connections to separate files. # do_test pcache2-1.2 { forcedelete test.db test.db-journal sqlite3 db test.db db eval {PRAGMA cache_size=10; SELECT 1 FROM sqlite_master;} lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 1 } {2} do_test pcache2-1.3 { forcedelete test2.db test2.db-journal sqlite3 db2 test2.db db2 eval {PRAGMA cache_size=50; SELECT 1 FROM sqlite_master;} lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 1 } {4} # Make lots of changes on the first connection. Verify that the # page cache usage does not grow to consume the page space set aside # for the second connection. |
︙ | ︙ |
Changes to tool/mkpragmatab.tcl.
︙ | ︙ | |||
166 167 168 169 170 171 172 | FLAG: NeedSchema IF: !defined(SQLITE_OMIT_PAGER_PRAGMAS) NAME: journal_size_limit IF: !defined(SQLITE_OMIT_PAGER_PRAGMAS) NAME: cache_size | < | 166 167 168 169 170 171 172 173 174 175 176 177 178 179 | FLAG: NeedSchema IF: !defined(SQLITE_OMIT_PAGER_PRAGMAS) NAME: journal_size_limit IF: !defined(SQLITE_OMIT_PAGER_PRAGMAS) NAME: cache_size IF: !defined(SQLITE_OMIT_PAGER_PRAGMAS) NAME: mmap_size IF: !defined(SQLITE_OMIT_PAGER_PRAGMAS) NAME: auto_vacuum FLAG: NeedSchema |
︙ | ︙ |
Changes to tool/mksqlite3c-noext.tcl.
︙ | ︙ | |||
176 177 178 179 180 181 182 183 184 185 186 187 188 189 | if {$hdr!="os_common.h" && $hdr!="hwtime.h"} { set available_hdr($hdr) 0 } section_comment "Include $hdr in the middle of $tail" copy_file tsrc/$hdr section_comment "Continuing where we left off in $tail" if {$linemacros} {puts $out "#line [expr {$ln+1}] \"$filename\""} } } elseif {![info exists seen_hdr($hdr)]} { if {![regexp {/\*\s+amalgamator:\s+dontcache\s+\*/} $line]} { set seen_hdr($hdr) 1 } puts $out $line } elseif {[regexp {/\*\s+amalgamator:\s+keep\s+\*/} $line]} { | > > > > | 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 | if {$hdr!="os_common.h" && $hdr!="hwtime.h"} { set available_hdr($hdr) 0 } section_comment "Include $hdr in the middle of $tail" copy_file tsrc/$hdr section_comment "Continuing where we left off in $tail" if {$linemacros} {puts $out "#line [expr {$ln+1}] \"$filename\""} } else { # Comment out the entire line, replacing any nested comment # begin/end markers with the harmless substring "**". puts $out "/* [string map [list /* ** */ **] $line] */" } } elseif {![info exists seen_hdr($hdr)]} { if {![regexp {/\*\s+amalgamator:\s+dontcache\s+\*/} $line]} { set seen_hdr($hdr) 1 } puts $out $line } elseif {[regexp {/\*\s+amalgamator:\s+keep\s+\*/} $line]} { |
︙ | ︙ |
Changes to tool/mksqlite3c.tcl.
︙ | ︙ | |||
184 185 186 187 188 189 190 191 192 193 194 195 196 197 | if {$hdr!="os_common.h" && $hdr!="hwtime.h"} { set available_hdr($hdr) 0 } section_comment "Include $hdr in the middle of $tail" copy_file tsrc/$hdr section_comment "Continuing where we left off in $tail" if {$linemacros} {puts $out "#line [expr {$ln+1}] \"$filename\""} } } elseif {![info exists seen_hdr($hdr)]} { if {![regexp {/\*\s+amalgamator:\s+dontcache\s+\*/} $line]} { set seen_hdr($hdr) 1 } puts $out $line } elseif {[regexp {/\*\s+amalgamator:\s+keep\s+\*/} $line]} { | > > > > | 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 | if {$hdr!="os_common.h" && $hdr!="hwtime.h"} { set available_hdr($hdr) 0 } section_comment "Include $hdr in the middle of $tail" copy_file tsrc/$hdr section_comment "Continuing where we left off in $tail" if {$linemacros} {puts $out "#line [expr {$ln+1}] \"$filename\""} } else { # Comment out the entire line, replacing any nested comment # begin/end markers with the harmless substring "**". puts $out "/* [string map [list /* ** */ **] $line] */" } } elseif {![info exists seen_hdr($hdr)]} { if {![regexp {/\*\s+amalgamator:\s+dontcache\s+\*/} $line]} { set seen_hdr($hdr) 1 } puts $out $line } elseif {[regexp {/\*\s+amalgamator:\s+keep\s+\*/} $line]} { |
︙ | ︙ |