Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Merge the Fts5MultiSegIter and Fts5IndexIter objects. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
6ace73541ae48aeb19e0c2d9f6772d03 |
User & Date: | dan 2015-07-07 19:35:11.437 |
Context
2015-07-08
| ||
12:25 | Early detection of file corruption in a debugging routine, before the invalid data can cause problems. (check-in: b27a47fef9 user: drh tags: trunk) | |
2015-07-07
| ||
19:35 | Merge the Fts5MultiSegIter and Fts5IndexIter objects. (check-in: 6ace73541a user: dan tags: trunk) | |
19:07 | Add a test case to verify that "PRAGMA data_version" works as expected when an OTA client writes to the database. (check-in: 6441f2dc9e user: dan tags: trunk) | |
Changes
Changes to ext/fts5/fts5_index.c.
︙ | ︙ | |||
290 291 292 293 294 295 296 | typedef struct Fts5BtreeIter Fts5BtreeIter; typedef struct Fts5BtreeIterLevel Fts5BtreeIterLevel; typedef struct Fts5Data Fts5Data; typedef struct Fts5DlidxIter Fts5DlidxIter; typedef struct Fts5DlidxLvl Fts5DlidxLvl; typedef struct Fts5DlidxWriter Fts5DlidxWriter; | < | 290 291 292 293 294 295 296 297 298 299 300 301 302 303 | typedef struct Fts5BtreeIter Fts5BtreeIter; typedef struct Fts5BtreeIterLevel Fts5BtreeIterLevel; typedef struct Fts5Data Fts5Data; typedef struct Fts5DlidxIter Fts5DlidxIter; typedef struct Fts5DlidxLvl Fts5DlidxLvl; typedef struct Fts5DlidxWriter Fts5DlidxWriter; typedef struct Fts5NodeIter Fts5NodeIter; typedef struct Fts5PageWriter Fts5PageWriter; typedef struct Fts5SegIter Fts5SegIter; typedef struct Fts5DoclistIter Fts5DoclistIter; typedef struct Fts5SegWriter Fts5SegWriter; typedef struct Fts5Structure Fts5Structure; typedef struct Fts5StructureLevel Fts5StructureLevel; |
︙ | ︙ | |||
344 345 346 347 348 349 350 | /* Output variables. aPoslist==0 at EOF */ i64 iRowid; u8 *aPoslist; int nPoslist; }; | < < < < < < < < < < > | 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 | /* Output variables. aPoslist==0 at EOF */ i64 iRowid; u8 *aPoslist; int nPoslist; }; /* ** The contents of the "structure" record for each index are represented ** using an Fts5Structure record in memory. Which uses instances of the ** other Fts5StructureXXX types as components. */ struct Fts5StructureSegment { int iSegid; /* Segment id */ int nHeight; /* Height of segment b-tree */ int pgnoFirst; /* First leaf page number in segment */ int pgnoLast; /* Last leaf page number in segment */ }; struct Fts5StructureLevel { int nMerge; /* Number of segments in incr-merge */ int nSeg; /* Total number of segments on level */ Fts5StructureSegment *aSeg; /* Array of segments. aSeg[0] is oldest. */ }; struct Fts5Structure { int nRef; /* Object reference count */ u64 nWriteCounter; /* Total leaves written to level 0 */ int nSegment; /* Total segments in this structure */ int nLevel; /* Number of levels in this index */ Fts5StructureLevel aLevel[0]; /* Array of nLevel level objects */ }; /* |
︙ | ︙ | |||
506 507 508 509 510 511 512 | int bDel; /* True if the delete flag is set */ }; #define FTS5_SEGITER_ONETERM 0x01 #define FTS5_SEGITER_REVERSE 0x02 | > > > > > | > > > > | 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 | int bDel; /* True if the delete flag is set */ }; #define FTS5_SEGITER_ONETERM 0x01 #define FTS5_SEGITER_REVERSE 0x02 /* ** poslist: ** Used by sqlite3Fts5IterPoslist() when the poslist needs to be buffered. ** There is no way to tell if this is populated or not. */ struct Fts5IndexIter { Fts5Index *pIndex; /* Index that owns this iterator */ Fts5Structure *pStruct; /* Database structure for this iterator */ Fts5Buffer poslist; /* Buffer containing current poslist */ int nSeg; /* Size of aSeg[] array */ int bRev; /* True to iterate in reverse order */ int bSkipEmpty; /* True to skip deleted entries */ int bEof; /* True at EOF */ Fts5CResult *aFirst; /* Current merge state (see above) */ Fts5SegIter aSeg[1]; /* Array of segment iterators */ }; |
︙ | ︙ | |||
855 856 857 858 859 860 861 | } /* ** Release a reference to an Fts5Structure object returned by an earlier ** call to fts5StructureRead() or fts5StructureDecode(). */ static void fts5StructureRelease(Fts5Structure *pStruct){ | | > > > > > | 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 | } /* ** Release a reference to an Fts5Structure object returned by an earlier ** call to fts5StructureRead() or fts5StructureDecode(). */ static void fts5StructureRelease(Fts5Structure *pStruct){ if( pStruct && 0>=(--pStruct->nRef) ){ int i; assert( pStruct->nRef==0 ); for(i=0; i<pStruct->nLevel; i++){ sqlite3_free(pStruct->aLevel[i].aSeg); } sqlite3_free(pStruct); } } static void fts5StructureRef(Fts5Structure *pStruct){ pStruct->nRef++; } /* ** Deserialize and return the structure record currently stored in serialized ** form within buffer pData/nData. ** ** The Fts5Structure.aLevel[] and each Fts5StructureLevel.aSeg[] array ** are over-allocated by one slot. This allows the structure contents |
︙ | ︙ | |||
905 906 907 908 909 910 911 912 913 914 915 916 917 918 | nByte = ( sizeof(Fts5Structure) + /* Main structure */ sizeof(Fts5StructureLevel) * (nLevel) /* aLevel[] array */ ); pRet = (Fts5Structure*)sqlite3Fts5MallocZero(&rc, nByte); if( pRet ){ pRet->nLevel = nLevel; pRet->nSegment = nSegment; i += sqlite3Fts5GetVarint(&pData[i], &pRet->nWriteCounter); for(iLvl=0; rc==SQLITE_OK && iLvl<nLevel; iLvl++){ Fts5StructureLevel *pLvl = &pRet->aLevel[iLvl]; int nTotal; | > | 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 | nByte = ( sizeof(Fts5Structure) + /* Main structure */ sizeof(Fts5StructureLevel) * (nLevel) /* aLevel[] array */ ); pRet = (Fts5Structure*)sqlite3Fts5MallocZero(&rc, nByte); if( pRet ){ pRet->nRef = 1; pRet->nLevel = nLevel; pRet->nSegment = nSegment; i += sqlite3Fts5GetVarint(&pData[i], &pRet->nWriteCounter); for(iLvl=0; rc==SQLITE_OK && iLvl<nLevel; iLvl++){ Fts5StructureLevel *pLvl = &pRet->aLevel[iLvl]; int nTotal; |
︙ | ︙ | |||
1763 1764 1765 1766 1767 1768 1769 | } /* ** Return true if the iterator passed as the second argument currently ** points to a delete marker. A delete marker is an entry with a 0 byte ** position-list. */ | | | 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 | } /* ** Return true if the iterator passed as the second argument currently ** points to a delete marker. A delete marker is an entry with a 0 byte ** position-list. */ static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5IndexIter *pIter){ Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst]; return (p->rc==SQLITE_OK && pSeg->pLeaf && pSeg->nPos==0); } /* ** Advance iterator pIter to the next entry. ** |
︙ | ︙ | |||
2406 2407 2408 2409 2410 2411 2412 | /* ** This function is used as part of the big assert() procedure implemented by ** fts5AssertMultiIterSetup(). It ensures that the result currently stored ** in *pRes is the correct result of comparing the current positions of the ** two iterators. */ static void fts5AssertComparisonResult( | | | 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 | /* ** This function is used as part of the big assert() procedure implemented by ** fts5AssertMultiIterSetup(). It ensures that the result currently stored ** in *pRes is the correct result of comparing the current positions of the ** two iterators. */ static void fts5AssertComparisonResult( Fts5IndexIter *pIter, Fts5SegIter *p1, Fts5SegIter *p2, Fts5CResult *pRes ){ int i1 = p1 - pIter->aSeg; int i2 = p2 - pIter->aSeg; |
︙ | ︙ | |||
2447 2448 2449 2450 2451 2452 2453 | /* ** This function is a no-op unless SQLITE_DEBUG is defined when this module ** is compiled. In that case, this function is essentially an assert() ** statement used to verify that the contents of the pIter->aFirst[] array ** are correct. */ | | | 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 | /* ** This function is a no-op unless SQLITE_DEBUG is defined when this module ** is compiled. In that case, this function is essentially an assert() ** statement used to verify that the contents of the pIter->aFirst[] array ** are correct. */ static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5IndexIter *pIter){ if( p->rc==SQLITE_OK ){ int i; assert( (pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0)==pIter->bEof ); for(i=0; i<pIter->nSeg; i+=2){ Fts5SegIter *p1 = &pIter->aSeg[i]; |
︙ | ︙ | |||
2480 2481 2482 2483 2484 2485 2486 | ** Do the comparison necessary to populate pIter->aFirst[iOut]. ** ** If the returned value is non-zero, then it is the index of an entry ** in the pIter->aSeg[] array that is (a) not at EOF, and (b) pointing ** to a key that is a duplicate of another, higher priority, ** segment-iterator in the pSeg->aSeg[] array. */ | | | 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 | ** Do the comparison necessary to populate pIter->aFirst[iOut]. ** ** If the returned value is non-zero, then it is the index of an entry ** in the pIter->aSeg[] array that is (a) not at EOF, and (b) pointing ** to a key that is a duplicate of another, higher priority, ** segment-iterator in the pSeg->aSeg[] array. */ static int fts5MultiIterDoCompare(Fts5IndexIter *pIter, int iOut){ int i1; /* Index of left-hand Fts5SegIter */ int i2; /* Index of right-hand Fts5SegIter */ int iRes; Fts5SegIter *p1; /* Left-hand Fts5SegIter */ Fts5SegIter *p2; /* Right-hand Fts5SegIter */ Fts5CResult *pRes = &pIter->aFirst[iOut]; |
︙ | ︙ | |||
2626 2627 2628 2629 2630 2631 2632 | } } /* ** Free the iterator object passed as the second argument. */ | | > > | | 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 | } } /* ** Free the iterator object passed as the second argument. */ static void fts5MultiIterFree(Fts5Index *p, Fts5IndexIter *pIter){ if( pIter ){ int i; for(i=0; i<pIter->nSeg; i++){ fts5SegIterClear(&pIter->aSeg[i]); } fts5StructureRelease(pIter->pStruct); fts5BufferFree(&pIter->poslist); sqlite3_free(pIter); } } static void fts5MultiIterAdvanced( Fts5Index *p, /* FTS5 backend to iterate within */ Fts5IndexIter *pIter, /* Iterator to update aFirst[] array for */ int iChanged, /* Index of sub-iterator just advanced */ int iMinset /* Minimum entry in aFirst[] to set */ ){ int i; for(i=(pIter->nSeg+iChanged)/2; i>=iMinset && p->rc==SQLITE_OK; i=i/2){ int iEq; if( (iEq = fts5MultiIterDoCompare(pIter, i)) ){ |
︙ | ︙ | |||
2664 2665 2666 2667 2668 2669 2670 | ** ** If non-zero is returned, the caller should call fts5MultiIterAdvanced() ** on the iterator instead. That function does the same as this one, except ** that it deals with more complicated cases as well. */ static int fts5MultiIterAdvanceRowid( Fts5Index *p, /* FTS5 backend to iterate within */ | | | 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 | ** ** If non-zero is returned, the caller should call fts5MultiIterAdvanced() ** on the iterator instead. That function does the same as this one, except ** that it deals with more complicated cases as well. */ static int fts5MultiIterAdvanceRowid( Fts5Index *p, /* FTS5 backend to iterate within */ Fts5IndexIter *pIter, /* Iterator to update aFirst[] array for */ int iChanged /* Index of sub-iterator just advanced */ ){ int i; Fts5SegIter *pNew = &pIter->aSeg[iChanged]; Fts5SegIter *pOther = &pIter->aSeg[iChanged ^ 0x0001]; for(i=(pIter->nSeg+iChanged)/2; 1; i=i/2){ |
︙ | ︙ | |||
2696 2697 2698 2699 2700 2701 2702 | return 0; } /* ** Set the pIter->bEof variable based on the state of the sub-iterators. */ | | | | 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 | return 0; } /* ** Set the pIter->bEof variable based on the state of the sub-iterators. */ static void fts5MultiIterSetEof(Fts5IndexIter *pIter){ pIter->bEof = pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0; } /* ** Move the iterator to the next entry. ** ** If an error occurs, an error code is left in Fts5Index.rc. It is not ** considered an error if the iterator reaches EOF, or if it is already at ** EOF when this function is called. */ static void fts5MultiIterNext( Fts5Index *p, Fts5IndexIter *pIter, int bFrom, /* True if argument iFrom is valid */ i64 iFrom /* Advance at least as far as this */ ){ if( p->rc==SQLITE_OK ){ int bUseFrom = bFrom; do { int iFirst = pIter->aFirst[1].iFirst; |
︙ | ︙ | |||
2739 2740 2741 2742 2743 2744 2745 | fts5AssertMultiIterSetup(p, pIter); bUseFrom = 0; }while( pIter->bSkipEmpty && fts5MultiIterIsEmpty(p, pIter) ); } } | | | | > | | | > > | 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 | fts5AssertMultiIterSetup(p, pIter); bUseFrom = 0; }while( pIter->bSkipEmpty && fts5MultiIterIsEmpty(p, pIter) ); } } static Fts5IndexIter *fts5MultiIterAlloc( Fts5Index *p, /* FTS5 backend to iterate within */ int nSeg ){ Fts5IndexIter *pNew; int nSlot; /* Power of two >= nSeg */ for(nSlot=2; nSlot<nSeg; nSlot=nSlot*2); pNew = fts5IdxMalloc(p, sizeof(Fts5IndexIter) + /* pNew */ sizeof(Fts5SegIter) * (nSlot-1) + /* pNew->aSeg[] */ sizeof(Fts5CResult) * nSlot /* pNew->aFirst[] */ ); if( pNew ){ pNew->nSeg = nSlot; pNew->aFirst = (Fts5CResult*)&pNew->aSeg[nSlot]; pNew->pIndex = p; } return pNew; } /* ** Allocate a new Fts5IndexIter object. ** ** The new object will be used to iterate through data in structure pStruct. ** If iLevel is -ve, then all data in all segments is merged. Or, if iLevel ** is zero or greater, data from the first nSegment segments on level iLevel ** is merged. ** ** The iterator initially points to the first term/rowid entry in the ** iterated data. */ static void fts5MultiIterNew( Fts5Index *p, /* FTS5 backend to iterate within */ Fts5Structure *pStruct, /* Structure of specific index */ int bSkipEmpty, /* True to ignore delete-keys */ int flags, /* FTS5INDEX_QUERY_XXX flags */ const u8 *pTerm, int nTerm, /* Term to seek to (or NULL/0) */ int iLevel, /* Level to iterate (-1 for all) */ int nSegment, /* Number of segments to merge (iLevel>=0) */ Fts5IndexIter **ppOut /* New object */ ){ int nSeg = 0; /* Number of segment-iters in use */ int iIter = 0; /* */ int iSeg; /* Used to iterate through segments */ Fts5StructureLevel *pLvl; Fts5IndexIter *pNew; assert( (pTerm==0 && nTerm==0) || iLevel<0 ); /* Allocate space for the new multi-seg-iterator. */ if( p->rc==SQLITE_OK ){ if( iLevel<0 ){ assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); nSeg = pStruct->nSegment; nSeg += (p->pHash ? 1 : 0); }else{ nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment); } } *ppOut = pNew = fts5MultiIterAlloc(p, nSeg); if( pNew==0 ) return; pNew->bRev = (0!=(flags & FTS5INDEX_QUERY_DESC)); pNew->bSkipEmpty = bSkipEmpty; pNew->pStruct = pStruct; fts5StructureRef(pStruct); /* Initialize each of the component segment iterators. */ if( iLevel<0 ){ Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel]; if( p->pHash ){ /* Add a segment iterator for the current contents of the hash table. */ Fts5SegIter *pIter = &pNew->aSeg[iIter++]; |
︙ | ︙ | |||
2855 2856 2857 2858 2859 2860 2861 | }else{ fts5MultiIterFree(p, pNew); *ppOut = 0; } } /* | | | | | 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 | }else{ fts5MultiIterFree(p, pNew); *ppOut = 0; } } /* ** Create an Fts5IndexIter that iterates through the doclist provided ** as the second argument. */ static void fts5MultiIterNew2( Fts5Index *p, /* FTS5 backend to iterate within */ Fts5Data *pData, /* Doclist to iterate through */ int bDesc, /* True for descending rowid order */ Fts5IndexIter **ppOut /* New object */ ){ Fts5IndexIter *pNew; pNew = fts5MultiIterAlloc(p, 2); if( pNew ){ Fts5SegIter *pIter = &pNew->aSeg[1]; pIter->flags = FTS5_SEGITER_ONETERM; if( pData->n>0 ){ pIter->pLeaf = pData; |
︙ | ︙ | |||
2896 2897 2898 2899 2900 2901 2902 | fts5DataRelease(pData); } /* ** Return true if the iterator is at EOF or if an error has occurred. ** False otherwise. */ | | | | | | 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 | fts5DataRelease(pData); } /* ** Return true if the iterator is at EOF or if an error has occurred. ** False otherwise. */ static int fts5MultiIterEof(Fts5Index *p, Fts5IndexIter *pIter){ assert( p->rc || (pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0)==pIter->bEof ); return (p->rc || pIter->bEof); } /* ** Return the rowid of the entry that the iterator currently points ** to. If the iterator points to EOF when this function is called the ** results are undefined. */ static i64 fts5MultiIterRowid(Fts5IndexIter *pIter){ assert( pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf ); return pIter->aSeg[ pIter->aFirst[1].iFirst ].iRowid; } /* ** Move the iterator to the next entry at or following iMatch. */ static void fts5MultiIterNextFrom( Fts5Index *p, Fts5IndexIter *pIter, i64 iMatch ){ while( 1 ){ i64 iRowid; fts5MultiIterNext(p, pIter, 1, iMatch); if( fts5MultiIterEof(p, pIter) ) break; iRowid = fts5MultiIterRowid(pIter); if( pIter->bRev==0 && iRowid>=iMatch ) break; if( pIter->bRev!=0 && iRowid<=iMatch ) break; } } /* ** Return a pointer to a buffer containing the term associated with the ** entry that the iterator currently points to. */ static const u8 *fts5MultiIterTerm(Fts5IndexIter *pIter, int *pn){ Fts5SegIter *p = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; *pn = p->term.n; return p->term.p; } static void fts5ChunkIterate( Fts5Index *p, /* Index object */ |
︙ | ︙ | |||
3563 3564 3565 3566 3567 3568 3569 | } /* ** Iterator pIter was used to iterate through the input segments of on an ** incremental merge operation. This function is called if the incremental ** merge step has finished but the input has not been completely exhausted. */ | | | 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 | } /* ** Iterator pIter was used to iterate through the input segments of on an ** incremental merge operation. This function is called if the incremental ** merge step has finished but the input has not been completely exhausted. */ static void fts5TrimSegments(Fts5Index *p, Fts5IndexIter *pIter){ int i; Fts5Buffer buf; memset(&buf, 0, sizeof(Fts5Buffer)); for(i=0; i<pIter->nSeg; i++){ Fts5SegIter *pSeg = &pIter->aSeg[i]; if( pSeg->pSeg==0 ){ /* no-op */ |
︙ | ︙ | |||
3623 3624 3625 3626 3627 3628 3629 | Fts5Structure **ppStruct, /* IN/OUT: Stucture of index */ int iLvl, /* Level to read input from */ int *pnRem /* Write up to this many output leaves */ ){ Fts5Structure *pStruct = *ppStruct; Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; Fts5StructureLevel *pLvlOut; | | | 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 | Fts5Structure **ppStruct, /* IN/OUT: Stucture of index */ int iLvl, /* Level to read input from */ int *pnRem /* Write up to this many output leaves */ ){ Fts5Structure *pStruct = *ppStruct; Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; Fts5StructureLevel *pLvlOut; Fts5IndexIter *pIter = 0; /* Iterator to read input data */ int nRem = pnRem ? *pnRem : 0; /* Output leaf pages left to write */ int nInput; /* Number of input segments */ Fts5SegWriter writer; /* Writer object */ Fts5StructureSegment *pSeg; /* Output segment */ Fts5Buffer term; int bRequireDoclistTerm = 0; /* Doclist terminator (0x00) required */ int bOldest; /* True if the output segment is the oldest */ |
︙ | ︙ | |||
4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 | pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte); } } if( pNew ){ Fts5StructureLevel *pLvl; int nByte = nSeg * sizeof(Fts5StructureSegment); pNew->nLevel = pStruct->nLevel+1; pNew->nWriteCounter = pStruct->nWriteCounter; pLvl = &pNew->aLevel[pStruct->nLevel]; pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte); if( pLvl->aSeg ){ int iLvl, iSeg; int iSegOut = 0; for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){ | > | 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 | pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte); } } if( pNew ){ Fts5StructureLevel *pLvl; int nByte = nSeg * sizeof(Fts5StructureSegment); pNew->nLevel = pStruct->nLevel+1; pNew->nRef = 1; pNew->nWriteCounter = pStruct->nWriteCounter; pLvl = &pNew->aLevel[pStruct->nLevel]; pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte); if( pLvl->aSeg ){ int iLvl, iSeg; int iSegOut = 0; for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){ |
︙ | ︙ | |||
4176 4177 4178 4179 4180 4181 4182 | ** currently points to to buffer pBuf. ** ** If an error occurs, an error code is left in p->rc. It is assumed ** no error has already occurred when this function is called. */ static void fts5MultiIterPoslist( Fts5Index *p, | | | 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 | ** currently points to to buffer pBuf. ** ** If an error occurs, an error code is left in p->rc. It is assumed ** no error has already occurred when this function is called. */ static void fts5MultiIterPoslist( Fts5Index *p, Fts5IndexIter *pMulti, int bSz, /* Append a size field before the data */ Fts5Buffer *pBuf ){ if( p->rc==SQLITE_OK ){ Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ]; assert( fts5MultiIterEof(p, pMulti)==0 ); |
︙ | ︙ | |||
4330 4331 4332 4333 4334 4335 4336 | } static void fts5SetupPrefixIter( Fts5Index *p, /* Index to read from */ int bDesc, /* True for "ORDER BY rowid DESC" */ const u8 *pToken, /* Buffer containing prefix to match */ int nToken, /* Size of buffer pToken in bytes */ | | | | 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 | } static void fts5SetupPrefixIter( Fts5Index *p, /* Index to read from */ int bDesc, /* True for "ORDER BY rowid DESC" */ const u8 *pToken, /* Buffer containing prefix to match */ int nToken, /* Size of buffer pToken in bytes */ Fts5IndexIter **ppIter /* OUT: New iterator */ ){ Fts5Structure *pStruct; Fts5Buffer *aBuf; const int nBuf = 32; aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf); pStruct = fts5StructureRead(p); if( aBuf && pStruct ){ const int flags = FTS5INDEX_QUERY_SCAN; int i; i64 iLastRowid = 0; Fts5IndexIter *p1 = 0; /* Iterator used to gather data from index */ Fts5Data *pData; Fts5Buffer doclist; memset(&doclist, 0, sizeof(doclist)); for(fts5MultiIterNew(p, pStruct, 1, flags, pToken, nToken, -1, 0, &p1); fts5MultiIterEof(p, p1)==0; fts5MultiIterNext(p, p1, 0, 0) |
︙ | ︙ | |||
4386 4387 4388 4389 4390 4391 4392 | fts5MultiIterFree(p, p1); pData = fts5IdxMalloc(p, sizeof(Fts5Data) + doclist.n); if( pData ){ pData->p = (u8*)&pData[1]; pData->n = doclist.n; memcpy(pData->p, doclist.p, doclist.n); | | | 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 | fts5MultiIterFree(p, p1); pData = fts5IdxMalloc(p, sizeof(Fts5Data) + doclist.n); if( pData ){ pData->p = (u8*)&pData[1]; pData->n = doclist.n; memcpy(pData->p, doclist.p, doclist.n); fts5MultiIterNew2(p, pData, bDesc, ppIter); } fts5BufferFree(&doclist); } fts5StructureRelease(pStruct); sqlite3_free(aBuf); } |
︙ | ︙ | |||
4596 4597 4598 4599 4600 4601 4602 | int sqlite3Fts5IndexQuery( Fts5Index *p, /* FTS index to query */ const char *pToken, int nToken, /* Token (or prefix) to query for */ int flags, /* Mask of FTS5INDEX_QUERY_X flags */ Fts5IndexIter **ppIter /* OUT: New iterator object */ ){ Fts5Config *pConfig = p->pConfig; | | | < | | | | | | | | | | < < < > < | < | | | | < | | | > | | | > | | < < | | | | | | < | < | > | < < | < | 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661 4662 4663 4664 4665 4666 4667 4668 4669 4670 4671 4672 4673 4674 4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695 4696 4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713 4714 4715 4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 4726 4727 4728 4729 4730 4731 4732 4733 4734 4735 4736 4737 4738 4739 4740 4741 4742 4743 4744 4745 4746 4747 4748 4749 4750 4751 4752 4753 4754 4755 4756 4757 4758 4759 4760 4761 4762 4763 4764 4765 4766 4767 4768 4769 4770 4771 4772 4773 4774 4775 4776 4777 4778 4779 4780 4781 4782 4783 4784 4785 | int sqlite3Fts5IndexQuery( Fts5Index *p, /* FTS index to query */ const char *pToken, int nToken, /* Token (or prefix) to query for */ int flags, /* Mask of FTS5INDEX_QUERY_X flags */ Fts5IndexIter **ppIter /* OUT: New iterator object */ ){ Fts5Config *pConfig = p->pConfig; Fts5IndexIter *pRet = 0; int iIdx = 0; Fts5Buffer buf = {0, 0, 0}; /* If the QUERY_SCAN flag is set, all other flags must be clear. */ assert( (flags & FTS5INDEX_QUERY_SCAN)==0 || (flags & FTS5INDEX_QUERY_SCAN)==FTS5INDEX_QUERY_SCAN ); if( sqlite3Fts5BufferGrow(&p->rc, &buf, nToken+1)==0 ){ memcpy(&buf.p[1], pToken, nToken); #ifdef SQLITE_DEBUG if( flags & FTS5INDEX_QUERY_TEST_NOIDX ){ assert( flags & FTS5INDEX_QUERY_PREFIX ); iIdx = 1+pConfig->nPrefix; }else #endif if( flags & FTS5INDEX_QUERY_PREFIX ){ int nChar = fts5IndexCharlen(pToken, nToken); for(iIdx=1; iIdx<=pConfig->nPrefix; iIdx++){ if( pConfig->aPrefix[iIdx-1]==nChar ) break; } } if( iIdx<=pConfig->nPrefix ){ Fts5Structure *pStruct = fts5StructureRead(p); buf.p[0] = FTS5_MAIN_PREFIX + iIdx; if( pStruct ){ fts5MultiIterNew(p, pStruct, 1, flags, buf.p, nToken+1, -1, 0, &pRet); fts5StructureRelease(pStruct); } }else{ int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0; buf.p[0] = FTS5_MAIN_PREFIX; fts5SetupPrefixIter(p, bDesc, buf.p, nToken+1, &pRet); } if( p->rc ){ sqlite3Fts5IterClose(pRet); pRet = 0; fts5CloseReader(p); } *ppIter = pRet; sqlite3Fts5BufferFree(&buf); } return fts5IndexReturn(p); } /* ** Return true if the iterator passed as the only argument is at EOF. */ int sqlite3Fts5IterEof(Fts5IndexIter *pIter){ assert( pIter->pIndex->rc==SQLITE_OK ); return pIter->bEof; } /* ** Move to the next matching rowid. */ int sqlite3Fts5IterNext(Fts5IndexIter *pIter){ assert( pIter->pIndex->rc==SQLITE_OK ); fts5MultiIterNext(pIter->pIndex, pIter, 0, 0); return fts5IndexReturn(pIter->pIndex); } /* ** Move to the next matching term/rowid. Used by the fts5vocab module. */ int sqlite3Fts5IterNextScan(Fts5IndexIter *pIter){ Fts5Index *p = pIter->pIndex; assert( pIter->pIndex->rc==SQLITE_OK ); fts5MultiIterNext(p, pIter, 0, 0); if( p->rc==SQLITE_OK ){ Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; if( pSeg->pLeaf && pSeg->term.p[0]!=FTS5_MAIN_PREFIX ){ fts5DataRelease(pSeg->pLeaf); pSeg->pLeaf = 0; pIter->bEof = 1; } } return fts5IndexReturn(pIter->pIndex); } /* ** Move to the next matching rowid that occurs at or after iMatch. The ** definition of "at or after" depends on whether this iterator iterates ** in ascending or descending rowid order. */ int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIter, i64 iMatch){ fts5MultiIterNextFrom(pIter->pIndex, pIter, iMatch); return fts5IndexReturn(pIter->pIndex); } /* ** Return the current rowid. */ i64 sqlite3Fts5IterRowid(Fts5IndexIter *pIter){ return fts5MultiIterRowid(pIter); } /* ** Return the current term. */ const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIter, int *pn){ int n; const char *z = (const char*)fts5MultiIterTerm(pIter, &n); *pn = n-1; return &z[1]; } /* ** Return a pointer to a buffer containing a copy of the position list for ** the current entry. Output variable *pn is set to the size of the buffer ** in bytes before returning. ** ** The returned position list does not include the "number of bytes" varint ** field that starts the position list on disk. */ int sqlite3Fts5IterPoslist( Fts5IndexIter *pIter, const u8 **pp, /* OUT: Pointer to position-list data */ int *pn, /* OUT: Size of position-list in bytes */ i64 *piRowid /* OUT: Current rowid */ ){ Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; assert( pIter->pIndex->rc==SQLITE_OK ); *piRowid = pSeg->iRowid; *pn = pSeg->nPos; if( pSeg->iLeafOffset+pSeg->nPos <= pSeg->pLeaf->n ){ *pp = &pSeg->pLeaf->p[pSeg->iLeafOffset]; }else{ fts5BufferZero(&pIter->poslist); fts5SegiterPoslist(pIter->pIndex, pSeg, &pIter->poslist); *pp = pIter->poslist.p; } return fts5IndexReturn(pIter->pIndex); } /* ** This function is similar to sqlite3Fts5IterPoslist(), except that it ** copies the position list into the buffer supplied as the second ** argument. */ int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf){ Fts5Index *p = pIter->pIndex; assert( p->rc==SQLITE_OK ); fts5BufferZero(pBuf); fts5MultiIterPoslist(p, pIter, 0, pBuf); return fts5IndexReturn(p); } /* ** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery(). */ void sqlite3Fts5IterClose(Fts5IndexIter *pIter){ if( pIter ){ Fts5Index *pIndex = pIter->pIndex; fts5MultiIterFree(pIter->pIndex, pIter); fts5CloseReader(pIndex); } } /* ** Read the "averages" record into the buffer supplied as the second ** argument. Return SQLITE_OK if successful, or an SQLite error code ** if an error occurs. |
︙ | ︙ | |||
5235 5236 5237 5238 5239 5240 5241 | ** checksum does not match. Return SQLITE_OK if all checks pass without ** error, or some other SQLite error code if another error (e.g. OOM) ** occurs. */ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ u64 cksum2 = 0; /* Checksum based on contents of indexes */ Fts5Buffer poslist = {0,0,0}; /* Buffer used to hold a poslist */ | | | 5236 5237 5238 5239 5240 5241 5242 5243 5244 5245 5246 5247 5248 5249 5250 | ** checksum does not match. Return SQLITE_OK if all checks pass without ** error, or some other SQLite error code if another error (e.g. OOM) ** occurs. */ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ u64 cksum2 = 0; /* Checksum based on contents of indexes */ Fts5Buffer poslist = {0,0,0}; /* Buffer used to hold a poslist */ Fts5IndexIter *pIter; /* Used to iterate through entire index */ Fts5Structure *pStruct; /* Index structure */ /* Used by extra internal tests only run if NDEBUG is not defined */ u64 cksum3 = 0; /* Checksum based on contents of indexes */ Fts5Buffer term = {0,0,0}; /* Buffer used to hold most recent term */ /* Load the FTS index structure */ |
︙ | ︙ |
Changes to ext/fts5/test/fts5version.test.
︙ | ︙ | |||
42 43 44 45 46 47 48 | do_test 1.5 { db close sqlite3 db test.db catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' } } {1 {invalid fts5 file format (found 3, expected 2) - run 'rebuild'}} | < | 42 43 44 45 46 47 48 49 50 51 52 53 54 55 | do_test 1.5 { db close sqlite3 db test.db catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' } } {1 {invalid fts5 file format (found 3, expected 2) - run 'rebuild'}} do_test 1.6 { db close sqlite3 db test.db catchsql { INSERT INTO t1 VALUES('x y z') } } {1 {invalid fts5 file format (found 3, expected 2) - run 'rebuild'}} do_test 1.7 { |
︙ | ︙ |