SQLite

Check-in [a5d5468c05]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a couple of build problems.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: a5d5468c0509d129e198bf9432190ee07cedb7af
User & Date: dan 2015-03-04 08:29:24.833
Context
2015-03-07
11:50
Fix the bm25() function so that it multiplies scores by -1 before returning them. This means better matches have a lower numerical score, so "ORDER BY rank" (not "ORDER BY rank DESC") does what you want. (check-in: 3ee7b5a9f9 user: dan tags: fts5)
2015-03-04
08:29
Fix a couple of build problems. (check-in: a5d5468c05 user: dan tags: fts5)
2015-02-27
09:41
Further minor optimizations to flushing fts5 data to disk. (check-in: a07dcca9ef user: dan tags: fts5)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts5/fts5.c.
9
10
11
12
13
14
15

16
17
18
19
20
21
22
**    May you share freely, never taking more than you give.
**
******************************************************************************
**
** This is an SQLite module implementing full-text search.
*/



#include "fts5Int.h"


typedef struct Fts5Table Fts5Table;
typedef struct Fts5Cursor Fts5Cursor;
typedef struct Fts5Global Fts5Global;







>







9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
**    May you share freely, never taking more than you give.
**
******************************************************************************
**
** This is an SQLite module implementing full-text search.
*/

#if defined(SQLITE_ENABLE_FTS5)

#include "fts5Int.h"


typedef struct Fts5Table Fts5Table;
typedef struct Fts5Cursor Fts5Cursor;
typedef struct Fts5Global Fts5Global;
1958
1959
1960
1961
1962
1963
1964

1965
1966
      rc = sqlite3_create_function(
          db, "fts5", 0, SQLITE_UTF8, p, fts5Fts5Func, 0, 0
      );
    }
  }
  return rc;
}










>


1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
      rc = sqlite3_create_function(
          db, "fts5", 0, SQLITE_UTF8, p, fts5Fts5Func, 0, 0
      );
    }
  }
  return rc;
}
#endif /* defined(SQLITE_ENABLE_FTS5) */


Changes to ext/fts5/fts5_aux.c.
1
2
3
4
5
6
7
8
9
10
11
12
13


14
15
16
17
18
19
20
/*
** 2014 May 31
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
*/



#include "fts5Int.h"
#include <math.h>

/*
** Object used to iterate through all "coalesced phrase instances" in 
** a single column of the current row. If the phrase instances in the
** column being considered do not overlap, this object simply iterates













>
>







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
/*
** 2014 May 31
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
*/

#ifdef SQLITE_ENABLE_FTS5

#include "fts5Int.h"
#include <math.h>

/*
** Object used to iterate through all "coalesced phrase instances" in 
** a single column of the current row. If the phrase instances in the
** column being considered do not overlap, this object simply iterates
545
546
547
548
549
550
551

552
553
        aBuiltin[i].xFunc,
        aBuiltin[i].xDestroy
    );
  }

  return rc;
}










>


547
548
549
550
551
552
553
554
555
556
        aBuiltin[i].xFunc,
        aBuiltin[i].xDestroy
    );
  }

  return rc;
}
#endif /* SQLITE_ENABLE_FTS5 */


Changes to ext/fts5/fts5_buffer.c.
8
9
10
11
12
13
14

15
16
17
18
19
20
21
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
*/




#include "fts5Int.h"

int sqlite3Fts5BufferGrow(int *pRc, Fts5Buffer *pBuf, int nByte){
  /* A no-op if an error has already occurred */
  if( *pRc ) return 1;








>







8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
*/


#ifdef SQLITE_ENABLE_FTS5

#include "fts5Int.h"

int sqlite3Fts5BufferGrow(int *pRc, Fts5Buffer *pBuf, int nByte){
  /* A no-op if an error has already occurred */
  if( *pRc ) return 1;

290
291
292
293
294
295
296
297
298
299
      *pRc = SQLITE_NOMEM;
    }else{
      memset(pRet, 0, nByte);
    }
  }
  return pRet;
}










|

<
291
292
293
294
295
296
297
298
299

      *pRc = SQLITE_NOMEM;
    }else{
      memset(pRet, 0, nByte);
    }
  }
  return pRet;
}
#endif /* SQLITE_ENABLE_FTS5 */


Changes to ext/fts5/fts5_config.c.
8
9
10
11
12
13
14



15
16
17
18
19
20
21
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
**
** This is an SQLite module implementing full-text search.
*/




#include "fts5Int.h"

#define FTS5_DEFAULT_PAGE_SIZE   1000
#define FTS5_DEFAULT_AUTOMERGE      4
#define FTS5_DEFAULT_CRISISMERGE   16








>
>
>







8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
**
** This is an SQLite module implementing full-text search.
*/

#ifdef SQLITE_ENABLE_FTS5


#include "fts5Int.h"

#define FTS5_DEFAULT_PAGE_SIZE   1000
#define FTS5_DEFAULT_AUTOMERGE      4
#define FTS5_DEFAULT_CRISISMERGE   16

785
786
787
788
789
790
791


  if( rc==SQLITE_OK ){
    pConfig->iCookie = iCookie;
  }
  return rc;
}









>
788
789
790
791
792
793
794
795

  if( rc==SQLITE_OK ){
    pConfig->iCookie = iCookie;
  }
  return rc;
}

#endif /* SQLITE_ENABLE_FTS5 */
Changes to ext/fts5/fts5_expr.c.
1
2
3
4
5
6
7
8
9
10
11
12
13



14
15
16
17
18
19
20
/*
** 2014 May 31
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
**
*/




#include "fts5Int.h"
#include "fts5parse.h"

/*
** All token types in the generated fts5parse.h file are greater than 0.
*/













>
>
>







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
/*
** 2014 May 31
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
**
*/

#ifdef SQLITE_ENABLE_FTS5


#include "fts5Int.h"
#include "fts5parse.h"

/*
** All token types in the generated fts5parse.h file are greater than 0.
*/
1691
1692
1693
1694
1695
1696
1697

      return pPhrase->poslist.n;
    }
  }
  *pa = 0;
  return 0;
}









>
1694
1695
1696
1697
1698
1699
1700
1701
      return pPhrase->poslist.n;
    }
  }
  *pa = 0;
  return 0;
}

#endif /* SQLITE_ENABLE_FTS5 */
Changes to ext/fts5/fts5_hash.c.
1
2
3
4
5
6
7
8
9
10
11
12
13



14
15
16
17
18
19
20
/*
** 2014 August 11
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
**
*/




#include "fts5Int.h"

typedef struct Fts5HashEntry Fts5HashEntry;

/*
** This file contains the implementation of an in-memory hash table used













>
>
>







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
/*
** 2014 August 11
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
**
*/

#ifdef SQLITE_ENABLE_FTS5


#include "fts5Int.h"

typedef struct Fts5HashEntry Fts5HashEntry;

/*
** This file contains the implementation of an in-memory hash table used
453
454
455
456
457
458
459

  }else{
    *pzTerm = 0;
    *ppDoclist = 0;
    *pnDoclist = 0;
  }
}









>
456
457
458
459
460
461
462
463
  }else{
    *pzTerm = 0;
    *ppDoclist = 0;
    *pnDoclist = 0;
  }
}

#endif /* SQLITE_ENABLE_FTS5 */
Changes to ext/fts5/fts5_index.c.
12
13
14
15
16
17
18


19
20
21
22
23
24
25
**
** Low level access to the FTS index stored in the database file. The 
** routines in this file file implement all read and write access to the
** %_data table. Other parts of the system access this functionality via
** the interface defined in fts5Int.h.
*/



#include "fts5Int.h"

/*
** Overview:
**
** The %_data table contains all the FTS indexes for an FTS5 virtual table.
** As well as the main term index, there may be up to 31 prefix indexes.







>
>







12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
**
** Low level access to the FTS index stored in the database file. The 
** routines in this file file implement all read and write access to the
** %_data table. Other parts of the system access this functionality via
** the interface defined in fts5Int.h.
*/

#ifdef SQLITE_ENABLE_FTS5

#include "fts5Int.h"

/*
** Overview:
**
** The %_data table contains all the FTS indexes for an FTS5 virtual table.
** As well as the main term index, there may be up to 31 prefix indexes.
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
**
**   FTS5_SEGITER_ONETERM:
**     If set, set the iterator to point to EOF after the current doclist 
**     has been exhausted. Do not proceed to the next term in the segment.
**
**   FTS5_SEGITER_REVERSE:
**     This flag is only ever set if FTS5_SEGITER_ONETERM is also set. If
**     it is set, iterate through docids in ascending order instead of the
**     default descending order.
**
** iRowidOffset/nRowidOffset/aRowidOffset:
**     These are used if the FTS5_SEGITER_REVERSE flag is set.
**
**     Each time a new page is loaded, the iterator is set to point to the
**     final rowid. Additionally, the aRowidOffset[] array is populated 
**     with the byte offsets of all relevant rowid fields on the page. 







|
|







446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
**
**   FTS5_SEGITER_ONETERM:
**     If set, set the iterator to point to EOF after the current doclist 
**     has been exhausted. Do not proceed to the next term in the segment.
**
**   FTS5_SEGITER_REVERSE:
**     This flag is only ever set if FTS5_SEGITER_ONETERM is also set. If
**     it is set, iterate through docids in descending order instead of the
**     default ascending order.
**
** iRowidOffset/nRowidOffset/aRowidOffset:
**     These are used if the FTS5_SEGITER_REVERSE flag is set.
**
**     Each time a new page is loaded, the iterator is set to point to the
**     final rowid. Additionally, the aRowidOffset[] array is populated 
**     with the byte offsets of all relevant rowid fields on the page. 
5056
5057
5058
5059
5060
5061
5062

    rc = sqlite3_create_function(
        db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0
    );
  }
  return rc;
}









>
5058
5059
5060
5061
5062
5063
5064
5065
    rc = sqlite3_create_function(
        db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0
    );
  }
  return rc;
}

#endif /* SQLITE_ENABLE_FTS5 */
Changes to ext/fts5/fts5_storage.c.
1
2
3
4
5
6
7
8
9
10
11
12
13



14
15
16
17
18
19
20
/*
** 2014 May 31
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
**
*/




#include "fts5Int.h"

struct Fts5Storage {
  Fts5Config *pConfig;
  Fts5Index *pIndex;
  int bTotalsValid;               /* True if nTotalRow/aTotalSize[] are valid */













>
>
>







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
/*
** 2014 May 31
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
**
*/

#ifdef SQLITE_ENABLE_FTS5


#include "fts5Int.h"

struct Fts5Storage {
  Fts5Config *pConfig;
  Fts5Index *pIndex;
  int bTotalsValid;               /* True if nTotalRow/aTotalSize[] are valid */
982
983
984
985
986
987
988

      p->pConfig->iCookie = iNew;
    }
  }
  return rc;
}










>
985
986
987
988
989
990
991
992
      p->pConfig->iCookie = iNew;
    }
  }
  return rc;
}


#endif /* SQLITE_ENABLE_FTS5 */
Changes to ext/fts5/fts5_tcl.c.
1
2
3
4
5
6
7
8
9
10
11
12
13
14

15

16

17
18
19
20
21
22
23
24
25
/*
** 2014 Dec 01
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
**
*/


#ifdef SQLITE_TEST



#include "fts5.h"
#include <tcl.h>
#include <string.h>
#include <assert.h>

/*************************************************************************
** This is a copy of the first part of the SqliteDb structure in 
** tclsqlite.c.  We need it here so that the get_sqlite_pointer routine
** can extract the sqlite3* pointer from an existing Tcl SQLite














>

>

>
|
|







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
/*
** 2014 Dec 01
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
**
*/


#ifdef SQLITE_TEST
#include <tcl.h>

#ifdef SQLITE_ENABLE_FTS5

#include "fts5.h"
#include <string.h>
#include <assert.h>

/*************************************************************************
** This is a copy of the first part of the SqliteDb structure in 
** tclsqlite.c.  We need it here so that the get_sqlite_pointer routine
** can extract the sqlite3* pointer from an existing Tcl SQLite
852
853
854
855
856
857
858



859

860
    void *pCtx = 0;
    if( p->bTokenizeCtx ) pCtx = (void*)pContext;
    Tcl_CreateObjCommand(interp, p->zName, p->xProc, pCtx, (i ? 0 : xF5tFree));
  }

  return TCL_OK;
}





#endif







>
>
>
|
>
|
855
856
857
858
859
860
861
862
863
864
865
866
867
    void *pCtx = 0;
    if( p->bTokenizeCtx ) pCtx = (void*)pContext;
    Tcl_CreateObjCommand(interp, p->zName, p->xProc, pCtx, (i ? 0 : xF5tFree));
  }

  return TCL_OK;
}
#else  /* SQLITE_ENABLE_FTS5 */
int Fts5tcl_Init(Tcl_Interp *interp){
  return TCL_OK;
}
#endif /* SQLITE_ENABLE_FTS5 */
#endif /* SQLITE_TEST */
Changes to ext/fts5/fts5_tokenize.c.
1
2
3
4
5
6
7
8
9
10
11
12
13


14
15
16
17
18
19
20
/*
** 2014 May 31
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
*/



#include "fts5.h"
#include <string.h>
#include <assert.h>

/**************************************************************************
** Start of ascii tokenizer implementation.
*/













>
>







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
/*
** 2014 May 31
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
*/

#if defined(SQLITE_ENABLE_FTS5)

#include "fts5.h"
#include <string.h>
#include <assert.h>

/**************************************************************************
** Start of ascii tokenizer implementation.
*/
1217
1218
1219
1220
1221
1222
1223

1224
1225
        &aBuiltin[i].x,
        0
    );
  }

  return SQLITE_OK;
}










>


1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
        &aBuiltin[i].x,
        0
    );
  }

  return SQLITE_OK;
}
#endif /* defined(SQLITE_ENABLE_FTS5) */


Changes to main.mk.
632
633
634
635
636
637
638

639
640

641
642
643
644
645
646
647
	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_unicode2.c

fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon 
	cp $(TOP)/ext/fts5/fts5parse.y .
	rm -f fts5parse.h
	./lemon $(OPTS) fts5parse.y
	mv fts5parse.c fts5parse.c.orig

	cat fts5parse.c.orig | sed 's/yy/fts5yy/g' | sed 's/YY/fts5YY/g' \
		| sed 's/TOKEN/FTS5TOKEN/g' > fts5parse.c



userauth.o:	$(TOP)/ext/userauth/userauth.c $(HDR) $(EXTHDR)
	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/userauth/userauth.c


# Rules for building test programs and for running tests







>

|
>







632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_unicode2.c

fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon 
	cp $(TOP)/ext/fts5/fts5parse.y .
	rm -f fts5parse.h
	./lemon $(OPTS) fts5parse.y
	mv fts5parse.c fts5parse.c.orig
	echo "#ifdef SQLITE_ENABLE_FTS5" > fts5parse.c
	cat fts5parse.c.orig | sed 's/yy/fts5yy/g' | sed 's/YY/fts5YY/g' \
		| sed 's/TOKEN/FTS5TOKEN/g' >> fts5parse.c
	echo "#endif /* SQLITE_ENABLE_FTS5 */" >> fts5parse.c


userauth.o:	$(TOP)/ext/userauth/userauth.c $(HDR) $(EXTHDR)
	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/userauth/userauth.c


# Rules for building test programs and for running tests