︙ | | |
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
|
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
|
+
-
-
-
-
+
-
-
+
|
** The xDisconnect() virtual table method.
*/
static int fts3DisconnectMethod(sqlite3_vtab *pVtab){
Fts3Table *p = (Fts3Table *)pVtab;
int i;
assert( p->nPendingData==0 );
assert( p->pSegments==0 );
/* Free any prepared statements held */
for(i=0; i<SizeofArray(p->aStmt); i++){
sqlite3_finalize(p->aStmt[i]);
}
for(i=0; i<p->nLeavesStmt; i++){
sqlite3_finalize(p->aLeavesStmt[i]);
}
sqlite3_free(p->zSelectLeaves);
sqlite3_free(p->zSegmentsTbl);
sqlite3_free(p->aLeavesStmt);
/* Invoke the tokenizer destructor to free the tokenizer. */
p->pTokenizer->pModule->xDestroy(p->pTokenizer);
sqlite3_free(p);
return SQLITE_OK;
}
/*
** Construct one or more SQL statements from the format string given
** and then evaluate those statements. The success code is writting
** and then evaluate those statements. The success code is written
** into *pRc.
**
** If *pRc is initially non-zero then this routine is a no-op.
*/
static void fts3DbExec(
int *pRc, /* Success code */
sqlite3 *db, /* Database in which to run SQL */
|
︙ | | |
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
|
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
|
+
+
+
+
-
-
-
-
-
+
+
+
+
+
+
-
-
-
-
-
+
+
+
+
+
-
-
-
-
+
+
+
+
-
-
-
-
-
-
+
+
+
+
+
-
-
-
+
+
+
+
-
-
-
-
-
-
-
+
+
+
+
+
+
-
-
-
-
-
-
+
+
+
+
+
+
-
|
}
/*
** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table
** passed as the first argument. This is done as part of the xConnect()
** and xCreate() methods.
**
** If *pRc is non-zero when this function is called, it is a no-op.
** Otherwise, if an error occurs, an SQLite error code is stored in *pRc
** before returning.
*/
static int fts3DeclareVtab(Fts3Table *p){
int i; /* Iterator variable */
int rc; /* Return code */
char *zSql; /* SQL statement passed to declare_vtab() */
char *zCols; /* List of user defined columns */
static void fts3DeclareVtab(int *pRc, Fts3Table *p){
if( *pRc==SQLITE_OK ){
int i; /* Iterator variable */
int rc; /* Return code */
char *zSql; /* SQL statement passed to declare_vtab() */
char *zCols; /* List of user defined columns */
/* Create a list of user columns for the virtual table */
zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]);
for(i=1; zCols && i<p->nColumn; i++){
zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]);
}
/* Create a list of user columns for the virtual table */
zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]);
for(i=1; zCols && i<p->nColumn; i++){
zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]);
}
/* Create the whole "CREATE TABLE" statement to pass to SQLite */
zSql = sqlite3_mprintf(
"CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName
);
/* Create the whole "CREATE TABLE" statement to pass to SQLite */
zSql = sqlite3_mprintf(
"CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName
);
if( !zCols || !zSql ){
rc = SQLITE_NOMEM;
}else{
rc = sqlite3_declare_vtab(p->db, zSql);
}
if( !zCols || !zSql ){
rc = SQLITE_NOMEM;
}else{
rc = sqlite3_declare_vtab(p->db, zSql);
}
sqlite3_free(zSql);
sqlite3_free(zCols);
return rc;
sqlite3_free(zSql);
sqlite3_free(zCols);
*pRc = rc;
}
}
/*
** Create the backing store tables (%_content, %_segments and %_segdir)
** required by the FTS3 table passed as the only argument. This is done
** as part of the vtab xCreate() method.
**
** If the p->bHasDocsize boolean is true (indicating that this is an
** FTS4 table, not an FTS3 table) then also create the %_docsize and
** %_stat tables required by FTS4.
*/
static int fts3CreateTables(Fts3Table *p){
int rc = SQLITE_OK; /* Return code */
int i; /* Iterator variable */
char *zContentCols; /* Columns of %_content table */
sqlite3 *db = p->db; /* The database connection */
/* Create a list of user columns for the content table */
if( p->bHasContent ){
zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY");
for(i=0; zContentCols && i<p->nColumn; i++){
char *z = p->azColumn[i];
zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z);
}
if( zContentCols==0 ) rc = SQLITE_NOMEM;
zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY");
for(i=0; zContentCols && i<p->nColumn; i++){
char *z = p->azColumn[i];
zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z);
}
if( zContentCols==0 ) rc = SQLITE_NOMEM;
/* Create the content table */
fts3DbExec(&rc, db,
"CREATE TABLE %Q.'%q_content'(%s)",
p->zDb, p->zName, zContentCols
);
sqlite3_free(zContentCols);
/* Create the content table */
fts3DbExec(&rc, db,
"CREATE TABLE %Q.'%q_content'(%s)",
p->zDb, p->zName, zContentCols
);
sqlite3_free(zContentCols);
}
/* Create other tables */
fts3DbExec(&rc, db,
"CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);",
p->zDb, p->zName
);
fts3DbExec(&rc, db,
"CREATE TABLE %Q.'%q_segdir'("
|
︙ | | |
634
635
636
637
638
639
640
641
642
643
644
645
646
647
|
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
|
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
|
zDb, zName, zSuffix
);
rc = sqlite3_exec(db, zSql, fts3TableExistsCallback, &res, 0);
sqlite3_free(zSql);
*pResult = (u8)(res & 0xff);
if( rc!=SQLITE_ABORT ) *pRc = rc;
}
/*
** Store the current database page-size in bytes in p->nPgsz.
**
** If *pRc is non-zero when this function is called, it is a no-op.
** Otherwise, if an error occurs, an SQLite error code is stored in *pRc
** before returning.
*/
static void fts3DatabasePageSize(int *pRc, Fts3Table *p){
if( *pRc==SQLITE_OK ){
int rc; /* Return code */
char *zSql; /* SQL text "PRAGMA %Q.page_size" */
sqlite3_stmt *pStmt; /* Compiled "PRAGMA %Q.page_size" statement */
zSql = sqlite3_mprintf("PRAGMA %Q.page_size", p->zDb);
if( !zSql ){
rc = SQLITE_NOMEM;
}else{
rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0);
if( rc==SQLITE_OK ){
sqlite3_step(pStmt);
p->nPgsz = sqlite3_column_int(pStmt, 0);
rc = sqlite3_finalize(pStmt);
}
}
assert( p->nPgsz>0 || rc!=SQLITE_OK );
sqlite3_free(zSql);
*pRc = rc;
}
}
/*
** This function is the implementation of both the xConnect and xCreate
** methods of the FTS3 virtual table.
**
** The argv[] array contains the following:
**
|
︙ | | |
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
|
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
|
-
+
+
+
+
-
+
-
+
-
+
+
+
|
p->bHasDocsize = argv[0][3]=='4';
rc = fts3CreateTables(p);
}else{
rc = SQLITE_OK;
fts3TableExists(&rc, db, argv[1], argv[2], "_content", &p->bHasContent);
fts3TableExists(&rc, db, argv[1], argv[2], "_docsize", &p->bHasDocsize);
}
if( rc!=SQLITE_OK ) goto fts3_init_out;
/* Figure out the page-size for the database. This is required in order to
** estimate the cost of loading large doclists from the database (see
** function sqlite3Fts3SegReaderCost() for details).
*/
rc = fts3DeclareVtab(p);
fts3DatabasePageSize(&rc, p);
if( rc!=SQLITE_OK ) goto fts3_init_out;
/* Declare the table schema to SQLite. */
*ppVTab = &p->base;
fts3DeclareVtab(&rc, p);
fts3_init_out:
assert( p || (pTokenizer && rc!=SQLITE_OK) );
if( rc!=SQLITE_OK ){
if( p ){
fts3DisconnectMethod((sqlite3_vtab *)p);
}else{
pTokenizer->pModule->xDestroy(pTokenizer);
}
}else{
*ppVTab = &p->base;
}
return rc;
}
/*
** The xConnect() and xCreate() methods for the virtual table. All the
** work is done in function fts3InitVtab().
|
︙ | | |
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
|
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
|
-
+
+
+
|
return SQLITE_OK;
}
/*
** Close the cursor. For additional information see the documentation
** on the xClose method of the virtual table interface.
*/
static int fulltextClose(sqlite3_vtab_cursor *pCursor){
static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){
Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
sqlite3_finalize(pCsr->pStmt);
sqlite3Fts3ExprFree(pCsr->pExpr);
sqlite3Fts3FreeDeferredTokens(pCsr);
sqlite3_free(pCsr->aDoclist);
sqlite3_free(pCsr->aMatchinfo);
sqlite3_free(pCsr);
return SQLITE_OK;
}
/*
|
︙ | | |
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
|
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
|
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
-
+
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
+
+
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
+
+
+
+
+
-
-
+
+
-
+
-
-
-
-
+
-
+
-
-
-
-
-
+
-
-
+
+
+
-
-
-
-
-
-
+
+
+
+
-
-
-
+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
-
-
-
-
+
-
-
-
-
-
-
-
-
-
-
-
+
-
-
-
+
+
-
-
-
-
-
+
+
-
+
-
+
-
-
-
-
-
+
-
|
}
}else{
return SQLITE_OK;
}
}
/*
** Advance the cursor to the next row in the %_content table that
** matches the search criteria. For a MATCH search, this will be
** the next row that matches. For a full-table scan, this will be
** simply the next row in the %_content table. For a docid lookup,
** this routine simply sets the EOF flag.
** This function is used to process a single interior node when searching
** a b-tree for a term or term prefix. The node data is passed to this
** function via the zNode/nNode parameters. The term to search for is
** passed in zTerm/nTerm.
**
** If piFirst is not NULL, then this function sets *piFirst to the blockid
** of the child node that heads the sub-tree that may contain the term.
**
** If piLast is not NULL, then *piLast is set to the right-most child node
** that heads a sub-tree that may contain a term for which zTerm/nTerm is
** a prefix.
**
** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned
** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK.
** even if we reach end-of-file. The fts3EofMethod() will be called
** subsequently to determine whether or not an EOF was hit.
*/
static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){
static int fts3ScanInteriorNode(
Fts3Table *p, /* Virtual table handle */
const char *zTerm, /* Term to select leaves for */
int nTerm, /* Size of term zTerm in bytes */
const char *zNode, /* Buffer containing segment interior node */
int nNode, /* Size of buffer at zNode */
sqlite3_int64 *piFirst, /* OUT: Selected child node */
sqlite3_int64 *piLast /* OUT: Selected child node */
){
int rc = SQLITE_OK; /* Return code */
const char *zCsr = zNode; /* Cursor to iterate through node */
const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
char *zBuffer = 0; /* Buffer to load terms into */
int nAlloc = 0; /* Size of allocated buffer */
int isFirstTerm = 1; /* True when processing first term on page */
sqlite3_int64 iChild; /* Block id of child node to descend to */
Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
if( pCsr->aDoclist==0 ){
if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){
pCsr->isEof = 1;
rc = sqlite3_reset(pCsr->pStmt);
}
}else if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){
pCsr->isEof = 1;
}else{
sqlite3_reset(pCsr->pStmt);
fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId);
pCsr->isRequireSeek = 1;
/* Skip over the 'height' varint that occurs at the start of every
** interior node. Then load the blockid of the left-child of the b-tree
** node into variable iChild. */
zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
while( zCsr<zEnd && (piFirst || piLast) ){
int cmp; /* memcmp() result */
int nSuffix; /* Size of term suffix */
int nPrefix = 0; /* Size of term prefix */
int nBuffer; /* Total term size */
/* Load the next term on the node into zBuffer. Use realloc() to expand
** the size of zBuffer if required. */
if( !isFirstTerm ){
zCsr += sqlite3Fts3GetVarint32(zCsr, &nPrefix);
}
isFirstTerm = 0;
zCsr += sqlite3Fts3GetVarint32(zCsr, &nSuffix);
if( nPrefix+nSuffix>nAlloc ){
char *zNew;
nAlloc = (nPrefix+nSuffix) * 2;
zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
if( !zNew ){
sqlite3_free(zBuffer);
return SQLITE_NOMEM;
}
zBuffer = zNew;
}
memcpy(&zBuffer[nPrefix], zCsr, nSuffix);
nBuffer = nPrefix + nSuffix;
zCsr += nSuffix;
/* Compare the term we are searching for with the term just loaded from
** the interior node. If the specified term is greater than or equal
** to the term from the interior node, then all terms on the sub-tree
** headed by node iChild are smaller than zTerm. No need to search
** iChild.
**
** If the interior node term is larger than the specified term, then
** the tree headed by iChild may contain the specified term.
*/
cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer));
if( piFirst && (cmp<0 || (cmp==0 && nBuffer>nTerm)) ){
*piFirst = iChild;
piFirst = 0;
}
if( piLast && cmp<0 ){
*piLast = iChild;
piLast = 0;
}
iChild++;
};
if( piFirst ) *piFirst = iChild;
if( piLast ) *piLast = iChild;
pCsr->isMatchinfoNeeded = 1;
}
sqlite3_free(zBuffer);
return rc;
}
/*
** The buffer pointed to by argument zNode (size nNode bytes) contains the
** root node of a b-tree segment. The segment is guaranteed to be at least
** one level high (i.e. the root node is not also a leaf). If successful,
** this function locates the leaf node of the segment that may contain the
** term specified by arguments zTerm and nTerm and writes its block number
** to *piLeaf.
** The buffer pointed to by argument zNode (size nNode bytes) contains an
** interior node of a b-tree segment. The zTerm buffer (size nTerm bytes)
** contains a term. This function searches the sub-tree headed by the zNode
** node for the range of leaf nodes that may contain the specified term
** or terms for which the specified term is a prefix.
**
** If piLeaf is not NULL, then *piLeaf is set to the blockid of the
** left-most leaf node in the tree that may contain the specified term.
** If piLeaf2 is not NULL, then *piLeaf2 is set to the blockid of the
** right-most leaf node that may contain a term for which the specified
** term is a prefix.
**
** It is possible that the returned leaf node does not contain the specified
** term. However, if the segment does contain said term, it is stored on
** the identified leaf node. Because this function only inspects interior
** segment nodes (and never loads leaf nodes into memory), it is not possible
** It is possible that the range of returned leaf nodes does not contain
** the specified term or any terms for which it is a prefix. However, if the
** segment does contain any such terms, they are stored within the identified
** range. Because this function only inspects interior segment nodes (and
** never loads leaf nodes into memory), it is not possible to be sure.
** to be sure.
**
** If an error occurs, an error code other than SQLITE_OK is returned.
*/
static int fts3SelectLeaf(
Fts3Table *p, /* Virtual table handle */
const char *zTerm, /* Term to select leaves for */
int nTerm, /* Size of term zTerm in bytes */
const char *zNode, /* Buffer containing segment interior node */
int nNode, /* Size of buffer at zNode */
sqlite3_int64 *piLeaf /* Selected leaf node */
sqlite3_int64 *piLeaf, /* Selected leaf node */
sqlite3_int64 *piLeaf2 /* Selected leaf node */
){
int rc = SQLITE_OK; /* Return code */
int rc; /* Return code */
const char *zCsr = zNode; /* Cursor to iterate through node */
const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
char *zBuffer = 0; /* Buffer to load terms into */
int nAlloc = 0; /* Size of allocated buffer */
int iHeight; /* Height of this node in tree */
while( 1 ){
assert( piLeaf || piLeaf2 );
int isFirstTerm = 1; /* True when processing first term on page */
int iHeight; /* Height of this node in tree */
sqlite3_int64 iChild; /* Block id of child node to descend to */
int nBlock; /* Size of child node in bytes */
zCsr += sqlite3Fts3GetVarint32(zCsr, &iHeight);
sqlite3Fts3GetVarint32(zNode, &iHeight);
zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
rc = fts3ScanInteriorNode(p, zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2);
assert( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) );
while( zCsr<zEnd ){
int cmp; /* memcmp() result */
int nSuffix; /* Size of term suffix */
int nPrefix = 0; /* Size of term prefix */
int nBuffer; /* Total term size */
if( rc==SQLITE_OK && iHeight>1 ){
char *zBlob = 0; /* Blob read from %_segments table */
int nBlob; /* Size of zBlob in bytes */
/* Load the next term on the node into zBuffer */
if( !isFirstTerm ){
zCsr += sqlite3Fts3GetVarint32(zCsr, &nPrefix);
if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){
rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob);
}
isFirstTerm = 0;
zCsr += sqlite3Fts3GetVarint32(zCsr, &nSuffix);
if( nPrefix+nSuffix>nAlloc ){
char *zNew;
nAlloc = (nPrefix+nSuffix) * 2;
zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
if( !zNew ){
sqlite3_free(zBuffer);
return SQLITE_NOMEM;
}
zBuffer = zNew;
}
if( rc==SQLITE_OK ){
rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0);
}
sqlite3_free(zBlob);
piLeaf = 0;
zBlob = 0;
}
memcpy(&zBuffer[nPrefix], zCsr, nSuffix);
nBuffer = nPrefix + nSuffix;
zCsr += nSuffix;
/* Compare the term we are searching for with the term just loaded from
** the interior node. If the specified term is greater than or equal
** to the term from the interior node, then all terms on the sub-tree
** headed by node iChild are smaller than zTerm. No need to search
** iChild.
**
** If the interior node term is larger than the specified term, then
** the tree headed by iChild may contain the specified term.
*/
cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer));
if( cmp<0 || (cmp==0 && nBuffer>nTerm) ) break;
if( rc==SQLITE_OK ){
iChild++;
};
rc = sqlite3Fts3ReadBlock(p, piLeaf ? *piLeaf : *piLeaf2, &zBlob, &nBlob);
}
/* If (iHeight==1), the children of this interior node are leaves. The
** specified term may be present on leaf node iChild.
*/
if( iHeight==1 ){
*piLeaf = iChild;
if( rc==SQLITE_OK ){
rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2);
break;
}
sqlite3_free(zBlob);
}
/* Descend to interior node iChild. */
rc = sqlite3Fts3ReadBlock(p, iChild, &zCsr, &nBlock);
if( rc!=SQLITE_OK ) break;
zEnd = &zCsr[nBlock];
}
sqlite3_free(zBuffer);
return rc;
}
/*
** This function is used to create delta-encoded serialized lists of FTS3
** varints. Each call to this function appends a single varint to a list.
*/
|
︙ | | |
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
|
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
|
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
+
-
-
+
+
-
+
+
+
+
|
*pp = p;
*pp1 = p1 + 1;
*pp2 = p2 + 1;
}
/*
** nToken==1 searches for adjacent positions.
**
** This function is used to merge two position lists into one. When it is
** called, *pp1 and *pp2 must both point to position lists. A position-list is
** the part of a doclist that follows each document id. For example, if a row
** contains:
**
** 'a b c'|'x y z'|'a b b a'
**
** Then the position list for this row for token 'b' would consist of:
**
** 0x02 0x01 0x02 0x03 0x03 0x00
**
** When this function returns, both *pp1 and *pp2 are left pointing to the
** byte following the 0x00 terminator of their respective position lists.
**
** If isSaveLeft is 0, an entry is added to the output position list for
** each position in *pp2 for which there exists one or more positions in
** *pp1 so that (pos(*pp2)>pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e.
** when the *pp1 token appears before the *pp2 token, but not more than nToken
** slots before it.
*/
static int fts3PoslistPhraseMerge(
char **pp, /* Output buffer */
char **pp, /* IN/OUT: Preallocated output buffer */
int nToken, /* Maximum difference in token positions */
int isSaveLeft, /* Save the left position */
int isExact, /* If *pp1 is exactly nTokens before *pp2 */
char **pp1, /* Left input list */
char **pp2 /* Right input list */
char **pp1, /* IN/OUT: Left input list */
char **pp2 /* IN/OUT: Right input list */
){
char *p = (pp ? *pp : 0);
char *p1 = *pp1;
char *p2 = *pp2;
int iCol1 = 0;
int iCol2 = 0;
/* Never set both isSaveLeft and isExact for the same invocation. */
assert( isSaveLeft==0 || isExact==0 );
assert( *p1!=0 && *p2!=0 );
if( *p1==POS_COLUMN ){
p1++;
p1 += sqlite3Fts3GetVarint32(p1, &iCol1);
}
if( *p2==POS_COLUMN ){
p2++;
|
︙ | | |
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
|
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
|
+
-
+
+
|
assert( *p1!=POS_END && *p1!=POS_COLUMN );
assert( *p2!=POS_END && *p2!=POS_COLUMN );
fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2;
fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2;
while( 1 ){
if( iPos2==iPos1+nToken
if( iPos2>iPos1 && iPos2<=iPos1+nToken ){
|| (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken)
){
sqlite3_int64 iSave;
if( !pp ){
fts3PoslistCopy(0, &p2);
fts3PoslistCopy(0, &p1);
*pp1 = p1;
*pp2 = p2;
return 1;
|
︙ | | |
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
|
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
|
-
+
-
+
-
+
-
+
|
char **pp1, /* IN/OUT: Left input list */
char **pp2 /* IN/OUT: Right input list */
){
char *p1 = *pp1;
char *p2 = *pp2;
if( !pp ){
if( fts3PoslistPhraseMerge(0, nRight, 0, pp1, pp2) ) return 1;
if( fts3PoslistPhraseMerge(0, nRight, 0, 0, pp1, pp2) ) return 1;
*pp1 = p1;
*pp2 = p2;
return fts3PoslistPhraseMerge(0, nLeft, 0, pp2, pp1);
return fts3PoslistPhraseMerge(0, nLeft, 0, 0, pp2, pp1);
}else{
char *pTmp1 = aTmp;
char *pTmp2;
char *aTmp2;
int res = 1;
fts3PoslistPhraseMerge(&pTmp1, nRight, 0, pp1, pp2);
fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2);
aTmp2 = pTmp2 = pTmp1;
*pp1 = p1;
*pp2 = p2;
fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, pp2, pp1);
fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1);
if( pTmp1!=aTmp && pTmp2!=aTmp2 ){
fts3PoslistMerge(pp, &aTmp, &aTmp2);
}else if( pTmp1!=aTmp ){
fts3PoslistCopy(pp, &aTmp);
}else if( pTmp2!=aTmp2 ){
fts3PoslistCopy(pp, &aTmp2);
}else{
|
︙ | | |
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
|
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
|
-
+
+
+
|
int nParam1, /* Used by MERGE_NEAR and MERGE_POS_NEAR */
int nParam2, /* Used by MERGE_NEAR and MERGE_POS_NEAR */
char *aBuffer, /* Pre-allocated output buffer */
int *pnBuffer, /* OUT: Bytes written to aBuffer */
char *a1, /* Buffer containing first doclist */
int n1, /* Size of buffer a1 */
char *a2, /* Buffer containing second doclist */
int n2 /* Size of buffer a2 */
int n2, /* Size of buffer a2 */
int *pnDoc /* OUT: Number of docids in output */
){
sqlite3_int64 i1 = 0;
sqlite3_int64 i2 = 0;
sqlite3_int64 iPrev = 0;
char *p = aBuffer;
char *p1 = a1;
char *p2 = a2;
char *pEnd1 = &a1[n1];
char *pEnd2 = &a2[n2];
int nDoc = 0;
assert( mergetype==MERGE_OR || mergetype==MERGE_POS_OR
|| mergetype==MERGE_AND || mergetype==MERGE_NOT
|| mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE
|| mergetype==MERGE_NEAR || mergetype==MERGE_POS_NEAR
);
|
︙ | | |
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
|
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
|
+
|
case MERGE_AND:
while( p1 && p2 ){
if( i1==i2 ){
fts3PutDeltaVarint(&p, &iPrev, i1);
fts3GetDeltaVarint2(&p1, pEnd1, &i1);
fts3GetDeltaVarint2(&p2, pEnd2, &i2);
nDoc++;
}else if( i1<i2 ){
fts3GetDeltaVarint2(&p1, pEnd1, &i1);
}else{
fts3GetDeltaVarint2(&p2, pEnd2, &i2);
}
}
break;
|
︙ | | |
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
|
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
|
-
+
+
+
|
case MERGE_PHRASE: {
char **ppPos = (mergetype==MERGE_PHRASE ? 0 : &p);
while( p1 && p2 ){
if( i1==i2 ){
char *pSave = p;
sqlite3_int64 iPrevSave = iPrev;
fts3PutDeltaVarint(&p, &iPrev, i1);
if( 0==fts3PoslistPhraseMerge(ppPos, 1, 0, &p1, &p2) ){
if( 0==fts3PoslistPhraseMerge(ppPos, nParam1, 0, 1, &p1, &p2) ){
p = pSave;
iPrev = iPrevSave;
}else{
nDoc++;
}
fts3GetDeltaVarint2(&p1, pEnd1, &i1);
fts3GetDeltaVarint2(&p2, pEnd2, &i2);
}else if( i1<i2 ){
fts3PoslistCopy(0, &p1);
fts3GetDeltaVarint2(&p1, pEnd1, &i1);
}else{
|
︙ | | |
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
|
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
|
+
|
}
}
sqlite3_free(aTmp);
break;
}
}
if( pnDoc ) *pnDoc = nDoc;
*pnBuffer = (int)(p-aBuffer);
return SQLITE_OK;
}
/*
** A pointer to an instance of this structure is used as the context
** argument to sqlite3Fts3SegReaderIterate()
|
︙ | | |
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
|
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
|
-
+
-
+
|
** into a single doclist.
*/
for(i=0; i<SizeofArray(pTS->aaOutput); i++){
if( pTS->aaOutput[i] ){
if( !aOut ){
aOut = pTS->aaOutput[i];
nOut = pTS->anOutput[i];
pTS->aaOutput[0] = 0;
pTS->aaOutput[i] = 0;
}else{
int nNew = nOut + pTS->anOutput[i];
char *aNew = sqlite3_malloc(nNew);
if( !aNew ){
sqlite3_free(aOut);
return SQLITE_NOMEM;
}
fts3DoclistMerge(mergetype, 0, 0,
aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut
aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, 0
);
sqlite3_free(pTS->aaOutput[i]);
sqlite3_free(aOut);
pTS->aaOutput[i] = 0;
aOut = aNew;
nOut = nNew;
}
|
︙ | | |
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
|
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
|
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
-
-
-
-
-
-
-
-
-
+
-
+
-
-
+
+
-
-
+
+
+
+
+
+
+
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
+
+
+
+
+
+
-
+
+
+
+
+
+
-
-
-
+
+
+
+
+
+
+
-
-
-
-
+
+
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
+
+
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
-
-
+
+
-
+
+
+
+
+
+
+
+
+
+
+
-
+
+
+
+
+
+
-
-
+
+
|
aNew = sqlite3_malloc(nNew);
if( !aNew ){
if( aMerge!=aDoclist ){
sqlite3_free(aMerge);
}
return SQLITE_NOMEM;
}
fts3DoclistMerge(mergetype, 0, 0,
aNew, &nNew, pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge
fts3DoclistMerge(mergetype, 0, 0, aNew, &nNew,
pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge, 0
);
if( iOut>0 ) sqlite3_free(aMerge);
sqlite3_free(pTS->aaOutput[iOut]);
pTS->aaOutput[iOut] = 0;
aMerge = aNew;
nMerge = nNew;
if( (iOut+1)==SizeofArray(pTS->aaOutput) ){
pTS->aaOutput[iOut] = aMerge;
pTS->anOutput[iOut] = nMerge;
}
}
}
return SQLITE_OK;
}
static int fts3DeferredTermSelect(
Fts3DeferredToken *pToken, /* Phrase token */
int isTermPos, /* True to include positions */
int *pnOut, /* OUT: Size of list */
char **ppOut /* OUT: Body of list */
){
char *aSource;
int nSource;
aSource = sqlite3Fts3DeferredDoclist(pToken, &nSource);
if( !aSource ){
*pnOut = 0;
*ppOut = 0;
}else if( isTermPos ){
*ppOut = sqlite3_malloc(nSource);
if( !*ppOut ) return SQLITE_NOMEM;
memcpy(*ppOut, aSource, nSource);
*pnOut = nSource;
}else{
sqlite3_int64 docid;
*pnOut = sqlite3Fts3GetVarint(aSource, &docid);
*ppOut = sqlite3_malloc(*pnOut);
if( !*ppOut ) return SQLITE_NOMEM;
sqlite3Fts3PutVarint(*ppOut, docid);
}
return SQLITE_OK;
}
/*
** An Fts3SegReaderArray is used to store an array of Fts3SegReader objects.
** Elements are added to the array using fts3SegReaderArrayAdd().
*/
struct Fts3SegReaderArray {
int nSegment; /* Number of valid entries in apSegment[] */
int nAlloc; /* Allocated size of apSegment[] */
int nCost; /* The cost of executing SegReaderIterate() */
Fts3SegReader *apSegment[1]; /* Array of seg-reader objects */
};
/*
** Free an Fts3SegReaderArray object. Also free all seg-readers in the
** array (using sqlite3Fts3SegReaderFree()).
*/
static void fts3SegReaderArrayFree(Fts3SegReaderArray *pArray){
if( pArray ){
int i;
for(i=0; i<pArray->nSegment; i++){
sqlite3Fts3SegReaderFree(0, pArray->apSegment[i]);
}
sqlite3_free(pArray);
}
}
static int fts3SegReaderArrayAdd(
Fts3SegReaderArray **ppArray,
Fts3SegReader *pNew
){
Fts3SegReaderArray *pArray = *ppArray;
if( !pArray || pArray->nAlloc==pArray->nSegment ){
int nNew = (pArray ? pArray->nAlloc+16 : 16);
pArray = (Fts3SegReaderArray *)sqlite3_realloc(pArray,
sizeof(Fts3SegReaderArray) + (nNew-1) * sizeof(Fts3SegReader*)
);
if( !pArray ){
sqlite3Fts3SegReaderFree(0, pNew);
return SQLITE_NOMEM;
}
if( nNew==16 ){
pArray->nSegment = 0;
pArray->nCost = 0;
}
pArray->nAlloc = nNew;
*ppArray = pArray;
}
pArray->apSegment[pArray->nSegment++] = pNew;
return SQLITE_OK;
}
static int fts3TermSegReaderArray(
Fts3Cursor *pCsr, /* Virtual table cursor handle */
const char *zTerm, /* Term to query for */
int nTerm, /* Size of zTerm in bytes */
int isPrefix, /* True for a prefix search */
Fts3SegReaderArray **ppArray /* OUT: Allocated seg-reader array */
){
Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
int rc; /* Return code */
Fts3SegReaderArray *pArray = 0; /* Array object to build */
Fts3SegReader *pReader = 0; /* Seg-reader to add to pArray */
sqlite3_stmt *pStmt = 0; /* SQL statement to scan %_segdir table */
int iAge = 0; /* Used to assign ages to segments */
/* Allocate a seg-reader to scan the pending terms, if any. */
rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &pReader);
if( rc==SQLITE_OK && pReader ) {
rc = fts3SegReaderArrayAdd(&pArray, pReader);
}
/* Loop through the entire %_segdir table. For each segment, create a
** Fts3SegReader to iterate through the subset of the segment leaves
** that may contain a term that matches zTerm/nTerm. For non-prefix
** searches, this is always a single leaf. For prefix searches, this
** may be a contiguous block of leaves.
*/
if( rc==SQLITE_OK ){
rc = sqlite3Fts3AllSegdirs(p, &pStmt);
}
while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
Fts3SegReader *pNew = 0;
int nRoot = sqlite3_column_bytes(pStmt, 4);
char const *zRoot = sqlite3_column_blob(pStmt, 4);
if( sqlite3_column_int64(pStmt, 1)==0 ){
/* The entire segment is stored on the root node (which must be a
** leaf). Do not bother inspecting any data in this case, just
** create a Fts3SegReader to scan the single leaf.
*/
rc = sqlite3Fts3SegReaderNew(p, iAge, 0, 0, 0, zRoot, nRoot, &pNew);
}else{
sqlite3_int64 i1; /* First leaf that may contain zTerm */
sqlite3_int64 i2; /* Final leaf that may contain zTerm */
rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1, (isPrefix?&i2:0));
if( isPrefix==0 ) i2 = i1;
if( rc==SQLITE_OK ){
rc = sqlite3Fts3SegReaderNew(p, iAge, i1, i2, 0, 0, 0, &pNew);
}
}
assert( (pNew==0)==(rc!=SQLITE_OK) );
/* If a new Fts3SegReader was allocated, add it to the array. */
if( rc==SQLITE_OK ){
rc = fts3SegReaderArrayAdd(&pArray, pNew);
}
if( rc==SQLITE_OK ){
rc = sqlite3Fts3SegReaderCost(pCsr, pNew, &pArray->nCost);
}
iAge++;
}
if( rc==SQLITE_DONE ){
rc = sqlite3_reset(pStmt);
}else{
sqlite3_reset(pStmt);
}
if( rc!=SQLITE_OK ){
fts3SegReaderArrayFree(pArray);
pArray = 0;
}
*ppArray = pArray;
return rc;
}
/*
** This function retreives the doclist for the specified term (or term
** prefix) from the database.
**
** The returned doclist may be in one of two formats, depending on the
** value of parameter isReqPos. If isReqPos is zero, then the doclist is
** a sorted list of delta-compressed docids (a bare doclist). If isReqPos
** is non-zero, then the returned list is in the same format as is stored
** in the database without the found length specifier at the start of on-disk
** doclists.
*/
static int fts3TermSelect(
Fts3Table *p, /* Virtual table handle */
Fts3PhraseToken *pTok, /* Token to query for */
int iColumn, /* Column to query (or -ve for all columns) */
const char *zTerm, /* Term to query for */
int nTerm, /* Size of zTerm in bytes */
int isPrefix, /* True for a prefix search */
int isReqPos, /* True to include position lists in output */
int *pnOut, /* OUT: Size of buffer at *ppOut */
char **ppOut /* OUT: Malloced result buffer */
){
int i;
TermSelect tsc;
Fts3SegFilter filter; /* Segment term filter configuration */
Fts3SegReader **apSegment; /* Array of segments to read data from */
int nSegment = 0; /* Size of apSegment array */
int nAlloc = 16; /* Allocated size of segment array */
int rc; /* Return code */
sqlite3_stmt *pStmt = 0; /* SQL statement to scan %_segdir table */
int iAge = 0; /* Used to assign ages to segments */
apSegment = (Fts3SegReader **)sqlite3_malloc(sizeof(Fts3SegReader*)*nAlloc);
if( !apSegment ) return SQLITE_NOMEM;
rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &apSegment[0]);
if( rc!=SQLITE_OK ) goto finished;
if( apSegment[0] ){
nSegment = 1;
}
/* Loop through the entire %_segdir table. For each segment, create a
** Fts3SegReader to iterate through the subset of the segment leaves
** that may contain a term that matches zTerm/nTerm. For non-prefix
** searches, this is always a single leaf. For prefix searches, this
** may be a contiguous block of leaves.
**
** The code in this loop does not actually load any leaves into memory
** (unless the root node happens to be a leaf). It simply examines the
** b-tree structure to determine which leaves need to be inspected.
*/
rc = sqlite3Fts3AllSegdirs(p, &pStmt);
while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
Fts3SegReader *pNew = 0;
Fts3SegReaderArray *pArray; /* Seg-reader array for this term */
int nRoot = sqlite3_column_bytes(pStmt, 4);
char const *zRoot = sqlite3_column_blob(pStmt, 4);
if( sqlite3_column_int64(pStmt, 1)==0 ){
/* The entire segment is stored on the root node (which must be a
** leaf). Do not bother inspecting any data in this case, just
** create a Fts3SegReader to scan the single leaf.
*/
rc = sqlite3Fts3SegReaderNew(p, iAge, 0, 0, 0, zRoot, nRoot, &pNew);
}else{
int rc2; /* Return value of sqlite3Fts3ReadBlock() */
sqlite3_int64 i1; /* Blockid of leaf that may contain zTerm */
rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1);
if( rc==SQLITE_OK ){
sqlite3_int64 i2 = sqlite3_column_int64(pStmt, 2);
rc = sqlite3Fts3SegReaderNew(p, iAge, i1, i2, 0, 0, 0, &pNew);
}
TermSelect tsc; /* Context object for fts3TermSelectCb() */
/* The following call to ReadBlock() serves to reset the SQL statement
** used to retrieve blocks of data from the %_segments table. If it is
** not reset here, then it may remain classified as an active statement
** by SQLite, which may lead to "DROP TABLE" or "DETACH" commands
** failing.
*/
rc2 = sqlite3Fts3ReadBlock(p, 0, 0, 0);
if( rc==SQLITE_OK ){
rc = rc2;
}
}
iAge++;
Fts3SegFilter filter; /* Segment term filter configuration */
/* If a new Fts3SegReader was allocated, add it to the apSegment array. */
assert( pNew!=0 || rc!=SQLITE_OK );
if( pNew ){
if( nSegment==nAlloc ){
Fts3SegReader **pArray;
nAlloc += 16;
pArray = (Fts3SegReader **)sqlite3_realloc(
apSegment, nAlloc*sizeof(Fts3SegReader *)
);
if( !pArray ){
sqlite3Fts3SegReaderFree(p, pNew);
rc = SQLITE_NOMEM;
goto finished;
}
apSegment = pArray;
}
apSegment[nSegment++] = pNew;
}
}
if( rc!=SQLITE_DONE ){
assert( rc!=SQLITE_OK );
goto finished;
}
pArray = pTok->pArray;
memset(&tsc, 0, sizeof(TermSelect));
tsc.isReqPos = isReqPos;
filter.flags = FTS3_SEGMENT_IGNORE_EMPTY
| (isPrefix ? FTS3_SEGMENT_PREFIX : 0)
| (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0)
| (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0)
| (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0);
filter.iCol = iColumn;
filter.zTerm = zTerm;
filter.nTerm = nTerm;
filter.zTerm = pTok->z;
filter.nTerm = pTok->n;
rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment, &filter,
fts3TermSelectCb, (void *)&tsc
rc = sqlite3Fts3SegReaderIterate(p, pArray->apSegment, pArray->nSegment,
&filter, fts3TermSelectCb, (void *)&tsc
);
if( rc==SQLITE_OK ){
rc = fts3TermSelectMerge(&tsc);
}
if( rc==SQLITE_OK ){
*ppOut = tsc.aaOutput[0];
*pnOut = tsc.anOutput[0];
}else{
int i;
for(i=0; i<SizeofArray(tsc.aaOutput); i++){
sqlite3_free(tsc.aaOutput[i]);
}
}
fts3SegReaderArrayFree(pArray);
pTok->pArray = 0;
return rc;
}
finished:
sqlite3_reset(pStmt);
for(i=0; i<nSegment; i++){
sqlite3Fts3SegReaderFree(p, apSegment[i]);
}
sqlite3_free(apSegment);
/*
** This function counts the total number of docids in the doclist stored
** in buffer aList[], size nList bytes.
**
** If the isPoslist argument is true, then it is assumed that the doclist
** contains a position-list following each docid. Otherwise, it is assumed
** that the doclist is simply a list of docids stored as delta encoded
** varints.
*/
static int fts3DoclistCountDocids(int isPoslist, char *aList, int nList){
int nDoc = 0; /* Return value */
if( aList ){
char *aEnd = &aList[nList]; /* Pointer to one byte after EOF */
char *p = aList; /* Cursor */
if( !isPoslist ){
/* The number of docids in the list is the same as the number of
** varints. In FTS3 a varint consists of a single byte with the 0x80
** bit cleared and zero or more bytes with the 0x80 bit set. So to
** count the varints in the buffer, just count the number of bytes
** with the 0x80 bit clear. */
while( p<aEnd ) nDoc += (((*p++)&0x80)==0);
}else{
while( p<aEnd ){
nDoc++;
while( (*p++)&0x80 ); /* Skip docid varint */
fts3PoslistCopy(0, &p); /* Skip over position list */
}
}
}
return nDoc;
}
/*
** Call sqlite3Fts3DeferToken() for each token in the expression pExpr.
*/
static int fts3DeferExpression(Fts3Cursor *pCsr, Fts3Expr *pExpr){
int rc = SQLITE_OK;
if( pExpr ){
rc = fts3DeferExpression(pCsr, pExpr->pLeft);
if( rc==SQLITE_OK ){
rc = fts3DeferExpression(pCsr, pExpr->pRight);
}
if( pExpr->eType==FTSQUERY_PHRASE ){
int iCol = pExpr->pPhrase->iColumn;
int i;
for(i=0; rc==SQLITE_OK && i<pExpr->pPhrase->nToken; i++){
Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i];
if( pToken->pDeferred==0 ){
rc = sqlite3Fts3DeferToken(pCsr, pToken, iCol);
}
}
}
}
return rc;
}
/*
** This function removes the position information from a doclist. When
** called, buffer aList (size *pnList bytes) contains a doclist that includes
** position information. This function removes the position information so
** that aList contains only docids, and adjusts *pnList to reflect the new
** (possibly reduced) size of the doclist.
*/
static void fts3DoclistStripPositions(
char *aList, /* IN/OUT: Buffer containing doclist */
int *pnList /* IN/OUT: Size of doclist in bytes */
){
if( aList ){
char *aEnd = &aList[*pnList]; /* Pointer to one byte after EOF */
char *p = aList; /* Input cursor */
char *pOut = aList; /* Output cursor */
while( p<aEnd ){
sqlite3_int64 delta;
p += sqlite3Fts3GetVarint(p, &delta);
fts3PoslistCopy(0, &p);
pOut += sqlite3Fts3PutVarint(pOut, delta);
}
*pnList = (pOut - aList);
}
}
/*
** Return a DocList corresponding to the phrase *pPhrase.
**
** If this function returns SQLITE_OK, but *pnOut is set to a negative value,
** then no tokens in the phrase were looked up in the full-text index. This
** is only possible when this function is called from within xFilter(). The
** caller should assume that all documents match the phrase. The actual
** filtering will take place in xNext().
*/
static int fts3PhraseSelect(
Fts3Table *p, /* Virtual table handle */
Fts3Cursor *pCsr, /* Virtual table cursor handle */
Fts3Phrase *pPhrase, /* Phrase to return a doclist for */
int isReqPos, /* True if output should contain positions */
char **paOut, /* OUT: Pointer to malloc'd result buffer */
int *pnOut /* OUT: Size of buffer at *paOut */
){
char *pOut = 0;
int nOut = 0;
int rc = SQLITE_OK;
int ii;
int iCol = pPhrase->iColumn;
int isTermPos = (pPhrase->nToken>1 || isReqPos);
Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
int isFirst = 1;
int iPrevTok = 0;
int nDoc = 0;
/* If this is an xFilter() evaluation, create a segment-reader for each
** phrase token. Or, if this is an xNext() or snippet/offsets/matchinfo
** evaluation, only create segment-readers if there are no Fts3DeferredToken
** objects attached to the phrase-tokens.
*/
for(ii=0; ii<pPhrase->nToken; ii++){
struct PhraseToken *pTok = &pPhrase->aToken[ii];
char *z = pTok->z; /* Next token of the phrase */
int n = pTok->n; /* Size of z in bytes */
int isPrefix = pTok->isPrefix;/* True if token is a prefix */
Fts3PhraseToken *pTok = &pPhrase->aToken[ii];
if( pTok->pArray==0 ){
if( (pCsr->eEvalmode==FTS3_EVAL_FILTER)
|| (pCsr->eEvalmode==FTS3_EVAL_NEXT && pCsr->pDeferred==0)
|| (pCsr->eEvalmode==FTS3_EVAL_MATCHINFO && pTok->bFulltext)
){
rc = fts3TermSegReaderArray(
pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pArray
);
if( rc!=SQLITE_OK ) return rc;
}
}
}
for(ii=0; ii<pPhrase->nToken; ii++){
Fts3PhraseToken *pTok; /* Token to find doclist for */
int iTok; /* The token being queried this iteration */
char *pList; /* Pointer to token doclist */
int nList; /* Size of buffer at pList */
/* Select a token to process. If this is an xFilter() call, then tokens
** are processed in order from least to most costly. Otherwise, tokens
** are processed in the order in which they occur in the phrase.
*/
if( pCsr->eEvalmode==FTS3_EVAL_MATCHINFO ){
assert( isReqPos );
iTok = ii;
pTok = &pPhrase->aToken[iTok];
if( pTok->bFulltext==0 ) continue;
}else if( pCsr->eEvalmode==FTS3_EVAL_NEXT || isReqPos ){
iTok = ii;
pTok = &pPhrase->aToken[iTok];
}else{
int nMinCost = 0x7FFFFFFF;
int jj;
/* Find the remaining token with the lowest cost. */
for(jj=0; jj<pPhrase->nToken; jj++){
Fts3SegReaderArray *pArray = pPhrase->aToken[jj].pArray;
if( pArray && pArray->nCost<nMinCost ){
iTok = jj;
nMinCost = pArray->nCost;
}
}
pTok = &pPhrase->aToken[iTok];
/* This branch is taken if it is determined that loading the doclist
** for the next token would require more IO than loading all documents
** currently identified by doclist pOut/nOut. No further doclists will
** be loaded from the full-text index for this phrase.
*/
if( nMinCost>nDoc && ii>0 ){
rc = fts3DeferExpression(pCsr, pCsr->pExpr);
break;
}
}
if( pCsr->eEvalmode==FTS3_EVAL_NEXT && pTok->pDeferred ){
rc = fts3TermSelect(p, iCol, z, n, isPrefix, isTermPos, &nList, &pList);
rc = fts3DeferredTermSelect(pTok->pDeferred, isTermPos, &nList, &pList);
}else{
assert( pTok->pArray );
rc = fts3TermSelect(p, pTok, iCol, isTermPos, &nList, &pList);
pTok->bFulltext = 1;
}
assert( rc!=SQLITE_OK || pCsr->eEvalmode || pTok->pArray==0 );
if( rc!=SQLITE_OK ) break;
if( ii==0 ){
if( isFirst ){
pOut = pList;
nOut = nList;
if( pCsr->eEvalmode==FTS3_EVAL_FILTER && pPhrase->nToken>1 ){
nDoc = fts3DoclistCountDocids(1, pOut, nOut);
}
isFirst = 0;
iPrevTok = iTok;
}else{
/* Merge the new term list and the current output. If this is the
** last term in the phrase, and positions are not required in the
** output of this function, the positions can be dropped as part
/* Merge the new term list and the current output. */
char *aLeft, *aRight;
int nLeft, nRight;
int nDist;
int mt;
/* If this is the final token of the phrase, and positions were not
** of this merge. Either way, the result of this merge will be
** smaller than nList bytes. The code in fts3DoclistMerge() is written
** so that it is safe to use pList as the output as well as an input
** in this case.
** requested by the caller, use MERGE_PHRASE instead of POS_PHRASE.
** This drops the position information from the output list.
*/
int mergetype = MERGE_POS_PHRASE;
if( ii==pPhrase->nToken-1 && !isReqPos ){
mergetype = MERGE_PHRASE;
mt = MERGE_POS_PHRASE;
if( ii==pPhrase->nToken-1 && !isReqPos ) mt = MERGE_PHRASE;
assert( iPrevTok!=iTok );
if( iPrevTok<iTok ){
aLeft = pOut;
nLeft = nOut;
aRight = pList;
nRight = nList;
nDist = iTok-iPrevTok;
iPrevTok = iTok;
}else{
aRight = pOut;
nRight = nOut;
aLeft = pList;
nLeft = nList;
nDist = iPrevTok-iTok;
}
pOut = aRight;
fts3DoclistMerge(mergetype, 0, 0, pList, &nOut, pOut, nOut, pList, nList);
sqlite3_free(pOut);
fts3DoclistMerge(
mt, nDist, 0, pOut, &nOut, aLeft, nLeft, aRight, nRight, &nDoc
);
sqlite3_free(aLeft);
pOut = pList;
}
assert( nOut==0 || pOut!=0 );
}
if( rc==SQLITE_OK ){
if( ii!=pPhrase->nToken ){
assert( pCsr->eEvalmode==FTS3_EVAL_FILTER && isReqPos==0 );
fts3DoclistStripPositions(pOut, &nOut);
}
*paOut = pOut;
*pnOut = nOut;
}else{
sqlite3_free(pOut);
}
return rc;
}
/*
** This function merges two doclists according to the requirements of a
** NEAR operator.
**
** Both input doclists must include position information. The output doclist
** includes position information if the first argument to this function
** is MERGE_POS_NEAR, or does not if it is MERGE_NEAR.
*/
static int fts3NearMerge(
int mergetype, /* MERGE_POS_NEAR or MERGE_NEAR */
int nNear, /* Parameter to NEAR operator */
int nTokenLeft, /* Number of tokens in LHS phrase arg */
char *aLeft, /* Doclist for LHS (incl. positions) */
int nLeft, /* Size of LHS doclist in bytes */
int nTokenRight, /* As nTokenLeft */
char *aRight, /* As aLeft */
int nRight, /* As nRight */
char **paOut, /* OUT: Results of merge (malloced) */
int *pnOut /* OUT: Sized of output buffer */
){
char *aOut;
int rc;
char *aOut; /* Buffer to write output doclist to */
int rc; /* Return code */
assert( mergetype==MERGE_POS_NEAR || MERGE_NEAR );
aOut = sqlite3_malloc(nLeft+nRight+1);
if( aOut==0 ){
rc = SQLITE_NOMEM;
}else{
rc = fts3DoclistMerge(mergetype, nNear+nTokenRight, nNear+nTokenLeft,
aOut, pnOut, aLeft, nLeft, aRight, nRight
aOut, pnOut, aLeft, nLeft, aRight, nRight, 0
);
if( rc!=SQLITE_OK ){
sqlite3_free(aOut);
aOut = 0;
}
}
*paOut = aOut;
return rc;
}
/*
** This function is used as part of the processing for the snippet() and
** offsets() functions.
**
** Both pLeft and pRight are expression nodes of type FTSQUERY_PHRASE. Both
** have their respective doclists (including position information) loaded
** in Fts3Expr.aDoclist/nDoclist. This function removes all entries from
** each doclist that are not within nNear tokens of a corresponding entry
** in the other doclist.
*/
int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){
int rc;
int rc; /* Return code */
assert( pLeft->eType==FTSQUERY_PHRASE );
assert( pRight->eType==FTSQUERY_PHRASE );
assert( pLeft->isLoaded && pRight->isLoaded );
if( pLeft->aDoclist==0 || pRight->aDoclist==0 ){
sqlite3_free(pLeft->aDoclist);
sqlite3_free(pRight->aDoclist);
pRight->aDoclist = 0;
pLeft->aDoclist = 0;
rc = SQLITE_OK;
}else{
char *aOut;
int nOut;
char *aOut; /* Buffer in which to assemble new doclist */
int nOut; /* Size of buffer aOut in bytes */
rc = fts3NearMerge(MERGE_POS_NEAR, nNear,
pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist,
pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist,
&aOut, &nOut
);
if( rc!=SQLITE_OK ) return rc;
|
︙ | | |
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
|
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
|
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
+
+
-
-
-
+
+
+
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
-
+
-
|
sqlite3_free(pLeft->aDoclist);
pLeft->aDoclist = aOut;
pLeft->nDoclist = nOut;
}
return rc;
}
/*
** Allocate an Fts3SegReaderArray for each token in the expression pExpr.
** The allocated objects are stored in the Fts3PhraseToken.pArray member
** variables of each token structure.
*/
static int fts3ExprAllocateSegReaders(
Fts3Cursor *pCsr, /* FTS3 table */
Fts3Expr *pExpr, /* Expression to create seg-readers for */
int *pnExpr /* OUT: Number of AND'd expressions */
){
int rc = SQLITE_OK; /* Return code */
assert( pCsr->eEvalmode==FTS3_EVAL_FILTER );
if( pnExpr && pExpr->eType!=FTSQUERY_AND ){
(*pnExpr)++;
pnExpr = 0;
}
if( pExpr->eType==FTSQUERY_PHRASE ){
Fts3Phrase *pPhrase = pExpr->pPhrase;
int ii;
for(ii=0; rc==SQLITE_OK && ii<pPhrase->nToken; ii++){
Fts3PhraseToken *pTok = &pPhrase->aToken[ii];
if( pTok->pArray==0 ){
rc = fts3TermSegReaderArray(
pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pArray
);
}
}
}else{
rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pLeft, pnExpr);
if( rc==SQLITE_OK ){
rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pRight, pnExpr);
}
}
return rc;
}
/*
** Free the Fts3SegReaderArray objects associated with each token in the
** expression pExpr. In other words, this function frees the resources
** allocated by fts3ExprAllocateSegReaders().
*/
static void fts3ExprFreeSegReaders(Fts3Expr *pExpr){
if( pExpr ){
Fts3Phrase *pPhrase = pExpr->pPhrase;
if( pPhrase ){
int kk;
for(kk=0; kk<pPhrase->nToken; kk++){
fts3SegReaderArrayFree(pPhrase->aToken[kk].pArray);
pPhrase->aToken[kk].pArray = 0;
}
}
fts3ExprFreeSegReaders(pExpr->pLeft);
fts3ExprFreeSegReaders(pExpr->pRight);
}
}
/*
** Return the sum of the costs of all tokens in the expression pExpr. This
** function must be called after Fts3SegReaderArrays have been allocated
** for all tokens using fts3ExprAllocateSegReaders().
*/
int fts3ExprCost(Fts3Expr *pExpr){
int nCost; /* Return value */
if( pExpr->eType==FTSQUERY_PHRASE ){
Fts3Phrase *pPhrase = pExpr->pPhrase;
int ii;
nCost = 0;
for(ii=0; ii<pPhrase->nToken; ii++){
nCost += pPhrase->aToken[ii].pArray->nCost;
}
}else{
nCost = fts3ExprCost(pExpr->pLeft) + fts3ExprCost(pExpr->pRight);
}
return nCost;
}
/*
** The following is a helper function (and type) for fts3EvalExpr(). It
** must be called after Fts3SegReaders have been allocated for every token
** in the expression. See the context it is called from in fts3EvalExpr()
** for further explanation.
*/
typedef struct ExprAndCost ExprAndCost;
struct ExprAndCost {
Fts3Expr *pExpr;
int nCost;
};
static void fts3ExprAssignCosts(
Fts3Expr *pExpr, /* Expression to create seg-readers for */
ExprAndCost **ppExprCost /* OUT: Write to *ppExprCost */
){
if( pExpr->eType==FTSQUERY_AND ){
fts3ExprAssignCosts(pExpr->pLeft, ppExprCost);
fts3ExprAssignCosts(pExpr->pRight, ppExprCost);
}else{
(*ppExprCost)->pExpr = pExpr;
(*ppExprCost)->nCost = fts3ExprCost(pExpr);;
(*ppExprCost)++;
}
}
/*
** Evaluate the full-text expression pExpr against fts3 table pTab. Store
** the resulting doclist in *paOut and *pnOut. This routine mallocs for
** the space needed to store the output. The caller is responsible for
** Evaluate the full-text expression pExpr against FTS3 table pTab. Store
** the resulting doclist in *paOut and *pnOut. This routine mallocs for
** the space needed to store the output. The caller is responsible for
** freeing the space when it has finished.
**
** This function is called in two distinct contexts:
**
** * From within the virtual table xFilter() method. In this case, the
** output doclist contains entries for all rows in the table, based on
** data read from the full-text index.
**
** In this case, if the query expression contains one or more tokens that
** are very common, then the returned doclist may contain a superset of
** the documents that actually match the expression.
**
** * From within the virtual table xNext() method. This call is only made
** if the call from within xFilter() found that there were very common
** tokens in the query expression and did return a superset of the
** matching documents. In this case the returned doclist contains only
** entries that correspond to the current row of the table. Instead of
** reading the data for each token from the full-text index, the data is
** already available in-memory in the Fts3PhraseToken.pDeferred structures.
** See fts3EvalDeferred() for how it gets there.
**
** In the first case above, Fts3Cursor.doDeferred==0. In the second (if it is
** required) Fts3Cursor.doDeferred==1.
**
** If the SQLite invokes the snippet(), offsets() or matchinfo() function
** as part of a SELECT on an FTS3 table, this function is called on each
** individual phrase expression in the query. If there were very common tokens
** found in the xFilter() call, then this function is called once for phrase
** for each row visited, and the returned doclist contains entries for the
** current row only. Otherwise, if there were no very common tokens, then this
** function is called once only for each phrase in the query and the returned
** doclist contains entries for all rows of the table.
**
** Fts3Cursor.doDeferred==1 when this function is called on phrases as a
** result of a snippet(), offsets() or matchinfo() invocation.
*/
static int evalFts3Expr(
Fts3Table *p, /* Virtual table handle */
static int fts3EvalExpr(
Fts3Cursor *p, /* Virtual table cursor handle */
Fts3Expr *pExpr, /* Parsed fts3 expression */
char **paOut, /* OUT: Pointer to malloc'd result buffer */
int *pnOut, /* OUT: Size of buffer at *paOut */
int isReqPos /* Require positions in output buffer */
){
int rc = SQLITE_OK; /* Return code */
/* Zero the output parameters. */
*paOut = 0;
*pnOut = 0;
if( pExpr ){
assert( pExpr->eType==FTSQUERY_PHRASE
|| pExpr->eType==FTSQUERY_NEAR
|| isReqPos==0
assert( pExpr->eType==FTSQUERY_NEAR || pExpr->eType==FTSQUERY_OR
|| pExpr->eType==FTSQUERY_AND || pExpr->eType==FTSQUERY_NOT
|| pExpr->eType==FTSQUERY_PHRASE
);
assert( pExpr->eType==FTSQUERY_PHRASE || isReqPos==0 );
if( pExpr->eType==FTSQUERY_PHRASE ){
rc = fts3PhraseSelect(p, pExpr->pPhrase,
rc = fts3PhraseSelect(p, pExpr->pPhrase,
isReqPos || (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR),
paOut, pnOut
);
fts3ExprFreeSegReaders(pExpr);
}else if( p->eEvalmode==FTS3_EVAL_FILTER && pExpr->eType==FTSQUERY_AND ){
ExprAndCost *aExpr = 0; /* Array of AND'd expressions and costs */
int nExpr = 0; /* Size of aExpr[] */
char *aRet = 0; /* Doclist to return to caller */
int nRet = 0; /* Length of aRet[] in bytes */
int nDoc = 0x7FFFFFFF;
assert( !isReqPos );
rc = fts3ExprAllocateSegReaders(p, pExpr, &nExpr);
if( rc==SQLITE_OK ){
assert( nExpr>1 );
aExpr = sqlite3_malloc(sizeof(ExprAndCost) * nExpr);
if( !aExpr ) rc = SQLITE_NOMEM;
}
if( rc==SQLITE_OK ){
int ii; /* Used to iterate through expressions */
fts3ExprAssignCosts(pExpr, &aExpr);
aExpr -= nExpr;
for(ii=0; ii<nExpr; ii++){
char *aNew;
int nNew;
int jj;
ExprAndCost *pBest = 0;
for(jj=0; jj<nExpr; jj++){
ExprAndCost *pCand = &aExpr[jj];
if( pCand->pExpr && (pBest==0 || pCand->nCost<pBest->nCost) ){
pBest = pCand;
}
}
if( pBest->nCost>nDoc ){
rc = fts3DeferExpression(p, p->pExpr);
break;
}else{
rc = fts3EvalExpr(p, pBest->pExpr, &aNew, &nNew, 0);
if( rc!=SQLITE_OK ) break;
pBest->pExpr = 0;
if( ii==0 ){
aRet = aNew;
nRet = nNew;
nDoc = fts3DoclistCountDocids(0, aRet, nRet);
}else{
fts3DoclistMerge(
MERGE_AND, 0, 0, aRet, &nRet, aRet, nRet, aNew, nNew, &nDoc
);
sqlite3_free(aNew);
}
}
}
}
*paOut = aRet;
*pnOut = nRet;
sqlite3_free(aExpr);
fts3ExprFreeSegReaders(pExpr);
}else{
char *aLeft;
char *aRight;
int nLeft;
int nRight;
if( 0==(rc = evalFts3Expr(p, pExpr->pRight, &aRight, &nRight, isReqPos))
&& 0==(rc = evalFts3Expr(p, pExpr->pLeft, &aLeft, &nLeft, isReqPos))
){
assert( pExpr->eType==FTSQUERY_NEAR || pExpr->eType==FTSQUERY_OR
|| pExpr->eType==FTSQUERY_AND || pExpr->eType==FTSQUERY_NOT
);
assert( pExpr->eType==FTSQUERY_NEAR
|| pExpr->eType==FTSQUERY_OR
|| pExpr->eType==FTSQUERY_NOT
|| (pExpr->eType==FTSQUERY_AND && p->eEvalmode==FTS3_EVAL_NEXT)
);
if( 0==(rc = fts3EvalExpr(p, pExpr->pRight, &aRight, &nRight, isReqPos))
&& 0==(rc = fts3EvalExpr(p, pExpr->pLeft, &aLeft, &nLeft, isReqPos))
){
switch( pExpr->eType ){
case FTSQUERY_NEAR: {
Fts3Expr *pLeft;
Fts3Expr *pRight;
int mergetype = isReqPos ? MERGE_POS_NEAR : MERGE_NEAR;
int mergetype = MERGE_NEAR;
if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){
mergetype = MERGE_POS_NEAR;
}
pLeft = pExpr->pLeft;
while( pLeft->eType==FTSQUERY_NEAR ){
pLeft=pLeft->pRight;
}
|
︙ | | |
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
|
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
|
-
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
|
/* Allocate a buffer for the output. The maximum size is the
** sum of the sizes of the two input buffers. The +1 term is
** so that a buffer of zero bytes is never allocated - this can
** cause fts3DoclistMerge() to incorrectly return SQLITE_NOMEM.
*/
char *aBuffer = sqlite3_malloc(nRight+nLeft+1);
rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut,
aLeft, nLeft, aRight, nRight
aLeft, nLeft, aRight, nRight, 0
);
*paOut = aBuffer;
sqlite3_free(aLeft);
break;
}
default: {
assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND );
fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut,
aLeft, nLeft, aRight, nRight
aLeft, nLeft, aRight, nRight, 0
);
*paOut = aLeft;
break;
}
}
}
sqlite3_free(aRight);
}
}
return rc;
}
/*
** This function is called from within xNext() for each row visited by
** an FTS3 query. If evaluating the FTS3 query expression within xFilter()
** was able to determine the exact set of matching rows, this function sets
** *pbRes to true and returns SQLITE_IO immediately.
**
** Otherwise, if evaluating the query expression within xFilter() returned a
** superset of the matching documents instead of an exact set (this happens
** when the query includes very common tokens and it is deemed too expensive to
** load their doclists from disk), this function tests if the current row
** really does match the FTS3 query.
**
** If an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK
** is returned and *pbRes is set to true if the current row matches the
** FTS3 query (and should be included in the results returned to SQLite), or
** false otherwise.
*/
static int fts3EvalDeferred(
Fts3Cursor *pCsr, /* FTS3 cursor pointing at row to test */
int *pbRes /* OUT: Set to true if row is a match */
){
int rc = SQLITE_OK;
if( pCsr->pDeferred==0 ){
*pbRes = 1;
}else{
rc = fts3CursorSeek(0, pCsr);
if( rc==SQLITE_OK ){
sqlite3Fts3FreeDeferredDoclists(pCsr);
rc = sqlite3Fts3CacheDeferredDoclists(pCsr);
}
if( rc==SQLITE_OK ){
char *a = 0;
int n = 0;
rc = fts3EvalExpr(pCsr, pCsr->pExpr, &a, &n, 0);
assert( n>=0 );
*pbRes = (n>0);
sqlite3_free(a);
}
}
return rc;
}
/*
** Advance the cursor to the next row in the %_content table that
** matches the search criteria. For a MATCH search, this will be
** the next row that matches. For a full-table scan, this will be
** simply the next row in the %_content table. For a docid lookup,
** this routine simply sets the EOF flag.
**
** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned
** even if we reach end-of-file. The fts3EofMethod() will be called
** subsequently to determine whether or not an EOF was hit.
*/
static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){
int res;
int rc = SQLITE_OK; /* Return code */
Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
pCsr->eEvalmode = FTS3_EVAL_NEXT;
do {
if( pCsr->aDoclist==0 ){
if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){
pCsr->isEof = 1;
rc = sqlite3_reset(pCsr->pStmt);
break;
}
pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0);
}else{
if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){
pCsr->isEof = 1;
break;
}
sqlite3_reset(pCsr->pStmt);
fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId);
pCsr->isRequireSeek = 1;
pCsr->isMatchinfoNeeded = 1;
}
}while( SQLITE_OK==(rc = fts3EvalDeferred(pCsr, &res)) && res==0 );
return rc;
}
/*
** This is the xFilter interface for the virtual table. See
** the virtual table xFilter method documentation for additional
** information.
**
** If idxNum==FTS3_FULLSCAN_SEARCH then do a full table scan against
** the %_content table.
**
** If idxNum==FTS3_DOCID_SEARCH then do a docid lookup for a single entry
** in the %_content table.
**
** If idxNum>=FTS3_FULLTEXT_SEARCH then use the full text index. The
** column on the left-hand side of the MATCH operator is column
** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand
** side of the MATCH operator.
*/
/* TODO(shess) Upgrade the cursor initialization and destruction to
** account for fts3FilterMethod() being called multiple times on the
** same cursor. The current solution is very fragile. Apply fix to
** fts3 as appropriate.
*/
static int fts3FilterMethod(
sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
int idxNum, /* Strategy index */
const char *idxStr, /* Unused */
int nVal, /* Number of elements in apVal */
sqlite3_value **apVal /* Arguments for the indexing scheme */
){
|
︙ | | |
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
|
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
|
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
-
|
UNUSED_PARAMETER(idxStr);
UNUSED_PARAMETER(nVal);
assert( idxNum>=0 && idxNum<=(FTS3_FULLTEXT_SEARCH+p->nColumn) );
assert( nVal==0 || nVal==1 );
assert( (nVal==0)==(idxNum==FTS3_FULLSCAN_SEARCH) );
assert( p->pSegments==0 );
/* In case the cursor has been used before, clear it now. */
sqlite3_finalize(pCsr->pStmt);
sqlite3_free(pCsr->aDoclist);
sqlite3Fts3ExprFree(pCsr->pExpr);
memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
/* Compile a SELECT statement for this cursor. For a full-table-scan, the
** statement loops through all rows of the %_content table. For a
** full-text query or docid lookup, the statement retrieves a single
** row by docid.
*/
zSql = sqlite3_mprintf(azSql[idxNum==FTS3_FULLSCAN_SEARCH], p->zDb, p->zName);
if( !zSql ){
rc = SQLITE_NOMEM;
}else{
rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
sqlite3_free(zSql);
}
if( rc!=SQLITE_OK ) return rc;
pCsr->eSearch = (i16)idxNum;
if( idxNum==FTS3_DOCID_SEARCH ){
if( idxNum!=FTS3_DOCID_SEARCH && idxNum!=FTS3_FULLSCAN_SEARCH ){
rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
}else if( idxNum!=FTS3_FULLSCAN_SEARCH ){
int iCol = idxNum-FTS3_FULLTEXT_SEARCH;
const char *zQuery = (const char *)sqlite3_value_text(apVal[0]);
if( zQuery==0 && sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
return SQLITE_NOMEM;
}
|
︙ | | |
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
|
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
|
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
|
}
return rc;
}
rc = sqlite3Fts3ReadLock(p);
if( rc!=SQLITE_OK ) return rc;
rc = evalFts3Expr(p, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0);
rc = fts3EvalExpr(pCsr, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0);
sqlite3Fts3SegmentsClose(p);
if( rc!=SQLITE_OK ) return rc;
pCsr->pNextId = pCsr->aDoclist;
pCsr->iPrevId = 0;
}
/* Compile a SELECT statement for this cursor. For a full-table-scan, the
** statement loops through all rows of the %_content table. For a
** full-text query or docid lookup, the statement retrieves a single
** row by docid.
*/
zSql = sqlite3_mprintf(azSql[idxNum==FTS3_FULLSCAN_SEARCH], p->zDb, p->zName);
if( !zSql ){
rc = SQLITE_NOMEM;
}else{
rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
sqlite3_free(zSql);
}
if( rc==SQLITE_OK && idxNum==FTS3_DOCID_SEARCH ){
rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
}
pCsr->eSearch = (i16)idxNum;
if( rc!=SQLITE_OK ) return rc;
return fts3NextMethod(pCursor);
}
/*
** This is the xEof method of the virtual table. SQLite calls this
|
︙ | | |
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
|
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
|
+
+
+
+
+
|
** rowid should be written to *pRowid.
*/
static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
Fts3Cursor *pCsr = (Fts3Cursor *) pCursor;
if( pCsr->aDoclist ){
*pRowid = pCsr->iPrevId;
}else{
/* This branch runs if the query is implemented using a full-table scan
** (not using the full-text index). In this case grab the rowid from the
** SELECT statement.
*/
assert( pCsr->isRequireSeek==0 );
*pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
}
return SQLITE_OK;
}
/*
** This is the xColumn method, called by SQLite to request a value from
|
︙ | | |
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
|
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
|
-
+
+
+
|
}
/*
** Implementation of xSync() method. Flush the contents of the pending-terms
** hash-table to the database.
*/
static int fts3SyncMethod(sqlite3_vtab *pVtab){
return sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab);
int rc = sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab);
sqlite3Fts3SegmentsClose((Fts3Table *)pVtab);
return rc;
}
/*
** Implementation of xBegin() method. This is a no-op.
*/
static int fts3BeginMethod(sqlite3_vtab *pVtab){
UNUSED_PARAMETER(pVtab);
|
︙ | | |
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
|
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
|
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
|
/*
** Load the doclist associated with expression pExpr to pExpr->aDoclist.
** The loaded doclist contains positions as well as the document ids.
** This is used by the matchinfo(), snippet() and offsets() auxillary
** functions.
*/
int sqlite3Fts3ExprLoadDoclist(Fts3Table *pTab, Fts3Expr *pExpr){
return evalFts3Expr(pTab, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1);
int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *pCsr, Fts3Expr *pExpr){
int rc;
assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase );
assert( pCsr->eEvalmode==FTS3_EVAL_NEXT );
rc = fts3EvalExpr(pCsr, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1);
return rc;
}
int sqlite3Fts3ExprLoadFtDoclist(
Fts3Cursor *pCsr,
Fts3Expr *pExpr,
char **paDoclist,
int *pnDoclist
){
int rc;
assert( pCsr->eEvalmode==FTS3_EVAL_NEXT );
assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase );
pCsr->eEvalmode = FTS3_EVAL_MATCHINFO;
rc = fts3EvalExpr(pCsr, pExpr, paDoclist, pnDoclist, 1);
pCsr->eEvalmode = FTS3_EVAL_NEXT;
return rc;
}
/*
** After ExprLoadDoclist() (see above) has been called, this function is
** used to iterate/search through the position lists that make up the doclist
** stored in pExpr->aDoclist.
*/
|
︙ | | |
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
|
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
|
-
+
|
** message is written to context pContext and SQLITE_ERROR returned. The
** string passed via zFunc is used as part of the error message.
*/
static int fts3FunctionArg(
sqlite3_context *pContext, /* SQL function call context */
const char *zFunc, /* Function name */
sqlite3_value *pVal, /* argv[0] passed to function */
Fts3Cursor **ppCsr /* OUT: Store cursor handle here */
Fts3Cursor **ppCsr /* OUT: Store cursor handle here */
){
Fts3Cursor *pRet;
if( sqlite3_value_type(pVal)!=SQLITE_BLOB
|| sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *)
){
char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc);
sqlite3_result_error(pContext, zErr, -1);
|
︙ | | |
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
|
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
|
-
-
+
-
-
-
-
-
|
*/
static void fts3MatchinfoFunc(
sqlite3_context *pContext, /* SQLite function call context */
int nVal, /* Size of argument array */
sqlite3_value **apVal /* Array of arguments */
){
Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */
if( nVal!=1 ){
assert( nVal==1 );
sqlite3_result_error(pContext,
"wrong number of arguments to function matchinfo()", -1);
return;
}
if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){
sqlite3Fts3Matchinfo(pContext, pCsr);
}
}
/*
** This routine implements the xFindFunction method for the FTS3
|
︙ | | |
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
|
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
|
-
|
return rc;
}
fts3DbExec(&rc, db,
"ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';",
p->zDb, p->zName, zName
);
if( rc==SQLITE_ERROR ) rc = SQLITE_OK;
if( p->bHasDocsize ){
fts3DbExec(&rc, db,
"ALTER TABLE %Q.'%q_docsize' RENAME TO '%q_docsize';",
p->zDb, p->zName, zName
);
fts3DbExec(&rc, db,
"ALTER TABLE %Q.'%q_stat' RENAME TO '%q_stat';",
|
︙ | | |
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
|
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
|
-
+
|
/* iVersion */ 0,
/* xCreate */ fts3CreateMethod,
/* xConnect */ fts3ConnectMethod,
/* xBestIndex */ fts3BestIndexMethod,
/* xDisconnect */ fts3DisconnectMethod,
/* xDestroy */ fts3DestroyMethod,
/* xOpen */ fts3OpenMethod,
/* xClose */ fulltextClose,
/* xClose */ fts3CloseMethod,
/* xFilter */ fts3FilterMethod,
/* xNext */ fts3NextMethod,
/* xEof */ fts3EofMethod,
/* xColumn */ fts3ColumnMethod,
/* xRowid */ fts3RowidMethod,
/* xUpdate */ fts3UpdateMethod,
/* xBegin */ fts3BeginMethod,
|
︙ | | |
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
|
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
|
-
-
-
-
+
+
+
+
-
-
+
+
+
|
static void hashDestroy(void *p){
Fts3Hash *pHash = (Fts3Hash *)p;
sqlite3Fts3HashClear(pHash);
sqlite3_free(pHash);
}
/*
** The fts3 built-in tokenizers - "simple" and "porter" - are implemented
** in files fts3_tokenizer1.c and fts3_porter.c respectively. The following
** two forward declarations are for functions declared in these files
** used to retrieve the respective implementations.
** The fts3 built-in tokenizers - "simple", "porter" and "icu"- are
** implemented in files fts3_tokenizer1.c, fts3_porter.c and fts3_icu.c
** respectively. The following three forward declarations are for functions
** declared in these files used to retrieve the respective implementations.
**
** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed
** to by the argument to point to the "simple" tokenizer implementation.
** Function ...PorterTokenizerModule() sets *pModule to point to the
** porter tokenizer/stemmer implementation.
** And so on.
*/
void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
#ifdef SQLITE_ENABLE_ICU
void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule);
#endif
/*
** Initialise the fts3 extension. If this extension is built as part
** of the sqlite library, then this function is called directly by
** SQLite. If fts3 is built as a dynamically loadable extension, this
** function is called by the sqlite3_extension_init() entry point.
*/
|
︙ | | |
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
|
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
|
-
+
|
** the two scalar functions. If this is successful, register the
** module with sqlite.
*/
if( SQLITE_OK==rc
&& SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer"))
&& SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
&& SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1))
&& SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", -1))
&& SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1))
&& SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1))
){
rc = sqlite3_create_module_v2(
db, "fts3", &fts3Module, (void *)pHash, hashDestroy
);
if( rc==SQLITE_OK ){
rc = sqlite3_create_module_v2(
|
︙ | | |
︙ | | |
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
|
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
|
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
+
+
+
+
+
-
+
+
|
char *aData;
int nSpace;
sqlite3_int64 iLastDocid;
sqlite3_int64 iLastCol;
sqlite3_int64 iLastPos;
};
/*
** Each cursor has a (possibly empty) linked list of the following objects.
*/
struct Fts3DeferredToken {
Fts3PhraseToken *pToken; /* Pointer to corresponding expr token */
int iCol; /* Column token must occur in */
Fts3DeferredToken *pNext; /* Next in list of deferred tokens */
PendingList *pList; /* Doclist is assembled here */
};
/*
** An instance of this structure is used to iterate through the terms on
** a contiguous set of segment b-tree leaf nodes. Although the details of
** this structure are only manipulated by code in this file, opaque handles
** of type Fts3SegReader* are also used by code in fts3.c to iterate through
** terms when querying the full-text index. See functions:
**
** sqlite3Fts3SegReaderNew()
** sqlite3Fts3SegReaderFree()
** sqlite3Fts3SegReaderCost()
** sqlite3Fts3SegReaderIterate()
**
** Methods used to manipulate Fts3SegReader structures:
**
** fts3SegReaderNext()
** fts3SegReaderFirstDocid()
** fts3SegReaderNextDocid()
*/
struct Fts3SegReader {
int iIdx; /* Index within level, or 0x7FFFFFFF for PT */
sqlite3_int64 iStartBlock;
sqlite3_int64 iEndBlock;
sqlite3_stmt *pStmt; /* SQL Statement to access leaf nodes */
sqlite3_int64 iStartBlock; /* Rowid of first leaf block to traverse */
sqlite3_int64 iLeafEndBlock; /* Rowid of final leaf block to traverse */
sqlite3_int64 iEndBlock; /* Rowid of final block in segment (or 0) */
sqlite3_int64 iCurrentBlock; /* Current leaf block (or 0) */
char *aNode; /* Pointer to node data (or NULL) */
int nNode; /* Size of buffer at aNode (or 0) */
int nTermAlloc; /* Allocated size of zTerm buffer */
Fts3HashElem **ppNextElem;
/* Variables set by fts3SegReaderNext(). These may be read directly
** by the caller. They are valid from the time SegmentReaderNew() returns
** until SegmentReaderNext() returns something other than SQLITE_OK
** (i.e. SQLITE_DONE).
*/
int nTerm; /* Number of bytes in current term */
char *zTerm; /* Pointer to current term */
int nTermAlloc; /* Allocated size of zTerm buffer */
char *aDoclist; /* Pointer to doclist of current entry */
int nDoclist; /* Size of doclist in current entry */
/* The following variables are used to iterate through the current doclist */
char *pOffsetList;
sqlite3_int64 iDocid;
};
#define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0)
#define fts3SegReaderIsRootOnly(p) ((p)->aNode==(char *)&(p)[1])
/*
** An instance of this structure is used to create a segment b-tree in the
** database. The internal details of this type are only accessed by the
** following functions:
**
** fts3SegWriterAdd()
|
︙ | | |
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
|
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
|
-
-
-
-
-
-
+
+
+
+
+
|
#define SQL_SELECT_LEVEL 12
#define SQL_SELECT_ALL_LEVEL 13
#define SQL_SELECT_LEVEL_COUNT 14
#define SQL_SELECT_SEGDIR_COUNT_MAX 15
#define SQL_DELETE_SEGDIR_BY_LEVEL 16
#define SQL_DELETE_SEGMENTS_RANGE 17
#define SQL_CONTENT_INSERT 18
#define SQL_GET_BLOCK 19
#define SQL_DELETE_DOCSIZE 20
#define SQL_REPLACE_DOCSIZE 21
#define SQL_SELECT_DOCSIZE 22
#define SQL_SELECT_DOCTOTAL 23
#define SQL_REPLACE_DOCTOTAL 24
#define SQL_DELETE_DOCSIZE 19
#define SQL_REPLACE_DOCSIZE 20
#define SQL_SELECT_DOCSIZE 21
#define SQL_SELECT_DOCTOTAL 22
#define SQL_REPLACE_DOCTOTAL 23
/*
** This function is used to obtain an SQLite prepared statement handle
** for the statement identified by the second argument. If successful,
** *pp is set to the requested statement handle and SQLITE_OK returned.
** Otherwise, an SQLite error code is returned and *pp is set to 0.
**
|
︙ | | |
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
|
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
|
-
-
-
-
-
-
+
+
+
+
+
|
/* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?",
/* 15 */ "SELECT count(*), max(level) FROM %Q.'%q_segdir'",
/* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?",
/* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?",
/* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%z)",
/* 19 */ "SELECT block FROM %Q.'%q_segments' WHERE blockid = ?",
/* 20 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?",
/* 21 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",
/* 22 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?",
/* 23 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0",
/* 24 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)",
/* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?",
/* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",
/* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?",
/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0",
/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)",
};
int rc = SQLITE_OK;
sqlite3_stmt *pStmt;
assert( SizeofArray(azSql)==SizeofArray(p->aStmt) );
assert( eStmt<SizeofArray(azSql) && eStmt>=0 );
|
︙ | | |
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
|
293
294
295
296
297
298
299
300
301
302
303
304
305
306
|
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
|
sqlite3_step(pStmt);
rc = sqlite3_reset(pStmt);
}
*pRC = rc;
}
/*
** Read a single block from the %_segments table. If the specified block
** does not exist, return SQLITE_CORRUPT. If some other error (malloc, IO
** etc.) occurs, return the appropriate SQLite error code.
**
** Otherwise, if successful, set *pzBlock to point to a buffer containing
** the block read from the database, and *pnBlock to the size of the read
** block in bytes.
**
** WARNING: The returned buffer is only valid until the next call to
** sqlite3Fts3ReadBlock().
*/
int sqlite3Fts3ReadBlock(
Fts3Table *p,
sqlite3_int64 iBlock,
char const **pzBlock,
int *pnBlock
){
sqlite3_stmt *pStmt;
int rc = fts3SqlStmt(p, SQL_GET_BLOCK, &pStmt, 0);
if( rc!=SQLITE_OK ) return rc;
sqlite3_reset(pStmt);
if( pzBlock ){
sqlite3_bind_int64(pStmt, 1, iBlock);
rc = sqlite3_step(pStmt);
if( rc!=SQLITE_ROW ){
return (rc==SQLITE_DONE ? SQLITE_CORRUPT : rc);
}
*pnBlock = sqlite3_column_bytes(pStmt, 0);
*pzBlock = (char *)sqlite3_column_blob(pStmt, 0);
if( sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){
return SQLITE_CORRUPT;
}
}
return SQLITE_OK;
}
/*
** This function ensures that the caller has obtained a shared-cache
** table-lock on the %_content table. This is required before reading
** data from the fts3 table. If this lock is not acquired first, then
** the caller may end up holding read-locks on the %_segments and %_segdir
** tables, but no read-lock on the %_content table. If this happens
** a second connection will be able to write to the fts3 table, but
|
︙ | | |
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
|
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
|
-
-
-
-
+
+
+
+
|
** Tokenize the nul-terminated string zText and add all tokens to the
** pending-terms hash-table. The docid used is that currently stored in
** p->iPrevDocid, and the column is specified by argument iCol.
**
** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code.
*/
static int fts3PendingTermsAdd(
Fts3Table *p, /* FTS table into which text will be inserted */
const char *zText, /* Text of document to be inseted */
int iCol, /* Column number into which text is inserted */
u32 *pnWord /* OUT: Number of tokens inserted */
Fts3Table *p, /* Table into which text will be inserted */
const char *zText, /* Text of document to be inserted */
int iCol, /* Column into which text is being inserted */
u32 *pnWord /* OUT: Number of tokens inserted */
){
int rc;
int iStart;
int iEnd;
int iPos;
int nWord = 0;
|
︙ | | |
574
575
576
577
578
579
580
581
582
583
584
585
586
587
|
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
|
+
+
+
|
int rc = sqlite3Fts3PendingTermsFlush(p);
if( rc!=SQLITE_OK ) return rc;
}
p->iPrevDocid = iDocid;
return SQLITE_OK;
}
/*
** Discard the contents of the pending-terms hash table.
*/
void sqlite3Fts3PendingTermsClear(Fts3Table *p){
Fts3HashElem *pElem;
for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){
sqlite3_free(fts3HashData(pElem));
}
fts3HashClear(&p->pendingTerms);
p->nPendingData = 0;
|
︙ | | |
601
602
603
604
605
606
607
608
609
610
611
612
613
614
|
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
|
+
|
const char *zText = (const char *)sqlite3_value_text(apVal[i]);
if( zText ){
int rc = fts3PendingTermsAdd(p, zText, i-2, &aSz[i-2]);
if( rc!=SQLITE_OK ){
return rc;
}
}
aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]);
}
return SQLITE_OK;
}
/*
** This function is called by the xUpdate() method for an INSERT operation.
** The apVal parameter is passed a copy of the apVal argument passed by
|
︙ | | |
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
|
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
|
-
+
|
}
/*
** The first element in the apVal[] array is assumed to contain the docid
** (an integer) of a row about to be deleted. Remove all terms from the
** full-text index.
*/
static void fts3DeleteTerms(
static void fts3DeleteTerms(
int *pRC, /* Result code */
Fts3Table *p, /* The FTS table to delete from */
sqlite3_value **apVal, /* apVal[] contains the docid to be deleted */
u32 *aSz /* Sizes of deleted document written here */
){
int rc;
sqlite3_stmt *pSelect;
|
︙ | | |
720
721
722
723
724
725
726
727
728
729
730
731
732
733
|
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
|
+
|
const char *zText = (const char *)sqlite3_column_text(pSelect, i);
rc = fts3PendingTermsAdd(p, zText, -1, &aSz[i-1]);
if( rc!=SQLITE_OK ){
sqlite3_reset(pSelect);
*pRC = rc;
return;
}
aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i);
}
}
rc = sqlite3_reset(pSelect);
}else{
sqlite3_reset(pSelect);
}
*pRC = rc;
|
︙ | | |
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
|
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
|
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
+
-
+
+
+
-
-
+
+
+
+
+
+
+
+
-
-
-
-
-
-
-
+
+
+
+
+
-
+
+
+
+
+
+
+
+
+
+
+
|
}else{
*piIdx = iNext;
}
}
return rc;
}
/*
** The %_segments table is declared as follows:
**
** CREATE TABLE %_segments(blockid INTEGER PRIMARY KEY, block BLOB)
**
** This function reads data from a single row of the %_segments table. The
** specific row is identified by the iBlockid parameter. If paBlob is not
** NULL, then a buffer is allocated using sqlite3_malloc() and populated
** with the contents of the blob stored in the "block" column of the
** identified table row is. Whether or not paBlob is NULL, *pnBlob is set
** to the size of the blob in bytes before returning.
**
** If an error occurs, or the table does not contain the specified row,
** an SQLite error code is returned. Otherwise, SQLITE_OK is returned. If
** paBlob is non-NULL, then it is the responsibility of the caller to
** eventually free the returned buffer.
**
** This function may leave an open sqlite3_blob* handle in the
** Fts3Table.pSegments variable. This handle is reused by subsequent calls
** to this function. The handle may be closed by calling the
** sqlite3Fts3SegmentsClose() function. Reusing a blob handle is a handy
** performance improvement, but the blob handle should always be closed
** before control is returned to the user (to prevent a lock being held
** on the database file for longer than necessary). Thus, any virtual table
** method (xFilter etc.) that may directly or indirectly call this function
** must call sqlite3Fts3SegmentsClose() before returning.
*/
int sqlite3Fts3ReadBlock(
Fts3Table *p, /* FTS3 table handle */
sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */
char **paBlob, /* OUT: Blob data in malloc'd buffer */
int *pnBlob /* OUT: Size of blob data */
){
int rc; /* Return code */
/* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */
assert( pnBlob);
if( p->pSegments ){
rc = sqlite3_blob_reopen(p->pSegments, iBlockid);
}else{
if( 0==p->zSegmentsTbl ){
p->zSegmentsTbl = sqlite3_mprintf("%s_segments", p->zName);
if( 0==p->zSegmentsTbl ) return SQLITE_NOMEM;
}
rc = sqlite3_blob_open(
p->db, p->zDb, p->zSegmentsTbl, "block", iBlockid, 0, &p->pSegments
);
}
if( rc==SQLITE_OK ){
int nByte = sqlite3_blob_bytes(p->pSegments);
if( paBlob ){
char *aByte = sqlite3_malloc(nByte);
if( !aByte ){
rc = SQLITE_NOMEM;
}else{
rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0);
if( rc!=SQLITE_OK ){
sqlite3_free(aByte);
aByte = 0;
}
}
*paBlob = aByte;
}
*pnBlob = nByte;
}
return rc;
}
/*
** Close the blob handle at p->pSegments, if it is open. See comments above
** the sqlite3Fts3ReadBlock() function for details.
*/
void sqlite3Fts3SegmentsClose(Fts3Table *p){
sqlite3_blob_close(p->pSegments);
p->pSegments = 0;
}
/*
** Move the iterator passed as the first argument to the next term in the
** segment. If successful, SQLITE_OK is returned. If there is no next term,
** SQLITE_DONE. Otherwise, an SQLite error code.
*/
static int fts3SegReaderNext(Fts3SegReader *pReader){
static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){
char *pNext; /* Cursor variable */
int nPrefix; /* Number of bytes in term prefix */
int nSuffix; /* Number of bytes in term suffix */
if( !pReader->aDoclist ){
pNext = pReader->aNode;
}else{
pNext = &pReader->aDoclist[pReader->nDoclist];
}
if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){
int rc; /* Return code from Fts3ReadBlock() */
int rc;
if( fts3SegReaderIsPending(pReader) ){
Fts3HashElem *pElem = *(pReader->ppNextElem);
if( pElem==0 ){
pReader->aNode = 0;
}else{
PendingList *pList = (PendingList *)fts3HashData(pElem);
pReader->zTerm = (char *)fts3HashKey(pElem);
pReader->nTerm = fts3HashKeysize(pElem);
pReader->nNode = pReader->nDoclist = pList->nData + 1;
pReader->aNode = pReader->aDoclist = pList->aData;
pReader->ppNextElem++;
assert( pReader->aNode );
}
return SQLITE_OK;
}
if( !fts3SegReaderIsRootOnly(pReader) ){
if( !pReader->pStmt ){
pReader->aNode = 0;
sqlite3_free(pReader->aNode);
}
pReader->aNode = 0;
/* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf
** blocks have already been traversed. */
assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock );
if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){
return SQLITE_OK;
}
rc = sqlite3_step(pReader->pStmt);
if( rc!=SQLITE_ROW ){
pReader->aNode = 0;
return (rc==SQLITE_DONE ? SQLITE_OK : rc);
}
pReader->nNode = sqlite3_column_bytes(pReader->pStmt, 0);
pReader->aNode = (char *)sqlite3_column_blob(pReader->pStmt, 0);
rc = sqlite3Fts3ReadBlock(
p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode
);
if( rc!=SQLITE_OK ) return rc;
pNext = pReader->aNode;
}
pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix);
pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix);
if( nPrefix+nSuffix>pReader->nTermAlloc ){
int nNew = (nPrefix+nSuffix)*2;
char *zNew = sqlite3_realloc(pReader->zTerm, nNew);
if( !zNew ){
return SQLITE_NOMEM;
}
pReader->zTerm = zNew;
pReader->nTermAlloc = nNew;
}
memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix);
pReader->nTerm = nPrefix+nSuffix;
pNext += nSuffix;
pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist);
assert( pNext<&pReader->aNode[pReader->nNode] );
pReader->aDoclist = pNext;
pReader->pOffsetList = 0;
/* Check that the doclist does not appear to extend past the end of the
** b-tree node. And that the final byte of the doclist is either an 0x00
** or 0x01. If either of these statements is untrue, then the data structure
** is corrupt.
*/
if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode]
|| (pReader->aDoclist[pReader->nDoclist-1]&0xFE)!=0
){
return SQLITE_CORRUPT;
}
return SQLITE_OK;
}
/*
** Set the SegReader to point to the first docid in the doclist associated
** with the current term.
*/
|
︙ | | |
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
|
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
|
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
+
+
-
-
-
-
-
-
-
-
-
+
+
-
+
+
+
-
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
|
pReader->pOffsetList = 0;
}else{
sqlite3_int64 iDelta;
pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta);
pReader->iDocid += iDelta;
}
}
/*
** This function is called to estimate the amount of data that will be
** loaded from the disk If SegReaderIterate() is called on this seg-reader,
** in units of average document size.
**
** This can be used as follows: If the caller has a small doclist that
** contains references to N documents, and is considering merging it with
** a large doclist (size X "average documents"), it may opt not to load
** the large doclist if X>N.
*/
int sqlite3Fts3SegReaderCost(
Fts3Cursor *pCsr, /* FTS3 cursor handle */
Fts3SegReader *pReader, /* Segment-reader handle */
int *pnCost /* IN/OUT: Number of bytes read */
){
Fts3Table *p = (Fts3Table*)pCsr->base.pVtab;
int rc = SQLITE_OK; /* Return code */
int nCost = 0; /* Cost in bytes to return */
int pgsz = p->nPgsz; /* Database page size */
/* If this seg-reader is reading the pending-terms table, or if all data
** for the segment is stored on the root page of the b-tree, then the cost
** is zero. In this case all required data is already in main memory.
*/
if( p->bHasDocsize
&& !fts3SegReaderIsPending(pReader)
&& !fts3SegReaderIsRootOnly(pReader)
){
int nBlob = 0;
sqlite3_int64 iBlock;
if( pCsr->nRowAvg==0 ){
/* The average document size, which is required to calculate the cost
** of each doclist, has not yet been determined. Read the required
** data from the %_stat table to calculate it.
**
** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3
** varints, where nCol is the number of columns in the FTS3 table.
** The first varint is the number of documents currently stored in
** the table. The following nCol varints contain the total amount of
** data stored in all rows of each column of the table, from left
** to right.
*/
sqlite3_stmt *pStmt;
rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0);
if( rc ) return rc;
if( sqlite3_step(pStmt)==SQLITE_ROW ){
sqlite3_int64 nDoc = 0;
sqlite3_int64 nByte = 0;
const char *a = sqlite3_column_blob(pStmt, 0);
if( a ){
const char *pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
a += sqlite3Fts3GetVarint(a, &nDoc);
while( a<pEnd ){
a += sqlite3Fts3GetVarint(a, &nByte);
}
}
pCsr->nRowAvg = (((nByte / nDoc) + pgsz - 1) / pgsz);
}
rc = sqlite3_reset(pStmt);
if( rc!=SQLITE_OK || pCsr->nRowAvg==0 ) return rc;
}
/* Assume that a blob flows over onto overflow pages if it is larger
** than (pgsz-35) bytes in size (the file-format documentation
** confirms this).
*/
for(iBlock=pReader->iStartBlock; iBlock<=pReader->iLeafEndBlock; iBlock++){
rc = sqlite3Fts3ReadBlock(p, iBlock, 0, &nBlob);
if( rc!=SQLITE_OK ) break;
if( (nBlob+35)>pgsz ){
int nOvfl = (nBlob + 34)/pgsz;
nCost += ((nOvfl + pCsr->nRowAvg - 1)/pCsr->nRowAvg);
}
}
}
*pnCost += nCost;
return rc;
}
/*
** Free all allocations associated with the iterator passed as the
** second argument.
*/
void sqlite3Fts3SegReaderFree(Fts3Table *p, Fts3SegReader *pReader){
if( pReader ){
if( pReader->pStmt ){
if( pReader && !fts3SegReaderIsPending(pReader) ){
sqlite3_free(pReader->zTerm);
/* Move the leaf-range SELECT statement to the aLeavesStmt[] array,
** so that it can be reused when required by another query.
*/
assert( p->nLeavesStmt<p->nLeavesTotal );
sqlite3_reset(pReader->pStmt);
p->aLeavesStmt[p->nLeavesStmt++] = pReader->pStmt;
}
if( !fts3SegReaderIsPending(pReader) ){
sqlite3_free(pReader->zTerm);
if( !fts3SegReaderIsRootOnly(pReader) ){
sqlite3_free(pReader->aNode);
}
sqlite3_free(pReader);
}
sqlite3_free(pReader);
}
/*
** Allocate a new SegReader object.
*/
int sqlite3Fts3SegReaderNew(
Fts3Table *p, /* Virtual table handle */
int iAge, /* Segment "age". */
sqlite3_int64 iStartLeaf, /* First leaf to traverse */
sqlite3_int64 iEndLeaf, /* Final leaf to traverse */
sqlite3_int64 iEndBlock, /* Final block of segment */
const char *zRoot, /* Buffer containing root node */
int nRoot, /* Size of buffer containing root node */
Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */
){
int rc = SQLITE_OK; /* Return code */
Fts3SegReader *pReader; /* Newly allocated SegReader object */
int nExtra = 0; /* Bytes to allocate segment root node */
assert( iStartLeaf<=iEndLeaf );
if( iStartLeaf==0 ){
nExtra = nRoot;
}
pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra);
if( !pReader ){
return SQLITE_NOMEM;
}
memset(pReader, 0, sizeof(Fts3SegReader));
pReader->iIdx = iAge;
pReader->iStartBlock = iStartLeaf;
pReader->iIdx = iAge;
pReader->iLeafEndBlock = iEndLeaf;
pReader->iEndBlock = iEndBlock;
if( nExtra ){
/* The entire segment is stored in the root node. */
pReader->aNode = (char *)&pReader[1];
pReader->nNode = nRoot;
memcpy(pReader->aNode, zRoot, nRoot);
}else{
/* If the text of the SQL statement to iterate through a contiguous
** set of entries in the %_segments table has not yet been composed,
** compose it now.
*/
if( !p->zSelectLeaves ){
p->zSelectLeaves = sqlite3_mprintf(
"SELECT block FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ? "
"ORDER BY blockid", p->zDb, p->zName
);
if( !p->zSelectLeaves ){
rc = SQLITE_NOMEM;
goto finished;
}
}
pReader->iCurrentBlock = iStartLeaf-1;
/* If there are no free statements in the aLeavesStmt[] array, prepare
** a new statement now. Otherwise, reuse a prepared statement from
** aLeavesStmt[].
*/
if( p->nLeavesStmt==0 ){
if( p->nLeavesTotal==p->nLeavesAlloc ){
int nNew = p->nLeavesAlloc + 16;
sqlite3_stmt **aNew = (sqlite3_stmt **)sqlite3_realloc(
p->aLeavesStmt, nNew*sizeof(sqlite3_stmt *)
);
if( !aNew ){
rc = SQLITE_NOMEM;
goto finished;
}
}
p->nLeavesAlloc = nNew;
p->aLeavesStmt = aNew;
}
rc = sqlite3_prepare_v2(p->db, p->zSelectLeaves, -1, &pReader->pStmt, 0);
if( rc!=SQLITE_OK ){
goto finished;
}
p->nLeavesTotal++;
}else{
pReader->pStmt = p->aLeavesStmt[--p->nLeavesStmt];
}
/* Bind the start and end leaf blockids to the prepared SQL statement. */
sqlite3_bind_int64(pReader->pStmt, 1, iStartLeaf);
sqlite3_bind_int64(pReader->pStmt, 2, iEndLeaf);
}
rc = fts3SegReaderNext(pReader);
rc = fts3SegReaderNext(p, pReader);
finished:
if( rc==SQLITE_OK ){
*ppReader = pReader;
}else{
sqlite3Fts3SegReaderFree(p, pReader);
}
return rc;
}
|
︙ | | |
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
|
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
|
-
+
|
if( !pReader ){
rc = SQLITE_NOMEM;
}else{
memset(pReader, 0, nByte);
pReader->iIdx = 0x7FFFFFFF;
pReader->ppNextElem = (Fts3HashElem **)&pReader[1];
memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *));
fts3SegReaderNext(pReader);
fts3SegReaderNext(p, pReader);
}
}
if( isPrefix ){
sqlite3_free(aElem);
}
*ppReader = pReader;
|
︙ | | |
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
|
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
|
-
+
|
}
/*
** Add term zTerm to the SegmentNode. It is guaranteed that zTerm is larger
** (according to memcmp) than the previous term.
*/
static int fts3NodeAddTerm(
Fts3Table *p, /* Virtual table handle */
Fts3Table *p, /* Virtual table handle */
SegmentNode **ppTree, /* IN/OUT: SegmentNode handle */
int isCopyTerm, /* True if zTerm/nTerm is transient */
const char *zTerm, /* Pointer to buffer containing term */
int nTerm /* Size of term in bytes */
){
SegmentNode *pTree = *ppTree;
int rc;
|
︙ | | |
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
|
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
|
-
+
|
*/
if( pFilter->zTerm ){
int nTerm = pFilter->nTerm;
const char *zTerm = pFilter->zTerm;
for(i=0; i<nSegment; i++){
Fts3SegReader *pSeg = apSegment[i];
while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ){
rc = fts3SegReaderNext(pSeg);
rc = fts3SegReaderNext(p, pSeg);
if( rc!=SQLITE_OK ) goto finished; }
}
}
fts3SegReaderSort(apSegment, nSegment, nSegment, fts3SegReaderCmp);
while( apSegment[0]->aNode ){
int nTerm = apSegment[0]->nTerm;
|
︙ | | |
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
|
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
|
-
+
|
** term (if such a term exists in the index) has already been made.
*/
if( pFilter->zTerm && !isPrefix ){
goto finished;
}
for(i=0; i<nMerge; i++){
rc = fts3SegReaderNext(apSegment[i]);
rc = fts3SegReaderNext(p, apSegment[i]);
if( rc!=SQLITE_OK ) goto finished;
}
fts3SegReaderSort(apSegment, nSegment, nMerge, fts3SegReaderCmp);
}
finished:
sqlite3_free(aBuffer);
|
︙ | | |
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
|
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
|
-
+
|
** Otherwise, if successful, SQLITE_OK is returned. If an error occurs,
** an SQLite error code is returned.
*/
static int fts3SegmentMerge(Fts3Table *p, int iLevel){
int i; /* Iterator variable */
int rc; /* Return code */
int iIdx; /* Index of new segment */
int iNewLevel; /* Level to create new segment at */
int iNewLevel = 0; /* Level to create new segment at */
sqlite3_stmt *pStmt = 0;
SegmentWriter *pWriter = 0;
int nSegment = 0; /* Number of segments being merged */
Fts3SegReader **apSegment = 0; /* Array of Segment iterators */
Fts3SegReader *pPending = 0; /* Iterator for pending-terms */
Fts3SegFilter filter; /* Segment term filter condition */
|
︙ | | |
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
|
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
|
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
+
+
+
+
+
+
+
-
+
-
+
-
+
-
+
-
+
-
+
|
sqlite3_bind_int64(pStmt, 1, p->iPrevDocid);
sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, sqlite3_free);
sqlite3_step(pStmt);
*pRC = sqlite3_reset(pStmt);
}
/*
** Update the 0 record of the %_stat table so that it holds a blob
** which contains the document count followed by the cumulative
** document sizes for all columns.
** Record 0 of the %_stat table contains a blob consisting of N varints,
** where N is the number of user defined columns in the fts3 table plus
** two. If nCol is the number of user defined columns, then values of the
** varints are set as follows:
**
** Varint 0: Total number of rows in the table.
**
** Varint 1..nCol: For each column, the total number of tokens stored in
** the column for all rows of the table.
**
** Varint 1+nCol: The total size, in bytes, of all text values in all
** columns of all rows of the table.
**
*/
static void fts3UpdateDocTotals(
int *pRC, /* The result code */
Fts3Table *p, /* Table being updated */
u32 *aSzIns, /* Size increases */
u32 *aSzDel, /* Size decreases */
int nChng /* Change in the number of documents */
int *pRC, /* The result code */
Fts3Table *p, /* Table being updated */
u32 *aSzIns, /* Size increases */
u32 *aSzDel, /* Size decreases */
int nChng /* Change in the number of documents */
){
char *pBlob; /* Storage for BLOB written into %_stat */
int nBlob; /* Size of BLOB written into %_stat */
u32 *a; /* Array of integers that becomes the BLOB */
sqlite3_stmt *pStmt; /* Statement for reading and writing */
int i; /* Loop counter */
int rc; /* Result code from subfunctions */
const int nStat = p->nColumn+2;
if( *pRC ) return;
a = sqlite3_malloc( (sizeof(u32)+10)*(p->nColumn+1) );
a = sqlite3_malloc( (sizeof(u32)+10)*nStat );
if( a==0 ){
*pRC = SQLITE_NOMEM;
return;
}
pBlob = (char*)&a[p->nColumn+1];
pBlob = (char*)&a[nStat];
rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0);
if( rc ){
sqlite3_free(a);
*pRC = rc;
return;
}
if( sqlite3_step(pStmt)==SQLITE_ROW ){
fts3DecodeIntArray(p->nColumn+1, a,
fts3DecodeIntArray(nStat, a,
sqlite3_column_blob(pStmt, 0),
sqlite3_column_bytes(pStmt, 0));
}else{
memset(a, 0, sizeof(u32)*(p->nColumn+1) );
memset(a, 0, sizeof(u32)*(nStat) );
}
sqlite3_reset(pStmt);
if( nChng<0 && a[0]<(u32)(-nChng) ){
a[0] = 0;
}else{
a[0] += nChng;
}
for(i=0; i<p->nColumn; i++){
for(i=0; i<p->nColumn+1; i++){
u32 x = a[i+1];
if( x+aSzIns[i] < aSzDel[i] ){
x = 0;
}else{
x = x + aSzIns[i] - aSzDel[i];
}
a[i+1] = x;
}
fts3EncodeIntArray(p->nColumn+1, a, pBlob, &nBlob);
fts3EncodeIntArray(nStat, a, pBlob, &nBlob);
rc = fts3SqlStmt(p, SQL_REPLACE_DOCTOTAL, &pStmt, 0);
if( rc ){
sqlite3_free(a);
*pRC = rc;
return;
}
sqlite3_bind_blob(pStmt, 1, pBlob, nBlob, SQLITE_STATIC);
|
︙ | | |
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
|
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
|
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
-
-
+
+
-
+
|
}else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){
p->nMaxPendingData = atoi(&zVal[11]);
rc = SQLITE_OK;
#endif
}else{
rc = SQLITE_ERROR;
}
sqlite3Fts3SegmentsClose(p);
return rc;
}
/*
** Return the deferred doclist associated with deferred token pDeferred.
** This function assumes that sqlite3Fts3CacheDeferredDoclists() has already
** been called to allocate and populate the doclist.
*/
char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *pDeferred, int *pnByte){
if( pDeferred->pList ){
*pnByte = pDeferred->pList->nData;
return pDeferred->pList->aData;
}
*pnByte = 0;
return 0;
}
/*
** Helper fucntion for FreeDeferredDoclists(). This function removes all
** references to deferred doclists from within the tree of Fts3Expr
** structures headed by
*/
static void fts3DeferredDoclistClear(Fts3Expr *pExpr){
if( pExpr ){
fts3DeferredDoclistClear(pExpr->pLeft);
fts3DeferredDoclistClear(pExpr->pRight);
if( pExpr->isLoaded ){
sqlite3_free(pExpr->aDoclist);
pExpr->isLoaded = 0;
pExpr->aDoclist = 0;
pExpr->nDoclist = 0;
pExpr->pCurrent = 0;
pExpr->iCurrent = 0;
}
}
}
/*
** Delete all cached deferred doclists. Deferred doclists are cached
** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function.
*/
void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){
Fts3DeferredToken *pDef;
for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){
sqlite3_free(pDef->pList);
pDef->pList = 0;
}
if( pCsr->pDeferred ){
fts3DeferredDoclistClear(pCsr->pExpr);
}
}
/*
** Free all entries in the pCsr->pDeffered list. Entries are added to
** this list using sqlite3Fts3DeferToken().
*/
void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){
Fts3DeferredToken *pDef;
Fts3DeferredToken *pNext;
for(pDef=pCsr->pDeferred; pDef; pDef=pNext){
pNext = pDef->pNext;
sqlite3_free(pDef->pList);
sqlite3_free(pDef);
}
pCsr->pDeferred = 0;
}
/*
** Generate deferred-doclists for all tokens in the pCsr->pDeferred list
** based on the row that pCsr currently points to.
**
** A deferred-doclist is like any other doclist with position information
** included, except that it only contains entries for a single row of the
** table, not for all rows.
*/
int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){
int rc = SQLITE_OK; /* Return code */
if( pCsr->pDeferred ){
int i; /* Used to iterate through table columns */
sqlite3_int64 iDocid; /* Docid of the row pCsr points to */
Fts3DeferredToken *pDef; /* Used to iterate through deferred tokens */
Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
sqlite3_tokenizer *pT = p->pTokenizer;
sqlite3_tokenizer_module const *pModule = pT->pModule;
assert( pCsr->isRequireSeek==0 );
iDocid = sqlite3_column_int64(pCsr->pStmt, 0);
for(i=0; i<p->nColumn && rc==SQLITE_OK; i++){
const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1);
sqlite3_tokenizer_cursor *pTC = 0;
rc = pModule->xOpen(pT, zText, -1, &pTC);
while( rc==SQLITE_OK ){
char const *zToken; /* Buffer containing token */
int nToken; /* Number of bytes in token */
int iDum1, iDum2; /* Dummy variables */
int iPos; /* Position of token in zText */
pTC->pTokenizer = pT;
rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos);
for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){
Fts3PhraseToken *pPT = pDef->pToken;
if( (pDef->iCol>=p->nColumn || pDef->iCol==i)
&& (pPT->n==nToken || (pPT->isPrefix && pPT->n<nToken))
&& (0==memcmp(zToken, pPT->z, pPT->n))
){
fts3PendingListAppend(&pDef->pList, iDocid, i, iPos, &rc);
}
}
}
if( pTC ) pModule->xClose(pTC);
if( rc==SQLITE_DONE ) rc = SQLITE_OK;
}
for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){
if( pDef->pList ){
rc = fts3PendingListAppendVarint(&pDef->pList, 0);
}
}
}
return rc;
}
/*
** Add an entry for token pToken to the pCsr->pDeferred list.
*/
int sqlite3Fts3DeferToken(
Fts3Cursor *pCsr, /* Fts3 table cursor */
Fts3PhraseToken *pToken, /* Token to defer */
int iCol /* Column that token must appear in (or -1) */
){
Fts3DeferredToken *pDeferred;
pDeferred = sqlite3_malloc(sizeof(*pDeferred));
if( !pDeferred ){
return SQLITE_NOMEM;
}
memset(pDeferred, 0, sizeof(*pDeferred));
pDeferred->pToken = pToken;
pDeferred->pNext = pCsr->pDeferred;
pDeferred->iCol = iCol;
pCsr->pDeferred = pDeferred;
assert( pToken->pDeferred==0 );
pToken->pDeferred = pDeferred;
return SQLITE_OK;
}
/*
** This function does the work for the xUpdate method of FTS3 virtual
** tables.
*/
int sqlite3Fts3UpdateMethod(
sqlite3_vtab *pVtab, /* FTS3 vtab object */
int nArg, /* Size of argument array */
sqlite3_value **apVal, /* Array of arguments */
sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */
){
Fts3Table *p = (Fts3Table *)pVtab;
int rc = SQLITE_OK; /* Return Code */
int isRemove = 0; /* True for an UPDATE or DELETE */
sqlite3_int64 iRemove = 0; /* Rowid removed by UPDATE or DELETE */
u32 *aSzIns; /* Sizes of inserted documents */
u32 *aSzDel; /* Sizes of deleted documents */
int nChng = 0; /* Net change in number of documents */
assert( p->pSegments==0 );
/* Allocate space to hold the change in document sizes */
aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*p->nColumn*2 );
aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 );
if( aSzIns==0 ) return SQLITE_NOMEM;
aSzDel = &aSzIns[p->nColumn];
memset(aSzIns, 0, sizeof(aSzIns[0])*p->nColumn*2);
aSzDel = &aSzIns[p->nColumn+1];
memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2);
/* If this is a DELETE or UPDATE operation, remove the old record. */
if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
int isEmpty;
int isEmpty = 0;
rc = fts3IsEmpty(p, apVal, &isEmpty);
if( rc==SQLITE_OK ){
if( isEmpty ){
/* Deleting this row means the whole table is empty. In this case
** delete the contents of all three tables and throw away any
** data in the pendingTerms hash table.
*/
|
︙ | | |
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
|
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
|
+
|
}
if( p->bHasDocsize ){
fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng);
}
sqlite3_free(aSzIns);
sqlite3Fts3SegmentsClose(p);
return rc;
}
/*
** Flush any data in the pending-terms hash table to disk. If successful,
** merge all segments in the database (including the new segment, if
** there was any data to flush) into a single segment.
|
︙ | | |
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
|
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
|
+
|
sqlite3Fts3PendingTermsClear(p);
}
}else{
sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0);
sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
}
}
sqlite3Fts3SegmentsClose(p);
return rc;
}
#endif
|