/ Check-in [fa0998e1]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix problems introduced into fts3 as part of the refactoring.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts3-refactor
Files: files | file ages | folders
SHA1: fa0998e19d984ee57f4f506c34eb858026cc49c3
User & Date: dan 2009-11-19 00:15:28
Context
2009-11-19
14:52
Merge the fts3-refactor branch with the trunk. check-in: c8d2bd37 user: dan tags: fts3-refactor
00:15
Fix problems introduced into fts3 as part of the refactoring. check-in: fa0998e1 user: dan tags: fts3-refactor
2009-11-18
15:35
Add some missing comments and fix some other issues in fts3 code. check-in: 2fe579e7 user: dan tags: fts3-refactor
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
...
930
931
932
933
934
935
936

937
938
939
940
941
942
943
944
945
946
947
...
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
....
1417
1418
1419
1420
1421
1422
1423
1424
1425

1426
1427
1428
1429
1430
1431
1432
....
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
....
1542
1543
1544
1545
1546
1547
1548

1549
1550
1551
1552
1553
1554
1555
  char *zContentCols;             /* Columns of %_content table */
  char *zSql;                     /* SQL script to create required tables */

  /* Create a list of user columns for the content table */
  zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY");
  for(i=0; zContentCols && i<p->nColumn; i++){
    char *z = p->azColumn[i];
    zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i+1, z);
  }

  /* Create the whole SQL script */
  zSql = sqlite3_mprintf(
      "CREATE TABLE %Q.'%q_content'(%s);"
      "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);"
      "CREATE TABLE %Q.'%q_segdir'("
................................................................................
    sqlite3_int64 iChild;         /* Block id of child node to descend to */
    int nBlock;                   /* Size of child node in bytes */

    zCsr += sqlite3Fts3GetVarint32(zCsr, &iHeight);
    zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
  
    while( zCsr<zEnd ){

      int nSuffix;                /* Size of term suffix */
      int nPrefix = 0;            /* Size of term prefix */
      int nBuffer;                /* Total term size */
      int nMin;                   /* Minimum of nBuffer and nTerm */
  
      /* Load the next term on the node into zBuffer */
      if( zBuffer ){
        zCsr += sqlite3Fts3GetVarint32(zCsr, &nPrefix);
      }
      zCsr += sqlite3Fts3GetVarint32(zCsr, &nSuffix);
      if( nPrefix+nSuffix>nAlloc ){
................................................................................
      ** to the term from the interior node, then all terms on the sub-tree 
      ** headed by node iChild are smaller than zTerm. No need to search 
      ** iChild.
      **
      ** If the interior node term is larger than the specified term, then
      ** the tree headed by iChild may contain the specified term.
      */
      nMin = (nBuffer>nTerm ? nTerm : nBuffer);
      if( memcmp(zTerm, zBuffer, nMin)<0 ) break;
      iChild++;
    };

    /* If (iHeight==1), the children of this interior node are leaves. The
    ** specified term may be present on leaf node iChild.
    */
    if( iHeight==1 ){
................................................................................
  char *zTerm,
  int nTerm,
  char *aDoclist,
  int nDoclist
){
  TermSelect *pTS = (TermSelect *)pContext;
  int nNew = pTS->nOutput + nDoclist;

  char *aNew = sqlite3_malloc(nNew);

  if( !aNew ){
    return SQLITE_NOMEM;
  }

  if( pTS->nOutput==0 ){
    /* If this is the first term selected, copy the doclist to the output
    ** buffer using memcpy(). TODO: Add a way to transfer control of the
................................................................................
      ** create a Fts3SegReader to scan the single leaf. 
      */
      rc = sqlite3Fts3SegReaderNew(p, iAge, 0, 0, 0, zRoot, nRoot, &pNew);
    }else{
      sqlite3_int64 i1;
      rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1);
      if( rc==SQLITE_OK ){
        sqlite3_int64 i2 = sqlite3_column_int64(pStmt, 3);
        rc = sqlite3Fts3SegReaderNew(p, iAge, i1, i2, 0, 0, 0, &pNew);
      }
    }
    iAge++;

    /* If a new Fts3SegReader was allocated, add it to the apSegment array. */
    assert( (rc==SQLITE_OK)==(pNew!=0) );
................................................................................
  filter.flags = FTS3_SEGMENT_IGNORE_EMPTY 
        | (isPrefix ? FTS3_SEGMENT_PREFIX : 0)
        | (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0)
        | (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0);
  filter.iCol = iColumn;
  filter.zTerm = zTerm;
  filter.nTerm = nTerm;

  rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment, &filter,
      fts3TermSelectCb, (void *)&tsc
  );

  if( rc==SQLITE_OK ){
    *ppOut = tsc.aOutput;
    *pnOut = tsc.nOutput;







|







 







>



<







 







|
|







 







<

>







 







|







 







>







584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
...
930
931
932
933
934
935
936
937
938
939
940

941
942
943
944
945
946
947
...
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
....
1417
1418
1419
1420
1421
1422
1423

1424
1425
1426
1427
1428
1429
1430
1431
1432
....
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
....
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
  char *zContentCols;             /* Columns of %_content table */
  char *zSql;                     /* SQL script to create required tables */

  /* Create a list of user columns for the content table */
  zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY");
  for(i=0; zContentCols && i<p->nColumn; i++){
    char *z = p->azColumn[i];
    zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z);
  }

  /* Create the whole SQL script */
  zSql = sqlite3_mprintf(
      "CREATE TABLE %Q.'%q_content'(%s);"
      "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);"
      "CREATE TABLE %Q.'%q_segdir'("
................................................................................
    sqlite3_int64 iChild;         /* Block id of child node to descend to */
    int nBlock;                   /* Size of child node in bytes */

    zCsr += sqlite3Fts3GetVarint32(zCsr, &iHeight);
    zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
  
    while( zCsr<zEnd ){
      int cmp;                    /* memcmp() result */
      int nSuffix;                /* Size of term suffix */
      int nPrefix = 0;            /* Size of term prefix */
      int nBuffer;                /* Total term size */

  
      /* Load the next term on the node into zBuffer */
      if( zBuffer ){
        zCsr += sqlite3Fts3GetVarint32(zCsr, &nPrefix);
      }
      zCsr += sqlite3Fts3GetVarint32(zCsr, &nSuffix);
      if( nPrefix+nSuffix>nAlloc ){
................................................................................
      ** to the term from the interior node, then all terms on the sub-tree 
      ** headed by node iChild are smaller than zTerm. No need to search 
      ** iChild.
      **
      ** If the interior node term is larger than the specified term, then
      ** the tree headed by iChild may contain the specified term.
      */
      cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer));
      if( cmp<0 || (cmp==0 && nBuffer>nTerm) ) break;
      iChild++;
    };

    /* If (iHeight==1), the children of this interior node are leaves. The
    ** specified term may be present on leaf node iChild.
    */
    if( iHeight==1 ){
................................................................................
  char *zTerm,
  int nTerm,
  char *aDoclist,
  int nDoclist
){
  TermSelect *pTS = (TermSelect *)pContext;
  int nNew = pTS->nOutput + nDoclist;

  char *aNew = sqlite3_malloc(nNew);

  if( !aNew ){
    return SQLITE_NOMEM;
  }

  if( pTS->nOutput==0 ){
    /* If this is the first term selected, copy the doclist to the output
    ** buffer using memcpy(). TODO: Add a way to transfer control of the
................................................................................
      ** create a Fts3SegReader to scan the single leaf. 
      */
      rc = sqlite3Fts3SegReaderNew(p, iAge, 0, 0, 0, zRoot, nRoot, &pNew);
    }else{
      sqlite3_int64 i1;
      rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1);
      if( rc==SQLITE_OK ){
        sqlite3_int64 i2 = sqlite3_column_int64(pStmt, 2);
        rc = sqlite3Fts3SegReaderNew(p, iAge, i1, i2, 0, 0, 0, &pNew);
      }
    }
    iAge++;

    /* If a new Fts3SegReader was allocated, add it to the apSegment array. */
    assert( (rc==SQLITE_OK)==(pNew!=0) );
................................................................................
  filter.flags = FTS3_SEGMENT_IGNORE_EMPTY 
        | (isPrefix ? FTS3_SEGMENT_PREFIX : 0)
        | (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0)
        | (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0);
  filter.iCol = iColumn;
  filter.zTerm = zTerm;
  filter.nTerm = nTerm;

  rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment, &filter,
      fts3TermSelectCb, (void *)&tsc
  );

  if( rc==SQLITE_OK ){
    *ppOut = tsc.aOutput;
    *pnOut = tsc.nOutput;

Changes to ext/fts3/fts3_write.c.

148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178

179
180
181
182
183
184
185
...
247
248
249
250
251
252
253








254
255
256
257
258
259
260
...
698
699
700
701
702
703
704

705
706
707
708
709
710
711
...
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
static int fts3SqlStmt(
  Fts3Table *p, 
  int eStmt, 
  sqlite3_stmt **pp, 
  sqlite3_value **apVal
){
  const char *azSql[] = {
    "DELETE FROM %Q.'%q_content' WHERE rowid = ?",
    "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)",
    "DELETE FROM %Q.'%q_content'",
    "DELETE FROM %Q.'%q_segments'",
    "DELETE FROM %Q.'%q_segdir'",
    "SELECT * FROM %Q.'%q_content' WHERE rowid=?",
    "SELECT coalesce(max(idx)+1, 0) FROM %Q.'%q_segdir' WHERE level=?",
    "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)",
    "SELECT coalesce(max(blockid)+1, 1) FROM %Q.'%q_segments'",
    "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)",

    /* Return segments in order from oldest to newest.*/ 
    "SELECT idx, start_block, leaves_end_block, end_block, root "
        "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC",
    "SELECT idx, start_block, leaves_end_block, end_block, root "
        "FROM %Q.'%q_segdir' ORDER BY level DESC, idx ASC",

    "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?",
    "SELECT count(*), max(level) FROM %Q.'%q_segdir'",

    "DELETE FROM %Q.'%q_segdir' WHERE level = ?",
    "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?",
    0, /* CONTENT_INSERT - generated elsewhere */
    "SELECT block FROM %Q.'%q_segments' WHERE blockid = ?",

  };
  int rc = SQLITE_OK;
  sqlite3_stmt *pStmt;

  assert( SizeofArray(azSql)==SizeofArray(p->aStmt) );
  assert( eStmt<SizeofArray(azSql) && eStmt>=0 );
  
................................................................................
** Set *ppStmt to a statement handle that may be used to iterate through
** all rows in the %_segdir table, from oldest to newest. If successful,
** return SQLITE_OK. If an error occurs while preparing the statement, 
** return an SQLite error code.
**
** There is only ever one instance of this SQL statement compiled for
** each FTS3 table.








*/
int sqlite3Fts3AllSegdirs(Fts3Table *p, sqlite3_stmt **ppStmt){
  return fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, ppStmt, 0);
}


static int fts3SqlExec(Fts3Table *p, int eStmt, sqlite3_value **apVal){
................................................................................
    pReader->zTerm = zNew;
    pReader->nTermAlloc = nNew;
  }
  memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix);
  pReader->nTerm = nPrefix+nSuffix;
  pNext += nSuffix;
  pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist);

  pReader->aDoclist = pNext;
  pReader->pOffsetList = 0;
  return SQLITE_OK;
}

static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){
  int n;
................................................................................
  }

  /* If there are no more entries in the doclist, set pOffsetList to
  ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and
  ** Fts3SegReader.pOffsetList to point to the next offset list before
  ** returning.
  */
  if( p==&pReader->aDoclist[pReader->nDoclist] ){
    pReader->pOffsetList = 0;
  }else{
    sqlite3_int64 iDelta;
    pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta);
    pReader->iDocid += iDelta;
  }
}







|
|
|
|
|
|
|
|
|
|

|
|
|
|
|

|
|

|
|
|
|
>







 







>
>
>
>
>
>
>
>







 







>







 







|







148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
...
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
...
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
...
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
static int fts3SqlStmt(
  Fts3Table *p, 
  int eStmt, 
  sqlite3_stmt **pp, 
  sqlite3_value **apVal
){
  const char *azSql[] = {
/* 0  */  "DELETE FROM %Q.'%q_content' WHERE rowid = ?",
/* 1  */  "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)",
/* 2  */  "DELETE FROM %Q.'%q_content'",
/* 3  */  "DELETE FROM %Q.'%q_segments'",
/* 4  */  "DELETE FROM %Q.'%q_segdir'",
/* 5  */  "SELECT * FROM %Q.'%q_content' WHERE rowid=?",
/* 6  */  "SELECT coalesce(max(idx)+1, 0) FROM %Q.'%q_segdir' WHERE level=?",
/* 7  */  "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)",
/* 8  */  "SELECT coalesce(max(blockid)+1, 1) FROM %Q.'%q_segments'",
/* 9  */  "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)",

          /* Return segments in order from oldest to newest.*/ 
/* 10 */  "SELECT idx, start_block, leaves_end_block, end_block, root "
            "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC",
/* 11 */  "SELECT idx, start_block, leaves_end_block, end_block, root "
            "FROM %Q.'%q_segdir' ORDER BY level DESC, idx ASC",

/* 12 */  "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?",
/* 13 */  "SELECT count(*), max(level) FROM %Q.'%q_segdir'",

/* 14 */  "DELETE FROM %Q.'%q_segdir' WHERE level = ?",
/* 15 */  "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?",
/* 16 */  0, /* CONTENT_INSERT - generated elsewhere */
/* 17 */  "SELECT block FROM %Q.'%q_segments' WHERE blockid = ?",

  };
  int rc = SQLITE_OK;
  sqlite3_stmt *pStmt;

  assert( SizeofArray(azSql)==SizeofArray(p->aStmt) );
  assert( eStmt<SizeofArray(azSql) && eStmt>=0 );
  
................................................................................
** Set *ppStmt to a statement handle that may be used to iterate through
** all rows in the %_segdir table, from oldest to newest. If successful,
** return SQLITE_OK. If an error occurs while preparing the statement, 
** return an SQLite error code.
**
** There is only ever one instance of this SQL statement compiled for
** each FTS3 table.
**
** The statement returns the following columns from the %_segdir table:
**
**   0: idx
**   1: start_block
**   2: leaves_end_block
**   3: end_block
**   4: root
*/
int sqlite3Fts3AllSegdirs(Fts3Table *p, sqlite3_stmt **ppStmt){
  return fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, ppStmt, 0);
}


static int fts3SqlExec(Fts3Table *p, int eStmt, sqlite3_value **apVal){
................................................................................
    pReader->zTerm = zNew;
    pReader->nTermAlloc = nNew;
  }
  memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix);
  pReader->nTerm = nPrefix+nSuffix;
  pNext += nSuffix;
  pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist);
  assert( pNext<&pReader->aNode[pReader->nNode] );
  pReader->aDoclist = pNext;
  pReader->pOffsetList = 0;
  return SQLITE_OK;
}

static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){
  int n;
................................................................................
  }

  /* If there are no more entries in the doclist, set pOffsetList to
  ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and
  ** Fts3SegReader.pOffsetList to point to the next offset list before
  ** returning.
  */
  if( p>=&pReader->aDoclist[pReader->nDoclist] ){
    pReader->pOffsetList = 0;
  }else{
    sqlite3_int64 iDelta;
    pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta);
    pReader->iDocid += iDelta;
  }
}