SQLite

*/
/*
** Read a single block from the %_segments table.
*/
static int fts3ReadBlock(
  Fts3Table *p,
  sqlite3_int64 iBlock,
  char const **pzBlock,
  int *pnBlock
){
  sqlite3_stmt *pStmt;
  int rc = sqlite3Fts3SqlStmt(p, FTS3_SQL_GET_BLOCK, &pStmt);
  if( rc!=SQLITE_OK ) return rc;
  sqlite3_reset(pStmt);

  if( !*pzBlock ){
    return SQLITE_NOMEM;
  }
  return SQLITE_OK;
}

/*
** The buffer pointed to by argument zNode (size nNode bytes) contains the
** root node of a b-tree segment. The segment is guaranteed to be at least
** one level high (i.e. the root node is not also a leaf). If successful,
** this function locates the leaf node of the segment that may contain the 
** term specified by arguments zTerm and nTerm and writes its block number 
** to *piLeaf.
**
** It is possible that the returned leaf node does not contain the specified
** term. However, if the segment does contain said term, it is stored on
** the identified leaf node. Because this function only inspects interior
** segment nodes (and never loads leaf nodes into memory), it is not possible
** to be sure.
**
** If an error occurs, an error code other than SQLITE_OK is returned.


*/ 
static int fts3SelectLeaf(
  Fts3Table *p,                   /* Virtual table handle */
  const char *zTerm,              /* Term to select leaves for */
  int nTerm,                      /* Size of term zTerm in bytes */

  const char *zNode,              /* Buffer containing segment interior node */
  int nNode,                      /* Size of buffer at zNode */
  sqlite3_int64 *piLeaf           /* Selected leaf node */

){
  int rc = SQLITE_OK;             /* Return code */
  const char *zCsr = zNode;       /* Cursor to iterate through node */
  const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
  char *zBuffer = 0;              /* Buffer to load terms into */
  int nAlloc = 0;                 /* Size of allocated buffer */

  while( 1 ){
    int iHeight;                  /* Height of this node in tree */
    sqlite3_int64 iChild;         /* Block id of child node to descend to */

    int nBlock;                   /* Size of child node in bytes */

    zCsr += sqlite3Fts3GetVarint32(zCsr, &iHeight);
    zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
  
    while( zCsr<zEnd ){
      int nSuffix;                /* Size of term suffix */
      int nPrefix = 0;            /* Size of term prefix */
      int nBuffer;                /* Total term size */
      int nMin;                   /* Minimum of nBuffer and nTerm */

  
      /* Load the next term on the node into zBuffer */
      if( zBuffer ){
        zCsr += sqlite3Fts3GetVarint32(zCsr, &nPrefix);
      }
      zCsr += sqlite3Fts3GetVarint32(zCsr, &nSuffix);
      if( nPrefix+nSuffix>nAlloc ){
        char *zNew;
        nAlloc = (nPrefix+nSuffix) * 2;
        zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
        if( !zNew ){
          sqlite3_free(zBuffer);
          return SQLITE_NOMEM;
        }
        zBuffer = zNew;
      }
      memcpy(&zBuffer[nPrefix], zCsr, nSuffix);
      nBuffer = nPrefix + nSuffix;
      zCsr += nSuffix;
  
      /* Compare the term we are searching for with the term just loaded from
      ** the interior node. If the specified term is greater than or equal
      ** to the term from the interior node, then all terms on the sub-tree 
      ** headed by node iChild are smaller than zTerm. No need to search 
      ** iChild.
      **
      ** If the interior node term is larger than the specified term, then
      ** the tree headed by iChild may contain the specified term.
      */
      nMin = (nBuffer>nTerm ? nTerm : nBuffer);
      if( memcmp(zTerm, zBuffer, nMin)<0 ) break;
      iChild++;
    };

    /* If (iHeight==1), the children of this interior node are leaves. The
    ** specified term may be present on leaf node iChild.
    */












    if( iHeight==1 ){
      *piLeaf = iChild;



      break;




    }


    /* Descend to interior node iChild. */
    rc = fts3ReadBlock(p, iChild, &zCsr, &nBlock);
    if( rc!=SQLITE_OK ) break;
    zEnd = &zCsr[nBlock];
  }


  sqlite3_free(zBuffer);
  return rc;
}

static void fts3PutDeltaVarint(
  char **pp, 
  sqlite3_int64 *piPrev, 
  sqlite3_int64 iVal

      if( mergetype==MERGE_POS_NEAR ){
        ppPos = &p;
        aTmp = sqlite3_malloc(2*(n1+n2));
        if( !aTmp ){
          return SQLITE_NOMEM;
        }
      }


      while( p1 && p2 ){
        if( i1==i2 ){
          char *pSave = p;
          sqlite3_int64 iPrevSave = iPrev;
          fts3PutDeltaVarint(&p, &iPrev, i1);

      assert(!"Invalid mergetype value passed to fts3DoclistMerge()");
  }

  *pnBuffer = (p-aBuffer);
  return SQLITE_OK;
}

/* 
** A pointer to an instance of this structure is used as the context 
** argument to sqlite3Fts3SegReaderIterate()
*/
typedef struct TermSelect TermSelect;
struct TermSelect {



  int isReqPos;
  char *aOutput;                  /* Malloc'd output buffer */
  int nOutput;                    /* Size of output in bytes */
};

static int fts3TermSelectCb(
  Fts3Table *p,                   /* Virtual table object */
  void *pContext,                 /* Pointer to TermSelect structure */
  char *zTerm,
  int nTerm,
  char *aDoclist,
  int nDoclist
){
  TermSelect *pTS = (TermSelect *)pContext;




  int nNew = pTS->nOutput + nDoclist;

  char *aNew = sqlite3_malloc(nNew);
  if( !aNew ){
    return SQLITE_NOMEM;
  }

  if( pTS->nOutput==0 ){
    /* If this is the first term selected, copy the doclist to the output
    ** buffer using memcpy(). TODO: Add a way to transfer control of the
    ** aDoclist buffer from the caller so as to avoid the memcpy().
    */
    memcpy(aNew, aDoclist, nDoclist);
  }else{
    /* The output buffer is not empty. Merge doclist aDoclist with the
    ** existing output. This can only happen with prefix-searches (as
    ** searches for exact terms return exactly one doclist).
    */
    int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR);

    fts3DoclistMerge(mergetype, 0, 0,
        aNew, &nNew, pTS->aOutput, pTS->nOutput, aDoclist, nDoclist
    );
  }

  sqlite3_free(pTS->aOutput);
  pTS->aOutput = aNew;
  pTS->nOutput = nNew;


  return SQLITE_OK;
}

/*
** This function retreives the doclist for the specified term (or term
** prefix) from the database. 

  int isPrefix,                   /* True for a prefix search */
  int isReqPos,                   /* True to include position lists in output */
  int *pnOut,                     /* OUT: Size of buffer at *ppOut */
  char **ppOut                    /* OUT: Malloced result buffer */
){
  int i;
  TermSelect tsc;
  Fts3SegFilter filter;           /* Segment term filter configuration */
  Fts3SegReader **apSegment = 0;  /* Array of segments to read data from */
  int nSegment = 0;               /* Size of apSegment array */
  int nAlloc = 0;                 /* Allocated size of segment array */
  int rc;                         /* Return code */
  sqlite3_stmt *pStmt;            /* SQL statement to scan %_segdir table */
  int iAge = 0;                   /* Used to assign ages to segments */


  /* Loop through the entire %_segdir table. For each segment, create a
  ** Fts3SegReader to iterate through the subset of the segment leaves
  ** that may contain a term that matches zTerm/nTerm. For non-prefix
  ** searches, this is always a single leaf. For prefix searches, this
  ** may be a contiguous block of leaves.
  **

    if( sqlite3_column_int64(pStmt, 1)==0 ){
      /* The entire segment is stored on the root node (which must be a
      ** leaf). Do not bother inspecting any data in this case, just
      ** create a Fts3SegReader to scan the single leaf. 
      */
      rc = sqlite3Fts3SegReaderNew(p, iAge, 0, 0, 0, zRoot, nRoot, &pNew);
    }else{
      sqlite3_int64 i1;
      rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1);
      if( rc==SQLITE_OK ){
        sqlite3_int64 i2 = sqlite3_column_int64(pStmt, 3);
        rc = sqlite3Fts3SegReaderNew(p, iAge, i1, i2, 0, 0, 0, &pNew);
      }
    }
    iAge++;

    /* If a new Fts3SegReader was allocated, add it to the apSegment array. */
    assert( (rc==SQLITE_OK)==(pNew!=0) );

  }
  if( rc!=SQLITE_DONE ){
    assert( rc!=SQLITE_OK );
    goto finished;
  }

  memset(&tsc, 0, sizeof(TermSelect));



  tsc.isReqPos = isReqPos;

  filter.flags = FTS3_SEGMENT_IGNORE_EMPTY 
        | (isPrefix ? FTS3_SEGMENT_PREFIX : 0)
        | (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0)
        | (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0);
  filter.iCol = iColumn;
  filter.zTerm = zTerm;
  filter.nTerm = nTerm;
  rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment, &filter,
      fts3TermSelectCb, (void *)&tsc
  );

  if( rc==SQLITE_OK ){
    *ppOut = tsc.aOutput;
    *pnOut = tsc.nOutput;
  }else{
    sqlite3_free(tsc.aOutput);

#define FTS3_VARINT_MAX 10

typedef struct Fts3Table Fts3Table;
typedef struct Fts3Cursor Fts3Cursor;
typedef struct Fts3Expr Fts3Expr;
typedef struct Fts3Phrase Fts3Phrase;
typedef struct Fts3SegReader Fts3SegReader;
typedef struct Fts3SegFilter Fts3SegFilter;

/*
** A connection to a fulltext index is an instance of the following
** structure. The xCreate and xConnect methods create an instance
** of this structure and xDestroy and xDisconnect free that instance.
** All other methods receive a pointer to the structure as one of their
** arguments.

/*
** A "phrase" is a sequence of one or more tokens that must match in
** sequence.  A single token is the base case and the most common case.
** For a sequence of tokens contained in "...", nToken will be the number
** of tokens in the string.
*/
struct Fts3Phrase {
  int nToken;                /* Number of tokens in the phrase */
  int iColumn;               /* Index of column this phrase must match */
  int isNot;                 /* Phrase prefixed by unary not (-) operator */
  struct PhraseToken {
    char *z;                 /* Text of the token */
    int n;                   /* Number of bytes in buffer pointed to by z */
    int isPrefix;            /* True if token ends in with a "*" character */
  } aToken[1];               /* One entry for each token in the phrase */
};

/*
** A tree of these objects forms the RHS of a MATCH operator.
*/
struct Fts3Expr {
  int eType;                 /* One of the FTSQUERY_XXX values defined below */

void sqlite3Fts3PendingTermsClear(Fts3Table *);
int sqlite3Fts3Optimize(Fts3Table *);

/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */
#define FTS3_SEGMENT_REQUIRE_POS   0x00000001
#define FTS3_SEGMENT_IGNORE_EMPTY  0x00000002
#define FTS3_SEGMENT_COLUMN_FILTER 0x00000004
#define FTS3_SEGMENT_PREFIX        0x00000008

struct Fts3SegFilter {
  const char *zTerm;
  int nTerm;
  int iCol;
  int flags;
};

int sqlite3Fts3SegReaderNew(Fts3Table *,int, sqlite3_int64,
  sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**);
void sqlite3Fts3SegReaderFree(Fts3SegReader *);

int sqlite3Fts3SegReaderIterate(
  Fts3Table *, Fts3SegReader **, int, Fts3SegFilter *,
  int (*)(Fts3Table *, void *, char *, int, char *, int),  void *
);

/* fts3.c */
int sqlite3Fts3PutVarint(char *, sqlite3_int64);
int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
int sqlite3Fts3GetVarint32(const char *, int *);

    }else{
      rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1;
    }
  }
  assert( pLhs->aNode && pRhs->aNode );
  return rc;
}

/*
** Compare the term that the Fts3SegReader object passed as the first argument
** points to with the term specified by arguments zTerm and nTerm. 
**
** If the pSeg iterator is already at EOF, return 0. Otherwise, return
** -ve if the pSeg term is less than zTerm/nTerm, 0 if the two terms are
** equal, or +ve if the pSeg term is greater than zTerm/nTerm.
*/
static int fts3SegReaderTermCmp(
  Fts3SegReader *pSeg,            /* Segment reader object */
  const char *zTerm,              /* Term to compare to */
  int nTerm                       /* Size of term zTerm in bytes */
){
  int res = 0;
  if( pSeg->aNode ){
    if( pSeg->nTerm>nTerm ){
      res = memcmp(pSeg->zTerm, zTerm, nTerm);
    }else{
      res = memcmp(pSeg->zTerm, zTerm, pSeg->nTerm);
    }
    if( res==0 ){
      res = pSeg->nTerm-nTerm;
    }
  }
  return res;
}

/*
** Argument apSegment is an array of nSegment elements. It is known that
** the final (nSegment-nSuspect) members are already in sorted order
** (according to the comparison function provided). This function shuffles
** the array around until all entries are in sorted order.
*/

}

static void fts3ColumnFilter(int iCol, char **ppList, int *pnList){
  char *pList = *ppList;
  int nList = *pnList;
  char *pEnd = &pList[nList];
  int iCurrent = 0;

  char *p = pList;

  assert( iCol>=0 );
  while( 1 ){
    char c = 0;
    while( p<pEnd && (c | *p)&0xFE ) c = *p++ & 0x80;
  
    if( iCol==iCurrent ){
      nList = (p - pList);
      break;

  return fts3LeafAdd(p, ppW, 1, zTerm, nTerm, aDoclist, nDoclist);
}

int sqlite3Fts3SegReaderIterate(
  Fts3Table *p,                   /* Virtual table handle */
  Fts3SegReader **apSegment,      /* Array of Fts3SegReader objects */
  int nSegment,                   /* Size of apSegment array */
  Fts3SegFilter *pFilter,         /* Restrictions on range of iteration */

  int (*xFunc)(Fts3Table *, void *, char *, int, char *, int),  /* Callback */
  void *pContext                  /* Callback context (2nd argument) */
){
  int i;                          /* Iterator variable */
  char *aBuffer = 0;              /* Buffer to merge doclists in */
  int nAlloc = 0;                 /* Allocated size of aBuffer buffer */
  int rc = SQLITE_OK;             /* Return code */

  int isIgnoreEmpty =  (pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY);
  int isRequirePos =   (pFilter->flags & FTS3_SEGMENT_REQUIRE_POS);
  int isColFilter =    (pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER);
  int isPrefix =       (pFilter->flags & FTS3_SEGMENT_PREFIX);

  /* If the Fts3SegFilter defines a specific term (or term prefix) to search 
  ** for, then advance each segment iterator until it points to a term of
  ** equal or greater value than the specified term. This prevents many
  ** unnecessary merge/sort operations for the case where single segment
  ** b-tree leaf nodes contain more than one term.
  */
  if( pFilter->zTerm ){
    int nTerm = pFilter->nTerm;
    char *zTerm = pFilter->zTerm;
    for(i=0; i<nSegment; i++){
      Fts3SegReader *pSeg = apSegment[i];
      while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ){
        rc = fts3SegReaderNext(pSeg);
        if( rc!=SQLITE_OK ) goto finished;
      }
    }
  }

  fts3SegReaderSort(apSegment, nSegment, nSegment, fts3SegReaderCmp);
  while( apSegment[0]->aNode ){
    int nTerm = apSegment[0]->nTerm;
    char *zTerm = apSegment[0]->zTerm;
    int nMerge = 1;

    /* If this is a prefix-search, and if the term that apSegment[0] points
    ** to does not share a suffix with pFilter->zTerm/nTerm, then all 
    ** required callbacks have been made. In this case exit early.
    **
    ** Similarly, if this is a search for an exact match, and the first term
    ** of segment apSegment[0] is not a match, exit early.
    */
    if( pFilter->zTerm ){
      if( nTerm<pFilter->nTerm 
       || (!isPrefix && nTerm>pFilter->nTerm)
       || memcmp(zTerm, pFilter->zTerm, pFilter->nTerm) 
    ){
        goto finished;
      }
    }

    while( nMerge<nSegment 
        && apSegment[nMerge]->aNode
        && apSegment[nMerge]->nTerm==nTerm 
        && 0==memcmp(zTerm, apSegment[nMerge]->zTerm, nTerm)
    ){
      nMerge++;

            && apSegment[j]->pOffsetList 
            && apSegment[j]->iDocid==iDocid 
        ){
          fts3SegReaderNextDocid(apSegment[j], 0, 0);
          j++;
        }


        if( isColFilter ){
          fts3ColumnFilter(pFilter->iCol, &pList, &nList);
        }

        if( !isIgnoreEmpty || nList>0 ){
          nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0);
          if( nDoclist+nByte>nAlloc ){
            char *aNew;
            nAlloc = nDoclist+nByte*2;

      }

      if( nDoclist>0 ){
        rc = xFunc(p, pContext, zTerm, nTerm, aBuffer, nDoclist);
        if( rc!=SQLITE_OK ) goto finished;
      }
    }

    /* If there is a term specified to filter on, and this is not a prefix
    ** search, return now. The callback that corresponds to the required
    ** term (if such a term exists in the index) has already been made.
    */
    if( pFilter->zTerm && !isPrefix ){
      goto finished;
    }

    for(i=0; i<nMerge; i++){
      rc = fts3SegReaderNext(apSegment[i]);
      if( rc!=SQLITE_OK ) goto finished;
    }
    fts3SegReaderSort(apSegment, nSegment, nMerge, fts3SegReaderCmp);
  }

  int rc;                         /* Return code */
  int iIdx;                       /* Index of new segment */
  int iNewLevel;                  /* Level to create new segment at */
  sqlite3_stmt *pStmt;
  SegmentWriter *pWriter = 0;
  int nSegment = 0;               /* Number of segments being merged */
  Fts3SegReader **apSegment = 0;  /* Array of Segment iterators */
  Fts3SegFilter filter;           /* Segment term filter condition */

  if( iLevel<0 ){
    /* This call is to merge all segments in the database to a single
    ** segment. The level of the new segment is equal to the the numerically 
    ** greatest segment level currently present in the database. The index
    ** of the new segment is always 0.
    */

      goto finished;
    }
  }
  rc = sqlite3_reset(pStmt);
  pStmt = 0;
  if( rc!=SQLITE_OK ) goto finished;

  memset(&filter, 0, sizeof(Fts3SegFilter));
  filter.flags = FTS3_SEGMENT_REQUIRE_POS;
  filter.flags |= (iLevel<0 ? FTS3_SEGMENT_IGNORE_EMPTY : 0);
  rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment,


      &filter, fts3MergeCallback, (void *)&pWriter
  );
  if( rc!=SQLITE_OK ) goto finished;

  rc = fts3DeleteSegdir(p, iLevel, apSegment, nSegment);
  if( rc==SQLITE_OK ){
    rc = fts3LeafWrite(p, pWriter, iNewLevel, iIdx);
  }

  CREATE VIRTUAL TABLE ft5 USING fts3(a, b, tokenize unknown)
} {unknown tokenizer: unknown}
do_write_test fts3_malloc-1.6 sqlite_master {
  CREATE VIRTUAL TABLE ft6 USING fts3(a, b, tokenize porter)
}

# Test the xConnect/xDisconnect methods:
#db eval { ATTACH 'test2.db' AS aux }
#do_write_test fts3_malloc-1.6 aux.sqlite_master {
#  CREATE VIRTUAL TABLE aux.ft7 USING fts3(a, b, c);

#}
#do_write_test fts3_malloc-1.6 aux.sqlite_master {
#  CREATE VIRTUAL TABLE aux.ft7 USING fts3(a, b, c);

#}



do_test fts3_malloc-2.0 {
  execsql { 
    DROP TABLE ft1;
    DROP TABLE ft2;
    DROP TABLE ft3;
    DROP TABLE ft4;
    DROP TABLE ft6;
  }
  execsql { CREATE VIRTUAL TABLE ft USING fts3(a, b) }
  for {set ii 1} {$ii < 32} {incr ii} {
    set a [list]
    set b [list]
    if {$ii & 0x01} {lappend a one   ; lappend b neung}
    if {$ii & 0x02} {lappend a two   ; lappend b song }
    if {$ii & 0x04} {lappend a three ; lappend b sahm }