SQLite

Check-in [311aeb9c2b]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Be a bit more susicious of invalid results from the tokenizer. Backports check-in (4514) from fts3. (CVS 5459)
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 311aeb9c2b75c420a37198a93e353c72e9166747
User & Date: shess 2008-07-22 23:54:51.000
Context
2008-07-23
00:52
Documentation updates. (CVS 5460) (check-in: fe8e9f53f4 user: drh tags: trunk)
2008-07-22
23:54
Be a bit more susicious of invalid results from the tokenizer. Backports check-in (4514) from fts3. (CVS 5459) (check-in: 311aeb9c2b user: shess tags: trunk)
23:49
Implement optimize() function. Backports check-in (5417) from fts3. (CVS 5458) (check-in: c16900dc76 user: shess tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts2/fts2.c.
︙︙
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893

3894
3895
3896

3897
3898
3899
3900
3901
3902
3903

  int iStartOffset, iEndOffset, iPosition;
  int rc;

  rc = pTokenizer->pModule->xOpen(pTokenizer, zText, -1, &pCursor);
  if( rc!=SQLITE_OK ) return rc;

  pCursor->pTokenizer = pTokenizer;
  while( SQLITE_OK==pTokenizer->pModule->xNext(pCursor,
                                               &pToken, &nTokenBytes,
                                               &iStartOffset, &iEndOffset,
                                               &iPosition) ){
    DLCollector *p;
    int nData;                   /* Size of doclist before our update. */

    /* Positions can't be negative; we use -1 as a terminator internally. */

    if( iPosition<0 ){
      pTokenizer->pModule->xClose(pCursor);
      return SQLITE_ERROR;

    }

    p = fts2HashFind(&v->pendingTerms, pToken, nTokenBytes);
    if( p==NULL ){
      nData = 0;
      p = dlcNew(iDocid, DL_DEFAULT);
      fts2HashInsert(&v->pendingTerms, pToken, nTokenBytes, p);








|
|
|
|



|
>
|
<
|
>








3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895

3896
3897
3898
3899
3900
3901
3902
3903
3904

  int iStartOffset, iEndOffset, iPosition;
  int rc;

  rc = pTokenizer->pModule->xOpen(pTokenizer, zText, -1, &pCursor);
  if( rc!=SQLITE_OK ) return rc;

  pCursor->pTokenizer = pTokenizer;
  while( SQLITE_OK==(rc=pTokenizer->pModule->xNext(pCursor,
                                                   &pToken, &nTokenBytes,
                                                   &iStartOffset, &iEndOffset,
                                                   &iPosition)) ){
    DLCollector *p;
    int nData;                   /* Size of doclist before our update. */

    /* Positions can't be negative; we use -1 as a terminator
     * internally.  Token can't be NULL or empty. */
    if( iPosition<0 || pToken == NULL || nTokenBytes == 0 ){

      rc = SQLITE_ERROR;
      break;
    }

    p = fts2HashFind(&v->pendingTerms, pToken, nTokenBytes);
    if( p==NULL ){
      nData = 0;
      p = dlcNew(iDocid, DL_DEFAULT);
      fts2HashInsert(&v->pendingTerms, pToken, nTokenBytes, p);
︙︙
3918
3919
3920
3921
3922
3923
3924

3925
3926
3927
3928
3929
3930
3931


  /* TODO(shess) Check return?  Should this be able to cause errors at
  ** this point?  Actually, same question about sqlite3_finalize(),
  ** though one could argue that failure there means that the data is
  ** not durable.  *ponder*
  */
  pTokenizer->pModule->xClose(pCursor);

  return rc;
}

/* Add doclists for all terms in [pValues] to pendingTerms table. */
static int insertTerms(fulltext_vtab *v, sqlite_int64 iRowid,
                       sqlite3_value **pValues){
  int i;








>








3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933


  /* TODO(shess) Check return?  Should this be able to cause errors at
  ** this point?  Actually, same question about sqlite3_finalize(),
  ** though one could argue that failure there means that the data is
  ** not durable.  *ponder*
  */
  pTokenizer->pModule->xClose(pCursor);
  if( SQLITE_DONE == rc ) return SQLITE_OK;
  return rc;
}

/* Add doclists for all terms in [pValues] to pendingTerms table. */
static int insertTerms(fulltext_vtab *v, sqlite_int64 iRowid,
                       sqlite3_value **pValues){
  int i;
︙︙