/ Check-in [deb8f56d]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Be a bit more susicious of invalid results from the tokenizer. (CVS 4514)
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: deb8f56d3adea0025d28b8effabec7c7b7fe3026
User & Date: shess 2007-10-24 23:24:22
Context
2007-10-27
16:25
Clarify the behavior of sqlite3_last_insert_rowid() when using INSERT OR IGNORE. (CVS 4515) check-in: c0fa0c8b user: drh tags: trunk
2007-10-24
23:24
Be a bit more susicious of invalid results from the tokenizer. (CVS 4514) check-in: deb8f56d user: shess tags: trunk
23:04
Fix os2CurrentTime() to give correct results for negative timezones. (CVS 4513) check-in: a3b82548 user: pweilbacher tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130

4131
4132
4133

4134
4135
4136
4137
4138
4139
4140
....
4155
4156
4157
4158
4159
4160
4161

4162
4163
4164
4165
4166
4167
4168
  int iStartOffset, iEndOffset, iPosition;
  int rc;

  rc = pTokenizer->pModule->xOpen(pTokenizer, zText, -1, &pCursor);
  if( rc!=SQLITE_OK ) return rc;

  pCursor->pTokenizer = pTokenizer;
  while( SQLITE_OK==pTokenizer->pModule->xNext(pCursor,
                                               &pToken, &nTokenBytes,
                                               &iStartOffset, &iEndOffset,
                                               &iPosition) ){
    DLCollector *p;
    int nData;                   /* Size of doclist before our update. */

    /* Positions can't be negative; we use -1 as a terminator internally. */

    if( iPosition<0 ){
      pTokenizer->pModule->xClose(pCursor);
      return SQLITE_ERROR;

    }

    p = fts3HashFind(&v->pendingTerms, pToken, nTokenBytes);
    if( p==NULL ){
      nData = 0;
      p = dlcNew(iDocid, DL_DEFAULT);
      fts3HashInsert(&v->pendingTerms, pToken, nTokenBytes, p);
................................................................................

  /* TODO(shess) Check return?  Should this be able to cause errors at
  ** this point?  Actually, same question about sqlite3_finalize(),
  ** though one could argue that failure there means that the data is
  ** not durable.  *ponder*
  */
  pTokenizer->pModule->xClose(pCursor);

  return rc;
}

/* Add doclists for all terms in [pValues] to pendingTerms table. */
static int insertTerms(fulltext_vtab *v, sqlite_int64 iDocid,
                       sqlite3_value **pValues){
  int i;







|
|
|
|



|
>
|
<
|
>







 







>







4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132

4133
4134
4135
4136
4137
4138
4139
4140
4141
....
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
  int iStartOffset, iEndOffset, iPosition;
  int rc;

  rc = pTokenizer->pModule->xOpen(pTokenizer, zText, -1, &pCursor);
  if( rc!=SQLITE_OK ) return rc;

  pCursor->pTokenizer = pTokenizer;
  while( SQLITE_OK==(rc=pTokenizer->pModule->xNext(pCursor,
                                                   &pToken, &nTokenBytes,
                                                   &iStartOffset, &iEndOffset,
                                                   &iPosition)) ){
    DLCollector *p;
    int nData;                   /* Size of doclist before our update. */

    /* Positions can't be negative; we use -1 as a terminator
     * internally.  Token can't be NULL or empty. */
    if( iPosition<0 || pToken == NULL || nTokenBytes == 0 ){

      rc = SQLITE_ERROR;
      break;
    }

    p = fts3HashFind(&v->pendingTerms, pToken, nTokenBytes);
    if( p==NULL ){
      nData = 0;
      p = dlcNew(iDocid, DL_DEFAULT);
      fts3HashInsert(&v->pendingTerms, pToken, nTokenBytes, p);
................................................................................

  /* TODO(shess) Check return?  Should this be able to cause errors at
  ** this point?  Actually, same question about sqlite3_finalize(),
  ** though one could argue that failure there means that the data is
  ** not durable.  *ponder*
  */
  pTokenizer->pModule->xClose(pCursor);
  if( SQLITE_DONE == rc ) return SQLITE_OK;
  return rc;
}

/* Add doclists for all terms in [pValues] to pendingTerms table. */
static int insertTerms(fulltext_vtab *v, sqlite_int64 iDocid,
                       sqlite3_value **pValues){
  int i;