SQLite

Check-in [8630996134]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add the "unindexed" column option to fts5.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: 86309961344f4076ddcf55d730d3600ec3b6e45c
User & Date: dan 2015-04-24 19:41:43.259
Context
2015-04-24
20:18
Merge latest trunk changes with this branch. (check-in: 1c78d8920f user: dan tags: fts5)
19:41
Add the "unindexed" column option to fts5. (check-in: 8630996134 user: dan tags: fts5)
15:56
Add extra tests for corrupt database handling in fts5. (check-in: 41449f7a0b user: dan tags: fts5)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts5/fts5Int.h.
88
89
90
91
92
93
94

95
96
97
98
99
100
101
*/
struct Fts5Config {
  sqlite3 *db;                    /* Database handle */
  char *zDb;                      /* Database holding FTS index (e.g. "main") */
  char *zName;                    /* Name of FTS index */
  int nCol;                       /* Number of columns */
  char **azCol;                   /* Column names */

  int nPrefix;                    /* Number of prefix indexes */
  int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */
  int eContent;                   /* An FTS5_CONTENT value */
  char *zContent;                 /* content table */ 
  char *zContentRowid;            /* "content_rowid=" option value */ 
  Fts5Tokenizer *pTok;
  fts5_tokenizer *pTokApi;







>







88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
*/
struct Fts5Config {
  sqlite3 *db;                    /* Database handle */
  char *zDb;                      /* Database holding FTS index (e.g. "main") */
  char *zName;                    /* Name of FTS index */
  int nCol;                       /* Number of columns */
  char **azCol;                   /* Column names */
  u8 *abUnindexed;                /* True for unindexed columns */
  int nPrefix;                    /* Number of prefix indexes */
  int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */
  int eContent;                   /* An FTS5_CONTENT value */
  char *zContent;                 /* content table */ 
  char *zContentRowid;            /* "content_rowid=" option value */ 
  Fts5Tokenizer *pTok;
  fts5_tokenizer *pTokApi;
Changes to ext/fts5/fts5_config.c.
130
131
132
133
134
135
136













137
138
139
140
141
142
143

144
145
146
147
148
149

150
151
152


153
154


155
156
157
158
159
160


161
162
163
164
165
166
167
168
        break;
    }
  }

  return p;
}














static int fts5Dequote(char *z){
  char q;
  int iIn = 1;
  int iOut = 0;
  int bRet = 1;
  q = z[0];


  assert( q=='[' || q=='\'' || q=='"' || q=='`' );
  if( q=='[' ) q = ']';  

  while( z[iIn] ){
    if( z[iIn]==q ){
      if( z[iIn+1]!=q ){

        if( z[iIn+1]=='\0' ) bRet = 0;
        break;
      }


      z[iOut++] = q;
      iIn += 2;


    }else{
      z[iOut++] = z[iIn++];
    }
  }
  z[iOut] = '\0';



  return bRet;
}

/*
** Convert an SQL-style quoted string into a normal string by removing
** the quote characters.  The conversion is done in-place.  If the
** input does not begin with a quote character, then this routine
** is a no-op.







>
>
>
>
>
>
>
>
>
>
>
>
>




<


>






>
|
|
|
>
>
|
|
>
>




<

>
>
|







130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153

154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176

177
178
179
180
181
182
183
184
185
186
187
        break;
    }
  }

  return p;
}

/*
** The first character of the string pointed to by argument z is guaranteed
** to be an open-quote character (see function fts5_isopenquote()).
**
** This function searches for the corresponding close-quote character within
** the string and, if found, dequotes the string in place and adds a new
** nul-terminator byte.
**
** If the close-quote is found, the value returned is the byte offset of
** the character immediately following it. Or, if the close-quote is not 
** found, -1 is returned. If -1 is returned, the buffer is left in an 
** undefined state.
*/
static int fts5Dequote(char *z){
  char q;
  int iIn = 1;
  int iOut = 0;

  q = z[0];

  /* Set stack variable q to the close-quote character */
  assert( q=='[' || q=='\'' || q=='"' || q=='`' );
  if( q=='[' ) q = ']';  

  while( z[iIn] ){
    if( z[iIn]==q ){
      if( z[iIn+1]!=q ){
        /* Character iIn was the close quote. */
        z[iOut] = '\0';
        return iIn+1;
      }else{
        /* Character iIn and iIn+1 form an escaped quote character. Skip
        ** the input cursor past both and copy a single quote character 
        ** to the output buffer. */
        iIn += 2;
        z[iOut++] = q;
      }
    }else{
      z[iOut++] = z[iIn++];
    }
  }


  /* Did not find the close-quote character. Return -1. */
  z[iOut] = '\0';
  return -1;
}

/*
** Convert an SQL-style quoted string into a normal string by removing
** the quote characters.  The conversion is done in-place.  If the
** input does not begin with a quote character, then this routine
** is a no-op.
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
  assert( 0==fts5_iswhitespace(z[0]) );
  quote = z[0];
  if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){
    fts5Dequote(z);
  }
}

/*
** Trim any white-space from the right of nul-terminated string z.
*/
static char *fts5TrimString(char *z){
  int n = strlen(z);
  while( n>0 && fts5_iswhitespace(z[n-1]) ){
    z[--n] = '\0';
  }
  while( fts5_iswhitespace(*z) ) z++;
  return z;
}

/*
** Duplicate the string passed as the only argument into a buffer allocated
** by sqlite3_malloc().
**
** Return 0 if an OOM error is encountered.
*/
static char *fts5Strdup(int *pRc, const char *z){







<
<
<
<
<
<
<
<
<
<
<
<







199
200
201
202
203
204
205












206
207
208
209
210
211
212
  assert( 0==fts5_iswhitespace(z[0]) );
  quote = z[0];
  if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){
    fts5Dequote(z);
  }
}













/*
** Duplicate the string passed as the only argument into a buffer allocated
** by sqlite3_malloc().
**
** Return 0 if an OOM error is encountered.
*/
static char *fts5Strdup(int *pRc, const char *z){
247
248
249
250
251
252
253
254
255
256
257

258
259
260
261
262
263
264
** may be left in *pzErr. It is the responsibility of the caller to
** eventually free any such error message using sqlite3_free().
*/
static int fts5ConfigParseSpecial(
  Fts5Global *pGlobal,
  Fts5Config *pConfig,            /* Configuration object to update */
  const char *zCmd,               /* Special command to parse */
  int nCmd,                       /* Size of zCmd in bytes */
  const char *zArg,               /* Argument to parse */
  char **pzErr                    /* OUT: Error message */
){

  if( sqlite3_strnicmp("prefix", zCmd, nCmd)==0 ){
    const int nByte = sizeof(int) * FTS5_MAX_PREFIX_INDEXES;
    int rc = SQLITE_OK;
    const char *p;
    if( pConfig->aPrefix ){
      *pzErr = sqlite3_mprintf("multiple prefix=... directives");
      rc = SQLITE_ERROR;







<



>







254
255
256
257
258
259
260

261
262
263
264
265
266
267
268
269
270
271
** may be left in *pzErr. It is the responsibility of the caller to
** eventually free any such error message using sqlite3_free().
*/
static int fts5ConfigParseSpecial(
  Fts5Global *pGlobal,
  Fts5Config *pConfig,            /* Configuration object to update */
  const char *zCmd,               /* Special command to parse */

  const char *zArg,               /* Argument to parse */
  char **pzErr                    /* OUT: Error message */
){
  int nCmd = strlen(zCmd);
  if( sqlite3_strnicmp("prefix", zCmd, nCmd)==0 ){
    const int nByte = sizeof(int) * FTS5_MAX_PREFIX_INDEXES;
    int rc = SQLITE_OK;
    const char *p;
    if( pConfig->aPrefix ){
      *pzErr = sqlite3_mprintf("multiple prefix=... directives");
      rc = SQLITE_ERROR;
379
380
381
382
383
384
385














































































386
387
388
389
390
391
392
*/
static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){
  assert( pConfig->pTok==0 && pConfig->pTokApi==0 );
  return sqlite3Fts5GetTokenizer(
      pGlobal, 0, 0, &pConfig->pTok, &pConfig->pTokApi
  );
}















































































/*
** Arguments nArg/azArg contain the string arguments passed to the xCreate
** or xConnect method of the virtual table. This function attempts to 
** allocate an instance of Fts5Config containing the results of parsing
** those arguments.
**







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
*/
static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){
  assert( pConfig->pTok==0 && pConfig->pTokApi==0 );
  return sqlite3Fts5GetTokenizer(
      pGlobal, 0, 0, &pConfig->pTok, &pConfig->pTokApi
  );
}

/*
** Gobble up the first bareword or quoted word from the input buffer zIn.
** Return a pointer to the character immediately following the last in
** the gobbled word if successful, or a NULL pointer otherwise (failed
** to find close-quote character).
**
** Before returning, set pzOut to point to a new buffer containing a
** nul-terminated, dequoted copy of the gobbled word. If the word was
** quoted, *pbQuoted is also set to 1 before returning.
**
** If *pRc is other than SQLITE_OK when this function is called, it is
** a no-op (NULL is returned). Otherwise, if an OOM occurs within this
** function, *pRc is set to SQLITE_NOMEM before returning. *pRc is *not*
** set if a parse error (failed to find close quote) occurs.
*/
static const char *fts5ConfigGobbleWord(
  int *pRc, 
  const char *zIn, 
  char **pzOut, 
  int *pbQuoted
){
  const char *zRet = 0;
  *pbQuoted = 0;
  *pzOut = 0;

  if( *pRc==SQLITE_OK ){
    int nIn = strlen(zIn);
    char *zOut = sqlite3_malloc(nIn+1);

    if( zOut==0 ){
      *pRc = SQLITE_NOMEM;
    }else{
      memcpy(zOut, zIn, nIn+1);
      if( fts5_isopenquote(zOut[0]) ){
        int ii = fts5Dequote(zOut);
        if( ii>0 ) zRet = &zIn[ii];
        *pbQuoted = 1;
      }else{
        zRet = fts5ConfigSkipBareword(zIn);
        zOut[zRet-zIn] = '\0';
      }
    }

    if( zRet==0 ){
      sqlite3_free(zOut);
    }else{
      *pzOut = zOut;
    }
  }

  return zRet;
}

static int fts5ConfigParseColumn(
  Fts5Config *p, 
  char *zCol, 
  char *zArg, 
  char **pzErr
){
  int rc = SQLITE_OK;
  if( 0==sqlite3_stricmp(zCol, FTS5_RANK_NAME) 
   || 0==sqlite3_stricmp(zCol, FTS5_ROWID_NAME) 
  ){
    *pzErr = sqlite3_mprintf("reserved fts5 column name: %s", zCol);
    rc = SQLITE_ERROR;
  }else if( zArg ){
    if( 0==sqlite3_stricmp(zArg, "unindexed") ){
      p->abUnindexed[p->nCol] = 1;
    }else{
      *pzErr = sqlite3_mprintf("unrecognized column option: %s", zArg);
      rc = SQLITE_ERROR;
    }
  }

  p->azCol[p->nCol++] = zCol;
  return rc;
}

/*
** Arguments nArg/azArg contain the string arguments passed to the xCreate
** or xConnect method of the virtual table. This function attempts to 
** allocate an instance of Fts5Config containing the results of parsing
** those arguments.
**
403
404
405
406
407
408
409

410
411
412
413
414
415
416

417

418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446

447
448
449

450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477


478
479
480
481
482
483
484
485
486
487
488
489
  const char **azArg,             /* Array of nArg CREATE VIRTUAL TABLE args */
  Fts5Config **ppOut,             /* OUT: Results of parse */
  char **pzErr                    /* OUT: Error message */
){
  int rc = SQLITE_OK;             /* Return code */
  Fts5Config *pRet;               /* New object to return */
  int i;


  *ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config));
  if( pRet==0 ) return SQLITE_NOMEM;
  memset(pRet, 0, sizeof(Fts5Config));
  pRet->db = db;
  pRet->iCookie = -1;


  pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg);

  pRet->zDb = fts5Strdup(&rc, azArg[1]);
  pRet->zName = fts5Strdup(&rc, azArg[2]);
  if( rc==SQLITE_OK && sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){
    *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName);
    rc = SQLITE_ERROR;
  }

  for(i=3; rc==SQLITE_OK && i<nArg; i++){
    char *zDup = fts5Strdup(&rc, azArg[i]);
    if( zDup ){
      char *zCol = 0;
      int bParseError = 0;

      /* Check if this is a quoted column name */
      if( fts5_isopenquote(zDup[0]) ){
        bParseError = fts5Dequote(zDup);
        zCol = zDup;
      }else{
        char *z = (char*)fts5ConfigSkipBareword(zDup);
        if( *z=='\0' ){
          zCol = zDup;
        }else{
          int nCmd = z - zDup;
          z = (char*)fts5ConfigSkipWhitespace(z);
          if( *z!='=' ){
            bParseError = 1;
          }else{
            z++;
            z = fts5TrimString(z);

            if( fts5_isopenquote(*z) ){
              if( fts5Dequote(z) ) bParseError = 1;
            }else{

              char *z2 = (char*)fts5ConfigSkipBareword(z);
              if( *z2 ) bParseError = 1;
            }
            if( bParseError==0 ){
              rc = fts5ConfigParseSpecial(pGlobal, pRet, zDup, nCmd, z, pzErr);
            }
          }
        }
      }

      if( bParseError ){
        assert( *pzErr==0 );
        *pzErr = sqlite3_mprintf("parse error in \"%s\"", zDup);
        rc = SQLITE_ERROR;
      }else if( zCol ){
        if( 0==sqlite3_stricmp(zCol, FTS5_RANK_NAME) 
         || 0==sqlite3_stricmp(zCol, FTS5_ROWID_NAME) 
        ){
          *pzErr = sqlite3_mprintf("reserved fts5 column name: %s", zCol);
          rc = SQLITE_ERROR;
        }else{
          pRet->azCol[pRet->nCol++] = zCol;
          zDup = 0;
        }
      }

      sqlite3_free(zDup);
    }


  }

  /* If a tokenizer= option was successfully parsed, the tokenizer has
  ** already been allocated. Otherwise, allocate an instance of the default
  ** tokenizer (simple) now.  */
  if( rc==SQLITE_OK && pRet->pTok==0 ){
    rc = fts5ConfigDefaultTokenizer(pGlobal, pRet);
  }

  /* If no zContent option was specified, fill in the default values. */
  if( rc==SQLITE_OK && pRet->eContent==FTS5_CONTENT_NORMAL ){
    pRet->zContent = sqlite3_mprintf("%Q.'%q_content'", pRet->zDb, pRet->zName);







>







>
|
>








|
<
|
|
|
<
<
|
|
|
|
<
<
<
<
|
|
|
<
|
|
>
|
|
<
>
|
|
|
<
<
|
<
<
<
|
<
|
|

|
<
<
|
<
|

|
|


|
<
|
>
>




|







488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514

515
516
517


518
519
520
521




522
523
524

525
526
527
528
529

530
531
532
533


534



535

536
537
538
539


540

541
542
543
544
545
546
547

548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
  const char **azArg,             /* Array of nArg CREATE VIRTUAL TABLE args */
  Fts5Config **ppOut,             /* OUT: Results of parse */
  char **pzErr                    /* OUT: Error message */
){
  int rc = SQLITE_OK;             /* Return code */
  Fts5Config *pRet;               /* New object to return */
  int i;
  int nByte;

  *ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config));
  if( pRet==0 ) return SQLITE_NOMEM;
  memset(pRet, 0, sizeof(Fts5Config));
  pRet->db = db;
  pRet->iCookie = -1;

  nByte = nArg * (sizeof(char*) + sizeof(u8));
  pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, nByte);
  pRet->abUnindexed = (u8*)&pRet->azCol[nArg];
  pRet->zDb = fts5Strdup(&rc, azArg[1]);
  pRet->zName = fts5Strdup(&rc, azArg[2]);
  if( rc==SQLITE_OK && sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){
    *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName);
    rc = SQLITE_ERROR;
  }

  for(i=3; rc==SQLITE_OK && i<nArg; i++){
    const char *zOrig = azArg[i];

    const char *z;
    char *zOne = 0;
    char *zTwo = 0;


    int bOption = 0;
    int bMustBeCol = 0;

    z = fts5ConfigGobbleWord(&rc, zOrig, &zOne, &bMustBeCol);




    z = fts5ConfigSkipWhitespace(z);
    if( z && *z=='=' ){
      bOption = 1;

      z++;
      if( bMustBeCol ) z = 0;
    }
    z = fts5ConfigSkipWhitespace(z);
    if( z && z[0] ){

      int bDummy;
      z = fts5ConfigGobbleWord(&rc, z, &zTwo, &bDummy);
      if( z && z[0] ) z = 0;
    }






    if( rc==SQLITE_OK ){

      if( z==0 ){
        *pzErr = sqlite3_mprintf("parse error in \"%s\"", zOrig);
        rc = SQLITE_ERROR;
      }else{


        if( bOption ){

          rc = fts5ConfigParseSpecial(pGlobal, pRet, zOne, zTwo, pzErr);
        }else{
          rc = fts5ConfigParseColumn(pRet, zOne, zTwo, pzErr);
          zOne = 0;
        }
      }
    }


    sqlite3_free(zOne);
    sqlite3_free(zTwo);
  }

  /* If a tokenizer= option was successfully parsed, the tokenizer has
  ** already been allocated. Otherwise, allocate an instance of the default
  ** tokenizer (unicode61) now.  */
  if( rc==SQLITE_OK && pRet->pTok==0 ){
    rc = fts5ConfigDefaultTokenizer(pGlobal, pRet);
  }

  /* If no zContent option was specified, fill in the default values. */
  if( rc==SQLITE_OK && pRet->eContent==FTS5_CONTENT_NORMAL ){
    pRet->zContent = sqlite3_mprintf("%Q.'%q_content'", pRet->zDb, pRet->zName);
Changes to ext/fts5/fts5_index.c.
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
      if( pIter->iRowidOffset>0 ){
        u8 *a = pIter->pLeaf->p;
        int iOff;
        int nPos;
        int bDummy;
        i64 iDelta;

        if( p->rc==SQLITE_OK ){
          pIter->iRowidOffset--;
          pIter->iLeafOffset = iOff = pIter->aRowidOffset[pIter->iRowidOffset];
          iOff += fts5GetPoslistSize(&a[iOff], &nPos, &bDummy);
          iOff += nPos;
          getVarint(&a[iOff], (u64*)&iDelta);
          pIter->iRowid -= iDelta;
          fts5SegIterLoadNPos(p, pIter);
        }
      }else{
        fts5SegIterReverseNewPage(p, pIter);
      }
    }else{
      Fts5Data *pLeaf = pIter->pLeaf;
      int iOff;
      int bNewTerm = 0;







<
|
|
|
|
|
|
|
<







1811
1812
1813
1814
1815
1816
1817

1818
1819
1820
1821
1822
1823
1824

1825
1826
1827
1828
1829
1830
1831
      if( pIter->iRowidOffset>0 ){
        u8 *a = pIter->pLeaf->p;
        int iOff;
        int nPos;
        int bDummy;
        i64 iDelta;


        pIter->iRowidOffset--;
        pIter->iLeafOffset = iOff = pIter->aRowidOffset[pIter->iRowidOffset];
        iOff += fts5GetPoslistSize(&a[iOff], &nPos, &bDummy);
        iOff += nPos;
        getVarint(&a[iOff], (u64*)&iDelta);
        pIter->iRowid -= iDelta;
        fts5SegIterLoadNPos(p, pIter);

      }else{
        fts5SegIterReverseNewPage(p, pIter);
      }
    }else{
      Fts5Data *pLeaf = pIter->pLeaf;
      int iOff;
      int bNewTerm = 0;
Changes to ext/fts5/fts5_storage.c.
319
320
321
322
323
324
325

326
327
328
329
330
331
332
    if( sqlite3_step(pSeek)==SQLITE_ROW ){
      int iCol;
      Fts5InsertCtx ctx;
      ctx.pStorage = p;
      ctx.iCol = -1;
      rc = sqlite3Fts5IndexBeginWrite(p->pIndex, iDel);
      for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){

        ctx.szCol = 0;
        rc = sqlite3Fts5Tokenize(pConfig, 
            (const char*)sqlite3_column_text(pSeek, iCol),
            sqlite3_column_bytes(pSeek, iCol),
            (void*)&ctx,
            fts5StorageInsertCallback
        );







>







319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
    if( sqlite3_step(pSeek)==SQLITE_ROW ){
      int iCol;
      Fts5InsertCtx ctx;
      ctx.pStorage = p;
      ctx.iCol = -1;
      rc = sqlite3Fts5IndexBeginWrite(p->pIndex, iDel);
      for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
        if( pConfig->abUnindexed[iCol-1] ) continue;
        ctx.szCol = 0;
        rc = sqlite3Fts5Tokenize(pConfig, 
            (const char*)sqlite3_column_text(pSeek, iCol),
            sqlite3_column_bytes(pSeek, iCol),
            (void*)&ctx,
            fts5StorageInsertCallback
        );
482
483
484
485
486
487
488

489
490
491
492
493
494
495
    int iCol;
    Fts5InsertCtx ctx;
    ctx.pStorage = p;
    ctx.iCol = -1;

    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, iDel);
    for(iCol=0; rc==SQLITE_OK && iCol<pConfig->nCol; iCol++){

      ctx.szCol = 0;
      rc = sqlite3Fts5Tokenize(pConfig, 
        (const char*)sqlite3_value_text(apVal[iCol]),
        sqlite3_value_bytes(apVal[iCol]),
        (void*)&ctx,
        fts5StorageInsertCallback
      );







>







483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
    int iCol;
    Fts5InsertCtx ctx;
    ctx.pStorage = p;
    ctx.iCol = -1;

    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, iDel);
    for(iCol=0; rc==SQLITE_OK && iCol<pConfig->nCol; iCol++){
      if( pConfig->abUnindexed[iCol] ) continue;
      ctx.szCol = 0;
      rc = sqlite3Fts5Tokenize(pConfig, 
        (const char*)sqlite3_value_text(apVal[iCol]),
        sqlite3_value_bytes(apVal[iCol]),
        (void*)&ctx,
        fts5StorageInsertCallback
      );
560
561
562
563
564
565
566

567
568
569
570
571
572

573
574
575
576
577
578
579
  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pScan) ){
    i64 iRowid = sqlite3_column_int64(pScan, 0);

    sqlite3Fts5BufferZero(&buf);
    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, iRowid);
    for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
      ctx.szCol = 0;

      rc = sqlite3Fts5Tokenize(pConfig, 
          (const char*)sqlite3_column_text(pScan, ctx.iCol+1),
          sqlite3_column_bytes(pScan, ctx.iCol+1),
          (void*)&ctx,
          fts5StorageInsertCallback
      );

      sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
      p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
    }
    p->nTotalRow++;

    if( rc==SQLITE_OK ){
      rc = fts5StorageInsertDocsize(p, iRowid, &buf);







>
|
|
|
|
|
|
>







562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pScan) ){
    i64 iRowid = sqlite3_column_int64(pScan, 0);

    sqlite3Fts5BufferZero(&buf);
    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, iRowid);
    for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
      ctx.szCol = 0;
      if( pConfig->abUnindexed[ctx.iCol]==0 ){
        rc = sqlite3Fts5Tokenize(pConfig, 
            (const char*)sqlite3_column_text(pScan, ctx.iCol+1),
            sqlite3_column_bytes(pScan, ctx.iCol+1),
            (void*)&ctx,
            fts5StorageInsertCallback
        );
      }
      sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
      p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
    }
    p->nTotalRow++;

    if( rc==SQLITE_OK ){
      rc = fts5StorageInsertDocsize(p, iRowid, &buf);
667
668
669
670
671
672
673

674
675
676
677
678
679

680
681
682
683
684
685
686
  /* Add new entries to the FTS index */
  if( rc==SQLITE_OK ){
    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, *piRowid);
    ctx.pStorage = p;
  }
  for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
    ctx.szCol = 0;

    rc = sqlite3Fts5Tokenize(pConfig, 
        (const char*)sqlite3_value_text(apVal[ctx.iCol+2]),
        sqlite3_value_bytes(apVal[ctx.iCol+2]),
        (void*)&ctx,
        fts5StorageInsertCallback
    );

    sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
    p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
  }
  p->nTotalRow++;

  /* Write the %_docsize record */
  if( rc==SQLITE_OK ){







>
|
|
|
|
|
|
>







671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
  /* Add new entries to the FTS index */
  if( rc==SQLITE_OK ){
    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, *piRowid);
    ctx.pStorage = p;
  }
  for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
    ctx.szCol = 0;
    if( pConfig->abUnindexed[ctx.iCol]==0 ){
      rc = sqlite3Fts5Tokenize(pConfig, 
          (const char*)sqlite3_value_text(apVal[ctx.iCol+2]),
          sqlite3_value_bytes(apVal[ctx.iCol+2]),
          (void*)&ctx,
          fts5StorageInsertCallback
      );
    }
    sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
    p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
  }
  p->nTotalRow++;

  /* Write the %_docsize record */
  if( rc==SQLITE_OK ){
779
780
781
782
783
784
785

786
787
788
789
790
791
792
    int rc2;
    while( SQLITE_ROW==sqlite3_step(pScan) ){
      int i;
      ctx.iRowid = sqlite3_column_int64(pScan, 0);
      ctx.szCol = 0;
      rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize);
      for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){

        ctx.iCol = i;
        ctx.szCol = 0;
        rc = sqlite3Fts5Tokenize(
            pConfig, 
            (const char*)sqlite3_column_text(pScan, i+1),
            sqlite3_column_bytes(pScan, i+1),
            (void*)&ctx,







>







785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
    int rc2;
    while( SQLITE_ROW==sqlite3_step(pScan) ){
      int i;
      ctx.iRowid = sqlite3_column_int64(pScan, 0);
      ctx.szCol = 0;
      rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize);
      for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
        if( pConfig->abUnindexed[i] ) continue;
        ctx.iCol = i;
        ctx.szCol = 0;
        rc = sqlite3Fts5Tokenize(
            pConfig, 
            (const char*)sqlite3_column_text(pScan, i+1),
            sqlite3_column_bytes(pScan, i+1),
            (void*)&ctx,
Changes to ext/fts5/test/fts5tokenizer.test.
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
}

do_catchsql_test 4.1 {
  CREATE VIRTUAL TABLE ft2 USING fts5(x, tokenize = tcl abc);
} {1 {parse error in "tokenize = tcl abc"}}
do_catchsql_test 4.2 {
  CREATE VIRTUAL TABLE ft2 USING fts5(x y)
} {1 {parse error in "x y"}}

#-------------------------------------------------------------------------
# Test the "separators" and "tokenchars" options a bit.
#
foreach {tn tokenizer} {1 ascii 2 unicode61} {
  reset_db
  set T "$tokenizer tokenchars ',.:' separators 'xyz'"







|







66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
}

do_catchsql_test 4.1 {
  CREATE VIRTUAL TABLE ft2 USING fts5(x, tokenize = tcl abc);
} {1 {parse error in "tokenize = tcl abc"}}
do_catchsql_test 4.2 {
  CREATE VIRTUAL TABLE ft2 USING fts5(x y)
} {1 {unrecognized column option: y}}

#-------------------------------------------------------------------------
# Test the "separators" and "tokenchars" options a bit.
#
foreach {tn tokenizer} {1 ascii 2 unicode61} {
  reset_db
  set T "$tokenizer tokenchars ',.:' separators 'xyz'"
Added ext/fts5/test/fts5unindexed.test.


















































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
# 2015 Apr 24
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# The tests in this file focus on "unindexed" columns.
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5unindexed


do_execsql_test 1.1 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, b UNINDEXED);
  INSERT INTO t1 VALUES('a b c', 'd e f');
  INSERT INTO t1 VALUES('g h i', 'j k l');
} {}

do_execsql_test 1.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'b' } {1}
do_execsql_test 1.3 { SELECT rowid FROM t1 WHERE t1 MATCH 'e' } {}

do_execsql_test 1.4 { INSERT INTO t1(t1) VALUES('integrity-check') } {}
do_execsql_test 1.5 { INSERT INTO t1(t1) VALUES('rebuild') } {}
do_execsql_test 1.6 { INSERT INTO t1(t1) VALUES('integrity-check') } {}

do_execsql_test 1.7 { SELECT rowid FROM t1 WHERE t1 MATCH 'b' } {1}
do_execsql_test 1.8 { SELECT rowid FROM t1 WHERE t1 MATCH 'e' } {}

do_execsql_test 1.9 { DELETE FROM t1 WHERE t1 MATCH 'b' } {}

do_execsql_test 1.10 { INSERT INTO t1(t1) VALUES('integrity-check') } {}
do_execsql_test 1.11 { INSERT INTO t1(t1) VALUES('rebuild') } {}
do_execsql_test 1.12 { INSERT INTO t1(t1) VALUES('integrity-check') } {}

do_execsql_test 1.13 { SELECT rowid FROM t1 WHERE t1 MATCH 'i' } {2}
do_execsql_test 1.14 { SELECT rowid FROM t1 WHERE t1 MATCH 'l' } {}

do_execsql_test 2.1 {
  CREATE VIRTUAL TABLE t2 USING fts5(a UNINDEXED, b UNINDEXED);
  INSERT INTO t1 VALUES('a b c', 'd e f');
  INSERT INTO t1 VALUES('g h i', 'j k l');
  SELECT rowid FROM t2_data;
} {1 10}
do_execsql_test 2.2 {
  INSERT INTO t2(t2) VALUES('rebuild');
  INSERT INTO t2(t2) VALUES('integrity-check');
  SELECT rowid FROM t2_data;
} {1 10}

do_execsql_test 3.1 {
  CREATE TABLE x4(i INTEGER PRIMARY KEY, a, b, c);
  CREATE VIRTUAL TABLE t4 USING fts5(a, b UNINDEXED, c, content=x4);
  INSERT INTO x4 VALUES(10, 'a b c', 'd e f', 'g h i');
  INSERT INTO x4 VALUES(20, 'j k l', 'm n o', 'p q r');
  INSERT INTO t4(t4) VALUES('rebuild');
  INSERT INTO t4(t4) VALUES('integrity-check');
} {}

do_execsql_test 3.2 {
  INSERT INTO t4(t4, rowid, a, b, c) VALUES('delete', 20, 'j k l', '', 'p q r');
  DELETE FROM x4 WHERE rowid=20;
  INSERT INTO t4(t4) VALUES('integrity-check');
} {}


finish_test