SQLite

Check-in [c3c672af]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a bug in the fts5 porter tokenizer preventing it from passing xCreate() arguments through to its parent tokenizer.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: c3c672af97edf2ae5d793f6fa47364370aa4f4ec
User & Date: dan 2015-07-31 14:43:02
Context
2015-07-31
15:13
Fix a problem causing [sqldiff --rbu] to fail on tables for which all columns are part of the PRIMARY KEY. (check-in: 93449e70 user: dan tags: trunk)
14:43
Fix a bug in the fts5 porter tokenizer preventing it from passing xCreate() arguments through to its parent tokenizer. (check-in: c3c672af user: dan tags: trunk)
2015-07-30
20:26
Add the --rbu switch to the "sqldiff" utility. (check-in: 098bea26 user: dan tags: trunk)
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_tokenize.c.

︙︙
533
534
535
536
537
538
539


540
541
542
543
544
545
546
547

  if( pRet ){
    memset(pRet, 0, sizeof(PorterTokenizer));
    rc = pApi->xFindTokenizer(pApi, zBase, &pUserdata, &pRet->tokenizer);
  }else{
    rc = SQLITE_NOMEM;
  }
  if( rc==SQLITE_OK ){


    rc = pRet->tokenizer.xCreate(pUserdata, 0, 0, &pRet->pTokenizer);
  }

  if( rc!=SQLITE_OK ){
    fts5PorterDelete((Fts5Tokenizer*)pRet);
    pRet = 0;
  }
  *ppOut = (Fts5Tokenizer*)pRet;








>
>
|








533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549

  if( pRet ){
    memset(pRet, 0, sizeof(PorterTokenizer));
    rc = pApi->xFindTokenizer(pApi, zBase, &pUserdata, &pRet->tokenizer);
  }else{
    rc = SQLITE_NOMEM;
  }
  if( rc==SQLITE_OK ){
    int nArg2 = (nArg>0 ? nArg-1 : 0);
    const char **azArg2 = (nArg2 ? &azArg[1] : 0);
    rc = pRet->tokenizer.xCreate(pUserdata, azArg2, nArg2, &pRet->pTokenizer);
  }

  if( rc!=SQLITE_OK ){
    fts5PorterDelete((Fts5Tokenizer*)pRet);
    pRet = 0;
  }
  *ppOut = (Fts5Tokenizer*)pRet;
︙︙

Changes to ext/fts5/test/fts5tokenizer.test.

︙︙
242
243
244
245
246
247
248















249
250
251

  INSERT INTO e6 VALUES('\u0E08\u0E07\u0E09');
  CREATE VIRTUAL TABLE e7 USING fts5vocab(e6, 'row');
  SELECT term FROM e7;
  ROLLBACK;
}] [subst {
  brown dog fox jumped lazy over quick the \u0E08 \u0E09
}]
















finish_test









>
>
>
>
>
>
>
>
>
>
>
>
>
>
>




242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266

  INSERT INTO e6 VALUES('\u0E08\u0E07\u0E09');
  CREATE VIRTUAL TABLE e7 USING fts5vocab(e6, 'row');
  SELECT term FROM e7;
  ROLLBACK;
}] [subst {
  brown dog fox jumped lazy over quick the \u0E08 \u0E09
}]

# Test that the porter tokenizer correctly passes arguments through to
# its parent tokenizer.
do_execsql_test 8.3 {
  BEGIN;
  CREATE VIRTUAL TABLE e6 USING fts5(x,
    tokenize="porter unicode61 separators ABCDEFGHIJKLMNOPQRSTUVWXYZ"
  );
  INSERT INTO e6 VALUES('theAquickBbrownCfoxDjumpedWoverXtheYlazyZdog');
  CREATE VIRTUAL TABLE e7 USING fts5vocab(e6, 'row');
  SELECT term FROM e7;
  ROLLBACK;
} {
  brown dog fox jump lazi over quick the
}

finish_test