SQLite

Check-in [701ef64b3d]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix two similar problems in fts3 that meant that an OOM error could cause a memory leak.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 701ef64b3dbf45e52043e79e528002bd4b7a21e2
User & Date: dan 2010-01-22 15:48:18.000
Context
2010-01-26
01:25
Make the TEMP file tables use the page size set for the main database. Ticket [b80eeab588c4]. Also copy over the changes from apple-osx check-in [7c3bede3f2]. (check-in: 5dcfb0c9e4 user: drh tags: trunk)
2010-01-22
15:48
Fix two similar problems in fts3 that meant that an OOM error could cause a memory leak. (check-in: 701ef64b3d user: dan tags: trunk)
2010-01-21
23:11
Fix a segfault that can occur when the LHS of a LIKE operator has an undefined collating sequence. Ticket [1258875e07553]. (check-in: a82e6b4585 user: drh tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts3/fts3_porter.c.
︙︙
601
602
603
604
605
606
607

608
609
610

611
612
613
614
615
616
617

    while( c->iOffset<c->nInput && !isDelim(z[c->iOffset]) ){
      c->iOffset++;
    }

    if( c->iOffset>iStartOffset ){
      int n = c->iOffset-iStartOffset;
      if( n>c->nAllocated ){

        c->nAllocated = n+20;
        c->zToken = sqlite3_realloc(c->zToken, c->nAllocated);
        if( c->zToken==NULL ) return SQLITE_NOMEM;

      }
      porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes);
      *pzToken = c->zToken;
      *piStartOffset = iStartOffset;
      *piEndOffset = c->iOffset;
      *piPosition = c->iToken++;
      return SQLITE_OK;








>

|
|
>








601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619

    while( c->iOffset<c->nInput && !isDelim(z[c->iOffset]) ){
      c->iOffset++;
    }

    if( c->iOffset>iStartOffset ){
      int n = c->iOffset-iStartOffset;
      if( n>c->nAllocated ){
        char *pNew;
        c->nAllocated = n+20;
        pNew = sqlite3_realloc(c->zToken, c->nAllocated);
        if( !pNew ) return SQLITE_NOMEM;
        c->zToken = pNew;
      }
      porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes);
      *pzToken = c->zToken;
      *piStartOffset = iStartOffset;
      *piEndOffset = c->iOffset;
      *piPosition = c->iToken++;
      return SQLITE_OK;
︙︙
Changes to ext/fts3/fts3_tokenizer1.c.
︙︙
178
179
180
181
182
183
184

185
186
187

188
189
190
191
192
193
194

    while( c->iOffset<c->nBytes && !simpleDelim(t, p[c->iOffset]) ){
      c->iOffset++;
    }

    if( c->iOffset>iStartOffset ){
      int i, n = c->iOffset-iStartOffset;
      if( n>c->nTokenAllocated ){

        c->nTokenAllocated = n+20;
        c->pToken = sqlite3_realloc(c->pToken, c->nTokenAllocated);
        if( c->pToken==NULL ) return SQLITE_NOMEM;

      }
      for(i=0; i<n; i++){
        /* TODO(shess) This needs expansion to handle UTF-8
        ** case-insensitivity.
        */
        unsigned char ch = p[iStartOffset+i];
        c->pToken[i] = (char)(ch<0x80 ? tolower(ch) : ch);








>

|
|
>








178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196

    while( c->iOffset<c->nBytes && !simpleDelim(t, p[c->iOffset]) ){
      c->iOffset++;
    }

    if( c->iOffset>iStartOffset ){
      int i, n = c->iOffset-iStartOffset;
      if( n>c->nTokenAllocated ){
        char *pNew;
        c->nTokenAllocated = n+20;
        pNew = sqlite3_realloc(c->pToken, c->nTokenAllocated);
        if( !pNew ) return SQLITE_NOMEM;
        c->pToken = pNew;
      }
      for(i=0; i<n; i++){
        /* TODO(shess) This needs expansion to handle UTF-8
        ** case-insensitivity.
        */
        unsigned char ch = p[iStartOffset+i];
        c->pToken[i] = (char)(ch<0x80 ? tolower(ch) : ch);
︙︙
Changes to test/fts3malloc.test.
︙︙
29
30
31
32
33
34
35
36

37
38
39
40
41
42
43


# Test organization:
#
# fts3_malloc-1.*: Test OOM during CREATE and DROP table statements.
# fts3_malloc-2.*: Test OOM during SELECT operations.
# fts3_malloc-3.*: Test OOM during SELECT operations with a larger database.
# fts3_malloc-4.*: Test OOM during database write operations.
#

#


proc normal_list {l} {
  set ret [list]
  foreach elem $l {lappend ret $elem}
  set ret








|
>








29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44


# Test organization:
#
# fts3_malloc-1.*: Test OOM during CREATE and DROP table statements.
# fts3_malloc-2.*: Test OOM during SELECT operations.
# fts3_malloc-3.*: Test OOM during SELECT operations with a larger database.
# fts3_malloc-4.*: Test OOM during database write operations.
# fts3_malloc-5.*: Test that a couple of memory leaks that could follow
#                  OOM in tokenizer code have been fixed.
#


proc normal_list {l} {
  set ret [list]
  foreach elem $l {lappend ret $elem}
  set ret
︙︙
283
284
285
286
287
288
289










290
291
292
293

  3 "DELETE FROM ft WHERE ft MATCH 'five'"
} {
  do_write_test fts3_malloc-4.1.$tn ft_content $sql
}
do_test fts3_malloc-4.2 {
  execsql { SELECT a FROM ft }
} {two four {two four}}












finish_test









>
>
>
>
>
>
>
>
>
>





284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304

  3 "DELETE FROM ft WHERE ft MATCH 'five'"
} {
  do_write_test fts3_malloc-4.1.$tn ft_content $sql
}
do_test fts3_malloc-4.2 {
  execsql { SELECT a FROM ft }
} {two four {two four}}

do_write_test fts3_malloc-5.1 ft_content {
  INSERT INTO ft VALUES('short alongertoken reallyquitealotlongerimeanit andthistokenisjustsolongthatonemightbeforgivenforimaginingthatitwasmerelyacontrivedexampleandnotarealtoken', 'cynics!')
}
do_test fts3_malloc-5.2 {
  execsql { CREATE VIRTUAL TABLE ft8 USING fts3(x, tokenize porter) }
} {}
do_write_test fts3_malloc-5.3 ft_content {
  INSERT INTO ft8 VALUES('short alongertoken reallyquitealotlongerimeanit andthistokenisjustsolongthatonemightbeforgivenforimaginingthatitwasmerelyacontrivedexampleandnotarealtoken')
}


finish_test