SQLite

Check-in [36ae510de4]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Avoid allocating a large object on the stack in the incremental merge code. Use sqlite3_malloc() instead.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | fts4-incr-merge
Files: files | file ages | folders
SHA1: 36ae510de45be44efd34cff242d02fb21b7419ac
User & Date: dan 2012-03-14 12:17:40.405
Context
2012-03-14
20:01
Add tests for incremental merge code. (check-in: 570473729d user: dan tags: fts4-incr-merge)
12:17
Avoid allocating a large object on the stack in the incremental merge code. Use sqlite3_malloc() instead. (check-in: 36ae510de4 user: dan tags: fts4-incr-merge)
11:51
Fix another bug in the incremental merge code. (check-in: f97b12e095 user: dan tags: fts4-incr-merge)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts3/fts3_write.c.
︙︙
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)

#include <string.h>
#include <assert.h>
#include <stdlib.h>


#define FTS_MAX_APPENDABLE_HEIGHT 10

/*
** When full-text index nodes are loaded from disk, the buffer that they
** are loaded into has the following number of bytes of padding at the end 
** of it. i.e. if a full-text index node is 900 bytes in size, then a buffer
** of 920 bytes is allocated for it.
**








|








21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)

#include <string.h>
#include <assert.h>
#include <stdlib.h>


#define FTS_MAX_APPENDABLE_HEIGHT 16

/*
** When full-text index nodes are loaded from disk, the buffer that they
** are loaded into has the following number of bytes of padding at the end 
** of it. i.e. if a full-text index node is 900 bytes in size, then a buffer
** of 920 bytes is allocated for it.
**
︙︙
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185







4186
4187

4188
4189
4190
4191
4192

4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205

4206
4207
4208
4209
4210
4211
4212
4213
4214

4215
4216
4217
4218
4219
4220
4221

  int nRem = nMerge;              /* Number of leaf pages yet to  be written */

  assert( nMin>=2 );

  while( rc==SQLITE_OK && nRem>0 ){
    sqlite3_int64 iAbsLevel;        /* Absolute level number to work on */
    sqlite3_stmt *pFindLevel = 0;   /* SQL used to determine iAbsLevel */
    Fts3MultiSegReader csr;         /* Cursor used to read input data */
    Fts3SegFilter filter;           /* Filter used with cursor csr */
    IncrmergeWriter writer;         /* Writer object */

    memset(&writer, 0, sizeof(IncrmergeWriter));

    /* Determine which level to merge segments from. Any level, from any
    ** prefix or language index may be selected. Stack variable iAbsLevel 
    ** is set to the absolute level number of the level to merge from.  */
    rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0);
    if( rc!=SQLITE_OK ) return rc;
    sqlite3_bind_int(pFindLevel, 1, nMin);
    if( sqlite3_step(pFindLevel)!=SQLITE_ROW ){
      return sqlite3_reset(pFindLevel);
    }
    iAbsLevel = sqlite3_column_int64(pFindLevel, 0);
    rc = sqlite3_reset(pFindLevel);
    if( rc!=SQLITE_OK ) return rc;








    /* Open a cursor to iterate through the contents of indexes 0 and 1 of
    ** the selected absolute level. */

    rc = fts3IncrmergeCsr(p, iAbsLevel, &csr);
    if( rc!=SQLITE_OK ) return rc;
    memset(&filter, 0, sizeof(Fts3SegFilter));
    filter.flags = FTS3_SEGMENT_REQUIRE_POS;


    rc = sqlite3Fts3SegReaderStart(p, &csr, &filter);
    assert( rc!=SQLITE_ABORT );
    if( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, &csr)) ){
      rc = fts3IncrmergeWriter(p, iAbsLevel, csr.zTerm, csr.nTerm, &writer);
      if( rc==SQLITE_OK ){
        do {
          rc = fts3IncrmergeAppend(p, &writer, &csr);
          if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, &csr);
          if( writer.nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK;
        }while( rc==SQLITE_ROW );
      }
    }
    assert( rc!=SQLITE_ABORT );

    fts3IncrmergeRelease(p, &writer, &rc);
    nRem -= (1 + writer.nWork);

    /* Update or delete the input segments */
    if( rc==SQLITE_OK ){
      rc = fts3IncrmergeChomp(p, iAbsLevel, &csr);
    }

    sqlite3Fts3SegReaderFinish(&csr);

  }

  return rc;
}

/*
** Process statements of the form:








|
|
|
|
<





<








>
>
>
>
>
>
>


>
|
<
<
<

>
|
<
|
|
|
|
|
|
|
|
|
|
<
>
|
|



|


|
>








4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170

4171
4172
4173
4174
4175

4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194



4195
4196
4197

4198
4199
4200
4201
4202
4203
4204
4205
4206
4207

4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225

  int nRem = nMerge;              /* Number of leaf pages yet to  be written */

  assert( nMin>=2 );

  while( rc==SQLITE_OK && nRem>0 ){
    sqlite3_int64 iAbsLevel;        /* Absolute level number to work on */
    sqlite3_stmt *pFindLevel = 0;   /* SQL used to determine iAbsLevel */
    Fts3MultiSegReader *pCsr;       /* Cursor used to read input data */
    Fts3SegFilter *pFilter;         /* Filter used with cursor pCsr */
    IncrmergeWriter *pWriter;       /* Writer object */
    const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter);


    /* Determine which level to merge segments from. Any level, from any
    ** prefix or language index may be selected. Stack variable iAbsLevel 
    ** is set to the absolute level number of the level to merge from.  */
    rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0);

    sqlite3_bind_int(pFindLevel, 1, nMin);
    if( sqlite3_step(pFindLevel)!=SQLITE_ROW ){
      return sqlite3_reset(pFindLevel);
    }
    iAbsLevel = sqlite3_column_int64(pFindLevel, 0);
    rc = sqlite3_reset(pFindLevel);
    if( rc!=SQLITE_OK ) return rc;

    /* Allocate space for the cursor, filter and writer objects */
    pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc);
    if( !pWriter ) return SQLITE_NOMEM;
    memset(pWriter, 0, nAlloc);
    pFilter = (Fts3SegFilter *)&pWriter[1];
    pCsr = (Fts3MultiSegReader *)&pFilter[1];

    /* Open a cursor to iterate through the contents of indexes 0 and 1 of
    ** the selected absolute level. */
    pFilter->flags = FTS3_SEGMENT_REQUIRE_POS;
    rc = fts3IncrmergeCsr(p, iAbsLevel, pCsr);




    if( rc==SQLITE_OK ){
      rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter);

      if( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr)) ){
        rc = fts3IncrmergeWriter(p, iAbsLevel, pCsr->zTerm,pCsr->nTerm,pWriter);
        if( rc==SQLITE_OK ){
          do {
            rc = fts3IncrmergeAppend(p, pWriter, pCsr);
            if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr);
            if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK;
          }while( rc==SQLITE_ROW );
        }
      }

    }
    fts3IncrmergeRelease(p, pWriter, &rc);
    nRem -= (1 + pWriter->nWork);

    /* Update or delete the input segments */
    if( rc==SQLITE_OK ){
      rc = fts3IncrmergeChomp(p, iAbsLevel, pCsr);
    }

    sqlite3Fts3SegReaderFinish(pCsr);
    sqlite3_free(pWriter);
  }

  return rc;
}

/*
** Process statements of the form:
︙︙