/ Check-in [fe904766]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a segfault that can occur in matchinfo if an fts4 table contains mostly zero-length documents. Specifically, if the table contains more rows than it does bytes of text.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1:fe9047668eaaf76e7aa1ef1f32dec7c7c4226e45
User & Date: dan 2011-01-13 10:58:27
Context
2011-01-13
11:20
Fix a couple of crashes in fts3 that can occur if the database contents are inconsistent. check-in: 811e12cd user: dan tags: trunk
10:58
Fix a segfault that can occur in matchinfo if an fts4 table contains mostly zero-length documents. Specifically, if the table contains more rows than it does bytes of text. check-in: fe904766 user: dan tags: trunk
2011-01-12
17:56
Do not raise an SQLITE_CORRUPT error in Recoverymode if the database size in the header is larger than the physical file size. This facilitates recovery of a database in which the database size field has been corrupted. check-in: 114640d9 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3_snippet.c.

952
953
954
955
956
957
958

959
960
961
962
963
964
965
  sqlite3_int64 nDoc;

  if( !*ppStmt ){
    int rc = sqlite3Fts3SelectDoctotal(pTab, ppStmt);
    if( rc!=SQLITE_OK ) return rc;
  }
  pStmt = *ppStmt;


  a = sqlite3_column_blob(pStmt, 0);
  a += sqlite3Fts3GetVarint(a, &nDoc);
  *pnDoc = (u32)nDoc;

  if( paLen ) *paLen = a;
  return SQLITE_OK;







>







952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
  sqlite3_int64 nDoc;

  if( !*ppStmt ){
    int rc = sqlite3Fts3SelectDoctotal(pTab, ppStmt);
    if( rc!=SQLITE_OK ) return rc;
  }
  pStmt = *ppStmt;
  assert( sqlite3_data_count(pStmt)==1 );

  a = sqlite3_column_blob(pStmt, 0);
  a += sqlite3Fts3GetVarint(a, &nDoc);
  *pnDoc = (u32)nDoc;

  if( paLen ) *paLen = a;
  return SQLITE_OK;

Changes to ext/fts3/fts3_write.c.

1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119

1120
1121
1122
1123
1124
1125
1126
      ** the table. The following nCol varints contain the total amount of
      ** data stored in all rows of each column of the table, from left
      ** to right.
      */
      sqlite3_stmt *pStmt;
      rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0);
      if( rc ) return rc;
      if( sqlite3_step(pStmt)==SQLITE_ROW ){
        sqlite3_int64 nDoc = 0;
        sqlite3_int64 nByte = 0;
        const char *a = sqlite3_column_blob(pStmt, 0);
        if( a ){
          const char *pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
          a += sqlite3Fts3GetVarint(a, &nDoc);
          while( a<pEnd ){
            a += sqlite3Fts3GetVarint(a, &nByte);
          }
        }

        pCsr->nRowAvg = (int)(((nByte / nDoc) + pgsz - 1) / pgsz);

      }
      rc = sqlite3_reset(pStmt);
      if( rc!=SQLITE_OK || pCsr->nRowAvg==0 ) return rc;
    }

    /* Assume that a blob flows over onto overflow pages if it is larger
    ** than (pgsz-35) bytes in size (the file-format documentation







|











|
>







1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
      ** the table. The following nCol varints contain the total amount of
      ** data stored in all rows of each column of the table, from left
      ** to right.
      */
      sqlite3_stmt *pStmt;
      rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0);
      if( rc ) return rc;
      if( sqlite3_data_count(pStmt) || sqlite3_step(pStmt)==SQLITE_ROW ){
        sqlite3_int64 nDoc = 0;
        sqlite3_int64 nByte = 0;
        const char *a = sqlite3_column_blob(pStmt, 0);
        if( a ){
          const char *pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
          a += sqlite3Fts3GetVarint(a, &nDoc);
          while( a<pEnd ){
            a += sqlite3Fts3GetVarint(a, &nByte);
          }
        }

        pCsr->nRowAvg = (int)(((nByte / nDoc) + pgsz) / pgsz);
        assert( pCsr->nRowAvg>0 ); 
      }
      rc = sqlite3_reset(pStmt);
      if( rc!=SQLITE_OK || pCsr->nRowAvg==0 ) return rc;
    }

    /* Assume that a blob flows over onto overflow pages if it is larger
    ** than (pgsz-35) bytes in size (the file-format documentation

install-sh became a regular file.

Changes to test/fts3matchinfo.test.

333
334
335
336
337
338
339






340




















341
342
  SELECT typeof(matchinfo(t10)), length(matchinfo(t10)) FROM t10 WHERE docid=1;
} {blob 0}
do_execsql_test 7.4 {
  SELECT typeof(matchinfo(t10)), length(matchinfo(t10)) 
  FROM t10 WHERE t10 MATCH 'record'
} {blob 20 blob 20}




























finish_test








>
>
>
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
  SELECT typeof(matchinfo(t10)), length(matchinfo(t10)) FROM t10 WHERE docid=1;
} {blob 0}
do_execsql_test 7.4 {
  SELECT typeof(matchinfo(t10)), length(matchinfo(t10)) 
  FROM t10 WHERE t10 MATCH 'record'
} {blob 20 blob 20}

#-------------------------------------------------------------------------
# Test a special case - matchinfo('nxa') with many zero length documents. 
# Special because "x" internally uses a statement used by both "n" and "a". 
# This was causing a problem at one point in the obscure case where the
# total number of bytes of data stored in an fts3 table was greater than
# the number of rows. i.e. when the following query returns true:
#
#   SELECT sum(length(content)) < count(*) FROM fts4table;
#
do_execsql_test 8.1 {
  CREATE VIRTUAL TABLE t11 USING fts4;
  INSERT INTO t11(t11) VALUES('nodesize=24');
  INSERT INTO t11 VALUES('quitealongstringoftext');
  INSERT INTO t11 VALUES('anotherquitealongstringoftext');
  INSERT INTO t11 VALUES('athirdlongstringoftext');
  INSERT INTO t11 VALUES('andonemoreforgoodluck');
}
do_test 8.2 {
  for {set i 0} {$i < 200} {incr i} {
    execsql { INSERT INTO t11 VALUES('') }
  }
  execsql { INSERT INTO t11(t11) VALUES('optimize') }
} {}
do_execsql_test 8.3 {
  SELECT mit(matchinfo(t11, 'nxa')) FROM t11 WHERE t11 MATCH 'a*'
} {{204 1 3 3 0} {204 1 3 3 0} {204 1 3 3 0}}

finish_test

test/progress.test became executable.

tool/mkopts.tcl became executable.