/ Check-in [85ca4409]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge in the latest enhancements from trunks, especially the use of _byteswap_ulong() and similar intrinsics on MSVC.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | sessions
Files: files | file ages | folders
SHA1:85ca4409bdca7aee801e9fba1c49a87fabbf2064
User & Date: drh 2015-07-02 18:47:12
Context
2015-07-14
15:39
Merge trunk changes, including the addition of FTS5 and pcache1 performance enhancements. check-in: db4cbefb user: drh tags: sessions
2015-07-02
18:47
Merge in the latest enhancements from trunks, especially the use of _byteswap_ulong() and similar intrinsics on MSVC. check-in: 85ca4409 user: drh tags: sessions
16:52
Fix harmless compiler warnings and one macro VVA_ONLY macro usage issue. check-in: bcc8a755 user: mistachkin tags: trunk
2015-06-30
16:29
Merge all the latest enhancements from trunk. This merge include FTS5 and a number of notable performance enhancements. check-in: 39936b33 user: drh tags: sessions
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to Makefile.in.

1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
   $(TOP)/ext/fts5/fts5_vocab.c  \

fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon 
	cp $(TOP)/ext/fts5/fts5parse.y .
	rm -f fts5parse.h
	./lemon $(OPTS) fts5parse.y
	mv fts5parse.c fts5parse.c.orig
	echo "#ifdef SQLITE_ENABLE_FTS5" > fts5parse.c
	cat fts5parse.c.orig | sed 's/yy/fts5yy/g' | sed 's/YY/fts5YY/g' \
		| sed 's/TOKEN/FTS5TOKEN/g' >> fts5parse.c
	echo "#endif /* SQLITE_ENABLE_FTS5 */" >> fts5parse.c

fts5parse.h: fts5parse.c

fts5.c: $(FTS5_SRC)
	$(TCLSH_CMD) $(TOP)/ext/fts5/tool/mkfts5c.tcl

fts5.lo:	fts5.c $(HDR) $(EXTHDR)







<


<







1007
1008
1009
1010
1011
1012
1013

1014
1015

1016
1017
1018
1019
1020
1021
1022
   $(TOP)/ext/fts5/fts5_vocab.c  \

fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon 
	cp $(TOP)/ext/fts5/fts5parse.y .
	rm -f fts5parse.h
	./lemon $(OPTS) fts5parse.y
	mv fts5parse.c fts5parse.c.orig

	cat fts5parse.c.orig | sed 's/yy/fts5yy/g' | sed 's/YY/fts5YY/g' \
		| sed 's/TOKEN/FTS5TOKEN/g' >> fts5parse.c


fts5parse.h: fts5parse.c

fts5.c: $(FTS5_SRC)
	$(TCLSH_CMD) $(TOP)/ext/fts5/tool/mkfts5c.tcl

fts5.lo:	fts5.c $(HDR) $(EXTHDR)

Changes to Makefile.msc.

1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
   $(TOP)\ext\fts5\fts5_vocab.c

fts5parse.c:	$(TOP)\ext\fts5\fts5parse.y lemon.exe
	copy $(TOP)\ext\fts5\fts5parse.y .
	del /Q fts5parse.h 2>NUL
	.\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(OPTS) fts5parse.y
	move fts5parse.c fts5parse.c.orig
	echo #ifdef SQLITE_ENABLE_FTS5 > $@
	type fts5parse.c.orig \
		| $(NAWK) "/.*/ { gsub(/yy/,\"fts5yy\");print }" \
		| $(NAWK) "/.*/ { gsub(/YY/,\"fts5YY\");print }" \
		| $(NAWK) "/.*/ { gsub(/TOKEN/,\"FTS5TOKEN\");print }" >> $@
	echo #endif /* SQLITE_ENABLE_FTS5 */ >> $@

fts5parse.h: fts5parse.c

fts5.c: $(FTS5_SRC)
	$(TCLSH_CMD) $(TOP)\ext\fts5\tool\mkfts5c.tcl

fts5.lo:	fts5.c $(HDR) $(EXTHDR)
	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c fts5.c

fts5_ext.lo:	fts5.c $(HDR) $(EXTHDR)
	$(LTCOMPILE) $(NO_WARN) -DSQLITE_ENABLE_FTS5 -c fts5.c

fts5.dll:	fts5_ext.lo
	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ fts5_ext.lo

# Rules to build the 'testfixture' application.
#
# If using the amalgamation, use sqlite3.c directly to build the test







<



|
<










|







1704
1705
1706
1707
1708
1709
1710

1711
1712
1713
1714

1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
   $(TOP)\ext\fts5\fts5_vocab.c

fts5parse.c:	$(TOP)\ext\fts5\fts5parse.y lemon.exe
	copy $(TOP)\ext\fts5\fts5parse.y .
	del /Q fts5parse.h 2>NUL
	.\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(OPTS) fts5parse.y
	move fts5parse.c fts5parse.c.orig

	type fts5parse.c.orig \
		| $(NAWK) "/.*/ { gsub(/yy/,\"fts5yy\");print }" \
		| $(NAWK) "/.*/ { gsub(/YY/,\"fts5YY\");print }" \
		| $(NAWK) "/.*/ { gsub(/TOKEN/,\"FTS5TOKEN\");print }" > $@


fts5parse.h: fts5parse.c

fts5.c: $(FTS5_SRC)
	$(TCLSH_CMD) $(TOP)\ext\fts5\tool\mkfts5c.tcl

fts5.lo:	fts5.c $(HDR) $(EXTHDR)
	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c fts5.c

fts5_ext.lo:	fts5.c $(HDR) $(EXTHDR)
	$(LTCOMPILE) $(NO_WARN) -c fts5.c

fts5.dll:	fts5_ext.lo
	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ fts5_ext.lo

# Rules to build the 'testfixture' application.
#
# If using the amalgamation, use sqlite3.c directly to build the test

Changes to ext/fts3/unicode/mkunicode.tcl.

586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
...
683
684
685
686
687
688
689
690
691
692
693
694

/*
** DO NOT EDIT THIS MACHINE GENERATED FILE.
*/
  }]
  puts ""
  if {$::generate_fts5_code} {
    puts "#if defined(SQLITE_ENABLE_FTS5)"
  } else {
    puts "#ifndef SQLITE_DISABLE_FTS3_UNICODE"
    puts "#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)"
  }
  puts ""
  puts "#include <assert.h>"
  puts ""
................................................................................
if {$::generate_test_code} {
  print_test_isalnum ${function_prefix}UnicodeIsalnum $lRange
  print_fold_test ${function_prefix}UnicodeFold $mappings
  print_test_main 
}

if {$generate_fts5_code} {
  puts "#endif /* defined(SQLITE_ENABLE_FTS5) */"
} else {
  puts "#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */"
  puts "#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */"
}







|







 







|




586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
...
683
684
685
686
687
688
689
690
691
692
693
694

/*
** DO NOT EDIT THIS MACHINE GENERATED FILE.
*/
  }]
  puts ""
  if {$::generate_fts5_code} {
    # no-op
  } else {
    puts "#ifndef SQLITE_DISABLE_FTS3_UNICODE"
    puts "#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)"
  }
  puts ""
  puts "#include <assert.h>"
  puts ""
................................................................................
if {$::generate_test_code} {
  print_test_isalnum ${function_prefix}UnicodeIsalnum $lRange
  print_fold_test ${function_prefix}UnicodeFold $mappings
  print_test_main 
}

if {$generate_fts5_code} {
  # no-op
} else {
  puts "#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */"
  puts "#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */"
}

Changes to ext/fts5/fts5Int.h.

10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
...
688
689
690
691
692
693
694
695
696
**
******************************************************************************
**
*/
#ifndef _FTS5INT_H
#define _FTS5INT_H

#ifdef SQLITE_ENABLE_FTS5

#include "fts5.h"
#include "sqlite3ext.h"
SQLITE_EXTENSION_INIT1

#include <string.h>
#include <assert.h>

................................................................................
int sqlite3Fts5UnicodeIsalnum(int c);
int sqlite3Fts5UnicodeIsdiacritic(int c);
int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic);
/*
** End of interface to code in fts5_unicode2.c.
**************************************************************************/

#endif
#endif







<
<







 








<
10
11
12
13
14
15
16


17
18
19
20
21
22
23
...
686
687
688
689
690
691
692
693

**
******************************************************************************
**
*/
#ifndef _FTS5INT_H
#define _FTS5INT_H



#include "fts5.h"
#include "sqlite3ext.h"
SQLITE_EXTENSION_INIT1

#include <string.h>
#include <assert.h>

................................................................................
int sqlite3Fts5UnicodeIsalnum(int c);
int sqlite3Fts5UnicodeIsdiacritic(int c);
int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic);
/*
** End of interface to code in fts5_unicode2.c.
**************************************************************************/

#endif

Changes to ext/fts5/fts5_aux.c.

7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
...
548
549
550
551
552
553
554
555
556
557
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
*/

#ifdef SQLITE_ENABLE_FTS5

#include "fts5Int.h"
#include <math.h>

/*
** Object used to iterate through all "coalesced phrase instances" in 
** a single column of the current row. If the phrase instances in the
................................................................................
        aBuiltin[i].xFunc,
        aBuiltin[i].xDestroy
    );
  }

  return rc;
}
#endif /* SQLITE_ENABLE_FTS5 */









<







 







<


7
8
9
10
11
12
13

14
15
16
17
18
19
20
...
547
548
549
550
551
552
553

554
555
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
*/



#include "fts5Int.h"
#include <math.h>

/*
** Object used to iterate through all "coalesced phrase instances" in 
** a single column of the current row. If the phrase instances in the
................................................................................
        aBuiltin[i].xFunc,
        aBuiltin[i].xDestroy
    );
  }

  return rc;
}



Changes to ext/fts5/fts5_buffer.c.

8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
...
301
302
303
304
305
306
307
308
309
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
*/


#ifdef SQLITE_ENABLE_FTS5

#include "fts5Int.h"

int sqlite3Fts5BufferGrow(int *pRc, Fts5Buffer *pBuf, int nByte){
  /* A no-op if an error has already occurred */
  if( *pRc ) return 1;

................................................................................
    0, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 1, 1, 1, 1, 1,   /* 0x60 .. 0x6F */
    1, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 0, 0, 0, 0, 0    /* 0x70 .. 0x7F */
  };

  return (t & 0x80) || aBareword[(int)t];
}

#endif /* SQLITE_ENABLE_FTS5 */








<







 







<

8
9
10
11
12
13
14

15
16
17
18
19
20
21
...
300
301
302
303
304
305
306

307
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
*/




#include "fts5Int.h"

int sqlite3Fts5BufferGrow(int *pRc, Fts5Buffer *pBuf, int nByte){
  /* A no-op if an error has already occurred */
  if( *pRc ) return 1;

................................................................................
    0, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 1, 1, 1, 1, 1,   /* 0x60 .. 0x6F */
    1, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 0, 0, 0, 0, 0    /* 0x70 .. 0x7F */
  };

  return (t & 0x80) || aBareword[(int)t];
}



Changes to ext/fts5/fts5_config.c.

9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
...
857
858
859
860
861
862
863
864
**    May you share freely, never taking more than you give.
**
******************************************************************************
**
** This is an SQLite module implementing full-text search.
*/

#ifdef SQLITE_ENABLE_FTS5


#include "fts5Int.h"

#define FTS5_DEFAULT_PAGE_SIZE   1000
#define FTS5_DEFAULT_AUTOMERGE      4
#define FTS5_DEFAULT_CRISISMERGE   16
................................................................................

  if( rc==SQLITE_OK ){
    pConfig->iCookie = iCookie;
  }
  return rc;
}

#endif /* SQLITE_ENABLE_FTS5 */







<







 







<
9
10
11
12
13
14
15

16
17
18
19
20
21
22
...
856
857
858
859
860
861
862

**    May you share freely, never taking more than you give.
**
******************************************************************************
**
** This is an SQLite module implementing full-text search.
*/




#include "fts5Int.h"

#define FTS5_DEFAULT_PAGE_SIZE   1000
#define FTS5_DEFAULT_AUTOMERGE      4
#define FTS5_DEFAULT_CRISISMERGE   16
................................................................................

  if( rc==SQLITE_OK ){
    pConfig->iCookie = iCookie;
  }
  return rc;
}


Changes to ext/fts5/fts5_expr.c.

8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
....
2029
2030
2031
2032
2033
2034
2035
2036
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
**
*/

#ifdef SQLITE_ENABLE_FTS5


#include "fts5Int.h"
#include "fts5parse.h"

/*
** All token types in the generated fts5parse.h file are greater than 0.
................................................................................
  }else{
    *pa = 0;
    nRet = 0;
  }
  return nRet;
}

#endif /* SQLITE_ENABLE_FTS5 */







<







 







<
8
9
10
11
12
13
14

15
16
17
18
19
20
21
....
2028
2029
2030
2031
2032
2033
2034

**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
**
*/




#include "fts5Int.h"
#include "fts5parse.h"

/*
** All token types in the generated fts5parse.h file are greater than 0.
................................................................................
  }else{
    *pa = 0;
    nRet = 0;
  }
  return nRet;
}


Changes to ext/fts5/fts5_hash.c.

8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
...
459
460
461
462
463
464
465
466
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
**
*/

#ifdef SQLITE_ENABLE_FTS5


#include "fts5Int.h"

typedef struct Fts5HashEntry Fts5HashEntry;

/*
................................................................................
  }else{
    *pzTerm = 0;
    *ppDoclist = 0;
    *pnDoclist = 0;
  }
}

#endif /* SQLITE_ENABLE_FTS5 */







<







 







<
8
9
10
11
12
13
14

15
16
17
18
19
20
21
...
458
459
460
461
462
463
464

**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
**
*/




#include "fts5Int.h"

typedef struct Fts5HashEntry Fts5HashEntry;

/*
................................................................................
  }else{
    *pzTerm = 0;
    *ppDoclist = 0;
    *pnDoclist = 0;
  }
}


Changes to ext/fts5/fts5_index.c.

12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
....
5404
5405
5406
5407
5408
5409
5410
5411
**
** Low level access to the FTS index stored in the database file. The 
** routines in this file file implement all read and write access to the
** %_data table. Other parts of the system access this functionality via
** the interface defined in fts5Int.h.
*/

#ifdef SQLITE_ENABLE_FTS5

#include "fts5Int.h"

/*
** Overview:
**
** The %_data table contains all the FTS indexes for an FTS5 virtual table.
................................................................................
    rc = sqlite3_create_function(
        db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0
    );
  }
  return rc;
}

#endif /* SQLITE_ENABLE_FTS5 */







<







 







<
12
13
14
15
16
17
18

19
20
21
22
23
24
25
....
5403
5404
5405
5406
5407
5408
5409

**
** Low level access to the FTS index stored in the database file. The 
** routines in this file file implement all read and write access to the
** %_data table. Other parts of the system access this functionality via
** the interface defined in fts5Int.h.
*/



#include "fts5Int.h"

/*
** Overview:
**
** The %_data table contains all the FTS indexes for an FTS5 virtual table.
................................................................................
    rc = sqlite3_create_function(
        db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0
    );
  }
  return rc;
}


Changes to ext/fts5/fts5_main.c.

9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
....
2284
2285
2286
2287
2288
2289
2290
2291
2292









2293


**    May you share freely, never taking more than you give.
**
******************************************************************************
**
** This is an SQLite module implementing full-text search.
*/

#if defined(SQLITE_ENABLE_FTS5)

#include "fts5Int.h"

/*
** This variable is set to false when running tests for which the on disk
** structures should not be corrupt. Otherwise, true. If it is false, extra
** assert() conditions in the fts5 code are activated - conditions that are
................................................................................
      rc = sqlite3_create_function(
          db, "fts5", 0, SQLITE_UTF8, p, fts5Fts5Func, 0, 0
      );
    }
  }
  return rc;
}
#endif /* defined(SQLITE_ENABLE_FTS5) */




















<







 







<

>
>
>
>
>
>
>
>
>
|
>
>
9
10
11
12
13
14
15

16
17
18
19
20
21
22
....
2283
2284
2285
2286
2287
2288
2289

2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
**    May you share freely, never taking more than you give.
**
******************************************************************************
**
** This is an SQLite module implementing full-text search.
*/



#include "fts5Int.h"

/*
** This variable is set to false when running tests for which the on disk
** structures should not be corrupt. Otherwise, true. If it is false, extra
** assert() conditions in the fts5 code are activated - conditions that are
................................................................................
      rc = sqlite3_create_function(
          db, "fts5", 0, SQLITE_UTF8, p, fts5Fts5Func, 0, 0
      );
    }
  }
  return rc;
}


#ifdef _WIN32
__declspec(dllexport)
#endif
int sqlite3_fts_init(
  sqlite3 *db,
  char **pzErrMsg,
  const sqlite3_api_routines *pApi
){
  return sqlite3_fts5_init(db, pzErrMsg, pApi);
}


Changes to ext/fts5/fts5_storage.c.

8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
....
1088
1089
1090
1091
1092
1093
1094
1095
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
**
*/

#ifdef SQLITE_ENABLE_FTS5


#include "fts5Int.h"

struct Fts5Storage {
  Fts5Config *pConfig;
  Fts5Index *pIndex;
................................................................................
      p->pConfig->iCookie = iNew;
    }
  }
  return rc;
}


#endif /* SQLITE_ENABLE_FTS5 */







<







 







<
8
9
10
11
12
13
14

15
16
17
18
19
20
21
....
1087
1088
1089
1090
1091
1092
1093

**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
**
*/




#include "fts5Int.h"

struct Fts5Storage {
  Fts5Config *pConfig;
  Fts5Index *pIndex;
................................................................................
      p->pConfig->iCookie = iNew;
    }
  }
  return rc;
}



Changes to ext/fts5/fts5_tcl.c.

413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
    /* These two - xSetAuxdataInt and xGetAuxdataInt - are similar to the
    ** xSetAuxdata and xGetAuxdata methods implemented above. The difference
    ** is that they may only save an integer value as auxiliary data, and
    ** do not specify a destructor function.  */
    CASE(14, "xSetAuxdataInt") {
      int iVal;
      if( Tcl_GetIntFromObj(interp, objv[2], &iVal) ) return TCL_ERROR;
      rc = p->pApi->xSetAuxdata(p->pFts, (void*)iVal, 0);
      break;
    }
    CASE(15, "xGetAuxdataInt") {
      int iVal;
      int bClear;
      if( Tcl_GetBooleanFromObj(interp, objv[2], &bClear) ) return TCL_ERROR;
      iVal = (int)p->pApi->xGetAuxdata(p->pFts, bClear);
      Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal));
      break;
    }

    default: 
      assert( 0 );
      break;







|






|







413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
    /* These two - xSetAuxdataInt and xGetAuxdataInt - are similar to the
    ** xSetAuxdata and xGetAuxdata methods implemented above. The difference
    ** is that they may only save an integer value as auxiliary data, and
    ** do not specify a destructor function.  */
    CASE(14, "xSetAuxdataInt") {
      int iVal;
      if( Tcl_GetIntFromObj(interp, objv[2], &iVal) ) return TCL_ERROR;
      rc = p->pApi->xSetAuxdata(p->pFts, (void*)((char*)0 + iVal), 0);
      break;
    }
    CASE(15, "xGetAuxdataInt") {
      int iVal;
      int bClear;
      if( Tcl_GetBooleanFromObj(interp, objv[2], &bClear) ) return TCL_ERROR;
      iVal = ((char*)p->pApi->xGetAuxdata(p->pFts, bClear) - (char*)0);
      Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal));
      break;
    }

    default: 
      assert( 0 );
      break;

Changes to ext/fts5/fts5_tokenize.c.

7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
....
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
*/

#if defined(SQLITE_ENABLE_FTS5)

#include "fts5Int.h"

/**************************************************************************
** Start of ascii tokenizer implementation.
*/

................................................................................
        &aBuiltin[i].x,
        0
    );
  }

  return SQLITE_OK;
}
#endif /* defined(SQLITE_ENABLE_FTS5) */









<







 







<


7
8
9
10
11
12
13

14
15
16
17
18
19
20
....
1221
1222
1223
1224
1225
1226
1227

1228
1229
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
*/



#include "fts5Int.h"

/**************************************************************************
** Start of ascii tokenizer implementation.
*/

................................................................................
        &aBuiltin[i].x,
        0
    );
  }

  return SQLITE_OK;
}



Changes to ext/fts5/fts5_unicode2.c.

11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
...
355
356
357
358
359
360
361
362
******************************************************************************
*/

/*
** DO NOT EDIT THIS MACHINE GENERATED FILE.
*/

#if defined(SQLITE_ENABLE_FTS5)

#include <assert.h>

/*
** Return true if the argument corresponds to a unicode codepoint
** classified as either a letter or a number. Otherwise false.
**
................................................................................
  
  else if( c>=66560 && c<66600 ){
    ret = c + 40;
  }

  return ret;
}
#endif /* defined(SQLITE_ENABLE_FTS5) */







<







 







<
11
12
13
14
15
16
17

18
19
20
21
22
23
24
...
354
355
356
357
358
359
360

******************************************************************************
*/

/*
** DO NOT EDIT THIS MACHINE GENERATED FILE.
*/



#include <assert.h>

/*
** Return true if the argument corresponds to a unicode codepoint
** classified as either a letter or a number. Otherwise false.
**
................................................................................
  
  else if( c>=66560 && c<66600 ){
    ret = c + 40;
  }

  return ret;
}

Changes to ext/fts5/fts5_varint.c.

9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
...
337
338
339
340
341
342
343
344
**    May you share freely, never taking more than you give.
**
******************************************************************************
**
** Routines for varint serialization and deserialization.
*/

#ifdef SQLITE_ENABLE_FTS5

#include "fts5Int.h"

/*
** This is a copy of the sqlite3GetVarint32() routine from the SQLite core.
** Except, this version does handle the single byte case that the core
** version depends on being handled before its function is called.
................................................................................
  if( iVal<(1 << 7 ) ) return 1;
  if( iVal<(1 << 14) ) return 2;
  if( iVal<(1 << 21) ) return 3;
  if( iVal<(1 << 28) ) return 4;
  return 5;
}

#endif /* SQLITE_ENABLE_FTS5 */







<







 







<
9
10
11
12
13
14
15

16
17
18
19
20
21
22
...
336
337
338
339
340
341
342

**    May you share freely, never taking more than you give.
**
******************************************************************************
**
** Routines for varint serialization and deserialization.
*/



#include "fts5Int.h"

/*
** This is a copy of the sqlite3GetVarint32() routine from the SQLite core.
** Except, this version does handle the single byte case that the core
** version depends on being handled before its function is called.
................................................................................
  if( iVal<(1 << 7 ) ) return 1;
  if( iVal<(1 << 14) ) return 2;
  if( iVal<(1 << 21) ) return 3;
  if( iVal<(1 << 28) ) return 4;
  return 5;
}


Changes to ext/fts5/fts5_vocab.c.

27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
...
482
483
484
485
486
487
488
489
490
491
**
**   One row for each term in the database. The value of $doc is set to
**   the number of fts5 rows that contain at least one instance of term
**   $term. Field $cnt is set to the total number of instances of term 
**   $term in the database.
*/

#if defined(SQLITE_ENABLE_FTS5)

#include "fts5Int.h"


typedef struct Fts5VocabTable Fts5VocabTable;
typedef struct Fts5VocabCursor Fts5VocabCursor;

................................................................................
    /* xRelease      */ 0,
    /* xRollbackTo   */ 0,
  };
  void *p = (void*)pGlobal;

  return sqlite3_create_module_v2(db, "fts5vocab", &fts5Vocab, p, 0);
}
#endif /* defined(SQLITE_ENABLE_FTS5) */









<







 







<


27
28
29
30
31
32
33

34
35
36
37
38
39
40
...
481
482
483
484
485
486
487

488
489
**
**   One row for each term in the database. The value of $doc is set to
**   the number of fts5 rows that contain at least one instance of term
**   $term. Field $cnt is set to the total number of instances of term 
**   $term in the database.
*/



#include "fts5Int.h"


typedef struct Fts5VocabTable Fts5VocabTable;
typedef struct Fts5VocabCursor Fts5VocabCursor;

................................................................................
    /* xRelease      */ 0,
    /* xRollbackTo   */ 0,
  };
  void *p = (void*)pGlobal;

  return sqlite3_create_module_v2(db, "fts5vocab", &fts5Vocab, p, 0);
}



Changes to ext/fts5/tool/mkfts5c.tcl.

20
21
22
23
24
25
26


27
28
29
30
31
32
33
34





35
36
37
38
39
40
41
..
59
60
61
62
63
64
65

66
67
68
69
70
71
72
73
74
75
76
77
78
  %dir%/fts5_varint.c
  %dir%/fts5_vocab.c
  fts5parse.c
}]

set G(hdr) {



#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) 
# define NDEBUG 1
#endif
#if defined(NDEBUG) && defined(SQLITE_DEBUG)
# undef NDEBUG
#endif

}






proc readfile {zFile} {
  set fd [open $zFile]
  set data [read $fd]
  close $fd
  return $data
}
................................................................................
    }
    puts $G(fd) $line
  }
}

proc fts5c_close {} {
  global G

  if {$G(fd)!="stdout"} {
    close $G(fd)
  }
}


fts5c_init fts5.c
foreach f $G(src) { fts5c_printfile $f }
fts5c_close











>
>








>
>
>
>
>







 







>













20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
..
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
  %dir%/fts5_varint.c
  %dir%/fts5_vocab.c
  fts5parse.c
}]

set G(hdr) {

#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5) 

#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) 
# define NDEBUG 1
#endif
#if defined(NDEBUG) && defined(SQLITE_DEBUG)
# undef NDEBUG
#endif

}

set G(footer) {
    
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5) */
}

proc readfile {zFile} {
  set fd [open $zFile]
  set data [read $fd]
  close $fd
  return $data
}
................................................................................
    }
    puts $G(fd) $line
  }
}

proc fts5c_close {} {
  global G
  puts -nonewline $G(fd) $G(footer)
  if {$G(fd)!="stdout"} {
    close $G(fd)
  }
}


fts5c_init fts5.c
foreach f $G(src) { fts5c_printfile $f }
fts5c_close




Changes to main.mk.

663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
   $(TOP)/ext/fts5/fts5_vocab.c  \

fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon 
	cp $(TOP)/ext/fts5/fts5parse.y .
	rm -f fts5parse.h
	./lemon $(OPTS) fts5parse.y
	mv fts5parse.c fts5parse.c.orig
	echo "#ifdef SQLITE_ENABLE_FTS5" > fts5parse.c
	cat fts5parse.c.orig | sed 's/yy/fts5yy/g' | sed 's/YY/fts5YY/g' \
		| sed 's/TOKEN/FTS5TOKEN/g' >> fts5parse.c
	echo "#endif /* SQLITE_ENABLE_FTS5 */" >> fts5parse.c

fts5parse.h: fts5parse.c

fts5.c: $(FTS5_SRC)
	tclsh $(TOP)/ext/fts5/tool/mkfts5c.tcl

fts5.o:	fts5.c $(HDR) $(EXTHDR)







<


<







663
664
665
666
667
668
669

670
671

672
673
674
675
676
677
678
   $(TOP)/ext/fts5/fts5_vocab.c  \

fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon 
	cp $(TOP)/ext/fts5/fts5parse.y .
	rm -f fts5parse.h
	./lemon $(OPTS) fts5parse.y
	mv fts5parse.c fts5parse.c.orig

	cat fts5parse.c.orig | sed 's/yy/fts5yy/g' | sed 's/YY/fts5YY/g' \
		| sed 's/TOKEN/FTS5TOKEN/g' >> fts5parse.c


fts5parse.h: fts5parse.c

fts5.c: $(FTS5_SRC)
	tclsh $(TOP)/ext/fts5/tool/mkfts5c.tcl

fts5.o:	fts5.c $(HDR) $(EXTHDR)

Changes to src/backup.c.

681
682
683
684
685
686
687
688
689
690




691
692
693
694
695
696
697
...
700
701
702
703
704
705
706

707


708
709
710
711
712
713
714
** to be updated with the new data before the backup operation is
** complete.
**
** It is assumed that the mutex associated with the BtShared object
** corresponding to the source database is held when this function is
** called.
*/
void sqlite3BackupUpdate(sqlite3_backup *pBackup, Pgno iPage, const u8 *aData){
  sqlite3_backup *p;                   /* Iterator variable */
  for(p=pBackup; p; p=p->pNext){




    assert( sqlite3_mutex_held(p->pSrc->pBt->mutex) );
    if( !isFatalError(p->rc) && iPage<p->iNext ){
      /* The backup process p has already copied page iPage. But now it
      ** has been modified by a transaction on the source pager. Copy
      ** the new data into the backup.
      */
      int rc;
................................................................................
      rc = backupOnePage(p, iPage, aData, 1);
      sqlite3_mutex_leave(p->pDestDb->mutex);
      assert( rc!=SQLITE_BUSY && rc!=SQLITE_LOCKED );
      if( rc!=SQLITE_OK ){
        p->rc = rc;
      }
    }

  }


}

/*
** Restart the backup process. This is called when the pager layer
** detects that the database has been modified by an external database
** connection. In this case there is no way of knowing which of the
** pages that have been copied into the destination database are still 







|
|
|
>
>
>
>







 







>
|
>
>







681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
...
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
** to be updated with the new data before the backup operation is
** complete.
**
** It is assumed that the mutex associated with the BtShared object
** corresponding to the source database is held when this function is
** called.
*/
static SQLITE_NOINLINE void backupUpdate(
  sqlite3_backup *p,
  Pgno iPage,
  const u8 *aData
){
  assert( p!=0 );
  do{
    assert( sqlite3_mutex_held(p->pSrc->pBt->mutex) );
    if( !isFatalError(p->rc) && iPage<p->iNext ){
      /* The backup process p has already copied page iPage. But now it
      ** has been modified by a transaction on the source pager. Copy
      ** the new data into the backup.
      */
      int rc;
................................................................................
      rc = backupOnePage(p, iPage, aData, 1);
      sqlite3_mutex_leave(p->pDestDb->mutex);
      assert( rc!=SQLITE_BUSY && rc!=SQLITE_LOCKED );
      if( rc!=SQLITE_OK ){
        p->rc = rc;
      }
    }
  }while( (p = p->pNext)!=0 );
}
void sqlite3BackupUpdate(sqlite3_backup *pBackup, Pgno iPage, const u8 *aData){
  if( pBackup ) backupUpdate(pBackup, iPage, aData);
}

/*
** Restart the backup process. This is called when the pager layer
** detects that the database has been modified by an external database
** connection. In this case there is no way of knowing which of the
** pages that have been copied into the destination database are still 

Changes to src/btree.c.

1033
1034
1035
1036
1037
1038
1039



1040
1041
1042
1043
1044
1045
1046
....
1230
1231
1232
1233
1234
1235
1236


1237
1238
1239
1240
1241
1242
1243
....
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
....
8915
8916
8917
8918
8919
8920
8921
8922
8923
8924
8925
8926
8927
8928
8929
8930
8931
8932
8933
8934
8935
8936
8937
8938
8939
8940
8941
8942
8943
8944












8945
8946
8947
8948
8949
8950
8951
8952
8953
8954
....
8956
8957
8958
8959
8960
8961
8962
8963
8964
8965
8966
8967
8968
8969
8970
8971
8972
8973
8974
8975
8976
8977


8978
8979
8980






























8981





8982
8983
8984
8985
8986
8987
8988
8989
8990

8991









8992
8993
8994
8995
8996







8997
8998
8999
9000

9001
9002
9003
9004
9005


9006
9007
9008



9009
9010
9011
9012
9013
9014
9015
9016
9017
9018
9019
9020
9021

9022
9023
9024
9025
9026
9027
9028

9029
9030
9031
9032
9033
9034
9035
9036
9037
9038
9039
9040
9041
9042
9043
9044
9045
9046
9047
9048
9049
9050
9051
9052
9053
9054
9055
9056
9057
9058
9059
9060
9061
9062
9063
9064
9065
9066
9067


9068
9069
9070
9071
9072
9073
9074
9075
9076
9077
9078
9079
9080
9081
9082
9083
9084
9085
9086
9087
9088
9089
9090
9091
9092





9093
9094
9095
9096
9097
9098
9099
9100
9101
9102
9103
9104
9105
9106
9107


9108
9109
9110
9111
9112
9113
9114
9115
9116
9117
9118
9119
9120


9121
9122
9123

9124
9125
9126
9127
9128
9129
9130
9131
9132
9133
9134
9135
9136
9137
9138
9139
9140
9141



9142
9143
9144
9145
9146
9147
9148
9149
9150
9151
9152
9153
9154
9155
9156
9157
9158
9159
9160
9161
9162
9163
9164
9165
9166
9167
9168
9169
9170
9171

9172
9173
9174
9175
9176
9177
9178
....
9194
9195
9196
9197
9198
9199
9200
9201
9202
9203

9204

9205
9206
9207
9208
9209
9210
9211
9212
9213
9214
9215
9216
9217
9218
9219
9220
9221

9222
9223
9224
9225
9226
9227
9228
9229






9230
9231
9232
9233
9234
9235
9236
9237
9238
9239
9240
9241
9242


9243

9244
9245
9246
9247
9248
9249
9250
9251
9252
9253

9254
9255
9256
9257
9258
9259
9260
....
9270
9271
9272
9273
9274
9275
9276
9277
9278
9279
9280
9281
9282
9283
9284
9285
9286
9287
9288
9289
9290


9291
9292
9293
9294
9295
9296
9297
9298



9299
9300
9301
9302
9303
9304
9305
  u8 *pCell,              /* Pointer to the cell text. */
  CellInfo *pInfo         /* Fill in this structure */
){
  assert( sqlite3_mutex_held(pPage->pBt->mutex) );
  assert( pPage->leaf==0 );
  assert( pPage->noPayload );
  assert( pPage->childPtrSize==4 );



  pInfo->nSize = 4 + getVarint(&pCell[4], (u64*)&pInfo->nKey);
  pInfo->nPayload = 0;
  pInfo->nLocal = 0;
  pInfo->iOverflow = 0;
  pInfo->pPayload = 0;
  return;
}
................................................................................
#ifdef SQLITE_DEBUG
  /* The value returned by this function should always be the same as
  ** the (CellInfo.nSize) value found by doing a full parse of the
  ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of
  ** this function verifies that this invariant is not violated. */
  CellInfo debuginfo;
  pPage->xParseCell(pPage, pCell, &debuginfo);


#endif

  assert( pPage->childPtrSize==4 );
  pEnd = pIter + 9;
  while( (*pIter++)&0x80 && pIter<pEnd );
  assert( debuginfo.nSize==(u16)(pIter - pCell) || CORRUPT_DB );
  return (u16)(pIter - pCell);
................................................................................
** the database file should be truncated to during the commit process. 
** i.e. the database has been reorganized so that only the first *pnTrunc
** pages are in use.
*/
static int autoVacuumCommit(BtShared *pBt){
  int rc = SQLITE_OK;
  Pager *pPager = pBt->pPager;
  VVA_ONLY( int nRef = sqlite3PagerRefcount(pPager) );

  assert( sqlite3_mutex_held(pBt->mutex) );
  invalidateAllOverflowCache(pBt);
  assert(pBt->autoVacuum);
  if( !pBt->incrVacuum ){
    Pgno nFin;         /* Number of pages in database after autovacuuming */
    Pgno nFree;        /* Number of pages on the freelist initially */
................................................................................
** the tree depth.  Root pages return 0.  Parents of root pages
** return 1, and so forth.
** 
** These checks are done:
**
**      1.  Make sure that cells and freeblocks do not overlap
**          but combine to completely cover the page.
**  NO  2.  Make sure cell keys are in order.
**  NO  3.  Make sure no key is less than or equal to zLowerBound.
**  NO  4.  Make sure no key is greater than or equal to zUpperBound.
**      5.  Check the integrity of overflow pages.
**      6.  Recursively call checkTreePage on all children.
**      7.  Verify that the depth of all children is the same.
**      8.  Make sure this page is at least 33% full or else it is
**          the root of the tree.
*/
static int checkTreePage(
  IntegrityCk *pCheck,  /* Context for the sanity check */
  int iPage,            /* Page number of the page to check */
  i64 *pnParentMinKey, 
  i64 *pnParentMaxKey
){
  MemPage *pPage;
  int i, rc, depth, d2, pgno, cnt;
  int hdr, cellStart;
  int nCell;
  u8 *data;
  BtShared *pBt;
  int usableSize;
  u32 *heap = 0;












  u32 x, prev = 0;
  i64 nMinKey = 0;
  i64 nMaxKey = 0;
  const char *saved_zPfx = pCheck->zPfx;
  int saved_v1 = pCheck->v1;
  int saved_v2 = pCheck->v2;

  /* Check that the page exists
  */
  pBt = pCheck->pBt;
................................................................................
  if( iPage==0 ) return 0;
  if( checkRef(pCheck, iPage) ) return 0;
  pCheck->zPfx = "Page %d: ";
  pCheck->v1 = iPage;
  if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0))!=0 ){
    checkAppendMsg(pCheck,
       "unable to get the page. error code=%d", rc);
    depth = -1;
    goto end_of_check;
  }

  /* Clear MemPage.isInit to make sure the corruption detection code in
  ** btreeInitPage() is executed.  */
  pPage->isInit = 0;
  if( (rc = btreeInitPage(pPage))!=0 ){
    assert( rc==SQLITE_CORRUPT );  /* The only possible error from InitPage */
    checkAppendMsg(pCheck,
                   "btreeInitPage() returns error code %d", rc);
    releasePage(pPage);
    depth = -1;
    goto end_of_check;
  }



  /* Check out all the cells.
  */






























  depth = 0;





  for(i=0; i<pPage->nCell && pCheck->mxErr; i++){
    u8 *pCell;
    u32 sz;
    CellInfo info;

    /* Check payload overflow pages
    */
    pCheck->zPfx = "On tree page %d cell %d: ";
    pCheck->v1 = iPage;

    pCheck->v2 = i;









    pCell = findCell(pPage,i);
    pPage->xParseCell(pPage, pCell, &info);
    sz = info.nPayload;
    /* For intKey pages, check that the keys are in order.
    */







    if( pPage->intKey ){
      if( i==0 ){
        nMinKey = nMaxKey = info.nKey;
      }else if( info.nKey <= nMaxKey ){

        checkAppendMsg(pCheck,
           "Rowid %lld out of order (previous was %lld)", info.nKey, nMaxKey);
      }
      nMaxKey = info.nKey;
    }


    if( (sz>info.nLocal) 
     && (&pCell[info.iOverflow]<=&pPage->aData[pBt->usableSize])
    ){



      int nPage = (sz - info.nLocal + usableSize - 5)/(usableSize - 4);
      Pgno pgnoOvfl = get4byte(&pCell[info.iOverflow]);
#ifndef SQLITE_OMIT_AUTOVACUUM
      if( pBt->autoVacuum ){
        checkPtrmap(pCheck, pgnoOvfl, PTRMAP_OVERFLOW1, iPage);
      }
#endif
      checkList(pCheck, 0, pgnoOvfl, nPage);
    }

    /* Check sanity of left child page.
    */
    if( !pPage->leaf ){

      pgno = get4byte(pCell);
#ifndef SQLITE_OMIT_AUTOVACUUM
      if( pBt->autoVacuum ){
        checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage);
      }
#endif
      d2 = checkTreePage(pCheck, pgno, &nMinKey, i==0?NULL:&nMaxKey);

      if( i>0 && d2!=depth ){
        checkAppendMsg(pCheck, "Child page depth differs");
      }
      depth = d2;
    }
  }

  if( !pPage->leaf ){
    pgno = get4byte(&pPage->aData[pPage->hdrOffset+8]);
    pCheck->zPfx = "On page %d at right child: ";
    pCheck->v1 = iPage;
#ifndef SQLITE_OMIT_AUTOVACUUM
    if( pBt->autoVacuum ){
      checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage);
    }
#endif
    checkTreePage(pCheck, pgno, NULL, !pPage->nCell?NULL:&nMaxKey);
  }
 
  /* For intKey leaf pages, check that the min/max keys are in order
  ** with any left/parent/right pages.
  */
  pCheck->zPfx = "Page %d: ";
  pCheck->v1 = iPage;
  if( pPage->leaf && pPage->intKey ){
    /* if we are a left child page */
    if( pnParentMinKey ){
      /* if we are the left most child page */
      if( !pnParentMaxKey ){
        if( nMaxKey > *pnParentMinKey ){
          checkAppendMsg(pCheck,
              "Rowid %lld out of order (max larger than parent min of %lld)",
              nMaxKey, *pnParentMinKey);
        }
      }else{
        if( nMinKey <= *pnParentMinKey ){
          checkAppendMsg(pCheck,
              "Rowid %lld out of order (min less than parent min of %lld)",
              nMinKey, *pnParentMinKey);


        }
        if( nMaxKey > *pnParentMaxKey ){
          checkAppendMsg(pCheck,
              "Rowid %lld out of order (max larger than parent max of %lld)",
              nMaxKey, *pnParentMaxKey);
        }
        *pnParentMinKey = nMaxKey;
      }
    /* else if we're a right child page */
    } else if( pnParentMaxKey ){
      if( nMinKey <= *pnParentMaxKey ){
        checkAppendMsg(pCheck,
            "Rowid %lld out of order (min less than parent max of %lld)",
            nMinKey, *pnParentMaxKey);
      }
    }
  }

  /* Check for complete coverage of the page
  */
  data = pPage->aData;
  hdr = pPage->hdrOffset;
  heap = (u32*)sqlite3PageMalloc( pBt->pageSize );
  pCheck->zPfx = 0;
  if( heap==0 ){





    pCheck->mallocFailed = 1;
  }else{
    int contentOffset = get2byteNotZero(&data[hdr+5]);
    assert( contentOffset<=usableSize );  /* Enforced by btreeInitPage() */
    heap[0] = 0;
    btreeHeapInsert(heap, contentOffset-1);
    /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the
    ** number of cells on the page. */
    nCell = get2byte(&data[hdr+3]);
    /* EVIDENCE-OF: R-23882-45353 The cell pointer array of a b-tree page
    ** immediately follows the b-tree page header. */
    cellStart = hdr + 12 - 4*pPage->leaf;
    /* EVIDENCE-OF: R-02776-14802 The cell pointer array consists of K 2-byte
    ** integer offsets to the cell contents. */
    for(i=0; i<nCell; i++){


      int pc = get2byteAligned(&data[cellStart+i*2]);
      u32 size = 65536;
      if( pc<=usableSize-4 ){
        size = pPage->xCellSize(pPage, &data[pc]);
      }
      if( (int)(pc+size-1)>=usableSize ){
        pCheck->zPfx = 0;
        checkAppendMsg(pCheck,
            "Corruption detected in cell %d on page %d",i,iPage);
      }else{
        btreeHeapInsert(heap, (pc<<16)|(pc+size-1));
      }
    }


    /* EVIDENCE-OF: R-20690-50594 The second field of the b-tree page header
    ** is the offset of the first freeblock, or zero if there are no
    ** freeblocks on the page. */

    i = get2byte(&data[hdr+1]);
    while( i>0 ){
      int size, j;
      assert( i<=usableSize-4 );     /* Enforced by btreeInitPage() */
      size = get2byte(&data[i+2]);
      assert( i+size<=usableSize );  /* Enforced by btreeInitPage() */
      btreeHeapInsert(heap, (i<<16)|(i+size-1));
      /* EVIDENCE-OF: R-58208-19414 The first 2 bytes of a freeblock are a
      ** big-endian integer which is the offset in the b-tree page of the next
      ** freeblock in the chain, or zero if the freeblock is the last on the
      ** chain. */
      j = get2byte(&data[i]);
      /* EVIDENCE-OF: R-06866-39125 Freeblocks are always connected in order of
      ** increasing offset. */
      assert( j==0 || j>i+size );  /* Enforced by btreeInitPage() */
      assert( j<=usableSize-4 );   /* Enforced by btreeInitPage() */
      i = j;
    }



    cnt = 0;
    assert( heap[0]>0 );
    assert( (heap[1]>>16)==0 );
    btreeHeapPull(heap,&prev);
    while( btreeHeapPull(heap,&x) ){
      if( (prev&0xffff)+1>(x>>16) ){
        checkAppendMsg(pCheck,
          "Multiple uses for byte %u of page %d", x>>16, iPage);
        break;
      }else{
        cnt += (x>>16) - (prev&0xffff) - 1;
        prev = x;
      }
    }
    cnt += usableSize - (prev&0xffff) - 1;
    /* EVIDENCE-OF: R-43263-13491 The total number of bytes in all fragments
    ** is stored in the fifth field of the b-tree page header.
    ** EVIDENCE-OF: R-07161-27322 The one-byte integer at offset 7 gives the
    ** number of fragmented free bytes within the cell content area.
    */
    if( heap[0]==0 && cnt!=data[hdr+7] ){
      checkAppendMsg(pCheck,
          "Fragmentation of %d bytes reported as %d on page %d",
          cnt, data[hdr+7], iPage);
    }
  }
  sqlite3PageFree(heap);
  releasePage(pPage);

end_of_check:

  pCheck->zPfx = saved_zPfx;
  pCheck->v1 = saved_v1;
  pCheck->v2 = saved_v2;
  return depth+1;
}
#endif /* SQLITE_OMIT_INTEGRITY_CHECK */

................................................................................
  Btree *p,     /* The btree to be checked */
  int *aRoot,   /* An array of root pages numbers for individual trees */
  int nRoot,    /* Number of entries in aRoot[] */
  int mxErr,    /* Stop reporting errors after this many */
  int *pnErr    /* Write number of errors seen to this variable */
){
  Pgno i;
  int nRef;
  IntegrityCk sCheck;
  BtShared *pBt = p->pBt;

  char zErr[100];


  sqlite3BtreeEnter(p);
  assert( p->inTrans>TRANS_NONE && pBt->inTransaction>TRANS_NONE );
  nRef = sqlite3PagerRefcount(pBt->pPager);
  sCheck.pBt = pBt;
  sCheck.pPager = pBt->pPager;
  sCheck.nPage = btreePagecount(sCheck.pBt);
  sCheck.mxErr = mxErr;
  sCheck.nErr = 0;
  sCheck.mallocFailed = 0;
  sCheck.zPfx = 0;
  sCheck.v1 = 0;
  sCheck.v2 = 0;
  *pnErr = 0;
  if( sCheck.nPage==0 ){
    sqlite3BtreeLeave(p);
    return 0;

  }

  sCheck.aPgRef = sqlite3MallocZero((sCheck.nPage / 8)+ 1);
  if( !sCheck.aPgRef ){
    *pnErr = 1;
    sqlite3BtreeLeave(p);
    return 0;
  }






  i = PENDING_BYTE_PAGE(pBt);
  if( i<=sCheck.nPage ) setPageReferenced(&sCheck, i);
  sqlite3StrAccumInit(&sCheck.errMsg, 0, zErr, sizeof(zErr), SQLITE_MAX_LENGTH);

  /* Check the integrity of the freelist
  */
  sCheck.zPfx = "Main freelist: ";
  checkList(&sCheck, 1, get4byte(&pBt->pPage1->aData[32]),
            get4byte(&pBt->pPage1->aData[36]));
  sCheck.zPfx = 0;

  /* Check all the tables.
  */


  for(i=0; (int)i<nRoot && sCheck.mxErr; i++){

    if( aRoot[i]==0 ) continue;
#ifndef SQLITE_OMIT_AUTOVACUUM
    if( pBt->autoVacuum && aRoot[i]>1 ){
      checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0);
    }
#endif
    sCheck.zPfx = "List of tree roots: ";
    checkTreePage(&sCheck, aRoot[i], NULL, NULL);
    sCheck.zPfx = 0;
  }


  /* Make sure every page in the file is referenced
  */
  for(i=1; i<=sCheck.nPage && sCheck.mxErr; i++){
#ifdef SQLITE_OMIT_AUTOVACUUM
    if( getPageReferenced(&sCheck, i)==0 ){
      checkAppendMsg(&sCheck, "Page %d is never used", i);
................................................................................
    if( getPageReferenced(&sCheck, i)!=0 && 
       (PTRMAP_PAGENO(pBt, i)==i && pBt->autoVacuum) ){
      checkAppendMsg(&sCheck, "Pointer map page %d is referenced", i);
    }
#endif
  }

  /* Make sure this analysis did not leave any unref() pages.
  ** This is an internal consistency check; an integrity check
  ** of the integrity check.
  */
  if( NEVER(nRef != sqlite3PagerRefcount(pBt->pPager)) ){
    checkAppendMsg(&sCheck,
      "Outstanding page count goes from %d to %d during this analysis",
      nRef, sqlite3PagerRefcount(pBt->pPager)
    );
  }

  /* Clean  up and report errors.
  */
  sqlite3BtreeLeave(p);


  sqlite3_free(sCheck.aPgRef);
  if( sCheck.mallocFailed ){
    sqlite3StrAccumReset(&sCheck.errMsg);
    *pnErr = sCheck.nErr+1;
    return 0;
  }
  *pnErr = sCheck.nErr;
  if( sCheck.nErr==0 ) sqlite3StrAccumReset(&sCheck.errMsg);



  return sqlite3StrAccumFinish(&sCheck.errMsg);
}
#endif /* SQLITE_OMIT_INTEGRITY_CHECK */

/*
** Return the full pathname of the underlying database file.  Return
** an empty string if the database is in-memory or a TEMP database.







>
>
>







 







>
>







 







|







 







|
<
<
|
|
|
<
<




|
|

|
|
|
|
|
|
|
|
>
>
>
>
>
>
>
>
>
>
>
>

<
<







 







<










<
<


>
>

|
<
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
>
>
>
>
>
|
<
<


<
<
<
<
>

>
>
>
>
>
>
>
>
>
|

<
<
<
>
>
>
>
>
>
>

<
<
<
>
|
<

|

>
>
|
<
<
>
>
>
|
|








<
<

>






|
>
|

<
|
|
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
|
<
<
<
<
>
>
|
<
<
<
<
|
|
|
<
<
<
<
<
<
<
<
<
<


<
<
<

<
>
>
>
>
>
|
<
<
<
|
|
<
<
<
<
<
<
<
<
<
>
>
|
<
<

<
<
<
<
<
<



>
>
|

|
>



|

|









|


>
>
>
|









|



|





|


|


<
<


>







 







<


>

>



|









|
|
|
|
>




|
|
<

>
>
>
>
>
>


<










>
>

>






<
|
<

>







 







<
<
<
<
<
<
<
<
<
<
<


<
>
>



|
<



>
>
>







1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
....
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
....
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
....
8920
8921
8922
8923
8924
8925
8926
8927


8928
8929
8930


8931
8932
8933
8934
8935
8936
8937
8938
8939
8940
8941
8942
8943
8944
8945
8946
8947
8948
8949
8950
8951
8952
8953
8954
8955
8956
8957
8958


8959
8960
8961
8962
8963
8964
8965
....
8967
8968
8969
8970
8971
8972
8973

8974
8975
8976
8977
8978
8979
8980
8981
8982
8983


8984
8985
8986
8987
8988
8989

8990
8991
8992
8993
8994
8995
8996
8997
8998
8999
9000
9001
9002
9003
9004
9005
9006
9007
9008
9009
9010
9011
9012
9013
9014
9015
9016
9017
9018
9019
9020
9021
9022
9023
9024
9025
9026


9027
9028




9029
9030
9031
9032
9033
9034
9035
9036
9037
9038
9039
9040
9041



9042
9043
9044
9045
9046
9047
9048
9049



9050
9051

9052
9053
9054
9055
9056
9057


9058
9059
9060
9061
9062
9063
9064
9065
9066
9067
9068
9069
9070


9071
9072
9073
9074
9075
9076
9077
9078
9079
9080
9081
9082

9083
9084





























9085




9086
9087
9088




9089
9090
9091










9092
9093



9094

9095
9096
9097
9098
9099
9100



9101
9102









9103
9104
9105


9106






9107
9108
9109
9110
9111
9112
9113
9114
9115
9116
9117
9118
9119
9120
9121
9122
9123
9124
9125
9126
9127
9128
9129
9130
9131
9132
9133
9134
9135
9136
9137
9138
9139
9140
9141
9142
9143
9144
9145
9146
9147
9148
9149
9150
9151
9152
9153
9154
9155
9156
9157
9158
9159
9160
9161
9162


9163
9164
9165
9166
9167
9168
9169
9170
9171
9172
....
9188
9189
9190
9191
9192
9193
9194

9195
9196
9197
9198
9199
9200
9201
9202
9203
9204
9205
9206
9207
9208
9209
9210
9211
9212
9213
9214
9215
9216
9217
9218
9219
9220
9221
9222
9223

9224
9225
9226
9227
9228
9229
9230
9231
9232

9233
9234
9235
9236
9237
9238
9239
9240
9241
9242
9243
9244
9245
9246
9247
9248
9249
9250
9251
9252

9253

9254
9255
9256
9257
9258
9259
9260
9261
9262
....
9272
9273
9274
9275
9276
9277
9278











9279
9280

9281
9282
9283
9284
9285
9286

9287
9288
9289
9290
9291
9292
9293
9294
9295
9296
9297
9298
9299
  u8 *pCell,              /* Pointer to the cell text. */
  CellInfo *pInfo         /* Fill in this structure */
){
  assert( sqlite3_mutex_held(pPage->pBt->mutex) );
  assert( pPage->leaf==0 );
  assert( pPage->noPayload );
  assert( pPage->childPtrSize==4 );
#ifndef SQLITE_DEBUG
  UNUSED_PARAMETER(pPage);
#endif
  pInfo->nSize = 4 + getVarint(&pCell[4], (u64*)&pInfo->nKey);
  pInfo->nPayload = 0;
  pInfo->nLocal = 0;
  pInfo->iOverflow = 0;
  pInfo->pPayload = 0;
  return;
}
................................................................................
#ifdef SQLITE_DEBUG
  /* The value returned by this function should always be the same as
  ** the (CellInfo.nSize) value found by doing a full parse of the
  ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of
  ** this function verifies that this invariant is not violated. */
  CellInfo debuginfo;
  pPage->xParseCell(pPage, pCell, &debuginfo);
#else
  UNUSED_PARAMETER(pPage);
#endif

  assert( pPage->childPtrSize==4 );
  pEnd = pIter + 9;
  while( (*pIter++)&0x80 && pIter<pEnd );
  assert( debuginfo.nSize==(u16)(pIter - pCell) || CORRUPT_DB );
  return (u16)(pIter - pCell);
................................................................................
** the database file should be truncated to during the commit process. 
** i.e. the database has been reorganized so that only the first *pnTrunc
** pages are in use.
*/
static int autoVacuumCommit(BtShared *pBt){
  int rc = SQLITE_OK;
  Pager *pPager = pBt->pPager;
  VVA_ONLY( int nRef = sqlite3PagerRefcount(pPager); )

  assert( sqlite3_mutex_held(pBt->mutex) );
  invalidateAllOverflowCache(pBt);
  assert(pBt->autoVacuum);
  if( !pBt->incrVacuum ){
    Pgno nFin;         /* Number of pages in database after autovacuuming */
    Pgno nFree;        /* Number of pages on the freelist initially */
................................................................................
** the tree depth.  Root pages return 0.  Parents of root pages
** return 1, and so forth.
** 
** These checks are done:
**
**      1.  Make sure that cells and freeblocks do not overlap
**          but combine to completely cover the page.
**      2.  Make sure integer cell keys are in order.


**      3.  Check the integrity of overflow pages.
**      4.  Recursively call checkTreePage on all children.
**      5.  Verify that the depth of all children is the same.


*/
static int checkTreePage(
  IntegrityCk *pCheck,  /* Context for the sanity check */
  int iPage,            /* Page number of the page to check */
  i64 *piMinKey,        /* Write minimum integer primary key here */
  i64 maxKey            /* Error if integer primary key greater than this */
){
  MemPage *pPage = 0;      /* The page being analyzed */
  int i;                   /* Loop counter */
  int rc;                  /* Result code from subroutine call */
  int depth = -1, d2;      /* Depth of a subtree */
  int pgno;                /* Page number */
  int nFrag;               /* Number of fragmented bytes on the page */
  int hdr;                 /* Offset to the page header */
  int cellStart;           /* Offset to the start of the cell pointer array */
  int nCell;               /* Number of cells */
  int doCoverageCheck = 1; /* True if cell coverage checking should be done */
  int keyCanBeEqual = 1;   /* True if IPK can be equal to maxKey
                           ** False if IPK must be strictly less than maxKey */
  u8 *data;                /* Page content */
  u8 *pCell;               /* Cell content */
  u8 *pCellIdx;            /* Next element of the cell pointer array */
  BtShared *pBt;           /* The BtShared object that owns pPage */
  u32 pc;                  /* Address of a cell */
  u32 usableSize;          /* Usable size of the page */
  u32 contentOffset;       /* Offset to the start of the cell content area */
  u32 *heap = 0;           /* Min-heap used for checking cell coverage */
  u32 x, prev = 0;


  const char *saved_zPfx = pCheck->zPfx;
  int saved_v1 = pCheck->v1;
  int saved_v2 = pCheck->v2;

  /* Check that the page exists
  */
  pBt = pCheck->pBt;
................................................................................
  if( iPage==0 ) return 0;
  if( checkRef(pCheck, iPage) ) return 0;
  pCheck->zPfx = "Page %d: ";
  pCheck->v1 = iPage;
  if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0))!=0 ){
    checkAppendMsg(pCheck,
       "unable to get the page. error code=%d", rc);

    goto end_of_check;
  }

  /* Clear MemPage.isInit to make sure the corruption detection code in
  ** btreeInitPage() is executed.  */
  pPage->isInit = 0;
  if( (rc = btreeInitPage(pPage))!=0 ){
    assert( rc==SQLITE_CORRUPT );  /* The only possible error from InitPage */
    checkAppendMsg(pCheck,
                   "btreeInitPage() returns error code %d", rc);


    goto end_of_check;
  }
  data = pPage->aData;
  hdr = pPage->hdrOffset;

  /* Set up for cell analysis */

  pCheck->zPfx = "On tree page %d cell %d: ";
  contentOffset = get2byteNotZero(&data[hdr+5]);
  assert( contentOffset<=usableSize );  /* Enforced by btreeInitPage() */

  /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the
  ** number of cells on the page. */
  nCell = get2byte(&data[hdr+3]);
  assert( pPage->nCell==nCell );

  /* EVIDENCE-OF: R-23882-45353 The cell pointer array of a b-tree page
  ** immediately follows the b-tree page header. */
  cellStart = hdr + 12 - 4*pPage->leaf;
  assert( pPage->aCellIdx==&data[cellStart] );
  pCellIdx = &data[cellStart + 2*(nCell-1)];

  if( !pPage->leaf ){
    /* Analyze the right-child page of internal pages */
    pgno = get4byte(&data[hdr+8]);
#ifndef SQLITE_OMIT_AUTOVACUUM
    if( pBt->autoVacuum ){
      pCheck->zPfx = "On page %d at right child: ";
      checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage);
    }
#endif
    depth = checkTreePage(pCheck, pgno, &maxKey, maxKey);
    keyCanBeEqual = 0;
  }else{
    /* For leaf pages, the coverage check will occur in the same loop
    ** as the other cell checks, so initialize the heap.  */
    heap = pCheck->heap;
    heap[0] = 0;
    btreeHeapInsert(heap, contentOffset-1);
  }

  /* EVIDENCE-OF: R-02776-14802 The cell pointer array consists of K 2-byte
  ** integer offsets to the cell contents. */
  for(i=nCell-1; i>=0 && pCheck->mxErr; i--){


    CellInfo info;





    /* Check cell size */
    pCheck->v2 = i;
    assert( pCellIdx==&data[cellStart + i*2] );
    pc = get2byteAligned(pCellIdx);
    pCellIdx -= 2;
    if( pc<contentOffset || pc>usableSize-4 ){
      checkAppendMsg(pCheck, "Offset %d out of range %d..%d",
                             pc, contentOffset, usableSize-4);
      doCoverageCheck = 0;
      continue;
    }
    pCell = &data[pc];
    pPage->xParseCell(pPage, pCell, &info);



    if( pc+info.nSize>usableSize ){
      checkAppendMsg(pCheck, "Extends off end of page");
      doCoverageCheck = 0;
      continue;
    }

    /* Check for integer primary key out of range */
    if( pPage->intKey ){



      if( keyCanBeEqual ? (info.nKey > maxKey) : (info.nKey >= maxKey) ){
        checkAppendMsg(pCheck, "Rowid %lld out of order", info.nKey);

      }
      maxKey = info.nKey;
    }

    /* Check the content overflow list */
    if( info.nPayload>info.nLocal ){


      int nPage;       /* Number of pages on the overflow chain */
      Pgno pgnoOvfl;   /* First page of the overflow chain */
      assert( pc + info.iOverflow <= usableSize );
      nPage = (info.nPayload - info.nLocal + usableSize - 5)/(usableSize - 4);
      pgnoOvfl = get4byte(&pCell[info.iOverflow]);
#ifndef SQLITE_OMIT_AUTOVACUUM
      if( pBt->autoVacuum ){
        checkPtrmap(pCheck, pgnoOvfl, PTRMAP_OVERFLOW1, iPage);
      }
#endif
      checkList(pCheck, 0, pgnoOvfl, nPage);
    }



    if( !pPage->leaf ){
      /* Check sanity of left child page for internal pages */
      pgno = get4byte(pCell);
#ifndef SQLITE_OMIT_AUTOVACUUM
      if( pBt->autoVacuum ){
        checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage);
      }
#endif
      d2 = checkTreePage(pCheck, pgno, &maxKey, maxKey);
      keyCanBeEqual = 0;
      if( d2!=depth ){
        checkAppendMsg(pCheck, "Child page depth differs");

        depth = d2;
      }





























    }else{




      /* Populate the coverage-checking heap for leaf pages */
      btreeHeapInsert(heap, (pc<<16)|(pc+info.nSize-1));
    }




  }
  *piMinKey = maxKey;











  /* Check for complete coverage of the page
  */



  pCheck->zPfx = 0;

  if( doCoverageCheck && pCheck->mxErr>0 ){
    /* For leaf pages, the min-heap has already been initialized and the
    ** cells have already been inserted.  But for internal pages, that has
    ** not yet been done, so do it now */
    if( !pPage->leaf ){
      heap = pCheck->heap;



      heap[0] = 0;
      btreeHeapInsert(heap, contentOffset-1);









      for(i=nCell-1; i>=0; i--){
        u32 size;
        pc = get2byteAligned(&data[cellStart+i*2]);


        size = pPage->xCellSize(pPage, &data[pc]);






        btreeHeapInsert(heap, (pc<<16)|(pc+size-1));
      }
    }
    /* Add the freeblocks to the min-heap
    **
    ** EVIDENCE-OF: R-20690-50594 The second field of the b-tree page header
    ** is the offset of the first freeblock, or zero if there are no
    ** freeblocks on the page. 
    */
    i = get2byte(&data[hdr+1]);
    while( i>0 ){
      int size, j;
      assert( (u32)i<=usableSize-4 );     /* Enforced by btreeInitPage() */
      size = get2byte(&data[i+2]);
      assert( (u32)(i+size)<=usableSize );  /* Enforced by btreeInitPage() */
      btreeHeapInsert(heap, (i<<16)|(i+size-1));
      /* EVIDENCE-OF: R-58208-19414 The first 2 bytes of a freeblock are a
      ** big-endian integer which is the offset in the b-tree page of the next
      ** freeblock in the chain, or zero if the freeblock is the last on the
      ** chain. */
      j = get2byte(&data[i]);
      /* EVIDENCE-OF: R-06866-39125 Freeblocks are always connected in order of
      ** increasing offset. */
      assert( j==0 || j>i+size );  /* Enforced by btreeInitPage() */
      assert( (u32)j<=usableSize-4 );   /* Enforced by btreeInitPage() */
      i = j;
    }
    /* Analyze the min-heap looking for overlap between cells and/or 
    ** freeblocks, and counting the number of untracked bytes in nFrag.
    */
    nFrag = 0;
    assert( heap[0]>0 );
    assert( (heap[1]>>16)==0 );
    btreeHeapPull(heap,&prev);
    while( btreeHeapPull(heap,&x) ){
      if( (prev&0xffff)+1>(x>>16) ){
        checkAppendMsg(pCheck,
          "Multiple uses for byte %u of page %d", x>>16, iPage);
        break;
      }else{
        nFrag += (x>>16) - (prev&0xffff) - 1;
        prev = x;
      }
    }
    nFrag += usableSize - (prev&0xffff) - 1;
    /* EVIDENCE-OF: R-43263-13491 The total number of bytes in all fragments
    ** is stored in the fifth field of the b-tree page header.
    ** EVIDENCE-OF: R-07161-27322 The one-byte integer at offset 7 gives the
    ** number of fragmented free bytes within the cell content area.
    */
    if( heap[0]==0 && nFrag!=data[hdr+7] ){
      checkAppendMsg(pCheck,
          "Fragmentation of %d bytes reported as %d on page %d",
          nFrag, data[hdr+7], iPage);
    }
  }



end_of_check:
  releasePage(pPage);
  pCheck->zPfx = saved_zPfx;
  pCheck->v1 = saved_v1;
  pCheck->v2 = saved_v2;
  return depth+1;
}
#endif /* SQLITE_OMIT_INTEGRITY_CHECK */

................................................................................
  Btree *p,     /* The btree to be checked */
  int *aRoot,   /* An array of root pages numbers for individual trees */
  int nRoot,    /* Number of entries in aRoot[] */
  int mxErr,    /* Stop reporting errors after this many */
  int *pnErr    /* Write number of errors seen to this variable */
){
  Pgno i;

  IntegrityCk sCheck;
  BtShared *pBt = p->pBt;
  int savedDbFlags = pBt->db->flags;
  char zErr[100];
  VVA_ONLY( int nRef );

  sqlite3BtreeEnter(p);
  assert( p->inTrans>TRANS_NONE && pBt->inTransaction>TRANS_NONE );
  assert( (nRef = sqlite3PagerRefcount(pBt->pPager))>=0 );
  sCheck.pBt = pBt;
  sCheck.pPager = pBt->pPager;
  sCheck.nPage = btreePagecount(sCheck.pBt);
  sCheck.mxErr = mxErr;
  sCheck.nErr = 0;
  sCheck.mallocFailed = 0;
  sCheck.zPfx = 0;
  sCheck.v1 = 0;
  sCheck.v2 = 0;
  sCheck.aPgRef = 0;
  sCheck.heap = 0;
  sqlite3StrAccumInit(&sCheck.errMsg, 0, zErr, sizeof(zErr), SQLITE_MAX_LENGTH);
  if( sCheck.nPage==0 ){
    goto integrity_ck_cleanup;
  }

  sCheck.aPgRef = sqlite3MallocZero((sCheck.nPage / 8)+ 1);
  if( !sCheck.aPgRef ){
    sCheck.mallocFailed = 1;
    goto integrity_ck_cleanup;

  }
  sCheck.heap = (u32*)sqlite3PageMalloc( pBt->pageSize );
  if( sCheck.heap==0 ){
    sCheck.mallocFailed = 1;
    goto integrity_ck_cleanup;
  }

  i = PENDING_BYTE_PAGE(pBt);
  if( i<=sCheck.nPage ) setPageReferenced(&sCheck, i);


  /* Check the integrity of the freelist
  */
  sCheck.zPfx = "Main freelist: ";
  checkList(&sCheck, 1, get4byte(&pBt->pPage1->aData[32]),
            get4byte(&pBt->pPage1->aData[36]));
  sCheck.zPfx = 0;

  /* Check all the tables.
  */
  testcase( pBt->db->flags & SQLITE_CellSizeCk );
  pBt->db->flags &= ~SQLITE_CellSizeCk;
  for(i=0; (int)i<nRoot && sCheck.mxErr; i++){
    i64 notUsed;
    if( aRoot[i]==0 ) continue;
#ifndef SQLITE_OMIT_AUTOVACUUM
    if( pBt->autoVacuum && aRoot[i]>1 ){
      checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0);
    }
#endif

    checkTreePage(&sCheck, aRoot[i], &notUsed, LARGEST_INT64);

  }
  pBt->db->flags = savedDbFlags;

  /* Make sure every page in the file is referenced
  */
  for(i=1; i<=sCheck.nPage && sCheck.mxErr; i++){
#ifdef SQLITE_OMIT_AUTOVACUUM
    if( getPageReferenced(&sCheck, i)==0 ){
      checkAppendMsg(&sCheck, "Page %d is never used", i);
................................................................................
    if( getPageReferenced(&sCheck, i)!=0 && 
       (PTRMAP_PAGENO(pBt, i)==i && pBt->autoVacuum) ){
      checkAppendMsg(&sCheck, "Pointer map page %d is referenced", i);
    }
#endif
  }












  /* Clean  up and report errors.
  */

integrity_ck_cleanup:
  sqlite3PageFree(sCheck.heap);
  sqlite3_free(sCheck.aPgRef);
  if( sCheck.mallocFailed ){
    sqlite3StrAccumReset(&sCheck.errMsg);
    sCheck.nErr++;

  }
  *pnErr = sCheck.nErr;
  if( sCheck.nErr==0 ) sqlite3StrAccumReset(&sCheck.errMsg);
  /* Make sure this analysis did not leave any unref() pages. */
  assert( nRef==sqlite3PagerRefcount(pBt->pPager) );
  sqlite3BtreeLeave(p);
  return sqlite3StrAccumFinish(&sCheck.errMsg);
}
#endif /* SQLITE_OMIT_INTEGRITY_CHECK */

/*
** Return the full pathname of the underlying database file.  Return
** an empty string if the database is in-memory or a TEMP database.

Changes to src/btreeInt.h.

678
679
680
681
682
683
684

685
686
687
688
689
690
691
...
697
698
699
700
701
702
703


704
705
706
  Pgno nPage;       /* Number of pages in the database */
  int mxErr;        /* Stop accumulating errors when this reaches zero */
  int nErr;         /* Number of messages written to zErrMsg so far */
  int mallocFailed; /* A memory allocation error has occurred */
  const char *zPfx; /* Error message prefix */
  int v1, v2;       /* Values for up to two %d fields in zPfx */
  StrAccum errMsg;  /* Accumulate the error message text here */

};

/*
** Routines to read or write a two- and four-byte big-endian integer values.
*/
#define get2byte(x)   ((x)[0]<<8 | (x)[1])
#define put2byte(p,v) ((p)[0] = (u8)((v)>>8), (p)[1] = (u8)(v))
................................................................................
** two-byte aligned address.  get2bytea() is only used for accessing the
** cell addresses in a btree header.
*/
#if SQLITE_BYTEORDER==4321
# define get2byteAligned(x)  (*(u16*)(x))
#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4008000
# define get2byteAligned(x)  __builtin_bswap16(*(u16*)(x))


#else
# define get2byteAligned(x)  ((x)[0]<<8 | (x)[1])
#endif







>







 







>
>



678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
...
698
699
700
701
702
703
704
705
706
707
708
709
  Pgno nPage;       /* Number of pages in the database */
  int mxErr;        /* Stop accumulating errors when this reaches zero */
  int nErr;         /* Number of messages written to zErrMsg so far */
  int mallocFailed; /* A memory allocation error has occurred */
  const char *zPfx; /* Error message prefix */
  int v1, v2;       /* Values for up to two %d fields in zPfx */
  StrAccum errMsg;  /* Accumulate the error message text here */
  u32 *heap;        /* Min-heap used for analyzing cell coverage */
};

/*
** Routines to read or write a two- and four-byte big-endian integer values.
*/
#define get2byte(x)   ((x)[0]<<8 | (x)[1])
#define put2byte(p,v) ((p)[0] = (u8)((v)>>8), (p)[1] = (u8)(v))
................................................................................
** two-byte aligned address.  get2bytea() is only used for accessing the
** cell addresses in a btree header.
*/
#if SQLITE_BYTEORDER==4321
# define get2byteAligned(x)  (*(u16*)(x))
#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4008000
# define get2byteAligned(x)  __builtin_bswap16(*(u16*)(x))
#elif SQLITE_BYTEORDER==1234 && defined(_MSC_VER) && _MSC_VER>=1300
# define get2byteAligned(x)  _byteswap_ushort(*(u16*)(x))
#else
# define get2byteAligned(x)  ((x)[0]<<8 | (x)[1])
#endif

Changes to src/pager.c.

6383
6384
6385
6386
6387
6388
6389

6390
6391
6392
6393
6394
6395

6396
6397
6398
6399
6400
6401
6402
** Return TRUE if the database file is opened read-only.  Return FALSE
** if the database is (in theory) writable.
*/
u8 sqlite3PagerIsreadonly(Pager *pPager){
  return pPager->readOnly;
}


/*
** Return the number of references to the pager.
*/
int sqlite3PagerRefcount(Pager *pPager){
  return sqlite3PcacheRefCount(pPager->pPCache);
}


/*
** Return the approximate number of bytes of memory currently
** used by the pager and its associated cache.
*/
int sqlite3PagerMemUsed(Pager *pPager){
  int perPageSize = pPager->pageSize + pPager->nExtra + sizeof(PgHdr)







>






>







6383
6384
6385
6386
6387
6388
6389
6390
6391
6392
6393
6394
6395
6396
6397
6398
6399
6400
6401
6402
6403
6404
** Return TRUE if the database file is opened read-only.  Return FALSE
** if the database is (in theory) writable.
*/
u8 sqlite3PagerIsreadonly(Pager *pPager){
  return pPager->readOnly;
}

#ifdef SQLITE_DEBUG
/*
** Return the number of references to the pager.
*/
int sqlite3PagerRefcount(Pager *pPager){
  return sqlite3PcacheRefCount(pPager->pPCache);
}
#endif

/*
** Return the approximate number of bytes of memory currently
** used by the pager and its associated cache.
*/
int sqlite3PagerMemUsed(Pager *pPager){
  int perPageSize = pPager->pageSize + pPager->nExtra + sizeof(PgHdr)

Changes to src/pager.h.

169
170
171
172
173
174
175

176

177
178
179
180
181
182
183
#ifdef SQLITE_ENABLE_ZIPVFS
  int sqlite3PagerWalFramesize(Pager *pPager);
#endif

/* Functions used to query pager state and configuration. */
u8 sqlite3PagerIsreadonly(Pager*);
u32 sqlite3PagerDataVersion(Pager*);

int sqlite3PagerRefcount(Pager*);

int sqlite3PagerMemUsed(Pager*);
const char *sqlite3PagerFilename(Pager*, int);
const sqlite3_vfs *sqlite3PagerVfs(Pager*);
sqlite3_file *sqlite3PagerFile(Pager*);
const char *sqlite3PagerJournalname(Pager*);
int sqlite3PagerNosync(Pager*);
void *sqlite3PagerTempSpace(Pager*);







>
|
>







169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
#ifdef SQLITE_ENABLE_ZIPVFS
  int sqlite3PagerWalFramesize(Pager *pPager);
#endif

/* Functions used to query pager state and configuration. */
u8 sqlite3PagerIsreadonly(Pager*);
u32 sqlite3PagerDataVersion(Pager*);
#ifdef SQLITE_DEBUG
  int sqlite3PagerRefcount(Pager*);
#endif
int sqlite3PagerMemUsed(Pager*);
const char *sqlite3PagerFilename(Pager*, int);
const sqlite3_vfs *sqlite3PagerVfs(Pager*);
sqlite3_file *sqlite3PagerFile(Pager*);
const char *sqlite3PagerJournalname(Pager*);
int sqlite3PagerNosync(Pager*);
void *sqlite3PagerTempSpace(Pager*);

Changes to src/select.c.

4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
  Select *p,             /* The SELECT statement being coded. */
  SelectDest *pDest      /* What to do with the query results */
){
  int i, j;              /* Loop counters */
  WhereInfo *pWInfo;     /* Return from sqlite3WhereBegin() */
  Vdbe *v;               /* The virtual machine under construction */
  int isAgg;             /* True for select lists like "count(*)" */
  ExprList *pEList;      /* List of columns to extract. */
  SrcList *pTabList;     /* List of tables to select from */
  Expr *pWhere;          /* The WHERE clause.  May be NULL */
  ExprList *pGroupBy;    /* The GROUP BY clause.  May be NULL */
  Expr *pHaving;         /* The HAVING clause.  May be NULL */
  int rc = 1;            /* Value to return from this function */
  DistinctCtx sDistinct; /* Info on how to code the DISTINCT keyword */
  SortCtx sSort;         /* Info on how to code the ORDER BY clause */







|







4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
  Select *p,             /* The SELECT statement being coded. */
  SelectDest *pDest      /* What to do with the query results */
){
  int i, j;              /* Loop counters */
  WhereInfo *pWInfo;     /* Return from sqlite3WhereBegin() */
  Vdbe *v;               /* The virtual machine under construction */
  int isAgg;             /* True for select lists like "count(*)" */
  ExprList *pEList = 0;  /* List of columns to extract. */
  SrcList *pTabList;     /* List of tables to select from */
  Expr *pWhere;          /* The WHERE clause.  May be NULL */
  ExprList *pGroupBy;    /* The GROUP BY clause.  May be NULL */
  Expr *pHaving;         /* The HAVING clause.  May be NULL */
  int rc = 1;            /* Value to return from this function */
  DistinctCtx sDistinct; /* Info on how to code the DISTINCT keyword */
  SortCtx sSort;         /* Info on how to code the ORDER BY clause */

Changes to src/sqliteInt.h.

181
182
183
184
185
186
187










188
189
190
191
192
193
194
#  define SQLITE_NOINLINE  __attribute__((noinline))
#elif defined(_MSC_VER) && _MSC_VER>=1310
#  define SQLITE_NOINLINE  __declspec(noinline)
#else
#  define SQLITE_NOINLINE
#endif











/*
** The SQLITE_THREADSAFE macro must be defined as 0, 1, or 2.
** 0 means mutexes are permanently disable and the library is never
** threadsafe.  1 means the library is serialized which is the highest
** level of threadsafety.  2 means the library is multithreaded - multiple
** threads can use SQLite as long as no two threads try to use the same
** database connection at the same time.







>
>
>
>
>
>
>
>
>
>







181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
#  define SQLITE_NOINLINE  __attribute__((noinline))
#elif defined(_MSC_VER) && _MSC_VER>=1310
#  define SQLITE_NOINLINE  __declspec(noinline)
#else
#  define SQLITE_NOINLINE
#endif

/*
** Make sure that the compiler intrinsics we desire are enabled when
** compiling with an appropriate version of MSVC.
*/
#if defined(_MSC_VER) && _MSC_VER>=1300
#  include <intrin.h>
#  pragma intrinsic(_byteswap_ushort)
#  pragma intrinsic(_byteswap_ulong)
#endif

/*
** The SQLITE_THREADSAFE macro must be defined as 0, 1, or 2.
** 0 means mutexes are permanently disable and the library is never
** threadsafe.  1 means the library is serialized which is the highest
** level of threadsafety.  2 means the library is multithreaded - multiple
** threads can use SQLite as long as no two threads try to use the same
** database connection at the same time.

Changes to src/util.c.

1082
1083
1084
1085
1086
1087
1088




1089
1090
1091
1092
1093
1094
1095
1096
1097
1098



1099
1100
1101
1102
1103
1104
1105
  u32 x;
  memcpy(&x,p,4);
  return x;
#elif SQLITE_BYTEORDER==1234 && defined(__GNUC__)
  u32 x;
  memcpy(&x,p,4);
  return __builtin_bswap32(x);




#else
  testcase( p[0]&0x80 );
  return ((unsigned)p[0]<<24) | (p[1]<<16) | (p[2]<<8) | p[3];
#endif
}
void sqlite3Put4byte(unsigned char *p, u32 v){
#if SQLITE_BYTEORDER==4321
  memcpy(p,&v,4);
#elif SQLITE_BYTEORDER==1234 && defined(__GNUC__)
  u32 x = __builtin_bswap32(v);



  memcpy(p,&x,4);
#else
  p[0] = (u8)(v>>24);
  p[1] = (u8)(v>>16);
  p[2] = (u8)(v>>8);
  p[3] = (u8)v;
#endif







>
>
>
>










>
>
>







1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
  u32 x;
  memcpy(&x,p,4);
  return x;
#elif SQLITE_BYTEORDER==1234 && defined(__GNUC__)
  u32 x;
  memcpy(&x,p,4);
  return __builtin_bswap32(x);
#elif SQLITE_BYTEORDER==1234 && defined(_MSC_VER) && _MSC_VER>=1300
  u32 x;
  memcpy(&x,p,4);
  return _byteswap_ulong(x);
#else
  testcase( p[0]&0x80 );
  return ((unsigned)p[0]<<24) | (p[1]<<16) | (p[2]<<8) | p[3];
#endif
}
void sqlite3Put4byte(unsigned char *p, u32 v){
#if SQLITE_BYTEORDER==4321
  memcpy(p,&v,4);
#elif SQLITE_BYTEORDER==1234 && defined(__GNUC__)
  u32 x = __builtin_bswap32(v);
  memcpy(p,&x,4);
#elif SQLITE_BYTEORDER==1234 && defined(_MSC_VER) && _MSC_VER>=1300
  u32 x = _byteswap_ulong(v);
  memcpy(p,&x,4);
#else
  p[0] = (u8)(v>>24);
  p[1] = (u8)(v>>16);
  p[2] = (u8)(v>>8);
  p[3] = (u8)v;
#endif

Changes to test/corrupt2.test.

245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
  db2 eval {SELECT rowid FROM t1} {
    set result [db2 eval {pragma integrity_check}]
    break
  }
  set result
} {{*** in database main ***
On tree page 2 cell 0: 2nd reference to page 10
On tree page 2 cell 1: Child page depth differs
Page 4 is never used}}

db2 close

proc corruption_test {args} {
  set A(-corrupt) {}
  set A(-sqlprep) {}







<







245
246
247
248
249
250
251

252
253
254
255
256
257
258
  db2 eval {SELECT rowid FROM t1} {
    set result [db2 eval {pragma integrity_check}]
    break
  }
  set result
} {{*** in database main ***
On tree page 2 cell 0: 2nd reference to page 10

Page 4 is never used}}

db2 close

proc corruption_test {args} {
  set A(-corrupt) {}
  set A(-sqlprep) {}

Changes to test/corrupt7.test.

61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
  hexio_get_int [hexio_read test.db 20 1]
} 0      ;# Unused bytes per page is 0

integrity_check corrupt7-1.4

# Deliberately corrupt some of the cell offsets in the btree page
# on page 2 of the database.
#
# The error message is different depending on whether or not the
# SQLITE_ENABLE_OVERSIZE_CELL_CHECK compile-time option is engaged.
#
ifcapable oversize_cell_check {
  do_test corrupt7-2.1 {
    db close
    hexio_write test.db 1062 FF
    sqlite3 db test.db
    db eval {PRAGMA integrity_check(1)}
  } {{*** in database main ***
Page 2: btreeInitPage() returns error code 11}}

  do_test corrupt7-2.2 {
    db close
    hexio_write test.db 1062 04
    sqlite3 db test.db
    db eval {PRAGMA integrity_check(1)}
  } {{*** in database main ***
Page 2: btreeInitPage() returns error code 11}}
} else {
  do_test corrupt7-2.1 {
    db close
    hexio_write test.db 1062 FF
    sqlite3 db test.db
    db eval {PRAGMA integrity_check(1)}
  } {{*** in database main ***
Corruption detected in cell 15 on page 2}}
  do_test corrupt7-2.2 {
    db close
    hexio_write test.db 1062 04
    sqlite3 db test.db
    db eval {PRAGMA integrity_check(1)}
  } {{*** in database main ***
On tree page 2 cell 15: Rowid 0 out of order (previous was 15)}}
}
  
# The code path that was causing the buffer overrun that this test
# case was checking for was removed.
#
#do_test corrupt7-3.1 {
#  execsql {
#    DROP TABLE t1;







<
<
<
<
<
|
|
|
|
|
|
<
>
|
|
|
|
|
|
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
|
<







61
62
63
64
65
66
67





68
69
70
71
72
73

74
75
76
77
78
79
80















81

82
83
84
85
86
87
88
  hexio_get_int [hexio_read test.db 20 1]
} 0      ;# Unused bytes per page is 0

integrity_check corrupt7-1.4

# Deliberately corrupt some of the cell offsets in the btree page
# on page 2 of the database.





do_test corrupt7-2.1 {
  db close
  hexio_write test.db 1062 FF
  sqlite3 db test.db
  db eval {PRAGMA integrity_check(1)}
} {{*** in database main ***

On tree page 2 cell 15: Offset 65457 out of range 945..1020}}
do_test corrupt7-2.2 {
  db close
  hexio_write test.db 1062 04
  sqlite3 db test.db
  db eval {PRAGMA integrity_check(1)}
} {{*** in database main ***















On tree page 2 cell 15: Offset 1201 out of range 945..1020}}

  
# The code path that was causing the buffer overrun that this test
# case was checking for was removed.
#
#do_test corrupt7-3.1 {
#  execsql {
#    DROP TABLE t1;

Changes to test/corruptE.test.

10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
..
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
...
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
#***********************************************************************
# This file implements regression tests for SQLite library.
#
# This file implements tests to make sure SQLite does not crash or
# segfault if it sees a corrupt database file.  It specifcally
# focuses on rowid order corruption.
#
# $Id: corruptE.test,v 1.14 2009/07/11 06:55:34 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# Do not use a codec for tests in this file, as the database file is
# manipulated directly using tcl scripts (using the [hexio_write] command).
#
................................................................................
  forcecopy test.bu test.db

  # insert corrupt byte(s)
  hexio_write test.db 2041 [format %02x 0x2e]

  sqlite3 db test.db

  set res [ catchsql {PRAGMA integrity_check} ]
  set ans [lindex $res 1]

  list [regexp {out of order.*previous was} $ans] \
       [regexp {out of order.*max larger than parent max} $ans]
} {1 1}

do_test corruptE-2.2 {
  db close
  forcecopy test.bu test.db

  # insert corrupt byte(s)
  hexio_write test.db 2047 [format %02x 0x84]

  sqlite3 db test.db

  set res [ catchsql {PRAGMA integrity_check} ]
  set ans [lindex $res 1]

  list [regexp {out of order.*previous was} $ans] \
       [regexp {out of order.*min less than parent min} $ans]
} {1 1}

do_test corruptE-2.3 {
  db close
  forcecopy test.bu test.db

  # insert corrupt byte(s)
  hexio_write test.db 7420 [format %02x 0xa8]
  hexio_write test.db 10459 [format %02x 0x8d]

  sqlite3 db test.db

  set res [ catchsql {PRAGMA integrity_check} ]
  set ans [lindex $res 1]

  list [regexp {out of order.*max larger than parent min} $ans]
} {1}

do_test corruptE-2.4 {
  db close
  forcecopy test.bu test.db

  # insert corrupt byte(s)
  hexio_write test.db 10233 [format %02x 0xd0]

  sqlite3 db test.db

  set res [ catchsql {PRAGMA integrity_check} ]
  set ans [lindex $res 1]

  list [regexp {out of order.*min less than parent max} $ans]
} {1}


set tests [list {10233 0xd0} \
                {941 0x42} \
                {1028 0x53} \
                {2041 0xd0} \
                {2042 0x1f} \
                {2047 0xaa} \
                {2263 0x29} \
                {2274 0x75} \
                {3267 0xf2} \
                {4104 0x2c} \
                {5113 0x36} \
                {10233 0x84} \
                {10234 0x74} \
                {10239 0x41} \
                {10453 0x11} \
                {11273 0x28} \
                {11455 0x11} \
                {11461 0xe6} \
                {12281 0x99} \
                {12296 0x9e} \
                {12297 0xd7} \
                {13303 0x53} ]

set tc 1
foreach test $tests {
  do_test corruptE-3.$tc {
    db close
................................................................................
    forcecopy test.bu test.db

    # insert corrupt byte(s)
    hexio_write test.db [lindex $test 0] [format %02x [lindex $test 1]]

    sqlite3 db test.db

    set res [ catchsql {PRAGMA integrity_check} ]
    set ans [lindex $res 1]

    list [regexp {out of order} $ans]
  } {1}
  incr tc 1
}

finish_test







<







 







|
|
<
<
<
<










|
|
<
<
<
<











|
|
<
<
<










|
|
<
<
<




<


<
<


<




<

<

<
<







 







|
<
<
|
<




10
11
12
13
14
15
16

17
18
19
20
21
22
23
..
74
75
76
77
78
79
80
81
82




83
84
85
86
87
88
89
90
91
92
93
94




95
96
97
98
99
100
101
102
103
104
105
106
107



108
109
110
111
112
113
114
115
116
117
118
119



120
121
122
123

124
125


126
127

128
129
130
131

132

133


134
135
136
137
138
139
140
...
141
142
143
144
145
146
147
148


149

150
151
152
153
#***********************************************************************
# This file implements regression tests for SQLite library.
#
# This file implements tests to make sure SQLite does not crash or
# segfault if it sees a corrupt database file.  It specifcally
# focuses on rowid order corruption.
#


set testdir [file dirname $argv0]
source $testdir/tester.tcl

# Do not use a codec for tests in this file, as the database file is
# manipulated directly using tcl scripts (using the [hexio_write] command).
#
................................................................................
  forcecopy test.bu test.db

  # insert corrupt byte(s)
  hexio_write test.db 2041 [format %02x 0x2e]

  sqlite3 db test.db

  catchsql {PRAGMA integrity_check}
} {/ out of order/}





do_test corruptE-2.2 {
  db close
  forcecopy test.bu test.db

  # insert corrupt byte(s)
  hexio_write test.db 2047 [format %02x 0x84]

  sqlite3 db test.db

  catchsql {PRAGMA integrity_check}
} {/ Extends off end of page/}





do_test corruptE-2.3 {
  db close
  forcecopy test.bu test.db

  # insert corrupt byte(s)
  hexio_write test.db 7420 [format %02x 0xa8]
  hexio_write test.db 10459 [format %02x 0x8d]

  sqlite3 db test.db

  catchsql {PRAGMA integrity_check}
} {/out of order/}




do_test corruptE-2.4 {
  db close
  forcecopy test.bu test.db

  # insert corrupt byte(s)
  hexio_write test.db 10233 [format %02x 0xd0]

  sqlite3 db test.db

  catchsql {PRAGMA integrity_check}
} {/out of order/}





set tests [list {10233 0xd0} \
                {941 0x42} \

                {2041 0xd0} \
                {2042 0x1f} \


                {2274 0x75} \
                {3267 0xf2} \

                {5113 0x36} \
                {10233 0x84} \
                {10234 0x74} \
                {10239 0x41} \

                {11273 0x28} \

                {11461 0xe6} \


                {12297 0xd7} \
                {13303 0x53} ]

set tc 1
foreach test $tests {
  do_test corruptE-3.$tc {
    db close
................................................................................
    forcecopy test.bu test.db

    # insert corrupt byte(s)
    hexio_write test.db [lindex $test 0] [format %02x [lindex $test 1]]

    sqlite3 db test.db

    catchsql {PRAGMA integrity_check}


  } {/out of order/}

  incr tc 1
}

finish_test

Changes to test/speedtest1.c.

19
20
21
22
23
24
25

26
27
28
29
30
31
32
..
57
58
59
60
61
62
63

64
65
66
67
68
69
70
...
319
320
321
322
323
324
325









326
327
328
329
330
331
332
...
336
337
338
339
340
341
342

343
344
345
346
347
348
349
...
388
389
390
391
392
393
394

395
396
397
398
399
400
401
....
1238
1239
1240
1241
1242
1243
1244


1245
1246
1247
1248
1249
1250
1251
  "  --notnull           Add NOT NULL constraints to table columns\n"
  "  --pagesize N        Set the page size to N\n"
  "  --pcache N SZ       Configure N pages of pagecache each of size SZ bytes\n"
  "  --primarykey        Use PRIMARY KEY instead of UNIQUE where appropriate\n"
  "  --reprepare         Reprepare each statement upon every invocation\n"
  "  --scratch N SZ      Configure scratch memory for N slots of SZ bytes each\n"
  "  --sqlonly           No-op.  Only show the SQL that would have been run.\n"

  "  --size N            Relative test size.  Default=100\n"
  "  --stats             Show statistics at the end\n"
  "  --testset T         Run test-set T\n"
  "  --trace             Turn on SQL tracing\n"
  "  --threads N         Use up to N threads for sorting\n"
  "  --utf16be           Set text encoding to UTF-16BE\n"
  "  --utf16le           Set text encoding to UTF-16LE\n"
................................................................................
  sqlite3_int64 iStart;      /* Start-time for the current test */
  sqlite3_int64 iTotal;      /* Total time */
  int bWithoutRowid;         /* True for --without-rowid */
  int bReprepare;            /* True to reprepare the SQL on each rerun */
  int bSqlOnly;              /* True to print the SQL once only */
  int bExplain;              /* Print SQL with EXPLAIN prefix */
  int bVerify;               /* Try to verify that results are correct */

  int szTest;                /* Scale factor for test iterations */
  const char *zWR;           /* Might be WITHOUT ROWID */
  const char *zNN;           /* Might be NOT NULL */
  const char *zPK;           /* Might be UNIQUE or PRIMARY KEY */
  unsigned int x, y;         /* Pseudo-random number generator state */
  int nResult;               /* Size of the current result */
  char zResult[3000];        /* Text of the current result */
................................................................................
     || sqlite3_strglob("ALTER *", zSql)==0
      )
#endif
  ){
    printf("%.*s;\n", n, zSql);
  }
}










/* Run SQL */
void speedtest1_exec(const char *zFormat, ...){
  va_list ap;
  char *zSql;
  va_start(ap, zFormat);
  zSql = sqlite3_vmprintf(zFormat, ap);
................................................................................
  }else{
    char *zErrMsg = 0;
    int rc = sqlite3_exec(g.db, zSql, 0, 0, &zErrMsg);
    if( zErrMsg ) fatal_error("SQL error: %s\n%s\n", zErrMsg, zSql);
    if( rc!=SQLITE_OK ) fatal_error("exec error: %s\n", sqlite3_errmsg(g.db));
  }
  sqlite3_free(zSql);

}

/* Prepare an SQL statement */
void speedtest1_prepare(const char *zFormat, ...){
  va_list ap;
  char *zSql;
  va_start(ap, zFormat);
................................................................................
    sqlite3_finalize(g.pStmt);
    g.pStmt = pNew;
  }else
#endif
  {
    sqlite3_reset(g.pStmt);
  }

}

/* The sqlite3_trace() callback function */
static void traceCallback(void *NotUsed, const char *zSql){
  int n = (int)strlen(zSql);
  while( n>0 && (zSql[n-1]==';' || isspace(zSql[n-1])) ) n--;
  fprintf(stderr,"%.*s;\n", n, zSql);
................................................................................
      }else if( strcmp(z,"scratch")==0 ){
        if( i>=argc-2 ) fatal_error("missing arguments on %s\n", argv[i]);
        nScratch = integerValue(argv[i+1]);
        szScratch = integerValue(argv[i+2]);
        i += 2;
      }else if( strcmp(z,"sqlonly")==0 ){
        g.bSqlOnly = 1;


      }else if( strcmp(z,"size")==0 ){
        if( i>=argc-1 ) fatal_error("missing argument on %s\n", argv[i]);
        g.szTest = integerValue(argv[++i]);
      }else if( strcmp(z,"stats")==0 ){
        showStats = 1;
      }else if( strcmp(z,"testset")==0 ){
        if( i>=argc-1 ) fatal_error("missing argument on %s\n", argv[i]);







>







 







>







 







>
>
>
>
>
>
>
>
>







 







>







 







>







 







>
>







19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
..
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
...
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
...
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
...
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
....
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
  "  --notnull           Add NOT NULL constraints to table columns\n"
  "  --pagesize N        Set the page size to N\n"
  "  --pcache N SZ       Configure N pages of pagecache each of size SZ bytes\n"
  "  --primarykey        Use PRIMARY KEY instead of UNIQUE where appropriate\n"
  "  --reprepare         Reprepare each statement upon every invocation\n"
  "  --scratch N SZ      Configure scratch memory for N slots of SZ bytes each\n"
  "  --sqlonly           No-op.  Only show the SQL that would have been run.\n"
  "  --shrink-memory     Invoke sqlite3_db_release_memory() frequently.\n"
  "  --size N            Relative test size.  Default=100\n"
  "  --stats             Show statistics at the end\n"
  "  --testset T         Run test-set T\n"
  "  --trace             Turn on SQL tracing\n"
  "  --threads N         Use up to N threads for sorting\n"
  "  --utf16be           Set text encoding to UTF-16BE\n"
  "  --utf16le           Set text encoding to UTF-16LE\n"
................................................................................
  sqlite3_int64 iStart;      /* Start-time for the current test */
  sqlite3_int64 iTotal;      /* Total time */
  int bWithoutRowid;         /* True for --without-rowid */
  int bReprepare;            /* True to reprepare the SQL on each rerun */
  int bSqlOnly;              /* True to print the SQL once only */
  int bExplain;              /* Print SQL with EXPLAIN prefix */
  int bVerify;               /* Try to verify that results are correct */
  int bMemShrink;            /* Call sqlite3_db_release_memory() often */
  int szTest;                /* Scale factor for test iterations */
  const char *zWR;           /* Might be WITHOUT ROWID */
  const char *zNN;           /* Might be NOT NULL */
  const char *zPK;           /* Might be UNIQUE or PRIMARY KEY */
  unsigned int x, y;         /* Pseudo-random number generator state */
  int nResult;               /* Size of the current result */
  char zResult[3000];        /* Text of the current result */
................................................................................
     || sqlite3_strglob("ALTER *", zSql)==0
      )
#endif
  ){
    printf("%.*s;\n", n, zSql);
  }
}

/* Shrink memory used, if appropriate and if the SQLite version is capable
** of doing so.
*/
void speedtest1_shrink_memory(void){
#if SQLITE_VERSION_NUMBER>=3007010
  if( g.bMemShrink ) sqlite3_db_release_memory(g.db);
#endif
}

/* Run SQL */
void speedtest1_exec(const char *zFormat, ...){
  va_list ap;
  char *zSql;
  va_start(ap, zFormat);
  zSql = sqlite3_vmprintf(zFormat, ap);
................................................................................
  }else{
    char *zErrMsg = 0;
    int rc = sqlite3_exec(g.db, zSql, 0, 0, &zErrMsg);
    if( zErrMsg ) fatal_error("SQL error: %s\n%s\n", zErrMsg, zSql);
    if( rc!=SQLITE_OK ) fatal_error("exec error: %s\n", sqlite3_errmsg(g.db));
  }
  sqlite3_free(zSql);
  speedtest1_shrink_memory();
}

/* Prepare an SQL statement */
void speedtest1_prepare(const char *zFormat, ...){
  va_list ap;
  char *zSql;
  va_start(ap, zFormat);
................................................................................
    sqlite3_finalize(g.pStmt);
    g.pStmt = pNew;
  }else
#endif
  {
    sqlite3_reset(g.pStmt);
  }
  speedtest1_shrink_memory();
}

/* The sqlite3_trace() callback function */
static void traceCallback(void *NotUsed, const char *zSql){
  int n = (int)strlen(zSql);
  while( n>0 && (zSql[n-1]==';' || isspace(zSql[n-1])) ) n--;
  fprintf(stderr,"%.*s;\n", n, zSql);
................................................................................
      }else if( strcmp(z,"scratch")==0 ){
        if( i>=argc-2 ) fatal_error("missing arguments on %s\n", argv[i]);
        nScratch = integerValue(argv[i+1]);
        szScratch = integerValue(argv[i+2]);
        i += 2;
      }else if( strcmp(z,"sqlonly")==0 ){
        g.bSqlOnly = 1;
      }else if( strcmp(z,"shrink-memory")==0 ){
        g.bMemShrink = 1;
      }else if( strcmp(z,"size")==0 ){
        if( i>=argc-1 ) fatal_error("missing argument on %s\n", argv[i]);
        g.szTest = integerValue(argv[++i]);
      }else if( strcmp(z,"stats")==0 ){
        showStats = 1;
      }else if( strcmp(z,"testset")==0 ){
        if( i>=argc-1 ) fatal_error("missing argument on %s\n", argv[i]);