SQLite

Changes On Branch mmap-warm
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Changes In Branch mmap-warm Excluding Merge-Ins

This is equivalent to a diff from 42f07775 to 3235835b

2017-09-18
18:17
Add the sqlite3_mmap_warm() function as an extension in the ext/misc/mmapwarm.c source file. (check-in: 1b2de414 user: drh tags: trunk)
18:08
Fix a typo in the MSVC makefile. (Closed-Leaf check-in: 3235835b user: drh tags: mmap-warm)
16:28
Add extension "mmapwarm.c". Provides function sqlite3_mmap_warm(), used to "warm up" the memory mapping used by SQLite in mmap mode to access db file content. (check-in: d4a30b91 user: dan tags: mmap-warm)
09:40
The out-of-bounds read on recovery fix of check-in [378afa16381a222a] caused problems for some corner-case error conditions. This alternative fix appears to work better. (check-in: 74f399d8 user: drh tags: trunk)
08:51
Merge latest trunk changes with this branch. (Leaf check-in: 2e573350 user: dan tags: shared-mapping-hack)
00:18
Fix the CSV virtual table extension so that it works when the default character is unsigned. (check-in: 42f07775 user: drh tags: trunk)
2017-09-17
19:45
Do not make the assumption (as check-in [4da49a95c0f07] incorrectly did) that the ExprList returned by sqlite3ExprListDup() would never be passed into sqlite3ExprListAppend(). Include a new test case that shows this sometimes does happen. (check-in: 29227d00 user: drh tags: trunk)

Changes to Makefile.in.

424
425
426
427
428
429
430

431
432
433
434
435
436
437
  $(TOP)/ext/misc/eval.c \
  $(TOP)/ext/misc/fileio.c \
  $(TOP)/ext/misc/fuzzer.c \
  $(TOP)/ext/fts5/fts5_tcl.c \
  $(TOP)/ext/fts5/fts5_test_mi.c \
  $(TOP)/ext/fts5/fts5_test_tok.c \
  $(TOP)/ext/misc/ieee754.c \

  $(TOP)/ext/misc/nextchar.c \
  $(TOP)/ext/misc/percentile.c \
  $(TOP)/ext/misc/regexp.c \
  $(TOP)/ext/misc/remember.c \
  $(TOP)/ext/misc/series.c \
  $(TOP)/ext/misc/spellfix.c \
  $(TOP)/ext/misc/totype.c \







>







424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
  $(TOP)/ext/misc/eval.c \
  $(TOP)/ext/misc/fileio.c \
  $(TOP)/ext/misc/fuzzer.c \
  $(TOP)/ext/fts5/fts5_tcl.c \
  $(TOP)/ext/fts5/fts5_test_mi.c \
  $(TOP)/ext/fts5/fts5_test_tok.c \
  $(TOP)/ext/misc/ieee754.c \
  $(TOP)/ext/misc/mmapwarm.c \
  $(TOP)/ext/misc/nextchar.c \
  $(TOP)/ext/misc/percentile.c \
  $(TOP)/ext/misc/regexp.c \
  $(TOP)/ext/misc/remember.c \
  $(TOP)/ext/misc/series.c \
  $(TOP)/ext/misc/spellfix.c \
  $(TOP)/ext/misc/totype.c \

Changes to Makefile.msc.

1407
1408
1409
1410
1411
1412
1413

1414
1415
1416
1417
1418
1419
1420
  $(TOP)\ext\misc\eval.c \
  $(TOP)\ext\misc\fileio.c \
  $(TOP)\ext\misc\fuzzer.c \
  $(TOP)\ext\fts5\fts5_tcl.c \
  $(TOP)\ext\fts5\fts5_test_mi.c \
  $(TOP)\ext\fts5\fts5_test_tok.c \
  $(TOP)\ext\misc\ieee754.c \

  $(TOP)\ext\misc\nextchar.c \
  $(TOP)\ext\misc\percentile.c \
  $(TOP)\ext\misc\regexp.c \
  $(TOP)\ext\misc\remember.c \
  $(TOP)\ext\misc\series.c \
  $(TOP)\ext\misc\spellfix.c \
  $(TOP)\ext\misc\totype.c \







>







1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
  $(TOP)\ext\misc\eval.c \
  $(TOP)\ext\misc\fileio.c \
  $(TOP)\ext\misc\fuzzer.c \
  $(TOP)\ext\fts5\fts5_tcl.c \
  $(TOP)\ext\fts5\fts5_test_mi.c \
  $(TOP)\ext\fts5\fts5_test_tok.c \
  $(TOP)\ext\misc\ieee754.c \
  $(TOP)\ext\misc\mmapwarm.c \
  $(TOP)\ext\misc\nextchar.c \
  $(TOP)\ext\misc\percentile.c \
  $(TOP)\ext\misc\regexp.c \
  $(TOP)\ext\misc\remember.c \
  $(TOP)\ext\misc\series.c \
  $(TOP)\ext\misc\spellfix.c \
  $(TOP)\ext\misc\totype.c \

Added ext/misc/mmapwarm.c.

























































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
/*
** 2017-09-18
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
**
*/

#include "sqlite3.h"


/*
** This function is used to touch each page of a mapping of a memory
** mapped SQLite database. Assuming that the system has sufficient free
** memory and supports sufficiently large mappings, this causes the OS 
** to cache the entire database in main memory, making subsequent 
** database accesses faster.
**
** If the second parameter to this function is not NULL, it is the name of
** the specific database to operate on (i.e. "main" or the name of an
** attached database).
**
** SQLITE_OK is returned if successful, or an SQLite error code otherwise.
** It is not considered an error if the file is not memory-mapped, or if
** the mapping does not span the entire file. If an error does occur, a
** transaction may be left open on the database file.
**
** It is illegal to call this function when the database handle has an 
** open transaction. SQLITE_MISUSE is returned in this case.
*/
int sqlite3_mmap_warm(sqlite3 *db, const char *zDb){
  int rc = SQLITE_OK;
  char *zSql = 0;
  int pgsz = 0;
  int nTotal = 0;

  if( 0==sqlite3_get_autocommit(db) ) return SQLITE_MISUSE;

  /* Open a read-only transaction on the file in question */
  zSql = sqlite3_mprintf("BEGIN; SELECT * FROM %s%q%ssqlite_master", 
      (zDb ? "'" : ""), (zDb ? zDb : ""), (zDb ? "'." : "")
  );
  if( zSql==0 ) return SQLITE_NOMEM;
  rc = sqlite3_exec(db, zSql, 0, 0, 0);
  sqlite3_free(zSql);

  /* Find the SQLite page size of the file */
  if( rc==SQLITE_OK ){
    zSql = sqlite3_mprintf("PRAGMA %s%q%spage_size", 
        (zDb ? "'" : ""), (zDb ? zDb : ""), (zDb ? "'." : "")
    );
    if( zSql==0 ){
      rc = SQLITE_NOMEM;
    }else{
      sqlite3_stmt *pPgsz = 0;
      rc = sqlite3_prepare_v2(db, zSql, -1, &pPgsz, 0);
      sqlite3_free(zSql);
      if( rc==SQLITE_OK ){
        if( sqlite3_step(pPgsz)==SQLITE_ROW ){
          pgsz = sqlite3_column_int(pPgsz, 0);
        }
        rc = sqlite3_finalize(pPgsz);
      }
      if( rc==SQLITE_OK && pgsz==0 ){
        rc = SQLITE_ERROR;
      }
    }
  }

  /* Touch each mmap'd page of the file */
  if( rc==SQLITE_OK ){
    int rc2;
    sqlite3_file *pFd = 0;
    rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_FILE_POINTER, &pFd);
    if( rc==SQLITE_OK && pFd->pMethods->iVersion>=3 ){
      sqlite3_int64 iPg = 1;
      sqlite3_io_methods const *p = pFd->pMethods;
      while( 1 ){
        unsigned char *pMap;
        rc = p->xFetch(pFd, pgsz*iPg, pgsz, (void**)&pMap);
        if( rc!=SQLITE_OK || pMap==0 ) break;

        nTotal += pMap[0];
        nTotal += pMap[pgsz-1];

        rc = p->xUnfetch(pFd, pgsz*iPg, (void*)pMap);
        if( rc!=SQLITE_OK ) break;
        iPg++;
      }
      sqlite3_log(SQLITE_OK, 
          "sqlite3_mmap_warm_cache: Warmed up %d pages of %s", iPg==1?0:iPg,
          sqlite3_db_filename(db, zDb)
      );
    }

    rc2 = sqlite3_exec(db, "END", 0, 0, 0);
    if( rc==SQLITE_OK ) rc = rc2;
  }

  return rc;
}

Changes to main.mk.

330
331
332
333
334
335
336

337
338
339
340
341
342
343
  $(TOP)/ext/misc/carray.c \
  $(TOP)/ext/misc/closure.c \
  $(TOP)/ext/misc/csv.c \
  $(TOP)/ext/misc/eval.c \
  $(TOP)/ext/misc/fileio.c \
  $(TOP)/ext/misc/fuzzer.c \
  $(TOP)/ext/misc/ieee754.c \

  $(TOP)/ext/misc/nextchar.c \
  $(TOP)/ext/misc/percentile.c \
  $(TOP)/ext/misc/regexp.c \
  $(TOP)/ext/misc/remember.c \
  $(TOP)/ext/misc/series.c \
  $(TOP)/ext/misc/spellfix.c \
  $(TOP)/ext/misc/totype.c \







>







330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
  $(TOP)/ext/misc/carray.c \
  $(TOP)/ext/misc/closure.c \
  $(TOP)/ext/misc/csv.c \
  $(TOP)/ext/misc/eval.c \
  $(TOP)/ext/misc/fileio.c \
  $(TOP)/ext/misc/fuzzer.c \
  $(TOP)/ext/misc/ieee754.c \
  $(TOP)/ext/misc/mmapwarm.c \
  $(TOP)/ext/misc/nextchar.c \
  $(TOP)/ext/misc/percentile.c \
  $(TOP)/ext/misc/regexp.c \
  $(TOP)/ext/misc/remember.c \
  $(TOP)/ext/misc/series.c \
  $(TOP)/ext/misc/spellfix.c \
  $(TOP)/ext/misc/totype.c \

Changes to src/test1.c.

7410
7411
7412
7413
7414
7415
7416





























7417
7418
7419
7420
7421
7422
7423
  }else{
    if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
    rc = sqlite3_db_config(db, SQLITE_DBCONFIG_MAINDBNAME, "icecube");
    Tcl_SetObjResult(interp, Tcl_NewIntObj(rc));
    return TCL_OK;
  }
}






























/*
** Register commands with the TCL interpreter.
*/
int Sqlitetest1_Init(Tcl_Interp *interp){
  extern int sqlite3_search_count;
  extern int sqlite3_found_count;







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







7410
7411
7412
7413
7414
7415
7416
7417
7418
7419
7420
7421
7422
7423
7424
7425
7426
7427
7428
7429
7430
7431
7432
7433
7434
7435
7436
7437
7438
7439
7440
7441
7442
7443
7444
7445
7446
7447
7448
7449
7450
7451
7452
  }else{
    if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
    rc = sqlite3_db_config(db, SQLITE_DBCONFIG_MAINDBNAME, "icecube");
    Tcl_SetObjResult(interp, Tcl_NewIntObj(rc));
    return TCL_OK;
  }
}

/*
** Usage: sqlite3_mmap_warm DB DBNAME
*/
static int SQLITE_TCLAPI test_mmap_warm(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
  Tcl_Obj *CONST objv[]
){
  extern int getDbPointer(Tcl_Interp*, const char*, sqlite3**);
  extern int sqlite3_mmap_warm(sqlite3 *db, const char *);

  if( objc!=2 && objc!=3 ){
    Tcl_WrongNumArgs(interp, 1, objv, "DB ?DBNAME?");
    return TCL_ERROR;
  }else{
    int rc;
    sqlite3 *db;
    const char *zDb = 0;
    if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
    if( objc==3 ){
      zDb = Tcl_GetString(objv[2]);
    }
    rc = sqlite3_mmap_warm(db, zDb);
    Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
    return TCL_OK;
  }
}

/*
** Register commands with the TCL interpreter.
*/
int Sqlitetest1_Init(Tcl_Interp *interp){
  extern int sqlite3_search_count;
  extern int sqlite3_found_count;
7680
7681
7682
7683
7684
7685
7686
7687
7688

7689
7690
7691
7692
7693
7694
7695
     { "sqlite3_snapshot_free", test_snapshot_free, 0 },
     { "sqlite3_snapshot_cmp", test_snapshot_cmp, 0 },
     { "sqlite3_snapshot_recover", test_snapshot_recover, 0 },
     { "sqlite3_snapshot_get_blob", test_snapshot_get_blob, 0 },
     { "sqlite3_snapshot_open_blob", test_snapshot_open_blob, 0 },
     { "sqlite3_snapshot_cmp_blob", test_snapshot_cmp_blob, 0 },
#endif
     { "sqlite3_delete_database", test_delete_database, 0 },
     { "atomic_batch_write",      test_atomic_batch_write,     0   },

  };
  static int bitmask_size = sizeof(Bitmask)*8;
  static int longdouble_size = sizeof(LONGDOUBLE_TYPE);
  int i;
  extern int sqlite3_sync_count, sqlite3_fullsync_count;
  extern int sqlite3_opentemp_count;
  extern int sqlite3_like_count;







|
|
>







7709
7710
7711
7712
7713
7714
7715
7716
7717
7718
7719
7720
7721
7722
7723
7724
7725
     { "sqlite3_snapshot_free", test_snapshot_free, 0 },
     { "sqlite3_snapshot_cmp", test_snapshot_cmp, 0 },
     { "sqlite3_snapshot_recover", test_snapshot_recover, 0 },
     { "sqlite3_snapshot_get_blob", test_snapshot_get_blob, 0 },
     { "sqlite3_snapshot_open_blob", test_snapshot_open_blob, 0 },
     { "sqlite3_snapshot_cmp_blob", test_snapshot_cmp_blob, 0 },
#endif
     { "sqlite3_delete_database", test_delete_database,    0 },
     { "atomic_batch_write",      test_atomic_batch_write, 0 },
     { "sqlite3_mmap_warm",       test_mmap_warm,          0 },
  };
  static int bitmask_size = sizeof(Bitmask)*8;
  static int longdouble_size = sizeof(LONGDOUBLE_TYPE);
  int i;
  extern int sqlite3_sync_count, sqlite3_fullsync_count;
  extern int sqlite3_opentemp_count;
  extern int sqlite3_like_count;