Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Minor optimizations to fts5 writes. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | fts5 |
| Files: | files | file ages | folders |
| SHA1: |
1fffe51fa92f1784365140d5b163ab6c |
| User & Date: | dan 2015-01-31 15:23:44 |
Context
|
2015-02-02
| ||
| 09:40 | Merge latest trunk changes with this branch. (check-in: 76212f2c user: dan tags: fts5) | |
|
2015-01-31
| ||
| 15:23 | Minor optimizations to fts5 writes. (check-in: 1fffe51f user: dan tags: fts5) | |
|
2015-01-29
| ||
| 20:59 | Fix some problems with transactions that both read and write an fts5 table. (check-in: 0e225b15 user: dan tags: fts5) | |
Changes
Changes to ext/fts5/fts5_hash.c.
72
73
74
75
76
77
78
79
80
81
82
83
84
85
...
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
|
*/
static void fts5Put4ByteVarint(u8 *a, int iVal){
a[0] = (0x80 | (u8)(iVal >> 21));
a[1] = (0x80 | (u8)(iVal >> 14));
a[2] = (0x80 | (u8)(iVal >> 7));
a[3] = (0x7F & (u8)(iVal));
}
/*
** Allocate a new hash table.
*/
int sqlite3Fts5HashNew(Fts5Hash **ppNew, int *pnByte){
int rc = SQLITE_OK;
Fts5Hash *pNew;
................................................................................
/* Issue the new-term callback */
rc = xTerm(pCtx, pList->zKey, nKey);
/* Issue the xEntry callbacks */
while( rc==SQLITE_OK && iOff<pList->nData ){
i64 iDelta; /* Rowid delta value */
int nPoslist; /* Size of position list in bytes */
iOff += getVarint(&pPtr[iOff], (u64*)&iDelta);
iRowid += iDelta;
iOff += fts5GetVarint32(&pPtr[iOff], nPoslist);
rc = xEntry(pCtx, iRowid, &pPtr[iOff], nPoslist);
iOff += nPoslist;
}
/* Issue the term-done callback */
if( rc==SQLITE_OK ) rc = xTermDone(pCtx);
}
sqlite3_free(pList);
|
>
>
>
>
>
>
>
>
>
>
>
>
<
>
>
|
|
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
...
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
|
*/
static void fts5Put4ByteVarint(u8 *a, int iVal){
a[0] = (0x80 | (u8)(iVal >> 21));
a[1] = (0x80 | (u8)(iVal >> 14));
a[2] = (0x80 | (u8)(iVal >> 7));
a[3] = (0x7F & (u8)(iVal));
}
static int fts5Get4ByteVarint(u8 *a, int *pnVarint){
int iRet = ((int)(a[0] & 0x7F) << 21) + ((int)(a[1] & 0x7F) << 14)
+ ((int)(a[2] & 0x7F) << 7) + ((int)(a[3]));
*pnVarint = (
(iRet & 0xFFFFFF80)==0 ? 1 :
(iRet & 0xFFFFC000)==0 ? 2 :
(iRet & 0xFFE00000)==0 ? 3 : 4
);
return iRet;
}
/*
** Allocate a new hash table.
*/
int sqlite3Fts5HashNew(Fts5Hash **ppNew, int *pnByte){
int rc = SQLITE_OK;
Fts5Hash *pNew;
................................................................................
/* Issue the new-term callback */
rc = xTerm(pCtx, pList->zKey, nKey);
/* Issue the xEntry callbacks */
while( rc==SQLITE_OK && iOff<pList->nData ){
i64 iDelta; /* Rowid delta value */
int nPoslist; /* Size of position list in bytes */
int nVarint;
iOff += getVarint(&pPtr[iOff], (u64*)&iDelta);
iRowid += iDelta;
nPoslist = fts5Get4ByteVarint(&pPtr[iOff], &nVarint);
iOff += 4;
rc = xEntry(pCtx, iRowid, &pPtr[iOff-nVarint], nPoslist+nVarint);
iOff += nPoslist;
}
/* Issue the term-done callback */
if( rc==SQLITE_OK ) rc = xTermDone(pCtx);
}
sqlite3_free(pList);
|
Changes to ext/fts5/fts5_index.c.
3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 |
void *pCtx,
i64 iRowid,
const u8 *aPoslist,
int nPoslist
){
Fts5FlushCtx *p = (Fts5FlushCtx*)pCtx;
Fts5Index *pIdx = p->pIdx;
/* Append the rowid itself */
fts5WriteAppendRowid(pIdx, &p->writer, iRowid);
/* Append the size of the position list in bytes */
fts5WriteAppendPoslistInt(pIdx, &p->writer, nPoslist);
/* And the poslist data */
fts5WriteAppendPoslistData(pIdx, &p->writer, aPoslist, nPoslist);
return pIdx->rc;
}
/*
** Flush the contents of in-memory hash table iHash to a new level-0
|
> > > > > > > > < < < |
3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 |
void *pCtx,
i64 iRowid,
const u8 *aPoslist,
int nPoslist
){
Fts5FlushCtx *p = (Fts5FlushCtx*)pCtx;
Fts5Index *pIdx = p->pIdx;
#ifdef SQLITE_DEBUG
/* The poslist-size varint should already be at the start of the
** aPoslist/nPoslist buffer. This assert verifies that. */
int n, i;
i = fts5GetVarint32(aPoslist, n);
assert( nPoslist==(n+i) );
#endif
/* Append the rowid itself */
fts5WriteAppendRowid(pIdx, &p->writer, iRowid);
/* And the poslist data */
fts5WriteAppendPoslistData(pIdx, &p->writer, aPoslist, nPoslist);
return pIdx->rc;
}
/*
** Flush the contents of in-memory hash table iHash to a new level-0
|
Changes to ext/fts5/tool/loadfts5.tcl.
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
..
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
|
set fd [open $f]
set data [read $fd]
close $fd
return $data
}
set ::nRow 0
proc load_hierachy {dir} {
foreach f [glob -nocomplain -dir $dir *] {
if {$::O(limit) && $::nRow>=$::O(limit)} break
if {[file isdir $f]} {
load_hierachy $f
} else {
db eval { INSERT INTO t1 VALUES($f, loadfile($f)) }
incr ::nRow
}
}
}
proc usage {} {
puts stderr "Usage: $::argv0 ?SWITCHES? DATABASE PATH"
puts stderr ""
................................................................................
}
}
sqlite3 db [lindex $argv end-1]
db func loadfile loadfile
db transaction {
db eval "CREATE VIRTUAL TABLE t1 USING $O(vtab) (path, content$O(tok))"
if {$O(automerge)>=0} {
if {$O(vtab) == "fts5"} {
db eval { INSERT INTO t1(t1, rank) VALUES('automerge', $O(automerge)) }
} else {
db eval { INSERT INTO t1(t1) VALUES('automerge=' || $O(automerge)) }
}
}
|
>
>
>
>
>
>
>
>
>
>
|
>
|
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
..
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
|
set fd [open $f] set data [read $fd] close $fd return $data } set ::nRow 0 set ::nRowPerDot 1000 proc load_hierachy {dir} { foreach f [glob -nocomplain -dir $dir *] { if {$::O(limit) && $::nRow>=$::O(limit)} break if {[file isdir $f]} { load_hierachy $f } else { db eval { INSERT INTO t1 VALUES($f, loadfile($f)) } incr ::nRow if {($::nRow % $::nRowPerDot)==0} { puts -nonewline . if {($::nRow % (65*$::nRowPerDot))==0} { puts "" } flush stdout } } } } proc usage {} { puts stderr "Usage: $::argv0 ?SWITCHES? DATABASE PATH" puts stderr "" ................................................................................ } } sqlite3 db [lindex $argv end-1] db func loadfile loadfile db transaction { catch { db eval "CREATE VIRTUAL TABLE t1 USING $O(vtab) (path, content$O(tok))" } if {$O(automerge)>=0} { if {$O(vtab) == "fts5"} { db eval { INSERT INTO t1(t1, rank) VALUES('automerge', $O(automerge)) } } else { db eval { INSERT INTO t1(t1) VALUES('automerge=' || $O(automerge)) } } } |