/ Check-in [4fbca3ab]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:The tokenizer should never return a negative size of the next token. Ticket #453. (CVS 1098)
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 4fbca3ab09596c530da7c50657f3bc9140178dd5
User & Date: drh 2003-09-12 02:08:15
Context
2003-09-18
21:59
modify the configure file to allow cross-compile to work at arm and xscale (CVS 1099) check-in: 760f2fa2 user: xdong tags: trunk
2003-09-12
02:08
The tokenizer should never return a negative size of the next token. Ticket #453. (CVS 1098) check-in: 4fbca3ab user: drh tags: trunk
2003-09-09
00:47
Add test cases that check out sqlite_get_table() on queries with results sets containing 100 columns. (CVS 1097) check-in: f9feedec user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/tokenize.c.

11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
...
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
...
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
*************************************************************************
** An tokenizer for SQL
**
** This file contains C code that splits an SQL input string up into
** individual tokens and sends those tokens one-by-one over to the
** parser for analysis.
**
** $Id: tokenize.c,v 1.62 2003/09/06 22:18:08 drh Exp $
*/
#include "sqliteInt.h"
#include "os.h"
#include <ctype.h>
#include <stdlib.h>

/*
................................................................................
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Dx */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Ex */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Fx */
};


/*
** Return the length of the token that begins at z[0].  Return
** -1 if the token is (or might be) incomplete.  Store the token
** type in *tokenType before returning.
*/
static int sqliteGetToken(const unsigned char *z, int *tokenType){
  int i;
  switch( *z ){
    case ' ': case '\t': case '\n': case '\f': case '\r': {
      for(i=1; isspace(z[i]); i++){}
      *tokenType = TK_SPACE;
      return i;
    }
    case '-': {
      if( z[1]==0 ) return -1;
      if( z[1]=='-' ){
        for(i=2; z[i] && z[i]!='\n'; i++){}
        *tokenType = TK_COMMENT;
        return i;
      }
      *tokenType = TK_MINUS;
      return 1;
................................................................................
  if( pEngine==0 ){
    sqliteSetString(pzErrMsg, "out of memory", 0);
    return 1;
  }
  pParse->sLastToken.dyn = 0;
  pParse->zTail = zSql;
  while( sqlite_malloc_failed==0 && zSql[i]!=0 ){
    
    assert( i>=0 );
    pParse->sLastToken.z = &zSql[i];
    assert( pParse->sLastToken.dyn==0 );
    pParse->sLastToken.n = sqliteGetToken((unsigned char*)&zSql[i], &tokenType);
    i += pParse->sLastToken.n;
    switch( tokenType ){
      case TK_SPACE:







|







 







|
<
|










<







 







<







11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
...
210
211
212
213
214
215
216
217

218
219
220
221
222
223
224
225
226
227
228

229
230
231
232
233
234
235
...
420
421
422
423
424
425
426

427
428
429
430
431
432
433
*************************************************************************
** An tokenizer for SQL
**
** This file contains C code that splits an SQL input string up into
** individual tokens and sends those tokens one-by-one over to the
** parser for analysis.
**
** $Id: tokenize.c,v 1.63 2003/09/12 02:08:15 drh Exp $
*/
#include "sqliteInt.h"
#include "os.h"
#include <ctype.h>
#include <stdlib.h>

/*
................................................................................
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Dx */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Ex */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Fx */
};


/*
** Return the length of the token that begins at z[0]. 

** Store the token type in *tokenType before returning.
*/
static int sqliteGetToken(const unsigned char *z, int *tokenType){
  int i;
  switch( *z ){
    case ' ': case '\t': case '\n': case '\f': case '\r': {
      for(i=1; isspace(z[i]); i++){}
      *tokenType = TK_SPACE;
      return i;
    }
    case '-': {

      if( z[1]=='-' ){
        for(i=2; z[i] && z[i]!='\n'; i++){}
        *tokenType = TK_COMMENT;
        return i;
      }
      *tokenType = TK_MINUS;
      return 1;
................................................................................
  if( pEngine==0 ){
    sqliteSetString(pzErrMsg, "out of memory", 0);
    return 1;
  }
  pParse->sLastToken.dyn = 0;
  pParse->zTail = zSql;
  while( sqlite_malloc_failed==0 && zSql[i]!=0 ){

    assert( i>=0 );
    pParse->sLastToken.z = &zSql[i];
    assert( pParse->sLastToken.dyn==0 );
    pParse->sLastToken.n = sqliteGetToken((unsigned char*)&zSql[i], &tokenType);
    i += pParse->sLastToken.n;
    switch( tokenType ){
      case TK_SPACE:

Changes to test/misc2.test.

9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
...
175
176
177
178
179
180
181








#
#***********************************************************************
# This file implements regression tests for SQLite library.
#
# This file implements tests for miscellanous features that were
# left out of other test files.
#
# $Id: misc2.test,v 1.8 2003/08/27 22:54:32 drh Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# Test for ticket #360
#
do_test misc2-1.1 {
................................................................................
  set rc [catch {
    db eval {SELECT rowid FROM t1} {} {
      db eval "INSERT INTO t1 VALUES(3)"
    }
  } msg]
  lappend rc $msg
} {1 {database table is locked}}















|







 







>
>
>
>
>
>
>
>
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
...
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
#
#***********************************************************************
# This file implements regression tests for SQLite library.
#
# This file implements tests for miscellanous features that were
# left out of other test files.
#
# $Id: misc2.test,v 1.9 2003/09/12 02:08:16 drh Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# Test for ticket #360
#
do_test misc2-1.1 {
................................................................................
  set rc [catch {
    db eval {SELECT rowid FROM t1} {} {
      db eval "INSERT INTO t1 VALUES(3)"
    }
  } msg]
  lappend rc $msg
} {1 {database table is locked}}

# Ticket #453.  If the SQL ended with "-", the tokenizer was calling that
# an incomplete token, which caused problem.  The solution was to just call
# it a minus sign.
#
do_test misc2-8.1 {
  catchsql {-}
} {1 {near "-": syntax error}}