SQLite: Check-in [1253a872]

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview

Comment:	Add support for measuring and reporting coverage of the parser state machine using the SQLITE_TESTCTRL_PARSER_COVERAGE test-control.
Downloads:	Tarball \| ZIP archive \| SQL archive
Timelines:	family \| ancestors \| descendants \| both \| lemon-improvements
Files:	files \| file ages \| folders
SHA3-256:	1253a872dbf48656d4efd588ab61223a5ac550d9b2b932249d6ba585276ba573
User & Date:	drh 2017-12-26 18:04:23

Context

2017-12-26
18:14		Merge bug fixes from trunk. (check-in: d76e1206 user: drh tags: lemon-improvements)
18:04		Add support for measuring and reporting coverage of the parser state machine using the SQLITE_TESTCTRL_PARSER_COVERAGE test-control. (check-in: 1253a872 user: drh tags: lemon-improvements)
2017-12-25
04:15		Enhance LEMON so that it generates the action table in such a way that no range check is needed on the lookahead table to verify that the next input token is valid. This makes the lookahead table slightly larger (about 120 bytes) but helps the parser to run faster. (check-in: 7eb0198d user: drh tags: lemon-improvements)

Changes

Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/main.c.

      db->init.newTnum = va_arg(ap,int);
      if( db->init.busy==0 && db->init.newTnum>0 ){
        sqlite3ResetAllSchemasOfConnection(db);
      }
      sqlite3_mutex_leave(db->mutex);
      break;
    }
















  }
  va_end(ap);
#endif /* SQLITE_UNTESTABLE */
  return rc;
}

/*








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

      db->init.newTnum = va_arg(ap,int);
      if( db->init.busy==0 && db->init.newTnum>0 ){
        sqlite3ResetAllSchemasOfConnection(db);
      }
      sqlite3_mutex_leave(db->mutex);
      break;
    }

#if defined(YYCOVERAGE)
    /*  sqlite3_test_control(SQLITE_TESTCTRL_PARSER_COVERAGE, FILE *out)
    **
    ** This test control (only available when SQLite is compiled with
    ** -DYYCOVERAGE) writes a report onto "out" that shows all
    ** state/lookahead combinations in the parser state machine
    ** which are never exercised.  If any state is missed, make the
    ** return code SQLITE_ERROR.
    */
    case SQLITE_TESTCTRL_PARSER_COVERAGE: {
      FILE *out = va_arg(ap, FILE*);
      if( sqlite3ParserCoverage(out) ) rc = SQLITE_ERROR;
      break;
    }
#endif /* defined(YYCOVERAGE) */
  }
  va_end(ap);
#endif /* SQLITE_UNTESTABLE */
  return rc;
}

/*

Changes to src/shell.c.in.

      { "imposter",           SQLITE_TESTCTRL_IMPOSTER,   "SCHEMA ON/OFF ROOTPAGE"},
#ifdef SQLITE_N_KEYWORD
      { "iskeyword",          SQLITE_TESTCTRL_ISKEYWORD,     "IDENTIFIER"         },
#endif
      { "localtime_fault",    SQLITE_TESTCTRL_LOCALTIME_FAULT,"BOOLEAN"           },
      { "never_corrupt",      SQLITE_TESTCTRL_NEVER_CORRUPT, "BOOLEAN"            },
      { "optimizations",      SQLITE_TESTCTRL_OPTIMIZATIONS, "DISABLE-MASK"       },



      { "pending_byte",       SQLITE_TESTCTRL_PENDING_BYTE,  "OFFSET  "           },
      { "prng_reset",         SQLITE_TESTCTRL_PRNG_RESET,    ""                   },
      { "prng_restore",       SQLITE_TESTCTRL_PRNG_RESTORE,  ""                   },
      { "prng_save",          SQLITE_TESTCTRL_PRNG_SAVE,     ""                   },
      { "reserve",            SQLITE_TESTCTRL_RESERVE,       "BYTES-OF-RESERVE"   },
    };
    int testctrl = -1;
................................................................................
            rc2 = sqlite3_test_control(testctrl, p->db,
                          azArg[2],
                          integerValue(azArg[3]),
                          integerValue(azArg[4]));
            isOk = 3;
          }
          break;








      }
    }
    if( isOk==0 && iCtrl>=0 ){
      utf8_printf(p->out, "Usage: .testctrl %s %s\n", zCmd, aCtrl[iCtrl].zUsage);
      rc = 1;
    }else if( isOk==1 ){
      raw_printf(p->out, "%d\n", rc2);








>
>
>







 







>
>
>
>
>
>
>
>

      { "imposter",           SQLITE_TESTCTRL_IMPOSTER,   "SCHEMA ON/OFF ROOTPAGE"},
#ifdef SQLITE_N_KEYWORD
      { "iskeyword",          SQLITE_TESTCTRL_ISKEYWORD,     "IDENTIFIER"         },
#endif
      { "localtime_fault",    SQLITE_TESTCTRL_LOCALTIME_FAULT,"BOOLEAN"           },
      { "never_corrupt",      SQLITE_TESTCTRL_NEVER_CORRUPT, "BOOLEAN"            },
      { "optimizations",      SQLITE_TESTCTRL_OPTIMIZATIONS, "DISABLE-MASK"       },
#ifdef YYCOVERAGE
      { "parser_coverage",    SQLITE_TESTCTRL_PARSER_COVERAGE, ""                 },
#endif
      { "pending_byte",       SQLITE_TESTCTRL_PENDING_BYTE,  "OFFSET  "           },
      { "prng_reset",         SQLITE_TESTCTRL_PRNG_RESET,    ""                   },
      { "prng_restore",       SQLITE_TESTCTRL_PRNG_RESTORE,  ""                   },
      { "prng_save",          SQLITE_TESTCTRL_PRNG_SAVE,     ""                   },
      { "reserve",            SQLITE_TESTCTRL_RESERVE,       "BYTES-OF-RESERVE"   },
    };
    int testctrl = -1;
................................................................................
            rc2 = sqlite3_test_control(testctrl, p->db,
                          azArg[2],
                          integerValue(azArg[3]),
                          integerValue(azArg[4]));
            isOk = 3;
          }
          break;

#ifdef YYCOVERAGE
        case SQLITE_TESTCTRL_PARSER_COVERAGE:
          if( nArg==2 ){
            sqlite3_test_control(testctrl, p->out);
            isOk = 3;
          }
#endif
      }
    }
    if( isOk==0 && iCtrl>=0 ){
      utf8_printf(p->out, "Usage: .testctrl %s %s\n", zCmd, aCtrl[iCtrl].zUsage);
      rc = 1;
    }else if( isOk==1 ){
      raw_printf(p->out, "%d\n", rc2);

Changes to src/sqlite.h.in.

#define SQLITE_TESTCTRL_ONCE_RESET_THRESHOLD    19
#define SQLITE_TESTCTRL_NEVER_CORRUPT           20
#define SQLITE_TESTCTRL_VDBE_COVERAGE           21
#define SQLITE_TESTCTRL_BYTEORDER               22
#define SQLITE_TESTCTRL_ISINIT                  23
#define SQLITE_TESTCTRL_SORTER_MMAP             24
#define SQLITE_TESTCTRL_IMPOSTER                25

#define SQLITE_TESTCTRL_LAST                    25  /* Largest TESTCTRL */

/*
** CAPI3REF: SQLite Runtime Status
**
** ^These interfaces are used to retrieve runtime status information
** about the performance of SQLite, and optionally to reset various
** highwater marks.  ^The first argument is an integer code for

>
|

#define SQLITE_TESTCTRL_ONCE_RESET_THRESHOLD    19
#define SQLITE_TESTCTRL_NEVER_CORRUPT           20
#define SQLITE_TESTCTRL_VDBE_COVERAGE           21
#define SQLITE_TESTCTRL_BYTEORDER               22
#define SQLITE_TESTCTRL_ISINIT                  23
#define SQLITE_TESTCTRL_SORTER_MMAP             24
#define SQLITE_TESTCTRL_IMPOSTER                25
#define SQLITE_TESTCTRL_PARSER_COVERAGE         26
#define SQLITE_TESTCTRL_LAST                    26  /* Largest TESTCTRL */

/*
** CAPI3REF: SQLite Runtime Status
**
** ^These interfaces are used to retrieve runtime status information
** about the performance of SQLite, and optionally to reset various
** highwater marks.  ^The first argument is an integer code for

Changes to src/sqliteInt.h.

  #define sqlite3ConnectionUnlocked(x)
  #define sqlite3ConnectionClosed(x)
#endif

#ifdef SQLITE_DEBUG
  void sqlite3ParserTrace(FILE*, char *);
#endif




/*
** If the SQLITE_ENABLE IOTRACE exists then the global variable
** sqlite3IoTrace is a pointer to a printf-like routine used to
** print I/O tracing messages.
*/
#ifdef SQLITE_ENABLE_IOTRACE








>
>
>

  #define sqlite3ConnectionUnlocked(x)
  #define sqlite3ConnectionClosed(x)
#endif

#ifdef SQLITE_DEBUG
  void sqlite3ParserTrace(FILE*, char *);
#endif
#if defined(YYCOVERAGE)
  int sqlite3ParserCoverage(FILE*);
#endif

/*
** If the SQLITE_ENABLE IOTRACE exists then the global variable
** sqlite3IoTrace is a pointer to a printf-like routine used to
** print I/O tracing messages.
*/
#ifdef SQLITE_ENABLE_IOTRACE

Changes to tool/lemon.c.

    }
  }

  /* Finish rendering the constants now that the action table has
  ** been computed */
  fprintf(out,"#define YYNSTATE             %d\n",lemp->nxstate);  lineno++;
  fprintf(out,"#define YYNRULE              %d\n",lemp->nrule);  lineno++;

  fprintf(out,"#define YY_MAX_SHIFT         %d\n",lemp->nxstate-1); lineno++;
  i = lemp->minShiftReduce;
  fprintf(out,"#define YY_MIN_SHIFTREDUCE   %d\n",i); lineno++;
  i += lemp->nrule;
  fprintf(out,"#define YY_MAX_SHIFTREDUCE   %d\n", i-1); lineno++;
  fprintf(out,"#define YY_ERROR_ACTION      %d\n", lemp->errAction); lineno++;
  fprintf(out,"#define YY_ACCEPT_ACTION     %d\n", lemp->accAction); lineno++;

    }
  }

  /* Finish rendering the constants now that the action table has
  ** been computed */
  fprintf(out,"#define YYNSTATE             %d\n",lemp->nxstate);  lineno++;
  fprintf(out,"#define YYNRULE              %d\n",lemp->nrule);  lineno++;
  fprintf(out,"#define YYNTOKEN             %d\n",lemp->nterminal); lineno++;
  fprintf(out,"#define YY_MAX_SHIFT         %d\n",lemp->nxstate-1); lineno++;
  i = lemp->minShiftReduce;
  fprintf(out,"#define YY_MIN_SHIFTREDUCE   %d\n",i); lineno++;
  i += lemp->nrule;
  fprintf(out,"#define YY_MAX_SHIFTREDUCE   %d\n", i-1); lineno++;
  fprintf(out,"#define YY_ERROR_ACTION      %d\n", lemp->errAction); lineno++;
  fprintf(out,"#define YY_ACCEPT_ACTION     %d\n", lemp->accAction); lineno++;

Changes to tool/lempar.c.

**    ParseARG_PDECL     A parameter declaration for the %extra_argument
**    ParseARG_STORE     Code to store %extra_argument into yypParser
**    ParseARG_FETCH     Code to extract %extra_argument from yypParser
**    YYERRORSYMBOL      is the code number of the error symbol.  If not
**                       defined, then do no error processing.
**    YYNSTATE           the combined number of states.
**    YYNRULE            the number of rules in the grammar

**    YY_MAX_SHIFT       Maximum value for shift actions
**    YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions
**    YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions
**    YY_ERROR_ACTION    The yy_action[] code for syntax error
**    YY_ACCEPT_ACTION   The yy_action[] code for accept
**    YY_NO_ACTION       The yy_action[] code for no-op
**    YY_MIN_REDUCE      Minimum value for reduce actions
................................................................................
  yyTraceFILE = TraceFILE;
  yyTracePrompt = zTracePrompt;
  if( yyTraceFILE==0 ) yyTracePrompt = 0;
  else if( yyTracePrompt==0 ) yyTraceFILE = 0;
}
#endif /* NDEBUG */

#ifndef NDEBUG
/* For tracing shifts, the names of all terminals and nonterminals
** are required.  The following table supplies these names */
static const char *const yyTokenName[] = { 
%%
};
#endif /* NDEBUG */

#ifndef NDEBUG
/* For tracing reduce actions, the names of all rules are required.
*/
static const char *const yyRuleName[] = {
%%
};
................................................................................
#ifdef YYTRACKMAXSTACKDEPTH
int ParseStackPeak(void *p){
  yyParser *pParser = (yyParser*)p;
  return pParser->yyhwm;
}
#endif






























/*
** Find the appropriate action for a parser given the terminal
** look-ahead token iLookAhead.
*/
static unsigned int yy_find_shift_action(
  yyParser *pParser,        /* The parser */
  YYCODETYPE iLookAhead     /* The look-ahead token */
){
  int i;
  int stateno = pParser->yytos->stateno;
 
  if( stateno>YY_MAX_SHIFT ) return stateno;
  assert( stateno <= YY_SHIFT_COUNT );



  do{
    i = yy_shift_ofst[stateno];
    assert( iLookAhead!=YYNOCODE );
    i += iLookAhead;
    assert( i>=0 && i<sizeof(yy_lookahead)/sizeof(yy_lookahead[0]) );
    if( yy_lookahead[i]!=iLookAhead ){
#ifdef YYFALLBACK








>







 







|





|







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>













>
>
>

**    ParseARG_PDECL     A parameter declaration for the %extra_argument
**    ParseARG_STORE     Code to store %extra_argument into yypParser
**    ParseARG_FETCH     Code to extract %extra_argument from yypParser
**    YYERRORSYMBOL      is the code number of the error symbol.  If not
**                       defined, then do no error processing.
**    YYNSTATE           the combined number of states.
**    YYNRULE            the number of rules in the grammar
**    YYNTOKEN           Number of terminal symbols
**    YY_MAX_SHIFT       Maximum value for shift actions
**    YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions
**    YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions
**    YY_ERROR_ACTION    The yy_action[] code for syntax error
**    YY_ACCEPT_ACTION   The yy_action[] code for accept
**    YY_NO_ACTION       The yy_action[] code for no-op
**    YY_MIN_REDUCE      Minimum value for reduce actions
................................................................................
  yyTraceFILE = TraceFILE;
  yyTracePrompt = zTracePrompt;
  if( yyTraceFILE==0 ) yyTracePrompt = 0;
  else if( yyTracePrompt==0 ) yyTraceFILE = 0;
}
#endif /* NDEBUG */

#if defined(YYCOVERAGE) || !defined(NDEBUG)
/* For tracing shifts, the names of all terminals and nonterminals
** are required.  The following table supplies these names */
static const char *const yyTokenName[] = { 
%%
};
#endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */

#ifndef NDEBUG
/* For tracing reduce actions, the names of all rules are required.
*/
static const char *const yyRuleName[] = {
%%
};
................................................................................
#ifdef YYTRACKMAXSTACKDEPTH
int ParseStackPeak(void *p){
  yyParser *pParser = (yyParser*)p;
  return pParser->yyhwm;
}
#endif

/* This array of booleans keeps track of the parser statement
** coverage.  The element yycoverage[X][Y] is set when the parser
** is in state X and has a lookahead token Y.  In a well-tested
** systems, every element of this matrix should end up being set.
*/
#if defined(YYCOVERAGE)
static unsigned char yycoverage[YYNSTATE][YYNTOKEN];
#endif

/*
** Write into out a description of every state/lookahead combination that
** has not previously been seen by the parser.  Return the number of 
** missed state/lookahead combinations.
*/
#if defined(YYCOVERAGE)
int ParseCoverage(FILE *out){
  int i, j;
  int nMissed = 0;
  for(i=0; i<YYNSTATE; i++){
    for(j=0; j<YYNTOKEN; j++){
      if( yycoverage[i][j] ) continue;
      nMissed++;
      if( out ) fprintf(out,"State %d lookahead %s\n", i, yyTokenName[j]);
    }
  }
  return nMissed;
}
#endif

/*
** Find the appropriate action for a parser given the terminal
** look-ahead token iLookAhead.
*/
static unsigned int yy_find_shift_action(
  yyParser *pParser,        /* The parser */
  YYCODETYPE iLookAhead     /* The look-ahead token */
){
  int i;
  int stateno = pParser->yytos->stateno;
 
  if( stateno>YY_MAX_SHIFT ) return stateno;
  assert( stateno <= YY_SHIFT_COUNT );
#if defined(YYCOVERAGE)
  yycoverage[stateno][iLookAhead] = 1;
#endif
  do{
    i = yy_shift_ofst[stateno];
    assert( iLookAhead!=YYNOCODE );
    i += iLookAhead;
    assert( i>=0 && i<sizeof(yy_lookahead)/sizeof(yy_lookahead[0]) );
    if( yy_lookahead[i]!=iLookAhead ){
#ifdef YYFALLBACK