/ Check-in [74183a5f]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Experimental changes to the command line shell to make it work better with Unicode on Windows.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | altShellFix
Files: files | file ages | folders
SHA1:74183a5f803999e1d4e288851c72519c3c53f6b5
User & Date: mistachkin 2016-04-04 02:16:44
Context
2016-04-04
02:33
Refinements to the previous check-in. check-in: b36dfdff user: mistachkin tags: altShellFix
02:16
Experimental changes to the command line shell to make it work better with Unicode on Windows. check-in: 74183a5f user: mistachkin tags: altShellFix
02:05
More refactoring and cleanup work on the Win32 string conversion and utility routines. check-in: 02ccb444 user: mistachkin tags: altShellFix
Changes
Hide Diffs Unified Diffs Show Whitespace Changes Patch

Changes to src/shell.c.

131
132
133
134
135
136
137









138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
...
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
...
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
...
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
...
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
...
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
....
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
....
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
....
4978
4979
4980
4981
4982
4983
4984









4985




4986
4987
4988
4989
4990
4991
4992
4993
4994
4995



4996
4997
4998
4999
5000
5001
5002
5003
5004














5005
5006
5007
5008
5009
5010
5011
5012
5013
....
5333
5334
5335
5336
5337
5338
5339




5340
5341
#define isatty(x) 1
#endif

/* ctype macros that work with signed characters */
#define IsSpace(X)  isspace((unsigned char)X)
#define IsDigit(X)  isdigit((unsigned char)X)
#define ToLower(X)  (char)tolower((unsigned char)X)










/* On Windows, we normally run with output mode of TEXT so that \n characters
** are automatically translated into \r\n.  However, this behavior needs
** to be disabled in some cases (ex: when generating CSV output and when
** rendering quoted strings that contain \n characters).  The following
** routines take care of that.
*/
#if defined(_WIN32) || defined(WIN32)
static void setBinaryMode(FILE *out){
  fflush(out);
  _setmode(_fileno(out), _O_BINARY);
}
static void setTextMode(FILE *out){
  fflush(out);
  _setmode(_fileno(out), _O_TEXT);
}
#else
# define setBinaryMode(X)
# define setTextMode(X)
#endif


/* True if the timer is enabled */
static int enableTimer = 0;

/* Return the current wall-clock time */
................................................................................

#define BEGIN_TIMER beginTimer()
#define END_TIMER endTimer()
#define HAS_TIMER 1

#elif (defined(_WIN32) || defined(WIN32))

#include <windows.h>

/* Saved resource information for the beginning of an operation */
static HANDLE hProcess;
static FILETIME ftKernelBegin;
static FILETIME ftUserBegin;
static sqlite3_int64 ftWallBegin;
typedef BOOL (WINAPI *GETPROCTIMES)(HANDLE, LPFILETIME, LPFILETIME,
                                    LPFILETIME, LPFILETIME);
................................................................................
** output from UTF-8 into MBCS.
*/
#if defined(_WIN32) || defined(WIN32)
void utf8_printf(FILE *out, const char *zFormat, ...){
  va_list ap;
  va_start(ap, zFormat);
  if( stdout_is_console && (out==stdout || out==stderr) ){
    extern char *sqlite3_win32_utf8_to_mbcs(const char*);
    char *z1 = sqlite3_vmprintf(zFormat, ap);
    char *z2 = sqlite3_win32_utf8_to_mbcs(z1);
    sqlite3_free(z1);
    fputs(z2, out);
    sqlite3_free(z2);
  }else{
    vfprintf(out, zFormat, ap);
  }
  va_end(ap);
................................................................................
      break;
    }
  }
#if defined(_WIN32) || defined(WIN32)
  /* For interactive input on Windows systems, translate the 
  ** multi-byte characterset characters into UTF-8. */
  if( stdin_is_interactive ){
    extern char *sqlite3_win32_mbcs_to_utf8(const char*);
    char *zTrans = sqlite3_win32_mbcs_to_utf8(zLine);
    if( zTrans ){
      int nTrans = strlen30(zTrans)+1;
      if( nTrans>nLine ){
        zLine = realloc(zLine, nTrans);
        if( zLine==0 ){
          sqlite3_free(zTrans);
          return 0;
................................................................................

/*
** Output the given string as a quoted string using SQL quoting conventions.
*/
static void output_quoted_string(FILE *out, const char *z){
  int i;
  int nSingle = 0;
  setBinaryMode(out);
  for(i=0; z[i]; i++){
    if( z[i]=='\'' ) nSingle++;
  }
  if( nSingle==0 ){
    utf8_printf(out,"'%s'",z);
  }else{
    raw_printf(out,"'");
................................................................................
      }else{
        utf8_printf(out,"%s",z);
        break;
      }
    }
    raw_printf(out,"'");
  }
  setTextMode(out);
}

/*
** Output the given string as a quoted according to C or TCL quoting rules.
*/
static void output_c_string(FILE *out, const char *z){
  unsigned int c;
................................................................................
        output_c_string(p->out, azArg[i] ? azArg[i] : p->nullValue);
        if(i<nArg-1) utf8_printf(p->out, "%s", p->colSeparator);
      }
      utf8_printf(p->out, "%s", p->rowSeparator);
      break;
    }
    case MODE_Csv: {
      setBinaryMode(p->out);
      if( p->cnt++==0 && p->showHeader ){
        for(i=0; i<nArg; i++){
          output_csv(p, azCol[i] ? azCol[i] : "", i<nArg-1);
        }
        utf8_printf(p->out, "%s", p->rowSeparator);
      }
      if( nArg>0 ){
        for(i=0; i<nArg; i++){
          output_csv(p, azArg[i], i<nArg-1);
        }
        utf8_printf(p->out, "%s", p->rowSeparator);
      }
      setTextMode(p->out);
      break;
    }
    case MODE_Insert: {
      p->cnt++;
      if( azArg==0 ) break;
      utf8_printf(p->out,"INSERT INTO %s",p->zDestTable);
      if( p->showHeader ){
................................................................................
      rc = 1;
    }
  }else

  if( c=='b' && n>=3 && strncmp(azArg[0], "binary", n)==0 ){
    if( nArg==2 ){
      if( booleanValue(azArg[1]) ){
        setBinaryMode(p->out);
      }else{
        setTextMode(p->out);
      }
    }else{
      raw_printf(stderr, "Usage: .binary on|off\n");
      rc = 1;
    }
  }else

................................................................................
    utf8_printf(stderr, "%s: Error: missing argument to %s\n",
            argv[0], argv[argc-1]);
    exit(1);
  }
  return argv[i];
}










int SQLITE_CDECL main(int argc, char **argv){




  char *zErrMsg = 0;
  ShellState data;
  const char *zInitFile = 0;
  int i;
  int rc = 0;
  int warnInmemoryDb = 0;
  int readStdin = 1;
  int nCmd = 0;
  char **azCmd = 0;




#if USE_SYSTEM_SQLITE+0!=1
  if( strcmp(sqlite3_sourceid(),SQLITE_SOURCE_ID)!=0 ){
    utf8_printf(stderr, "SQLite header and source version mismatch\n%s\n%s\n",
            sqlite3_sourceid(), SQLITE_SOURCE_ID);
    exit(1);
  }
#endif
  setBinaryMode(stdin);
  setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */














  Argv0 = argv[0];
  main_init(&data);
  stdin_is_interactive = isatty(0);
  stdout_is_console = isatty(1);

  /* Make sure we have a valid signal handler early, before anything
  ** else is done.
  */
#ifdef SIGINT
................................................................................
  }
  set_table_name(&data, 0);
  if( data.db ){
    session_close_all(&data);
    sqlite3_close(data.db);
  }
  sqlite3_free(data.zFreeOnClose); 




  return rc;
}







>
>
>
>
>
>
>
>
>








|
|
|

|
|
|


|
|







 







<
<







 







<

|







 







<
|







 







|







 







|







 







|












|







 







|

|







 







>
>
>
>
>
>
>
>
>

>
>
>
>










>
>
>







|
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>

<







 







>
>
>
>


131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
...
234
235
236
237
238
239
240


241
242
243
244
245
246
247
...
374
375
376
377
378
379
380

381
382
383
384
385
386
387
388
389
...
521
522
523
524
525
526
527

528
529
530
531
532
533
534
535
...
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
...
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
....
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
....
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
....
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040

5041
5042
5043
5044
5045
5046
5047
....
5367
5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
#define isatty(x) 1
#endif

/* ctype macros that work with signed characters */
#define IsSpace(X)  isspace((unsigned char)X)
#define IsDigit(X)  isdigit((unsigned char)X)
#define ToLower(X)  (char)tolower((unsigned char)X)

#if defined(_WIN32) || defined(WIN32)
#include <windows.h>

/* string conversion routines only needed on Win32 */
extern char *sqlite3_win32_unicode_to_utf8(LPCWSTR);
extern char *sqlite3_win32_mbcs_to_utf8_v2(const char *, int);
extern char *sqlite3_win32_utf8_to_mbcs_v2(const char *, int);
#endif

/* On Windows, we normally run with output mode of TEXT so that \n characters
** are automatically translated into \r\n.  However, this behavior needs
** to be disabled in some cases (ex: when generating CSV output and when
** rendering quoted strings that contain \n characters).  The following
** routines take care of that.
*/
#if defined(_WIN32) || defined(WIN32)
static void setBinaryMode(FILE *file, int isOutput){
  if( isOutput ) fflush(file);
  _setmode(_fileno(file), _O_BINARY);
}
static void setTextMode(FILE *file, int isOutput){
  if( isOutput ) fflush(file);
  _setmode(_fileno(file), _O_TEXT);
}
#else
# define setBinaryMode(X,Y)
# define setTextMode(X,Y)
#endif


/* True if the timer is enabled */
static int enableTimer = 0;

/* Return the current wall-clock time */
................................................................................

#define BEGIN_TIMER beginTimer()
#define END_TIMER endTimer()
#define HAS_TIMER 1

#elif (defined(_WIN32) || defined(WIN32))



/* Saved resource information for the beginning of an operation */
static HANDLE hProcess;
static FILETIME ftKernelBegin;
static FILETIME ftUserBegin;
static sqlite3_int64 ftWallBegin;
typedef BOOL (WINAPI *GETPROCTIMES)(HANDLE, LPFILETIME, LPFILETIME,
                                    LPFILETIME, LPFILETIME);
................................................................................
** output from UTF-8 into MBCS.
*/
#if defined(_WIN32) || defined(WIN32)
void utf8_printf(FILE *out, const char *zFormat, ...){
  va_list ap;
  va_start(ap, zFormat);
  if( stdout_is_console && (out==stdout || out==stderr) ){

    char *z1 = sqlite3_vmprintf(zFormat, ap);
    char *z2 = sqlite3_win32_utf8_to_mbcs_v2(z1, 0);
    sqlite3_free(z1);
    fputs(z2, out);
    sqlite3_free(z2);
  }else{
    vfprintf(out, zFormat, ap);
  }
  va_end(ap);
................................................................................
      break;
    }
  }
#if defined(_WIN32) || defined(WIN32)
  /* For interactive input on Windows systems, translate the
  ** multi-byte characterset characters into UTF-8. */
  if( stdin_is_interactive ){

    char *zTrans = sqlite3_win32_mbcs_to_utf8_v2(zLine, 0);
    if( zTrans ){
      int nTrans = strlen30(zTrans)+1;
      if( nTrans>nLine ){
        zLine = realloc(zLine, nTrans);
        if( zLine==0 ){
          sqlite3_free(zTrans);
          return 0;
................................................................................

/*
** Output the given string as a quoted string using SQL quoting conventions.
*/
static void output_quoted_string(FILE *out, const char *z){
  int i;
  int nSingle = 0;
  setBinaryMode(out, 1);
  for(i=0; z[i]; i++){
    if( z[i]=='\'' ) nSingle++;
  }
  if( nSingle==0 ){
    utf8_printf(out,"'%s'",z);
  }else{
    raw_printf(out,"'");
................................................................................
      }else{
        utf8_printf(out,"%s",z);
        break;
      }
    }
    raw_printf(out,"'");
  }
  setTextMode(out, 1);
}

/*
** Output the given string as a quoted according to C or TCL quoting rules.
*/
static void output_c_string(FILE *out, const char *z){
  unsigned int c;
................................................................................
        output_c_string(p->out, azArg[i] ? azArg[i] : p->nullValue);
        if(i<nArg-1) utf8_printf(p->out, "%s", p->colSeparator);
      }
      utf8_printf(p->out, "%s", p->rowSeparator);
      break;
    }
    case MODE_Csv: {
      setBinaryMode(p->out, 1);
      if( p->cnt++==0 && p->showHeader ){
        for(i=0; i<nArg; i++){
          output_csv(p, azCol[i] ? azCol[i] : "", i<nArg-1);
        }
        utf8_printf(p->out, "%s", p->rowSeparator);
      }
      if( nArg>0 ){
        for(i=0; i<nArg; i++){
          output_csv(p, azArg[i], i<nArg-1);
        }
        utf8_printf(p->out, "%s", p->rowSeparator);
      }
      setTextMode(p->out, 1);
      break;
    }
    case MODE_Insert: {
      p->cnt++;
      if( azArg==0 ) break;
      utf8_printf(p->out,"INSERT INTO %s",p->zDestTable);
      if( p->showHeader ){
................................................................................
      rc = 1;
    }
  }else

  if( c=='b' && n>=3 && strncmp(azArg[0], "binary", n)==0 ){
    if( nArg==2 ){
      if( booleanValue(azArg[1]) ){
        setBinaryMode(p->out, 1);
      }else{
        setTextMode(p->out, 1);
      }
    }else{
      raw_printf(stderr, "Usage: .binary on|off\n");
      rc = 1;
    }
  }else

................................................................................
    utf8_printf(stderr, "%s: Error: missing argument to %s\n",
            argv[0], argv[argc-1]);
    exit(1);
  }
  return argv[i];
}

#ifndef SQLITE_SHELL_IS_UTF8
#  if (defined(_WIN32) || defined(WIN32)) && defined(_MSC_VER)
#    define SQLITE_SHELL_IS_UTF8          (0)
#  else
#    define SQLITE_SHELL_IS_UTF8          (1)
#  endif
#endif

#if SQLITE_SHELL_IS_UTF8
int SQLITE_CDECL main(int argc, char **argv){
#else
int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
  char **argv = 0;
#endif
  char *zErrMsg = 0;
  ShellState data;
  const char *zInitFile = 0;
  int i;
  int rc = 0;
  int warnInmemoryDb = 0;
  int readStdin = 1;
  int nCmd = 0;
  char **azCmd = 0;

  setBinaryMode(stdin, 0);
  setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */

#if USE_SYSTEM_SQLITE+0!=1
  if( strcmp(sqlite3_sourceid(),SQLITE_SOURCE_ID)!=0 ){
    utf8_printf(stderr, "SQLite header and source version mismatch\n%s\n%s\n",
            sqlite3_sourceid(), SQLITE_SOURCE_ID);
    exit(1);
  }
#endif
  main_init(&data);
#if !SQLITE_SHELL_IS_UTF8
  sqlite3_initialize();
  argv = sqlite3_malloc64(sizeof(argv[0])*argc);
  if( argv==0 ){
    raw_printf(stderr, "out of memory\n");
    exit(1);
  }
  for(i=0; i<argc; i++){
    argv[i] = sqlite3_win32_unicode_to_utf8(wargv[i]);
    if( argv[i]==0 ){
      raw_printf(stderr, "out of memory\n");
      exit(1);
    }
  }
#endif
  Argv0 = argv[0];

  stdin_is_interactive = isatty(0);
  stdout_is_console = isatty(1);

  /* Make sure we have a valid signal handler early, before anything
  ** else is done.
  */
#ifdef SIGINT
................................................................................
  }
  set_table_name(&data, 0);
  if( data.db ){
    session_close_all(&data);
    sqlite3_close(data.db);
  }
  sqlite3_free(data.zFreeOnClose);
#if !SQLITE_SHELL_IS_UTF8
  for(i=0; i<argc; i++) sqlite3_free(argv[i]);
  sqlite3_free(argv);
#endif
  return rc;
}

Changes to test/shell1.test.

905
906
907
908
909
910
911
912



















913
      error "failed with error: $res"
    }
    if {$res ne "$oldChar\n"} {
      error "failed with byte $hex mismatch"
    }
  }
} {}




















finish_test








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
      error "failed with error: $res"
    }
    if {$res ne "$oldChar\n"} {
      error "failed with byte $hex mismatch"
    }
  }
} {}

do_test shell1-6.0 {
  # The file name here is the word "test" in Chinese.
  # In UTF-8, it is encoded as: \xE6\xB5\x8B\xE8\xAF\x95
  set fileName \u6D4B\u8BD5; append fileName .db
  set x [catchcmdex $fileName "CREATE TABLE t1(x);\n.schema\n"]
  set code [lindex $x 0]
  set res [string trim [lindex $x 1]]
  if {$code ne "0"} {
    error "failed with error: $res"
  }
  if {$res ne "CREATE TABLE t1(x);"} {
    error "failed with mismatch: $res"
  }
  if {![file exists $fileName]} {
    error "file \"$fileName\" (MBCS) does not exist"
  }
  forcedelete $fileName
} {}

finish_test