SQLite

Check-in [414010d236]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add -utf8 option to CLI, for improved console behavior on Windows.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: 414010d2366477285cb238f7af1e3d511ab4cda4bd4493441bd9868387dd5b66
User & Date: larrybr 2023-04-15 17:56:43.253
Context
2023-04-15
19:13
Avoid dropping the error code following an OOM in fts5 when using 'secure-delete' mode. (check-in: 9ee24a499c user: dan tags: trunk)
17:56
Add -utf8 option to CLI, for improved console behavior on Windows. (check-in: 414010d236 user: larrybr tags: trunk)
17:47
Add the 'secure-delete' option to the fts5 extension. For configuring fts5 to delete old entries directly from the full-text index instead of using delete keys. (check-in: 394980e4fe user: dan tags: trunk)
16:12
For CLI -utf8, set output codepage too. Adjust PP vars so that the code can be entirely omitted for targets pretending to be "WIN32" but not quite doing so. (check-in: 543594a727 user: larrybr tags: cli-utf8)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/shell.c.in.
224
225
226
227
228
229
230

231
232
233
234
235
236
237
#define IsDigit(X)  isdigit((unsigned char)X)
#define ToLower(X)  (char)tolower((unsigned char)X)

#if defined(_WIN32) || defined(WIN32)
#if SQLITE_OS_WINRT
#include <intrin.h>
#endif

#include <windows.h>

/* string conversion routines only needed on Win32 */
extern char *sqlite3_win32_unicode_to_utf8(LPCWSTR);
extern char *sqlite3_win32_mbcs_to_utf8_v2(const char *, int);
extern char *sqlite3_win32_utf8_to_mbcs_v2(const char *, int);
extern LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText);







>







224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
#define IsDigit(X)  isdigit((unsigned char)X)
#define ToLower(X)  (char)tolower((unsigned char)X)

#if defined(_WIN32) || defined(WIN32)
#if SQLITE_OS_WINRT
#include <intrin.h>
#endif
#define WIN32_LEAN_AND_MEAN
#include <windows.h>

/* string conversion routines only needed on Win32 */
extern char *sqlite3_win32_unicode_to_utf8(LPCWSTR);
extern char *sqlite3_win32_mbcs_to_utf8_v2(const char *, int);
extern char *sqlite3_win32_utf8_to_mbcs_v2(const char *, int);
extern LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText);
438
439
440
441
442
443
444
445
446
447
448














449
450
451
452
453
454
455
/*
** If the following flag is set, then command execution stops
** at an error if we are not interactive.
*/
static int bail_on_error = 0;

/*
** Threat stdin as an interactive input if the following variable
** is true.  Otherwise, assume stdin is connected to a file or pipe.
*/
static int stdin_is_interactive = 1;















/*
** On Windows systems we have to know if standard output is a console
** in order to translate UTF-8 into MBCS.  The following variable is
** true if translation is required.
*/
static int stdout_is_console = 1;







|



>
>
>
>
>
>
>
>
>
>
>
>
>
>







439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
/*
** If the following flag is set, then command execution stops
** at an error if we are not interactive.
*/
static int bail_on_error = 0;

/*
** Treat stdin as an interactive input if the following variable
** is true.  Otherwise, assume stdin is connected to a file or pipe.
*/
static int stdin_is_interactive = 1;

#if (defined(_WIN32) || defined(WIN32)) && SHELL_USE_LOCAL_GETLINE \
  && !defined(SHELL_OMIT_WIN_UTF8)
# define SHELL_WIN_UTF8_OPT 1
#else
# define SHELL_WIN_UTF8_OPT 0
#endif

#if SHELL_WIN_UTF8_OPT
/*
** Setup console for UTF-8 input/output when following variable true.
*/
static int console_utf8 = 0;
#endif

/*
** On Windows systems we have to know if standard output is a console
** in order to translate UTF-8 into MBCS.  The following variable is
** true if translation is required.
*/
static int stdout_is_console = 1;
575
576
577
578
579
580
581






















































































































582
583
584
585
586
587
588
      shell_strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3, PROMPT_LEN_MAX-4);
    }
  }
  return dynPrompt.dynamicPrompt;
}
#endif /* !defined(SQLITE_OMIT_DYNAPROMPT) */























































































































/*
** Render output like fprintf().  Except, if the output is going to the
** console and if this is running on a Windows machine, translate the
** output from UTF-8 into MBCS.
*/
#if defined(_WIN32) || defined(WIN32)
void utf8_printf(FILE *out, const char *zFormat, ...){







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
      shell_strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3, PROMPT_LEN_MAX-4);
    }
  }
  return dynPrompt.dynamicPrompt;
}
#endif /* !defined(SQLITE_OMIT_DYNAPROMPT) */

#if SHELL_WIN_UTF8_OPT
/* Following struct is used for -utf8 operation. */
static struct ConsoleState {
  int stdinEof;      /* EOF has been seen on console input */
  int infsMode;      /* Input file stream mode upon shell start */
  UINT inCodePage;   /* Input code page upon shell start */
  UINT outCodePage;  /* Output code page upon shell start */
  HANDLE hConsoleIn; /* Console input handle */
  DWORD consoleMode; /* Console mode upon shell start */
} conState = { 0, 0, 0, 0, INVALID_HANDLE_VALUE, 0 };

/*
** Prepare console, (if known to be a WIN32 console), for UTF-8
** input (from either typing or suitable paste operations) and for
** UTF-8 rendering. This may "fail" with a message to stderr, where
** the preparation is not done and common "code page" issues occur.
*/
static void console_prepare(void){
  conState.hConsoleIn = GetStdHandle(STD_INPUT_HANDLE);
  if( isatty(0) && GetFileType(conState.hConsoleIn)==FILE_TYPE_CHAR ){
    if( !IsValidCodePage(CP_UTF8) ){
      fprintf(stderr, "Cannot use UTF-8 code page.\n");
      console_utf8 = 0;
      return;
    }
    conState.inCodePage = GetConsoleCP();
    conState.outCodePage = GetConsoleOutputCP();
    SetConsoleCP(CP_UTF8);
    SetConsoleOutputCP(CP_UTF8);
    GetConsoleMode( conState.hConsoleIn, &conState.consoleMode);
    SetConsoleMode( conState.hConsoleIn,
                    conState.consoleMode | ENABLE_LINE_INPUT );
    conState.infsMode = _setmode(_fileno(stdin), _O_U16TEXT);
    console_utf8 = 1;
  }else{
    console_utf8 = 0;
  }
}

/*
** Undo the effects of console_prepare(), if any.
*/
static void SQLITE_CDECL console_restore(void){
  if( console_utf8 && conState.inCodePage!=0
      && conState.hConsoleIn != INVALID_HANDLE_VALUE ){
    _setmode(_fileno(stdin), conState.infsMode);
    SetConsoleCP(conState.inCodePage);
    SetConsoleOutputCP(conState.outCodePage);
    SetConsoleMode( conState.hConsoleIn, conState.consoleMode );
    console_utf8 = 0; /* Avoid multiple calls. */
  }
}

/*
** Collect input like fgets(...) with special provisions for input
** from the Windows console to get around its strange coding issues.
** Defers to plain fgets() when input is not interactive or when the
** startup option, -utf8, has not been provided or taken effect.
*/
static char* utf8_fgets(char *buf, int ncmax, FILE *fin){
  if( fin==0 ) fin = stdin;
  if( fin==stdin && stdin_is_interactive && console_utf8 ){
# define SQLITE_IALIM 150
    wchar_t wbuf[SQLITE_IALIM];
    int lend = 0;
    int noc = 0;
    if( ncmax == 0 || conState.stdinEof ) return 0;
    buf[0] = 0;
    while( noc < ncmax-7-1 && !lend ){
      /* There is room for at least 2 more characters and a 0-terminator. */
      int na = (ncmax > SQLITE_IALIM*4+1 + noc)
        ? SQLITE_IALIM : (ncmax-1 - noc)/4;
# undef SQLITE_IALIM
      DWORD nbr = 0;
      BOOL bRC = ReadConsoleW(conState.hConsoleIn, wbuf, na, &nbr, 0);
      if( !bRC || (noc==0 && nbr==0) ) return 0;
      if( nbr > 0 ){
        int nmb = WideCharToMultiByte(CP_UTF8, WC_COMPOSITECHECK|WC_DEFAULTCHAR,
                                      wbuf,nbr, 0,0, 0, 0);
        if( nmb !=0 && noc+nmb <= ncmax ){
          int iseg = noc;
          nmb = WideCharToMultiByte(CP_UTF8, WC_COMPOSITECHECK|WC_DEFAULTCHAR,
                                    wbuf,nbr, buf+noc,nmb, 0,0);
          noc += nmb;
          /* Fixup line-ends as coded by Windows for CR (or "Enter".)*/
          if( noc > 0 ){
            if( buf[noc-1]=='\n' ){
              lend = 1;
              if( noc > 1 && buf[noc-2]=='\r' ){
                buf[noc-2] = '\n';
                --noc;
              }
            }
          }
          /* Check for ^Z (anywhere in line) too. */
          while( iseg < noc ){
            if( buf[iseg]==0x1a ){
              conState.stdinEof = 1;
              noc = iseg; /* Chop ^Z and anything following. */
              break;
            }
            ++iseg;
          }
        }else break; /* Drop apparent garbage in. (Could assert.) */
      }else break;
    }
    /* If got nothing, (after ^Z chop), must be at end-of-file. */
    if( noc == 0 ) return 0;
    buf[noc] = 0;
    return buf;
  }else{
    return fgets(buf, ncmax, fin);
  }
}

# define fgets(b,n,f) utf8_fgets(b,n,f)
#endif /* SHELL_WIN_UTF8_OPT */

/*
** Render output like fprintf().  Except, if the output is going to the
** console and if this is running on a Windows machine, translate the
** output from UTF-8 into MBCS.
*/
#if defined(_WIN32) || defined(WIN32)
void utf8_printf(FILE *out, const char *zFormat, ...){
795
796
797
798
799
800
801
802
803

804




805
806
807
808
809
810
811
      n--;
      if( n>0 && zLine[n-1]=='\r' ) n--;
      zLine[n] = 0;
      break;
    }
  }
#if defined(_WIN32) || defined(WIN32)
  /* For interactive input on Windows systems, translate the
  ** multi-byte characterset characters into UTF-8. */

  if( stdin_is_interactive && in==stdin ){




    char *zTrans = sqlite3_win32_mbcs_to_utf8_v2(zLine, 0);
    if( zTrans ){
      i64 nTrans = strlen(zTrans)+1;
      if( nTrans>nLine ){
        zLine = realloc(zLine, nTrans);
        shell_check_oom(zLine);
      }







|
|
>
|
>
>
>
>







928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
      n--;
      if( n>0 && zLine[n-1]=='\r' ) n--;
      zLine[n] = 0;
      break;
    }
  }
#if defined(_WIN32) || defined(WIN32)
  /* For interactive input on Windows systems, without -utf8,
  ** translate the multi-byte characterset characters into UTF-8.
  ** This is the translation that predates the -utf8 option. */
  if( stdin_is_interactive && in==stdin
# if SHELL_WIN_UTF8_OPT
      && !console_utf8
# endif /* SHELL_WIN_UTF8_OPT */
      ){
    char *zTrans = sqlite3_win32_mbcs_to_utf8_v2(zLine, 0);
    if( zTrans ){
      i64 nTrans = strlen(zTrans)+1;
      if( nTrans>nLine ){
        zLine = realloc(zLine, nTrans);
        shell_check_oom(zLine);
      }
11581
11582
11583
11584
11585
11586
11587



11588
11589
11590
11591
11592
11593
11594
  "   -separator SEP       set output column separator. Default: '|'\n"
#ifdef SQLITE_ENABLE_SORTER_REFERENCES
  "   -sorterref SIZE      sorter references threshold size\n"
#endif
  "   -stats               print memory stats before each finalize\n"
  "   -table               set output mode to 'table'\n"
  "   -tabs                set output mode to 'tabs'\n"



  "   -version             show SQLite version\n"
  "   -vfs NAME            use NAME as the default VFS\n"
#ifdef SQLITE_ENABLE_VFSTRACE
  "   -vfstrace            enable tracing of all VFS calls\n"
#endif
#ifdef SQLITE_HAVE_ZLIB
  "   -zip                 open the file as a ZIP Archive\n"







>
>
>







11719
11720
11721
11722
11723
11724
11725
11726
11727
11728
11729
11730
11731
11732
11733
11734
11735
  "   -separator SEP       set output column separator. Default: '|'\n"
#ifdef SQLITE_ENABLE_SORTER_REFERENCES
  "   -sorterref SIZE      sorter references threshold size\n"
#endif
  "   -stats               print memory stats before each finalize\n"
  "   -table               set output mode to 'table'\n"
  "   -tabs                set output mode to 'tabs'\n"
#if SHELL_WIN_UTF8_OPT
  "   -utf8                setup interactive console code page for UTF-8\n"
#endif
  "   -version             show SQLite version\n"
  "   -vfs NAME            use NAME as the default VFS\n"
#ifdef SQLITE_ENABLE_VFSTRACE
  "   -vfstrace            enable tracing of all VFS calls\n"
#endif
#ifdef SQLITE_HAVE_ZLIB
  "   -zip                 open the file as a ZIP Archive\n"
11693
11694
11695
11696
11697
11698
11699
11700
11701
11702
11703
11704
11705
11706
11707
#if SQLITE_SHELL_IS_UTF8
int SQLITE_CDECL main(int argc, char **argv){
#else
int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
  char **argv;
#endif
#ifdef SQLITE_DEBUG
  sqlite3_int64 mem_main_enter = sqlite3_memory_used();
#endif
  char *zErrMsg = 0;
#ifdef SQLITE_SHELL_FIDDLE
#  define data shellState
#else
  ShellState data;
#endif







|







11834
11835
11836
11837
11838
11839
11840
11841
11842
11843
11844
11845
11846
11847
11848
#if SQLITE_SHELL_IS_UTF8
int SQLITE_CDECL main(int argc, char **argv){
#else
int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
  char **argv;
#endif
#ifdef SQLITE_DEBUG
  sqlite3_int64 mem_main_enter = 0;
#endif
  char *zErrMsg = 0;
#ifdef SQLITE_SHELL_FIDDLE
#  define data shellState
#else
  ShellState data;
#endif
11715
11716
11717
11718
11719
11720
11721
11722
11723
11724
11725
11726
11727
11728
11729
11730
11731
11732





11733
11734
11735
11736
11737
11738
11739
  char **azCmd = 0;
  const char *zVfs = 0;           /* Value of -vfs command-line option */
#if !SQLITE_SHELL_IS_UTF8
  char **argvToFree = 0;
  int argcToFree = 0;
#endif

  setBinaryMode(stdin, 0);
  setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */
#ifdef SQLITE_SHELL_FIDDLE
  stdin_is_interactive = 0;
  stdout_is_console = 1;
  data.wasm.zDefaultDbName = "/fiddle.sqlite3";
#else
  stdin_is_interactive = isatty(0);
  stdout_is_console = isatty(1);
#endif






#if !defined(_WIN32_WCE)
  if( getenv("SQLITE_DEBUG_BREAK") ){
    if( isatty(0) && isatty(2) ){
      fprintf(stderr,
          "attach debugger to process %d and press any key to continue.\n",
          GETPID());
      fgetc(stdin);







<









|
>
>
>
>
>







11856
11857
11858
11859
11860
11861
11862

11863
11864
11865
11866
11867
11868
11869
11870
11871
11872
11873
11874
11875
11876
11877
11878
11879
11880
11881
11882
11883
11884
  char **azCmd = 0;
  const char *zVfs = 0;           /* Value of -vfs command-line option */
#if !SQLITE_SHELL_IS_UTF8
  char **argvToFree = 0;
  int argcToFree = 0;
#endif


  setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */
#ifdef SQLITE_SHELL_FIDDLE
  stdin_is_interactive = 0;
  stdout_is_console = 1;
  data.wasm.zDefaultDbName = "/fiddle.sqlite3";
#else
  stdin_is_interactive = isatty(0);
  stdout_is_console = isatty(1);
#endif
#if SHELL_WIN_UTF8_OPT
  atexit(console_restore); /* Needs revision for CLI as library call */
#endif
#ifdef SQLITE_DEBUG
  mem_main_enter = sqlite3_memory_used();
#endif
#if !defined(_WIN32_WCE)
  if( getenv("SQLITE_DEBUG_BREAK") ){
    if( isatty(0) && isatty(2) ){
      fprintf(stderr,
          "attach debugger to process %d and press any key to continue.\n",
          GETPID());
      fgetc(stdin);
12112
12113
12114
12115
12116
12117
12118




12119
12120
12121
12122
12123
12124
12125
    }else if( cli_strcmp(z,"-version")==0 ){
      printf("%s %s\n", sqlite3_libversion(), sqlite3_sourceid());
      return 0;
    }else if( cli_strcmp(z,"-interactive")==0 ){
      stdin_is_interactive = 1;
    }else if( cli_strcmp(z,"-batch")==0 ){
      stdin_is_interactive = 0;




    }else if( cli_strcmp(z,"-heap")==0 ){
      i++;
    }else if( cli_strcmp(z,"-pagecache")==0 ){
      i+=2;
    }else if( cli_strcmp(z,"-lookaside")==0 ){
      i+=2;
    }else if( cli_strcmp(z,"-threadsafe")==0 ){







>
>
>
>







12257
12258
12259
12260
12261
12262
12263
12264
12265
12266
12267
12268
12269
12270
12271
12272
12273
12274
    }else if( cli_strcmp(z,"-version")==0 ){
      printf("%s %s\n", sqlite3_libversion(), sqlite3_sourceid());
      return 0;
    }else if( cli_strcmp(z,"-interactive")==0 ){
      stdin_is_interactive = 1;
    }else if( cli_strcmp(z,"-batch")==0 ){
      stdin_is_interactive = 0;
    }else if( cli_strcmp(z,"-utf8")==0 ){
#if SHELL_WIN_UTF8_OPT
      console_utf8 = 1;
#endif /* SHELL_WIN_UTF8_OPT */
    }else if( cli_strcmp(z,"-heap")==0 ){
      i++;
    }else if( cli_strcmp(z,"-pagecache")==0 ){
      i+=2;
    }else if( cli_strcmp(z,"-lookaside")==0 ){
      i+=2;
    }else if( cli_strcmp(z,"-threadsafe")==0 ){
12189
12190
12191
12192
12193
12194
12195








12196
12197
12198
12199
12200
12201
12202
    }else{
      utf8_printf(stderr,"%s: Error: unknown option: %s\n", Argv0, z);
      raw_printf(stderr,"Use -help for a list of options.\n");
      return 1;
    }
    data.cMode = data.mode;
  }









  if( !readStdin ){
    /* Run all arguments that do not begin with '-' as if they were separate
    ** command-line inputs, except for the argToSkip argument which contains
    ** the database filename.
    */
    for(i=0; i<nCmd; i++){







>
>
>
>
>
>
>
>







12338
12339
12340
12341
12342
12343
12344
12345
12346
12347
12348
12349
12350
12351
12352
12353
12354
12355
12356
12357
12358
12359
    }else{
      utf8_printf(stderr,"%s: Error: unknown option: %s\n", Argv0, z);
      raw_printf(stderr,"Use -help for a list of options.\n");
      return 1;
    }
    data.cMode = data.mode;
  }
#if SHELL_WIN_UTF8_OPT
  if( console_utf8 && stdin_is_interactive ){
    console_prepare();
  }else{
    setBinaryMode(stdin, 0);
    console_utf8 = 0;
  }
#endif

  if( !readStdin ){
    /* Run all arguments that do not begin with '-' as if they were separate
    ** command-line inputs, except for the argToSkip argument which contains
    ** the database filename.
    */
    for(i=0; i<nCmd; i++){