SQLite

Check-in [543594a727]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:For CLI -utf8, set output codepage too. Adjust PP vars so that the code can be entirely omitted for targets pretending to be "WIN32" but not quite doing so.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | cli-utf8
Files: files | file ages | folders
SHA3-256: 543594a7277b12d1c58af8e3cba3841667cf924172995536489ce7da704a11d7
User & Date: larrybr 2023-04-15 16:12:46.806
Context
2023-04-17
21:18
When CLI -utf8 active, do no translation to MBCS for output to Windows console. (check-in: cc1d4296d7 user: larrybr tags: cli-utf8)
2023-04-15
17:56
Add -utf8 option to CLI, for improved console behavior on Windows. (check-in: 414010d236 user: larrybr tags: trunk)
16:12
For CLI -utf8, set output codepage too. Adjust PP vars so that the code can be entirely omitted for targets pretending to be "WIN32" but not quite doing so. (check-in: 543594a727 user: larrybr tags: cli-utf8)
2023-04-14
21:34
Fix stray edits in comments in last check-in. (check-in: fa3ce7f013 user: larrybr tags: cli-utf8)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/shell.c.in.
444
445
446
447
448
449
450
451

452
453
454
455
456
457
458

/*
** Treat stdin as an interactive input if the following variable
** is true.  Otherwise, assume stdin is connected to a file or pipe.
*/
static int stdin_is_interactive = 1;

#if (defined(_WIN32) || defined(WIN32)) && SHELL_USE_LOCAL_GETLINE

# define SHELL_WIN_UTF8_OPT 1
#else
# define SHELL_WIN_UTF8_OPT 0
#endif

#if SHELL_WIN_UTF8_OPT
/*







|
>







444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459

/*
** Treat stdin as an interactive input if the following variable
** is true.  Otherwise, assume stdin is connected to a file or pipe.
*/
static int stdin_is_interactive = 1;

#if (defined(_WIN32) || defined(WIN32)) && SHELL_USE_LOCAL_GETLINE \
  && !defined(SHELL_OMIT_WIN_UTF8)
# define SHELL_WIN_UTF8_OPT 1
#else
# define SHELL_WIN_UTF8_OPT 0
#endif

#if SHELL_WIN_UTF8_OPT
/*
590
591
592
593
594
595
596
597

598
599
600

601
602

603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618

619


620
621
622
623
624
625
626
627
628
629
630
631
632

633
634
635

636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
    }
  }
  return dynPrompt.dynamicPrompt;
}
#endif /* !defined(SQLITE_OMIT_DYNAPROMPT) */

#if SHELL_WIN_UTF8_OPT
/* Following variables are used for -utf8 operation. */

static int stdinEof = 0;  /* EOF seen on console input */
static int infsMode;      /* Input file stream mode upon shell start */
static UINT codePage = 0; /* Input code page upon shell start */

static HANDLE hConsoleIn = INVALID_HANDLE_VALUE; /* Console input handle */
static DWORD consoleMode; /* Console mode upon shell start */


/*
** Prepare input stream, (if known to be a WIN32 console), for UTF-8
** input (from either typing or suitable paste operations) and for
** UTF-8 rendering. This may "fail" with a message to stderr, where
** the preparation is not done and common "code page" issues occur.
*/
static void instream_prepare(void){
  hConsoleIn = GetStdHandle(STD_INPUT_HANDLE);
  if( isatty(0) && GetFileType(hConsoleIn)==FILE_TYPE_CHAR ){
    if( !IsValidCodePage(CP_UTF8) ){
      fprintf(stderr, "Cannot use UTF-8 code page.\n");
      console_utf8 = 0;
      return;
    }
    codePage = GetConsoleCP();

    SetConsoleCP(CP_UTF8);


    GetConsoleMode( hConsoleIn, &consoleMode);
    SetConsoleMode( hConsoleIn, consoleMode | ENABLE_LINE_INPUT );
    infsMode = _setmode(_fileno(stdin), _O_U16TEXT);
    console_utf8 = 1;
  }else{
    console_utf8 = 0;
  }
}

/*
** Undo the effects of instream_prepare(), if any.
*/
static void SQLITE_CDECL instream_restore(void){

  if( console_utf8 && codePage!=0 &&hConsoleIn != INVALID_HANDLE_VALUE ){
    _setmode(_fileno(stdin), infsMode);
    SetConsoleCP(codePage);

    SetConsoleMode( hConsoleIn, consoleMode );
    console_utf8 = 0; /* Avoid multiple calls. */
  }
}

/*
** Collect input like fgets(...) with special provisions for input
** from the Windows console to get around its strange coding issues.
** Defers to plain fgets() when input is not interactive or when the
** startup option, -utf8, has not been provided or taken effect.
*/
static char* utf8_fgets(char *buf, int ncmax, FILE *fin){
  if( fin==0 ) fin = stdin;
  if( fin==stdin && stdin_is_interactive && console_utf8 ){
# define SQLITE_IALIM 150
    wchar_t wbuf[SQLITE_IALIM];
    int lend = 0;
    int noc = 0;
    if( ncmax == 0 || stdinEof ) return 0;
    buf[0] = 0;
    while( noc < ncmax-7-1 && !lend ){
      /* There is room for at least 2 more characters and a 0-terminator. */
      int na = (ncmax > SQLITE_IALIM*4+1 + noc)
        ? SQLITE_IALIM : (ncmax-1 - noc)/4;
# undef SQLITE_IALIM
      DWORD nbr = 0;
      BOOL bRC = ReadConsoleW(hConsoleIn, wbuf, na, &nbr, 0);
      if( !bRC || (noc==0 && nbr==0) ) return 0;
      if( nbr > 0 ){
        int nmb = WideCharToMultiByte(CP_UTF8, WC_COMPOSITECHECK|WC_DEFAULTCHAR,
                                      wbuf,nbr, 0,0, 0, 0);
        if( nmb !=0 && noc+nmb <= ncmax ){
          int iseg = noc;
          nmb = WideCharToMultiByte(CP_UTF8, WC_COMPOSITECHECK|WC_DEFAULTCHAR,







|
>
|
|
|
>
|
|
>


|




|
|
|





|
>

>
>
|
|
|







|

|
>
|
|
|
>
|

















|







|







591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
    }
  }
  return dynPrompt.dynamicPrompt;
}
#endif /* !defined(SQLITE_OMIT_DYNAPROMPT) */

#if SHELL_WIN_UTF8_OPT
/* Following struct is used for -utf8 operation. */
static struct ConsoleState {
  int stdinEof;      /* EOF has been seen on console input */
  int infsMode;      /* Input file stream mode upon shell start */
  UINT inCodePage;   /* Input code page upon shell start */
  UINT outCodePage;  /* Output code page upon shell start */
  HANDLE hConsoleIn; /* Console input handle */
  DWORD consoleMode; /* Console mode upon shell start */
} conState = { 0, 0, 0, 0, INVALID_HANDLE_VALUE, 0 };

/*
** Prepare console, (if known to be a WIN32 console), for UTF-8
** input (from either typing or suitable paste operations) and for
** UTF-8 rendering. This may "fail" with a message to stderr, where
** the preparation is not done and common "code page" issues occur.
*/
static void console_prepare(void){
  conState.hConsoleIn = GetStdHandle(STD_INPUT_HANDLE);
  if( isatty(0) && GetFileType(conState.hConsoleIn)==FILE_TYPE_CHAR ){
    if( !IsValidCodePage(CP_UTF8) ){
      fprintf(stderr, "Cannot use UTF-8 code page.\n");
      console_utf8 = 0;
      return;
    }
    conState.inCodePage = GetConsoleCP();
    conState.outCodePage = GetConsoleOutputCP();
    SetConsoleCP(CP_UTF8);
    SetConsoleOutputCP(CP_UTF8);
    GetConsoleMode( conState.hConsoleIn, &conState.consoleMode);
    SetConsoleMode( conState.hConsoleIn,
                    conState.consoleMode | ENABLE_LINE_INPUT );
    conState.infsMode = _setmode(_fileno(stdin), _O_U16TEXT);
    console_utf8 = 1;
  }else{
    console_utf8 = 0;
  }
}

/*
** Undo the effects of console_prepare(), if any.
*/
static void SQLITE_CDECL console_restore(void){
  if( console_utf8 && conState.inCodePage!=0
      && conState.hConsoleIn != INVALID_HANDLE_VALUE ){
    _setmode(_fileno(stdin), conState.infsMode);
    SetConsoleCP(conState.inCodePage);
    SetConsoleOutputCP(conState.outCodePage);
    SetConsoleMode( conState.hConsoleIn, conState.consoleMode );
    console_utf8 = 0; /* Avoid multiple calls. */
  }
}

/*
** Collect input like fgets(...) with special provisions for input
** from the Windows console to get around its strange coding issues.
** Defers to plain fgets() when input is not interactive or when the
** startup option, -utf8, has not been provided or taken effect.
*/
static char* utf8_fgets(char *buf, int ncmax, FILE *fin){
  if( fin==0 ) fin = stdin;
  if( fin==stdin && stdin_is_interactive && console_utf8 ){
# define SQLITE_IALIM 150
    wchar_t wbuf[SQLITE_IALIM];
    int lend = 0;
    int noc = 0;
    if( ncmax == 0 || conState.stdinEof ) return 0;
    buf[0] = 0;
    while( noc < ncmax-7-1 && !lend ){
      /* There is room for at least 2 more characters and a 0-terminator. */
      int na = (ncmax > SQLITE_IALIM*4+1 + noc)
        ? SQLITE_IALIM : (ncmax-1 - noc)/4;
# undef SQLITE_IALIM
      DWORD nbr = 0;
      BOOL bRC = ReadConsoleW(conState.hConsoleIn, wbuf, na, &nbr, 0);
      if( !bRC || (noc==0 && nbr==0) ) return 0;
      if( nbr > 0 ){
        int nmb = WideCharToMultiByte(CP_UTF8, WC_COMPOSITECHECK|WC_DEFAULTCHAR,
                                      wbuf,nbr, 0,0, 0, 0);
        if( nmb !=0 && noc+nmb <= ncmax ){
          int iseg = noc;
          nmb = WideCharToMultiByte(CP_UTF8, WC_COMPOSITECHECK|WC_DEFAULTCHAR,
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
                --noc;
              }
            }
          }
          /* Check for ^Z (anywhere in line) too. */
          while( iseg < noc ){
            if( buf[iseg]==0x1a ){
              stdinEof = 1;
              noc = iseg; /* Chop ^Z and anything following. */
              break;
            }
            ++iseg;
          }
        }else break; /* Drop apparent garbage in. (Could assert.) */
      }else break;







|







687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
                --noc;
              }
            }
          }
          /* Check for ^Z (anywhere in line) too. */
          while( iseg < noc ){
            if( buf[iseg]==0x1a ){
              conState.stdinEof = 1;
              noc = iseg; /* Chop ^Z and anything following. */
              break;
            }
            ++iseg;
          }
        }else break; /* Drop apparent garbage in. (Could assert.) */
      }else break;
11857
11858
11859
11860
11861
11862
11863
11864
11865
11866
11867
11868
11869
11870
11871
  stdout_is_console = 1;
  data.wasm.zDefaultDbName = "/fiddle.sqlite3";
#else
  stdin_is_interactive = isatty(0);
  stdout_is_console = isatty(1);
#endif
#if SHELL_WIN_UTF8_OPT
  atexit(instream_restore); /* Needs revision for CLI as library call */
#endif
#ifdef SQLITE_DEBUG
  mem_main_enter = sqlite3_memory_used();
#endif
#if !defined(_WIN32_WCE)
  if( getenv("SQLITE_DEBUG_BREAK") ){
    if( isatty(0) && isatty(2) ){







|







11866
11867
11868
11869
11870
11871
11872
11873
11874
11875
11876
11877
11878
11879
11880
  stdout_is_console = 1;
  data.wasm.zDefaultDbName = "/fiddle.sqlite3";
#else
  stdin_is_interactive = isatty(0);
  stdout_is_console = isatty(1);
#endif
#if SHELL_WIN_UTF8_OPT
  atexit(console_restore); /* Needs revision for CLI as library call */
#endif
#ifdef SQLITE_DEBUG
  mem_main_enter = sqlite3_memory_used();
#endif
#if !defined(_WIN32_WCE)
  if( getenv("SQLITE_DEBUG_BREAK") ){
    if( isatty(0) && isatty(2) ){
12331
12332
12333
12334
12335
12336
12337
12338
12339
12340
12341
12342
12343
12344
12345
      raw_printf(stderr,"Use -help for a list of options.\n");
      return 1;
    }
    data.cMode = data.mode;
  }
#if SHELL_WIN_UTF8_OPT
  if( console_utf8 && stdin_is_interactive ){
    instream_prepare();
  }else{
    setBinaryMode(stdin, 0);
    console_utf8 = 0;
  }
#endif

  if( !readStdin ){







|







12340
12341
12342
12343
12344
12345
12346
12347
12348
12349
12350
12351
12352
12353
12354
      raw_printf(stderr,"Use -help for a list of options.\n");
      return 1;
    }
    data.cMode = data.mode;
  }
#if SHELL_WIN_UTF8_OPT
  if( console_utf8 && stdin_is_interactive ){
    console_prepare();
  }else{
    setBinaryMode(stdin, 0);
    console_utf8 = 0;
  }
#endif

  if( !readStdin ){