SQLite

**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** This file contains routines used to translate between UTF-8, 
** UTF-16, UTF-16BE, and UTF-16LE.
**
** $Id: utf.c,v 1.51 2007/05/23 16:23:09 danielk1977 Exp $
**
** Notes on UTF-8:
**
**   Byte-0    Byte-1    Byte-2    Byte-3    Value
**  0xxxxxxx                                 00000000 00000000 0xxxxxxx
**  110yyyyy  10xxxxxx                       00000000 00000yyy yyxxxxxx
**  1110zzzz  10yyyyyy  10xxxxxx             00000000 zzzzyyyy yyxxxxxx

    if( !zOut ) return SQLITE_NOMEM;
  }else{
    zOut = zShort;
  }
  z = zOut;

  if( pMem->enc==SQLITE_UTF8 ){
    unsigned int iExtra = 0xD800;

    if( 0==(pMem->flags&MEM_Term) && zTerm>zIn && (zTerm[-1]&0x80) ){
      /* This UTF8 string is not nul-terminated, and the last byte is
      ** not a character in the ascii range (codpoints 0..127). This
      ** means the SQLITE_READ_UTF8() macro might read past the end
      ** of the allocated buffer.
      **
      ** There are four possibilities:
      **
      **   1. The last byte is the first byte of a non-ASCII character,
      **
      **   2. The final N bytes of the input string are continuation bytes
      **      and immediately preceding them is the first byte of a 
      **      non-ASCII character.
      **
      **   3. The final N bytes of the input string are continuation bytes
      **      and immediately preceding them is a byte that encodes a 
      **      character in the ASCII range.
      **
      **   4. The entire string consists of continuation characters.
      **
      ** Cases (3) and (4) require no special handling. The SQLITE_READ_UTF8()
      ** macro will not overread the buffer in these cases.
      */
      unsigned char *zExtra = &zTerm[-1];
      while( zExtra>zIn && (zExtra[0]&0xC0)==0x80 ){
        zExtra--;
      }

      if( (zExtra[0]&0xC0)==0xC0 ){
        /* Make a copy of the last character encoding in the input string.
        ** Then make sure it is nul-terminated and use SQLITE_READ_UTF8()
        ** to decode the codepoint. Store the codepoint in variable iExtra,
        ** it will be appended to the output string later.
        */
        unsigned char *zFree = 0;
        unsigned char zBuf[16];
        int nExtra = (pMem->n+zIn-zExtra);
        zTerm = zExtra;
        if( nExtra>15 ){
          zExtra = sqliteMallocRaw(nExtra+1);
          if( !zExtra ){
            return SQLITE_NOMEM;
          }
          zFree = zExtra;
        }else{
          zExtra = zBuf;
        }
        memcpy(zExtra, zTerm, nExtra);
        zExtra[nExtra] = '\0';
        SQLITE_READ_UTF8(zExtra, iExtra);
        sqliteFree(zFree);
      }
    }

    if( desiredEnc==SQLITE_UTF16LE ){
      /* UTF-8 -> UTF-16 Little-endian */
      while( zIn<zTerm ){
        SQLITE_READ_UTF8(zIn, c); 
        WRITE_UTF16LE(z, c);
      }
      if( iExtra!=0xD800 ){
        WRITE_UTF16LE(z, iExtra);
      }
    }else{
      assert( desiredEnc==SQLITE_UTF16BE );
      /* UTF-8 -> UTF-16 Big-endian */
      while( zIn<zTerm ){
        SQLITE_READ_UTF8(zIn, c); 
        WRITE_UTF16BE(z, c);
      }
      if( iExtra!=0xD800 ){
        WRITE_UTF16BE(z, iExtra);
      }
    }
    pMem->n = z - zOut;
    *z++ = 0;
  }else{
    assert( desiredEnc==SQLITE_UTF8 );
    if( pMem->enc==SQLITE_UTF16LE ){
      /* UTF-16 Little-endian -> UTF-8 */

#
#***********************************************************************
# This file implements regression tests for SQLite library.  The focus of
# this file is testing the SQLite routines used for converting between the
# various suported unicode encodings (UTF-8, UTF-16, UTF-16le and
# UTF-16be).
#
# $Id: enc.test,v 1.7 2007/05/23 16:23:09 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# Skip this test if the build does not support multiple encodings.
#
ifcapable {!utf16} {

test_conversion enc-X "\u0100"
test_conversion enc-4 "\u1234"
test_conversion enc-5 "\u4321abc"
test_conversion enc-6 "\u4321\u1234"
test_conversion enc-7 [string repeat "abcde\u00EF\u00EE\uFFFCabc" 100]
test_conversion enc-8 [string repeat "\u007E\u007F\u0080\u0081" 100]
test_conversion enc-9 [string repeat "\u07FE\u07FF\u0800\u0801\uFFF0" 100]
test_conversion enc-10 [string repeat "\uE000" 100]

proc test_collate {enc zLeft zRight} {
  return [string compare $zLeft $zRight]
}
add_test_collate $::DB 0 0 1
do_test enc-11.1 {
  execsql {
    CREATE TABLE ab(a COLLATE test_collate, b);
    INSERT INTO ab VALUES(CAST (X'C388' AS TEXT), X'888800');
    INSERT INTO ab VALUES(CAST (X'C0808080808080808080808080808080808080808080808080808080808080808080808080808080808080808080808080808080808388' AS TEXT), X'888800');
    CREATE INDEX ab_i ON ab(a, b);
  }
} {}
do_test enc-11.2 {
  set cp200 "\u00C8"
  execsql {
    SELECT count(*) FROM ab WHERE a = $::cp200;
  }
} {2}

finish_test

# This file implements regression tests for SQLite library.  The
# focus of this file is testing for correct handling of I/O errors
# such as writes failing because the disk is full.
# 
# The tests in this file use special facilities that are only
# available in the SQLite test fixture.
#
# $Id: ioerr2.test,v 1.5 2007/05/23 16:23:09 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

do_test ioerr2-1.1 {
  execsql {
    PRAGMA cache_size = 10;

  set ::go 1
  for {set ::N 1} {$::go} {incr ::N} {
    set ::sqlite_io_error_hit 0
    set ::sqlite_io_error_persist $bPersist
    set ::sqlite_io_error_pending $::N

    foreach {::go res} [catchsql $sql] {}
    check_db ioerr2-4.[expr {$bPersist+2}].$::N
  }
}

finish_test