/ Check-in [802d65af]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Tests for the functions in utf.c. (CVS 1436)
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 802d65affcafffda33e2ff1cbd4e4869dc3814df
User & Date: danielk1977 2004-05-22 08:16:11
Context
2004-05-22
09:21
Use the new form of the sqlite3_open() API everywhere. (CVS 1437) check-in: b4492173 user: danielk1977 tags: trunk
08:16
Tests for the functions in utf.c. (CVS 1436) check-in: 802d65af user: danielk1977 tags: trunk
08:09
Store the text encoding in the database (as meta value 4). (CVS 1435) check-in: 7f00ca58 user: danielk1977 tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Added test/enc.test.







































































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
# 2002 May 24
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.  The focus of
# this file is testing the SQLite routines used for converting between the
# various suported unicode encodings (UTF-8, UTF-16, UTF-16le and
# UTF-16be).
#
# $Id: enc.test,v 1.1 2004/05/22 08:16:11 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

proc do_bincmp_test {testname got expect} {
  binary scan $expect \c* expectvals
  binary scan $got \c* gotvals
  do_test $testname [list set dummy $gotvals] $expectvals
}

# $utf16 is a UTF-16 encoded string. Swap each pair of bytes around
# to change the byte-order of the string.
proc swap_byte_order {utf16} {
  binary scan $utf16 \c* ints

  foreach {a b} $ints {
    lappend ints2 $b
    lappend ints2 $a
  }

  return [binary format \c* $ints2]
}

#
# Test that the SQLite routines for converting between UTF encodings
# produce the same results as their TCL counterparts.
#
# $testname is the prefix to be used for the test names.
# $str is a string to use for testing (encoded in UTF-8, as normal for TCL).
#
# The test procedure is:
# 1. Convert the string from UTF-8 to UTF-16le and check that the TCL and
#    SQLite routines produce the same results.
#
# 2. Convert the string from UTF-8 to UTF-16be and check that the TCL and
#    SQLite routines produce the same results.
#
# 3. Use the SQLite routines to convert the native machine order UTF-16
#    representation back to the original UTF-8. Check that the result
#    matches the original representation.
#
# 4. Add a byte-order mark to each of the UTF-16 representations and
#    check that the SQLite routines can convert them back to UTF-8.  For
#    byte-order mark info, refer to section 3.10 of the unicode standard.
#
# 5. Take the byte-order marked UTF-16 strings from step 4 and ensure
#    that SQLite can convert them both to native byte order UTF-16 
#    strings, sans BOM.
#
# Coverage:
#
# sqlite_utf8to16be (step 2)
# sqlite_utf8to16le (step 1)
# sqlite_utf16to8 (steps 3, 4)
# sqlite_utf16to16le (step 5)
# sqlite_utf16to16be (step 5)
#
proc test_conversion {testname str} {
 
  # Step 1.
  set utf16le_sqlite [sqlite_utf8to16le $str]
  set utf16le_tcl [encoding convertto unicode $str]
  append utf16le_tcl "\x00\x00"
  if { $::tcl_platform(byteOrder)!="littleEndian" } {
    set utf16le_tcl [swap_byte_order $utf16le_tcl]
  }
  do_bincmp_test $testname.1 $utf16le_sqlite $utf16le_tcl
  set utf16le $utf16le_tcl

  # Step 2.
  set utf16be_sqlite [sqlite_utf8to16be $str]
  set utf16be_tcl [encoding convertto unicode $str]
  append utf16be_tcl "\x00\x00"
  if { $::tcl_platform(byteOrder)=="littleEndian" } {
    set utf16be_tcl [swap_byte_order $utf16be_tcl]
  }
  do_bincmp_test $testname.2 $utf16be_sqlite $utf16be_tcl
  set utf16be $utf16be_tcl
 
  # Step 3.
  if { $::tcl_platform(byteOrder)=="littleEndian" } {
    set utf16 $utf16le
  } else {
    set utf16 $utf16be
  }
  set utf8_sqlite [sqlite_utf16to8 $utf16]
  do_bincmp_test $testname.3 $utf8_sqlite [binarize $str]

  # Step 4 (little endian).
  append utf16le_bom "\xFF\xFE" $utf16le
  set utf8_sqlite [sqlite_utf16to8 $utf16le_bom]
  do_bincmp_test $testname.4.le $utf8_sqlite [binarize $str]

  # Step 4 (big endian).
  append utf16be_bom "\xFE\xFF" $utf16be
  set utf8_sqlite [sqlite_utf16to8 $utf16be_bom]
  do_bincmp_test $testname.4.be $utf8_sqlite [binarize $str]

  # Step 5 (little endian to little endian).
  set utf16_sqlite [sqlite_utf16to16le $utf16le_bom]
  do_bincmp_test $testname.5.le.le $utf16_sqlite $utf16le

  # Step 5 (big endian to big endian).
  set utf16_sqlite [sqlite_utf16to16be $utf16be_bom]
  do_bincmp_test $testname.5.be.be $utf16_sqlite $utf16be

  # Step 5 (big endian to little endian).
  set utf16_sqlite [sqlite_utf16to16le $utf16be_bom]
  do_bincmp_test $testname.5.be.le $utf16_sqlite $utf16le

  # Step 5 (little endian to big endian).
  set utf16_sqlite [sqlite_utf16to16be $utf16le_bom]
  do_bincmp_test $testname.5.le.be $utf16_sqlite $utf16be
}


test_conversion enc-1 "hello world"
test_conversion enc-2 "sqlite"
test_conversion enc-3 ""
test_conversion enc-4 "\u1234"
test_conversion enc-5 "\u4321abc"
test_conversion enc-6 "\u4321\u1234"
test_conversion enc-7 [string repeat "abcde\u00EF\u00EE\uFFFCabc" 100]
test_conversion enc-8 [string repeat "\u007E\u007F\u0080\u0081" 100]
test_conversion enc-9 [string repeat "\u07FE\u07FF\u0800\u0801\uFFF0" 100]

finish_test