SQLite

Check-in [8104baf23d]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix for retrieving UTF-16 little-endian text from a big-endian database. (CVS 1446)
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 8104baf23dd28fc982cf260e3e8c90f0c582f602
User & Date: danielk1977 2004-05-24 07:34:48.000
Context
2004-05-24
09:10
Add the sqlite3_value_*() access functions. (CVS 1447) (check-in: 4bf925fcfc user: danielk1977 tags: trunk)
07:34
Fix for retrieving UTF-16 little-endian text from a big-endian database. (CVS 1446) (check-in: 8104baf23d user: danielk1977 tags: trunk)
07:04
Have the vdbe handle strings in the same encoding as the database. (CVS 1445) (check-in: b7155db2b1 user: danielk1977 tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/vdbe.c.
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
**
** Various scripts scan this source file in order to generate HTML
** documentation, headers files, or other derived files.  The formatting
** of the code in this file is, therefore, important.  See other comments
** in this file for details.  If in doubt, do not deviate from existing
** commenting and indentation practices when changing or adding code.
**
** $Id: vdbe.c,v 1.322 2004/05/24 07:04:27 danielk1977 Exp $
*/
#include "sqliteInt.h"
#include "os.h"
#include <ctype.h>
#include "vdbeInt.h"

/*







|







39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
**
** Various scripts scan this source file in order to generate HTML
** documentation, headers files, or other derived files.  The formatting
** of the code in this file is, therefore, important.  See other comments
** in this file for details.  If in doubt, do not deviate from existing
** commenting and indentation practices when changing or adding code.
**
** $Id: vdbe.c,v 1.323 2004/05/24 07:34:48 danielk1977 Exp $
*/
#include "sqliteInt.h"
#include "os.h"
#include <ctype.h>
#include "vdbeInt.h"

/*
138
139
140
141
142
143
144






























































































145
146
147
148
149
150
151
/*
** Set the encoding flags of memory cell "pMem" to the correct values
** for the database encoding "enc" (one of TEXT_Utf8, TEXT_Utf16le or
** TEXT_Utf16be).
*/
#define SetEncodingFlags(pMem, enc) ((pMem)->flags = \
((pMem->flags & ~(MEM_Utf8|MEM_Utf16le|MEM_Utf16be))) | encToFlags(enc))































































































/*
** If pMem is a string object, this routine sets the encoding of the string
** (to one of UTF-8 or UTF16) and whether or not the string is
** nul-terminated. If pMem is not a string object, then this routine is
** a no-op.
**







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
/*
** Set the encoding flags of memory cell "pMem" to the correct values
** for the database encoding "enc" (one of TEXT_Utf8, TEXT_Utf16le or
** TEXT_Utf16be).
*/
#define SetEncodingFlags(pMem, enc) ((pMem)->flags = \
((pMem->flags & ~(MEM_Utf8|MEM_Utf16le|MEM_Utf16be))) | encToFlags(enc))
static int SetEncoding(Mem*, int);

/*
** Convert the given stack entity into a string if it isn't one
** already. Return non-zero if a malloc() fails.
*/
#define Stringify(P, enc) \
(!((P)->flags&(MEM_Str|MEM_Blob)) && hardStringify(P, enc))
static int hardStringify(Mem *pStack, u8 enc){
  int rc = SQLITE_OK;
  int fg = pStack->flags;

  assert( !(fg&(MEM_Str|MEM_Blob)) );
  assert( fg&(MEM_Int|MEM_Real|MEM_Null) );

  if( fg & MEM_Null ){      
    /* A NULL value is converted to a zero length string */
    pStack->zShort[0] = 0;
    pStack->zShort[1] = 0;
    pStack->flags = MEM_Str | MEM_Short | MEM_Term;
    pStack->z = pStack->zShort;
    pStack->n = (enc==TEXT_Utf8?1:2);
  }else{
    /* For a Real or Integer, use sqlite3_snprintf() to produce the UTF-8
    ** string representation of the value. Then, if the required encoding
    ** is UTF-16le or UTF-16be do a translation.
    ** 
    ** FIX ME: It would be better if sqlite3_snprintf() could do UTF-16.
    */
    if( fg & MEM_Real ){
      sqlite3_snprintf(NBFS, pStack->zShort, "%.15g", pStack->r);
    }else if( fg & MEM_Int ){
      sqlite3_snprintf(NBFS, pStack->zShort, "%lld", pStack->i);
    }
    pStack->n = strlen(pStack->zShort) + 1;
    pStack->z = pStack->zShort;
    pStack->flags = MEM_Str | MEM_Short | MEM_Term;

    /* Flip the string to UTF-16 if required */
    SetEncodingFlags(pStack, TEXT_Utf8);
    rc = SetEncoding(pStack, encToFlags(enc)|MEM_Term);
  }

  return rc;
}

/*
** Convert the given stack entity into a string that has been obtained
** from sqliteMalloc().  This is different from Stringify() above in that
** Stringify() will use the NBFS bytes of static string space if the string
** will fit but this routine always mallocs for space.
** Return non-zero if we run out of memory.
*/
#define Dynamicify(P, enc) \
(((P)->flags & MEM_Dyn)==0 ? hardDynamicify(P, enc):0)
static int hardDynamicify(Mem *pStack, u8 enc){
  int fg = pStack->flags;
  char *z;
  if( (fg & MEM_Str)==0 ){
    hardStringify(pStack, enc);
  }
  assert( (fg & MEM_Dyn)==0 );
  z = sqliteMallocRaw( pStack->n );
  if( z==0 ) return 1;
  memcpy(z, pStack->z, pStack->n);
  pStack->z = z;
  pStack->flags |= MEM_Dyn;
  return 0;
}

/*
** An ephemeral string value (signified by the MEM_Ephem flag) contains
** a pointer to a dynamically allocated string where some other entity
** is responsible for deallocating that string.  Because the stack entry
** does not control the string, it might be deleted without the stack
** entry knowing it.
**
** This routine converts an ephemeral string into a dynamically allocated
** string that the stack entry itself controls.  In other words, it
** converts an MEM_Ephem string into an MEM_Dyn string.
*/
#define Deephemeralize(P) \
   if( ((P)->flags&MEM_Ephem)!=0 && hardDeephem(P) ){ goto no_mem;}
static int hardDeephem(Mem *pStack){
  char *z;
  assert( (pStack->flags & MEM_Ephem)!=0 );
  z = sqliteMallocRaw( pStack->n );
  if( z==0 ) return 1;
  memcpy(z, pStack->z, pStack->n);
  pStack->z = z;
  pStack->flags &= ~MEM_Ephem;
  pStack->flags |= MEM_Dyn;
  return 0;
}

/*
** If pMem is a string object, this routine sets the encoding of the string
** (to one of UTF-8 or UTF16) and whether or not the string is
** nul-terminated. If pMem is not a string object, then this routine is
** a no-op.
**
167
168
169
170
171
172
173

174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190













191
192
193
194
195
196
197
    return SQLITE_OK;
  }

  enc1 = flagsToEnc(pMem->flags);
  enc2 = flagsToEnc(flags);

  if( enc1!=enc2 ){

    /* If the current encoding does not match the desired encoding, then
    ** we will need to do some translation between encodings.
    */
    char *z;
    int n;
    int rc = sqlite3utfTranslate(pMem->z,pMem->n,enc1,(void **)&z,&n,enc2);
    if( rc!=SQLITE_OK ){
      return rc;
    }

    /* Result of sqlite3utfTranslate is currently always dynamically
    ** allocated and nul terminated. This might be altered as a performance
    ** enhancement later.
    */
    pMem->z = z;
    pMem->n = n;
    pMem->flags = (MEM_Str | MEM_Dyn | MEM_Term | flags);













  }

  if( (flags&MEM_Term) && !(pMem->flags&MEM_Term) ){
    /* If we did not do any translation, but currently the string is
    ** not nul terminated (and is required to be), then we add the
    ** nul terminator now. We never have to do this if we translated
    ** the encoding of the string, as the translation functions return







>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
>
>
>
>
>
>
>
>
>
>
>
>
>







261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
    return SQLITE_OK;
  }

  enc1 = flagsToEnc(pMem->flags);
  enc2 = flagsToEnc(flags);

  if( enc1!=enc2 ){
    if( enc1==TEXT_Utf8 || enc2==TEXT_Utf8 ){
      /* If the current encoding does not match the desired encoding, then
      ** we will need to do some translation between encodings.
      */
      char *z;
      int n;
      int rc = sqlite3utfTranslate(pMem->z,pMem->n,enc1,(void **)&z,&n,enc2);
      if( rc!=SQLITE_OK ){
        return rc;
      }
  
      /* Result of sqlite3utfTranslate is currently always dynamically
      ** allocated and nul terminated. This might be altered as a performance
      ** enhancement later.
      */
      pMem->z = z;
      pMem->n = n;
      pMem->flags = (MEM_Str | MEM_Dyn | MEM_Term | flags);
    }else{
      /* Must be translating between UTF-16le and UTF-16be. */
      int i;
      if( pMem->flags&MEM_Static ){
        Dynamicify(pMem, enc1);
      }
      for(i=0; i<pMem->n; i+=2){
        char c = pMem->z[i];
        pMem->z[i] = pMem->z[i+1];
        pMem->z[i+1] = c;
      }
      SetEncodingFlags(pMem, enc2);
    }
  }

  if( (flags&MEM_Term) && !(pMem->flags&MEM_Term) ){
    /* If we did not do any translation, but currently the string is
    ** not nul terminated (and is required to be), then we add the
    ** nul terminator now. We never have to do this if we translated
    ** the encoding of the string, as the translation functions return
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
  }else{
    pStack->r = 0.0;
  }
/*  pStack->flags |= MEM_Real; */
  pStack->flags = MEM_Real;
}


/*
** Convert the given stack entity into a string if it isn't one
** already. Return non-zero if a malloc() fails.
*/
#define Stringify(P, enc) \
(!((P)->flags&(MEM_Str|MEM_Blob)) && hardStringify(P, enc))
static int hardStringify(Mem *pStack, u8 enc){
  int rc = SQLITE_OK;
  int fg = pStack->flags;

  assert( !(fg&(MEM_Str|MEM_Blob)) );
  assert( fg&(MEM_Int|MEM_Real|MEM_Null) );

  if( fg & MEM_Null ){      
    /* A NULL value is converted to a zero length string */
    pStack->zShort[0] = 0;
    pStack->zShort[1] = 0;
    pStack->flags = MEM_Str | MEM_Short | MEM_Term;
    pStack->z = pStack->zShort;
    pStack->n = (enc==TEXT_Utf8?1:2);
  }else{
    /* For a Real or Integer, use sqlite3_snprintf() to produce the UTF-8
    ** string representation of the value. Then, if the required encoding
    ** is UTF-16le or UTF-16be do a translation.
    ** 
    ** FIX ME: It would be better if sqlite3_snprintf() could do UTF-16.
    */
    if( fg & MEM_Real ){
      sqlite3_snprintf(NBFS, pStack->zShort, "%.15g", pStack->r);
    }else if( fg & MEM_Int ){
      sqlite3_snprintf(NBFS, pStack->zShort, "%lld", pStack->i);
    }
    pStack->n = strlen(pStack->zShort) + 1;
    pStack->z = pStack->zShort;
    pStack->flags = MEM_Str | MEM_Short | MEM_Term;

    /* Flip the string to UTF-16 if required */
    SetEncodingFlags(pStack, TEXT_Utf8);
    rc = SetEncoding(pStack, encToFlags(enc)|MEM_Term);
  }

  return rc;
}


/*
** Convert the given stack entity into a string that has been obtained
** from sqliteMalloc().  This is different from Stringify() above in that
** Stringify() will use the NBFS bytes of static string space if the string
** will fit but this routine always mallocs for space.
** Return non-zero if we run out of memory.
*/
#define Dynamicify(P, enc) \
(((P)->flags & MEM_Dyn)==0 ? hardDynamicify(P, enc):0)
static int hardDynamicify(Mem *pStack, u8 enc){
  int fg = pStack->flags;
  char *z;
  if( (fg & MEM_Str)==0 ){
    hardStringify(pStack, enc);
  }
  assert( (fg & MEM_Dyn)==0 );
  z = sqliteMallocRaw( pStack->n );
  if( z==0 ) return 1;
  memcpy(z, pStack->z, pStack->n);
  pStack->z = z;
  pStack->flags |= MEM_Dyn;
  return 0;
}

/*
** An ephemeral string value (signified by the MEM_Ephem flag) contains
** a pointer to a dynamically allocated string where some other entity
** is responsible for deallocating that string.  Because the stack entry
** does not control the string, it might be deleted without the stack
** entry knowing it.
**
** This routine converts an ephemeral string into a dynamically allocated
** string that the stack entry itself controls.  In other words, it
** converts an MEM_Ephem string into an MEM_Dyn string.
*/
#define Deephemeralize(P) \
   if( ((P)->flags&MEM_Ephem)!=0 && hardDeephem(P) ){ goto no_mem;}
static int hardDeephem(Mem *pStack){
  char *z;
  assert( (pStack->flags & MEM_Ephem)!=0 );
  z = sqliteMallocRaw( pStack->n );
  if( z==0 ) return 1;
  memcpy(z, pStack->z, pStack->n);
  pStack->z = z;
  pStack->flags &= ~MEM_Ephem;
  pStack->flags |= MEM_Dyn;
  return 0;
}

/*
** Advance the virtual machine to the next output row.
**
** The return vale will be either SQLITE_BUSY, SQLITE_DONE, 
** SQLITE_ROW, SQLITE_ERROR, or SQLITE_MISUSE.
**
** SQLITE_BUSY means that the virtual machine attempted to open







<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<







393
394
395
396
397
398
399































































































400
401
402
403
404
405
406
  }else{
    pStack->r = 0.0;
  }
/*  pStack->flags |= MEM_Real; */
  pStack->flags = MEM_Real;
}
































































































/*
** Advance the virtual machine to the next output row.
**
** The return vale will be either SQLITE_BUSY, SQLITE_DONE, 
** SQLITE_ROW, SQLITE_ERROR, or SQLITE_MISUSE.
**
** SQLITE_BUSY means that the virtual machine attempted to open
Changes to test/quick.test.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
# 2001 September 15
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file runs all tests.
#
# $Id: quick.test,v 1.18 2004/05/23 13:30:59 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl
rename finish_test really_finish_test
proc finish_test {} {}
set ISQUICK 1













|







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
# 2001 September 15
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file runs all tests.
#
# $Id: quick.test,v 1.19 2004/05/24 07:34:49 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl
rename finish_test really_finish_test
proc finish_test {} {}
set ISQUICK 1

36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
lappend EXCLUDE auth.test         ;# Cannot attach empty databases.
lappend EXCLUDE tableapi.test     ;# sqlite3_XX vs sqlite_XX problem
lappend EXCLUDE version.test      ;# uses the btree_meta API (not updated)

# Some tests fail in these file as a result of the partial manifest types
# implementation.
lappend EXCLUDE capi2.test 
lappend EXCLUDE enc2.test 


if {[sqlite -has-codec]} {
  lappend EXCLUDE \
    attach.test \
    attach2.test \
    auth.test \
    format3.test \







<
<







36
37
38
39
40
41
42


43
44
45
46
47
48
49
lappend EXCLUDE auth.test         ;# Cannot attach empty databases.
lappend EXCLUDE tableapi.test     ;# sqlite3_XX vs sqlite_XX problem
lappend EXCLUDE version.test      ;# uses the btree_meta API (not updated)

# Some tests fail in these file as a result of the partial manifest types
# implementation.
lappend EXCLUDE capi2.test 



if {[sqlite -has-codec]} {
  lappend EXCLUDE \
    attach.test \
    attach2.test \
    auth.test \
    format3.test \