SQLite

Check-in [c5ed5ebdf6]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Change the (machine-generated) keywordhash.h file to increase the scope of the tables used for keyword matching, so that the tables are accessible to functions other then keywordCode().
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: c5ed5ebdf660501fde7cc8aefaaaeae2a68e5899a64ac93f26684842a235281d
User & Date: drh 2017-07-06 16:33:14.765
Context
2017-07-06
22:43
Small adjustment to main.mk that facilitates giving non-standard compile-time options to the shell. (check-in: 7c7d53a9bb user: drh tags: trunk)
17:36
Initial implementation of a highly experimental interface for listing all keywords and symbolic names for an SQLite database connection. (check-in: 04ef6783a5 user: drh tags: experimental-namelist)
16:33
Change the (machine-generated) keywordhash.h file to increase the scope of the tables used for keyword matching, so that the tables are accessible to functions other then keywordCode(). (check-in: c5ed5ebdf6 user: drh tags: trunk)
13:51
More compact implementation of the typeof() SQL function. (check-in: efb4aab0ca user: drh tags: trunk)
Changes
Side-by-Side Diff Ignore Whitespace Patch
Changes to tool/mkkeywordhash.c.
332
333
334
335
336
337
338
339
340


341
342
343
344
345
346
347
332
333
334
335
336
337
338


339
340
341
342
343
344
345
346
347







-
-
+
+







*/
int main(int argc, char **argv){
  int i, j, k, h;
  int bestSize, bestCount;
  int count;
  int nChar;
  int totalLen = 0;
  int aHash[1000];  /* 1000 is much bigger than nKeyword */
  char zText[2000];
  int aKWHash[1000];  /* 1000 is much bigger than nKeyword */
  char zKWText[2000];

  /* Remove entries from the list of keywords that have mask==0 */
  for(i=j=0; i<nKeyword; i++){
    if( aKeywordTable[i].mask==0 ) continue;
    if( j<i ){
      aKeywordTable[j] = aKeywordTable[i];
    }
437
438
439
440
441
442
443
444

445
446
447
448


449
450

451
452
453
454
455
456
457
458

459
460
461
462


463
464
465
466
467
468
469

470
471
472
473
474

475
476
477
478
479
480
481

482
483
484
485
486
487
488
489
490
491


492
493
494

495
496

497
498
499

500
501
502
503
504
505
506
507
508

509

510

511
512
513


514
515
516
517
518
519
520

521



522

523
524

525
526
527
528
529
530
531
532

533

534

535
536

537
538
539
540
541
542
543
544

545


546

547
548

549
550
551
552
553
554
555
556

557

558

559
560
561

562
563
564
565
566
567
568
569
570





571
572
573
574
575
576
577


578
579

580
581
582
583
584
585
586
587
588
589
590
591

592
593
594
595
596
597
598
437
438
439
440
441
442
443

444
445
446


447
448
449

450
451
452
453
454
455
456
457

458
459
460


461
462
463
464
465
466
467


468
469
470
471
472

473
474
475
476
477
478
479

480
481
482
483
484
485
486
487
488


489
490
491
492

493
494

495
496
497

498
499
500
501
502
503
504
505
506

507
508
509

510
511


512
513
514
515
516
517
518
519

520
521
522
523
524

525
526

527
528
529
530
531
532
533
534

535
536
537

538
539

540
541
542
543
544
545
546
547

548
549
550
551

552
553

554
555
556
557
558
559
560
561

562
563
564

565
566
567

568
569
570
571
572
573
574
575


576
577
578
579
580
581
582
583
584
585


586
587
588

589
590
591
592
593
594
595
596
597
598
599
600

601
602
603
604
605
606
607
608







-
+


-
-
+
+

-
+







-
+


-
-
+
+





-
-
+




-
+






-
+








-
-
+
+


-
+

-
+


-
+








-
+

+
-
+

-
-
+
+






-
+

+
+
+
-
+

-
+







-
+

+
-
+

-
+







-
+

+
+
-
+

-
+







-
+

+
-
+


-
+







-
-
+
+
+
+
+





-
-
+
+

-
+











-
+







  qsort(aKeywordTable, nKeyword, sizeof(aKeywordTable[0]), keywordCompare3);

  /* Figure out how big to make the hash table in order to minimize the
  ** number of collisions */
  bestSize = nKeyword;
  bestCount = nKeyword*nKeyword;
  for(i=nKeyword/2; i<=2*nKeyword; i++){
    for(j=0; j<i; j++) aHash[j] = 0;
    for(j=0; j<i; j++) aKWHash[j] = 0;
    for(j=0; j<nKeyword; j++){
      h = aKeywordTable[j].hash % i;
      aHash[h] *= 2;
      aHash[h]++;
      aKWHash[h] *= 2;
      aKWHash[h]++;
    }
    for(j=count=0; j<i; j++) count += aHash[j];
    for(j=count=0; j<i; j++) count += aKWHash[j];
    if( count<bestCount ){
      bestCount = count;
      bestSize = i;
    }
  }

  /* Compute the hash */
  for(i=0; i<bestSize; i++) aHash[i] = 0;
  for(i=0; i<bestSize; i++) aKWHash[i] = 0;
  for(i=0; i<nKeyword; i++){
    h = aKeywordTable[i].hash % bestSize;
    aKeywordTable[i].iNext = aHash[h];
    aHash[h] = i+1;
    aKeywordTable[i].iNext = aKWHash[h];
    aKWHash[h] = i+1;
  }

  /* Begin generating code */
  printf("%s", zHdr);
  printf("/* Hash score: %d */\n", bestCount);
  printf("static int keywordCode(const char *z, int n, int *pType){\n");
  printf("  /* zText[] encodes %d bytes of keywords in %d bytes */\n",
  printf("/* zKWText[] encodes %d bytes of keyword text in %d bytes */\n",
          totalLen + nKeyword, nChar+1 );
  for(i=j=k=0; i<nKeyword; i++){
    Keyword *p = &aKeywordTable[i];
    if( p->substrId ) continue;
    memcpy(&zText[k], p->zName, p->len);
    memcpy(&zKWText[k], p->zName, p->len);
    k += p->len;
    if( j+p->len>70 ){
      printf("%*s */\n", 74-j, "");
      j = 0;
    }
    if( j==0 ){
      printf("  /*   ");
      printf("/*   ");
      j = 8;
    }
    printf("%s", p->zName);
    j += p->len;
  }
  if( j>0 ){
    printf("%*s */\n", 74-j, "");
  }
  printf("  static const char zText[%d] = {\n", nChar);
  zText[nChar] = 0;
  printf("static const char zKWText[%d] = {\n", nChar);
  zKWText[nChar] = 0;
  for(i=j=0; i<k; i++){
    if( j==0 ){
      printf("    ");
      printf("  ");
    }
    if( zText[i]==0 ){
    if( zKWText[i]==0 ){
      printf("0");
    }else{
      printf("'%c',", zText[i]);
      printf("'%c',", zKWText[i]);
    }
    j += 4;
    if( j>68 ){
      printf("\n");
      j = 0;
    }
  }
  if( j>0 ) printf("\n");
  printf("  };\n");
  printf("};\n");

  printf("/* aKWHash[i] is the hash value for the i-th keyword */\n");
  printf("  static const unsigned char aHash[%d] = {\n", bestSize);
  printf("static const unsigned char aKWHash[%d] = {\n", bestSize);
  for(i=j=0; i<bestSize; i++){
    if( j==0 ) printf("    ");
    printf(" %3d,", aHash[i]);
    if( j==0 ) printf("  ");
    printf(" %3d,", aKWHash[i]);
    j++;
    if( j>12 ){
      printf("\n");
      j = 0;
    }
  }
  printf("%s  };\n", j==0 ? "" : "\n");    
  printf("%s};\n", j==0 ? "" : "\n");    

  printf("/* aKWNext[] forms the hash collision chain.  If aKWHash[i]==0\n");
  printf("** then the i-th keyword has no more hash collisions.  Otherwise,\n");
  printf("** the next keyword with the same hash is aKWHash[i]-1. */\n");
  printf("  static const unsigned char aNext[%d] = {\n", nKeyword);
  printf("static const unsigned char aKWNext[%d] = {\n", nKeyword);
  for(i=j=0; i<nKeyword; i++){
    if( j==0 ) printf("    ");
    if( j==0 ) printf("  ");
    printf(" %3d,", aKeywordTable[i].iNext);
    j++;
    if( j>12 ){
      printf("\n");
      j = 0;
    }
  }
  printf("%s  };\n", j==0 ? "" : "\n");    
  printf("%s};\n", j==0 ? "" : "\n");    

  printf("/* aKWLen[i] is the length (in bytes) of the i-th keyword */\n");
  printf("  static const unsigned char aLen[%d] = {\n", nKeyword);
  printf("static const unsigned char aKWLen[%d] = {\n", nKeyword);
  for(i=j=0; i<nKeyword; i++){
    if( j==0 ) printf("    ");
    if( j==0 ) printf("  ");
    printf(" %3d,", aKeywordTable[i].len+aKeywordTable[i].prefix);
    j++;
    if( j>12 ){
      printf("\n");
      j = 0;
    }
  }
  printf("%s  };\n", j==0 ? "" : "\n");    
  printf("%s};\n", j==0 ? "" : "\n");    

  printf("/* aKWOffset[i] is the index into zKWText[] of the start of\n");
  printf("** the text for the i-th keyword. */\n");
  printf("  static const unsigned short int aOffset[%d] = {\n", nKeyword);
  printf("static const unsigned short int aKWOffset[%d] = {\n", nKeyword);
  for(i=j=0; i<nKeyword; i++){
    if( j==0 ) printf("    ");
    if( j==0 ) printf("  ");
    printf(" %3d,", aKeywordTable[i].offset);
    j++;
    if( j>12 ){
      printf("\n");
      j = 0;
    }
  }
  printf("%s  };\n", j==0 ? "" : "\n");
  printf("%s};\n", j==0 ? "" : "\n");

  printf("/* aKWCode[i] is the parser symbol code for the i-th keyword */\n");
  printf("  static const unsigned char aCode[%d] = {\n", nKeyword);
  printf("static const unsigned char aKWCode[%d] = {\n", nKeyword);
  for(i=j=0; i<nKeyword; i++){
    char *zToken = aKeywordTable[i].zTokenType;
    if( j==0 ) printf("    ");
    if( j==0 ) printf("  ");
    printf("%s,%*s", zToken, (int)(14-strlen(zToken)), "");
    j++;
    if( j>=5 ){
      printf("\n");
      j = 0;
    }
  }
  printf("%s  };\n", j==0 ? "" : "\n");

  printf("%s};\n", j==0 ? "" : "\n");
  printf("/* Check to see if z[0..n-1] is a keyword. If it is, write the\n");
  printf("** parser symbol code for that keyword into *pType.  Always\n");
  printf("** return the integer n (the length of the token). */\n");
  printf("static int keywordCode(const char *z, int n, int *pType){\n");
  printf("  int i, j;\n");
  printf("  const char *zKW;\n");
  printf("  if( n>=2 ){\n");
  printf("    i = ((charMap(z[0])*4) ^ (charMap(z[n-1])*3) ^ n) %% %d;\n",
          bestSize);
  printf("    for(i=((int)aHash[i])-1; i>=0; i=((int)aNext[i])-1){\n");
  printf("      if( aLen[i]!=n ) continue;\n");
  printf("    for(i=((int)aKWHash[i])-1; i>=0; i=((int)aKWNext[i])-1){\n");
  printf("      if( aKWLen[i]!=n ) continue;\n");
  printf("      j = 0;\n");
  printf("      zKW = &zText[aOffset[i]];\n");
  printf("      zKW = &zKWText[aKWOffset[i]];\n");
  printf("#ifdef SQLITE_ASCII\n");
  printf("      while( j<n && (z[j]&~0x20)==zKW[j] ){ j++; }\n");
  printf("#endif\n");
  printf("#ifdef SQLITE_EBCDIC\n");
  printf("      while( j<n && toupper(z[j])==zKW[j] ){ j++; }\n");
  printf("#endif\n");
  printf("      if( j<n ) continue;\n");
  for(i=0; i<nKeyword; i++){
    printf("      testcase( i==%d ); /* %s */\n",
           i, aKeywordTable[i].zOrigName);
  }
  printf("      *pType = aCode[i];\n");
  printf("      *pType = aKWCode[i];\n");
  printf("      break;\n");
  printf("    }\n");
  printf("  }\n");
  printf("  return n;\n");
  printf("}\n");
  printf("int sqlite3KeywordCode(const unsigned char *z, int n){\n");
  printf("  int id = TK_ID;\n");