SQLite

/*
** 2012 April 10
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
**
** This module implements a VIRTUAL TABLE that can be used to search
** a large vocabulary for close matches.  For example, this virtual
** table can be used to suggest corrections to misspelled words.  Or,
** it could be used with FTS4 to do full-text search using potentially
** misspelled words.
**
** Create an instance of the virtual table this way:
**
**    CREATE VIRTUAL TABLE demo USING spellfix1;
**
** The "spellfix1" term is the name of this module.  The "demo" is the
** name of the virtual table you will be creating.  The table is initially
** empty.  You have to populate it with your vocabulary.  Suppose you
** have a list of words in a table named "big_vocabulary".  Then do this:
**
**    INSERT INTO demo(word) SELECT word FROM big_vocabulary;
**
** If you intend to use this virtual table in cooperation with an FTS4
** table (for spelling correctly of search terms) then you can extract
** the vocabulary using an fts3aux table:
**
**    INSERT INTO demo(word) SELECT term FROM search_aux WHERE col='*';
**
** You can also provide the virtual table with a "rank" for each word.
** The "rank" is an estimate of how common the word is.  Larger numbers
** mean the word is more common.  If you omit the rank when populating
** the table, then a rank of 1 is assumed.  But if you have rank 
** information, you can supply it and the virtual table will show a
** slight preference for selecting more commonly used terms.  To
** populate the rank from an fts4aux table "search_aux" do something
** like this:
**
**    INSERT INTO demo(word,rank)
**        SELECT term, documents FROM search_aux WHERE col='*';
**
** To query the virtual table, include a MATCH operator in the WHERE
** clause.  For example:
**
**    SELECT word FROM demo WHERE word MATCH 'kennasaw';
**
** Using a dataset of American place names (derived from
** http://geonames.usgs.gov/domestic/download_data.htm) the query above
** returns 20 results beginning with:
**
**    kennesaw
**    kenosha
**    kenesaw
**    kenaga
**    keanak
**
** If you append the character '*' to the end of the pattern, then
** a prefix search is performed.  For example:
**
**    SELECT word FROM demo WHERE word MATCH 'kennes*';
**
** Yields 20 results beginning with:
**
**    kennesaw
**    kennestone
**    kenneson
**    kenneys
**    keanes
**    keenes
**
** The virtual table actually has a unique rowid with five columns plus three
** extra hidden columns.  The columns are as follows:
**
**    rowid         A unique integer number associated with each
**                  vocabulary item in the table.  This can be used
**                  as a foreign key on other tables in the database.
**
**    word          The text of the word that matches the pattern.
**                  Both word and pattern can contains unicode characters
**                  and can be mixed case.
**
**    rank          This is the rank of the word, as specified in the
**                  original INSERT statement.
**
**    distance      This is an edit distance or Levensthein distance going
**                  from the pattern to the word.
**
**    langid        This is the language-id of the word.  All queries are
**                  against a single language-id, which defaults to 0.
**                  For any given query this value is the same on all rows.
**
**    score         The score is a combination of rank and distance.  The
**                  idea is that a lower score is better.  The virtual table
**                  attempts to find words with the lowest score and 
**                  by default (unless overridden by ORDER BY) returns
**                  results in order of increasing score.
**
**    matchlen      For prefix queries, the number of characters in the prefix
**                  of the returned value (word) that matched the query term.
**                  For non-prefix queries, the number of characters in the 
**                  returned value.
**
**    top           (HIDDEN)  For any query, this value is the same on all
**                  rows.  It is an integer which is the maximum number of
**                  rows that will be output.  The actually number of rows
**                  output might be less than this number, but it will never
**                  be greater.  The default value for top is 20, but that
**                  can be changed for each query by including a term of
**                  the form "top=N" in the WHERE clause of the query.
**
**    scope         (HIDDEN)  For any query, this value is the same on all
**                  rows.  The scope is a measure of how widely the virtual
**                  table looks for matching words.  Smaller values of
**                  scope cause a broader search.  The scope is normally
**                  choosen automatically and is capped at 4.  Applications
**                  can change the scope by including a term of the form
**                  "scope=N" in the WHERE clause of the query.  Increasing
**                  the scope will make the query run faster, but will reduce
**                  the possible corrections.
**
**    srchcnt       (HIDDEN)  For any query, this value is the same on all
**                  rows.  This value is an integer which is the number of
**                  of words examined using the edit-distance algorithm to
**                  find the top matches that are ultimately displayed.  This
**                  value is for diagnostic use only.
**
**    soundslike    (HIDDEN)  When inserting vocabulary entries, this field
**                  can be set to an spelling that matches what the word
**                  sounds like.  See the DEALING WITH UNUSUAL AND DIFFICULT
**                  SPELLINGS section below for details.
**
** When inserting into or updating the virtual table, only the rowid, word,
** rank, and langid may be changes.  Any attempt to set or modify the values
** of distance, score, top, scope, or srchcnt is silently ignored.
**
** ALGORITHM
**
** A shadow table named "%_vocab" (where the % is replaced by the name of
** the virtual table; Ex: "demo_vocab" for the "demo" virtual table) is
** constructed with these columns:
**
**    id            The unique id (INTEGER PRIMARY KEY)
**
**    rank          The rank of word.
**
**    langid        The language id for this entry.
**
**    word          The original UTF8 text of the vocabulary word
**
**    k1            The word transliterated into lower-case ASCII.  
**                  There is a standard table of mappings from non-ASCII
**                  characters into ASCII.  Examples: "æ" -> "ae",
**                  "þ" -> "th", "ß" -> "ss", "á" -> "a", ...  The
**                  accessory function spellfix1_translit(X) will do
**                  the non-ASCII to ASCII mapping.  The built-in lower(X)
**                  function will convert to lower-case.  Thus:
**                  k1 = lower(spellfix1_translit(word)).
**
**    k2            This field holds a phonetic code derived from k1.  Letters
**                  that have similar sounds are mapped into the same symbol.
**                  For example, all vowels and vowel clusters become the
**                  single symbol "A".  And the letters "p", "b", "f", and
**                  "v" all become "B".  All nasal sounds are represented
**                  as "N".  And so forth.  The mapping is base on
**                  ideas found in Soundex, Metaphone, and other
**                  long-standing phonetic matching systems.  This key can
**                  be generated by the function spellfix1_phonehash(X).  
**                  Hence: k2 = spellfix1_phonehash(k1)
**
** There is also a function for computing the Wagner edit distance or the
** Levenshtein distance between a pattern and a word.  This function
** is exposed as spellfix1_editdist(X,Y).  The edit distance function
** returns the "cost" of converting X into Y.  Some transformations
** cost more than others.  Changing one vowel into a different vowel,
** for example is relatively cheap, as is doubling a constant, or
** omitting the second character of a double-constant.  Other transformations
** or more expensive.  The idea is that the edit distance function returns
** a low cost of words that are similar and a higher cost for words
** that are futher apart.  In this implementation, the maximum cost
** of any single-character edit (delete, insert, or substitute) is 100,
** with lower costs for some edits (such as transforming vowels).
**
** The "score" for a comparison is the edit distance between the pattern
** and the word, adjusted down by the base-2 logorithm of the word rank.
** For example, a match with distance 100 but rank 1000 would have a
** score of 122 (= 100 - log2(1000) + 32) where as a match with distance
** 100 with a rank of 1 would have a score of 131 (100 - log2(1) + 32).
** (NB:  The constant 32 is added to each score to keep it from going
** negative in case the edit distance is zero.)  In this way, frequently
** used words get a slightly lower cost which tends to move them toward
** the top of the list of alternative spellings.
**
** A straightforward implementation of a spelling corrector would be
** to compare the search term against every word in the vocabulary
** and select the 20 with the lowest scores.  However, there will 
** typically be hundreds of thousands or millions of words in the
** vocabulary, and so this approach is not fast enough.
**
** Suppose the term that is being spell-corrected is X.  To limit
** the search space, X is converted to a k2-like key using the
** equivalent of:
**
**    key = spellfix1_phonehash(lower(spellfix1_translit(X)))
**
** This key is then limited to "scope" characters.  The default scope
** value is 4, but an alternative scope can be specified using the
** "scope=N" term in the WHERE clause.  After the key has been truncated,
** the edit distance is run against every term in the vocabulary that
** has a k2 value that begins with the abbreviated key.
**
** For example, suppose the input word is "Paskagula".  The phonetic 
** key is "BACACALA" which is then truncated to 4 characters "BACA".
** The edit distance is then run on the 4980 entries (out of
** 272,597 entries total) of the vocabulary whose k2 values begin with
** BACA, yielding "Pascagoula" as the best match.
** 
** Only terms of the vocabulary with a matching langid are searched.
** Hence, the same table can contain entries from multiple languages
** and only the requested language will be used.  The default langid
** is 0.
**
** DEALING WITH UNUSUAL AND DIFFICULT SPELLINGS
**
** The algorithm above works quite well for most cases, but there are
** exceptions.  These exceptions can be dealt with by making additional
** entries in the virtual table using the "soundslike" column.
**
** For example, many words of Greek origin begin with letters "ps" where
** the "p" is silent.  Ex:  psalm, pseudonym, psoriasis, psyche.  In
** another example, many Scottish surnames can be spelled with an
** initial "Mac" or "Mc".  Thus, "MacKay" and "McKay" are both pronounced
** the same.
**
** Accommodation can be made for words that are not spelled as they
** sound by making additional entries into the virtual table for the
** same word, but adding an alternative spelling in the "soundslike"
** column.  For example, the canonical entry for "psalm" would be this:
**
**   INSERT INTO demo(word) VALUES('psalm');
**
** To enhance the ability to correct the spelling of "salm" into
** "psalm", make an addition entry like this:
**
**   INSERT INTO demo(word,soundslike) VALUES('psalm','salm');
**
** It is ok to make multiple entries for the same word as long as
** each entry has a different soundslike value.  Note that if no
** soundslike value is specified, the soundslike defaults to the word
** itself.
**
** Listed below are some cases where it might make sense to add additional
** soundslike entries.  The specific entries will depend on the application
** and the target language.
**
**   *   Silent "p" in words beginning with "ps":  psalm, psyche
**
**   *   Silent "p" in words beginning with "pn":  pneumonia, pneumatic
**
**   *   Silent "p" in words beginning with "pt":  pterodactyl, ptolemaic
**
**   *   Silent "d" in words beginning with "dj":  djinn, Djikarta
**
**   *   Silent "k" in words beginning with "kn":  knight, Knuthson
**
**   *   Silent "g" in words beginning with "gn":  gnarly, gnome, gnat
**
**   *   "Mac" versus "Mc" beginning Scottish surnames
**
**   *   "Tch" sounds in Slavic words:  Tchaikovsky vs. Chaykovsky
**
**   *   The letter "j" pronounced like "h" in Spanish:  LaJolla
**
**   *   Words beginning with "wr" versus "r":  write vs. rite
**
**   *   Miscellanous problem words such as "debt", "tsetse",
**       "Nguyen", "Van Nuyes".
*/
#if SQLITE_CORE
# include "sqliteInt.h"
#else
# include <string.h>
# include <stdio.h>
# include <stdlib.h>

**   4   'D'       Alveolar stops:  D T
**   5   'H'       Letter H at the beginning of a word
**   6   'L'       Glide:  L
**   7   'R'       Semivowel:  R
**   8   'M'       Nasals:  M N
**   9   'W'       Letter W at the beginning of a word
**   10  'Y'       Letter Y at the beginning of a word.
**   11  '9'       A digit: 0 1 2 3 4 5 6 7 8 9
**   12  ' '       White space
**   13  '?'       Other.
*/
#define CCLASS_SILENT         0
#define CCLASS_VOWEL          1
#define CCLASS_B              2
#define CCLASS_C              3

      case 'g': 
      case 'k': {
        if( zIn[1]=='n' ){ zIn++; nIn--; }
        break;
      }
    }
  }
  if( zIn[0]=='k' && zIn[1]=='n' ){ zIn++, nIn--; }
  for(i=0; i<nIn; i++){
    unsigned char c = zIn[i];
    if( i+1<nIn ){
      if( c=='w' && zIn[i+1]=='r' ) continue;
      if( c=='d' && (zIn[i+1]=='j' || zIn[i+1]=='g') ) continue;
      if( i+2<nIn ){
        if( c=='t' && zIn[i+1]=='c' && zIn[i+2]=='h' ) continue;

    /* differ only in case */
    return 0;
  }
  classFrom = characterClass(cPrev, cFrom);
  classTo = characterClass(cPrev, cTo);
  if( classFrom==classTo ){
    /* Same character class */
    return classFrom=='A' ? 25 : 40;
  }
  if( classFrom>=CCLASS_B && classFrom<=CCLASS_Y
      && classTo>=CCLASS_B && classTo<=CCLASS_Y ){
    /* Convert from one consonant to another, but in a different class */
    return 75;
  }
  /* Any other subsitution */

**
** If pnMatch is not NULL, then *pnMatch is set to the number of bytes
** of zB that matched the pattern in zA. If zA does not end with a '*',
** then this value is always the number of bytes in zB (i.e. strlen(zB)).
** If zA does end in a '*', then it is the number of bytes in the prefix
** of zB that was deemed to match zA.
*/
static int editdist1(const char *zA, const char *zB, int iLangId, int *pnMatch){
  int nA, nB;            /* Number of characters in zA[] and zB[] */
  int xA, xB;            /* Loop counters for zA[] and zB[] */
  char cA, cB;           /* Current character of zA and zB */
  char cAprev, cBprev;   /* Previous character of zA and zB */
  char cAnext, cBnext;   /* Next character in zA and zB */
  int d;                 /* North-west cost value */
  int dc = 0;            /* North-west character value */

#if 0
  printf("A=\"%s\" B=\"%s\" dc=%c\n", zA, zB, dc?dc:' ');
#endif

  /* Verify input strings and measure their lengths */
  for(nA=0; zA[nA]; nA++){
    if( zA[nA]>127 ) return -2;
  }
  for(nB=0; zB[nB]; nB++){
    if( zB[nB]>127 ) return -2;
  }

  /* Special processing if either string is empty */
  if( nA==0 ){
    cBprev = dc;
    for(xB=res=0; (cB = zB[xB])!=0; xB++){
      res += insertOrDeleteCost(cBprev, cB, zB[xB+1])/FINAL_INS_COST_DIV;

      if( m[xB]<res ){
        res = m[xB];
        if( pnMatch ) *pnMatch = xB+nMatch;
      }
    }
  }else{
    res = m[nB];


    if( pnMatch ) *pnMatch = -1;
  }
  sqlite3_free(toFree);
  return res;
}

/*
** Function:    editdist(A,B)
**              editdist(A,B,langid)
**
** Return the cost of transforming string A into string B.  Both strings
** must be pure ASCII text.  If A ends with '*' then it is assumed to be
** a prefix of B and extra characters on the end of B have minimal additional
** cost.
*/
static void editdistSqlFunc(
  sqlite3_context *context,
  int argc,
  sqlite3_value **argv
){
  int langid = argc==2 ? 0 : sqlite3_value_int(argv[2]);
  int res = editdist1(
                    (const char*)sqlite3_value_text(argv[0]),
                    (const char*)sqlite3_value_text(argv[1]),
                    langid, 0);
  if( res<0 ){
    if( res==(-3) ){
      sqlite3_result_error_nomem(context);
    }else if( res==(-2) ){
      sqlite3_result_error(context, "non-ASCII input to editdist()", -1);
    }else{
      sqlite3_result_error(context, "NULL input to editdist()", -1);

*/
static int editDist3ConfigLoad(
  EditDist3Config *p,      /* The edit distance configuration to load */
  sqlite3 *db,            /* Load from this database */
  const char *zTable      /* Name of the table from which to load */
){
  sqlite3_stmt *pStmt;
  int rc;
  char *zSql;
  int iLangPrev = -9999;
  EditDist3Lang *pLang;

  zSql = sqlite3_mprintf("SELECT iLang, cFrom, cTo, iCost"
                         " FROM \"%w\" WHERE iLang>=0 ORDER BY iLang", zTable);
  if( zSql==0 ) return SQLITE_NOMEM;
  rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0);
  sqlite3_free(zSql);
  if( rc ) return rc;
  editDist3ConfigClear(p);
  while( sqlite3_step(pStmt)==SQLITE_ROW ){
    int iLang = sqlite3_column_int(pStmt, 0);
    const char *zFrom = (const char*)sqlite3_column_text(pStmt, 1);
    int nFrom = sqlite3_column_bytes(pStmt, 1);
    const char *zTo = (const char*)sqlite3_column_text(pStmt, 2);
    int nTo = sqlite3_column_bytes(pStmt, 2);
    int iCost = sqlite3_column_int(pStmt, 3);



    if( nFrom>100 || nFrom<0 || nTo>100 || nTo<0 ) continue;
    if( iCost<0 ) continue;
    if( iLang!=iLangPrev ){
      EditDist3Lang *pNew;
      p->nLang++;
      pNew = sqlite3_realloc(p->a, p->nLang*sizeof(p->a[0]));
      if( pNew==0 ){ rc = SQLITE_NOMEM; break; }
      p->a = pNew;
      pLang = &p->a[p->nLang-1];

      pLang->iLang = iLang;
      pLang->iInsCost = 100;
      pLang->iDelCost = 100;
      pLang->iSubCost = 200;
      pLang->pCost = 0;
      iLangPrev = iLang;
    }
    if( nFrom==1 && zFrom[0]=='?' && nTo==0 ){
      pLang->iDelCost = iCost;
    }else if( nFrom==0 && nTo==1 && zTo[0]=='?' ){
      pLang->iInsCost = iCost;

      pCost->iCost = iCost;
      memcpy(pCost->a, zFrom, nFrom);
      memcpy(pCost->a + nFrom, zTo, nTo);
      pCost->pNext = pLang->pCost;
      pLang->pCost = pCost; 
    }
  }
  sqlite3_finalize(pStmt);

  return rc;
}

/*
** Return the length (in bytes) of a utf-8 character.  Or return a maximum
** of N.
*/

}

/*
** Return TRUE (non-zero) of the To side of the given cost matches
** the given string.
*/
static int matchFrom(EditDist3Cost *p, const char *z, int n){
  if( p->nFrom>n ) return 0;
  if( memcmp(p->a, z, p->nFrom)!=0 ) return 0;
  return 1;
}

/*
** Return TRUE (non-zero) of the next FROM character and the next TO
** character are the same.

  const char *z,
  int n
){
  EditDist3FromString *pStr;
  EditDist3Cost *p;
  int i;


  if( n<0 ) n = (int)strlen(z);
  pStr = sqlite3_malloc( sizeof(*pStr) + sizeof(pStr->a[0])*n + n + 1 );
  if( pStr==0 ) return 0;
  pStr->a = (EditDist3From*)&pStr[1];

  pStr->n = n;
  pStr->z = (char*)&pStr->a[n];
  memcpy(pStr->z, z, n+1);
  if( n && z[n-1]=='*' ){
    pStr->isPrefix = 1;
    n--;
    pStr->n--;

      pStr = 0;
      break;
    }
  }
  return pStr;
}

#if 0 /* No longer used */
/*
** Return the number of bytes in the common prefix of two UTF8 strings.
** Only complete characters are considered.
*/
static int editDist3PrefixLen(const char *z1, const char *z2){
  int n = 0;
  while( z1[n] && z1[n]==z2[n] ){ n++; }
  while( n && (z1[n]&0xc0)==0x80 ){ n--; }
  return n;
}

/*
** Return the number of bytes in the common suffix of two UTF8 strings.
** Only complete characters are considered.
*/
static int editDist3SuffixLen(const char *z1, int n1, const char *z2, int n2){
  int origN1 = n1;
  while( n1>0 && n2>0 && z1[n1-1]==z2[n2-1] ){ n1--; n2--; }
  while( n1<origN1 && (z1[n1]&0xc0)==0x80 ){ n1++; n2++; }
  return origN1 - n1;
}
#endif /* 0 */

/*
** Update entry m[i] such that it is the minimum of its current value
** and m[j]+iCost.
**
** If the iCost is 1,000,000 or greater, then consider the cost to be
** infinite and skip the update.
*/

  EditDist3FromString f = *pFrom;
  EditDist3To *a2;
  unsigned int *m;
  int szRow;
  EditDist3Cost *p;
  int res;

#if 0
  /* Remove comment prefix and suffix */
  n = editDist3PrefixLen(f.z, z2);
  if( f.n==n2 && n2==n ) return 0;  /* Identical strings */
  f.n -= n;
  f.z += n;
  f.a += n;
  n2 -= n;
  z2 += n;
  if( f.isPrefix==0 ){
    n = editDist3SuffixLen(f.z, f.n, z2, n2);
    f.n -= n;
    n2 -= n;
  }
#endif

  /* allocate the Wagner matrix and the aTo[] array for the TO string */
  n = (f.n+1)*(n2+1);
  n = (n+1)&~1;
  m = sqlite3_malloc( n*sizeof(m[0]) + sizeof(a2[0])*n2 );
  if( m==0 ) return -1;            /* Out of memory */
  a2 = (EditDist3To*)&m[n];
  memset(a2, 0, sizeof(a2[0])*n2);

        if( matchTo(p, z2+i2, n2-i2) ){
          updateCost(m, cxd+p->nFrom+szRow*p->nTo, cxd, p->iCost);
        }
      }
    }
  }

#if 0
  printf("         ^");
  for(i1=0; i1<f.n; i1++) printf(" %c-%2x", f.z[i1], f.z[i1]&0xff);
  printf("\n   ^:");
  for(i1=0; i1<szRow; i1++){
    int v = m[i1];
    if( v>9999 ) printf(" ****");
    else         printf(" %4d", v);

    pFrom = editDist3FromStringNew(pLang, zA, nA);
    if( pFrom==0 ){
      sqlite3_result_error_nomem(context);
      return;
    }
    dist = editDist3Core(pFrom, zB, nB, pLang, 0);
    editDist3FromStringDelete(pFrom);



    sqlite3_result_int(context, dist);

  } 
}

/*
** Register the editDist3 function with SQLite
*/
static int editDist3Install(sqlite3 *db){

/*
** Return the value of the first UTF-8 character in the string.
*/
static int utf8Read(const unsigned char *z, int n, int *pSize){
  int c, i;



  if( n==0 ){
    c = i = 0;
  }else{
    c = z[0];
    i = 1;
    if( c>=0xc0 ){
      c = sqlite3Utf8Trans1[c-0xc0];
      while( i<n && (z[i] & 0xc0)==0x80 ){

** The returned string might contain more characters than the input.
**
** Space to hold the returned string comes from sqlite3_malloc() and
** should be freed by the caller.
*/
static unsigned char *transliterate(const unsigned char *zIn, int nIn){
  unsigned char *zOut = sqlite3_malloc( nIn*4 + 1 );
  int i, c, sz, nOut;
  if( zOut==0 ) return 0;
  i = nOut = 0;
  while( i<nIn ){
    c = utf8Read(zIn, nIn, &sz);
    zIn += sz;
    nIn -= sz;
    if( c<=127 ){
      zOut[nOut++] = c;
    }else{
      int xTop, xBtm, x;

  }
  sqlite3_result_int(context, res);
}

/* End transliterate
******************************************************************************
******************************************************************************
** Begin Polloc & Zamora SPEEDCOP style keying functions.
*/
/*
** The Pollock & Zamora skeleton function.  Move all consonants to the
** front and all vowels to the end, removing duplicates.  Except if the
** first letter is a vowel then it remains as the first letter.
*/
static void pollockSkeletonKey(const char *zIn, char *zOut){
  int i, j;
  unsigned char c;
  char seen[26];
  static const unsigned char isVowel[] = { 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0,
    0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0 };
  memset(seen, 0, sizeof(seen));
  for(i=j=0; (c = (unsigned char)zIn[i])!=0; i++){
    if( c<'a' || c>'z' ) continue;
    if( j>0 || isVowel[c-'a'] ) continue;
    if( seen[c-'a'] ) continue;
    seen[c-'a'] = 1;
    zOut[j++] = c;
  }
  for(i=0; (c = (unsigned char)zIn[i])!=0; i++){
    if( c<'a' || c>'z' ) continue;
    if( seen[c-'a'] ) continue;
    if( !isVowel[c-'a'] ) continue;
    seen[c-'a'] = 1;
    zOut[j++] = c;
  }
  zOut[j] = 0;
}

/*
** Function:    pollock_skeleton(X)
**
** Return the Pollock and Zamora skeleton key for a string X of all
** lower-case letters.
*/
static void pollockSkeletonSqlFunc(
  sqlite3_context *context,
  int argc,
  sqlite3_value **argv
){
  const char *zIn = (const char*)sqlite3_value_text(argv[0]);
  int nIn = sqlite3_value_bytes(argv[0]);
  char *zOut;
  if( zIn ){
    zOut = sqlite3_malloc( nIn + 1 );
    if( zOut==0 ){
      sqlite3_result_error_nomem(context);
    }else{
      pollockSkeletonKey(zIn, zOut);
      sqlite3_result_text(context, (char*)zOut, -1, sqlite3_free);
    }
  }
}  

/*
** The Pollock & Zamora omission key.
**
** The key consists of unique consonants in the following order:
**
**         jkqxzvwybfmgpdhclntsr
**
** These are followed by unique vowels in input order.
*/
static void pollockOmissionKey(const char *zIn, char *zOut){
  int i, j;
  unsigned char c;
  char seen[26];
  static const unsigned char isVowel[] = { 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0,
    0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0 };
  static const unsigned char constOrder[] = "jkqxzvwybfmgpdhclntsr";

  memset(seen, 0, sizeof(seen));
  for(i=j=0; (c = (unsigned char)zIn[i])!=0; i++){
    if( c<'a' || c>'z' ) continue;
    if( isVowel[c-'a'] ) continue;
    if( seen[c-'a'] ) continue;
    seen[c-'a'] = 1;
  }
  for(i=0; (c = constOrder[i])!=0; i++){
    if( seen[c-'a'] ) zOut[j++] = c;
  }
  for(i=0; (c = (unsigned char)zIn[i])!=0; i++){
    if( c<'a' || c>'z' ) continue;
    if( seen[c-'a'] ) continue;
    if( !isVowel[c-'a'] ) continue;
    seen[c-'a'] = 1;
    zOut[j++] = c;
  }
  zOut[j] = 0;
}

/*
** Function:    pollock_omission(X)
**
** Return the Pollock and Zamora omission key for a string X of all
** lower-case letters.
*/
static void pollockOmissionSqlFunc(
  sqlite3_context *context,
  int argc,
  sqlite3_value **argv
){
  const char *zIn = (const char*)sqlite3_value_text(argv[0]);
  int nIn = sqlite3_value_bytes(argv[0]);
  char *zOut;
  if( zIn ){
    zOut = sqlite3_malloc( nIn + 1 );
    if( zOut==0 ){
      sqlite3_result_error_nomem(context);
    }else{
      pollockOmissionKey(zIn, zOut);
      sqlite3_result_text(context, (char*)zOut, -1, sqlite3_free);
    }
  }
}  


/* End SPEEDCOP keying functions
******************************************************************************
******************************************************************************
** Begin spellfix1 virtual table.
*/

/* Maximum length of a phonehash used for querying the shadow table */
#define SPELLFIX_MX_HASH  8

/* Maximum number of hash strings to examine per query */
#define SPELLFIX_MX_RUN   8

typedef struct spellfix1_vtab spellfix1_vtab;
typedef struct spellfix1_cursor spellfix1_cursor;

/* Fuzzy-search virtual table object */
struct spellfix1_vtab {
  sqlite3_vtab base;         /* Base class - must be first */

struct spellfix1_cursor {
  sqlite3_vtab_cursor base;    /* Base class - must be first */
  spellfix1_vtab *pVTab;       /* The table to which this cursor belongs */
  char *zPattern;              /* rhs of MATCH clause */
  int nRow;                    /* Number of rows of content */
  int nAlloc;                  /* Number of allocated rows */
  int iRow;                    /* Current row of content */
  int iLang;                   /* Value of the lang= constraint */
  int iTop;                    /* Value of the top= constraint */
  int iScope;                  /* Value of the scope= constraint */
  int nSearch;                 /* Number of vocabulary items checked */

  struct spellfix1_row {       /* For each row of content */
    sqlite3_int64 iRowid;         /* Rowid for this row */
    char *zWord;                  /* Text for this row */
    int iRank;                    /* Rank for this row */
    int iDistance;                /* Distance from pattern for this row */
    int iScore;                   /* Score for sorting */
    int iMatchlen;                /* Value of matchlen column (or -1) */

  char *zOut;
  int i, j;
  char c;
  while( isspace(zIn[0]) ) zIn++;
  zOut = sqlite3_mprintf("%s", zIn);
  if( zOut==0 ) return 0;
  i = (int)strlen(zOut);

  while( i>0 && isspace(zOut[i-1]) ){ i--; }

  zOut[i] = 0;
  c = zOut[0];
  if( c=='\'' || c=='"' ){
    for(i=1, j=0; zOut[i]; i++){
      zOut[j++] = zOut[i];
      if( zOut[i]==c ){
        if( zOut[i+1]==c ){
          i++;
        }else{
          zOut[j-1] = 0;
          break;

  const char *zModule = argv[0];
  const char *zDbName = argv[1];
  const char *zTableName = argv[2];
  int nDbName;
  int rc = SQLITE_OK;
  int i;

  if( argc<3 ){
    *pzErr = sqlite3_mprintf(
        "%s: wrong number of CREATE VIRTUAL TABLE arguments", argv[0]
    );
    rc = SQLITE_ERROR;
  }else{
    nDbName = strlen(zDbName);
    pNew = sqlite3_malloc( sizeof(*pNew) + nDbName + 1);
    if( pNew==0 ){
      rc = SQLITE_NOMEM;
    }else{
      memset(pNew, 0, sizeof(*pNew));
      pNew->zDbName = (char*)&pNew[1];
      memcpy(pNew->zDbName, zDbName, nDbName+1);
      pNew->zTableName = sqlite3_mprintf("%s", zTableName);
      pNew->db = db;
      if( pNew->zTableName==0 ){
        rc = SQLITE_NOMEM;
      }else{
        rc = sqlite3_declare_vtab(db, 
             "CREATE TABLE x(word,rank,distance,langid, "
             "score, matchlen, phonehash, "
             "top HIDDEN, scope HIDDEN, srchcnt HIDDEN, "
             "soundslike HIDDEN, command HIDDEN)"
        );
#define SPELLFIX_COL_WORD            0
#define SPELLFIX_COL_RANK            1
#define SPELLFIX_COL_DISTANCE        2
#define SPELLFIX_COL_LANGID          3
#define SPELLFIX_COL_SCORE           4
#define SPELLFIX_COL_MATCHLEN        5
#define SPELLFIX_COL_PHONEHASH       6
#define SPELLFIX_COL_TOP             7
#define SPELLFIX_COL_SCOPE           8
#define SPELLFIX_COL_SRCHCNT         9
#define SPELLFIX_COL_SOUNDSLIKE     10
#define SPELLFIX_COL_COMMAND        11
      }
      if( rc==SQLITE_OK && isCreate ){
        sqlite3_uint64 r;
        spellfix1DbExec(&rc, db,
           "CREATE TABLE IF NOT EXISTS \"%w\".\"%w_vocab\"(\n"
           "  id INTEGER PRIMARY KEY,\n"
           "  rank INT,\n"
           "  langid INT,\n"
           "  word TEXT,\n"
           "  k1 TEXT,\n"
           "  k2 TEXT\n"
           ");\n",
           zDbName, zTableName
        );
        sqlite3_randomness(sizeof(r), &r);
        spellfix1DbExec(&rc, db,
           "CREATE INDEX IF NOT EXISTS \"%w\".\"%w_index_%llx\" "
              "ON \"%w_vocab\"(langid,k2);",
           zDbName, zModule, r, zTableName
        );
      }
      for(i=3; rc==SQLITE_OK && i<argc; i++){
        if( memcmp(argv[i],"edit_cost_table=",16)==0 && pNew->zCostTable==0 ){
          pNew->zCostTable = spellfix1Dequote(&argv[i][16]);
          if( pNew->zCostTable==0 ) rc = SQLITE_NOMEM;
          continue;
        }

        rc = SQLITE_ERROR; 
      }
    }
  }




  *ppVTab = (sqlite3_vtab *)pNew;

  return rc;
}

/*
** The xConnect and xCreate methods
*/
static int spellfix1Connect(

  int i;
  for(i=0; i<pCur->nRow; i++){
    sqlite3_free(pCur->a[i].zWord);
  }
  pCur->nRow = 0;
  pCur->iRow = 0;
  pCur->nSearch = 0;




}

/*
** Resize the cursor to hold up to N rows of content
*/
static void spellfix1ResizeCursor(spellfix1_cursor *pCur, int N){
  struct spellfix1_row *aNew;

      iDistTerm = i;
    }
  }
  if( iPlan&1 ){
    int idx = 2;
    pIdxInfo->idxNum = iPlan;
    if( pIdxInfo->nOrderBy==1
     && pIdxInfo->aOrderBy[0].iColumn==4
     && pIdxInfo->aOrderBy[0].desc==0
    ){
      pIdxInfo->orderByConsumed = 1;  /* Default order by iScore */
    }
    if( iPlan&2 ){
      pIdxInfo->aConstraintUsage[iLangTerm].argvIndex = idx++;
      pIdxInfo->aConstraintUsage[iLangTerm].omit = 1;

  int iScope = p->iScope;
  spellfix1_cursor *pCur = p->pCur;
  sqlite3_stmt *pStmt = p->pStmt;
  char zHash1[SPELLFIX_MX_HASH];
  char zHash2[SPELLFIX_MX_HASH];
  char *zClass;
  int nClass;


  if( pCur->a==0 || p->rc ) return;   /* Prior memory allocation failure */
  if( p->nRun>=SPELLFIX_MX_RUN ) return;
  zClass = (char*)phoneticHash((unsigned char*)zQuery, nQuery);
  if( zClass==0 ){
    p->rc = SQLITE_NOMEM;
    return;
  }
  nClass = strlen(zClass);
  if( nClass>SPELLFIX_MX_HASH-2 ){

  }
  memcpy(zHash1, zClass, iScope);
  sqlite3_free(zClass);
  zHash1[iScope] = 0;
  memcpy(zHash2, zHash1, iScope);
  zHash2[iScope] = 'Z';
  zHash2[iScope+1] = 0;

  for(i=0; i<p->nRun; i++){
    if( strcmp(p->azPrior[i], zHash1)==0 ) return;
  }


  memcpy(p->azPrior[p->nRun++], zHash1, iScope+1);
  sqlite3_bind_text(pStmt, 1, zHash1, -1, SQLITE_STATIC);
  sqlite3_bind_text(pStmt, 2, zHash2, -1, SQLITE_STATIC);





  for(i=0; i<pCur->nRow; i++){
    if( pCur->a[i].iScore>iWorst ){
      iWorst = pCur->a[i].iScore;
      idxWorst = i;
    }
  }

  while( sqlite3_step(pStmt)==SQLITE_ROW ){
    int iMatchlen = -1;
    iRank = sqlite3_column_int(pStmt, 2);
    if( p->pMatchStr3 ){
      int nWord = sqlite3_column_bytes(pStmt, 1);
      zWord = (const char*)sqlite3_column_text(pStmt, 1);
      iDist = editDist3Core(p->pMatchStr3, zWord, nWord, p->pLang, &iMatchlen);
    }else{
      zK1 = (const char*)sqlite3_column_text(pStmt, 3);
      if( zK1==0 ) continue;
      iDist = editdist1(p->zPattern, zK1, pCur->iLang, 0);




    }
    pCur->nSearch++;
    iScore = spellfix1Score(iDist,iRank);
    if( p->iMaxDist>=0 ){
      if( iDist>p->iMaxDist ) continue;
      if( pCur->nRow>=pCur->nAlloc-1 ){
        spellfix1ResizeCursor(pCur, pCur->nAlloc*2 + 10);
        if( pCur->a==0 ) break;
      }
      idx = pCur->nRow;
    }else if( pCur->nRow<pCur->nAlloc ){
      idx = pCur->nRow;
    }else if( iScore<iWorst ){
      idx = idxWorst;
      sqlite3_free(pCur->a[idx].zWord);
    }else{
      continue;
    }
    pCur->a[idx].zWord = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1));




    pCur->a[idx].iRowid = sqlite3_column_int64(pStmt, 0);
    pCur->a[idx].iRank = iRank;
    pCur->a[idx].iDistance = iDist;
    pCur->a[idx].iScore = iScore;
    pCur->a[idx].iMatchlen = iMatchlen;
    memcpy(pCur->a[idx].zHash, zHash1, iScope+1);
    if( pCur->nRow<pCur->nAlloc ) pCur->nRow++;
    if( pCur->nRow==pCur->nAlloc ){
      iWorst = pCur->a[0].iScore;
      idxWorst = 0;
      for(i=1; i<pCur->nRow; i++){
        iScore = pCur->a[i].iScore;
        if( iWorst<iScore ){
          iWorst = iScore;
          idxWorst = i;
        }
      }
    }
  }
  sqlite3_reset(pStmt);

}

/*
** This version of the xFilter method work if the MATCH term is present
** and we are doing a scan.
*/
static int spellfix1FilterForMatch(
  spellfix1_cursor *pCur,
  int idxNum,
  int argc,
  sqlite3_value **argv
){
  const unsigned char *zMatchThis;   /* RHS of the MATCH operator */
  EditDist3FromString *pMatchStr3 = 0; /* zMatchThis as an editdist string */
  char *zPattern;                    /* Transliteration of zMatchThis */
  int nPattern;                      /* Length of zPattern */
  int iLimit = 20;                   /* Max number of rows of output */
  int iScope = 3;                    /* Use this many characters of zClass */
  int iLang = 0;                     /* Language code */
  char *zSql;                        /* SQL of shadow table query */
  sqlite3_stmt *pStmt;               /* Shadow table query */
  int rc;                            /* Result code */
  int idx = 1;                       /* Next available filter parameter */
  spellfix1_vtab *p = pCur->pVTab;   /* The virtual table that owns pCur */
  MatchQuery x;                      /* For passing info to RunQuery() */

  /* Load the cost table if we have not already done so */
  if( p->zCostTable!=0 && p->pConfig3==0 ){

  spellfix1ResetCursor(pCur);
  spellfix1ResizeCursor(pCur, iLimit);
  zMatchThis = sqlite3_value_text(argv[0]);
  if( zMatchThis==0 ) return SQLITE_OK;
  if( p->pConfig3 ){
    x.pLang = editDist3FindLang(p->pConfig3, iLang);
    pMatchStr3 = editDist3FromStringNew(x.pLang, (const char*)zMatchThis, -1);




  }else{
    x.pLang = 0;
  }
  zPattern = (char*)transliterate(zMatchThis, sqlite3_value_bytes(argv[0]));
  sqlite3_free(pCur->zPattern);
  pCur->zPattern = zPattern;
  if( zPattern==0 ) return SQLITE_NOMEM;



  nPattern = strlen(zPattern);
  if( zPattern[nPattern-1]=='*' ) nPattern--;
  zSql = sqlite3_mprintf(
     "SELECT id, word, rank, k1"
     "  FROM \"%w\".\"%w_vocab\""
     " WHERE langid=%d AND k2>=?1 AND k2<?2",
     p->zDbName, p->zTableName, iLang
  );





  rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
  sqlite3_free(zSql);
  pCur->iLang = iLang;
  x.pCur = pCur;
  x.pStmt = pStmt;
  x.zPattern = zPattern;
  x.nPattern = nPattern;
  x.pMatchStr3 = pMatchStr3;
  x.iLang = iLang;
  x.rc = rc;
  x.pConfig3 = p->pConfig3;
  if( x.rc==SQLITE_OK ){
    spellfix1RunQuery(&x, zPattern, nPattern);
  }

#if 0
  /* Convert "ght" to "t" in the original pattern and try again */
  if( x.rc==SQLITE_OK ){
    int i, j;                         /* Loop counters */
    char zQuery[50];                  /* Space for alternative query string */
    for(i=j=0; i<nPattern && i<sizeof(zQuery)-1; i++){
      char c = zPattern[i];
      if( c=='g' && i<nPattern-2 && zPattern[i+1]=='h' && zPattern[i+2]=='t' ){
        i += 2;
        c= 't';
      }
      zQuery[j++] = c;
    }
    zQuery[j] = 0;
    if( j<i ){
      spellfix1RunQuery(&x, zQuery, j);
    }
  }
#endif

  if( pCur->a ){
    qsort(pCur->a, pCur->nRow, sizeof(pCur->a[0]), spellfix1RowCompare);
    pCur->iTop = iLimit;
    pCur->iScope = iScope;


  }


  sqlite3_finalize(pStmt);
  editDist3FromStringDelete(pMatchStr3);
  return pCur->a ? x.rc : SQLITE_NOMEM;
}

/*
** This version of xFilter handles a full-table scan case
*/
static int spellfix1FilterForFullScan(
  spellfix1_cursor *pCur,
  int idxNum,
  int argc,
  sqlite3_value **argv
){



  spellfix1ResetCursor(pCur);












  spellfix1ResizeCursor(pCur, 0);

  return SQLITE_OK;
}


/*
** Called to "rewind" a cursor back to the beginning so that
** it starts its output over again.  Always called at least once
** prior to any spellfix1Column, spellfix1Rowid, or spellfix1Eof call.

/*
** Advance a cursor to its next row of output
*/
static int spellfix1Next(sqlite3_vtab_cursor *cur){
  spellfix1_cursor *pCur = (spellfix1_cursor *)cur;
  if( pCur->iRow < pCur->nRow ) pCur->iRow++;







  return SQLITE_OK;
}

/*
** Return TRUE if we are at the end-of-file
*/
static int spellfix1Eof(sqlite3_vtab_cursor *cur){
  spellfix1_cursor *pCur = (spellfix1_cursor *)cur;
  return pCur->iRow>=pCur->nRow;
}

/*
** Return columns from the current row.
*/
static int spellfix1Column(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){




  spellfix1_cursor *pCur = (spellfix1_cursor*)cur;








  switch( i ){
    case SPELLFIX_COL_WORD: {
      sqlite3_result_text(ctx, pCur->a[pCur->iRow].zWord, -1, SQLITE_STATIC);
      break;
    }
    case SPELLFIX_COL_RANK: {
      sqlite3_result_int(ctx, pCur->a[pCur->iRow].iRank);

        int nWord = strlen(zWord);

        if( nPattern>0 && pCur->zPattern[nPattern-1]=='*' ){
          char *zTranslit;
          int res;
          zTranslit = (char *)transliterate((unsigned char *)zWord, nWord);
          if( !zTranslit ) return SQLITE_NOMEM;
          res = editdist1(pCur->zPattern, zTranslit, pCur->iLang, &iMatchlen);
          sqlite3_free(zTranslit);
          if( res<0 ) return SQLITE_NOMEM;
          iMatchlen = translen_to_charlen(zWord, nWord, iMatchlen);
        }else{
          iMatchlen = utf8Charlen(zWord, nWord);
        }
      }

}

/*
** The rowid.
*/
static int spellfix1Rowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
  spellfix1_cursor *pCur = (spellfix1_cursor*)cur;



  *pRowid = pCur->a[pCur->iRow].iRowid;

  return SQLITE_OK;
}

/*
** The xUpdate() method.
*/
static int spellfix1Update(

             iRank, iLang, zWord, zK1, zK2
      );
      *pRowid = sqlite3_last_insert_rowid(db);
    }else{
      rowid = sqlite3_value_int64(argv[0]);
      newRowid = *pRowid = sqlite3_value_int64(argv[1]);
      spellfix1DbExec(&rc, db,
             "UPDATE \"%w\".\"%w_vocab\" SET id=%lld, rank=%d, lang=%d,"
             " word=%Q, rank=%d, k1=%Q, k2=%Q WHERE id=%lld",
             p->zDbName, p->zTableName, newRowid, iRank, iLang,
             zWord, zK1, zK2, rowid
      );
    }
    sqlite3_free(zK1);
    sqlite3_free(zK2);
  }

  spellfix1DbExec(&rc, db, 
     "ALTER TABLE \"%w\".\"%w_vocab\" RENAME TO \"%w_vocab\"",
     p->zDbName, p->zTableName, zNewName
  );
  if( rc==SQLITE_OK ){
    sqlite3_free(p->zTableName);
    p->zTableName = zNewName;


  }
  return rc;
}


/*
** A virtual table module that provides fuzzy search.

static int spellfix1Register(sqlite3 *db){
  int nErr = 0;
  int i;
  nErr += sqlite3_create_function(db, "spellfix1_translit", 1, SQLITE_UTF8, 0,
                                  transliterateSqlFunc, 0, 0);
  nErr += sqlite3_create_function(db, "spellfix1_editdist", 2, SQLITE_UTF8, 0,
                                  editdistSqlFunc, 0, 0);
  nErr += sqlite3_create_function(db, "spellfix1_editdist", 3, SQLITE_UTF8, 0,
                                  editdistSqlFunc, 0, 0);
  nErr += sqlite3_create_function(db, "spellfix1_phonehash", 1, SQLITE_UTF8, 0,
                                  phoneticHashSqlFunc, 0, 0);
  nErr += sqlite3_create_function(db, "spellfix1_scriptcode", 1, SQLITE_UTF8, 0,
                                  scriptCodeSqlFunc, 0, 0);
  nErr += sqlite3_create_function(db, "pollock_skeleton", 1, SQLITE_UTF8, 0,
                                  pollockSkeletonSqlFunc, 0, 0);
  nErr += sqlite3_create_function(db, "pollock_omission", 1, SQLITE_UTF8, 0,
                                  pollockOmissionSqlFunc, 0, 0);
  nErr += sqlite3_create_module(db, "spellfix1", &spellfix1Module, 0);
  nErr += editDist3Install(db);

  /* Verify sanity of the translit[] table */
  for(i=0; i<sizeof(translit)/sizeof(translit[0])-1; i++){
    assert( translit[i].cFrom<translit[i+1].cFrom );
  }

  foreach w $vocab {
    execsql { INSERT INTO t3(word) VALUES($w) }
  }
} {}

breakpoint
foreach {tn word res} {
  1   kos*     {kosher 3 kiosk 4 kudo 2 kappa 1 keypad 1}
  2   kellj*   {killjoy 5 killed 4 killingly 4 kill 4 killer 4}
  3   kellj    {kill 4 kills 5 killjoy 7 keel 4 killed 6}
} {
  do_execsql_test 1.2.$tn {
    SELECT word, matchlen FROM t3 WHERE word MATCH $word LIMIT 5

  } $res
} 

finish_test