SQLite

Check-in [ae6794ffa2]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add extra tests and fixes for multi-column matches.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: ae6794ffa23ef6191bd8834422abf322d978c11b
User & Date: dan 2015-05-29 19:00:22.571
Context
2015-05-30
11:49
Remove the "#include sqlite3Int.h" from fts5Int.h. (check-in: e008c3c8e2 user: dan tags: fts5)
2015-05-29
19:00
Add extra tests and fixes for multi-column matches. (check-in: ae6794ffa2 user: dan tags: fts5)
15:55
Add syntax to fts5 used to specify that a phrase or NEAR group should match a subset of columns. For example "[col1 col2 ...] : <phrase>". (check-in: 0fc0ea2092 user: dan tags: fts5)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts5/fts5_expr.c.
328
329
330
331
332
333
334








335
336
337
338
339
340
341
void sqlite3Fts5ExprFree(Fts5Expr *p){
  if( p ){
    sqlite3Fts5ParseNodeFree(p->pRoot);
    sqlite3_free(p->apExprPhrase);
    sqlite3_free(p);
  }
}









/*
** All individual term iterators in pPhrase are guaranteed to be valid and
** pointing to the same rowid when this function is called. This function 
** checks if the current rowid really is a match, and if so populates
** the pPhrase->poslist buffer accordingly. Output parameter *pbMatch
** is set to true if this is really a match, or false otherwise.







>
>
>
>
>
>
>
>







328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
void sqlite3Fts5ExprFree(Fts5Expr *p){
  if( p ){
    sqlite3Fts5ParseNodeFree(p->pRoot);
    sqlite3_free(p->apExprPhrase);
    sqlite3_free(p);
  }
}

static int fts5ExprColsetTest(Fts5ExprColset *pColset, int iCol){
  int i;
  for(i=0; i<pColset->nCol; i++){
    if( pColset->aiCol[i]==iCol ) return 1;
  }
  return 0;
}

/*
** All individual term iterators in pPhrase are guaranteed to be valid and
** pointing to the same rowid when this function is called. This function 
** checks if the current rowid really is a match, and if so populates
** the pPhrase->poslist buffer accordingly. Output parameter *pbMatch
** is set to true if this is really a match, or false otherwise.
351
352
353
354
355
356
357
358





359
360
361
362
363
364
365
  int *pbMatch                    /* OUT: Set to true if really a match */
){
  Fts5PoslistWriter writer = {0};
  Fts5PoslistReader aStatic[4];
  Fts5PoslistReader *aIter = aStatic;
  int i;
  int rc = SQLITE_OK;
  int iCol = pColset ? pColset->aiCol[0] : -1;






  fts5BufferZero(&pPhrase->poslist);

  /* If the aStatic[] array is not large enough, allocate a large array
  ** using sqlite3_malloc(). This approach could be improved upon. */
  if( pPhrase->nTerm>(sizeof(aStatic) / sizeof(aStatic[0])) ){
    int nByte = sizeof(Fts5PoslistReader) * pPhrase->nTerm;







|
>
>
>
>
>







359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
  int *pbMatch                    /* OUT: Set to true if really a match */
){
  Fts5PoslistWriter writer = {0};
  Fts5PoslistReader aStatic[4];
  Fts5PoslistReader *aIter = aStatic;
  int i;
  int rc = SQLITE_OK;
  int iCol = -1;
  
  if( pColset && pColset->nCol==1 ){
    iCol = pColset->aiCol[0];
    pColset = 0;
  }

  fts5BufferZero(&pPhrase->poslist);

  /* If the aStatic[] array is not large enough, allocate a large array
  ** using sqlite3_malloc(). This approach could be improved upon. */
  if( pPhrase->nTerm>(sizeof(aStatic) / sizeof(aStatic[0])) ){
    int nByte = sizeof(Fts5PoslistReader) * pPhrase->nTerm;
392
393
394
395
396
397
398

399
400
401

402
403
404
405
406
407
408
            if( sqlite3Fts5PoslistReaderNext(pPos) ) goto ismatch_out;
          }
          if( pPos->iPos>iAdj ) iPos = pPos->iPos-i;
        }
      }
    }while( bMatch==0 );


    /* Append position iPos to the output */
    rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos);
    if( rc!=SQLITE_OK ) goto ismatch_out;


    for(i=0; i<pPhrase->nTerm; i++){
      if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) goto ismatch_out;
    }
  }

 ismatch_out:







>
|
|
|
>







405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
            if( sqlite3Fts5PoslistReaderNext(pPos) ) goto ismatch_out;
          }
          if( pPos->iPos>iAdj ) iPos = pPos->iPos-i;
        }
      }
    }while( bMatch==0 );

    if( pColset==0 || fts5ExprColsetTest(pColset, FTS5_POS2COLUMN(iPos)) ){
      /* Append position iPos to the output */
      rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos);
      if( rc!=SQLITE_OK ) goto ismatch_out;
    }

    for(i=0; i<pPhrase->nTerm; i++){
      if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) goto ismatch_out;
    }
  }

 ismatch_out:
455
456
457
458
459
460
461
462
463
464
465
466


467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493




494
495
496
497
498
499
500
};

/*
** The near-set object passed as the first argument contains more than
** one phrase. All phrases currently point to the same row. The
** Fts5ExprPhrase.poslist buffers are populated accordingly. This function
** tests if the current row contains instances of each phrase sufficiently
** close together to meet the NEAR constraint. Output variable *pbMatch
** is set to true if it does, or false otherwise.
**
** If no error occurs, SQLITE_OK is returned. Or, if an error does occur,
** an SQLite error code. If a value other than SQLITE_OK is returned, the


** final value of *pbMatch is undefined.
**
** TODO: This function should also edit the position lists associated
** with each phrase to remove any phrase instances that are not part of
** a set of intances that collectively matches the NEAR constraint.
*/
static int fts5ExprNearIsMatch(Fts5ExprNearset *pNear, int *pbMatch){
  Fts5NearTrimmer aStatic[4];
  Fts5NearTrimmer *a = aStatic;

  Fts5ExprPhrase **apPhrase = pNear->apPhrase;

  int i;
  int rc = SQLITE_OK;
  int bMatch;

  assert( pNear->nPhrase>1 );

  /* If the aStatic[] array is not large enough, allocate a large array
  ** using sqlite3_malloc(). This approach could be improved upon. */
  if( pNear->nPhrase>(sizeof(aStatic) / sizeof(aStatic[0])) ){
    int nByte = sizeof(Fts5NearTrimmer) * pNear->nPhrase;
    a = (Fts5NearTrimmer*)sqlite3_malloc(nByte);
    if( !a ) return SQLITE_NOMEM;
    memset(a, 0, nByte);
  }else{
    memset(aStatic, 0, sizeof(aStatic));




  }

  /* Initialize a lookahead iterator for each phrase. After passing the
  ** buffer and buffer size to the lookaside-reader init function, zero
  ** the phrase poslist buffer. The new poslist for the phrase (containing
  ** the same entries as the original with some entries removed on account 
  ** of the NEAR constraint) is written over the original even as it is







|
|

<
|
>
>
|
|
|
|
|

|


<



|








|
<
<


>
>
>
>







470
471
472
473
474
475
476
477
478
479

480
481
482
483
484
485
486
487
488
489
490
491

492
493
494
495
496
497
498
499
500
501
502
503
504


505
506
507
508
509
510
511
512
513
514
515
516
517
};

/*
** The near-set object passed as the first argument contains more than
** one phrase. All phrases currently point to the same row. The
** Fts5ExprPhrase.poslist buffers are populated accordingly. This function
** tests if the current row contains instances of each phrase sufficiently
** close together to meet the NEAR constraint. Non-zero is returned if it
** does, or zero otherwise.
**

** If in/out parameter (*pRc) is set to other than SQLITE_OK when this
** function is called, it is a no-op. Or, if an error (e.g. SQLITE_NOMEM)
** occurs within this function (*pRc) is set accordingly before returning.
** The return value is undefined in both these cases.
** 
** If no error occurs and non-zero (a match) is returned, the position-list
** of each phrase object is edited to contain only those entries that
** meet the constraint before returning.
*/
static int fts5ExprNearIsMatch(int *pRc, Fts5ExprNearset *pNear){
  Fts5NearTrimmer aStatic[4];
  Fts5NearTrimmer *a = aStatic;

  Fts5ExprPhrase **apPhrase = pNear->apPhrase;

  int i;
  int rc = *pRc;
  int bMatch;

  assert( pNear->nPhrase>1 );

  /* If the aStatic[] array is not large enough, allocate a large array
  ** using sqlite3_malloc(). This approach could be improved upon. */
  if( pNear->nPhrase>(sizeof(aStatic) / sizeof(aStatic[0])) ){
    int nByte = sizeof(Fts5NearTrimmer) * pNear->nPhrase;
    a = (Fts5NearTrimmer*)sqlite3Fts5MallocZero(&rc, nByte);


  }else{
    memset(aStatic, 0, sizeof(aStatic));
  }
  if( rc!=SQLITE_OK ){
    *pRc = rc;
    return 0;
  }

  /* Initialize a lookahead iterator for each phrase. After passing the
  ** buffer and buffer size to the lookaside-reader init function, zero
  ** the phrase poslist buffer. The new poslist for the phrase (containing
  ** the same entries as the original with some entries removed on account 
  ** of the NEAR constraint) is written over the original even as it is
547
548
549
550
551
552
553
554
555

556
557

558
559
560
561
562
563
564
        iMin = a[i].reader.iLookahead;
        iAdv = i;
      }
    }
    if( fts5LookaheadReaderNext(&a[iAdv].reader) ) goto ismatch_out;
  }

 ismatch_out:
  *pbMatch = (a[0].pOut->n>0);

  if( a!=aStatic ) sqlite3_free(a);
  return rc;

}

/*
** Advance the first term iterator in the first phrase of pNear. Set output
** variable *pbEof to true if it reaches EOF or if an error occurs.
**
** Return SQLITE_OK if successful, or an SQLite error code if an error







|
|
>
|
|
>







564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
        iMin = a[i].reader.iLookahead;
        iAdv = i;
      }
    }
    if( fts5LookaheadReaderNext(&a[iAdv].reader) ) goto ismatch_out;
  }

  ismatch_out: {
    int bRet = a[0].pOut->n>0;
    *pRc = rc;
    if( a!=aStatic ) sqlite3_free(a);
    return bRet;
  }
}

/*
** Advance the first term iterator in the first phrase of pNear. Set output
** variable *pbEof to true if it reaches EOF or if an error occurs.
**
** Return SQLITE_OK if successful, or an SQLite error code if an error
696
697
698
699
700
701
702


















703
704
705
706
707
708
709
  ** not part of a varint */
  while( p<pEnd && !(prev & 0x80) && *p!=0x01 ){
    prev = *p++;
  }
  return p - (*pa);
}





















/*
** Argument pNode points to a NEAR node. All individual term iterators 
** point to valid entries (not EOF).
*
** This function tests if the term iterators currently all point to the







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
  ** not part of a varint */
  while( p<pEnd && !(prev & 0x80) && *p!=0x01 ){
    prev = *p++;
  }
  return p - (*pa);
}

static int fts5ExprExtractColset (
  Fts5ExprColset *pColset,        /* Colset to filter on */
  const u8 *pPos, int nPos,       /* Position list */
  Fts5Buffer *pBuf                /* Output buffer */
){
  int rc = SQLITE_OK;
  int i;

  fts5BufferZero(pBuf);
  for(i=0; i<pColset->nCol; i++){
    const u8 *pSub = pPos;
    int nSub = fts5ExprExtractCol(&pSub, nPos, pColset->aiCol[i]);
    if( nSub ){
      fts5BufferAppendBlob(&rc, pBuf, nSub, pSub);
    }
  }
  return rc;
}


/*
** Argument pNode points to a NEAR node. All individual term iterators 
** point to valid entries (not EOF).
*
** This function tests if the term iterators currently all point to the
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
        assert( pPhrase->poslist.nSpace==0 );
        pPhrase->poslist.p = (u8*)pPos;
        pPhrase->poslist.n = nPos;
      }else if( pColset->nCol==1 ){
        assert( pPhrase->poslist.nSpace==0 );
        pPhrase->poslist.n = fts5ExprExtractCol(&pPos, nPos, pColset->aiCol[0]);
        pPhrase->poslist.p = (u8*)pPos;
      }else{
        int i;
        fts5BufferZero(&pPhrase->poslist);
        for(i=0; i<pColset->nCol; i++){
          const u8 *pSub = pPos;
          int nSub = fts5ExprExtractCol(&pSub, nPos, pColset->aiCol[i]);
          if( nSub ){
            fts5BufferAppendBlob(&rc, &pPhrase->poslist, nSub, pSub);
          }
        }
      }

      if( pPhrase->poslist.n ) return rc;
    }else{
      int i;

      assert( pNear->pColset==0 || pNear->pColset->nCol==1 );

      /* Advance the iterators until they all point to the same rowid */
      rc = fts5ExprNearNextRowidMatch(pExpr, pNode);
      if( rc!=SQLITE_OK || pNode->bEof ) break;

      /* Check that each phrase in the nearset matches the current row.
      ** Populate the pPhrase->poslist buffers at the same time. If any
      ** phrase is not a match, break out of the loop early.  */
      for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
        Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
        if( pPhrase->nTerm>1 || pNear->pColset ){
          int bMatch = 0;
          rc = fts5ExprPhraseIsMatch(pExpr, pNear->pColset, pPhrase, &bMatch);
          if( bMatch==0 ) break;
        }else{
          rc = sqlite3Fts5IterPoslistBuffer(
              pPhrase->aTerm[0].pIter, &pPhrase->poslist
          );
        }
      }

      if( rc==SQLITE_OK && i==pNear->nPhrase ){
        int bMatch = 1;
        if( pNear->nPhrase>1 ){
          rc = fts5ExprNearIsMatch(pNear, &bMatch);
        }
        if( rc!=SQLITE_OK || bMatch ) break;
      }
    }

    /* If control flows to here, then the current rowid is not a match.
    ** Advance all term iterators in all phrases to the next rowid. */
    if( rc==SQLITE_OK ){
      rc = fts5ExprNearAdvanceFirst(pExpr, pNode, 0, 0);







|
<
|
<
<
<
<
<
<
<






<
<




















|
<
|
|
<
<







789
790
791
792
793
794
795
796

797







798
799
800
801
802
803


804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824

825
826


827
828
829
830
831
832
833
        assert( pPhrase->poslist.nSpace==0 );
        pPhrase->poslist.p = (u8*)pPos;
        pPhrase->poslist.n = nPos;
      }else if( pColset->nCol==1 ){
        assert( pPhrase->poslist.nSpace==0 );
        pPhrase->poslist.n = fts5ExprExtractCol(&pPos, nPos, pColset->aiCol[0]);
        pPhrase->poslist.p = (u8*)pPos;
      }else if( rc==SQLITE_OK ){

        rc = fts5ExprExtractColset(pColset, pPos, nPos, &pPhrase->poslist);







      }

      if( pPhrase->poslist.n ) return rc;
    }else{
      int i;



      /* Advance the iterators until they all point to the same rowid */
      rc = fts5ExprNearNextRowidMatch(pExpr, pNode);
      if( rc!=SQLITE_OK || pNode->bEof ) break;

      /* Check that each phrase in the nearset matches the current row.
      ** Populate the pPhrase->poslist buffers at the same time. If any
      ** phrase is not a match, break out of the loop early.  */
      for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
        Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
        if( pPhrase->nTerm>1 || pNear->pColset ){
          int bMatch = 0;
          rc = fts5ExprPhraseIsMatch(pExpr, pNear->pColset, pPhrase, &bMatch);
          if( bMatch==0 ) break;
        }else{
          rc = sqlite3Fts5IterPoslistBuffer(
              pPhrase->aTerm[0].pIter, &pPhrase->poslist
          );
        }
      }

      if( i==pNear->nPhrase ){

        if( i==1 ) break;
        if( fts5ExprNearIsMatch(&rc, pNear) ) break;


      }
    }

    /* If control flows to here, then the current rowid is not a match.
    ** Advance all term iterators in all phrases to the next rowid. */
    if( rc==SQLITE_OK ){
      rc = fts5ExprNearAdvanceFirst(pExpr, pNode, 0, 0);
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
){
  char *zRet = 0;
  if( pExpr->eType==FTS5_STRING ){
    Fts5ExprNearset *pNear = pExpr->pNear;
    int i; 
    int iTerm;

    zRet = fts5PrintfAppend(zRet, "[%s ", zNearsetCmd);
    if( zRet==0 ) return 0;
    if( pNear->pColset ){
      int *aiCol = pNear->pColset->aiCol;
      int nCol = pNear->pColset->nCol;
      if( nCol==1 ){
        zRet = fts5PrintfAppend(zRet, "-col %d ", aiCol[0]);
      }else{







|







1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
){
  char *zRet = 0;
  if( pExpr->eType==FTS5_STRING ){
    Fts5ExprNearset *pNear = pExpr->pNear;
    int i; 
    int iTerm;

    zRet = fts5PrintfAppend(zRet, "%s ", zNearsetCmd);
    if( zRet==0 ) return 0;
    if( pNear->pColset ){
      int *aiCol = pNear->pColset->aiCol;
      int nCol = pNear->pColset->nCol;
      if( nCol==1 ){
        zRet = fts5PrintfAppend(zRet, "-col %d ", aiCol[0]);
      }else{
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
        zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm);
      }

      if( zRet ) zRet = fts5PrintfAppend(zRet, "}");
      if( zRet==0 ) return 0;
    }

    zRet = fts5PrintfAppend(zRet, "]");
    if( zRet==0 ) return 0;

  }else{
    char *zOp = 0;
    char *z1 = 0;
    char *z2 = 0;
    switch( pExpr->eType ){
      case FTS5_AND: zOp = "&&"; break;
      case FTS5_NOT: zOp = "&& !"; break;
      default: 
        assert( pExpr->eType==FTS5_OR );
        zOp = "||"; 
        break;
    }

    z1 = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->pLeft);
    z2 = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->pRight);
    if( z1 && z2 ){
      int b1 = pExpr->pLeft->eType!=FTS5_STRING;
      int b2 = pExpr->pRight->eType!=FTS5_STRING;
      zRet = sqlite3_mprintf("%s%s%s %s %s%s%s", 
          b1 ? "(" : "", z1, b1 ? ")" : "",
          zOp, 
          b2 ? "(" : "", z2, b2 ? ")" : ""
      );
    }
    sqlite3_free(z1);
    sqlite3_free(z2);
  }

  return zRet;
}







<







|
|


|






<
<
|
<
<
<
<







1616
1617
1618
1619
1620
1621
1622

1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640


1641




1642
1643
1644
1645
1646
1647
1648
        zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm);
      }

      if( zRet ) zRet = fts5PrintfAppend(zRet, "}");
      if( zRet==0 ) return 0;
    }


    if( zRet==0 ) return 0;

  }else{
    char *zOp = 0;
    char *z1 = 0;
    char *z2 = 0;
    switch( pExpr->eType ){
      case FTS5_AND: zOp = "AND"; break;
      case FTS5_NOT: zOp = "NOT"; break;
      default: 
        assert( pExpr->eType==FTS5_OR );
        zOp = "OR"; 
        break;
    }

    z1 = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->pLeft);
    z2 = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->pRight);
    if( z1 && z2 ){


      zRet = sqlite3_mprintf("%s [%s] [%s]", zOp, z1, z2);




    }
    sqlite3_free(z1);
    sqlite3_free(z2);
  }

  return zRet;
}
Changes to ext/fts5/test/fts5_common.tcl.
142
143
144
145
146
147
148








































































































































149
  set map [list 0 a  1 b  2 c  3 d  4 e  5 f  6 g  7 h  8 i  9 j]
  set doc [list]
  for {set i 0} {$i < $n} {incr i} {
    lappend doc "x[string map $map [format %.3d [expr int(rand()*1000)]]]"
  }
  set doc
}
















































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
  set map [list 0 a  1 b  2 c  3 d  4 e  5 f  6 g  7 h  8 i  9 j]
  set doc [list]
  for {set i 0} {$i < $n} {incr i} {
    lappend doc "x[string map $map [format %.3d [expr int(rand()*1000)]]]"
  }
  set doc
}

#-------------------------------------------------------------------------
# Usage:
#
#   nearset aCol ?-pc VARNAME? ?-near N? ?-col C? -- phrase1 phrase2...
#
# This command is used to test if a document (set of column values) matches
# the logical equivalent of a single FTS5 NEAR() clump and, if so, return
# the equivalent of an FTS5 position list.
#
# Parameter $aCol is passed a list of the column values for the document
# to test. Parameters $phrase1 and so on are the phrases.
#
# The result is a list of phrase hits. Each phrase hit is formatted as
# three integers separated by "." characters, in the following format:
#
#   <phrase number> . <column number> . <token offset>
#
# Options:
#
#   -near N        (NEAR distance. Default 10)
#   -col  C        (List of column indexes to match against)
#   -pc   VARNAME  (variable in caller frame to use for phrase numbering)
#
proc nearset {aCol args} {
  set O(-near) 10
  set O(-col)  {}
  set O(-pc)   ""

  set nOpt [lsearch -exact $args --]
  if {$nOpt<0} { error "no -- option" }

  foreach {k v} [lrange $args 0 [expr $nOpt-1]] {
    if {[info exists O($k)]==0} { error "unrecognized option $k" }
    set O($k) $v
  }

  if {$O(-pc) == ""} {
    set counter 0
  } else {
    upvar $O(-pc) counter
  }

  # Set $phraselist to be a list of phrases. $nPhrase its length.
  set phraselist [lrange $args [expr $nOpt+1] end]
  set nPhrase [llength $phraselist]

  for {set j 0} {$j < [llength $aCol]} {incr j} {
    for {set i 0} {$i < $nPhrase} {incr i} { 
      set A($j,$i) [list]
    }
  }

  set iCol -1
  foreach col $aCol {
    incr iCol
    if {$O(-col)!="" && [lsearch $O(-col) $iCol]<0} continue
    set nToken [llength $col]

    set iFL [expr $O(-near) >= $nToken ? $nToken - 1 : $O(-near)]
    for { } {$iFL < $nToken} {incr iFL} {
      for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
        set B($iPhrase) [list]
      }
      
      for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
        set p [lindex $phraselist $iPhrase]
        set nPm1 [expr {[llength $p] - 1}]
        set iFirst [expr $iFL - $O(-near) - [llength $p]]

        for {set i $iFirst} {$i <= $iFL} {incr i} {
          if {[lrange $col $i [expr $i+$nPm1]] == $p} { lappend B($iPhrase) $i }
        }
        if {[llength $B($iPhrase)] == 0} break
      }

      if {$iPhrase==$nPhrase} {
        for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
          set A($iCol,$iPhrase) [concat $A($iCol,$iPhrase) $B($iPhrase)]
          set A($iCol,$iPhrase) [lsort -integer -uniq $A($iCol,$iPhrase)]
        }
      }
    }
  }

  set res [list]
  #puts [array names A]

  for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
    for {set iCol 0} {$iCol < [llength $aCol]} {incr iCol} {
      foreach a $A($iCol,$iPhrase) {
        lappend res "$counter.$iCol.$a"
      }
    }
    incr counter
  }

  #puts $res
  sort_poslist $res
}

#-------------------------------------------------------------------------
# Usage:
#
#   sort_poslist LIST
#
# Sort a position list of the type returned by command [nearset]
#
proc sort_poslist {L} {
  lsort -command instcompare $L
}
proc instcompare {lhs rhs} {
  foreach {p1 c1 o1} [split $lhs .] {}
  foreach {p2 c2 o2} [split $rhs .] {}

  set res [expr $c1 - $c2]
  if {$res==0} { set res [expr $o1 - $o2] }
  if {$res==0} { set res [expr $p1 - $p2] }

  return $res
}

#-------------------------------------------------------------------------
# Logical operators used by the commands returned by fts5_tcl_expr().
#
proc AND {a b} {
  if {[llength $a]==0 || [llength $b]==0} { return [list] }
  sort_poslist [concat $a $b]
}
proc OR {a b} {
  sort_poslist [concat $a $b]
}
proc NOT {a b} {
  if {[llength $b]} { return [list] }
  return $a
}

Changes to ext/fts5/test/fts5ac.test.
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265


266
267
268
269
270


271

272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
    95  {h b n j t k i h o q u}             {w n g i t o k c a m y p f l x c p}
    96  {f c x p y r b m o l m o a}         {p c a q s u n n x d c f a o}
    97  {u h h k m n k}                     {u b v n u a o c}
    98  {s p e t c z d f n w f}             {l s f j b l c e s h}
    99  {r c v w i v h a t a c v c r e}     {h h u m g o f b a e o}
}

#-------------------------------------------------------------------------
# Usage:
#
#   poslist aCol ?-pc VARNAME? ?-near N? ?-col C? -- phrase1 phrase2...
#
# This command is used to test if a document (set of column values) matches
# the logical equivalent of a single FTS5 NEAR() clump and, if so, return
# the equivalent of an FTS5 position list.
#
# Parameter $aCol is passed a list of the column values for the document
# to test. Parameters $phrase1 and so on are the phrases.
#
# The result is a list of phrase hits. Each phrase hit is formatted as
# three integers separated by "." characters, in the following format:
#
#   <phrase number> . <column number> . <token offset>
#
# Options:
#
#   -near N        (NEAR distance. Default 10)
#   -col  C        (List of column indexes to match against)
#   -pc   VARNAME  (variable in caller frame to use for phrase numbering)
#
proc poslist {aCol args} {
  set O(-near) 10
  set O(-col)  {}
  set O(-pc)   ""

  set nOpt [lsearch -exact $args --]
  if {$nOpt<0} { error "no -- option" }

  foreach {k v} [lrange $args 0 [expr $nOpt-1]] {
    if {[info exists O($k)]==0} { error "unrecognized option $k" }
    set O($k) $v
  }

  if {$O(-pc) == ""} {
    set counter 0
  } else {
    upvar $O(-pc) counter
  }

  # Set $phraselist to be a list of phrases. $nPhrase its length.
  set phraselist [lrange $args [expr $nOpt+1] end]
  set nPhrase [llength $phraselist]

  for {set j 0} {$j < [llength $aCol]} {incr j} {
    for {set i 0} {$i < $nPhrase} {incr i} { 
      set A($j,$i) [list]
    }
  }

  set iCol -1
  foreach col $aCol {
    incr iCol
    if {$O(-col)!="" && [lsearch $O(-col) $iCol]<0} continue
    set nToken [llength $col]

    set iFL [expr $O(-near) >= $nToken ? $nToken - 1 : $O(-near)]
    for { } {$iFL < $nToken} {incr iFL} {
      for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
        set B($iPhrase) [list]
      }
      
      for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
        set p [lindex $phraselist $iPhrase]
        set nPm1 [expr {[llength $p] - 1}]
        set iFirst [expr $iFL - $O(-near) - [llength $p]]

        for {set i $iFirst} {$i <= $iFL} {incr i} {
          if {[lrange $col $i [expr $i+$nPm1]] == $p} { lappend B($iPhrase) $i }
        }
        if {[llength $B($iPhrase)] == 0} break
      }

      if {$iPhrase==$nPhrase} {
        for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
          set A($iCol,$iPhrase) [concat $A($iCol,$iPhrase) $B($iPhrase)]
          set A($iCol,$iPhrase) [lsort -integer -uniq $A($iCol,$iPhrase)]
        }
      }
    }
  }

  set res [list]
#puts [array names A]

  for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
    for {set iCol 0} {$iCol < [llength $aCol]} {incr iCol} {
      foreach a $A($iCol,$iPhrase) {
        lappend res "$counter.$iCol.$a"
      }
    }
    incr counter
  }

    #puts $res
  return $res
}

# Usage:
#
#   nearset aCol ?-near N? ?-col C? -- phrase1 phrase2...
#
proc nearset {args} {
  set plist [poslist {*}$args]
  return [expr [llength [lindex $plist 0]]>0]
}

proc instcompare {lhs rhs} {
  foreach {p1 c1 o1} [split $lhs .] {}
  foreach {p2 c2 o2} [split $rhs .] {}

  set res [expr $c1 - $c2]
  if {$res==0} { set res [expr $o1 - $o2] }
  if {$res==0} { set res [expr $p1 - $p2] }

  return $res
}

# Argument $expr is an FTS5 match expression designed to be executed against
# an FTS5 table with the following schema:
# 
#   CREATE VIRTUAL TABLE xy USING fts5(x, y);
#
# Assuming the table contains the same records as stored int the global 
# $::data array (see above), this function returns a list containing one
# element for each match in the dataset. The elements are themselves lists
# formatted as follows:
#
#   <rowid> {<phrase 0 matches> <phrase 1 matches>...}
#
# where each <phrase X matches> element is a list of phrase matches in the
# same form as returned by auxiliary scalar function fts5_test().
#
proc matchdata {bPos expr {bAsc 1}} {

  set tclexpr [db one {SELECT fts5_expr_tcl($expr, 'nearset $cols', 'x', 'y')}]


  set res [list]

  #puts $tclexpr
  foreach {id x y} $::data {
    set cols [list $x $y]


    if $tclexpr {

      if {$bPos} {
        set N [regexp -all -inline {\[nearset [^\]]*\]} $tclexpr]
        set rowres [list]
        set cnt 0
        foreach phrase $N {
          set arglist [string range $phrase 9 end-1]
          set cmd "poslist [lindex $arglist 0] -pc cnt [lrange $arglist 1 end]"
          set pos [eval $cmd]
          set rowres [concat $rowres $pos]
        }
        set rowres [lsort -command instcompare $rowres]
        lappend res [list $id $rowres]
      } else {
        lappend res $id
      }
    }
  }

  if {$bAsc} {







<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<

















|
>
>





>
>
|
>

<
<
<
<
<
<
<
<
<
<
|







121
122
123
124
125
126
127
























































































































128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157










158
159
160
161
162
163
164
165
    95  {h b n j t k i h o q u}             {w n g i t o k c a m y p f l x c p}
    96  {f c x p y r b m o l m o a}         {p c a q s u n n x d c f a o}
    97  {u h h k m n k}                     {u b v n u a o c}
    98  {s p e t c z d f n w f}             {l s f j b l c e s h}
    99  {r c v w i v h a t a c v c r e}     {h h u m g o f b a e o}
}

























































































































# Argument $expr is an FTS5 match expression designed to be executed against
# an FTS5 table with the following schema:
# 
#   CREATE VIRTUAL TABLE xy USING fts5(x, y);
#
# Assuming the table contains the same records as stored int the global 
# $::data array (see above), this function returns a list containing one
# element for each match in the dataset. The elements are themselves lists
# formatted as follows:
#
#   <rowid> {<phrase 0 matches> <phrase 1 matches>...}
#
# where each <phrase X matches> element is a list of phrase matches in the
# same form as returned by auxiliary scalar function fts5_test().
#
proc matchdata {bPos expr {bAsc 1}} {

  set tclexpr [db one {
    SELECT fts5_expr_tcl($expr, 'nearset $cols -pc ::pc', 'x', 'y')
  }]
  set res [list]

  #puts $tclexpr
  foreach {id x y} $::data {
    set cols [list $x $y]
    set ::pc 0
    #set hits [lsort -command instcompare [eval $tclexpr]]
    set hits [eval $tclexpr]
    if {[llength $hits]>0} {
      if {$bPos} {










        lappend res [list $id $hits]
      } else {
        lappend res $id
      }
    }
  }

  if {$bAsc} {
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
  } {
    set res [matchdata 1 $expr]
    do_execsql_test $tn2.4.1.$tn.[llength $res] { 
      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
    } $res
  }

  do_test $tn2.4.1  { poslist {{a b c}} -- a } {0.0.0}
  do_test $tn2.4.2  { poslist {{a b c}} -- c } {0.0.2}

  foreach {tn expr tclexpr} {
    1 {a b} {[N $x -- {a}] && [N $x -- {b}]}
  } {
    do_execsql_test $tn2.5.$tn {
      SELECT fts5_expr_tcl($expr, 'N $x')
    } [list $tclexpr]
  }

  #-------------------------------------------------------------------------







|
|


|







296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
  } {
    set res [matchdata 1 $expr]
    do_execsql_test $tn2.4.1.$tn.[llength $res] { 
      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
    } $res
  }

  do_test $tn2.4.1  { nearset {{a b c}} -- a } {0.0.0}
  do_test $tn2.4.2  { nearset {{a b c}} -- c } {0.0.2}

  foreach {tn expr tclexpr} {
    1 {a b} {AND [N $x -- {a}] [N $x -- {b}]}
  } {
    do_execsql_test $tn2.5.$tn {
      SELECT fts5_expr_tcl($expr, 'N $x')
    } [list $tclexpr]
  }

  #-------------------------------------------------------------------------
473
474
475
476
477
478
479
480
481
482
483
      do_execsql_test $tn2.6.$bAsc.$tn.[llength $res] $sql $res
    }
  }
}

do_execsql_test 3.1 {
  SELECT fts5_expr_tcl('a AND b');
} {{[nearset -- {a}] && [nearset -- {b}]}}

finish_test








|



348
349
350
351
352
353
354
355
356
357
358
      do_execsql_test $tn2.6.$bAsc.$tn.[llength $res] $sql $res
    }
  }
}

do_execsql_test 3.1 {
  SELECT fts5_expr_tcl('a AND b');
} {{AND [nearset -- {a}] [nearset -- {b}]}}

finish_test

Added ext/fts5/test/fts5auto.test.






































































































































































































































































































































































































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
# 2015 May 30
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#*************************************************************************
#
# This file contains automatically generated tests for various types
# of MATCH expressions.
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5auto

# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}


set data {
    -4026076
    {n x w k b p x b n t t d s}     {f j j s p j o}               
    {w v i y r}                     {i p y s}                     
    {a o q v e n q r}               {q v g u c y a z y}           
    3995120
    {c}                             {e e w d t}                   
    {x c p f w r s m l r b f d}     {g g u e}                     
    {s n u t d v p d}               {b k v p m f}                 
    -2913881
    {k m}                           {a}                           
    {w r j z n s l}                 {m j i w d t w e l}           
    {z n c}                         {v f b m}                     
    174082
    {j}                             {q l w u k e q v r i}         
    {j l}                           {u v w r s p e l}             
    {p i k j k q c t g u s}         {g u y s m h q k g t e s o r} 
    3207399
    {e t}                           {}                            
    {p}                             {y v r b e k h d e v}         
    {t m w z b g q t s d d h}       {o n v u i t o y k j}         
    182399
    {}                              {m o s o x d y f a x j z}     
    {x n z r c d}                   {n r x i r}                   
    {s v s}                         {a u}                         
    768994
    {e u t q v z q k j p u f j p}   {y c b}                       
    {p s d}                         {k n w p m p p}               
    {u o x s d}                     {f s g r d b d r m m m z y}   
    3931037
    {c j p x e}                     {c n k t h z o i}             
    {}                              {r r p j k x w q}             
    {o r d z d}                     {x}                           
    3105748
    {p x r u}                       {x i s w o t o g x m z i w}   
    {q x m z}                       {h c j w b l y w x c o}       
    {m b k v}                       {t v q i s a d x}             
    -2501642
    {o u d n w o m o o s n t r h}   {k p e u y p e z d j r y g}   
    {v b b h d d q y j q j}         {a m w d t}                   
    {y e f n}                       {a k x i x}                   
    -1745680
    {z u w j f d b f}               {j w i c g u d w e}           
    {m f p v m a s p v c o s}       {s c r z o t w l b e a q}     
    {m k q}                         {k b a v o}                   
    -932328
    {r v i u m q d r}               {f z u v h c m r f g}         
    {r x r}                         {k p i d h h w h z u a x}     
    {k m j p}                       {h l j a e u c i q x x f x g} 
    -3923818
    {t t p b n u i h e c k}         {m z}                         
    {v u d c}                       {v y y j s g}                 
    {o a f k k q p h g x e n z x}   {h d w c o l}                 
    -2145922
    {z z l f a l g e d c d h}       {j b j p k o o u b q}         
    {d i g q t f d r h k}           {n w g j c x r p t y f l c t} 
    {d o c u k f o}                 {r y s x z s p p h g t p y c} 
    4552917
    {j w j y h l k u}               {n a}                         
    {y h w c n k}                   {b}                           
    {w}                             {z l r t s i m v c y}         
    2292008
    {q v q j w y y x u t}           {r q z n h a b o}             
    {d q y}                         {y v o e j}                   
    {}                              {a b h c d l p d x}           
    1407892
    {n j j u q d o a u c f}         {r d b w o q n g}             
    {d e v w s}                     {v d v o u o x s l s j z y}   
    {j y w h i f g i h m}           {v n z b n y}                 
    -4412544
    {g h h r s}                     {h e r e}                     
    {n q s}                         {o p z r m l l t}             
    {p}                             {f s u o b j}                 
    1209110
    {o a a z t t u h j}             {z z i r k r}                 
    {i c x q w g v o x z i z p}     {q o g k i n z x e d v w v}   
    {p f v b g f e d n p u c y k}   {q z z a i p a a s r e z}     
    3448977
    {i v}                           {l u x t b o k}               
    {f h u v p}                     {k a o y j}                   
    {d m k c j}                     {v c e r u e f i t}           
    -4703774
    {d h v w u z r e h x o l t}     {p s f y w y r q d a m w}     
    {c h g c g j j f t b i c q}     {s e}                         
    {c t q j g f}                   {v n r w y r a g e j d}       
    2414151
    {s o o s d s k q b f q v p e}   {j r o b t o p d l o o x}     
    {d d k t v e}                   {}                            
    {t v o d w}                     {w e q w h y c y y i j b a m} 
    -3342407
    {m c h n e p d o c r w n t}     {j d k s p q l}               
    {t g s r w x j l r z r}         {h}                           
    {r q v x i r a n h s}           {m y p b v w r a u o g q r}   
    -993951
    {l n p u o j d x t u u c o j}   {k r n a r e k v i t o e}     
    {q f t t a a c z v f}           {o n m p v f o e n}           
    {h z h i p s b j z h}           {i t w m k c u g n i}         
    1575251
    {}                              {z s i j d o x j a r t}       
    {h g j u j n v e n z}           {p z j n n f}                 
    {s q q f d w r l y i z d o m}   {b a n d h t b y g h d}       
    4263668
    {q g t h f s}                   {s g x p f q z i s o f l i}   
    {q k}                           {w v h a x n a r b}           
    {m j a h o b i x k r w z q u}   {m t r g j o e q t m p u l}   
    2487819
    {m w g x r n e u t s r}         {b x a t u u j c r n}         
    {j}                             {w f j r e e y l p}           
    {o u h b}                       {o c a c a b v}               
    167966
    {o d b s d o a u m o x y}       {c}                           
    {r w d o b v}                   {z e b}                       
    {i n z a f g z o}               {m u b a g}                   
    1948599
    {n r g q d j s}                 {n k}                         
    {l b p d v t k h y y}           {u m k e c}                   
    {t b n y o t b}                 {j w c i r x x}               
    2941631
    {l d p l b g f}                 {e k e}                       
    {p j}                           {m c s w t b k n l d x}       
    {f o v y v l}                   {c w p s w j w c u t y}       
    3561104
    {d r j j r j i g p}             {u}                           
    {g r j q}                       {z l p d s n f c h t d c v z} 
    {w r c f s x z y}               {g f o k g g}                 
    -2223281
    {y e t j j z f p o m m z}       {h k o g o}                   
    {m x a t}                       {l q x l}                     
    {r w k d l s y b}               {q g k b}                     
    -4502874
    {k k b x k l f}                 {r}                           
    {}                              {q m z b k h k u n e z}       
    {z q g y m y u}                 {}                            
    1757599
    {d p z j y u r}                 {z p l q w j t j}             
    {n i r x r y j}                 {}                            
    {h}                             {w t d q c x z z x e e}       
    -4809589
    {}                              {z p x u h i i n g}           
    {w q s u d b f x n}             {l y k b b r x t i}           
    {n d v j q o t o d p z e}       {u r y u v u c}               
    1068408
    {y e}                           {e g s k e w t p v o b k}     
    {z c m s}                       {r u r u h n h b p q g b}     
    {j k b l}                       {m c d t s r s q a d b o f}   
    -1972554
    {m s w}                         {d k v s a r k p a r i v}     
    {g j z k p}                     {y k c v r e u o q f i b a}   
    {i p i}                         {c z w c y b n z i v}         
    -2052385
    {}                              {x e u f f g n c i x n e i e} 
    {}                              {p s w d x p g}               
    {}                              {s j a h n}                   
    2805981
    {m x g c w o e}                 {k g u y r y i u e g g}       
    {f k j v t x p h x k u}         {w i}                         
    {b l f z f v t n}               {i u d o d p h s m u}         
    2507621
    {}                              {u b n l x f n j t}           
    {u r x l h}                     {h r l m r}                   
    {d y e n b s q v t k n q q}     {x l t v w h a s k}           
    -3138375
    {e o f j y x u w v e w z}       {r d q g k n n v r c z n e w} 
    {l y i q z k j p u f q s k}     {c i l l i m a a g a z r x f} 
    {a v k h m q z b y n z}         {q g w c y r r o a}           
    -457971
    {j x a w e c s h f l f}         {q}                           
    {j f v j u m d q r v v}         {x n v a w}                   
    {i e h d h f u w t t z}         {v s u l s v o v i k n e}     
    2265221
    {z t c y w n y r t}             {n b a x s}                   
    {q w a v}                       {a b s d x i g w t e z h}     
    {t l}                           {j k r w f f y j o k u}       
    -3941280
    {r x t o z}                     {f j n z k}                   
    {t x e b t d b k w i s}         {j t y h i h}                 
    {y q g n g s u v c z j z n g}   {n n g t l p h}               
    2084745
    {z d z d}                       {j}                           
    {o e k t b k a z l w}           {o p i h k c x}               
    {c r b t i j f}                 {z e n m}                     
    1265843
    {}                              {j s g j j x u y}             
    {u q t f}                       {g o g}                       
    {w o j e d}                     {w q n a c t q x j}           
    -2941116
    {i n c u o}                     {f b}                         
    {o m s q d o z a q}             {f s v o b b}                 
    {o a z c h r}                   {j e w h b f z}               
    -1265441
    {p g z q v a o a x a}           {s t h}                       
    {w i p o c}                     {s n d g f z w q o d v v l j} 
    {y f b i a s v}                 {u m o z k k s t s d p b l p} 
    -1989158
    {r i c n}                       {r e w w i n z}               
    {q u s y b w u g y g f o}       {y}                           
    {d}                             {j x i b x u y d c p v a h}   
    2391989
    {b n w x w f q h p i}           {e u b b i n a i o c d g}     
    {v a z o i e n l x l r}         {r u f o r k w m d w}         
    {k s}                           {r f e j q p w}               
}

do_test 1.0 {
  execsql {
    BEGIN;
    CREATE VIRTUAL TABLE tt USING fts5(a, b, c, d, e, f);
  }
  foreach {rowid a b c d e f} $data {
    execsql {
      INSERT INTO tt(rowid, a, b, c, d, e, f) 
      VALUES($rowid, $a, $b, $c, $d, $e, $f)
    }
  }
  execsql {
    COMMIT;
  }
} {}

proc fts5_test_poslist {cmd} {
  set res [list]
  for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
    lappend res [string map {{ } .} [$cmd xInst $i]]
  }
  set res
}
sqlite3_fts5_create_function db fts5_test_poslist fts5_test_poslist

proc matchdata {expr} {
  set tclexpr [db one {
    SELECT fts5_expr_tcl(
      $expr, 'nearset $cols -pc ::pc', 'a','b','c','d','e','f'
    )
  }]
  set res [list]

  db eval {SELECT rowid, * FROM tt} {
    set cols [list $a $b $c $d $e $f]
    set ::pc 0
    set rowdata [eval $tclexpr]

    if {$rowdata != ""} {
      lappend res $rowid $rowdata
    }
  }

  set res
}

#-------------------------------------------------------------------------
#

do_execsql_test 2.0 {
  SELECT rowid, fts5_test_poslist(tt) FROM tt WHERE tt MATCH 'a AND b';
} [matchdata "a AND b"]

do_test 2.1 {
  llength [matchdata "a AND b"]
} 62

foreach {tn expr} {
  1 { [a] : x }
  2 { [a b] : x }
  3 { [a b f] : x }
  4 { [f a b] : x }
  5 { [f a b] : x y }
  6 { [f a b] : x + y }
  7 { [c a b] : x + c }
  8 { [c d] : "l m" }
  9 { [c e] : "l m" }
} {
  set res [matchdata $expr]
  do_test 3.$tn.[llength $res] {
    execsql {
      SELECT rowid, fts5_test_poslist(tt) FROM tt WHERE tt MATCH $expr
    }
  } $res
}


finish_test

Changes to ext/fts5/test/fts5fault4.test.
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
} -test {
  faultsim_test_result {0 {"a" AND ("b" AND NEAR("a" "b", 10))}} 
}

do_faultsim_test 10.2 -faults oom-t* -body {
  db one { SELECT fts5_expr_tcl('x:"a b c" AND b NEAR(a b)', 'ns', 'x') }
} -test {
  set res {[ns -col 0 -- {a b c}] && ([ns -- {b}] && [ns -near 10 -- {a} {b}])}
  faultsim_test_result [list 0 $res]
}

do_faultsim_test 10.3 -faults oom-t* -body {
  db one { SELECT fts5_expr('x:a', 'x') }
} -test {
  faultsim_test_result {0 {x : "a"}}







|







308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
} -test {
  faultsim_test_result {0 {"a" AND ("b" AND NEAR("a" "b", 10))}} 
}

do_faultsim_test 10.2 -faults oom-t* -body {
  db one { SELECT fts5_expr_tcl('x:"a b c" AND b NEAR(a b)', 'ns', 'x') }
} -test {
  set res {AND [ns -col 0 -- {a b c}] [AND [ns -- {b}] [ns -near 10 -- {a} {b}]]}
  faultsim_test_result [list 0 $res]
}

do_faultsim_test 10.3 -faults oom-t* -body {
  db one { SELECT fts5_expr('x:a', 'x') }
} -test {
  faultsim_test_result {0 {x : "a"}}