/ Check-in [b90dbaed]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix problems causing undefined left-shift operations in the fts3 snippet() function.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: b90dbaed3092236e97f9796fa63989a3648060e16189e1267c430f4a7e799fac
User & Date: dan 2019-01-18 19:26:48
Context
2019-01-18
19:33
Typecasts added to the GeoPoly extension to avoid harmless UBSAN warnings. check-in: a1f6a093 user: drh tags: trunk
19:26
Fix problems causing undefined left-shift operations in the fts3 snippet() function. check-in: b90dbaed user: dan tags: trunk
18:52
Avoid integer overflow when computing the array of a bounding box with the rtree_i32 virtual table. check-in: b352f159 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3_snippet.c.

429
430
431
432
433
434
435
436

437
438
439
440
441
442
443
...
656
657
658
659
660
661
662

663
664
665
666
667
668
669
....
1428
1429
1430
1431
1432
1433
1434




1435
1436
1437
1438
1439
1440
1441
      char *pCsr = pPhrase->pTail;
      int iCsr = pPhrase->iTail;

      while( iCsr<(iStart+pIter->nSnippet) ){
        int j;
        u64 mPhrase = (u64)1 << i;
        u64 mPos = (u64)1 << (iCsr - iStart);
        assert( iCsr>=iStart );

        if( (mCover|mCovered)&mPhrase ){
          iScore++;
        }else{
          iScore += 1000;
        }
        mCover |= mPhrase;

................................................................................
  if( hlmask ){
    int nLeft;                    /* Tokens to the left of first highlight */
    int nRight;                   /* Tokens to the right of last highlight */
    int nDesired;                 /* Ideal number of tokens to shift forward */

    for(nLeft=0; !(hlmask & ((u64)1 << nLeft)); nLeft++);
    for(nRight=0; !(hlmask & ((u64)1 << (nSnippet-1-nRight))); nRight++);

    nDesired = (nLeft-nRight)/2;

    /* Ideally, the start of the snippet should be pushed forward in the
    ** document nDesired tokens. This block checks if there are actually
    ** nDesired tokens to the right of the snippet. If so, *piPos and
    ** *pHlMask are updated to shift the snippet nDesired tokens to the
    ** right. Otherwise, the snippet is shifted by the number of tokens
................................................................................
  SnippetFragment aSnippet[4];    /* Maximum of 4 fragments per snippet */
  int nFToken = -1;               /* Number of tokens in each fragment */

  if( !pCsr->pExpr ){
    sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC);
    return;
  }





  for(nSnippet=1; 1; nSnippet++){

    int iSnip;                    /* Loop counter 0..nSnippet-1 */
    u64 mCovered = 0;             /* Bitmask of phrases covered by snippet */
    u64 mSeen = 0;                /* Bitmask of phrases seen by BestSnippet() */








|
>







 







>







 







>
>
>
>







429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
...
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
....
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
      char *pCsr = pPhrase->pTail;
      int iCsr = pPhrase->iTail;

      while( iCsr<(iStart+pIter->nSnippet) ){
        int j;
        u64 mPhrase = (u64)1 << i;
        u64 mPos = (u64)1 << (iCsr - iStart);
        assert( iCsr>=iStart && (iCsr - iStart)<=64 );
        assert( i>=0 && i<=64 );
        if( (mCover|mCovered)&mPhrase ){
          iScore++;
        }else{
          iScore += 1000;
        }
        mCover |= mPhrase;

................................................................................
  if( hlmask ){
    int nLeft;                    /* Tokens to the left of first highlight */
    int nRight;                   /* Tokens to the right of last highlight */
    int nDesired;                 /* Ideal number of tokens to shift forward */

    for(nLeft=0; !(hlmask & ((u64)1 << nLeft)); nLeft++);
    for(nRight=0; !(hlmask & ((u64)1 << (nSnippet-1-nRight))); nRight++);
    assert( (nSnippet-1-nRight)<=63 && (nSnippet-1-nRight)>=0 );
    nDesired = (nLeft-nRight)/2;

    /* Ideally, the start of the snippet should be pushed forward in the
    ** document nDesired tokens. This block checks if there are actually
    ** nDesired tokens to the right of the snippet. If so, *piPos and
    ** *pHlMask are updated to shift the snippet nDesired tokens to the
    ** right. Otherwise, the snippet is shifted by the number of tokens
................................................................................
  SnippetFragment aSnippet[4];    /* Maximum of 4 fragments per snippet */
  int nFToken = -1;               /* Number of tokens in each fragment */

  if( !pCsr->pExpr ){
    sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC);
    return;
  }

  /* Limit the snippet length to 64 tokens. */
  if( nToken<-64 ) nToken = -64;
  if( nToken>+64 ) nToken = +64;

  for(nSnippet=1; 1; nSnippet++){

    int iSnip;                    /* Loop counter 0..nSnippet-1 */
    u64 mCovered = 0;             /* Bitmask of phrases covered by snippet */
    u64 mSeen = 0;                /* Bitmask of phrases seen by BestSnippet() */

Changes to test/fts3snippet.test.

551
552
553
554
555
556
557





558
559
560
561
562
  set x35 [string trim [string repeat "x " 35]]
  execsql "INSERT INTO t4 VALUES('$x35 E $x35 F $x35 G $x35');"
  llength [db one {
    SELECT snippet(t4, '', '', '', 0, 64) FROM t4 WHERE t4 MATCH 'E'
  }]
} {64}









set sqlite_fts3_enable_parentheses 0
finish_test







>
>
>
>
>





551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
  set x35 [string trim [string repeat "x " 35]]
  execsql "INSERT INTO t4 VALUES('$x35 E $x35 F $x35 G $x35');"
  llength [db one {
    SELECT snippet(t4, '', '', '', 0, 64) FROM t4 WHERE t4 MATCH 'E'
  }]
} {64}

do_test 4.3 {
  llength [db one {
    SELECT snippet(t4, '', '', '', 0, 150) FROM t4 WHERE t4 MATCH 'E'
  }]
} {64}



set sqlite_fts3_enable_parentheses 0
finish_test