/ Check-in [61bd2a88]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge the latest trunk changes, especially the RTREE enhancement to use sqlite3_blob objects.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | sqlite3_blob_reset
Files: files | file ages | folders
SHA1: 61bd2a885da45e7fa4f02ea1c44d6706faa6bf46
User & Date: drh 2017-02-04 14:30:57
Context
2017-02-04
14:30
Merge the latest trunk changes, especially the RTREE enhancement to use sqlite3_blob objects. Leaf check-in: 61bd2a88 user: drh tags: sqlite3_blob_reset
14:24
In RTREE, use an sqlite3_blob object rather than an sqlite3_stmt object for reading content out of the %_node shadow table. check-in: 97ccf3e4 user: drh tags: trunk
2017-02-02
23:57
Add the sqlite3_blob_reset() interface. Enhance the behavior of sqlite3_blob objects so that they can go active again after encountering an error by rerunning sqlite3_blob_reopen(). More work needed on the documentation. check-in: 53b77838 user: drh tags: sqlite3_blob_reset
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/rtree/rtree.c.

   115    115     sqlite3_vtab base;          /* Base class.  Must be first */
   116    116     sqlite3 *db;                /* Host database connection */
   117    117     int iNodeSize;              /* Size in bytes of each node in the node table */
   118    118     u8 nDim;                    /* Number of dimensions */
   119    119     u8 nDim2;                   /* Twice the number of dimensions */
   120    120     u8 eCoordType;              /* RTREE_COORD_REAL32 or RTREE_COORD_INT32 */
   121    121     u8 nBytesPerCell;           /* Bytes consumed per cell */
          122  +  u8 inWrTrans;               /* True if inside write transaction */
   122    123     int iDepth;                 /* Current depth of the r-tree structure */
   123    124     char *zDb;                  /* Name of database containing r-tree table */
   124    125     char *zName;                /* Name of r-tree table */ 
   125         -  int nBusy;                  /* Current number of users of this structure */
          126  +  u32 nBusy;                  /* Current number of users of this structure */
   126    127     i64 nRowEst;                /* Estimated number of rows in this table */
          128  +  u32 nCursor;                /* Number of open cursors */
   127    129   
   128    130     /* List of nodes removed during a CondenseTree operation. List is
   129    131     ** linked together via the pointer normally used for hash chains -
   130    132     ** RtreeNode.pNext. RtreeNode.iNode stores the depth of the sub-tree 
   131    133     ** headed by the node (leaf nodes have RtreeNode.iNode==0).
   132    134     */
   133    135     RtreeNode *pDeleted;
   134    136     int iReinsertHeight;        /* Height of sub-trees Reinsert() has run on */
          137  +
          138  +  /* Blob I/O on xxx_node */
          139  +  sqlite3_blob *pNodeBlob;
   135    140   
   136    141     /* Statements to read/write/delete a record from xxx_node */
   137         -  sqlite3_stmt *pReadNode;
   138    142     sqlite3_stmt *pWriteNode;
   139    143     sqlite3_stmt *pDeleteNode;
   140    144   
   141    145     /* Statements to read/write/delete a record from xxx_rowid */
   142    146     sqlite3_stmt *pReadRowid;
   143    147     sqlite3_stmt *pWriteRowid;
   144    148     sqlite3_stmt *pDeleteRowid;
................................................................................
   381    385   # define CLANG_VERSION 0
   382    386   #endif
   383    387   #endif
   384    388   
   385    389   /* The testcase() macro should already be defined in the amalgamation.  If
   386    390   ** it is not, make it a no-op.
   387    391   */
   388         -#ifndef SQLITE_AMALGMATION
          392  +#ifndef SQLITE_AMALGAMATION
   389    393   # define testcase(X)
   390    394   #endif
   391    395   
   392    396   /*
   393    397   ** Macros to determine whether the machine is big or little endian,
   394    398   ** and whether or not that determination is run-time or compile-time.
   395    399   **
................................................................................
   610    614       pNode->nRef = 1;
   611    615       pNode->pParent = pParent;
   612    616       pNode->isDirty = 1;
   613    617       nodeReference(pParent);
   614    618     }
   615    619     return pNode;
   616    620   }
          621  +
          622  +/*
          623  +** Clear the Rtree.pNodeBlob object
          624  +*/
          625  +static void nodeBlobReset(Rtree *pRtree){
          626  +  if( pRtree->pNodeBlob && pRtree->inWrTrans==0 && pRtree->nCursor==0 ){
          627  +    sqlite3_blob *pBlob = pRtree->pNodeBlob;
          628  +    pRtree->pNodeBlob = 0;
          629  +    sqlite3_blob_close(pBlob);
          630  +  }
          631  +}
   617    632   
   618    633   /*
   619    634   ** Obtain a reference to an r-tree node.
   620    635   */
   621    636   static int nodeAcquire(
   622    637     Rtree *pRtree,             /* R-tree structure */
   623    638     i64 iNode,                 /* Node number to load */
   624    639     RtreeNode *pParent,        /* Either the parent node or NULL */
   625    640     RtreeNode **ppNode         /* OUT: Acquired node */
   626    641   ){
   627         -  int rc;
   628         -  int rc2 = SQLITE_OK;
   629         -  RtreeNode *pNode;
          642  +  int rc = SQLITE_OK;
          643  +  RtreeNode *pNode = 0;
   630    644   
   631    645     /* Check if the requested node is already in the hash table. If so,
   632    646     ** increase its reference count and return it.
   633    647     */
   634    648     if( (pNode = nodeHashLookup(pRtree, iNode)) ){
   635    649       assert( !pParent || !pNode->pParent || pNode->pParent==pParent );
   636    650       if( pParent && !pNode->pParent ){
................................................................................
   638    652         pNode->pParent = pParent;
   639    653       }
   640    654       pNode->nRef++;
   641    655       *ppNode = pNode;
   642    656       return SQLITE_OK;
   643    657     }
   644    658   
   645         -  sqlite3_bind_int64(pRtree->pReadNode, 1, iNode);
   646         -  rc = sqlite3_step(pRtree->pReadNode);
   647         -  if( rc==SQLITE_ROW ){
   648         -    const u8 *zBlob = sqlite3_column_blob(pRtree->pReadNode, 0);
   649         -    if( pRtree->iNodeSize==sqlite3_column_bytes(pRtree->pReadNode, 0) ){
   650         -      pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode)+pRtree->iNodeSize);
   651         -      if( !pNode ){
   652         -        rc2 = SQLITE_NOMEM;
   653         -      }else{
   654         -        pNode->pParent = pParent;
   655         -        pNode->zData = (u8 *)&pNode[1];
   656         -        pNode->nRef = 1;
   657         -        pNode->iNode = iNode;
   658         -        pNode->isDirty = 0;
   659         -        pNode->pNext = 0;
   660         -        memcpy(pNode->zData, zBlob, pRtree->iNodeSize);
   661         -        nodeReference(pParent);
   662         -      }
   663         -    }
   664         -  }
   665         -  rc = sqlite3_reset(pRtree->pReadNode);
   666         -  if( rc==SQLITE_OK ) rc = rc2;
          659  +  if( pRtree->pNodeBlob ){
          660  +    sqlite3_blob *pBlob = pRtree->pNodeBlob;
          661  +    pRtree->pNodeBlob = 0;
          662  +    rc = sqlite3_blob_reopen(pBlob, iNode);
          663  +    pRtree->pNodeBlob = pBlob;
          664  +    if( rc ){
          665  +      nodeBlobReset(pRtree);
          666  +      if( rc==SQLITE_NOMEM ) return SQLITE_NOMEM;
          667  +    }
          668  +  }
          669  +  if( pRtree->pNodeBlob==0 ){
          670  +    char *zTab = sqlite3_mprintf("%s_node", pRtree->zName);
          671  +    if( zTab==0 ) return SQLITE_NOMEM;
          672  +    rc = sqlite3_blob_open(pRtree->db, pRtree->zDb, zTab, "data", iNode, 0,
          673  +                           &pRtree->pNodeBlob);
          674  +    sqlite3_free(zTab);
          675  +  }
          676  +  if( rc ){
          677  +    nodeBlobReset(pRtree);
          678  +    *ppNode = 0;
          679  +    /* If unable to open an sqlite3_blob on the desired row, that can only
          680  +    ** be because the shadow tables hold erroneous data. */
          681  +    if( rc==SQLITE_ERROR ) rc = SQLITE_CORRUPT_VTAB;
          682  +  }else if( pRtree->iNodeSize==sqlite3_blob_bytes(pRtree->pNodeBlob) ){
          683  +    pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode)+pRtree->iNodeSize);
          684  +    if( !pNode ){
          685  +      rc = SQLITE_NOMEM;
          686  +    }else{
          687  +      pNode->pParent = pParent;
          688  +      pNode->zData = (u8 *)&pNode[1];
          689  +      pNode->nRef = 1;
          690  +      pNode->iNode = iNode;
          691  +      pNode->isDirty = 0;
          692  +      pNode->pNext = 0;
          693  +      rc = sqlite3_blob_read(pRtree->pNodeBlob, pNode->zData,
          694  +                             pRtree->iNodeSize, 0);
          695  +      nodeReference(pParent);
          696  +    }
          697  +  }
   667    698   
   668    699     /* If the root node was just loaded, set pRtree->iDepth to the height
   669    700     ** of the r-tree structure. A height of zero means all data is stored on
   670    701     ** the root node. A height of one means the children of the root node
   671    702     ** are the leaves, and so on. If the depth as specified on the root node
   672    703     ** is greater than RTREE_MAX_DEPTH, the r-tree structure must be corrupt.
   673    704     */
................................................................................
   905    936   /*
   906    937   ** Decrement the r-tree reference count. When the reference count reaches
   907    938   ** zero the structure is deleted.
   908    939   */
   909    940   static void rtreeRelease(Rtree *pRtree){
   910    941     pRtree->nBusy--;
   911    942     if( pRtree->nBusy==0 ){
   912         -    sqlite3_finalize(pRtree->pReadNode);
          943  +    pRtree->inWrTrans = 0;
          944  +    pRtree->nCursor = 0;
          945  +    nodeBlobReset(pRtree);
   913    946       sqlite3_finalize(pRtree->pWriteNode);
   914    947       sqlite3_finalize(pRtree->pDeleteNode);
   915    948       sqlite3_finalize(pRtree->pReadRowid);
   916    949       sqlite3_finalize(pRtree->pWriteRowid);
   917    950       sqlite3_finalize(pRtree->pDeleteRowid);
   918    951       sqlite3_finalize(pRtree->pReadParent);
   919    952       sqlite3_finalize(pRtree->pWriteParent);
................................................................................
   943    976       pRtree->zDb, pRtree->zName, 
   944    977       pRtree->zDb, pRtree->zName,
   945    978       pRtree->zDb, pRtree->zName
   946    979     );
   947    980     if( !zCreate ){
   948    981       rc = SQLITE_NOMEM;
   949    982     }else{
          983  +    nodeBlobReset(pRtree);
   950    984       rc = sqlite3_exec(pRtree->db, zCreate, 0, 0, 0);
   951    985       sqlite3_free(zCreate);
   952    986     }
   953    987     if( rc==SQLITE_OK ){
   954    988       rtreeRelease(pRtree);
   955    989     }
   956    990   
................................................................................
   958    992   }
   959    993   
   960    994   /* 
   961    995   ** Rtree virtual table module xOpen method.
   962    996   */
   963    997   static int rtreeOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
   964    998     int rc = SQLITE_NOMEM;
          999  +  Rtree *pRtree = (Rtree *)pVTab;
   965   1000     RtreeCursor *pCsr;
   966   1001   
   967   1002     pCsr = (RtreeCursor *)sqlite3_malloc(sizeof(RtreeCursor));
   968   1003     if( pCsr ){
   969   1004       memset(pCsr, 0, sizeof(RtreeCursor));
   970   1005       pCsr->base.pVtab = pVTab;
   971   1006       rc = SQLITE_OK;
         1007  +    pRtree->nCursor++;
   972   1008     }
   973   1009     *ppCursor = (sqlite3_vtab_cursor *)pCsr;
   974   1010   
   975   1011     return rc;
   976   1012   }
   977   1013   
   978   1014   
................................................................................
   997   1033   /* 
   998   1034   ** Rtree virtual table module xClose method.
   999   1035   */
  1000   1036   static int rtreeClose(sqlite3_vtab_cursor *cur){
  1001   1037     Rtree *pRtree = (Rtree *)(cur->pVtab);
  1002   1038     int ii;
  1003   1039     RtreeCursor *pCsr = (RtreeCursor *)cur;
         1040  +  assert( pRtree->nCursor>0 );
  1004   1041     freeCursorConstraints(pCsr);
  1005   1042     sqlite3_free(pCsr->aPoint);
  1006   1043     for(ii=0; ii<RTREE_CACHE_SZ; ii++) nodeRelease(pRtree, pCsr->aNode[ii]);
  1007   1044     sqlite3_free(pCsr);
         1045  +  pRtree->nCursor--;
         1046  +  nodeBlobReset(pRtree);
  1008   1047     return SQLITE_OK;
  1009   1048   }
  1010   1049   
  1011   1050   /*
  1012   1051   ** Rtree virtual table module xEof method.
  1013   1052   **
  1014   1053   ** Return non-zero if the cursor does not currently point to a valid 
................................................................................
  1073   1112     RtreeConstraint *pConstraint,  /* The constraint to test */
  1074   1113     int eInt,                      /* True if RTree holding integer coordinates */
  1075   1114     u8 *pCellData,                 /* Raw cell content */
  1076   1115     RtreeSearchPoint *pSearch,     /* Container of this cell */
  1077   1116     sqlite3_rtree_dbl *prScore,    /* OUT: score for the cell */
  1078   1117     int *peWithin                  /* OUT: visibility of the cell */
  1079   1118   ){
  1080         -  int i;                                                /* Loop counter */
  1081   1119     sqlite3_rtree_query_info *pInfo = pConstraint->pInfo; /* Callback info */
  1082   1120     int nCoord = pInfo->nCoord;                           /* No. of coordinates */
  1083   1121     int rc;                                             /* Callback return code */
  1084   1122     RtreeCoord c;                                       /* Translator union */
  1085   1123     sqlite3_rtree_dbl aCoord[RTREE_MAX_DIMENSIONS*2];   /* Decoded coordinates */
  1086   1124   
  1087   1125     assert( pConstraint->op==RTREE_MATCH || pConstraint->op==RTREE_QUERY );
................................................................................
  1118   1156         case 4:   readCoord(pCellData+12, &c); aCoord[3] = c.i;
  1119   1157                   readCoord(pCellData+8,  &c); aCoord[2] = c.i;
  1120   1158         default:  readCoord(pCellData+4,  &c); aCoord[1] = c.i;
  1121   1159                   readCoord(pCellData,    &c); aCoord[0] = c.i;
  1122   1160       }
  1123   1161     }
  1124   1162     if( pConstraint->op==RTREE_MATCH ){
         1163  +    int eWithin = 0;
  1125   1164       rc = pConstraint->u.xGeom((sqlite3_rtree_geometry*)pInfo,
  1126         -                              nCoord, aCoord, &i);
  1127         -    if( i==0 ) *peWithin = NOT_WITHIN;
         1165  +                              nCoord, aCoord, &eWithin);
         1166  +    if( eWithin==0 ) *peWithin = NOT_WITHIN;
  1128   1167       *prScore = RTREE_ZERO;
  1129   1168     }else{
  1130   1169       pInfo->aCoord = aCoord;
  1131   1170       pInfo->iLevel = pSearch->iLevel - 1;
  1132   1171       pInfo->rScore = pInfo->rParentScore = pSearch->rScore;
  1133   1172       pInfo->eWithin = pInfo->eParentWithin = pSearch->eWithin;
  1134   1173       rc = pConstraint->u.xQueryFunc(pInfo);
................................................................................
  3147   3186       }
  3148   3187     }
  3149   3188   
  3150   3189   constraint:
  3151   3190     rtreeRelease(pRtree);
  3152   3191     return rc;
  3153   3192   }
         3193  +
         3194  +/*
         3195  +** Called when a transaction starts.
         3196  +*/
         3197  +static int rtreeBeginTransaction(sqlite3_vtab *pVtab){
         3198  +  Rtree *pRtree = (Rtree *)pVtab;
         3199  +  assert( pRtree->inWrTrans==0 );
         3200  +  pRtree->inWrTrans++;
         3201  +  return SQLITE_OK;
         3202  +}
         3203  +
         3204  +/*
         3205  +** Called when a transaction completes (either by COMMIT or ROLLBACK).
         3206  +** The sqlite3_blob object should be released at this point.
         3207  +*/
         3208  +static int rtreeEndTransaction(sqlite3_vtab *pVtab){
         3209  +  Rtree *pRtree = (Rtree *)pVtab;
         3210  +  pRtree->inWrTrans = 0;
         3211  +  nodeBlobReset(pRtree);
         3212  +  return SQLITE_OK;
         3213  +}
  3154   3214   
  3155   3215   /*
  3156   3216   ** The xRename method for rtree module virtual tables.
  3157   3217   */
  3158   3218   static int rtreeRename(sqlite3_vtab *pVtab, const char *zNewName){
  3159   3219     Rtree *pRtree = (Rtree *)pVtab;
  3160   3220     int rc = SQLITE_NOMEM;
................................................................................
  3168   3228     );
  3169   3229     if( zSql ){
  3170   3230       rc = sqlite3_exec(pRtree->db, zSql, 0, 0, 0);
  3171   3231       sqlite3_free(zSql);
  3172   3232     }
  3173   3233     return rc;
  3174   3234   }
         3235  +
  3175   3236   
  3176   3237   /*
  3177   3238   ** This function populates the pRtree->nRowEst variable with an estimate
  3178   3239   ** of the number of rows in the virtual table. If possible, this is based
  3179   3240   ** on sqlite_stat1 data. Otherwise, use RTREE_DEFAULT_ROWEST.
  3180   3241   */
  3181   3242   static int rtreeQueryStat1(sqlite3 *db, Rtree *pRtree){
................................................................................
  3228   3289     rtreeClose,                 /* xClose - close a cursor */
  3229   3290     rtreeFilter,                /* xFilter - configure scan constraints */
  3230   3291     rtreeNext,                  /* xNext - advance a cursor */
  3231   3292     rtreeEof,                   /* xEof */
  3232   3293     rtreeColumn,                /* xColumn - read data */
  3233   3294     rtreeRowid,                 /* xRowid - read data */
  3234   3295     rtreeUpdate,                /* xUpdate - write data */
  3235         -  0,                          /* xBegin - begin transaction */
  3236         -  0,                          /* xSync - sync transaction */
  3237         -  0,                          /* xCommit - commit transaction */
  3238         -  0,                          /* xRollback - rollback transaction */
         3296  +  rtreeBeginTransaction,      /* xBegin - begin transaction */
         3297  +  rtreeEndTransaction,        /* xSync - sync transaction */
         3298  +  rtreeEndTransaction,        /* xCommit - commit transaction */
         3299  +  rtreeEndTransaction,        /* xRollback - rollback transaction */
  3239   3300     0,                          /* xFindFunction - function overloading */
  3240   3301     rtreeRename,                /* xRename - rename the table */
  3241   3302     0,                          /* xSavepoint */
  3242   3303     0,                          /* xRelease */
  3243         -  0                           /* xRollbackTo */
         3304  +  0,                          /* xRollbackTo */
  3244   3305   };
  3245   3306   
  3246   3307   static int rtreeSqlInit(
  3247   3308     Rtree *pRtree, 
  3248   3309     sqlite3 *db, 
  3249   3310     const char *zDb, 
  3250   3311     const char *zPrefix, 
  3251   3312     int isCreate
  3252   3313   ){
  3253   3314     int rc = SQLITE_OK;
  3254   3315   
  3255         -  #define N_STATEMENT 9
         3316  +  #define N_STATEMENT 8
  3256   3317     static const char *azSql[N_STATEMENT] = {
  3257         -    /* Read and write the xxx_node table */
  3258         -    "SELECT data FROM '%q'.'%q_node' WHERE nodeno = :1",
         3318  +    /* Write the xxx_node table */
  3259   3319       "INSERT OR REPLACE INTO '%q'.'%q_node' VALUES(:1, :2)",
  3260   3320       "DELETE FROM '%q'.'%q_node' WHERE nodeno = :1",
  3261   3321   
  3262   3322       /* Read and write the xxx_rowid table */
  3263   3323       "SELECT nodeno FROM '%q'.'%q_rowid' WHERE rowid = :1",
  3264   3324       "INSERT OR REPLACE INTO '%q'.'%q_rowid' VALUES(:1, :2)",
  3265   3325       "DELETE FROM '%q'.'%q_rowid' WHERE rowid = :1",
................................................................................
  3289   3349       rc = sqlite3_exec(db, zCreate, 0, 0, 0);
  3290   3350       sqlite3_free(zCreate);
  3291   3351       if( rc!=SQLITE_OK ){
  3292   3352         return rc;
  3293   3353       }
  3294   3354     }
  3295   3355   
  3296         -  appStmt[0] = &pRtree->pReadNode;
  3297         -  appStmt[1] = &pRtree->pWriteNode;
  3298         -  appStmt[2] = &pRtree->pDeleteNode;
  3299         -  appStmt[3] = &pRtree->pReadRowid;
  3300         -  appStmt[4] = &pRtree->pWriteRowid;
  3301         -  appStmt[5] = &pRtree->pDeleteRowid;
  3302         -  appStmt[6] = &pRtree->pReadParent;
  3303         -  appStmt[7] = &pRtree->pWriteParent;
  3304         -  appStmt[8] = &pRtree->pDeleteParent;
         3356  +  appStmt[0] = &pRtree->pWriteNode;
         3357  +  appStmt[1] = &pRtree->pDeleteNode;
         3358  +  appStmt[2] = &pRtree->pReadRowid;
         3359  +  appStmt[3] = &pRtree->pWriteRowid;
         3360  +  appStmt[4] = &pRtree->pDeleteRowid;
         3361  +  appStmt[5] = &pRtree->pReadParent;
         3362  +  appStmt[6] = &pRtree->pWriteParent;
         3363  +  appStmt[7] = &pRtree->pDeleteParent;
  3305   3364   
  3306   3365     rc = rtreeQueryStat1(db, pRtree);
  3307   3366     for(i=0; i<N_STATEMENT && rc==SQLITE_OK; i++){
  3308   3367       char *zSql = sqlite3_mprintf(azSql[i], zDb, zPrefix);
  3309   3368       if( zSql ){
  3310   3369         rc = sqlite3_prepare_v2(db, zSql, -1, appStmt[i], 0); 
  3311   3370       }else{
................................................................................
  3436   3495     }
  3437   3496     memset(pRtree, 0, sizeof(Rtree)+nDb+nName+2);
  3438   3497     pRtree->nBusy = 1;
  3439   3498     pRtree->base.pModule = &rtreeModule;
  3440   3499     pRtree->zDb = (char *)&pRtree[1];
  3441   3500     pRtree->zName = &pRtree->zDb[nDb+1];
  3442   3501     pRtree->nDim = (u8)((argc-4)/2);
  3443         -  pRtree->nDim2 = argc - 4;
         3502  +  pRtree->nDim2 = pRtree->nDim*2;
  3444   3503     pRtree->nBytesPerCell = 8 + pRtree->nDim2*4;
  3445   3504     pRtree->eCoordType = (u8)eCoordType;
  3446   3505     memcpy(pRtree->zDb, argv[1], nDb);
  3447   3506     memcpy(pRtree->zName, argv[2], nName);
  3448   3507   
  3449   3508     /* Figure out the node size to use. */
  3450   3509     rc = getNodeSize(db, pRtree, isCreate, pzErr);

Changes to ext/rtree/rtreeA.test.

   105    105     1   "SELECT * FROM t1"
   106    106     2   "SELECT * FROM t1 WHERE rowid=5"
   107    107     3   "INSERT INTO t1 VALUES(1000, 1, 2, 3, 4)"
   108    108     4   "SELECT * FROM t1 WHERE x1<10 AND x2>12"
   109    109   }
   110    110   
   111    111   do_execsql_test  rtreeA-1.2.0 { DROP TABLE t1_node } {}
   112         -do_corruption_tests rtreeA-1.2 -error "SQL logic error or missing database" {
          112  +do_corruption_tests rtreeA-1.2 -error "database disk image is malformed" {
   113    113     1   "SELECT * FROM t1"
   114    114     2   "SELECT * FROM t1 WHERE rowid=5"
   115    115     3   "INSERT INTO t1 VALUES(1000, 1, 2, 3, 4)"
   116    116     4   "SELECT * FROM t1 WHERE x1<10 AND x2>12"
   117    117   }
   118    118   
   119    119   #-------------------------------------------------------------------------

Changes to src/expr.c.

  1414   1414       struct IdList_item *pNewItem = &pNew->a[i];
  1415   1415       struct IdList_item *pOldItem = &p->a[i];
  1416   1416       pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName);
  1417   1417       pNewItem->idx = pOldItem->idx;
  1418   1418     }
  1419   1419     return pNew;
  1420   1420   }
  1421         -Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){
  1422         -  Select *pNew, *pPrior;
         1421  +Select *sqlite3SelectDup(sqlite3 *db, Select *pDup, int flags){
         1422  +  Select *pRet = 0;
         1423  +  Select *pNext = 0;
         1424  +  Select **pp = &pRet;
         1425  +  Select *p;
         1426  +
  1423   1427     assert( db!=0 );
  1424         -  if( p==0 ) return 0;
  1425         -  pNew = sqlite3DbMallocRawNN(db, sizeof(*p) );
  1426         -  if( pNew==0 ) return 0;
  1427         -  pNew->pEList = sqlite3ExprListDup(db, p->pEList, flags);
  1428         -  pNew->pSrc = sqlite3SrcListDup(db, p->pSrc, flags);
  1429         -  pNew->pWhere = sqlite3ExprDup(db, p->pWhere, flags);
  1430         -  pNew->pGroupBy = sqlite3ExprListDup(db, p->pGroupBy, flags);
  1431         -  pNew->pHaving = sqlite3ExprDup(db, p->pHaving, flags);
  1432         -  pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy, flags);
  1433         -  pNew->op = p->op;
  1434         -  pNew->pPrior = pPrior = sqlite3SelectDup(db, p->pPrior, flags);
  1435         -  if( pPrior ) pPrior->pNext = pNew;
  1436         -  pNew->pNext = 0;
  1437         -  pNew->pLimit = sqlite3ExprDup(db, p->pLimit, flags);
  1438         -  pNew->pOffset = sqlite3ExprDup(db, p->pOffset, flags);
  1439         -  pNew->iLimit = 0;
  1440         -  pNew->iOffset = 0;
  1441         -  pNew->selFlags = p->selFlags & ~SF_UsesEphemeral;
  1442         -  pNew->addrOpenEphm[0] = -1;
  1443         -  pNew->addrOpenEphm[1] = -1;
  1444         -  pNew->nSelectRow = p->nSelectRow;
  1445         -  pNew->pWith = withDup(db, p->pWith);
  1446         -  sqlite3SelectSetName(pNew, p->zSelName);
  1447         -  return pNew;
         1428  +  for(p=pDup; p; p=p->pPrior){
         1429  +    Select *pNew = sqlite3DbMallocRawNN(db, sizeof(*p) );
         1430  +    if( pNew==0 ) break;
         1431  +    pNew->pEList = sqlite3ExprListDup(db, p->pEList, flags);
         1432  +    pNew->pSrc = sqlite3SrcListDup(db, p->pSrc, flags);
         1433  +    pNew->pWhere = sqlite3ExprDup(db, p->pWhere, flags);
         1434  +    pNew->pGroupBy = sqlite3ExprListDup(db, p->pGroupBy, flags);
         1435  +    pNew->pHaving = sqlite3ExprDup(db, p->pHaving, flags);
         1436  +    pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy, flags);
         1437  +    pNew->op = p->op;
         1438  +    pNew->pNext = pNext;
         1439  +    pNew->pPrior = 0;
         1440  +    pNew->pLimit = sqlite3ExprDup(db, p->pLimit, flags);
         1441  +    pNew->pOffset = sqlite3ExprDup(db, p->pOffset, flags);
         1442  +    pNew->iLimit = 0;
         1443  +    pNew->iOffset = 0;
         1444  +    pNew->selFlags = p->selFlags & ~SF_UsesEphemeral;
         1445  +    pNew->addrOpenEphm[0] = -1;
         1446  +    pNew->addrOpenEphm[1] = -1;
         1447  +    pNew->nSelectRow = p->nSelectRow;
         1448  +    pNew->pWith = withDup(db, p->pWith);
         1449  +    sqlite3SelectSetName(pNew, p->zSelName);
         1450  +    *pp = pNew;
         1451  +    pp = &pNew->pPrior;
         1452  +    pNext = pNew;
         1453  +  }
         1454  +
         1455  +  return pRet;
  1448   1456   }
  1449   1457   #else
  1450   1458   Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){
  1451   1459     assert( p==0 );
  1452   1460     return 0;
  1453   1461   }
  1454   1462   #endif

Changes to src/select.c.

  4182   4182         return SQLITE_ERROR;
  4183   4183       }
  4184   4184       assert( pTab->nTabRef==1 || ((pSel->selFlags&SF_Recursive) && pTab->nTabRef==2 ));
  4185   4185   
  4186   4186       pCte->zCteErr = "circular reference: %s";
  4187   4187       pSavedWith = pParse->pWith;
  4188   4188       pParse->pWith = pWith;
  4189         -    sqlite3WalkSelect(pWalker, bMayRecursive ? pSel->pPrior : pSel);
         4189  +    if( bMayRecursive ){
         4190  +      Select *pPrior = pSel->pPrior;
         4191  +      assert( pPrior->pWith==0 );
         4192  +      pPrior->pWith = pSel->pWith;
         4193  +      sqlite3WalkSelect(pWalker, pPrior);
         4194  +      pPrior->pWith = 0;
         4195  +    }else{
         4196  +      sqlite3WalkSelect(pWalker, pSel);
         4197  +    }
  4190   4198       pParse->pWith = pWith;
  4191   4199   
  4192   4200       for(pLeft=pSel; pLeft->pPrior; pLeft=pLeft->pPrior);
  4193   4201       pEList = pLeft->pEList;
  4194   4202       if( pCte->pCols ){
  4195   4203         if( pEList && pEList->nExpr!=pCte->pCols->nExpr ){
  4196   4204           sqlite3ErrorMsg(pParse, "table %s has %d values for %d columns",
................................................................................
  4226   4234   **
  4227   4235   ** This function is used as the xSelectCallback2() callback by
  4228   4236   ** sqlite3SelectExpand() when walking a SELECT tree to resolve table
  4229   4237   ** names and other FROM clause elements. 
  4230   4238   */
  4231   4239   static void selectPopWith(Walker *pWalker, Select *p){
  4232   4240     Parse *pParse = pWalker->pParse;
  4233         -  With *pWith = findRightmost(p)->pWith;
  4234         -  if( pWith!=0 ){
  4235         -    assert( pParse->pWith==pWith );
  4236         -    pParse->pWith = pWith->pOuter;
         4241  +  if( pParse->pWith && p->pPrior==0 ){
         4242  +    With *pWith = findRightmost(p)->pWith;
         4243  +    if( pWith!=0 ){
         4244  +      assert( pParse->pWith==pWith );
         4245  +      pParse->pWith = pWith->pOuter;
         4246  +    }
  4237   4247     }
  4238   4248   }
  4239   4249   #else
  4240   4250   #define selectPopWith 0
  4241   4251   #endif
  4242   4252   
  4243   4253   /*
................................................................................
  4279   4289       return WRC_Abort;
  4280   4290     }
  4281   4291     if( NEVER(p->pSrc==0) || (selFlags & SF_Expanded)!=0 ){
  4282   4292       return WRC_Prune;
  4283   4293     }
  4284   4294     pTabList = p->pSrc;
  4285   4295     pEList = p->pEList;
  4286         -  if( pWalker->xSelectCallback2==selectPopWith ){
  4287         -    sqlite3WithPush(pParse, findRightmost(p)->pWith, 0);
         4296  +  if( p->pWith ){
         4297  +    sqlite3WithPush(pParse, p->pWith, 0);
  4288   4298     }
  4289   4299   
  4290   4300     /* Make sure cursor numbers have been assigned to all entries in
  4291   4301     ** the FROM clause of the SELECT statement.
  4292   4302     */
  4293   4303     sqlite3SrcListAssignCursors(pParse, pTabList);
  4294   4304   
................................................................................
  4567   4577     w.xExprCallback = sqlite3ExprWalkNoop;
  4568   4578     w.pParse = pParse;
  4569   4579     if( pParse->hasCompound ){
  4570   4580       w.xSelectCallback = convertCompoundSelectToSubquery;
  4571   4581       sqlite3WalkSelect(&w, pSelect);
  4572   4582     }
  4573   4583     w.xSelectCallback = selectExpander;
  4574         -  if( (pSelect->selFlags & SF_MultiValue)==0 ){
  4575         -    w.xSelectCallback2 = selectPopWith;
  4576         -  }
         4584  +  w.xSelectCallback2 = selectPopWith;
  4577   4585     sqlite3WalkSelect(&w, pSelect);
  4578   4586   }
  4579   4587   
  4580   4588   
  4581   4589   #ifndef SQLITE_OMIT_SUBQUERY
  4582   4590   /*
  4583   4591   ** This is a Walker.xSelectCallback callback for the sqlite3SelectTypeInfo()

Changes to test/tabfunc01.test.

   144    144     WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100)
   145    145       INSERT INTO t600(a,b) SELECT x, printf('(%03d)',x) FROM c;
   146    146     SELECT b FROM t600 WHERE a IN generate_series(2,52,10);
   147    147   } {(002) (012) (022) (032) (042) (052)}
   148    148   
   149    149   
   150    150   do_test tabfunc01-700 {
   151         -  set PTR [intarray_addr 5 7 13 17 23]
          151  +  set PTR1 [intarray_addr 5 7 13 17 23]
   152    152     db eval {
   153         -    SELECT b FROM t600, carray($PTR,5) WHERE a=value;
          153  +    SELECT b FROM t600, carray($PTR1,5) WHERE a=value;
   154    154     }
   155    155   } {(005) (007) (013) (017) (023)}
   156    156   do_test tabfunc01-701 {
   157    157     db eval {
   158         -    SELECT b FROM t600 WHERE a IN carray($PTR,5,'int32');
          158  +    SELECT b FROM t600 WHERE a IN carray($PTR1,5,'int32');
   159    159     }
   160    160   } {(005) (007) (013) (017) (023)}
   161    161   do_test tabfunc01-702 {
   162    162     db eval {
   163         -    SELECT b FROM t600 WHERE a IN carray($PTR,4,'int32');
          163  +    SELECT b FROM t600 WHERE a IN carray($PTR1,4,'int32');
   164    164     }
   165    165   } {(005) (007) (013) (017)}
   166    166   do_catchsql_test tabfunc01-710 {
   167         -  SELECT b FROM t600 WHERE a IN carray($PTR,5,'int33');
          167  +  SELECT b FROM t600 WHERE a IN carray($PTR1,5,'int33');
   168    168   } {1 {unknown datatype: 'int33'}}
   169    169   
   170    170   do_test tabfunc01-720 {
   171         -  set PTR [int64array_addr 5 7 13 17 23]
          171  +  set PTR2 [int64array_addr 5 7 13 17 23]
   172    172     db eval {
   173         -    SELECT b FROM t600, carray($PTR,5,'int64') WHERE a=value;
          173  +    SELECT b FROM t600, carray($PTR2,5,'int64') WHERE a=value;
   174    174     }
   175    175   } {(005) (007) (013) (017) (023)}
   176    176   do_test tabfunc01-721 {
   177    177     db eval {
   178         -    SELECT remember(123,$PTR);
   179         -    SELECT value FROM carray($PTR,5,'int64');
          178  +    SELECT remember(123,$PTR2);
          179  +    SELECT value FROM carray($PTR2,5,'int64');
   180    180     }
   181    181   } {123 123 7 13 17 23}
   182    182   do_test tabfunc01-722 {
   183         -  set PTR2 [expr {$PTR+16}]
          183  +  set PTR3 [expr {$PTR2+16}]
   184    184     db eval {
   185         -    SELECT remember(987,$PTR2);
   186         -    SELECT value FROM carray($PTR,5,'int64');
          185  +    SELECT remember(987,$PTR3);
          186  +    SELECT value FROM carray($PTR2,5,'int64');
   187    187     }
   188    188   } {987 123 7 987 17 23}
   189    189   
   190    190   do_test tabfunc01-730 {
   191         -  set PTR [doublearray_addr 5.0 7.0 13.0 17.0 23.0]
          191  +  set PTR4 [doublearray_addr 5.0 7.0 13.0 17.0 23.0]
   192    192     db eval {
   193         -    SELECT b FROM t600, carray($PTR,5,'double') WHERE a=value;
          193  +    SELECT b FROM t600, carray($PTR4,5,'double') WHERE a=value;
   194    194     }
   195    195   } {(005) (007) (013) (017) (023)}
   196    196   
   197    197   do_test tabfunc01-740 {
   198         -  set PTR [textarray_addr 5 7 13 17 23]
          198  +  set PTR5 [textarray_addr x5 x7 x13 x17 x23]
   199    199     db eval {
   200         -    SELECT b FROM t600, carray($PTR,5,'char*') WHERE a=value;
          200  +    SELECT b FROM t600, carray($PTR5,5,'char*') WHERE a=trim(value,'x');
   201    201     }
   202    202   } {(005) (007) (013) (017) (023)}
   203    203   
          204  +do_test tabfunc01-750 {
          205  +  db eval {
          206  +    SELECT aa.value, bb.value, '|'
          207  +      FROM carray($PTR4,5,'double') AS aa
          208  +      JOIN carray($PTR5,5,'char*') AS bb ON aa.rowid=bb.rowid;
          209  +  }
          210  +} {5.0 x5 | 7.0 x7 | 13.0 x13 | 17.0 x17 | 23.0 x23 |}
   204    211   
          212  +# Free up memory allocations
   205    213   intarray_addr
   206    214   int64array_addr
   207    215   doublearray_addr
   208    216   textarray_addr
   209    217   
   210    218   finish_test