/ Check-in [3ba9e7ab]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Enhance the json_extract() function to reuse parses of the same JSON when the function appears multiple times in the same query.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: 3ba9e7ab79e32090a71c7aa152da9bd5951940ae5e2cf433bceadeb6bfca193f
User & Date: drh 2017-05-11 18:42:19
Context
2017-05-11
19:09
Add another test case for the new authorizer functionality. check-in: 339df63f user: dan tags: trunk
18:49
Merge recent enhancements from trunk. check-in: b55c0f14 user: drh tags: apple-osx
18:42
Enhance the json_extract() function to reuse parses of the same JSON when the function appears multiple times in the same query. check-in: 3ba9e7ab user: drh tags: trunk
18:14
Do not save the state of an fts5 merge operation mid-way through an input term, even if no previous entry for that term has caused any output. Doing so may corrupt the FTS index. check-in: 9a2de4f0 user: dan tags: trunk
16:49
Cache the JSON parse used by json_extract(). Closed-Leaf check-in: 44ca6c2c user: drh tags: auxdata-cache
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/misc/json1.c.

   167    167     u32 nAlloc;        /* Number of slots of aNode[] allocated */
   168    168     JsonNode *aNode;   /* Array of nodes containing the parse */
   169    169     const char *zJson; /* Original JSON string */
   170    170     u32 *aUp;          /* Index of parent of each node */
   171    171     u8 oom;            /* Set to true if out of memory */
   172    172     u8 nErr;           /* Number of errors seen */
   173    173     u16 iDepth;        /* Nesting depth */
          174  +  int nJson;         /* Length of the zJson string in bytes */
   174    175   };
   175    176   
   176    177   /*
   177    178   ** Maximum nesting depth of JSON for this implementation.
   178    179   **
   179    180   ** This limit is needed to avoid a stack overflow in the recursive
   180    181   ** descent parser.  A depth of 2000 is far deeper than any sane JSON
................................................................................
   408    409     sqlite3_free(pParse->aNode);
   409    410     pParse->aNode = 0;
   410    411     pParse->nNode = 0;
   411    412     pParse->nAlloc = 0;
   412    413     sqlite3_free(pParse->aUp);
   413    414     pParse->aUp = 0;
   414    415   }
          416  +
          417  +/*
          418  +** Free a JsonParse object that was obtained from sqlite3_malloc().
          419  +*/
          420  +static void jsonParseFree(JsonParse *pParse){
          421  +  jsonParseReset(pParse);
          422  +  sqlite3_free(pParse);
          423  +}
   415    424   
   416    425   /*
   417    426   ** Convert the JsonNode pNode into a pure JSON string and
   418    427   ** append to pOut.  Subsubstructure is also included.  Return
   419    428   ** the number of JsonNode objects that are encoded.
   420    429   */
   421    430   static void jsonRenderNode(
................................................................................
   959    968     if( aUp==0 ){
   960    969       pParse->oom = 1;
   961    970       return SQLITE_NOMEM;
   962    971     }
   963    972     jsonParseFillInParentage(pParse, 0, 0);
   964    973     return SQLITE_OK;
   965    974   }
          975  +
          976  +/*
          977  +** Magic number used for the JSON parse cache in sqlite3_get_auxdata()
          978  +*/
          979  +#define JSON_CACHE_ID  (-429938)
          980  +
          981  +/*
          982  +** Obtain a complete parse of the JSON found in the first argument
          983  +** of the argv array.  Use the sqlite3_get_auxdata() cache for this
          984  +** parse if it is available.  If the cache is not available or if it
          985  +** is no longer valid, parse the JSON again and return the new parse,
          986  +** and also register the new parse so that it will be available for
          987  +** future sqlite3_get_auxdata() calls.
          988  +*/
          989  +static JsonParse *jsonParseCached(
          990  +  sqlite3_context *pCtx,
          991  +  sqlite3_value **argv
          992  +){
          993  +  const char *zJson = (const char*)sqlite3_value_text(argv[0]);
          994  +  int nJson = sqlite3_value_bytes(argv[0]);
          995  +  JsonParse *p;
          996  +  if( zJson==0 ) return 0;
          997  +  p = (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID);
          998  +  if( p && p->nJson==nJson && memcmp(p->zJson,zJson,nJson)==0 ){
          999  +    p->nErr = 0;
         1000  +    return p; /* The cached entry matches, so return it */
         1001  +  }
         1002  +  p = sqlite3_malloc( sizeof(*p) + nJson + 1 );
         1003  +  if( p==0 ){
         1004  +    sqlite3_result_error_nomem(pCtx);
         1005  +    return 0;
         1006  +  }
         1007  +  memset(p, 0, sizeof(*p));
         1008  +  p->zJson = (char*)&p[1];
         1009  +  memcpy((char*)p->zJson, zJson, nJson+1);
         1010  +  if( jsonParse(p, pCtx, p->zJson) ){
         1011  +    sqlite3_free(p);
         1012  +    return 0;
         1013  +  }
         1014  +  p->nJson = nJson;
         1015  +  sqlite3_set_auxdata(pCtx, JSON_CACHE_ID, p, (void(*)(void*))jsonParseFree);
         1016  +  return (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID);
         1017  +}
   966   1018   
   967   1019   /*
   968   1020   ** Compare the OBJECT label at pNode against zKey,nKey.  Return true on
   969   1021   ** a match.
   970   1022   */
   971   1023   static int jsonLabelCompare(JsonNode *pNode, const char *zKey, u32 nKey){
   972   1024     if( pNode->jnFlags & JNODE_RAW ){
................................................................................
  1325   1377   ** Return 0 if the input is not a well-formed JSON array.
  1326   1378   */
  1327   1379   static void jsonArrayLengthFunc(
  1328   1380     sqlite3_context *ctx,
  1329   1381     int argc,
  1330   1382     sqlite3_value **argv
  1331   1383   ){
  1332         -  JsonParse x;          /* The parse */
         1384  +  JsonParse *p;          /* The parse */
  1333   1385     sqlite3_int64 n = 0;
  1334   1386     u32 i;
  1335   1387     JsonNode *pNode;
  1336   1388   
  1337         -  if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return;
  1338         -  assert( x.nNode );
         1389  +  p = jsonParseCached(ctx, argv);
         1390  +  if( p==0 ) return;
         1391  +  assert( p->nNode );
  1339   1392     if( argc==2 ){
  1340   1393       const char *zPath = (const char*)sqlite3_value_text(argv[1]);
  1341         -    pNode = jsonLookup(&x, zPath, 0, ctx);
         1394  +    pNode = jsonLookup(p, zPath, 0, ctx);
  1342   1395     }else{
  1343         -    pNode = x.aNode;
         1396  +    pNode = p->aNode;
  1344   1397     }
  1345   1398     if( pNode==0 ){
  1346         -    x.nErr = 1;
  1347         -  }else if( pNode->eType==JSON_ARRAY ){
         1399  +    return;
         1400  +  }
         1401  +  if( pNode->eType==JSON_ARRAY ){
  1348   1402       assert( (pNode->jnFlags & JNODE_APPEND)==0 );
  1349   1403       for(i=1; i<=pNode->n; n++){
  1350   1404         i += jsonNodeSize(&pNode[i]);
  1351   1405       }
  1352   1406     }
  1353         -  if( x.nErr==0 ) sqlite3_result_int64(ctx, n);
  1354         -  jsonParseReset(&x);
         1407  +  sqlite3_result_int64(ctx, n);
  1355   1408   }
  1356   1409   
  1357   1410   /*
  1358   1411   ** json_extract(JSON, PATH, ...)
  1359   1412   **
  1360   1413   ** Return the element described by PATH.  Return NULL if there is no
  1361   1414   ** PATH element.  If there are multiple PATHs, then return a JSON array
................................................................................
  1363   1416   ** is malformed.
  1364   1417   */
  1365   1418   static void jsonExtractFunc(
  1366   1419     sqlite3_context *ctx,
  1367   1420     int argc,
  1368   1421     sqlite3_value **argv
  1369   1422   ){
  1370         -  JsonParse x;          /* The parse */
         1423  +  JsonParse *p;          /* The parse */
  1371   1424     JsonNode *pNode;
  1372   1425     const char *zPath;
  1373   1426     JsonString jx;
  1374   1427     int i;
  1375   1428   
  1376   1429     if( argc<2 ) return;
  1377         -  if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return;
         1430  +  p = jsonParseCached(ctx, argv);
         1431  +  if( p==0 ) return;
  1378   1432     jsonInit(&jx, ctx);
  1379   1433     jsonAppendChar(&jx, '[');
  1380   1434     for(i=1; i<argc; i++){
  1381   1435       zPath = (const char*)sqlite3_value_text(argv[i]);
  1382         -    pNode = jsonLookup(&x, zPath, 0, ctx);
  1383         -    if( x.nErr ) break;
         1436  +    pNode = jsonLookup(p, zPath, 0, ctx);
         1437  +    if( p->nErr ) break;
  1384   1438       if( argc>2 ){
  1385   1439         jsonAppendSeparator(&jx);
  1386   1440         if( pNode ){
  1387   1441           jsonRenderNode(pNode, &jx, 0);
  1388   1442         }else{
  1389   1443           jsonAppendRaw(&jx, "null", 4);
  1390   1444         }
................................................................................
  1394   1448     }
  1395   1449     if( argc>2 && i==argc ){
  1396   1450       jsonAppendChar(&jx, ']');
  1397   1451       jsonResult(&jx);
  1398   1452       sqlite3_result_subtype(ctx, JSON_SUBTYPE);
  1399   1453     }
  1400   1454     jsonReset(&jx);
  1401         -  jsonParseReset(&x);
  1402   1455   }
  1403   1456   
  1404   1457   /* This is the RFC 7396 MergePatch algorithm.
  1405   1458   */
  1406   1459   static JsonNode *jsonMergePatch(
  1407   1460     JsonParse *pParse,   /* The JSON parser that contains the TARGET */
  1408   1461     u32 iTarget,         /* Node of the TARGET in pParse */

Changes to src/sqlite.h.in.

  4759   4759   ** of where this might be useful is in a regular-expression matching
  4760   4760   ** function. The compiled version of the regular expression can be stored as
  4761   4761   ** metadata associated with the pattern string.  
  4762   4762   ** Then as long as the pattern string remains the same,
  4763   4763   ** the compiled regular expression can be reused on multiple
  4764   4764   ** invocations of the same function.
  4765   4765   **
  4766         -** ^The sqlite3_get_auxdata() interface returns a pointer to the metadata
  4767         -** associated by the sqlite3_set_auxdata() function with the Nth argument
  4768         -** value to the application-defined function. ^If there is no metadata
  4769         -** associated with the function argument, this sqlite3_get_auxdata() interface
         4766  +** ^The sqlite3_get_auxdata(C,N) interface returns a pointer to the metadata
         4767  +** associated by the sqlite3_set_auxdata(C,N,P,X) function with the Nth argument
         4768  +** value to the application-defined function.  ^N is zero for the left-most
         4769  +** function argument.  ^If there is no metadata
         4770  +** associated with the function argument, the sqlite3_get_auxdata(C,N) interface
  4770   4771   ** returns a NULL pointer.
  4771   4772   **
  4772   4773   ** ^The sqlite3_set_auxdata(C,N,P,X) interface saves P as metadata for the N-th
  4773   4774   ** argument of the application-defined function.  ^Subsequent
  4774   4775   ** calls to sqlite3_get_auxdata(C,N) return P from the most recent
  4775   4776   ** sqlite3_set_auxdata(C,N,P,X) call if the metadata is still valid or
  4776   4777   ** NULL if the metadata has been discarded.
................................................................................
  4792   4793   ** should be called near the end of the function implementation and the
  4793   4794   ** function implementation should not make any use of P after
  4794   4795   ** sqlite3_set_auxdata() has been called.
  4795   4796   **
  4796   4797   ** ^(In practice, metadata is preserved between function calls for
  4797   4798   ** function parameters that are compile-time constants, including literal
  4798   4799   ** values and [parameters] and expressions composed from the same.)^
         4800  +**
         4801  +** The value of the N parameter to these interfaces should be non-negative.
         4802  +** Future enhancements may make use of negative N values to define new
         4803  +** kinds of function caching behavior.
  4799   4804   **
  4800   4805   ** These routines must be called from the same thread in which
  4801   4806   ** the SQL function is running.
  4802   4807   */
  4803   4808   void *sqlite3_get_auxdata(sqlite3_context*, int N);
  4804   4809   void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*));
  4805   4810   

Changes to src/vdbeapi.c.

   800    800       return (void*)p->pMem->z;
   801    801     }
   802    802   }
   803    803   
   804    804   /*
   805    805   ** Return the auxiliary data pointer, if any, for the iArg'th argument to
   806    806   ** the user-function defined by pCtx.
          807  +**
          808  +** The left-most argument is 0.
          809  +**
          810  +** Undocumented behavior:  If iArg is negative then access a cache of
          811  +** auxiliary data pointers that is available to all functions within a
          812  +** single prepared statement.  The iArg values must match.
   807    813   */
   808    814   void *sqlite3_get_auxdata(sqlite3_context *pCtx, int iArg){
   809    815     AuxData *pAuxData;
   810    816   
   811    817     assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
   812    818   #if SQLITE_ENABLE_STAT3_OR_STAT4
   813    819     if( pCtx->pVdbe==0 ) return 0;
   814    820   #else
   815    821     assert( pCtx->pVdbe!=0 );
   816    822   #endif
   817    823     for(pAuxData=pCtx->pVdbe->pAuxData; pAuxData; pAuxData=pAuxData->pNextAux){
   818         -    if( pAuxData->iAuxOp==pCtx->iOp && pAuxData->iAuxArg==iArg ){
          824  +    if(  pAuxData->iAuxArg==iArg && (pAuxData->iAuxOp==pCtx->iOp || iArg<0) ){
   819    825         return pAuxData->pAux;
   820    826       }
   821    827     }
   822    828     return 0;
   823    829   }
   824    830   
   825    831   /*
   826    832   ** Set the auxiliary data pointer and delete function, for the iArg'th
   827    833   ** argument to the user-function defined by pCtx. Any previous value is
   828    834   ** deleted by calling the delete function specified when it was set.
          835  +**
          836  +** The left-most argument is 0.
          837  +**
          838  +** Undocumented behavior:  If iArg is negative then make the data available
          839  +** to all functions within the current prepared statement using iArg as an
          840  +** access code.
   829    841   */
   830    842   void sqlite3_set_auxdata(
   831    843     sqlite3_context *pCtx, 
   832    844     int iArg, 
   833    845     void *pAux, 
   834    846     void (*xDelete)(void*)
   835    847   ){
   836    848     AuxData *pAuxData;
   837    849     Vdbe *pVdbe = pCtx->pVdbe;
   838    850   
   839    851     assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
   840         -  if( iArg<0 ) goto failed;
   841    852   #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
   842    853     if( pVdbe==0 ) goto failed;
   843    854   #else
   844    855     assert( pVdbe!=0 );
   845    856   #endif
   846    857   
   847    858     for(pAuxData=pVdbe->pAuxData; pAuxData; pAuxData=pAuxData->pNextAux){
   848         -    if( pAuxData->iAuxOp==pCtx->iOp && pAuxData->iAuxArg==iArg ) break;
          859  +    if( pAuxData->iAuxArg==iArg && (pAuxData->iAuxOp==pCtx->iOp || iArg<0) ){
          860  +      break;
          861  +    }
   849    862     }
   850    863     if( pAuxData==0 ){
   851    864       pAuxData = sqlite3DbMallocZero(pVdbe->db, sizeof(AuxData));
   852    865       if( !pAuxData ) goto failed;
   853    866       pAuxData->iAuxOp = pCtx->iOp;
   854    867       pAuxData->iAuxArg = iArg;
   855    868       pAuxData->pNextAux = pVdbe->pAuxData;

Changes to src/vdbeaux.c.

  2964   2964   **    * the corresponding bit in argument mask is clear (where the first
  2965   2965   **      function parameter corresponds to bit 0 etc.).
  2966   2966   */
  2967   2967   void sqlite3VdbeDeleteAuxData(sqlite3 *db, AuxData **pp, int iOp, int mask){
  2968   2968     while( *pp ){
  2969   2969       AuxData *pAux = *pp;
  2970   2970       if( (iOp<0)
  2971         -     || (pAux->iAuxOp==iOp 
         2971  +     || (pAux->iAuxOp==iOp
         2972  +          && pAux->iAuxArg>=0
  2972   2973             && (pAux->iAuxArg>31 || !(mask & MASKBIT32(pAux->iAuxArg))))
  2973   2974       ){
  2974   2975         testcase( pAux->iAuxArg==31 );
  2975   2976         if( pAux->xDeleteAux ){
  2976   2977           pAux->xDeleteAux(pAux->pAux);
  2977   2978         }
  2978   2979         *pp = pAux->pNextAux;