Enhance the JSON parser cache such that it is able to extract lines from
the cache and use them for writing, though they then have to be evicted from
the cache. This was an experiment in trying to reduce the amount of parsing
needed for a big UPDATE, but it does not seem to help any. Retained for
reference only.
FossilOrigin-Name: 2e6fbebc41f1a24f8233b136131e6c392d7474f24946ef164fe3600afbd02357
diff --git a/manifest b/manifest
index 575934e..042925f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Two\sminor\simprovements\sto\ssum(),\sone\sof\swhich\swas\sinspired\sby\n[forum:/forumpost/af5be98dbc|forum\spost\saf5be98dbc].
-D 2023-07-19T09:52:10.467
+C Enhance\sthe\sJSON\sparser\scache\ssuch\sthat\sit\sis\sable\sto\sextract\slines\sfrom\nthe\scache\sand\suse\sthem\sfor\swriting,\sthough\sthey\sthen\shave\sto\sbe\sevicted\sfrom\nthe\scache.\s\sThis\swas\san\sexperiment\sin\strying\sto\sreduce\sthe\samount\sof\sparsing\nneeded\sfor\sa\sbig\sUPDATE,\sbut\sit\sdoes\snot\sseem\sto\shelp\sany.\s\sRetained\sfor\nreference\sonly.
+D 2023-07-19T12:52:09.770
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -597,7 +597,7 @@
F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
-F src/json.c 14c474fb1249a46eb44e878e2361f36abfe686b134039b0d1883d93d61505b4a
+F src/json.c c011e8a5c2a6ef3b310412dd16de933370655d643c0dff33c81cf96a1088e223
F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465
F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002
@@ -2043,8 +2043,11 @@
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 76152ad2ffe56034f2fd93d9a1ce9358e1677a7e9cd3dcd9f3a34a5c956a463e
-R 47d8d8ca9ec8151ad0210262a08ac711
+P a0d3e7571aded8d1e03908059d2d5aa5d62ec49bff099cb38f6f35df5e4b18b5
+R 1ad9eb2d7a5390b633470ac87bec02c6
+T *branch * json-write-cache
+T *sym-json-write-cache *
+T -sym-trunk *
U drh
-Z b98de7129167c58b9385610fbdec3119
+Z 11a0a92f5fb8088da9a1185f8dea3559
# Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 4a5750f..529b0a4 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a0d3e7571aded8d1e03908059d2d5aa5d62ec49bff099cb38f6f35df5e4b18b5
\ No newline at end of file
+2e6fbebc41f1a24f8233b136131e6c392d7474f24946ef164fe3600afbd02357
\ No newline at end of file
diff --git a/src/json.c b/src/json.c
index 6bad1c1..b1d5be2 100644
--- a/src/json.c
+++ b/src/json.c
@@ -135,9 +135,10 @@
u8 nErr; /* Number of errors seen */
u8 oom; /* Set to true if out of memory */
u8 hasNonstd; /* True if input uses non-standard features like JSON5 */
+ u8 nRef; /* Number of references to this object */
int nJson; /* Length of the zJson string in bytes */
u32 iErr; /* Error location in zJson[] */
- u32 iHold; /* Replace cache line with the lowest iHold value */
+ u32 iHold; /* Creation sequence number, for LRU replacement */
};
/*
@@ -505,14 +506,19 @@
pParse->nAlloc = 0;
sqlite3_free(pParse->aUp);
pParse->aUp = 0;
+ pParse->nRef = 0;
}
/*
** Free a JsonParse object that was obtained from sqlite3_malloc().
*/
static void jsonParseFree(JsonParse *pParse){
- jsonParseReset(pParse);
- sqlite3_free(pParse);
+ if( pParse->nRef>1 ){
+ pParse->nRef--;
+ }else{
+ jsonParseReset(pParse);
+ sqlite3_free(pParse);
+ }
}
/*
@@ -1516,6 +1522,7 @@
memset(pParse, 0, sizeof(*pParse));
if( zJson==0 ) return 1;
pParse->zJson = zJson;
+ /* printf("JSON parse: %s\n", zJson); fflush(stdout); */
i = jsonParseValue(pParse, 0);
if( pParse->oom ) i = -1;
if( i>0 ){
@@ -1596,11 +1603,21 @@
** Obtain a complete parse of the JSON found in the first argument
** of the argv array. Use the sqlite3_get_auxdata() cache for this
** parse if it is available. If the cache is not available or if it
-** is no longer valid, parse the JSON again and return the new parse,
-** and also register the new parse so that it will be available for
-** future sqlite3_get_auxdata() calls.
+** is no longer valid, parse the JSON again and return the new parse.
**
-** If an error occurs and pErrCtx!=0 then report the error on pErrCtx
+** If a new parse is created, it is stored in the cache unless bWrite
+** is true. If the parse is cached and pErrCtx!=0 then the caller does
+** not need to free the returned parse since the parse belongs to the
+** cache and will be freed when the cache is cleared. On the other hand,
+** the returned parse should be treated as read-only.
+**
+** If the bWrite field is true, then it is assumed that the caller will
+** make changes to the parse. In this case, the parse is not cached (or
+** it is removed from the cache). Ownership of the parse passes to the
+** caller and the caller is expected to call jsonParseFree() when it has
+** finished using the parse.
+**
+** If an error occurs and pErrCtx!=0 then an error is reported on pErrCtx
** and return NULL.
**
** If an error occurs and pErrCtx==0 then return the Parse object with
@@ -1612,7 +1629,8 @@
static JsonParse *jsonParseCached(
sqlite3_context *pCtx,
sqlite3_value **argv,
- sqlite3_context *pErrCtx
+ sqlite3_context *pErrCtx,
+ int bWrite
){
const char *zJson = (const char*)sqlite3_value_text(argv[0]);
int nJson = sqlite3_value_bytes(argv[0]);
@@ -1629,12 +1647,19 @@
iMinKey = iKey;
break;
}
+ assert( p->nRef==1 );
if( pMatch==0
&& p->nJson==nJson
&& memcmp(p->zJson,zJson,nJson)==0
){
p->nErr = 0;
pMatch = p;
+ if( bWrite ){
+ p->nRef = 2;
+ sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iKey, 0, 0);
+ /* printf("JSON cache hit and purge (%d)\n", iKey); fflush(stdout); */
+ }
+ /* else{ printf("JSON cache hit (%d)\n", iKey); fflush(stdout); } */
}else if( p->iHold<iMinHold ){
iMinHold = p->iHold;
iMinKey = iKey;
@@ -1646,6 +1671,7 @@
if( pMatch ){
pMatch->nErr = 0;
pMatch->iHold = iMaxHold+1;
+ assert( pMatch->nRef==1 );
return pMatch;
}
p = sqlite3_malloc64( sizeof(*p) + nJson + 1 );
@@ -1656,9 +1682,11 @@
memset(p, 0, sizeof(*p));
p->zJson = (char*)&p[1];
memcpy((char*)p->zJson, zJson, nJson+1);
+ /* printf("JSON cache miss\n"); fflush(stdout); */
if( jsonParse(p, pErrCtx, p->zJson) ){
if( pErrCtx==0 ){
p->nErr = 1;
+ p->nRef = 1;
return p;
}
sqlite3_free(p);
@@ -1666,9 +1694,15 @@
}
p->nJson = nJson;
p->iHold = iMaxHold+1;
- sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iMinKey, p,
- (void(*)(void*))jsonParseFree);
- return (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iMinKey);
+ p->nRef = 1;
+ if( !bWrite ){
+ /* printf("JSON cache insert (%d)\n", iMinKey); fflush(stdout); */
+ sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iMinKey, p,
+ (void(*)(void*))jsonParseFree);
+ p = (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iMinKey);
+ }
+ assert( p==0 || p->nRef==1 );
+ return p;
}
/*
@@ -2095,7 +2129,7 @@
u32 i;
JsonNode *pNode;
- p = jsonParseCached(ctx, argv, ctx);
+ p = jsonParseCached(ctx, argv, ctx, 0);
if( p==0 ) return;
assert( p->nNode );
if( argc==2 ){
@@ -2157,7 +2191,7 @@
JsonString jx;
if( argc<2 ) return;
- p = jsonParseCached(ctx, argv, ctx);
+ p = jsonParseCached(ctx, argv, ctx, 0);
if( p==0 ) return;
if( argc==2 ){
/* With a single PATH argument */
@@ -2313,25 +2347,27 @@
int argc,
sqlite3_value **argv
){
- JsonParse x; /* The JSON that is being patched */
- JsonParse y; /* The patch */
+ JsonParse *pX; /* The JSON that is being patched */
+ JsonParse *pY; /* The patch */
JsonNode *pResult; /* The result of the merge */
UNUSED_PARAMETER(argc);
- if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return;
- if( jsonParse(&y, ctx, (const char*)sqlite3_value_text(argv[1])) ){
- jsonParseReset(&x);
+ pX = jsonParseCached(ctx, argv, ctx, 1);
+ if( pX==0 ) return;
+ pY = jsonParseCached(ctx, argv+1, ctx, 1);
+ if( pY==0 ){
+ jsonParseFree(pX);
return;
}
- pResult = jsonMergePatch(&x, 0, y.aNode);
- assert( pResult!=0 || x.oom );
+ pResult = jsonMergePatch(pX, 0, pY->aNode);
+ assert( pResult!=0 || pX->oom );
if( pResult ){
jsonReturnJson(pResult, ctx, 0);
}else{
sqlite3_result_error_nomem(ctx);
}
- jsonParseReset(&x);
- jsonParseReset(&y);
+ jsonParseFree(pX);
+ jsonParseFree(pY);
}
@@ -2387,26 +2423,27 @@
int argc,
sqlite3_value **argv
){
- JsonParse x; /* The parse */
+ JsonParse *p; /* The parse */
JsonNode *pNode;
const char *zPath;
u32 i;
if( argc<1 ) return;
- if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return;
- assert( x.nNode );
+ p = jsonParseCached(ctx, argv, ctx, 1);
+ if( p==0 ) return;
+ assert( p->nNode );
for(i=1; i<(u32)argc; i++){
zPath = (const char*)sqlite3_value_text(argv[i]);
if( zPath==0 ) goto remove_done;
- pNode = jsonLookup(&x, zPath, 0, ctx);
- if( x.nErr ) goto remove_done;
+ pNode = jsonLookup(p, zPath, 0, ctx);
+ if( p->nErr ) goto remove_done;
if( pNode ) pNode->jnFlags |= JNODE_REMOVE;
}
- if( (x.aNode[0].jnFlags & JNODE_REMOVE)==0 ){
- jsonReturnJson(x.aNode, ctx, 0);
+ if( (p->aNode[0].jnFlags & JNODE_REMOVE)==0 ){
+ jsonReturnJson(p->aNode, ctx, 0);
}
remove_done:
- jsonParseReset(&x);
+ jsonParseFree(p);
}
/*
@@ -2420,7 +2457,7 @@
int argc,
sqlite3_value **argv
){
- JsonParse x; /* The parse */
+ JsonParse *p; /* The parse */
JsonNode *pNode;
const char *zPath;
u32 i;
@@ -2430,12 +2467,13 @@
jsonWrongNumArgs(ctx, "replace");
return;
}
- if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return;
- assert( x.nNode );
+ p = jsonParseCached(ctx, argv, ctx, 1);
+ if( p==0 ) return;
+ assert( p->nNode );
for(i=1; i<(u32)argc; i+=2){
zPath = (const char*)sqlite3_value_text(argv[i]);
- pNode = jsonLookup(&x, zPath, 0, ctx);
- if( x.nErr ) goto replace_err;
+ pNode = jsonLookup(p, zPath, 0, ctx);
+ if( p->nErr ) goto replace_err;
if( pNode ){
assert( pNode->eU==0 || pNode->eU==1 || pNode->eU==4 );
testcase( pNode->eU!=0 && pNode->eU!=1 );
@@ -2444,14 +2482,14 @@
pNode->u.iReplace = i + 1;
}
}
- if( x.aNode[0].jnFlags & JNODE_REPLACE ){
- assert( x.aNode[0].eU==4 );
- sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]);
+ if( p->aNode[0].jnFlags & JNODE_REPLACE ){
+ assert( p->aNode[0].eU==4 );
+ sqlite3_result_value(ctx, argv[p->aNode[0].u.iReplace]);
}else{
- jsonReturnJson(x.aNode, ctx, argv);
+ jsonReturnJson(p->aNode, ctx, argv);
}
replace_err:
- jsonParseReset(&x);
+ jsonParseFree(p);
}
@@ -2472,7 +2510,7 @@
int argc,
sqlite3_value **argv
){
- JsonParse x; /* The parse */
+ JsonParse *p; /* The parse */
JsonNode *pNode;
const char *zPath;
u32 i;
@@ -2484,16 +2522,17 @@
jsonWrongNumArgs(ctx, bIsSet ? "set" : "insert");
return;
}
- if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return;
- assert( x.nNode );
+ p = jsonParseCached(ctx, argv, ctx, 1);
+ if( p==0 ) return;
+ assert( p->nNode );
for(i=1; i<(u32)argc; i+=2){
zPath = (const char*)sqlite3_value_text(argv[i]);
bApnd = 0;
- pNode = jsonLookup(&x, zPath, &bApnd, ctx);
- if( x.oom ){
+ pNode = jsonLookup(p, zPath, &bApnd, ctx);
+ if( p->oom ){
sqlite3_result_error_nomem(ctx);
goto jsonSetDone;
- }else if( x.nErr ){
+ }else if( p->nErr ){
goto jsonSetDone;
}else if( pNode && (bApnd || bIsSet) ){
testcase( pNode->eU!=0 && pNode->eU!=1 );
@@ -2503,14 +2542,14 @@
pNode->u.iReplace = i + 1;
}
}
- if( x.aNode[0].jnFlags & JNODE_REPLACE ){
- assert( x.aNode[0].eU==4 );
- sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]);
+ if( p->aNode[0].jnFlags & JNODE_REPLACE ){
+ assert( p->aNode[0].eU==4 );
+ sqlite3_result_value(ctx, argv[p->aNode[0].u.iReplace]);
}else{
- jsonReturnJson(x.aNode, ctx, argv);
+ jsonReturnJson(p->aNode, ctx, argv);
}
jsonSetDone:
- jsonParseReset(&x);
+ jsonParseFree(p);
}
/*
@@ -2529,7 +2568,7 @@
const char *zPath;
JsonNode *pNode;
- p = jsonParseCached(ctx, argv, ctx);
+ p = jsonParseCached(ctx, argv, ctx, 0);
if( p==0 ) return;
if( argc==2 ){
zPath = (const char*)sqlite3_value_text(argv[1]);
@@ -2556,7 +2595,7 @@
JsonParse *p; /* The parse */
UNUSED_PARAMETER(argc);
if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return;
- p = jsonParseCached(ctx, argv, 0);
+ p = jsonParseCached(ctx, argv, 0, 0);
if( p==0 || p->oom ){
sqlite3_result_error_nomem(ctx);
sqlite3_free(p);
@@ -2602,7 +2641,7 @@
JsonParse *p; /* The parse */
UNUSED_PARAMETER(argc);
if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return;
- p = jsonParseCached(ctx, argv, 0);
+ p = jsonParseCached(ctx, argv, 0, 0);
if( p==0 || p->oom ){
sqlite3_result_error_nomem(ctx);
sqlite3_free(p);
