From 6f458062d4b94281b04e7802f721f94c2ad3c0a9 Mon Sep 17 00:00:00 2001 From: slayerjain Date: Mon, 8 Jun 2026 11:15:47 +0530 Subject: [PATCH 1/2] fix(restheart-mongo): deterministic GraphQL app cache + expires_in noise MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The keploy restheart-mongo lane flaked on two recorded fields; both are app/JWT non-determinism, fixed the same way the lane already handles the ACL cache and the access_token JWT: 1. post-graphql-halpeople (mongo mock-miss -> 500 MongoSocketReadException): RESTHeart's GraphQL service caches gql-apps app definitions with a 60s time-to-revalidate (app-cache-ttr, Caffeine nanoTime ticker) — the exact same time-freeze x cache-ttl interaction already documented for /mongoAclAuthorizer. Under `keploy test --freezeTime` the ticker is frozen, so the app cache never revalidates and RESTHeart issues a different number/order of `find gql-apps` mongo queries than were recorded; the unmatched find has no mock so keploy's mongo proxy closes the socket and RESTHeart returns 500 instead of the recorded response. Fix: add /graphql/app-cache-enabled->false to RHO so gql-apps is read-through every request (identical mongo stream at record & replay), mirroring the existing /mongoAclAuthorizer/cache-enabled->false. 2. post-token (expires_in 872 vs 871): /tokens returns expires_in = seconds-until-exp computed at request time; it can differ by 1s across record/replay when the request straddles a second boundary, even under --freezeTime. The JWT itself (access_token) is already noised; add expires_in to globalNoise.body for the same reason. Signed-off-by: Shubham Jain Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: slayerjain --- restheart-mongo/docker-compose.yml | 18 +++++++++++++++++- restheart-mongo/keploy.yml.template | 7 +++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/restheart-mongo/docker-compose.yml b/restheart-mongo/docker-compose.yml index 44797e9c..e5e0a8e6 100644 --- a/restheart-mongo/docker-compose.yml +++ b/restheart-mongo/docker-compose.yml @@ -27,7 +27,7 @@ services: # --freezeTime keeps `exp` valid. Pinning the secret keeps # the bearer signature verifiable across record→replay # container restarts (deterministic key, deterministic JWT). - RHO: '/mclient/connection-string->"mongodb://${RESTHEART_MONGO_IP:-172.36.0.10}:27017";/http-listener/host->"0.0.0.0";/core/log-level->"INFO";/jwtConfigProvider/key->"keploy-fixed-jwt-secret-for-deterministic-recordings";/mongoAclAuthorizer/cache-enabled->false' + RHO: '/mclient/connection-string->"mongodb://${RESTHEART_MONGO_IP:-172.36.0.10}:27017";/http-listener/host->"0.0.0.0";/core/log-level->"INFO";/jwtConfigProvider/key->"keploy-fixed-jwt-secret-for-deterministic-recordings";/mongoAclAuthorizer/cache-enabled->false;/graphql/app-cache-enabled->false' # Bound RESTHeart's JVM heap. RESTHeart 9.x's default uses # `MaxRAMPercentage=25` of cgroup memory, which on a typical # Woodpecker runner cgroup (~8GB) lands ~2GB heap. With three @@ -59,6 +59,22 @@ services: # despite recorded 200. Disabling the cache makes ACL rules # read-through from mongo on every request — small perf cost, # but eliminates the time-freeze x cache-ttl interaction. + # + # Note on /graphql/app-cache-enabled->false in RHO: + # RESTHeart's GraphQL service caches gql-apps app definitions with a + # 60s time-to-revalidate (app-cache-ttr, Caffeine, nanoTime ticker) — + # the SAME time-freeze x cache-ttl interaction as the ACL cache above. + # flow.sh POSTs a "halpeople" GraphQL app to gql-apps and then fires + # POST /graphql/halpeople queries. At record the wall clock advances, + # so the app cache revalidates (re-reading gql-apps) at TTR + # boundaries; under `keploy test --freezeTime` the ticker is frozen so + # the cache never revalidates, producing a DIFFERENT number/order of + # `find gql-apps` mongo queries than were recorded. The unmatched + # find has no recorded mock -> keploy's mongo proxy closes the socket + # -> RESTHeart returns 500 (MongoSocketReadException) instead of the + # recorded response (the post-graphql-halpeople flakiness). Disabling + # the app cache makes gql-apps read-through on every request, so the + # mongo query stream is identical at record and replay. depends_on: mongo: condition: service_healthy diff --git a/restheart-mongo/keploy.yml.template b/restheart-mongo/keploy.yml.template index a16bc5e2..8b5a3736 100644 --- a/restheart-mongo/keploy.yml.template +++ b/restheart-mongo/keploy.yml.template @@ -61,3 +61,10 @@ test: client_ip: [] latencyMs: [] access_token: [] + # RESTHeart's /tokens endpoint returns expires_in = seconds until + # the JWT `exp`, computed as (exp - now) at request time. Even under + # --freezeTime the recorded vs replayed value can differ by 1s when + # the request straddles a second boundary (observed 872 vs 871 on + # post-token), so the countdown must be treated as noise just like + # the access_token JWT itself. + expires_in: [] From 458e0d2d872779ebc13abff22c55e331534ec02d Mon Sep 17 00:00:00 2001 From: slayerjain Date: Thu, 9 Jul 2026 20:09:01 +0530 Subject: [PATCH 2/2] fix(restheart-mongo): disable frozen /mongo metadata caches for deterministic replay RESTHeart's /mongo local-cache (db/collection _properties, 60s ttl) and schema-cache (JSON schemas, 60s ttl) are Caffeine caches ticking on System.nanoTime(), which --freezeTime freezes. The frozen ttl shifts which per-test window each 'find _properties' mongo query lands in between record and replay, so keploy's strict per-test mock pool misses -> mongo socket close -> 500 MongoSocketReadException. Observed on keploy/enterprise pipeline 5867 (record-pr-replay-stable): 13/306 flaked on get-root/post-users/delete-* legs that COMPAT_TEST_RETRIES could not absorb. Same frozen-ticker class as the ACL and GraphQL cache disables already in this file; makes _properties/schema reads read-through (deterministic, always correct, small perf cost). get-collection-cache left enabled (inert: needs ?cache, which flow.sh never sends). Signed-off-by: slayerjain --- restheart-mongo/docker-compose.yml | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/restheart-mongo/docker-compose.yml b/restheart-mongo/docker-compose.yml index e5e0a8e6..6db8e4ea 100644 --- a/restheart-mongo/docker-compose.yml +++ b/restheart-mongo/docker-compose.yml @@ -27,7 +27,7 @@ services: # --freezeTime keeps `exp` valid. Pinning the secret keeps # the bearer signature verifiable across record→replay # container restarts (deterministic key, deterministic JWT). - RHO: '/mclient/connection-string->"mongodb://${RESTHEART_MONGO_IP:-172.36.0.10}:27017";/http-listener/host->"0.0.0.0";/core/log-level->"INFO";/jwtConfigProvider/key->"keploy-fixed-jwt-secret-for-deterministic-recordings";/mongoAclAuthorizer/cache-enabled->false;/graphql/app-cache-enabled->false' + RHO: '/mclient/connection-string->"mongodb://${RESTHEART_MONGO_IP:-172.36.0.10}:27017";/http-listener/host->"0.0.0.0";/core/log-level->"INFO";/jwtConfigProvider/key->"keploy-fixed-jwt-secret-for-deterministic-recordings";/mongoAclAuthorizer/cache-enabled->false;/graphql/app-cache-enabled->false;/mongo/local-cache-enabled->false;/mongo/schema-cache-enabled->false' # Bound RESTHeart's JVM heap. RESTHeart 9.x's default uses # `MaxRAMPercentage=25` of cgroup memory, which on a typical # Woodpecker runner cgroup (~8GB) lands ~2GB heap. With three @@ -75,6 +75,34 @@ services: # recorded response (the post-graphql-halpeople flakiness). Disabling # the app cache makes gql-apps read-through on every request, so the # mongo query stream is identical at record and replay. + # + # Note on /mongo/{local,schema}-cache-enabled->false in RHO: + # The SAME time-freeze x cache-ttl interaction as the ACL and GraphQL + # caches above, but for the mongo service's own Caffeine caches (default + # ttls tick on the frozen System.nanoTime()). + # - local-cache (db/collection _properties metadata, 60s ttl): this is + # the confirmed culprit. Every request routes/validates via the target + # collection's _properties — `GET /` (get-root) reads ALL collections' + # _properties, and writes/deletes (post-users, delete-users, + # delete-acl-*) read the target collection's. With local-cache on, that + # `find _properties` mongo stream is served from cache at record (wall + # clock warm) but the frozen ttl at replay shifts WHICH per-test window + # each find lands in, so keploy's strict per-test mock pool has no + # matching mock -> mongo proxy closes the socket -> 500 + # MongoSocketReadException. Observed on keploy/enterprise pipeline 5867 + # (record-pr-replay-stable): 13/306 failed — get-root-1..4, + # post-users-1/2, delete-users-*, delete-acl-* — a different leg each + # run (COMPAT_TEST_RETRIES=3 could not absorb it). + # - schema-cache (JSON schemas, 60s ttl): same frozen-ticker class; the + # sample bootstraps _schemas and posts a schema (flow.sh), so a + # schema-validated write can shift its `_schemas` find the same way. + # Disabled proactively to close the class on a path the sample exercises. + # (get-collection-cache is left enabled: it only activates on ?cache + # requests, which flow.sh never sends, so it is inert here.) + # Disabling makes _properties/schema reads read-through on every request, + # so the mongo query stream is identical at record and replay. Read-through + # is always correct (only a small perf cost), so this cannot regress + # behaviour. depends_on: mongo: condition: service_healthy