From 1bfd25ed4cc6e979c0f7efd85505e8fa9a83bb4c Mon Sep 17 00:00:00 2001
From: Sagar Ghimire <info@sagarg.com.np>
Date: Wed, 28 Oct 2026 12:33:00 +0545
Subject: [PATCH] feat(api): cap request body at 10 MB with CKAN 413 envelope

BigQuery's `jobs.insert` request body is hard-capped at 10 MB, so a
`datastore_create` / `datastore_upsert` whose records (shipped as a
single `@rows` STRING query parameter) overflow that cap fails deep
in the engine with a raw HTML 413 and surfaces as a 500. Lower the
default `MAX_REQUEST_BODY_MB` from 50 to 10 so the middleware catches
oversize writes at the boundary, and rewrite its 413 to emit the CKAN
error envelope (`{help, success, error}`) with an actionable message
naming the actual body size and the configured limit.
---
 .env.example                |  2 +-
 datastore/api/middleware.py | 53 ++++++++++++++++++++++++++-----------
 datastore/core/config.py    |  2 +-
 3 files changed, 39 insertions(+), 18 deletions(-)

diff --git a/.env.example b/.env.example
index 0e0a758..9d703d6 100644
--- a/.env.example
+++ b/.env.example
@@ -2,7 +2,7 @@
 # Every field below maps 1:1 to a field on app.core.config.Config.
 
 # --- Application ---
-MAX_REQUEST_BODY_MB=50
+MAX_REQUEST_BODY_MB=10
 LOG_LEVEL=INFO
 # One JSON analytics event per datastore action / dump request.
 # false leaves the analytics middleware unmounted.
diff --git a/datastore/api/middleware.py b/datastore/api/middleware.py
index 079fe10..1c13096 100644
--- a/datastore/api/middleware.py
+++ b/datastore/api/middleware.py
@@ -1,14 +1,16 @@
 from __future__ import annotations
 
+import orjson
 from starlette.types import ASGIApp, Receive, Scope, Send
 
 
 class BodySizeLimitMiddleware:
     """Reject requests whose `Content-Length` exceeds `max_bytes` with 413.
 
-    Streaming bodies that omit `Content-Length` are not inspected here; cap
-    those at the ASGI server (uvicorn `--limit-max-request-size`) or behind
-    an ingress layer.
+
+    Streaming bodies that omit `Content-Length` are not inspected here;
+    cap those at the ASGI server (uvicorn `--limit-max-request-size`)
+    or behind an ingress layer.
     """
 
     def __init__(self, app: ASGIApp, max_bytes: int) -> None:
@@ -24,19 +26,38 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
                     except ValueError:
                         break
                     if size > self.max_bytes:
-                        await send(
-                            {
-                                "type": "http.response.start",
-                                "status": 413,
-                                "headers": [(b"content-type", b"application/json")],
-                            }
-                        )
-                        await send(
-                            {
-                                "type": "http.response.body",
-                                "body": b'{"detail":"request body too large"}',
-                            }
-                        )
+                        await self._reject(scope, send, size)
                         return
                     break
         await self.app(scope, receive, send)
+
+    async def _reject(self, scope: Scope, send: Send, size: int) -> None:
+        limit_mb = self.max_bytes / (1024 * 1024)
+        size_mb = size / (1024 * 1024)
+        path = scope.get("path", "")
+        body = orjson.dumps(
+            {
+                "help": path,
+                "success": False,
+                "error": {
+                    "__type": "Payload Too Large",
+                    "message": (
+                        f"Request body is {size_mb:.2f} MB; limit is "
+                        f"{limit_mb:.0f} MB. Single write "
+                        "at 10 MB — split the records into smaller "
+                        "batches and retry."
+                    ),
+                },
+            }
+        )
+        await send(
+            {
+                "type": "http.response.start",
+                "status": 413,
+                "headers": [
+                    (b"content-type", b"application/json"),
+                    (b"content-length", str(len(body)).encode()),
+                ],
+            }
+        )
+        await send({"type": "http.response.body", "body": body})
diff --git a/datastore/core/config.py b/datastore/core/config.py
index 19851ff..32c8e11 100644
--- a/datastore/core/config.py
+++ b/datastore/core/config.py
@@ -54,7 +54,7 @@ class Config(BaseSettings):
 
     # Request limits
     MAX_REQUEST_BODY_MB: int = Field(
-        default=50,
+        default=10,
         ge=1,
         le=1000,
         description="Maximum request body size in MB",