diff --git a/.env.example b/.env.example index 0e0a758..df77da1 100644 --- a/.env.example +++ b/.env.example @@ -48,6 +48,7 @@ BIGQUERY_DATASET= BIGQUERY_CREDENTIALS= BIGQUERY_CREDENTIALS_RO= BIGQUERY_USE_QUERY_CACHE=true +BIGQUERY_TRANSLATE_PG_SQL=true BIGQUERY_EXPORT_BUCKET= BIGQUERY_EXPORT_URL_EXPIRY_HOURS=1 SQL_FUNCTIONS_ALLOW_FILE= diff --git a/.gitignore b/.gitignore index 11585df..5e4ae70 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,11 @@ htmlcov/ .idea/ *.swp +# Local-only testing / exploration (prod replication scripts, pulled +# fixtures, production job logs). Everything here stays on the dev's +# machine — never committed. +/local_test/ + # OS .DS_Store Thumbs.db diff --git a/CLAUDE.md b/CLAUDE.md index b06ffaf..2227c27 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -653,6 +653,8 @@ is easy to follow. "result": { "resource_id": "balancing_auction_results_2025", "fields": [ + {"id": "_id", "type": "integer", "info": {"title": "Record Identifier", "...": "..."}}, + {"id": "_updated_at", "type": "datetime", "info": {"title": "Last Updated Timestamp", "...": "..."}}, {"id": "auction_id", "type": "integer", "info": {"title": "Auction ID", "...": "..."}}, {"id": "product_code", "type": "string", "info": {"...": "..."}}, {"id": "delivery_start", "type": "datetime", "info": {"...": "..."}}, @@ -668,6 +670,13 @@ is easy to follow. } ``` +System columns are engine-managed — the request body must omit them +(validation rejects `_id` / `_updated_at` in `fields`), and the service +prepends them to the stored schema with service-provided `info` +(title / description / comment / example / unit). The same `info` block +rides through every read endpoint so clients see consistent column +documentation. + Optional response fields (omitted from the body when not requested): - `records` — echoes the input rows back when the request sets `include_records: true`. - `total_records` — total row count after the write, populated when `include_total: true`. @@ -888,6 +897,9 @@ can confirm the table's new shape without a follow-up `datastore_info`: Returns the same field shape that was supplied to `datastore_create`, including the `info` data dictionary verbatim — clients can use this as a column-level metadata catalog (titles, descriptions, units, examples) without a side store. +The two engine-managed system columns (`_id`, `_updated_at`) are prepended +to `fields` with service-provided `info` so every read endpoint carries the +same documentation for them. **Response** ```json @@ -897,6 +909,28 @@ metadata catalog (titles, descriptions, units, examples) without a side store. "result": { "resource_id": "balancing_auction_results_2025", "fields": [ + { + "id": "_id", + "type": "integer", + "info": { + "title": "Record Identifier", + "description": "System-generated unique identifier assigned to each record in the database. Used to uniquely identify a row within a resource table.", + "comment": "Automatically populated. Not publisher-maintained. Values are unique within a resource and may be used for record retrieval and updates.", + "example": "12345", + "unit": "N/A" + } + }, + { + "id": "_updated_at", + "type": "datetime", + "info": { + "title": "Last Updated Timestamp", + "description": "System-generated timestamp indicating when the record was last created or modified in the database.", + "comment": "Automatically maintained. Useful for identifying recently changed records. Stored in UTC.", + "example": "2026-08-27T12:45:30", + "unit": "ISO 8601 UTC Timestamp" + } + }, { "id": "auction_id", "type": "integer", diff --git a/datastore/analytics.py b/datastore/analytics.py index 33e18b7..165df52 100644 --- a/datastore/analytics.py +++ b/datastore/analytics.py @@ -46,12 +46,26 @@ from datastore.core.constants import API_PREFIX, DUMP_PREFIX +# DEPRECATED legacy surface — this import, the two `LEGACY_*` constants +# below, and the three `LEGACY_*` references downstream in this file +# are the only legacy touch points in analytics. See +# `datastore/legacy/__init__.py` for the full removal plan: when the +# legacy URLs retire, delete lines marked `LEGACY ONLY` here plus the +# single import above and analytics returns to the pure v2 shape. +from datastore.legacy.constants import LEGACY_API_PREFIX, LEGACY_DUMP_PREFIX + log = logging.getLogger(__name__) ACTION_PREFIX = f"{API_PREFIX}/" DUMP_PATH_PREFIX = f"{DUMP_PREFIX}/" DUMP_QUERY_PATH = f"{DUMP_PREFIX}/query" +# LEGACY ONLY — CKAN-compatible surface. Recorded with the same action +# names and event shape as v2 — downstream can split v2 vs legacy by +# the `endpoint` path when it needs to. +LEGACY_ACTION_PREFIX = f"{LEGACY_API_PREFIX}/" +LEGACY_DUMP_PATH_PREFIX = f"{LEGACY_DUMP_PREFIX}/" + # Served under the versioned prefix but not part of the API surface, so # they must not be recorded as action calls. DOCS_PATHS = frozenset({"docs", "redoc", "openapi.json", "static"}) @@ -71,6 +85,17 @@ def action_name(path: str) -> str | None: if not name or name in DOCS_PATHS: return None return name + # LEGACY ONLY — CKAN-compatible surface. Same action names as v2 so + # one event stream covers both API versions. The `endpoint` field + # preserves the raw path so a downstream filter can isolate legacy + # traffic. Delete this block when the legacy URLs retire. + if path.startswith(LEGACY_DUMP_PATH_PREFIX): + return "datastore_dump" + if path.startswith(LEGACY_ACTION_PREFIX): + name = path[len(LEGACY_ACTION_PREFIX) :].strip("/").split("/", 1)[0] + if not name: + return None + return name return None @@ -172,7 +197,12 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: state: dict[str, Any] = scope.setdefault("state", {}) status = 500 # what reaches the caller if the app dies before a response body = bytearray() - capture = scope["method"] == "POST" and scope["path"].startswith(ACTION_PREFIX) + # LEGACY ONLY: `LEGACY_ACTION_PREFIX` in the tuple keeps POST + # body capture working for `/api/3/action/*`. Drop that element + # when the legacy URLs retire. + capture = scope["method"] == "POST" and scope["path"].startswith( + (ACTION_PREFIX, LEGACY_ACTION_PREFIX) + ) async def tee_receive() -> Any: message = await receive() @@ -256,9 +286,16 @@ def _resource_ref(self, scope: Scope, body: bytes) -> str | None: those events rely on ``authorization_dict`` alone. """ path: str = scope["path"] - if path.startswith(DUMP_PATH_PREFIX): - ref = path[len(DUMP_PATH_PREFIX) :].split("/", 1)[0] - return ref if ref and ref != "query" else None + # Both v2 (`/datastore/api/v2/dump/`) and legacy + # (`/datastore/dump/`) carry the resource id as the first + # path segment after the dump prefix. `query` is reserved on v2 + # (SQL download endpoint) and not a resource id. + # LEGACY ONLY: drop `LEGACY_DUMP_PATH_PREFIX` from this tuple + # when the legacy URLs retire. + for prefix in (DUMP_PATH_PREFIX, LEGACY_DUMP_PATH_PREFIX): + if path.startswith(prefix): + ref = path[len(prefix) :].split("/", 1)[0] + return ref if ref and ref != "query" else None query: bytes = scope.get("query_string", b"") params = parse_qs(query.decode("latin-1")) diff --git a/datastore/api/endpoints/datastore.py b/datastore/api/endpoints/datastore.py index 6fb6618..252f4e6 100644 --- a/datastore/api/endpoints/datastore.py +++ b/datastore/api/endpoints/datastore.py @@ -184,13 +184,22 @@ async def datastore_search_sql( Accepts a single `sql` query parameter. To export the result as a file instead, use `GET /query`. """ + # Collect each referenced resource's authorize payload — under + # AUTH_TYPE=ckan it carries the resource record (with its synced + # schema), which the service merges into the response so the output + # columns inherit CKAN-side data-dictionary edits by name. + resources: list[dict] = [] for resource_id in params.resource_ids: - await context.authorize(resource_id=resource_id, permission="read") + auth_result = await context.authorize(resource_id=resource_id, permission="read") + resource = (auth_result or {}).get("resource") + if isinstance(resource, dict): + resources.append(resource) data_dict = params.model_dump() | { "function_names": params.function_names, "limit": params.limit, "offset": params.offset, + "resources": resources, } body_iter = await search_sql_datastore( diff --git a/datastore/api/responses.py b/datastore/api/responses.py index aac205e..c3976ee 100644 --- a/datastore/api/responses.py +++ b/datastore/api/responses.py @@ -34,7 +34,10 @@ def _orjson_default(obj: Any) -> Any: # values past ~15 significant digits round to the nearest IEEE-754 # double (full-precision callers should CAST to STRING in SQL). if isinstance(obj, Decimal): - return float(obj) + f = float(obj) + if f.is_integer() and abs(f) < 2**63: + return int(f) + return f # `BYTES` columns come back as raw `bytes`; base64-encode so the # response stays UTF-8 and round-trippable. if isinstance(obj, bytes): diff --git a/datastore/api/routes.py b/datastore/api/routes.py index 89e33bc..3543e5c 100644 --- a/datastore/api/routes.py +++ b/datastore/api/routes.py @@ -5,6 +5,14 @@ from datastore.api.endpoints import datastore, dump, health from datastore.core.constants import API_BASE_PREFIX, API_PREFIX, DUMP_PREFIX +# DEPRECATED legacy CKAN surface. These two imports and the two +# `legacy_endpoints` `include_router` calls below are the ONLY references +# to `datastore.legacy` in the entire v2 codebase — remove them (plus +# `datastore/legacy/` and `tests/test_legacy_api.py`) when the legacy URLs +# retire. See `datastore/legacy/__init__.py` for the removal plan. +from datastore.legacy import endpoints as legacy_endpoints +from datastore.legacy.constants import LEGACY_API_PREFIX, LEGACY_DUMP_PREFIX + api_router = APIRouter() # Unversioned: a probe URL is stable regardless of the action contract. api_router.include_router(health.probe_router, prefix=API_BASE_PREFIX) @@ -14,3 +22,9 @@ api_router.include_router(dump.router, prefix=DUMP_PREFIX) # Versioned: the datastore actions are the compatibility contract. api_router.include_router(datastore.router, prefix=API_PREFIX) + +# DEPRECATED — see `datastore/legacy/__init__.py`. These two mounts +# expose read-only CKAN-compatible URLs so scripts in the wild keep +# working while callers migrate to v2. +api_router.include_router(legacy_endpoints.router, prefix=LEGACY_API_PREFIX) +api_router.include_router(legacy_endpoints.dump_router, prefix=LEGACY_DUMP_PREFIX) diff --git a/datastore/core/config.py b/datastore/core/config.py index 19851ff..18e9f98 100644 --- a/datastore/core/config.py +++ b/datastore/core/config.py @@ -264,6 +264,19 @@ def _check_engine_available(cls, v: str) -> str: "force a fresh scan in tests." ), ) + BIGQUERY_TRANSLATE_PG_SQL: bool = Field( + default=True, + description=( + "Translate Postgres-dialect SQL into BigQuery-dialect SQL before " + "running `datastore_search_sql` and `dump/query`. Covers the " + "pg-isms that broke customer scripts carried over from the " + "Postgres-backed CKAN datastore (double-quoted identifiers, " + "`::type` casts, `to_char`, `INTERVAL '10 days'`, " + "`CURRENT_DATE - 1`, …). Set False to disable the translator " + "per environment without a code change — SQL is then sent to " + "BigQuery as written." + ), + ) BIGQUERY_EXPORT_BUCKET: str = Field( default="", description=("GCS bucket name (no `gs://` prefix) that `/` "), diff --git a/datastore/core/constants.py b/datastore/core/constants.py index 471aa5d..3b9df49 100644 --- a/datastore/core/constants.py +++ b/datastore/core/constants.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Literal +from typing import Any, Literal # Default for the `API_URL` setting — the public base URL used to render # absolute URLs in the OpenAPI examples. Deployments override it via env; @@ -27,6 +27,9 @@ API_VERSION = "v2" API_PREFIX = f"{API_BASE_PREFIX}/{API_VERSION}" DUMP_PREFIX = f"{API_PREFIX}/dump" +# Note: legacy CKAN URL prefixes + their format tables live in +# `datastore/legacy/constants.py`, kept out of this file so the whole +# backwards-compat surface is one `rm -rf datastore/legacy/` away. # Download formats served by the export pipeline (`/…` and # `datastore_search_sql?download=…`). Lives here — not in `api/` — because # both the request schemas (pydantic layer) and the endpoints (starlette @@ -156,10 +159,10 @@ # Frictionless field type → canonical Postgres type. Used when deriving the # legacy `fields` shape from a Frictionless schema. Lossy in the other # direction (e.g., a Frictionless `integer` could have been any width); we -# pick the widest/most-permissive Postgres type so the column accepts -# anything the Frictionless type implies. +# pick the type CKAN classic emits so the legacy wire shape matches — +# `integer` → `int4` (both CKAN's user integers and `_id serial`). FRICTIONLESS_TO_POSTGRES: dict[str, str] = { - "integer": "int8", + "integer": "int4", "number": "numeric", "boolean": "bool", "string": "text", @@ -207,3 +210,48 @@ "_updated_at", } ) + + +# Service-provided documentation for the two engine-managed system +# columns, shaped as full Frictionless field entries with an `info` +# sub-dict so the read-side merger overlays them like any +# publisher-supplied field. CKAN's sync strips these columns from a +# resource's own schema, so without the seed `_id` / `_updated_at` +# would arrive on every read as bare `{name, type}` entries with no +# title or description. +SYSTEM_FIELD_INFO: dict[str, dict[str, Any]] = { + "_id": { + "name": "_id", + "info": { + "title": "Record Identifier", + "description": ( + "System-generated unique identifier assigned to each " + "record in the database. Used to uniquely identify a " + "row within a resource table." + ), + "comment": ( + "Automatically populated. Not publisher-maintained. " + "Values are unique within a resource and may be used " + "for record retrieval and updates." + ), + "example": "12345", + "unit": "N/A", + }, + }, + "_updated_at": { + "name": "_updated_at", + "info": { + "title": "Last Updated Timestamp", + "description": ( + "System-generated timestamp indicating when the record " + "was last created or modified in the database." + ), + "comment": ( + "Automatically maintained. Useful for identifying " + "recently changed records. Stored in UTC." + ), + "example": "2026-08-27T12:45:30", + "unit": "ISO 8601 UTC Timestamp", + }, + }, +} diff --git a/datastore/infrastructure/engines/bigquery/backend.py b/datastore/infrastructure/engines/bigquery/backend.py index 395c18b..7aff47a 100644 --- a/datastore/infrastructure/engines/bigquery/backend.py +++ b/datastore/infrastructure/engines/bigquery/backend.py @@ -714,6 +714,7 @@ def search_sql(self, sql: str, limit: int) -> SearchResult: sql, project=self.config.BIGQUERY_PROJECT, dataset=self.config.BIGQUERY_DATASET, + translate=self.config.BIGQUERY_TRANSLATE_PG_SQL, ) except Exception as e: raise ServerError( diff --git a/datastore/infrastructure/engines/bigquery/export.py b/datastore/infrastructure/engines/bigquery/export.py index 539d600..ad0bc3a 100644 --- a/datastore/infrastructure/engines/bigquery/export.py +++ b/datastore/infrastructure/engines/bigquery/export.py @@ -194,6 +194,7 @@ async def dump_sql( sql, project=backend.config.BIGQUERY_PROJECT, dataset=backend.config.BIGQUERY_DATASET, + translate=backend.config.BIGQUERY_TRANSLATE_PG_SQL, ) except Exception as e: raise ServerError( diff --git a/datastore/infrastructure/engines/bigquery/lib.py b/datastore/infrastructure/engines/bigquery/lib.py index b1fe8dd..ab9a6c4 100644 --- a/datastore/infrastructure/engines/bigquery/lib.py +++ b/datastore/infrastructure/engines/bigquery/lib.py @@ -29,7 +29,7 @@ # JSON / array / geojson absent — equality on those is rejected. _FILTER_PARAM_TYPE: dict[str, str] = { "integer": "INT64", - "number": "FLOAT64", + "number": "BIGNUMERIC", "boolean": "BOOL", "string": "STRING", "date": "DATE", @@ -78,6 +78,8 @@ def format_select_column(name: str, bq_type: str | None) -> str: return f"FORMAT_TIMESTAMP('%Y-%m-%dT%H:%M:%S', `{name}`, 'UTC') AS `{name}`" if bq == "DATETIME": return f"FORMAT_DATETIME('%Y-%m-%dT%H:%M:%S', `{name}`) AS `{name}`" + if bq in ("NUMERIC", "BIGNUMERIC"): + return f"CAST(`{name}` AS FLOAT64) AS `{name}`" return f"`{name}`" @@ -511,18 +513,49 @@ def strip_limit_offset(sql: str, *, dialect: str = "bigquery") -> str: return tree.sql(dialect=dialect) -def qualify_table_refs(sql: str, project: str, dataset: str) -> str: +def qualify_table_refs( + sql: str, + project: str, + dataset: str, + *, + translate: bool = True, +) -> str: """Rewrite unqualified table refs to `project.dataset.` for BigQuery. User SQL refs look like CKAN resource_ids (`FROM "uuid"` / `FROM uuid`); we parse as postgres (accepts double-quoted idents) and re-emit as BigQuery. Already-qualified refs and CTE aliases are left alone. + + `translate=True` (default) also runs the Postgres→BigQuery dialect + translator from `translate.py` first — the single hook that lets + customer scripts carried over from the Postgres-backed CKAN + datastore keep working (`"col"::numeric`, `CURRENT_DATE - 1`, + `to_char(..., 'YYYY-MM-DD')`, …). When the translator rewrites the + SQL, a DEBUG log line prints both the before/after and the list of + applied steps so operators can see exactly what changed. Pass + `translate=False` (wired to `Config.BIGQUERY_TRANSLATE_PG_SQL`) to + disable it per environment without a code change. """ import sqlglot from sqlglot import expressions as exp - tree = sqlglot.parse_one(sql, dialect="postgres") + from datastore.infrastructure.engines.bigquery.translate import translate_pg_to_bq + + translated, notes = translate_pg_to_bq(sql, enabled=translate) + if notes: + log.debug( + "pg→bq translation applied (%s): %r → %r", + "; ".join(notes), + sql, + translated, + ) + + # After translation the SQL is BigQuery-shaped (backticks, DATE_SUB, + # etc.) so parse under the BQ dialect; when translation is off we + # fall back to postgres so a user's `"ident"` still parses. + parse_dialect = "bigquery" if notes else "postgres" + tree = sqlglot.parse_one(translated, dialect=parse_dialect) cte_aliases = {cte.alias_or_name for cte in tree.find_all(exp.CTE) if cte.alias_or_name} for table in tree.find_all(exp.Table): name = table.name diff --git a/datastore/infrastructure/engines/bigquery/search.py b/datastore/infrastructure/engines/bigquery/search.py index bd21073..97e51e9 100644 --- a/datastore/infrastructure/engines/bigquery/search.py +++ b/datastore/infrastructure/engines/bigquery/search.py @@ -26,7 +26,7 @@ # `build_search`). Anything else falls back to STRING. _PARAM_TYPE: dict[str, str] = { "integer": "INT64", - "number": "FLOAT64", + "number": "BIGNUMERIC", "boolean": "BOOL", "string": "STRING", "date": "DATE", @@ -238,8 +238,9 @@ def build_search( list of `ScalarQueryParameter` / `ArrayQueryParameter` ready to drop into a `QueryJobConfig`. - Layout: ``SELECT [DISTINCT] cols FROM target AS t [WHERE ...] - [ORDER BY ...] LIMIT N OFFSET M``. Sort defaults to `_id ASC` when + Layout: ``SELECT [DISTINCT] cols FROM target AS _src [WHERE ...] + [ORDER BY ...] LIMIT N OFFSET M``. Alias is `_src` (not `t`) so a + user column named `t` cannot be shadowed by the table alias. Sort defaults to `_id ASC` when `_id` is projected (CKAN's row-id ordering convention); otherwise no default sort — caller must specify or accept BigQuery's order (undefined). @@ -275,23 +276,27 @@ def build_search( filters=filters, q=q, type_map=type_map, - table_alias="t", + table_alias="_src", params=params, ) parts: list[str] = [] projection = ", ".join(_project_column(c, type_map) for c in projected) - parts.append(f"SELECT {'DISTINCT ' if distinct else ''}{projection} FROM {table_ref} AS t") + # Alias is `_src` (not `t`) so a user column literally named `t` + # cannot collide: `SELECT \`t\` FROM tbl AS t` resolves `` `t` `` to + # the table-alias STRUCT (the whole row) instead of the column — BQ + # treats a bare table alias in SELECT as a row-shaped STRUCT. + parts.append(f"SELECT {'DISTINCT ' if distinct else ''}{projection} FROM {table_ref} AS _src") if where: parts.append(f"WHERE {where}") if sort_pairs: - # Sort against the underlying table column (`t.`) rather than - # the projected alias — the datetime projection rewrites + # Sort against the underlying table column (`_src.`) rather + # than the projected alias — the datetime projection rewrites # `col` → `FORMAT_TIMESTAMP(...) AS col`, and we want ORDER BY to # operate on the native TIMESTAMP, not the formatted string. (The # ISO format happens to sort lexicographically the same way, but # the explicit reference future-proofs us if the format changes.) - parts.append("ORDER BY " + ", ".join(f"t.`{c}` {d}" for c, d in sort_pairs)) + parts.append("ORDER BY " + ", ".join(f"_src.`{c}` {d}" for c, d in sort_pairs)) parts.append(f"LIMIT {int(limit)} OFFSET {int(offset)}") sql = " ".join(parts) @@ -335,7 +340,7 @@ def build_count( filters=filters, q=q, type_map=type_map, - table_alias="t", + table_alias="_src", params=params, ) @@ -344,7 +349,7 @@ def build_count( # then dedupes on the formatted (second-precision) string, matching # what the user sees in the data response. projection = ", ".join(_project_column(c, type_map) for c in projected) - inner_parts = [f"SELECT {'DISTINCT ' if distinct else ''}{projection} FROM {table_ref} AS t"] + inner_parts = [f"SELECT {'DISTINCT ' if distinct else ''}{projection} FROM {table_ref} AS _src"] if where: inner_parts.append(f"WHERE {where}") inner = " ".join(inner_parts) diff --git a/datastore/infrastructure/engines/bigquery/translate.py b/datastore/infrastructure/engines/bigquery/translate.py new file mode 100644 index 0000000..5808af8 --- /dev/null +++ b/datastore/infrastructure/engines/bigquery/translate.py @@ -0,0 +1,440 @@ +"""PostgreSQL → BigQuery SQL translator for `datastore_search_sql`. + +Many customer scripts in the wild were written against a Postgres-backed +CKAN datastore. When the backend was swapped to BigQuery those scripts +started 400ing on dialect quirks that are harmless on Postgres but +illegal on BigQuery — double-quoted identifiers, `"col"::numeric` casts, +`to_char(..., 'YYYY-MM-DD')`, `CURRENT_DATE - 1`, `INTERVAL '10 days'`. + +This module does one job: take the user's SQL and return a BigQuery- +compatible version plus a list of plain-English notes about what was +rewritten. Both the v2 and legacy `datastore_search_sql` paths flow +through [bigquery/lib.py](lib.py)'s `qualify_table_refs`, which invokes +the translator first — so there is one translation point for both APIs. + +Design notes +------------ +- **One sqlglot pass** (`transpile(read='postgres', write='bigquery')`) + handles most pg-isms (identifier quoting, `::` casts, `to_char`, + `INTERVAL '10 days'`, window functions, …). Verified against a real + customer query log before this landed. +- **One targeted AST rewrite** fixes ` ± ` + (e.g. `CURRENT_DATE - 1`) → `DATE_SUB(..., INTERVAL n DAY)`. sqlglot + leaves integer-day arithmetic unchanged and BigQuery rejects it; + this is the only hole big enough to break real traffic. +- **Transparent**: `translate_pg_to_bq` returns `(sql, notes)` where + `notes` is a human-readable list of rewrites. Callers can log it; the + engine logs at DEBUG when the SQL changes. Grep `notes` to see every + translation path exercised by production. +- **Reversible kill switch**: `translate_pg_to_bq(..., enabled=False)` + short-circuits to `(sql, [])`. Wired to `Config.BIGQUERY_TRANSLATE_PG_SQL` + so operators can flip it off per environment without a code change. +- **Fail soft**: if sqlglot can't parse the input, return the original + string. The downstream parser (`qualify_table_refs`) raises its own + error; we don't want to turn a parse failure into a translation + failure the user can't action. +""" + +from __future__ import annotations + + +def translate_pg_to_bq(sql: str, *, enabled: bool = True) -> tuple[str, list[str]]: + """Rewrite Postgres-dialect `sql` into BigQuery-dialect SQL. + + Returns `(translated_sql, notes)` where `notes` is a list of + human-readable strings describing each rewrite that was applied + (empty when the SQL was already BigQuery-compatible or the kill + switch is off). Never raises — a parse failure returns `(sql, [])` + so the downstream BigQuery parse surfaces the real error. + """ + if not enabled or not sql or not sql.strip(): + return sql, [] + + import sqlglot + from sqlglot import expressions as exp + + try: + tree = sqlglot.parse_one(sql, read="postgres") + except Exception: + return sql, [] + + notes: list[str] = [] + + # Rewrite ` ± ` → DATE_SUB/ADD with INTERVAL. + # sqlglot's postgres→bigquery transpile does NOT fix this on its own + # (verified against sqlglot 25.x) and it is the one hole we have seen + # in real customer traffic. Target the specific shape so we don't + # perturb unrelated integer arithmetic (e.g. a numeric column minus 1). + _rewrite_date_integer_arithmetic(tree, notes, exp=exp) + + # Pg allows `"timestamp_col" > CURRENT_DATE` through implicit coercion; + # BigQuery doesn't ("No matching signature for operator > for argument + # types: TIMESTAMP, DATE"). When one side of a comparison is a bare + # column reference and the other is a DATE-returning expression, wrap + # the DATE side in `TIMESTAMP(...)` so BQ sees TIMESTAMP vs TIMESTAMP. + # Runs after the arithmetic rewrite so DATE_SUB(...) from it gets + # wrapped too — the single shape `"col" > CURRENT_DATE - 1` ends up as + # `col > TIMESTAMP(DATE_SUB(CURRENT_DATE, INTERVAL 1 DAY))`. + _wrap_date_vs_column_comparison(tree, notes, exp=exp) + + # Pg's `EXTRACT(EPOCH FROM a - b)` returns a numeric count of seconds + # between two timestamps; BQ has no EPOCH extraction and no interval- + # typed subtraction, so this is `TIMESTAMP_DIFF(a, b, SECOND)`. + _rewrite_extract_epoch(tree, notes, exp=exp) + + + try: + translated = tree.sql(dialect="bigquery") + except Exception: + return sql, [] + + # Pg's `::timestamp` is semantically closer to BQ `DATETIME` (zone-less + # civil time), which is what sqlglot picks on write. But the CKAN + # BigQuery datastore backend creates `timestamp`-typed columns as BQ + # `TIMESTAMP` (zone-aware), so a predicate like `"ts_col" > CAST('…' AS + # DATETIME)` is a type mismatch at execution time. Retarget every + # `CAST(... AS DATETIME)` the writer emitted back to `CAST(... AS + # TIMESTAMP)` so comparisons stay legal. Done post-emit because the + # DATETIME shows up on the write side, not in the parsed AST. + translated, retargeted = _retarget_datetime_text(translated) + if retargeted: + notes.append( + f"retargeted {retargeted} `CAST(... AS DATETIME)` to " + "`CAST(... AS TIMESTAMP)` to match the backend's `TIMESTAMP` column type" + ) + + # Fix the fallout of sqlglot's `to_timestamp(ms/1000)` → `TIMESTAMP_SECONDS(ms/1000)` + # mapping: BigQuery's `/` operator returns FLOAT64, but TIMESTAMP_SECONDS + # requires INT64. Customer scripts carry the `to_timestamp(epoch_ms/1000)` + # idiom from pg; we collapse it to the direct `TIMESTAMP_MILLIS(epoch_ms)` + # equivalent to sidestep the type issue entirely. Runs post-emit because + # TIMESTAMP_SECONDS only appears after sqlglot's bigquery writer stage. + translated, ts_fixes = _fix_timestamp_seconds_text(translated) + if ts_fixes: + notes.append( + f"converted {ts_fixes} `TIMESTAMP_SECONDS(x / 1000)` call(s) to " + "`TIMESTAMP_MILLIS(x)` so BigQuery accepts the INT64 argument" + ) + + # Identifier quoting: pg uses `"ident"`, bq uses backticks. sqlglot + # does this on the write side; attribute it in `notes` so a reader + # of the log sees why identifiers changed. + if '"' in sql and "`" in translated and "`" not in sql: + notes.append("quoted identifiers rewritten from `\"ident\"` to backticks") + + if "::" in sql and "::" not in translated: + notes.append("`::type` casts rewritten to `CAST(... AS type)`") + + # Only report "sql dialect translated" (the catch-all) if the string + # actually changed and nothing more specific has been reported — a + # pure whitespace reformat doesn't count as a translation. + if _normalize(translated) != _normalize(sql) and not notes: + notes.append("sql dialect translated (postgres → bigquery)") + + return translated, notes + + +def _rewrite_date_integer_arithmetic(tree, notes: list[str], *, exp) -> None: + """In-place AST rewrite: ` ± ` → DATE_SUB/ADD. + + BigQuery rejects `CURRENT_DATE - 1` with "No matching signature for + operator - for argument types: DATE, INT64." Customer scripts rely on + it heavily (`WHERE deliveryStart > CURRENT_DATE - 1`), so we rewrite + just that shape. We match on the LEFT side being a date-returning + expression (CURRENT_DATE/CURRENT_DATETIME/CURRENT_TIMESTAMP or an + explicit DATE()/DATETIME()/TIMESTAMP() cast) and the RIGHT being a + bare integer literal — nothing else is touched, so a column of type + INT64 minus 1 keeps its original meaning. + """ + _DATE_FUNC_NAMES = { + "CurrentDate", + "CurrentDatetime", + "CurrentTimestamp", + "Date", + "Datetime", + "Timestamp", + } + + def is_date_expr(node) -> bool: + return type(node).__name__ in _DATE_FUNC_NAMES + + def is_int_literal(node) -> bool: + return isinstance(node, exp.Literal) and node.is_int + + rewrites = 0 + for node in list(tree.find_all(exp.Sub, exp.Add)): + left = node.this + right = node.expression + if not (is_date_expr(left) and is_int_literal(right)): + continue + interval = exp.Interval( + this=exp.Literal.number(right.this), + unit=exp.Var(this="DAY"), + ) + func_name = "DATE_SUB" if isinstance(node, exp.Sub) else "DATE_ADD" + replacement = exp.Anonymous(this=func_name, expressions=[left.copy(), interval]) + node.replace(replacement) + rewrites += 1 + + if rewrites: + notes.append( + f"rewrote {rewrites} date±integer expression(s) to " + "`DATE_SUB/ADD(..., INTERVAL n DAY)`" + ) + + +def _wrap_date_vs_column_comparison(tree, notes: list[str], *, exp) -> None: + """In-place AST rewrite: ` ` → ` TIMESTAMP()`. + + BigQuery rejects `TIMESTAMP > DATE` comparisons strictly. Customer SQL + carried over from Postgres relies on pg's implicit coercion + (`WHERE deliveryStart > CURRENT_DATE`, `WHERE delivery_date >= DATE '2024-01-01'`, + `WHERE col BETWEEN CURRENT_DATE AND CURRENT_DATE + 1`). We don't know + the column's actual BQ type at translation time, but wrapping the DATE + side in `TIMESTAMP(...)` is safe for both TIMESTAMP and DATETIME column + types, and is a no-op if the column really is a DATE (BQ accepts + `DATE > TIMESTAMP` via its own coercion in that direction). + + Trigger conditions: + - A comparison node (`>`, `>=`, `<`, `<=`, `=`, `!=`), or a BETWEEN. + - One side is an `exp.Column` (a bare quoted identifier). + - The other side is a DATE-returning expression we recognise. + + Non-triggers (left untouched to avoid perturbing legitimate SQL): + - Both sides are columns. + - The non-column side is already a TIMESTAMP/DATETIME expression. + - The non-column side is a function result we can't classify. + """ + _DATE_PRODUCING = { + "CurrentDate", # CURRENT_DATE + "Date", # DATE('...'), DATE(col) + } + _ALREADY_TIMESTAMP = { + "CurrentTimestamp", + "CurrentDatetime", + "Timestamp", # TIMESTAMP(...) wrapper, or ::timestamp cast + "Datetime", + "StrToTime", # to_timestamp(...) + "Cast", # ::timestamp / ::datetime (checked below) + } + + def is_date_producing(node) -> bool: + name = type(node).__name__ + if name in _DATE_PRODUCING: + return True + # `DATE_SUB(x, INTERVAL n DAY)` / `DATE_ADD` get emitted by the + # earlier arithmetic rewrite; they are `exp.Anonymous`. + if isinstance(node, exp.Anonymous): + func_name = (node.this or "").upper() + if func_name in ("DATE_SUB", "DATE_ADD"): + return True + # `DATE 'literal'` and a cast to DATE. + if isinstance(node, exp.Cast): + to = node.args.get("to") + if to is not None and str(to).strip().upper() == "DATE": + return True + # `CURRENT_DATE + INTERVAL '2 day'` — sqlglot models this as + # exp.Add(this=, expression=exp.Interval). Pg accepts it, + # BQ needs it wrapped; we recurse on the date-side so a nested + # `CURRENT_DATE + 1 + INTERVAL '…'` chain still classifies. + if isinstance(node, (exp.Add, exp.Sub)): + other = node.expression + if isinstance(other, exp.Interval) and is_date_producing(node.this): + return True + return False + + def is_already_timestamp(node) -> bool: + if type(node).__name__ in _ALREADY_TIMESTAMP: + # For a Cast, only "already timestamp" when the target type says so. + if isinstance(node, exp.Cast): + to = node.args.get("to") + return to is not None and str(to).strip().upper() in ( + "TIMESTAMP", "DATETIME", "TIMESTAMPTZ" + ) + return True + return False + + def wrap(node): + return exp.Anonymous(this="TIMESTAMP", expressions=[node.copy()]) + + rewrites = 0 + comparison_types = (exp.GT, exp.GTE, exp.LT, exp.LTE, exp.EQ, exp.NEQ) + + for cmp_node in list(tree.find_all(*comparison_types)): + left = cmp_node.this + right = cmp_node.expression + if left is None or right is None: + continue + + # Only act when exactly one side is a bare column reference. + left_is_col = isinstance(left, exp.Column) + right_is_col = isinstance(right, exp.Column) + if left_is_col == right_is_col: + continue + + col_side, other_side = (left, right) if left_is_col else (right, left) + if is_already_timestamp(other_side): + continue + if not is_date_producing(other_side): + continue + + if left_is_col: + cmp_node.set("expression", wrap(other_side)) + else: + cmp_node.set("this", wrap(other_side)) + rewrites += 1 + + # BETWEEN gets its own AST node and a `low`/`high` pair. Same shape + # argument as above — a column BETWEEN two DATE expressions wants both + # bounds wrapped. + for between in list(tree.find_all(exp.Between)): + subject = between.this + if not isinstance(subject, exp.Column): + continue + for key in ("low", "high"): + bound = between.args.get(key) + if bound is None or is_already_timestamp(bound) or not is_date_producing(bound): + continue + between.set(key, wrap(bound)) + rewrites += 1 + + if rewrites: + notes.append( + f"wrapped {rewrites} DATE expression(s) in `TIMESTAMP(...)` " + "to match TIMESTAMP column comparisons" + ) + + +import re as _re # noqa: E402 + +# Post-emit text rewrite: `TIMESTAMP_SECONDS( / 1000)` → `TIMESTAMP_MILLIS()`. +# `` is the shortest run of chars that balances parens; we restrict to a +# single argument (no commas) and no quotes so we never accidentally splice +# through a string literal. The `/ 1000` divisor is matched tightly — only +# bare `1000`, not `1000.0` or variables — because that is the specific pg +# idiom (`to_timestamp(epoch_ms/1000)`) we are compensating for. +_TIMESTAMP_SECONDS_MS_RE = _re.compile( + r"\bTIMESTAMP_SECONDS\(\s*(?P[^(),'\"]+?)\s*/\s*1000\s*\)", + _re.IGNORECASE, +) + + +def _fix_timestamp_seconds_text(translated: str) -> tuple[str, int]: + """Collapse the `TIMESTAMP_SECONDS(x / 1000)` idiom to `TIMESTAMP_MILLIS(x)`. + + BigQuery `/` returns FLOAT64 whereas TIMESTAMP_SECONDS wants INT64, so + the literal `TIMESTAMP_SECONDS(x / 1000)` the writer emits for pg's + `to_timestamp(epoch_ms/1000)` always fails. TIMESTAMP_MILLIS takes the + numerator directly in INT64 and matches the intended semantics exactly. + + Rewrite is applied only outside single-quoted string literals so a + column aliased with the literal text `TIMESTAMP_SECONDS(x / 1000)` + stays verbatim. + """ + out_parts: list[str] = [] + count = 0 + pos = 0 + n = len(translated) + while pos < n: + ch = translated[pos] + # Walk past a single-quoted string literal untouched. Pg accepts + # `''` as an embedded single quote; BQ also accepts `\'`. + if ch == "'": + end = pos + 1 + while end < n: + if translated[end] == "\\" and end + 1 < n: + end += 2 + continue + if translated[end] == "'": + if end + 1 < n and translated[end + 1] == "'": + end += 2 + continue + end += 1 + break + end += 1 + out_parts.append(translated[pos:end]) + pos = end + continue + # In code: look for the next string literal or the end, and run + # the regex only within that span. + next_quote = translated.find("'", pos) + span_end = n if next_quote == -1 else next_quote + chunk = translated[pos:span_end] + new_chunk, c = _TIMESTAMP_SECONDS_MS_RE.subn( + lambda m: f"TIMESTAMP_MILLIS({m.group('expr').strip()})", + chunk, + ) + out_parts.append(new_chunk) + count += c + pos = span_end + return "".join(out_parts), count + +# Post-emit text rewrite: `CAST(expr AS DATETIME)` → `CAST(expr AS TIMESTAMP)`. +# The `AS DATETIME)` suffix is distinctive enough that this is safe against +# the specific SQL shapes sqlglot emits (always UPPER-cased type names, always +# a single space before AS). The pattern won't match an identifier literally +# named "DATETIME" inside a string, since those are either single-quoted or +# backtick-quoted, not bare keywords after an AS. +_DATETIME_CAST_RE = _re.compile(r"\bAS\s+DATETIME\s*\)") + + +def _retarget_datetime_text(translated: str) -> tuple[str, int]: + """Flip every CAST(... AS DATETIME) in the emitted SQL to AS TIMESTAMP. + + Returns the rewritten SQL and the number of substitutions made so the + caller can decide whether to attribute a note. + """ + def _sub(_m: _re.Match[str]) -> str: + return "AS TIMESTAMP)" + + new, count = _DATETIME_CAST_RE.subn(_sub, translated) + return new, count + + +def _rewrite_extract_epoch(tree, notes: list[str], *, exp) -> None: + """In-place AST rewrite: `EXTRACT(EPOCH FROM a - b)` → `TIMESTAMP_DIFF(a, b, SECOND)`. + + Pg's EXTRACT(EPOCH FROM interval) returns the interval's seconds as a + number. BQ has no EPOCH extraction unit, and TIMESTAMP subtraction isn't + an operator at all — the equivalent is `TIMESTAMP_DIFF(a, b, SECOND)`. + + Covers two shapes seen in real customer traffic: + - `EXTRACT(EPOCH FROM a - b)` → `TIMESTAMP_DIFF(a, b, SECOND)` + - `EXTRACT(EPOCH FROM col)` → `UNIX_SECONDS(col)` (bare timestamp, + pg's "seconds since epoch for a timestamp" idiom) + """ + rewrites = 0 + for node in list(tree.find_all(exp.Extract)): + unit = node.this + if unit is None: + continue + unit_name = (unit.name if hasattr(unit, "name") else str(unit)).upper() + if unit_name != "EPOCH": + continue + src = node.expression + if src is None: + continue + # Unwrap one layer of parentheses that sqlglot models as exp.Paren. + inner = src.this if isinstance(src, exp.Paren) else src + if isinstance(inner, exp.Sub): + left_arg = inner.this + right_arg = inner.expression + replacement = exp.Anonymous( + this="TIMESTAMP_DIFF", + expressions=[left_arg.copy(), right_arg.copy(), exp.Var(this="SECOND")], + ) + else: + replacement = exp.Anonymous(this="UNIX_SECONDS", expressions=[inner.copy()]) + node.replace(replacement) + rewrites += 1 + + if rewrites: + notes.append( + f"rewrote {rewrites} `EXTRACT(EPOCH FROM …)` expression(s) to " + "`TIMESTAMP_DIFF(a, b, SECOND)` / `UNIX_SECONDS(col)`" + ) + + +def _normalize(sql: str) -> str: + """Collapse whitespace so a dialect change isn't masked by formatting.""" + return " ".join(sql.split()) diff --git a/datastore/infrastructure/engines/bigquery/types.py b/datastore/infrastructure/engines/bigquery/types.py index d78d085..20577f1 100644 --- a/datastore/infrastructure/engines/bigquery/types.py +++ b/datastore/infrastructure/engines/bigquery/types.py @@ -20,7 +20,7 @@ # BigQuery has no native equivalent that round-trips losslessly. FRICTIONLESS_TO_BIGQUERY: dict[str, str] = { "integer": "INT64", - "number": "FLOAT64", + "number": "BIGNUMERIC", "string": "STRING", "boolean": "BOOL", "date": "DATE", diff --git a/datastore/legacy/__init__.py b/datastore/legacy/__init__.py new file mode 100644 index 0000000..805c363 --- /dev/null +++ b/datastore/legacy/__init__.py @@ -0,0 +1,105 @@ +"""Legacy CKAN datastore API — DEPRECATED. + +================================================================ + WHY THIS PACKAGE EXISTS +================================================================ + +This whole subtree is a backwards-compatibility shim for callers who +hit the public CKAN datastore URLs — `/api/3/action/datastore_*` and +`/datastore/dump/{resource_id}`. It is not part of the v2 contract and +it is scheduled for removal. + +Everything here will be deleted — the folder in one shot, nothing from +it imported elsewhere — once customers migrate off the legacy URLs. +Do not add new features here; new work lives in the v2 surface +(`datastore/api/`, `datastore/services/`, `datastore/schemas/`). + +================================================================ + LAYOUT +================================================================ + + legacy/ + ├── __init__.py # this file — package docstring + banner helper + ├── constants.py # URL prefixes, dump formats, hidden columns + ├── schemas.py # legacy-only Pydantic subclasses + ├── streaming.py # envelope + dump byte writers (pure Python) + └── endpoints.py # FastAPI routers (the ONE fastapi-aware module) + +Legacy follows the same one-way layering v2 uses: + endpoints.py → streaming.py → (plain data) + +Only `endpoints.py` imports from `fastapi`/`starlette`. The layer- +invariant grep in CI continues to pass as-is. + +================================================================ + WHAT LEGACY ALTERS vs. v2 +================================================================ + +- Response envelope: no `schema`, no `records_format` beyond + `objects`, no `warnings`. +- `fields` entries are the minimal `{"type", "id"}` shape; v2's + `info` dictionary is dropped. +- Engine-managed `_id` / `_updated_at` columns are stripped from both + `fields` and `records`. +- `datastore_search_sql` accepts SQL without a LIMIT clause (v2 + requires one). +- Dump supports CSV / TSV / JSON (array) / XML and the full + `datastore_search` filter param set, plus `bom=true` for Excel. + +================================================================ + REMOVAL PLAN +================================================================ + +When no customer traffic is observed on the legacy URLs for an agreed +window, run these steps in order. Each step is independent — the app +boots after any one of them, so the removal can be split across PRs. + + 1. **routes.py** — remove the three lines in `datastore/api/routes.py` + that mount `legacy_endpoints.router` and `legacy_endpoints.dump_router` + (search for the `DEPRECATED legacy CKAN surface` block). + 2. **main.py** — remove the `log_startup_banner` import at the top and + the `_log_legacy_banner()` call in the lifespan (both marked + `DEPRECATED legacy surface`). + 3. **analytics.py** — remove the `from datastore.legacy.constants` + import and every block marked `LEGACY ONLY` in + `datastore/analytics.py`. Three sites total: the two prefix + constants near the top, the two `if path.startswith(LEGACY_*)` + branches in `action_name`, the `LEGACY_ACTION_PREFIX` entry in + the POST-capture tuple, and the `LEGACY_DUMP_PATH_PREFIX` entry + in `_resource_ref`'s prefix loop. + 4. **tests/test_legacy_api.py** — delete the file. + 5. **datastore/legacy/** — `rm -rf` the whole folder. + 6. **pyproject.toml** — drop the `snowballstemmer` dependency; it + exists only for byte-for-byte `_full_text` tsvector parity. + +Verification: `rg "datastore\.legacy|/api/3/action|/datastore/dump" +datastore/ tests/` must return nothing after the above. Then run +`pytest` + `ruff check` to confirm no stale references leak through. + +Nothing in v2 imports from this package. The removal cannot break the +v2 surface because every touch point outside this folder is marked +with either `DEPRECATED` (file-level) or `LEGACY ONLY` (line-level). +""" + +from __future__ import annotations + +import logging + +from datastore.legacy.constants import LEGACY_API_PREFIX, LEGACY_DUMP_PREFIX + +_log = logging.getLogger(__name__) + + +def log_startup_banner() -> None: + """Emit a WARNING at app boot so the deprecated surface is obvious + in the first page of logs — matches `main.py`'s own ready-log tone. + + Called from `create_app()` after the engines log their state. One + log line per process start; no per-request overhead. + """ + _log.warning( + "LEGACY CKAN routes mounted (deprecated, pending removal): " + "%s/datastore_{search,search_sql,info} %s/{resource_id}", + LEGACY_API_PREFIX, + LEGACY_DUMP_PREFIX, + ) diff --git a/datastore/legacy/constants.py b/datastore/legacy/constants.py new file mode 100644 index 0000000..01d38b4 --- /dev/null +++ b/datastore/legacy/constants.py @@ -0,0 +1,75 @@ +"""Constants used ONLY by the legacy CKAN compatibility surface. + +Kept in a legacy-local module (not `datastore/core/constants.py`) so +anyone grepping core/ for live config doesn't trip over values that are +about to be removed, and so the removal step is `rm -rf datastore/legacy` +with no scattered follow-ups. + +DEPRECATED — see `datastore/legacy/__init__.py` for the removal plan. +""" + +from __future__ import annotations + +from typing import Literal + +# Legacy CKAN URL prefixes. Mounted so scripts hitting the legacy CKAN +# datastore (`/api/3/action/...` and `/datastore/dump/...`) keep working +# byte-for-byte while callers migrate to v2. +LEGACY_API_PREFIX = "/api/3/action" +LEGACY_DUMP_PREFIX = "/datastore/dump" + +# Engine-managed columns the legacy response must NOT expose on the +# JSON action API (`datastore_search`, `datastore_search_sql`, +# `datastore_info`). `_updated_at` is datastore-internal (last-write +# timestamp used by the dump cache invalidation path), and real CKAN +# never surfaces it to clients. `_id` is kept here — real CKAN exposes +# it as a stable row id that scripts in the wild filter and sort by +# (`?sort=_id asc`). +LEGACY_HIDDEN_COLUMNS: frozenset[str] = frozenset({"_updated_at"}) + +# The dump endpoint (`/datastore/dump/{id}?format=…`) hides one more +# column than the action API: `_id`. Real CKAN's dump omits it because +# a download is meant to look like the "clean" dataset a publisher +# would hand over — the datastore's row-id serial is an implementation +# detail that shouldn't leak into CSVs people archive and share. +LEGACY_DUMP_HIDDEN_COLUMNS: frozenset[str] = LEGACY_HIDDEN_COLUMNS | {"_id"} + +# CKAN synthesises a `_full_text` column on every Postgres-backed +# datastore table (a `tsvector` built from the row's text columns). +# BigQuery has no equivalent, so we compute an approximation at +# response time: a space-joined concatenation of the row's non-null +# text values. Callers that filter on `?q=` are driven by the engine's +# own search index — this synthesised column is for display parity +# only, so a client script that reads `record["_full_text"]` sees a +# value in the shape it expects. Emitted on `datastore_search` only; +# `datastore_search_sql` echoes the user's SELECT shape verbatim. +LEGACY_FULL_TEXT_FIELD: dict[str, str] = {"type": "tsvector", "id": "_full_text"} + +# Legacy CKAN dump formats. Mirrors what CKAN 2.9 serves at +# `/datastore/dump/{id}?format=…` (see §9 of the CKAN maintainer docs). +# All four stream API-side — no GCS redirect — because tsv/json/xml have +# no engine-native export and even csv needs row-level filtering (hidden +# system columns, user `filters` / `q`), which the GCS export pipeline +# does not do. Keeping one code path across formats is the win. +LegacyDumpFormat = Literal["csv", "tsv", "json", "xml"] +LEGACY_DUMP_FORMATS: tuple[str, ...] = ("csv", "tsv", "json", "xml") + +LEGACY_DUMP_MEDIA_TYPES: dict[str, str] = { + "csv": "text/csv; charset=utf-8", + "tsv": "text/tab-separated-values; charset=utf-8", + "json": "application/json", + "xml": "application/xml; charset=utf-8", +} + +LEGACY_DUMP_EXTENSIONS: dict[str, str] = { + "csv": "csv", + "tsv": "tsv", + "json": "json", + "xml": "xml", +} + +# UTF-8 byte-order mark. CKAN's `?bom=true` prepends it so Excel on +# Windows opens the file as UTF-8 instead of mis-decoding it as cp1252. +# Only meaningful for the text formats; `?bom=true&format=json`/`xml` +# still prepends — matching CKAN rather than second-guessing callers. +UTF8_BOM = b"\xef\xbb\xbf" diff --git a/datastore/legacy/endpoints.py b/datastore/legacy/endpoints.py new file mode 100644 index 0000000..c1dce7b --- /dev/null +++ b/datastore/legacy/endpoints.py @@ -0,0 +1,614 @@ +"""Legacy CKAN-compatible HTTP endpoints. + +DEPRECATED — see `datastore/legacy/__init__.py` for the removal plan. + +The one FastAPI-aware module in the legacy package: everything that +touches `Request`, `Response`, `StreamingResponse`, routing, or +`Depends` lives here. The streaming writers and the request schema +subclass are in sibling modules (`streaming.py`, `schemas.py`) so +future maintainers don't accidentally couple business logic to the +legacy wrapper. + +Mounted at `/api/3/action/` and `/datastore/dump/` by +`datastore/api/routes.py`. Hidden from the v2 OpenAPI document via +`include_in_schema=False` so Swagger users see only v2 — the legacy +URLs keep working for scripts in the wild, but new integrators are +steered to the current contract. +""" + +from __future__ import annotations + +import asyncio +import logging +from collections.abc import Callable, Coroutine +from typing import Annotated, Any + +from fastapi import APIRouter, Body, Query +from fastapi.exceptions import RequestValidationError +from fastapi.routing import APIRoute +from starlette.requests import Request +from starlette.responses import Response, StreamingResponse + +from datastore.api.context import Context +from datastore.api.error_handlers import _format_loc, _group_errors +from datastore.api.responses import ERROR_RESPONSES, ORJSONResponse, _success_response +from datastore.core.exceptions import APIError, ValidationError +from datastore.core.helper import content_disposition, dump_filename_base +from datastore.infrastructure.engines import get_datastore_engine +from datastore.infrastructure.engines.registry import get_allowed_sql_functions +from datastore.legacy.constants import ( + LEGACY_DUMP_EXTENSIONS, + LEGACY_DUMP_MEDIA_TYPES, + LEGACY_FULL_TEXT_FIELD, + LegacyDumpFormat, +) +from datastore.legacy.schemas import DatastoreLegacySearchSQLRequest +from datastore.legacy.streaming import ( + build_legacy_pagination_links, + dump_csv, + dump_json, + dump_tsv, + dump_xml, + legacy_info_result, + stream_legacy_search, + stream_legacy_search_sql, +) +from datastore.schemas.request import DatastoreInfoRequest, DatastoreSearchRequest +from datastore.schemas.validators import ( + ResourceId, + frictionless_schema_to_fields, + to_csv_list, + to_json_object, + to_str_or_json_object, +) +from datastore.services.read import enrich_schema + +# Signed-int32 max — well under BigQuery's LIMIT ceiling, lets a legacy +# dump without an explicit `?limit=` drain the whole table while still +# passing a concrete int to `engine.search` (which takes `limit: int`, +# not `int | None`). Callers wanting a smaller cap pass `?limit=N`. +_LEGACY_DUMP_UNBOUNDED = 2**31 - 1 + + +_log = logging.getLogger(__name__) + + +def _legacy_help_url(request: Request) -> str: + """CKAN legacy help URL for a legacy action request.""" + action = request.url.path.rsplit("/", 1)[-1] + return f"{request.url.scheme}://{request.url.netloc}/api/3/action/help_show?name={action}" + + +# HTTP status code the legacy surface should emit for each logical error +# type. These match what the legacy CKAN API returns on the same shapes +# (verified end-to-end): +# +# - unknown resource via datastore_search → 404 +# - unknown resource via datastore_search_sql → 409 +# - validation error on action args / SQL → 409 +# - validation on datastore_search bad body → 409 +# +# so a client script doing +# +# if 200 <= status < 300: ... # success +# elif status == 404: ... # resource gone +# elif status == 409: ... # validation / query error +# +# behaves identically against our service and the legacy CKAN. Previously +# we masked everything as 200 (classic upstream CKAN parity), which broke +# clients that mirror the legacy CKAN's status-code taxonomy. +_LEGACY_STATUS_BY_TYPE: dict[str, int] = { + "Validation Error": 409, + "Conflict Error": 409, + "Not Found Error": 404, + "Authorization Error": 403, + "Internal Error": 500, +} + + +class _CKANCompatRoute(APIRoute): + """FastAPI route that mimics the legacy CKAN API's action-API error + status codes exactly. + + Historical note: upstream CKAN's action API returns **HTTP 200** with + `success:false` for every logical error. The legacy CKAN API we're + replacing sits behind a gateway that promotes those back to real HTTP + codes — 409 for Validation / Conflict, 404 for Not Found, 403 for + Auth. We match the legacy shape rather than the classic CKAN one, so + scripts tested against the legacy API behave identically against us. + + Body shape stays the one real CKAN emits — `{help, success:false, + error: {, __type}}` with per-field details spread directly + on `error` (not nested under `error.fields`). Only the HTTP status + moves. + + Applied via `APIRouter(route_class=_CKANCompatRoute)` on the legacy + action router only. The dump router keeps standard HTTP semantics + because it serves raw files, not the action envelope. + """ + + def get_route_handler(self) -> Callable[[Request], Coroutine[Any, Any, Response]]: + original = super().get_route_handler() + + async def ckan_handler(request: Request) -> Response: + is_sql_path = request.url.path.endswith("/datastore_search_sql") + try: + return await original(request) + except APIError as exc: + if exc.status_code >= 500: + # Server fault — let the global 5xx handler log it + # and emit a 5xx so monitoring stays honest. CKAN + # itself does the same. + raise + type_label = exc.type_label + fields = exc.fields or _fields_from_engine_message(exc.message) + if is_sql_path: + # Any error on `datastore_search_sql` is reported by + # CKAN under `error.query` as a 409 Validation Error. + # Covers unknown table (which the engine raises as + # NotFoundError), engine-level DB errors, and the + # schema-level `sql` field key alike. + type_label = "Validation Error" + fields = {"query": [exc.message]} + status = _LEGACY_STATUS_BY_TYPE.get(type_label, 409) + _log.debug( + "legacy APIError %s %s -> %s (HTTP %d)", + request.method, + request.url.path, + type_label, + status, + ) + return _legacy_error_envelope( + request, + type_label=type_label, + message=exc.message, + fields=fields, + status_code=status, + ) + except RequestValidationError as exc: + errors = list(exc.errors()) + fields = _group_errors(errors) + first = errors[0] if errors else {"msg": "invalid request"} + message = f"{_format_loc(tuple(first.get('loc', ())))}: {first.get('msg')}" + if is_sql_path: + # CKAN keys SQL-content errors (bad SQL, non-SELECT, + # multi-statement, …) under `error.query`, but keeps + # a plain missing-value error under `error.sql`. + # Mirror that: rename `sql` → `query` only for + # errors that aren't the "field required" kind. + sql_errors_are_content = any( + e.get("type") != "missing" + for e in errors + if e.get("loc") and e["loc"][-1] == "sql" + ) + if sql_errors_are_content and "sql" in fields: + fields = { + ("query" if k == "sql" else k): v for k, v in fields.items() + } + _log.debug( + "legacy RequestValidationError %s %s -> HTTP 409", + request.method, + request.url.path, + ) + return _legacy_error_envelope( + request, + type_label="Validation Error", + message=message, + fields=fields, + status_code=_LEGACY_STATUS_BY_TYPE["Validation Error"], + ) + + return ckan_handler + + +_ENGINE_FIELD_KEYED_PARAMS = ("filters", "q", "fields", "sort") + + +def _with_full_text_field( + fields: list[dict[str, Any]], +) -> list[dict[str, Any]]: + """Return a copy of `fields` with `_full_text` spliced in right + after `_id` (or at the head when `_id` isn't present), matching + the ordering CKAN emits for `SELECT *` responses. + """ + pos = 0 + for i, f in enumerate(fields): + if f.get("id") == "_id": + pos = i + 1 + break + return fields[:pos] + [LEGACY_FULL_TEXT_FIELD] + fields[pos:] + + +def _fields_from_engine_message(message: str) -> dict[str, list[str]] | None: + """Parse `" references unknown column 'X'"` into a legacy + field-keyed detail map so the error envelope matches CKAN: + `{"": ["invalid value \"X\""]}`. Returns None otherwise. + """ + for param in _ENGINE_FIELD_KEYED_PARAMS: + prefix = f"{param} references unknown column " + if message.startswith(prefix): + col = message[len(prefix):].strip().strip("'\"") + return {param: [f'invalid value "{col}"']} + return None + + +def _legacy_error_envelope( + request: Request, + *, + type_label: str, + message: str, + fields: dict[str, list[str]] | None = None, + status_code: int = 409, +) -> ORJSONResponse: + """CKAN-shaped error envelope at the HTTP status the legacy CKAN API + emits. + + Matches the exact body shape the legacy CKAN API returns, verified + end-to-end: + + { + "help": "...", + "success": false, + "error": { + "": ["", ...], # per-field keys at ERROR ROOT + "__type": "..." + } + } + + Specifics preserved for 1:1 parity: + + - No top-level `message` key. CKAN encodes the human-readable + reason inside the per-field arrays (or, when there's no field + context, under a generic `message` key — but only when fields + are absent). + - Per-field details are spread **directly on the error dict** + (not nested under `fields`). A caller doing + `error["sql"]` or `error["query"]` must find them there. + - `__type` is the last key by convention; `orjson` preserves + insertion order, so we write it last. + + `help` echoes the request URL — same thing the success envelope + does, so a legacy caller can't tell the two envelopes apart by + looking at the help key. + + `status_code` defaults to 409 (what prod emits for Validation / + Conflict). `_CKANCompatRoute` picks the right code per error type; + callers that build an envelope directly can override as needed. + """ + error: dict[str, Any] = {} + if fields: + # Per-field details go at the error-dict root (CKAN shape). + # Field keys never collide with `__type` because we never + # declare a field literally named `__type`; a defensive guard + # would be strict-but-wrong (CKAN itself doesn't do it). + error.update(fields) + else: + # No field context — surface the human message under `message`. + # Dropping it entirely would make the error unparseable for + # callers that only look at the error dict. + error["message"] = message + error["__type"] = type_label + + return ORJSONResponse( + {"help": str(request.url), "success": False, "error": error}, + status_code=status_code, + ) + + +# `include_in_schema=False` keeps the legacy surface out of the v2 +# OpenAPI document. Legacy exists purely for backwards compatibility — +# surfacing it in Swagger would advertise URLs we actively want callers +# to migrate off. Scripts targeting the legacy paths keep working; new +# integrators see only v2. +# +# `route_class=_CKANCompatRoute` makes the actions return HTTP 200 on +# logical errors (CKAN semantics). The dump router stays on the default +# route class — it serves raw files, not the JSON envelope, and HTTP +# error codes are the right thing for a browser hitting a bad download +# URL. +router = APIRouter( + tags=["Datastore (Legacy)"], + responses=ERROR_RESPONSES, + include_in_schema=False, + route_class=_CKANCompatRoute, +) +dump_router = APIRouter( + tags=["Datastore (Legacy)"], + responses=ERROR_RESPONSES, + include_in_schema=False, +) + + +# --- datastore_search ------------------------------------------------------ + + +@router.get("/datastore_search", summary="Legacy CKAN datastore_search (GET)") +async def datastore_search_get( + request: Request, + context: Context, + params: Annotated[DatastoreSearchRequest, Query()], +) -> StreamingResponse: + return await _legacy_search_impl(request, context, params) + + +@router.post("/datastore_search", summary="Legacy CKAN datastore_search (POST)") +async def datastore_search_post( + request: Request, + context: Context, + payload: Annotated[DatastoreSearchRequest, Body()], +) -> StreamingResponse: + return await _legacy_search_impl(request, context, payload) + + +async def _legacy_search_impl( + request: Request, + context: Context, + params: DatastoreSearchRequest, +) -> StreamingResponse: + """Shared body: authorize, run the engine, stream the legacy envelope.""" + auth_result = await context.authorize(resource_id=params.resource_id, permission="read") + + engine = get_datastore_engine(context, mode="ro") + result = await asyncio.to_thread( + engine.search, + resource_id=params.resource_id, + filters=to_json_object(params.filters), + q=to_str_or_json_object(params.q), + fields=to_csv_list(params.fields), + distinct=params.distinct, + plain=params.plain, + language=params.language, + limit=params.limit, + offset=params.offset, + sort=params.sort, + include_total=params.include_total, + ) + + schema = enrich_schema(result.schema, (auth_result or {}).get("resource")) + fields, _ = frictionless_schema_to_fields(schema) + body_iter = stream_legacy_search( + help_url=_legacy_help_url(request), + resource_id=params.resource_id, + fields=fields, + records=result.records, + limit=params.limit, + offset=params.offset, + total=result.total, + include_total=params.include_total, + records_format=params.records_format, + links=build_legacy_pagination_links( + str(request.url), + limit=params.limit, + offset=params.offset, + ), + ) + return StreamingResponse(body_iter, media_type="application/json") + + +# --- datastore_search_sql -------------------------------------------------- + + +@router.get("/datastore_search_sql", summary="Legacy CKAN datastore_search_sql (GET)") +async def datastore_search_sql_get( + request: Request, + context: Context, + params: Annotated[DatastoreLegacySearchSQLRequest, Query()], +) -> StreamingResponse: + return await _legacy_search_sql_impl(request, context, params) + + +@router.post("/datastore_search_sql", summary="Legacy CKAN datastore_search_sql (POST)") +async def datastore_search_sql_post( + request: Request, + context: Context, + payload: Annotated[DatastoreLegacySearchSQLRequest, Body()], +) -> StreamingResponse: + return await _legacy_search_sql_impl(request, context, payload) + + +async def _legacy_search_sql_impl( + request: Request, + context: Context, + params: DatastoreLegacySearchSQLRequest, +) -> StreamingResponse: + """Authorize every referenced table, enforce the function allow-list, + run the SQL and stream the legacy SQL envelope. + + LIMIT is optional (legacy CKAN behaviour). **No row cap applies**: + `datastore_search_sql` is where CKAN historically let callers pull + arbitrary query results, so a legacy script doing + `SELECT * FROM ""` on a 10M-row table streams the whole set. + `SEARCH_RESULT_ROWS_MAX` still gates `datastore_search`; it does + not apply here. The engine streams row-by-row so peak memory is + bounded regardless of result size. + """ + # Collect each referenced resource's authorize payload — under + # AUTH_TYPE=ckan it carries the resource record (with its synced + # schema), merged into the output below so computed-but-same-named + # columns inherit CKAN-side data-dictionary edits. + resources: list[dict[str, Any]] = [] + for resource_id in params.resource_ids: + auth_result = await context.authorize(resource_id=resource_id, permission="read") + resource = (auth_result or {}).get("resource") + if isinstance(resource, dict): + resources.append(resource) + + allowed = get_allowed_sql_functions( + context.config.DATASTORE_ENGINE, + override_path=context.config.SQL_FUNCTIONS_ALLOW_FILE, + ) + disallowed = sorted(set(params.function_names) - allowed) + if disallowed: + raise ValidationError( + f"sql uses disallowed function(s): {', '.join(disallowed)}", + fields={"sql": [f"disallowed: {', '.join(disallowed)}"]}, + ) + + # No cap: when the caller omits LIMIT we pass int32-max so the + # engine's `limit: int` signature stays non-optional, matching the + # dump endpoint's approach. The engine streams; it does not + # materialise the result set. + engine = get_datastore_engine(context, mode="ro") + effective_limit = params.limit if params.limit is not None else _LEGACY_DUMP_UNBOUNDED + result = await asyncio.to_thread( + engine.search_sql, + sql=params.sql, + limit=effective_limit, + ) + schema = enrich_schema(result.schema, resources) + fields, _ = frictionless_schema_to_fields(schema) + if params.has_select_star: + fields = _with_full_text_field(fields) + body_iter = stream_legacy_search_sql( + help_url=_legacy_help_url(request), + sql=params.sql, + fields=fields, + records=result.records, + ) + return StreamingResponse(body_iter, media_type="application/json") + + +# --- datastore_info -------------------------------------------------------- + + +@router.get("/datastore_info", summary="Legacy CKAN datastore_info (GET)") +async def datastore_info_get( + request: Request, + context: Context, + params: Annotated[DatastoreInfoRequest, Query()], +): + return await _legacy_info_impl(request, context, params) + + +@router.post("/datastore_info", summary="Legacy CKAN datastore_info (POST)") +async def datastore_info_post( + request: Request, + context: Context, + payload: Annotated[DatastoreInfoRequest, Body()], +): + return await _legacy_info_impl(request, context, payload) + + +async def _legacy_info_impl( + request: Request, + context: Context, + params: DatastoreInfoRequest, +): + """Legacy info: `result.{resource_id, fields, meta}`, fields filtered.""" + # `_require_resource_id_or_id` on `DatastoreInfoRequest` normalises + # `id` → `resource_id` and rejects the all-None case, so by this point + # `resource_id` is always set. Narrow the type for mypy. + assert params.resource_id is not None + resource_id: str = params.resource_id + data_dict = await context.authorize(resource_id=resource_id, permission="read") + engine = get_datastore_engine(context, mode="ro") + info = await asyncio.to_thread(engine.info, resource_id=resource_id) + + schema = info.schema + if context.config.AUTH_TYPE == "ckan": + ckan_schema = (data_dict.get("resource") or {}).get("schema") + if ckan_schema: + schema = ckan_schema + fields, _ = frictionless_schema_to_fields(schema) + + return _success_response( + request, + legacy_info_result( + fields=fields, + meta=info.meta, + ), + ) + + +# --- dump ------------------------------------------------------------------ + + +_DUMP_WRITERS = { + "csv": dump_csv, + "tsv": dump_tsv, + "json": dump_json, + "xml": dump_xml, +} + + +@dump_router.get( + "/{resource_id}", + summary="Legacy CKAN datastore dump", +) +async def legacy_dump( + request: Request, + context: Context, + resource_id: ResourceId, + fmt: Annotated[LegacyDumpFormat, Query(alias="format")] = "csv", + bom: bool = False, + # datastore_search filter / projection params — all optional. + filters: str | None = None, + q: str | None = None, + distinct: bool = False, + plain: bool = True, + language: str = "english", + limit: int | None = None, + offset: int = 0, + fields: str | None = None, + sort: str | None = None, +) -> StreamingResponse: + """Export a resource as a file. Mirrors CKAN's `/datastore/dump/{id}`. + + Formats: `csv` (default) · `tsv` · `json` (JSON array) · `xml`. + + All downloads stream API-side rather than redirecting to GCS — the + engine's native GCS export doesn't do row-level filtering (hidden + `_id` / `_updated_at` columns, user `filters` / `q`) and only + produces csv/ndjson/parquet. One streaming path across formats keeps + the response behaviour identical for every caller. + + All of `datastore_search`'s filter / projection params are accepted + (`filters`, `q`, `distinct`, `plain`, `language`, `limit`, `offset`, + `fields`, `sort`). Omitting `limit` streams the whole table. + + `bom=true` prepends the UTF-8 byte-order mark for Excel-on-Windows + compatibility. + """ + await context.authorize(resource_id=resource_id, permission="read") + + # Validate filter payloads up front — same parsers v2 search uses. + parsed_filters = to_json_object(filters) + parsed_q = to_str_or_json_object(q) + parsed_fields = to_csv_list(fields) + + if offset < 0: + raise ValidationError("offset must be >= 0", fields={"offset": ["must be >= 0"]}) + if limit is not None and limit < 0: + raise ValidationError("limit must be >= 0", fields={"limit": ["must be >= 0"]}) + + effective_limit = limit if limit is not None else _LEGACY_DUMP_UNBOUNDED + + engine = get_datastore_engine(context, mode="ro") + result = await asyncio.to_thread( + engine.search, + resource_id=resource_id, + filters=parsed_filters, + q=parsed_q, + fields=parsed_fields, + distinct=distinct, + plain=plain, + language=language, + limit=effective_limit, + offset=offset, + sort=sort, + # Row counts are expensive on a full-table dump and the output + # format carries no envelope to put them in. + include_total=False, + ) + + legacy_fields, _ = frictionless_schema_to_fields(result.schema) + writer = _DUMP_WRITERS[fmt] + body_iter = writer(fields=legacy_fields, records=result.records, bom=bom) + + filename_base = dump_filename_base(None, LEGACY_DUMP_EXTENSIONS[fmt]) or resource_id + filename = f"{filename_base}.{LEGACY_DUMP_EXTENSIONS[fmt]}" + return StreamingResponse( + body_iter, + media_type=LEGACY_DUMP_MEDIA_TYPES[fmt], + headers={"Content-Disposition": content_disposition(filename)}, + ) diff --git a/datastore/legacy/schemas.py b/datastore/legacy/schemas.py new file mode 100644 index 0000000..56ad930 --- /dev/null +++ b/datastore/legacy/schemas.py @@ -0,0 +1,73 @@ +"""Pydantic schemas used ONLY by the legacy CKAN surface. + +DEPRECATED — see `datastore/legacy/__init__.py`. + +Lives here (not `datastore/schemas/request.py`) so the v2 request +schemas don't carry back-compat subclasses after legacy is removed. +""" + +from __future__ import annotations + +from typing import ClassVar + +from pydantic import PrivateAttr, model_validator + +from datastore.schemas.request import DatastoreSearchSQLRequest + + +class DatastoreLegacySearchSQLRequest(DatastoreSearchSQLRequest): + """Legacy-API variant of `datastore_search_sql`. + + Real CKAN's `datastore_search_sql` accepts SQL without a LIMIT clause — + customer scripts in the wild rely on that. Flip `_REQUIRE_LIMIT` off + so a bare `SELECT * FROM ""` parses; every other guard (SELECT- + only, single statement, function allow-list, per-table authorize) is + inherited unchanged. + """ + + _REQUIRE_LIMIT: ClassVar[bool] = False + + _has_select_star: bool = PrivateAttr(default=False) + + @property + def has_select_star(self) -> bool: + """True when the top-level projection contains `*` or `t.*`. + + CKAN emits the synthesised `_full_text` column only when the + caller's SELECT list expands to the whole row (via `*`); the + legacy writer gates its `_full_text` injection on this flag. + """ + return self._has_select_star + + @model_validator(mode="after") + def _detect_select_star(self) -> "DatastoreLegacySearchSQLRequest": + """`_full_text` only exists as a column on the underlying + datastore table, so emit it only when the outer `SELECT *` + is reading from a real table — not from a CTE / subquery / + join where the inner projection may have dropped it. + """ + import sqlglot + from sqlglot import expressions as exp + + try: + tree = sqlglot.parse_one(self.sql, dialect="postgres") + except Exception: + return self + select = tree.find(exp.Select) + if select is None: + return self + has_star = any( + isinstance(expr, exp.Star) or isinstance(expr.this, exp.Star) + for expr in select.expressions + ) + from_clause = select.args.get("from_") + source = from_clause.this if from_clause else None + cte_aliases = {cte.alias_or_name for cte in tree.find_all(exp.CTE)} + reads_real_table = ( + isinstance(source, exp.Table) + and source.name + and source.name not in cte_aliases + and not select.args.get("joins") + ) + self._has_select_star = has_star and reads_real_table + return self diff --git a/datastore/legacy/streaming.py b/datastore/legacy/streaming.py new file mode 100644 index 0000000..08a0f8f --- /dev/null +++ b/datastore/legacy/streaming.py @@ -0,0 +1,655 @@ +"""Streaming response writers for the legacy CKAN datastore API. + +DEPRECATED — see `datastore/legacy/__init__.py` for the removal plan. + +A sibling of `datastore/services/streaming.py` (the v2 writers), but +kept in the legacy package so the whole surface deletes together. +Same chunked-bytes discipline (peak memory ≈ 1 row) and the same +reuse of `_json_default` from `services/streaming.py` for Decimal / +bytes handling. The envelope differs — the one real CKAN emits — and +engine-managed system columns (`_id`, `_updated_at`) are stripped so +legacy callers see exactly the shape they see today against the +public CKAN API. + +Deltas vs `datastore/services/streaming.py`: + + - No `schema` block. Legacy CKAN only knows the `fields` list. + - All four `records_format` values are honored (`objects` / + `lists` / `csv` / `tsv`), matching the public CKAN API. + - No `warnings` envelope — legacy clients never read it. + - `fields` entries carry `{"type", "id", "info"?}` — `info` is + passed through when the resource schema set one, so `datastore_search` + and `datastore_search_sql` surface the same per-column metadata + (title, description, unit, example, …) that `datastore_info` + returns. The system columns are filtered out by id. + - Each row tuple is projected to the kept-column indices before + serialisation so `_id` / `_updated_at` never land in `records`. + +The second half of this module (`dump_*`) are file-download writers +rather than envelope writers — they emit raw CSV / TSV / JSON array / +XML bytes rather than the CKAN JSON envelope. CKAN 2.9 serves these +at `/datastore/dump/{id}?format=…`; see the maintainer docs §9. +""" + +from __future__ import annotations + +import csv +import io +import re +from collections.abc import Iterator +from typing import Any +from urllib.parse import parse_qsl, urlencode, urlparse + +import orjson +import snowballstemmer + +from datastore.legacy.constants import ( + LEGACY_DUMP_HIDDEN_COLUMNS, + LEGACY_FULL_TEXT_FIELD, + LEGACY_HIDDEN_COLUMNS, + UTF8_BOM, +) +from datastore.services.streaming import ( + _coerce, + _delimited_row, + _json_default, + _json_string_inner, +) + +# Postgres's `english_stem` dictionary (`tsearch_data/english.stop` in the +# PG source). Used verbatim so our tsvector output matches what a Postgres- +# backed CKAN datastore emits byte-for-byte. Words in this set are dropped +# BUT the token's position in the sequence is still consumed — so later +# tokens keep their original indices, which is why CKAN can emit +# `'restrict':14` even though position 13 (`no`) was removed. +_PG_ENGLISH_STOPWORDS: frozenset[str] = frozenset({ + "i", "me", "my", "myself", "we", "our", "ours", "ourselves", "you", + "your", "yours", "yourself", "yourselves", "he", "him", "his", + "himself", "she", "her", "hers", "herself", "it", "its", "itself", + "they", "them", "their", "theirs", "themselves", "what", "which", + "who", "whom", "this", "that", "these", "those", "am", "is", "are", + "was", "were", "be", "been", "being", "have", "has", "had", "having", + "do", "does", "did", "doing", "a", "an", "the", "and", "but", "if", + "or", "because", "as", "until", "while", "of", "at", "by", "for", + "with", "about", "against", "between", "into", "through", "during", + "before", "after", "above", "below", "to", "from", "up", "down", + "in", "out", "on", "off", "over", "under", "again", "further", + "then", "once", "here", "there", "when", "where", "why", "how", + "all", "any", "both", "each", "few", "more", "most", "other", "some", + "such", "no", "nor", "not", "only", "own", "same", "so", "than", + "too", "very", "s", "t", "can", "will", "just", "don", "should", + "now", +}) + +# One shared stemmer instance (thread-safe for read-only `stemWord` +# calls — snowballstemmer's state is per-call, not per-instance). +_SNOWBALL = snowballstemmer.stemmer("english") + +def build_legacy_pagination_links( + url: str, + *, + limit: int, + offset: int, +) -> dict[str, str]: + """Legacy `_links` dict: path-only URL strings, no page counters. + + `start` is always present; `prev` when `offset > 0`; `next` + whenever `limit > 0` (last page included). + """ + parsed = urlparse(url) + pairs = parse_qsl(parsed.query, keep_blank_values=True) + base_pairs = [(k, v) for k, v in pairs if k != "offset"] + + def _link(pairs: list[tuple[str, str]]) -> str: + qs = urlencode(pairs) + return f"{parsed.path}?{qs}" if qs else parsed.path + + out: dict[str, str] = {"start": _link(base_pairs)} + if offset > 0: + prev_offset = max(0, offset - limit) + out["prev"] = _link([("offset", str(prev_offset))] + base_pairs) + if limit > 0: + out["next"] = _link([("offset", str(offset + limit))] + base_pairs) + return out + + +def stream_legacy_search( + *, + help_url: str, + resource_id: str, + fields: list[dict[str, Any]], + records: Iterator[tuple[Any, ...]], + limit: int, + offset: int, + total: int | None, + include_total: bool, + links: dict[str, Any], + records_format: str = "objects", +) -> Iterator[bytes]: + """CKAN legacy `datastore_search` envelope writer. + + `records_format` picks the shape of the `records` field (default + `objects`). The other three — `lists`, `csv`, `tsv` — match what + the public CKAN API emits. + """ + kept_fields, keep_indices = _project_fields(fields) + columns = [f["id"] for f in kept_fields] + + yield b'{"help":' + yield orjson.dumps(help_url) + yield b',"success":true,"result":{' + if include_total: + yield b'"include_total":true,' + yield b'"resource_id":' + yield orjson.dumps(resource_id) + yield b',"fields":' + yield orjson.dumps(kept_fields) + yield b',"records_format":' + yield orjson.dumps(records_format) + yield b',"records":' + yield from _records_for_format(records_format, columns, records, keep_indices) + yield b',"limit":' + yield orjson.dumps(limit) + yield b',"offset":' + yield orjson.dumps(offset) + yield b',"_links":' + yield orjson.dumps(links) + if include_total and total is not None: + yield b',"total":' + yield orjson.dumps(total) + yield b"}" + yield b"}" + + +def _records_for_format( + records_format: str, + columns: list[str], + records: Iterator[tuple[Any, ...]], + keep_indices: list[int], +) -> Iterator[bytes]: + """Dispatch the `records` field writer by `records_format`. + + - `objects` → `[{col: v, ...}, ...]` + - `lists` → `[[v1, v2, ...], ...]` + - `csv` → JSON string of CSV data rows (no header; column names + on `result.fields`), matching real CKAN + - `tsv` → JSON string of TSV data rows (same) + """ + if records_format == "objects": + yield from _records_object_array(columns, records, keep_indices) + elif records_format == "lists": + yield from _records_array_array(records, keep_indices) + elif records_format == "csv": + yield from _records_delimited_string(records, keep_indices, delimiter=",") + elif records_format == "tsv": + yield from _records_delimited_string(records, keep_indices, delimiter="\t") + else: + yield from _records_object_array(columns, records, keep_indices) + + +def _records_array_array( + records: Iterator[tuple[Any, ...]], + keep_indices: list[int], +) -> Iterator[bytes]: + """`[[v1, v2, ...], ...]` projected to kept columns.""" + yield b"[" + first = True + for row in records: + if first: + first = False + else: + yield b"," + yield orjson.dumps( + [_coerce(row[i]) for i in keep_indices], + default=_json_default, + option=orjson.OPT_PASSTHROUGH_DATETIME, + ) + yield b"]" + + +def _records_delimited_string( + records: Iterator[tuple[Any, ...]], + keep_indices: list[int], + *, + delimiter: str, +) -> Iterator[bytes]: + """One JSON string containing CSV / TSV data rows (no header). + + `_delimited_row` has to see post-serialisation primitives, so + datetime / dict / list / bytes / Decimal cells are coerced via + `_stringify` first — the dump writers do the same. + """ + yield b'"' + for row in records: + projected = [_stringify(row[i]) for i in keep_indices] + yield _json_string_inner(_delimited_row(projected, delimiter=delimiter)) + yield b'"' + + +def stream_legacy_search_sql( + *, + help_url: str, + sql: str, + fields: list[dict[str, Any]], + records: Iterator[tuple[Any, ...]], +) -> Iterator[bytes]: + """CKAN legacy `datastore_search_sql` envelope writer. + + No `_links` (legacy SQL responses don't paginate) and no `limit` / + `offset` echoes — the SQL carries them. `sql` is echoed back so + callers can confirm what ran, matching real CKAN. + + No `records_truncated` key: real CKAN's `datastore_search_sql` + doesn't emit one. + + `_full_text` is treated as a synthesised column: whenever the + caller lists it in `fields` (right after `_id`, matching CKAN's + `SELECT *` output), the writer computes a tsvector-shaped value + per row from the row's other non-null cells. The engine never + materialises it; the field list is the single source of truth. + """ + # `_full_text` is virtual (synthesised per row). Pop it before + # `_project_fields` runs so column→engine-row indices stay aligned, + # then splice it back in with a sentinel index after projection. + real_fields, full_text_at = _pop_full_text(fields) + kept_fields, keep_indices = _project_fields(real_fields) + if full_text_at is not None: + slot = min(full_text_at, len(kept_fields)) + kept_fields = kept_fields[:slot] + [LEGACY_FULL_TEXT_FIELD] + kept_fields[slot:] + keep_indices = keep_indices[:slot] + [-1] + keep_indices[slot:] + columns = [f["id"] for f in kept_fields] + + yield b'{"help":' + yield orjson.dumps(help_url) + yield b',"success":true,"result":{"records":' + if full_text_at is None: + yield from _records_object_array(columns, records, keep_indices) + else: + yield from _records_with_full_text(columns, records, keep_indices, slot) + yield b',"fields":' + yield orjson.dumps(kept_fields) + yield b',"sql":' + yield orjson.dumps(sql.strip()) + yield b"}}" + + +def _pop_full_text( + fields: list[dict[str, Any]], +) -> tuple[list[dict[str, Any]], int | None]: + """Strip the virtual `_full_text` entry from `fields` and return + `(real_fields, original_index)`. Returns `(fields, None)` unchanged + when `_full_text` is absent — the common case for every SQL shape + other than `SELECT *`. + """ + for i, f in enumerate(fields): + if f.get("id") == "_full_text": + return fields[:i] + fields[i + 1:], i + return fields, None + + +def legacy_info_result( + *, + fields: list[dict[str, Any]], + meta: dict[str, Any], +) -> dict[str, Any]: + """CKAN legacy `datastore_info` result body.""" + return { + "meta": {"count": meta.get("total_records", 0)}, + "primary_keys": list(meta.get("primary_key") or []), + "schema": { + f["id"]: f.get("type", "text") + for f in fields + if f["id"] not in ("_id", "_updated_at", "_full_text") + }, + } + + +def _records_with_full_text( + columns: list[str], + records: Iterator[tuple[Any, ...]], + keep_indices: list[int], + full_text_at: int, +) -> Iterator[bytes]: + """`[{col: value, ...}, ...]` with a synthesised `_full_text` per row. + + `keep_indices[full_text_at] == -1` by construction (the writer + splices a sentinel at the slot it got from `_full_text_slot`), so + we substitute the computed value at that position. Every other + position is a real engine-tuple index. + + `_full_text` is formatted as a Postgres tsvector lexeme string + (`'word':pos1,pos2 'word2':pos3 ...`) built from the row's + non-null values. We don't stem (Postgres uses English + morphological stemming via `tsearch`) but we do lowercase, + tokenise on non-alphanumerics, and 1-index positions in column + order — which is close enough for clients that parse the shape. + """ + yield b"[" + first = True + for row in records: + if first: + first = False + else: + yield b"," + projected: list[Any] = [] + for pos, src_idx in enumerate(keep_indices): + if pos == full_text_at: + projected.append(None) # placeholder — overwritten below + else: + projected.append(_coerce(row[src_idx])) + projected[full_text_at] = _compute_full_text( + [v for i, v in enumerate(projected) if i != full_text_at and columns[i] != "_id"] + ) + yield orjson.dumps( + dict(zip(columns, projected)), + default=_json_default, + option=orjson.OPT_PASSTHROUGH_DATETIME, + ) + yield b"]" + + +# Token: an alphanumeric run, or a sign-prefixed numeric run. The +# sign prefix is intentional — Postgres's parser treats `-23` from +# `2024-09-23` as its own token (the leading dash is kept), matching +# the real CKAN sample where `'-09':2 '-23':3` appear as distinct +# lexemes. Trailing punctuation and spaces split; a lone `-` doesn't +# produce an empty token. +_FULL_TEXT_TOKEN_RE = re.compile(r"-?[A-Za-z0-9]+") + + +def _compute_full_text(values: list[Any]) -> str: + """Build a Postgres-tsvector `_full_text` string, matching + `to_tsvector('english', …)` byte-for-byte. + + Pipeline (mirrors Postgres's `english_stem` dictionary): + + 1. Walk values in column order, stringify, tokenise. + 2. Lowercase each token and assign a 1-indexed position in walk + order. **Positions advance for stopwords too** — they're + counted, then dropped, so later tokens keep the index they + would have had if the stopword had stayed. This is why the + real CKAN sample has `'restrict':14` with no 13. + 3. Drop stopwords (Postgres's `english.stop` list). + 4. Stem each surviving token with Snowball English (`restrictions` + → `restrict`, `networks` → `network`). + 5. Group same-stem positions, sort alphabetically by stem, emit + `'stem':pos1,pos2 ...`. + + `_id` is excluded by the caller before invocation. + """ + positions: dict[str, list[int]] = {} + pos = 0 + for v in values: + if v is None: + continue + if isinstance(v, (dict, list)): + text = orjson.dumps(v, default=_json_default).decode("utf-8") + elif isinstance(v, bytes): + text = v.decode("latin-1", errors="replace") + else: + text = str(v) + for match in _FULL_TEXT_TOKEN_RE.finditer(text): + token = match.group(0).lower() + pos += 1 # advance even for stopwords + if token in _PG_ENGLISH_STOPWORDS: + continue + stemmed = _SNOWBALL.stemWord(token) + positions.setdefault(stemmed, []).append(pos) + + return " ".join( + f"'{stem}':{','.join(str(p) for p in positions[stem])}" + for stem in sorted(positions) + ) + + +def _project_fields( + fields: list[dict[str, Any]], + *, + hidden: frozenset[str] = LEGACY_HIDDEN_COLUMNS, +) -> tuple[list[dict[str, Any]], list[int]]: + """Reduce v2 `fields` to the CKAN-legacy shape, remembering which + positions survived so row tuples can be sliced to match. + + Returns `(kept_fields, keep_indices)`. `kept_fields` entries carry + `type`, `id`, and (when present) `info` — same key order the + legacy CKAN API emits. Any other v2 extras are dropped. + `keep_indices` is the list of original tuple positions to retain, + in order. + + `info` is passed through so clients hitting `datastore_search` and + `datastore_search_sql` see the same per-column metadata (title, + description, unit, example, …) that `datastore_info` already + returns. One schema update at the datastore_create / datastore_info + layer now reaches every read endpoint without a second round-trip. + + `hidden` picks the column set to drop. Default + `LEGACY_HIDDEN_COLUMNS` is the action-API set (just `_updated_at`); + dump writers pass `LEGACY_DUMP_HIDDEN_COLUMNS` instead, which also + drops `_id` to match CKAN's download layout. + """ + kept_fields: list[dict[str, Any]] = [] + keep_indices: list[int] = [] + for i, f in enumerate(fields): + fid = f.get("id") + if fid in hidden: + continue + entry: dict[str, Any] = {} + if "type" in f: + entry["type"] = f["type"] + if fid is not None: + entry["id"] = fid + info = f.get("info") + if isinstance(info, dict) and info: + entry["info"] = info + kept_fields.append(entry) + keep_indices.append(i) + return kept_fields, keep_indices + + +def _records_object_array( + columns: list[str], + records: Iterator[tuple[Any, ...]], + keep_indices: list[int], +) -> Iterator[bytes]: + """`[{col: value, ...}, ...]` with each row projected to kept columns. + + `keep_indices` is applied inline so the hidden columns never reach + `orjson.dumps`. Uses a per-row generator-expression tuple slice + (tuples don't take an iterable of indices directly) which stays + O(kept_columns) per row. + """ + yield b"[" + first = True + for row in records: + if first: + first = False + else: + yield b"," + projected = tuple(_coerce(row[i]) for i in keep_indices) + yield orjson.dumps( + dict(zip(columns, projected)), + default=_json_default, + option=orjson.OPT_PASSTHROUGH_DATETIME, + ) + yield b"]" + + +# --------------------------------------------------------------------------- +# File-download writers for `/datastore/dump/{id}?format=…` +# --------------------------------------------------------------------------- +# +# One writer per CKAN-legacy format. All four yield raw bytes a chunk at +# a time so a 10M-row dump never leaves peak memory above ~1 row + any +# per-format buffer (constant-size `StringIO` for csv/tsv). They never +# emit the CKAN JSON envelope — these are file downloads, not actions. +# +# Shared contract: +# fields — v2 legacy-shape list `[{"id", "type", "info"?}, ...]` +# in engine column order. Projected to the kept subset +# via `_project_fields` first, so hidden columns never +# reach the writer. +# records — lazy tuple iterator, engine column order. +# bom (csv/tsv/json/xml) — prepend UTF-8 BOM to the first chunk. CKAN's +# `?bom=true` for Excel; applied uniformly across +# formats because we don't second-guess the caller. + + +def dump_csv( + *, + fields: list[dict[str, Any]], + records: Iterator[tuple[Any, ...]], + bom: bool = False, +) -> Iterator[bytes]: + """Stream a CSV file: header row + data rows, RFC 4180 quoting.""" + yield from _dump_delimited(fields=fields, records=records, delimiter=",", bom=bom) + + +def dump_tsv( + *, + fields: list[dict[str, Any]], + records: Iterator[tuple[Any, ...]], + bom: bool = False, +) -> Iterator[bytes]: + """Stream a TSV file: header row + data rows, tab-delimited.""" + yield from _dump_delimited(fields=fields, records=records, delimiter="\t", bom=bom) + + +def dump_json( + *, + fields: list[dict[str, Any]], + records: Iterator[tuple[Any, ...]], + bom: bool = False, +) -> Iterator[bytes]: + """Stream a JSON array of `{col: value, ...}` objects. + + Not NDJSON — CKAN's dump format is a single JSON array for + compatibility with pandas / jq / standard JSON parsers. The + streaming layout keeps the per-row cost identical to NDJSON. + """ + kept_fields, keep_indices = _project_fields(fields, hidden=LEGACY_DUMP_HIDDEN_COLUMNS) + columns = [f["id"] for f in kept_fields] + if bom: + yield UTF8_BOM + yield from _records_object_array(columns, records, keep_indices) + + +def dump_xml( + *, + fields: list[dict[str, Any]], + records: Iterator[tuple[Any, ...]], + bom: bool = False, +) -> Iterator[bytes]: + """Stream `val……`. + + Matches CKAN's ``-root / per-row `` layout. Column values + are rendered as text nodes; `None` becomes an empty element so the + column still appears (so a consumer walking the XML by tag always + finds every column). The declaration carries `encoding="UTF-8"`. + """ + kept_fields, keep_indices = _project_fields(fields, hidden=LEGACY_DUMP_HIDDEN_COLUMNS) + columns = [f["id"] for f in kept_fields] + + if bom: + yield UTF8_BOM + yield b'\n' + yield b"\n" + for row in records: + projected = tuple(row[i] for i in keep_indices) + yield b" " + for col, val in zip(columns, projected): + tag = _xml_safe_tag(col) + if val is None: + yield f"<{tag}/>".encode("utf-8") + else: + yield f"<{tag}>{_xml_escape(_stringify(val))}".encode("utf-8") + yield b"\n" + yield b"\n" + + +# --- helpers for the dump writers ----------------------------------------- + + +def _dump_delimited( + *, + fields: list[dict[str, Any]], + records: Iterator[tuple[Any, ...]], + delimiter: str, + bom: bool, +) -> Iterator[bytes]: + """Shared implementation for csv + tsv. + + Streams via a reused `StringIO` + `csv.writer` so quoting + escaping + follow RFC 4180; the buffer is reset each row so memory stays O(1). + Non-primitive values (dicts, lists, bytes, Decimal) are stringified + through `_stringify` first — csv has no concept of a nested object. + """ + kept_fields, keep_indices = _project_fields(fields, hidden=LEGACY_DUMP_HIDDEN_COLUMNS) + columns = [f["id"] for f in kept_fields] + + buf = io.StringIO() + writer = csv.writer( + buf, delimiter=delimiter, quoting=csv.QUOTE_MINIMAL, lineterminator="\r\n" + ) + + def _emit_current_row() -> bytes: + payload = buf.getvalue().encode("utf-8") + buf.seek(0) + buf.truncate(0) + return payload + + if bom: + yield UTF8_BOM + + writer.writerow(columns) + yield _emit_current_row() + + for row in records: + projected = [_stringify(row[i]) for i in keep_indices] + writer.writerow(projected) + yield _emit_current_row() + + +def _stringify(value: Any) -> str: + """Coerce a scalar / Decimal / dict / list / bytes to a text form + safe for a delimited cell or an XML text node. + + - `None` → `""` (csv has no `null`; XML path checks `None` separately). + - dict / list → compact JSON (what CKAN does for its CSV dumps too). + - bytes → latin-1 decode (lossless, bytewise); same intent as CKAN's + fallback for `BYTEA` columns. + - Decimal / numbers / bool → `str(value)` (bool as `True`/`False` + matches CKAN's Python repr). + - datetime / date / time → their ISO strings via `str()` are the + shape BigQuery already hands us for text columns. + """ + if value is None: + return "" + if isinstance(value, (dict, list)): + return orjson.dumps(value, default=_json_default).decode("utf-8") + if isinstance(value, bytes): + return value.decode("latin-1", errors="replace") + return str(_coerce(value)) + + +# XML tag names must start with a letter / underscore and contain only +# letters, digits, hyphens, underscores and periods — anything else is +# rewritten to `_` so the output is always well-formed. Column ids go +# through `check_field_name` upstream (letters/digits/underscores + +# spaces), so the only character we actively rewrite is the space. +_XML_TAG_SAFE = str.maketrans({" ": "_"}) + + +def _xml_safe_tag(name: str) -> str: + """Turn a legacy field id into a valid XML element name.""" + safe = name.translate(_XML_TAG_SAFE) + if not safe or not (safe[0].isalpha() or safe[0] == "_"): + safe = "_" + safe + return safe + + +# Minimal XML text-node escapes — ampersand first so later escapes aren't +# double-encoded, then the two angle brackets. We do NOT escape quotes +# because text nodes don't need it (XML only requires `&` and `<`; `>` is +# escaped by convention to avoid the `]]>` CDATA-end sequence). +def _xml_escape(text: str) -> str: + return text.replace("&", "&").replace("<", "<").replace(">", ">") diff --git a/datastore/main.py b/datastore/main.py index 1bf54f2..c769ce0 100644 --- a/datastore/main.py +++ b/datastore/main.py @@ -33,6 +33,9 @@ warmup_engines, ) +# DEPRECATED legacy surface — single import so removal is one line. +from datastore.legacy import log_startup_banner as _log_legacy_banner + logging.basicConfig( level=get_config().LOG_LEVEL, format="%(asctime)s %(levelname)s %(name)s %(message)s", @@ -81,6 +84,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: "redis" if config.REDIS_URL else "memory", "on" if config.ANALYTICS_ENABLED else "off", ) + _log_legacy_banner() yield diff --git a/datastore/schemas/request.py b/datastore/schemas/request.py index db6ba3c..99e9784 100644 --- a/datastore/schemas/request.py +++ b/datastore/schemas/request.py @@ -21,6 +21,7 @@ fields_to_frictionless_schema, parse_sql_pagination, parse_sql_references, + to_csv_list, to_json_object, to_str_or_json_object, validate_frictionless_schema, @@ -50,9 +51,33 @@ class DatastoreCreateRequest(BaseModel): "resource_id": "balancing_auction_results_2025", "schema": { "fields": [ - {"name": "auction_id", "type": "integer"}, - {"name": "product_code", "type": "string"}, - {"name": "clearing_price_gbp_per_mwh", "type": "number"}, + { + "name": "auction_id", + "type": "integer", + "title": "Auction ID", + "description": "Unique auction identifier. Stable across all products auctioned in the same market window.", # noqa: E501 + "comment": "Primary-key component; pairs with product_code to uniquely reference a cleared bid.", # noqa: E501 + "example": "144", + "unit": "N/A", + }, + { + "name": "product_code", + "type": "string", + "title": "Product Code", + "description": "Product mnemonic for the balancing service (e.g. DCL, DCH, FFR).", # noqa: E501 + "comment": "Primary-key component; controlled vocabulary maintained by the market operator.", # noqa: E501 + "example": "DCL", + "unit": "N/A", + }, + { + "name": "clearing_price_gbp_per_mwh", + "type": "number", + "title": "Clearing Price", + "description": "Pay-as-cleared price for the auction. Negative values are possible during oversupply.", # noqa: E501 + "comment": "Pay-as-cleared: every accepted bid settles at this price, not its own offer.", # noqa: E501 + "example": "47.82", + "unit": "GBP/MWh", + }, ], "primaryKey": ["auction_id", "product_code"], }, @@ -60,7 +85,7 @@ class DatastoreCreateRequest(BaseModel): { "auction_id": 144, "product_code": "DCL", - "clearing_price_gbp_per_mwh": 47.82, + "clearing_price_gbp_per_mwh": 48.05, } ], } @@ -268,7 +293,11 @@ class DatastoreSearchRequest(BaseModel): offset: int = Field(default=0, ge=0, description="Rows to skip — pagination offset.") fields: str | None = Field( default=None, - description="Comma-separated columns to project. Default: all columns.", + description=( + "Columns to project. Accepts a comma-separated string " + "(`a,b,c`) or a JSON array of strings (`[\"a\",\"b\"]`). " + "Default: all columns." + ), examples=["auction_id,product_code,clearing_price_gbp_per_mwh"], ) sort: str | None = Field( @@ -299,6 +328,13 @@ def _check_q(cls, v: str | None) -> str | None: to_str_or_json_object(v) # raises if it looks like JSON but isn't return v + @field_validator("fields") + @classmethod + def _check_fields(cls, v: str | None) -> str | None: + if v: + to_csv_list(v) # raises if JSON-array form is malformed + return v + # Strip leading SQL comments (line `-- ...` and block `/* ... */`) before # checking the first keyword. The check is coarse — real safety comes from diff --git a/datastore/schemas/validators.py b/datastore/schemas/validators.py index 4855595..ea79f70 100644 --- a/datastore/schemas/validators.py +++ b/datastore/schemas/validators.py @@ -115,12 +115,22 @@ def to_str_or_json_object(value: Any) -> str | dict[str, Any] | None: def to_csv_list(value: Any) -> list[str] | None: - """Coerce a comma-separated string to a list; pass-through lists.""" + """Coerce a comma-separated string, JSON-array string, or list to a list of names. + """ if value is None or value == "": return None if isinstance(value, list): return [str(v) for v in value] if isinstance(value, str): + stripped = value.strip() + if stripped.startswith("["): + try: + parsed = json.loads(stripped) + except json.JSONDecodeError as e: + raise ValueError(f"invalid JSON array: {e}") from e + if not isinstance(parsed, list): + raise ValueError("must be a JSON array of strings") + return [str(v) for v in parsed] parts = [token.strip() for token in value.split(",")] return [p for p in parts if p] raise ValueError("must be a comma-separated string or list of strings") diff --git a/datastore/services/read.py b/datastore/services/read.py index 06e92f0..1e5db7c 100644 --- a/datastore/services/read.py +++ b/datastore/services/read.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse +from datastore.core.constants import SYSTEM_FIELD_INFO from datastore.core.exceptions import ValidationError from datastore.infrastructure.engines import get_datastore_engine from datastore.infrastructure.engines.registry import get_allowed_sql_functions @@ -86,12 +87,13 @@ async def search_datastore( include_total=data_dict["include_total"], ) - fields, _ = frictionless_schema_to_fields(result.schema) + schema = enrich_schema(result.schema, data_dict.get("resource")) + fields, _ = frictionless_schema_to_fields(schema) envelope_kwargs = dict( help_url=request_url, resource_id=data_dict["resource_id"], - schema=result.schema, + schema=schema, fields=fields, records=result.records, limit=data_dict["limit"], @@ -151,11 +153,13 @@ async def search_sql_datastore( sql=data_dict["sql"], limit=limit, ) - fields, _ = frictionless_schema_to_fields(result.schema) + schema = enrich_schema(result.schema, data_dict.get("resources")) + fields, _ = frictionless_schema_to_fields(schema) + return stream_objects( help_url=request_url, resource_id="", - schema=result.schema, + schema=schema, fields=fields, records=result.records, limit=limit, @@ -202,6 +206,39 @@ async def dump_sql_datastore(context: RequestContext, data_dict: dict[str, Any]) ) +def enrich_schema( + table_schema: dict[str, Any], + resources: dict[str, Any] | list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Overlay publisher + system-column metadata onto the engine schema. + Publisher wins on everything except `type`/`format`. `_id` and + `_updated_at` get service-provided docs when the publisher hasn't + described them. On name collisions across multiple resources, + first-match wins (FROM order). + """ + if isinstance(resources, dict): + resources = [resources] + sources: dict[str, dict[str, Any]] = {} + for resource in resources or (): + for src in (resource.get("schema") or {}).get("fields") or (): + name = src.get("name") + if name: + sources.setdefault(name, src) + for name, src in SYSTEM_FIELD_INFO.items(): + sources.setdefault(name, src) + + merged_fields: list[dict[str, Any]] = [] + for field in table_schema.get("fields") or (): + src = sources.get(field.get("name")) + if src is None: + merged_fields.append(field) + continue + merged_fields.append( + {**src, **{k: field[k] for k in ("type", "format") if k in field}} + ) + return {**table_schema, "fields": merged_fields} + + def _ensure_allowed_sql_functions( context: RequestContext, function_names: list[str], diff --git a/datastore/services/streaming.py b/datastore/services/streaming.py index 267706f..cba1cf6 100644 --- a/datastore/services/streaming.py +++ b/datastore/services/streaming.py @@ -39,6 +39,7 @@ import io import zipfile from collections.abc import AsyncIterator, Iterator +from datetime import date, datetime, time, timezone from decimal import Decimal from typing import Any @@ -46,7 +47,8 @@ def _json_default(obj: Any) -> Any: - """Serialise types `orjson` refuses out of the box. + """Serialise types `orjson` refuses out of the box, plus `datetime` + / `date` so our shape matches `datastore_search`'s engine-side cast. BigQuery `NUMERIC` / `BIGNUMERIC` columns come back as `decimal.Decimal`. Emit them as JSON numbers so clients can do @@ -57,11 +59,37 @@ def _json_default(obj: Any) -> Any: `bytes` (BigQuery `BYTES` columns) are base64-encoded so the response stays UTF-8 and round-trippable. + + `datetime` / `date` cells get the same offset-stripped ISO 8601 + shape `datastore_search` emits. The search path wraps TIMESTAMP / + DATETIME columns in `FORMAT_TIMESTAMP(..., 'UTC')` / + `FORMAT_DATETIME(...)` via `format_select_column` in + `bigquery/lib.py`, so cells arrive pre-stringified there and this + branch is a no-op. The `datastore_search_sql` path runs raw user + SQL — no engine wrap, so cells arrive as live `datetime` objects. + `orjson.OPT_PASSTHROUGH_DATETIME` (set on the row-level dumps + below) routes every `datetime` here instead of letting orjson emit + its native tz-aware `+00:00` suffix, giving the two endpoints the + same `YYYY-MM-DDTHH:MM:SS` wire shape. """ if isinstance(obj, Decimal): - return float(obj) + f = float(obj) + if f.is_integer() and abs(f) < 2**63: + return int(f) + return f if isinstance(obj, bytes): return base64.b64encode(obj).decode("ascii") + if isinstance(obj, datetime): + # Normalise aware datetimes to UTC first, then drop tzinfo so + # the serialised form has no offset — same contract + # `format_select_column` emits. + if obj.tzinfo is not None: + obj = obj.astimezone(timezone.utc).replace(tzinfo=None) + return obj.isoformat() + if isinstance(obj, date): + return obj.isoformat() + if isinstance(obj, time): + return obj.isoformat() raise TypeError( f"orjson cannot serialise {type(obj).__name__}; " "extend `_json_default` if a new BigQuery type comes through." @@ -264,6 +292,22 @@ def _stream_envelope( yield b"}" # close envelope +# Route every `date` / `datetime` cell through `_json_default` instead +# of letting orjson emit its native tz-aware `+00:00` suffix — see the +# `_json_default` docstring for the full rationale. No-op on +# `datastore_search` (where `format_select_column` pre-stringifies at +# the engine); the fix lands on `datastore_search_sql` where no engine +# cast exists. +_ROW_OPTS = orjson.OPT_PASSTHROUGH_DATETIME + + +def _coerce(v: Any) -> Any: + """Integer-valued floats serialise without `.0` to match CKAN wire shape.""" + if isinstance(v, float) and v.is_integer() and abs(v) < 2**63: + return int(v) + return v + + def _records_object_array(columns: list[str], records: Iterator[tuple]) -> Iterator[bytes]: """`[{col: value, ...}, ...]`.""" yield b"[" @@ -273,7 +317,11 @@ def _records_object_array(columns: list[str], records: Iterator[tuple]) -> Itera first = False else: yield b"," - yield orjson.dumps(dict(zip(columns, row)), default=_json_default) + yield orjson.dumps( + {c: _coerce(v) for c, v in zip(columns, row)}, + default=_json_default, + option=_ROW_OPTS, + ) yield b"]" @@ -286,7 +334,7 @@ def _records_array_array(records: Iterator[tuple]) -> Iterator[bytes]: first = False else: yield b"," - yield orjson.dumps(list(row), default=_json_default) + yield orjson.dumps([_coerce(v) for v in row], default=_json_default, option=_ROW_OPTS) yield b"]" diff --git a/example_payload/datastore_create/with_resource.json b/example_payload/datastore_create/with_resource.json index d92af4d..1e1a5d6 100644 --- a/example_payload/datastore_create/with_resource.json +++ b/example_payload/datastore_create/with_resource.json @@ -14,6 +14,7 @@ "title": "Auction ID", "description": "Unique auction identifier. Stable across all products auctioned in the same market window.", "comment": "Primary-key component; pairs with product_code to uniquely reference a cleared bid.", + "example": "144", "unit": null } }, @@ -24,6 +25,7 @@ "title": "Product Code", "description": "Product mnemonic for the balancing service (e.g. DCL, DCH, FFR).", "comment": "Primary-key component; controlled vocabulary maintained by the market operator.", + "example": "DCL", "unit": null } }, @@ -34,6 +36,7 @@ "title": "Delivery Start (UTC)", "description": "First instant of the delivery window. Stored as UTC; clients render local time.", "comment": "Always persisted as UTC \u2014 do not assume the caller's timezone.", + "example": "2025-11-04T16:00:00Z", "unit": null } }, @@ -44,6 +47,7 @@ "title": "Delivery Duration", "description": "Length of the delivery window.", "comment": "Add to delivery_start to derive the delivery end instant.", + "example": "30", "unit": "minutes" } }, @@ -54,6 +58,7 @@ "title": "Clearing Price", "description": "Pay-as-cleared price for the auction. Negative values are possible during oversupply.", "comment": "Pay-as-cleared: every accepted bid settles at this price, not its own offer.", + "example": "47.82", "unit": "GBP/MWh" } }, @@ -64,6 +69,7 @@ "title": "Cleared Volume", "description": "Total volume cleared in this auction.", "comment": "Sum across accepted bids for the (auction_id, product_code) pair.", + "example": "120.0", "unit": "MWh" } }, @@ -74,6 +80,7 @@ "title": "Accepted", "description": "Whether the bid cleared (true) or was rejected (false).", "comment": "Rejected rows are retained for audit; filter accepted=true for settlement.", + "example": "true", "unit": null } }, @@ -84,6 +91,7 @@ "title": "Bidder Metadata", "description": "Free-form provider-specific metadata captured at submission time.", "comment": "Schema not enforced; kept opaque for downstream analytics.", + "example": "{\"unit_id\": \"DRAX-1\", \"submission_lag_ms\": 412}", "unit": null } } diff --git a/example_payload/datastore_create/with_resource_id.json b/example_payload/datastore_create/with_resource_id.json index b337cb4..b6ad770 100644 --- a/example_payload/datastore_create/with_resource_id.json +++ b/example_payload/datastore_create/with_resource_id.json @@ -8,6 +8,7 @@ "title": "Auction ID", "description": "Unique auction identifier. Stable across all products auctioned in the same market window.", "comment": "Primary-key component; pairs with product_code to uniquely reference a cleared bid.", + "example": "144", "unit": null } }, @@ -18,6 +19,7 @@ "title": "Product Code", "description": "Product mnemonic for the balancing service (e.g. DCL, DCH, FFR).", "comment": "Primary-key component; controlled vocabulary maintained by the market operator.", + "example": "DCL", "unit": null } }, @@ -28,6 +30,7 @@ "title": "Delivery Start (UTC)", "description": "First instant of the delivery window. Stored as UTC; clients render local time.", "comment": "Always persisted as UTC \u2014 do not assume the caller's timezone.", + "example": "2025-11-04T16:00:00Z", "unit": null } }, @@ -38,6 +41,7 @@ "title": "Delivery Duration", "description": "Length of the delivery window.", "comment": "Add to delivery_start to derive the delivery end instant.", + "example": "30", "unit": "minutes" } }, @@ -48,6 +52,7 @@ "title": "Clearing Price", "description": "Pay-as-cleared price for the auction. Negative values are possible during oversupply.", "comment": "Pay-as-cleared: every accepted bid settles at this price, not its own offer.", + "example": "47.82", "unit": "GBP/MWh" } }, @@ -58,6 +63,7 @@ "title": "Cleared Volume", "description": "Total volume cleared in this auction.", "comment": "Sum across accepted bids for the (auction_id, product_code) pair.", + "example": "120.0", "unit": "MWh" } }, @@ -68,6 +74,7 @@ "title": "Accepted", "description": "Whether the bid cleared (true) or was rejected (false).", "comment": "Rejected rows are retained for audit; filter accepted=true for settlement.", + "example": "true", "unit": null } }, @@ -78,6 +85,7 @@ "title": "Bidder Metadata", "description": "Free-form provider-specific metadata captured at submission time.", "comment": "Schema not enforced; kept opaque for downstream analytics.", + "example": "{\"unit_id\": \"DRAX-1\", \"submission_lag_ms\": 412}", "unit": null } } diff --git a/example_payload/datastore_create/with_resource_id_response.json b/example_payload/datastore_create/with_resource_id_response.json new file mode 100644 index 0000000..2f07669 --- /dev/null +++ b/example_payload/datastore_create/with_resource_id_response.json @@ -0,0 +1,127 @@ +{ + "help": "http://localhost:8000/datastore/api/v2/docs#/Datastore/datastore_create", + "success": true, + "result": { + "resource_id": "balancing_auction_results_2025", + "fields": [ + { + "id": "_id", + "type": "int8", + "info": { + "title": "Record Identifier", + "description": "System-generated unique identifier assigned to each record in the database. Used to uniquely identify a row within a resource table.", + "comment": "Automatically populated. Not publisher-maintained. Values are unique within a resource and may be used for record retrieval and updates.", + "example": "12345", + "unit": "N/A" + } + }, + { + "id": "_updated_at", + "type": "timestamp", + "info": { + "title": "Last Updated Timestamp", + "description": "System-generated timestamp indicating when the record was last created or modified in the database.", + "comment": "Automatically maintained. Useful for identifying recently changed records. Stored in UTC.", + "example": "2026-08-27T12:45:30", + "unit": "ISO 8601 UTC Timestamp" + } + }, + { + "id": "auction_id", + "type": "int4", + "info": { + "title": "Auction ID", + "description": "Unique auction identifier. Stable across all products auctioned in the same market window.", + "comment": "Primary-key component; pairs with product_code to uniquely reference a cleared bid.", + "example": "144", + "unit": null + } + }, + { + "id": "product_code", + "type": "text", + "info": { + "title": "Product Code", + "description": "Product mnemonic for the balancing service (e.g. DCL, DCH, FFR).", + "comment": "Primary-key component; controlled vocabulary maintained by the market operator.", + "example": "DCL", + "unit": null + } + }, + { + "id": "delivery_start", + "type": "timestamptz", + "info": { + "title": "Delivery Start (UTC)", + "description": "First instant of the delivery window. Stored as UTC; clients render local time.", + "comment": "Always persisted as UTC \u2014 do not assume the caller's timezone.", + "example": "2025-11-04T16:00:00Z", + "unit": null + } + }, + { + "id": "duration_minutes", + "type": "int4", + "info": { + "title": "Delivery Duration", + "description": "Length of the delivery window.", + "comment": "Add to delivery_start to derive the delivery end instant.", + "example": "30", + "unit": "minutes" + } + }, + { + "id": "clearing_price_gbp_per_mwh", + "type": "float8", + "info": { + "title": "Clearing Price", + "description": "Pay-as-cleared price for the auction. Negative values are possible during oversupply.", + "comment": "Pay-as-cleared: every accepted bid settles at this price, not its own offer.", + "example": "47.82", + "unit": "GBP/MWh" + } + }, + { + "id": "volume_mwh", + "type": "float8", + "info": { + "title": "Cleared Volume", + "description": "Total volume cleared in this auction.", + "comment": "Sum across accepted bids for the (auction_id, product_code) pair.", + "example": "120.0", + "unit": "MWh" + } + }, + { + "id": "accepted", + "type": "bool", + "info": { + "title": "Accepted", + "description": "Whether the bid cleared (true) or was rejected (false).", + "comment": "Rejected rows are retained for audit; filter accepted=true for settlement.", + "example": "true", + "unit": null + } + }, + { + "id": "bidder_metadata", + "type": "jsonb", + "info": { + "title": "Bidder Metadata", + "description": "Free-form provider-specific metadata captured at submission time.", + "comment": "Schema not enforced; kept opaque for downstream analytics.", + "example": "{\"unit_id\": \"DRAX-1\", \"submission_lag_ms\": 412}", + "unit": null + } + } + ], + "primary_key": [ + "auction_id", + "product_code" + ], + "unique_key": [ + "auction_id", + "product_code" + ] + } +} diff --git a/example_payload/datastore_create/with_schema.json b/example_payload/datastore_create/with_schema.json index 538455d..23f2097 100644 --- a/example_payload/datastore_create/with_schema.json +++ b/example_payload/datastore_create/with_schema.json @@ -8,6 +8,7 @@ "title": "Auction ID", "description": "Unique auction identifier. Stable across all products auctioned in the same market window.", "comment": "Primary-key component; pairs with product_code to uniquely reference a cleared bid.", + "example": "144", "unit": null }, { @@ -16,6 +17,7 @@ "title": "Product Code", "description": "Product mnemonic for the balancing service (e.g. DCL, DCH, FFR).", "comment": "Primary-key component; controlled vocabulary maintained by the market operator.", + "example": "DCL", "unit": null }, { @@ -24,6 +26,7 @@ "title": "Delivery Start (UTC)", "description": "First instant of the delivery window. Stored as UTC; clients render local time.", "comment": "Always persisted as UTC \u2014 do not assume the caller's timezone.", + "example": "2025-11-04T16:00:00Z", "unit": null }, { @@ -32,6 +35,7 @@ "title": "Delivery Duration", "description": "Length of the delivery window.", "comment": "Add to delivery_start to derive the delivery end instant.", + "example": "30", "unit": "minutes", "info": { "unit": "minutes" @@ -43,6 +47,7 @@ "title": "Clearing Price", "description": "Pay-as-cleared price for the auction. Negative values are possible during oversupply.", "comment": "Pay-as-cleared: every accepted bid settles at this price, not its own offer.", + "example": "47.82", "unit": "GBP/MWh", "info": { "unit": "GBP/MWh" @@ -54,6 +59,7 @@ "title": "Cleared Volume", "description": "Total volume cleared in this auction.", "comment": "Sum across accepted bids for the (auction_id, product_code) pair.", + "example": "120.0", "unit": "MWh", "info": { "unit": "MWh" @@ -65,6 +71,7 @@ "title": "Accepted", "description": "Whether the bid cleared (true) or was rejected (false).", "comment": "Rejected rows are retained for audit; filter accepted=true for settlement.", + "example": "true", "unit": null }, { @@ -73,6 +80,7 @@ "title": "Bidder Metadata", "description": "Free-form provider-specific metadata captured at submission time.", "comment": "Schema not enforced; kept opaque for downstream analytics.", + "example": "{\"unit_id\": \"DRAX-1\", \"submission_lag_ms\": 412}", "unit": null } ], diff --git a/example_payload/datastore_info/basic_response.json b/example_payload/datastore_info/basic_response.json new file mode 100644 index 0000000..0526727 --- /dev/null +++ b/example_payload/datastore_info/basic_response.json @@ -0,0 +1,114 @@ +{ + "help": "http://localhost:8000/datastore/api/v2/docs#/Datastore/datastore_info", + "success": true, + "result": { + "resource_id": "balancing_auction_results_2025", + "fields": [ + { + "id": "_id", + "type": "int8", + "info": { + "title": "Record Identifier", + "description": "System-generated unique identifier assigned to each record in the database. Used to uniquely identify a row within a resource table.", + "comment": "Automatically populated. Not publisher-maintained. Values are unique within a resource and may be used for record retrieval and updates.", + "example": "12345", + "unit": "N/A" + } + }, + { + "id": "_updated_at", + "type": "timestamp", + "info": { + "title": "Last Updated Timestamp", + "description": "System-generated timestamp indicating when the record was last created or modified in the database.", + "comment": "Automatically maintained. Useful for identifying recently changed records. Stored in UTC.", + "example": "2026-08-27T12:45:30", + "unit": "ISO 8601 UTC Timestamp" + } + }, + { + "id": "auction_id", + "type": "int4", + "info": { + "title": "Auction ID", + "description": "Unique auction identifier. Stable across all products auctioned in the same market window.", + "comment": "Primary-key component; pairs with product_code to uniquely reference a cleared bid.", + "example": "144", + "unit": "N/A" + } + }, + { + "id": "product_code", + "type": "text", + "info": { + "title": "Product Code", + "description": "Product mnemonic for the balancing service (e.g. DCL, DCH, FFR).", + "comment": "Primary-key component; controlled vocabulary maintained by the market operator.", + "example": "DCL", + "unit": "N/A" + } + }, + { + "id": "delivery_start", + "type": "timestamptz", + "info": { + "title": "Delivery Start (UTC)", + "description": "First instant of the delivery window. Stored as UTC; clients render local time.", + "example": "2025-11-04T16:00:00Z" + } + }, + { + "id": "duration_minutes", + "type": "int4", + "info": { + "title": "Delivery Duration", + "example": "30", + "unit": "minutes" + } + }, + { + "id": "clearing_price_gbp_per_mwh", + "type": "float8", + "info": { + "title": "Clearing Price", + "example": "47.82", + "unit": "GBP/MWh" + } + }, + { + "id": "volume_mwh", + "type": "float8", + "info": { + "title": "Cleared Volume", + "example": "120.0", + "unit": "MWh" + } + }, + { + "id": "accepted", + "type": "bool", + "info": { + "title": "Accepted", + "example": "true" + } + }, + { + "id": "bidder_metadata", + "type": "jsonb", + "info": { + "title": "Bidder Metadata", + "example": "{\"unit_id\": \"DRAX-1\", \"submission_lag_ms\": 412}" + } + } + ], + "unique_key": [ + "auction_id", + "product_code" + ], + "primary_key": [ + "auction_id", + "product_code" + ], + "total": 110 + } +} diff --git a/example_payload/datastore_search/basic_response.json b/example_payload/datastore_search/basic_response.json new file mode 100644 index 0000000..45d0a8f --- /dev/null +++ b/example_payload/datastore_search/basic_response.json @@ -0,0 +1,82 @@ +{ + "help": "http://localhost:8000/datastore/api/v2/docs#/Datastore/datastore_search", + "success": true, + "result": { + "include_total": true, + "resource_id": "balancing_auction_results_2025", + "fields": [ + { + "id": "_id", + "type": "int8", + "info": { + "title": "Record Identifier", + "description": "System-generated unique identifier assigned to each record in the database. Used to uniquely identify a row within a resource table.", + "comment": "Automatically populated. Not publisher-maintained. Values are unique within a resource and may be used for record retrieval and updates.", + "example": "12345", + "unit": "N/A" + } + }, + { + "id": "_updated_at", + "type": "timestamp", + "info": { + "title": "Last Updated Timestamp", + "description": "System-generated timestamp indicating when the record was last created or modified in the database.", + "comment": "Automatically maintained. Useful for identifying recently changed records. Stored in UTC.", + "example": "2026-08-27T12:45:30", + "unit": "ISO 8601 UTC Timestamp" + } + }, + { + "id": "auction_id", + "type": "int4", + "info": { + "title": "Auction ID", + "example": "144", + "unit": "N/A" + } + }, + { + "id": "product_code", + "type": "text", + "info": { + "title": "Product Code", + "example": "DCL" + } + }, + { + "id": "clearing_price_gbp_per_mwh", + "type": "float8", + "info": { + "title": "Clearing Price", + "example": "47.82", + "unit": "GBP/MWh" + } + } + ], + "records_format": "objects", + "records": [ + { + "_id": 1, + "_updated_at": "2026-08-27T12:45:30", + "auction_id": 1, + "product_code": "DCL", + "clearing_price_gbp_per_mwh": 50.99 + }, + { + "_id": 2, + "_updated_at": "2026-08-27T12:45:30", + "auction_id": 2, + "product_code": "DCL", + "clearing_price_gbp_per_mwh": 45.55 + } + ], + "limit": 100, + "offset": 0, + "total": 110, + "_links": { + "start": "http://localhost:8000/datastore/api/v2/datastore_search?resource_id=balancing_auction_results_2025&limit=100", + "next": "http://localhost:8000/datastore/api/v2/datastore_search?resource_id=balancing_auction_results_2025&limit=100&offset=100" + } + } +} diff --git a/example_payload/datastore_search_sql/basic_response.json b/example_payload/datastore_search_sql/basic_response.json new file mode 100644 index 0000000..48220b2 --- /dev/null +++ b/example_payload/datastore_search_sql/basic_response.json @@ -0,0 +1,49 @@ +{ + "help": "http://localhost:8000/datastore/api/v2/docs#/Datastore/datastore_search_sql", + "success": true, + "result": { + "sql": "SELECT auction_id, product_code, clearing_price_gbp_per_mwh FROM \"balancing_auction_results_2025\" WHERE accepted = true LIMIT 100", + "fields": [ + { + "id": "auction_id", + "type": "int4", + "info": { + "title": "Auction ID", + "example": "144", + "unit": "N/A" + } + }, + { + "id": "product_code", + "type": "text", + "info": { + "title": "Product Code", + "example": "DCL" + } + }, + { + "id": "clearing_price_gbp_per_mwh", + "type": "float8", + "info": { + "title": "Clearing Price", + "example": "47.82", + "unit": "GBP/MWh" + } + } + ], + "records_format": "objects", + "records": [ + { + "auction_id": 2, + "product_code": "DCH", + "clearing_price_gbp_per_mwh": 35.66 + }, + { + "auction_id": 3, + "product_code": "DCL", + "clearing_price_gbp_per_mwh": 40.51 + } + ], + "records_truncated": false + } +} diff --git a/postman/collection.json b/postman/collection.json index b95274d..a71d4ab 100644 --- a/postman/collection.json +++ b/postman/collection.json @@ -1,6 +1,6 @@ { "info": { - "_postman_id": "bfb8a805-c676-4c5b-9832-520972b8d184", + "_postman_id": "d4419942-bb08-4d90-82d9-653aa88e61b5", "name": "Datastore API", "description": "CKAN-compatible datastore API \u2014 auto-generated from `example_payload/`. Set `baseUrl` to your server, `apiKey` to a CKAN API key (anonymous reads are allowed; writes require a key), and `resourceId` to the table you want to hit.", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" @@ -105,7 +105,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"resource_id\": \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\",\n \"schema\": {\n \"fields\": [\n {\n \"name\": \"auction_id\",\n \"type\": \"integer\",\n \"title\": \"Auction ID\",\n \"description\": \"Unique auction identifier. Stable across all products auctioned in the same market window.\",\n \"comment\": \"Primary-key component; pairs with product_code to uniquely reference a cleared bid.\",\n \"unit\": null\n },\n {\n \"name\": \"product_code\",\n \"type\": \"string\",\n \"title\": \"Product Code\",\n \"description\": \"Product mnemonic for the balancing service (e.g. DCL, DCH, FFR).\",\n \"comment\": \"Primary-key component; controlled vocabulary maintained by the market operator.\",\n \"unit\": null\n },\n {\n \"name\": \"delivery_start\",\n \"type\": \"datetime\",\n \"title\": \"Delivery Start (UTC)\",\n \"description\": \"First instant of the delivery window. Stored as UTC; clients render local time.\",\n \"comment\": \"Always persisted as UTC \\u2014 do not assume the caller's timezone.\",\n \"unit\": null\n },\n {\n \"name\": \"duration_minutes\",\n \"type\": \"integer\",\n \"title\": \"Delivery Duration\",\n \"description\": \"Length of the delivery window.\",\n \"comment\": \"Add to delivery_start to derive the delivery end instant.\",\n \"unit\": \"minutes\",\n \"info\": {\n \"unit\": \"minutes\"\n }\n },\n {\n \"name\": \"clearing_price_gbp_per_mwh\",\n \"type\": \"number\",\n \"title\": \"Clearing Price\",\n \"description\": \"Pay-as-cleared price for the auction. Negative values are possible during oversupply.\",\n \"comment\": \"Pay-as-cleared: every accepted bid settles at this price, not its own offer.\",\n \"unit\": \"GBP/MWh\",\n \"info\": {\n \"unit\": \"GBP/MWh\"\n }\n },\n {\n \"name\": \"volume_mwh\",\n \"type\": \"number\",\n \"title\": \"Cleared Volume\",\n \"description\": \"Total volume cleared in this auction.\",\n \"comment\": \"Sum across accepted bids for the (auction_id, product_code) pair.\",\n \"unit\": \"MWh\",\n \"info\": {\n \"unit\": \"MWh\"\n }\n },\n {\n \"name\": \"accepted\",\n \"type\": \"boolean\",\n \"title\": \"Accepted\",\n \"description\": \"Whether the bid cleared (true) or was rejected (false).\",\n \"comment\": \"Rejected rows are retained for audit; filter accepted=true for settlement.\",\n \"unit\": null\n },\n {\n \"name\": \"bidder_metadata\",\n \"type\": \"object\",\n \"title\": \"Bidder Metadata\",\n \"description\": \"Free-form provider-specific metadata captured at submission time.\",\n \"comment\": \"Schema not enforced; kept opaque for downstream analytics.\",\n \"unit\": null\n }\n ],\n \"primaryKey\": [\n \"auction_id\",\n \"product_code\"\n ]\n },\n \"records\": [\n {\n \"auction_id\": 1,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-01T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.99,\n \"volume_mwh\": 52.5,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 171\n }\n },\n {\n \"auction_id\": 1,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-01T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.41,\n \"volume_mwh\": 117.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 402\n }\n },\n {\n \"auction_id\": 2,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-01T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.55,\n \"volume_mwh\": 53.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 408\n }\n },\n {\n \"auction_id\": 2,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-01T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 35.66,\n \"volume_mwh\": 69.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 314\n }\n },\n {\n \"auction_id\": 3,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-01T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 40.51,\n \"volume_mwh\": 108.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 488\n }\n },\n {\n \"auction_id\": 3,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-01T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 55.15,\n \"volume_mwh\": 119.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 210\n }\n },\n {\n \"auction_id\": 4,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-01T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.93,\n \"volume_mwh\": 83.7,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 283\n }\n },\n {\n \"auction_id\": 4,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-01T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.19,\n \"volume_mwh\": 110.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 374\n }\n },\n {\n \"auction_id\": 5,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 38.12,\n \"volume_mwh\": 142.2,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 524\n }\n },\n {\n \"auction_id\": 5,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.72,\n \"volume_mwh\": 138.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 460\n }\n },\n {\n \"auction_id\": 6,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.74,\n \"volume_mwh\": 116.1,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 537\n }\n },\n {\n \"auction_id\": 6,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 40.82,\n \"volume_mwh\": 60.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 183\n }\n },\n {\n \"auction_id\": 7,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 44.25,\n \"volume_mwh\": 71.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 411\n }\n },\n {\n \"auction_id\": 7,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.87,\n \"volume_mwh\": 103.4,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 294\n }\n },\n {\n \"auction_id\": 8,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.75,\n \"volume_mwh\": 142.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 450\n }\n },\n {\n \"auction_id\": 8,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 43.11,\n \"volume_mwh\": 126.8,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 512\n }\n },\n {\n \"auction_id\": 9,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 42.89,\n \"volume_mwh\": 76.8,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 548\n }\n },\n {\n \"auction_id\": 9,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.95,\n \"volume_mwh\": 71.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 173\n }\n },\n {\n \"auction_id\": 10,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.62,\n \"volume_mwh\": 74.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 482\n }\n },\n {\n \"auction_id\": 10,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 49.61,\n \"volume_mwh\": 139.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 170\n }\n },\n {\n \"auction_id\": 11,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.74,\n \"volume_mwh\": 59.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 178\n }\n },\n {\n \"auction_id\": 11,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.69,\n \"volume_mwh\": 129.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 297\n }\n },\n {\n \"auction_id\": 12,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 44.54,\n \"volume_mwh\": 149.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 540\n }\n },\n {\n \"auction_id\": 12,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.56,\n \"volume_mwh\": 118.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 484\n }\n },\n {\n \"auction_id\": 13,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.67,\n \"volume_mwh\": 114.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 180\n }\n },\n {\n \"auction_id\": 13,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 46.34,\n \"volume_mwh\": 145.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 597\n }\n },\n {\n \"auction_id\": 14,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.51,\n \"volume_mwh\": 67.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 530\n }\n },\n {\n \"auction_id\": 14,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.97,\n \"volume_mwh\": 110.9,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 376\n }\n },\n {\n \"auction_id\": 15,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.84,\n \"volume_mwh\": 142.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 265\n }\n },\n {\n \"auction_id\": 15,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.22,\n \"volume_mwh\": 61.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 222\n }\n },\n {\n \"auction_id\": 16,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.45,\n \"volume_mwh\": 137.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 474\n }\n },\n {\n \"auction_id\": 16,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.15,\n \"volume_mwh\": 56.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 165\n }\n },\n {\n \"auction_id\": 17,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.49,\n \"volume_mwh\": 144.7,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 546\n }\n },\n {\n \"auction_id\": 17,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.16,\n \"volume_mwh\": 146.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 465\n }\n },\n {\n \"auction_id\": 18,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 42.79,\n \"volume_mwh\": 149.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 560\n }\n },\n {\n \"auction_id\": 18,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.94,\n \"volume_mwh\": 62.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 110\n }\n },\n {\n \"auction_id\": 19,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 49.71,\n \"volume_mwh\": 73.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 462\n }\n },\n {\n \"auction_id\": 19,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.78,\n \"volume_mwh\": 72.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 136\n }\n },\n {\n \"auction_id\": 20,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.85,\n \"volume_mwh\": 77.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 167\n }\n },\n {\n \"auction_id\": 20,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.08,\n \"volume_mwh\": 138.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 501\n }\n },\n {\n \"auction_id\": 21,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 46.82,\n \"volume_mwh\": 90.7,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 281\n }\n },\n {\n \"auction_id\": 21,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.59,\n \"volume_mwh\": 96.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 131\n }\n },\n {\n \"auction_id\": 22,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.07,\n \"volume_mwh\": 83.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 198\n }\n },\n {\n \"auction_id\": 22,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.76,\n \"volume_mwh\": 94.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 336\n }\n },\n {\n \"auction_id\": 23,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.25,\n \"volume_mwh\": 142.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 150\n }\n },\n {\n \"auction_id\": 23,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.26,\n \"volume_mwh\": 149.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 574\n }\n },\n {\n \"auction_id\": 24,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.84,\n \"volume_mwh\": 73.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 209\n }\n },\n {\n \"auction_id\": 24,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.62,\n \"volume_mwh\": 140.2,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 299\n }\n },\n {\n \"auction_id\": 25,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.63,\n \"volume_mwh\": 128.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 456\n }\n },\n {\n \"auction_id\": 25,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.93,\n \"volume_mwh\": 149.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 179\n }\n },\n {\n \"auction_id\": 26,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.75,\n \"volume_mwh\": 71.8,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 131\n }\n },\n {\n \"auction_id\": 26,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.7,\n \"volume_mwh\": 55.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 570\n }\n },\n {\n \"auction_id\": 27,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.32,\n \"volume_mwh\": 65.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 535\n }\n },\n {\n \"auction_id\": 27,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.65,\n \"volume_mwh\": 109.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 306\n }\n },\n {\n \"auction_id\": 28,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 38.0,\n \"volume_mwh\": 139.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 120\n }\n },\n {\n \"auction_id\": 28,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.48,\n \"volume_mwh\": 91.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 261\n }\n },\n {\n \"auction_id\": 29,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.37,\n \"volume_mwh\": 70.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 235\n }\n },\n {\n \"auction_id\": 29,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 44.89,\n \"volume_mwh\": 117.2,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 575\n }\n },\n {\n \"auction_id\": 30,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.8,\n \"volume_mwh\": 57.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 151\n }\n },\n {\n \"auction_id\": 30,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.83,\n \"volume_mwh\": 71.3,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 551\n }\n },\n {\n \"auction_id\": 31,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.72,\n \"volume_mwh\": 74.4,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 526\n }\n },\n {\n \"auction_id\": 31,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 48.58,\n \"volume_mwh\": 80.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 104\n }\n },\n {\n \"auction_id\": 32,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.7,\n \"volume_mwh\": 105.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 580\n }\n },\n {\n \"auction_id\": 32,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.95,\n \"volume_mwh\": 76.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 179\n }\n },\n {\n \"auction_id\": 33,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.81,\n \"volume_mwh\": 110.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 451\n }\n },\n {\n \"auction_id\": 33,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.86,\n \"volume_mwh\": 76.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 147\n }\n },\n {\n \"auction_id\": 34,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.86,\n \"volume_mwh\": 132.9,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 494\n }\n },\n {\n \"auction_id\": 34,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 38.27,\n \"volume_mwh\": 148.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 382\n }\n },\n {\n \"auction_id\": 35,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.64,\n \"volume_mwh\": 106.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 379\n }\n },\n {\n \"auction_id\": 35,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 35.9,\n \"volume_mwh\": 86.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 165\n }\n },\n {\n \"auction_id\": 36,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.05,\n \"volume_mwh\": 86.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 560\n }\n },\n {\n \"auction_id\": 36,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 43.95,\n \"volume_mwh\": 118.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 499\n }\n },\n {\n \"auction_id\": 37,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 49.0,\n \"volume_mwh\": 137.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 573\n }\n },\n {\n \"auction_id\": 37,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.25,\n \"volume_mwh\": 136.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 551\n }\n },\n {\n \"auction_id\": 38,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.31,\n \"volume_mwh\": 67.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 510\n }\n },\n {\n \"auction_id\": 38,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.75,\n \"volume_mwh\": 123.5,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 503\n }\n },\n {\n \"auction_id\": 39,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.53,\n \"volume_mwh\": 88.3,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 213\n }\n },\n {\n \"auction_id\": 39,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.99,\n \"volume_mwh\": 141.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 214\n }\n },\n {\n \"auction_id\": 40,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 35.59,\n \"volume_mwh\": 69.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 595\n }\n },\n {\n \"auction_id\": 40,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 54.33,\n \"volume_mwh\": 85.1,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 269\n }\n },\n {\n \"auction_id\": 41,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.48,\n \"volume_mwh\": 61.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 397\n }\n },\n {\n \"auction_id\": 41,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 59.06,\n \"volume_mwh\": 76.5,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 276\n }\n },\n {\n \"auction_id\": 42,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.21,\n \"volume_mwh\": 81.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 159\n }\n },\n {\n \"auction_id\": 42,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 44.63,\n \"volume_mwh\": 107.7,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 100\n }\n },\n {\n \"auction_id\": 43,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 48.0,\n \"volume_mwh\": 130.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 286\n }\n },\n {\n \"auction_id\": 43,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.78,\n \"volume_mwh\": 144.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 260\n }\n },\n {\n \"auction_id\": 44,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.58,\n \"volume_mwh\": 62.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 258\n }\n },\n {\n \"auction_id\": 44,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.67,\n \"volume_mwh\": 82.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 165\n }\n },\n {\n \"auction_id\": 45,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.8,\n \"volume_mwh\": 116.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 415\n }\n },\n {\n \"auction_id\": 45,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 49.23,\n \"volume_mwh\": 90.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 246\n }\n },\n {\n \"auction_id\": 46,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 40.25,\n \"volume_mwh\": 128.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 338\n }\n },\n {\n \"auction_id\": 46,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 46.04,\n \"volume_mwh\": 117.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 437\n }\n },\n {\n \"auction_id\": 47,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 37.12,\n \"volume_mwh\": 101.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 147\n }\n },\n {\n \"auction_id\": 47,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 55.46,\n \"volume_mwh\": 125.1,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 512\n }\n },\n {\n \"auction_id\": 48,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.98,\n \"volume_mwh\": 52.4,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 412\n }\n },\n {\n \"auction_id\": 48,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.24,\n \"volume_mwh\": 57.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 199\n }\n },\n {\n \"auction_id\": 49,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.96,\n \"volume_mwh\": 88.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 435\n }\n },\n {\n \"auction_id\": 49,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.19,\n \"volume_mwh\": 139.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 498\n }\n },\n {\n \"auction_id\": 50,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.63,\n \"volume_mwh\": 67.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 337\n }\n },\n {\n \"auction_id\": 50,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.26,\n \"volume_mwh\": 74.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 168\n }\n },\n {\n \"auction_id\": 51,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 55.04,\n \"volume_mwh\": 116.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 262\n }\n },\n {\n \"auction_id\": 51,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.75,\n \"volume_mwh\": 139.1,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 318\n }\n },\n {\n \"auction_id\": 52,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 55.76,\n \"volume_mwh\": 104.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 330\n }\n },\n {\n \"auction_id\": 52,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.48,\n \"volume_mwh\": 74.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 348\n }\n },\n {\n \"auction_id\": 53,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-10T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.67,\n \"volume_mwh\": 77.5,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 220\n }\n },\n {\n \"auction_id\": 53,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-10T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.79,\n \"volume_mwh\": 82.0,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 177\n }\n },\n {\n \"auction_id\": 54,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-10T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 40.78,\n \"volume_mwh\": 119.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 312\n }\n },\n {\n \"auction_id\": 54,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-10T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.19,\n \"volume_mwh\": 104.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 526\n }\n },\n {\n \"auction_id\": 55,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-10T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.5,\n \"volume_mwh\": 140.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 538\n }\n },\n {\n \"auction_id\": 55,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-10T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 57.02,\n \"volume_mwh\": 107.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 252\n }\n }\n ]\n}", + "raw": "{\n \"resource_id\": \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\",\n \"schema\": {\n \"fields\": [\n {\n \"name\": \"auction_id\",\n \"type\": \"integer\",\n \"title\": \"Auction ID\",\n \"description\": \"Unique auction identifier. Stable across all products auctioned in the same market window.\",\n \"comment\": \"Primary-key component; pairs with product_code to uniquely reference a cleared bid.\",\n \"example\": \"144\",\n \"unit\": null\n },\n {\n \"name\": \"product_code\",\n \"type\": \"string\",\n \"title\": \"Product Code\",\n \"description\": \"Product mnemonic for the balancing service (e.g. DCL, DCH, FFR).\",\n \"comment\": \"Primary-key component; controlled vocabulary maintained by the market operator.\",\n \"example\": \"DCL\",\n \"unit\": null\n },\n {\n \"name\": \"delivery_start\",\n \"type\": \"datetime\",\n \"title\": \"Delivery Start (UTC)\",\n \"description\": \"First instant of the delivery window. Stored as UTC; clients render local time.\",\n \"comment\": \"Always persisted as UTC \\u2014 do not assume the caller's timezone.\",\n \"example\": \"2025-11-04T16:00:00Z\",\n \"unit\": null\n },\n {\n \"name\": \"duration_minutes\",\n \"type\": \"integer\",\n \"title\": \"Delivery Duration\",\n \"description\": \"Length of the delivery window.\",\n \"comment\": \"Add to delivery_start to derive the delivery end instant.\",\n \"example\": \"30\",\n \"unit\": \"minutes\",\n \"info\": {\n \"unit\": \"minutes\"\n }\n },\n {\n \"name\": \"clearing_price_gbp_per_mwh\",\n \"type\": \"number\",\n \"title\": \"Clearing Price\",\n \"description\": \"Pay-as-cleared price for the auction. Negative values are possible during oversupply.\",\n \"comment\": \"Pay-as-cleared: every accepted bid settles at this price, not its own offer.\",\n \"example\": \"47.82\",\n \"unit\": \"GBP/MWh\",\n \"info\": {\n \"unit\": \"GBP/MWh\"\n }\n },\n {\n \"name\": \"volume_mwh\",\n \"type\": \"number\",\n \"title\": \"Cleared Volume\",\n \"description\": \"Total volume cleared in this auction.\",\n \"comment\": \"Sum across accepted bids for the (auction_id, product_code) pair.\",\n \"example\": \"120.0\",\n \"unit\": \"MWh\",\n \"info\": {\n \"unit\": \"MWh\"\n }\n },\n {\n \"name\": \"accepted\",\n \"type\": \"boolean\",\n \"title\": \"Accepted\",\n \"description\": \"Whether the bid cleared (true) or was rejected (false).\",\n \"comment\": \"Rejected rows are retained for audit; filter accepted=true for settlement.\",\n \"example\": \"true\",\n \"unit\": null\n },\n {\n \"name\": \"bidder_metadata\",\n \"type\": \"object\",\n \"title\": \"Bidder Metadata\",\n \"description\": \"Free-form provider-specific metadata captured at submission time.\",\n \"comment\": \"Schema not enforced; kept opaque for downstream analytics.\",\n \"example\": \"{\\\"unit_id\\\": \\\"DRAX-1\\\", \\\"submission_lag_ms\\\": 412}\",\n \"unit\": null\n }\n ],\n \"primaryKey\": [\n \"auction_id\",\n \"product_code\"\n ]\n },\n \"records\": [\n {\n \"auction_id\": 1,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-01T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.99,\n \"volume_mwh\": 52.5,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 171\n }\n },\n {\n \"auction_id\": 1,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-01T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.41,\n \"volume_mwh\": 117.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 402\n }\n },\n {\n \"auction_id\": 2,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-01T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.55,\n \"volume_mwh\": 53.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 408\n }\n },\n {\n \"auction_id\": 2,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-01T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 35.66,\n \"volume_mwh\": 69.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 314\n }\n },\n {\n \"auction_id\": 3,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-01T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 40.51,\n \"volume_mwh\": 108.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 488\n }\n },\n {\n \"auction_id\": 3,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-01T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 55.15,\n \"volume_mwh\": 119.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 210\n }\n },\n {\n \"auction_id\": 4,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-01T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.93,\n \"volume_mwh\": 83.7,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 283\n }\n },\n {\n \"auction_id\": 4,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-01T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.19,\n \"volume_mwh\": 110.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 374\n }\n },\n {\n \"auction_id\": 5,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 38.12,\n \"volume_mwh\": 142.2,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 524\n }\n },\n {\n \"auction_id\": 5,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.72,\n \"volume_mwh\": 138.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 460\n }\n },\n {\n \"auction_id\": 6,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.74,\n \"volume_mwh\": 116.1,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 537\n }\n },\n {\n \"auction_id\": 6,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 40.82,\n \"volume_mwh\": 60.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 183\n }\n },\n {\n \"auction_id\": 7,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 44.25,\n \"volume_mwh\": 71.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 411\n }\n },\n {\n \"auction_id\": 7,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.87,\n \"volume_mwh\": 103.4,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 294\n }\n },\n {\n \"auction_id\": 8,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.75,\n \"volume_mwh\": 142.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 450\n }\n },\n {\n \"auction_id\": 8,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 43.11,\n \"volume_mwh\": 126.8,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 512\n }\n },\n {\n \"auction_id\": 9,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 42.89,\n \"volume_mwh\": 76.8,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 548\n }\n },\n {\n \"auction_id\": 9,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.95,\n \"volume_mwh\": 71.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 173\n }\n },\n {\n \"auction_id\": 10,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.62,\n \"volume_mwh\": 74.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 482\n }\n },\n {\n \"auction_id\": 10,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 49.61,\n \"volume_mwh\": 139.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 170\n }\n },\n {\n \"auction_id\": 11,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.74,\n \"volume_mwh\": 59.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 178\n }\n },\n {\n \"auction_id\": 11,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.69,\n \"volume_mwh\": 129.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 297\n }\n },\n {\n \"auction_id\": 12,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 44.54,\n \"volume_mwh\": 149.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 540\n }\n },\n {\n \"auction_id\": 12,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.56,\n \"volume_mwh\": 118.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 484\n }\n },\n {\n \"auction_id\": 13,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.67,\n \"volume_mwh\": 114.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 180\n }\n },\n {\n \"auction_id\": 13,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 46.34,\n \"volume_mwh\": 145.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 597\n }\n },\n {\n \"auction_id\": 14,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.51,\n \"volume_mwh\": 67.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 530\n }\n },\n {\n \"auction_id\": 14,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.97,\n \"volume_mwh\": 110.9,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 376\n }\n },\n {\n \"auction_id\": 15,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.84,\n \"volume_mwh\": 142.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 265\n }\n },\n {\n \"auction_id\": 15,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.22,\n \"volume_mwh\": 61.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 222\n }\n },\n {\n \"auction_id\": 16,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.45,\n \"volume_mwh\": 137.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 474\n }\n },\n {\n \"auction_id\": 16,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.15,\n \"volume_mwh\": 56.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 165\n }\n },\n {\n \"auction_id\": 17,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.49,\n \"volume_mwh\": 144.7,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 546\n }\n },\n {\n \"auction_id\": 17,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.16,\n \"volume_mwh\": 146.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 465\n }\n },\n {\n \"auction_id\": 18,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 42.79,\n \"volume_mwh\": 149.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 560\n }\n },\n {\n \"auction_id\": 18,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.94,\n \"volume_mwh\": 62.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 110\n }\n },\n {\n \"auction_id\": 19,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 49.71,\n \"volume_mwh\": 73.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 462\n }\n },\n {\n \"auction_id\": 19,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.78,\n \"volume_mwh\": 72.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 136\n }\n },\n {\n \"auction_id\": 20,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.85,\n \"volume_mwh\": 77.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 167\n }\n },\n {\n \"auction_id\": 20,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.08,\n \"volume_mwh\": 138.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 501\n }\n },\n {\n \"auction_id\": 21,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 46.82,\n \"volume_mwh\": 90.7,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 281\n }\n },\n {\n \"auction_id\": 21,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.59,\n \"volume_mwh\": 96.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 131\n }\n },\n {\n \"auction_id\": 22,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.07,\n \"volume_mwh\": 83.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 198\n }\n },\n {\n \"auction_id\": 22,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.76,\n \"volume_mwh\": 94.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 336\n }\n },\n {\n \"auction_id\": 23,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.25,\n \"volume_mwh\": 142.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 150\n }\n },\n {\n \"auction_id\": 23,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.26,\n \"volume_mwh\": 149.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 574\n }\n },\n {\n \"auction_id\": 24,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.84,\n \"volume_mwh\": 73.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 209\n }\n },\n {\n \"auction_id\": 24,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.62,\n \"volume_mwh\": 140.2,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 299\n }\n },\n {\n \"auction_id\": 25,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.63,\n \"volume_mwh\": 128.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 456\n }\n },\n {\n \"auction_id\": 25,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.93,\n \"volume_mwh\": 149.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 179\n }\n },\n {\n \"auction_id\": 26,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.75,\n \"volume_mwh\": 71.8,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 131\n }\n },\n {\n \"auction_id\": 26,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.7,\n \"volume_mwh\": 55.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 570\n }\n },\n {\n \"auction_id\": 27,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.32,\n \"volume_mwh\": 65.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 535\n }\n },\n {\n \"auction_id\": 27,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.65,\n \"volume_mwh\": 109.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 306\n }\n },\n {\n \"auction_id\": 28,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 38.0,\n \"volume_mwh\": 139.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 120\n }\n },\n {\n \"auction_id\": 28,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.48,\n \"volume_mwh\": 91.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 261\n }\n },\n {\n \"auction_id\": 29,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.37,\n \"volume_mwh\": 70.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 235\n }\n },\n {\n \"auction_id\": 29,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 44.89,\n \"volume_mwh\": 117.2,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 575\n }\n },\n {\n \"auction_id\": 30,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.8,\n \"volume_mwh\": 57.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 151\n }\n },\n {\n \"auction_id\": 30,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.83,\n \"volume_mwh\": 71.3,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 551\n }\n },\n {\n \"auction_id\": 31,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.72,\n \"volume_mwh\": 74.4,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 526\n }\n },\n {\n \"auction_id\": 31,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 48.58,\n \"volume_mwh\": 80.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 104\n }\n },\n {\n \"auction_id\": 32,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.7,\n \"volume_mwh\": 105.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 580\n }\n },\n {\n \"auction_id\": 32,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.95,\n \"volume_mwh\": 76.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 179\n }\n },\n {\n \"auction_id\": 33,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.81,\n \"volume_mwh\": 110.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 451\n }\n },\n {\n \"auction_id\": 33,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.86,\n \"volume_mwh\": 76.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 147\n }\n },\n {\n \"auction_id\": 34,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.86,\n \"volume_mwh\": 132.9,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 494\n }\n },\n {\n \"auction_id\": 34,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 38.27,\n \"volume_mwh\": 148.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 382\n }\n },\n {\n \"auction_id\": 35,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.64,\n \"volume_mwh\": 106.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 379\n }\n },\n {\n \"auction_id\": 35,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 35.9,\n \"volume_mwh\": 86.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 165\n }\n },\n {\n \"auction_id\": 36,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.05,\n \"volume_mwh\": 86.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 560\n }\n },\n {\n \"auction_id\": 36,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 43.95,\n \"volume_mwh\": 118.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 499\n }\n },\n {\n \"auction_id\": 37,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 49.0,\n \"volume_mwh\": 137.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 573\n }\n },\n {\n \"auction_id\": 37,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.25,\n \"volume_mwh\": 136.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 551\n }\n },\n {\n \"auction_id\": 38,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.31,\n \"volume_mwh\": 67.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 510\n }\n },\n {\n \"auction_id\": 38,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.75,\n \"volume_mwh\": 123.5,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 503\n }\n },\n {\n \"auction_id\": 39,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.53,\n \"volume_mwh\": 88.3,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 213\n }\n },\n {\n \"auction_id\": 39,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.99,\n \"volume_mwh\": 141.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 214\n }\n },\n {\n \"auction_id\": 40,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 35.59,\n \"volume_mwh\": 69.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 595\n }\n },\n {\n \"auction_id\": 40,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 54.33,\n \"volume_mwh\": 85.1,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 269\n }\n },\n {\n \"auction_id\": 41,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.48,\n \"volume_mwh\": 61.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 397\n }\n },\n {\n \"auction_id\": 41,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 59.06,\n \"volume_mwh\": 76.5,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 276\n }\n },\n {\n \"auction_id\": 42,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.21,\n \"volume_mwh\": 81.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 159\n }\n },\n {\n \"auction_id\": 42,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 44.63,\n \"volume_mwh\": 107.7,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 100\n }\n },\n {\n \"auction_id\": 43,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 48.0,\n \"volume_mwh\": 130.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 286\n }\n },\n {\n \"auction_id\": 43,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.78,\n \"volume_mwh\": 144.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 260\n }\n },\n {\n \"auction_id\": 44,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.58,\n \"volume_mwh\": 62.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 258\n }\n },\n {\n \"auction_id\": 44,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.67,\n \"volume_mwh\": 82.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 165\n }\n },\n {\n \"auction_id\": 45,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.8,\n \"volume_mwh\": 116.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 415\n }\n },\n {\n \"auction_id\": 45,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 49.23,\n \"volume_mwh\": 90.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 246\n }\n },\n {\n \"auction_id\": 46,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 40.25,\n \"volume_mwh\": 128.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 338\n }\n },\n {\n \"auction_id\": 46,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 46.04,\n \"volume_mwh\": 117.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 437\n }\n },\n {\n \"auction_id\": 47,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 37.12,\n \"volume_mwh\": 101.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 147\n }\n },\n {\n \"auction_id\": 47,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 55.46,\n \"volume_mwh\": 125.1,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 512\n }\n },\n {\n \"auction_id\": 48,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.98,\n \"volume_mwh\": 52.4,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 412\n }\n },\n {\n \"auction_id\": 48,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.24,\n \"volume_mwh\": 57.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 199\n }\n },\n {\n \"auction_id\": 49,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.96,\n \"volume_mwh\": 88.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 435\n }\n },\n {\n \"auction_id\": 49,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.19,\n \"volume_mwh\": 139.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 498\n }\n },\n {\n \"auction_id\": 50,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.63,\n \"volume_mwh\": 67.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 337\n }\n },\n {\n \"auction_id\": 50,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.26,\n \"volume_mwh\": 74.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 168\n }\n },\n {\n \"auction_id\": 51,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 55.04,\n \"volume_mwh\": 116.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 262\n }\n },\n {\n \"auction_id\": 51,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.75,\n \"volume_mwh\": 139.1,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 318\n }\n },\n {\n \"auction_id\": 52,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 55.76,\n \"volume_mwh\": 104.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 330\n }\n },\n {\n \"auction_id\": 52,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.48,\n \"volume_mwh\": 74.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 348\n }\n },\n {\n \"auction_id\": 53,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-10T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.67,\n \"volume_mwh\": 77.5,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 220\n }\n },\n {\n \"auction_id\": 53,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-10T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.79,\n \"volume_mwh\": 82.0,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 177\n }\n },\n {\n \"auction_id\": 54,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-10T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 40.78,\n \"volume_mwh\": 119.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 312\n }\n },\n {\n \"auction_id\": 54,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-10T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.19,\n \"volume_mwh\": 104.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 526\n }\n },\n {\n \"auction_id\": 55,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-10T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.5,\n \"volume_mwh\": 140.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 538\n }\n },\n {\n \"auction_id\": 55,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-10T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 57.02,\n \"volume_mwh\": 107.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 252\n }\n }\n ]\n}", "options": { "raw": { "language": "json" @@ -140,7 +140,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"resource_id\": \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\",\n \"fields\": [\n {\n \"id\": \"auction_id\",\n \"type\": \"int4\",\n \"info\": {\n \"title\": \"Auction ID\",\n \"description\": \"Unique auction identifier. Stable across all products auctioned in the same market window.\",\n \"comment\": \"Primary-key component; pairs with product_code to uniquely reference a cleared bid.\",\n \"unit\": null\n }\n },\n {\n \"id\": \"product_code\",\n \"type\": \"text\",\n \"info\": {\n \"title\": \"Product Code\",\n \"description\": \"Product mnemonic for the balancing service (e.g. DCL, DCH, FFR).\",\n \"comment\": \"Primary-key component; controlled vocabulary maintained by the market operator.\",\n \"unit\": null\n }\n },\n {\n \"id\": \"delivery_start\",\n \"type\": \"timestamptz\",\n \"info\": {\n \"title\": \"Delivery Start (UTC)\",\n \"description\": \"First instant of the delivery window. Stored as UTC; clients render local time.\",\n \"comment\": \"Always persisted as UTC \\u2014 do not assume the caller's timezone.\",\n \"unit\": null\n }\n },\n {\n \"id\": \"duration_minutes\",\n \"type\": \"int4\",\n \"info\": {\n \"title\": \"Delivery Duration\",\n \"description\": \"Length of the delivery window.\",\n \"comment\": \"Add to delivery_start to derive the delivery end instant.\",\n \"unit\": \"minutes\"\n }\n },\n {\n \"id\": \"clearing_price_gbp_per_mwh\",\n \"type\": \"float8\",\n \"info\": {\n \"title\": \"Clearing Price\",\n \"description\": \"Pay-as-cleared price for the auction. Negative values are possible during oversupply.\",\n \"comment\": \"Pay-as-cleared: every accepted bid settles at this price, not its own offer.\",\n \"unit\": \"GBP/MWh\"\n }\n },\n {\n \"id\": \"volume_mwh\",\n \"type\": \"float8\",\n \"info\": {\n \"title\": \"Cleared Volume\",\n \"description\": \"Total volume cleared in this auction.\",\n \"comment\": \"Sum across accepted bids for the (auction_id, product_code) pair.\",\n \"unit\": \"MWh\"\n }\n },\n {\n \"id\": \"accepted\",\n \"type\": \"bool\",\n \"info\": {\n \"title\": \"Accepted\",\n \"description\": \"Whether the bid cleared (true) or was rejected (false).\",\n \"comment\": \"Rejected rows are retained for audit; filter accepted=true for settlement.\",\n \"unit\": null\n }\n },\n {\n \"id\": \"bidder_metadata\",\n \"type\": \"jsonb\",\n \"info\": {\n \"title\": \"Bidder Metadata\",\n \"description\": \"Free-form provider-specific metadata captured at submission time.\",\n \"comment\": \"Schema not enforced; kept opaque for downstream analytics.\",\n \"unit\": null\n }\n }\n ],\n \"primary_key\": [\n \"auction_id\",\n \"product_code\"\n ],\n \"records\": [\n {\n \"auction_id\": 1,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-01T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.99,\n \"volume_mwh\": 52.5,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 171\n }\n },\n {\n \"auction_id\": 2,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-01T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.55,\n \"volume_mwh\": 53.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 408\n }\n },\n {\n \"auction_id\": 2,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-01T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 35.66,\n \"volume_mwh\": 69.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 314\n }\n },\n {\n \"auction_id\": 3,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-01T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 40.51,\n \"volume_mwh\": 108.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 488\n }\n },\n {\n \"auction_id\": 3,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-01T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 55.15,\n \"volume_mwh\": 119.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 210\n }\n },\n {\n \"auction_id\": 4,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-01T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.93,\n \"volume_mwh\": 83.7,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 283\n }\n },\n {\n \"auction_id\": 4,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-01T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.19,\n \"volume_mwh\": 110.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 374\n }\n },\n {\n \"auction_id\": 5,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 38.12,\n \"volume_mwh\": 142.2,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 524\n }\n },\n {\n \"auction_id\": 5,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.72,\n \"volume_mwh\": 138.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 460\n }\n },\n {\n \"auction_id\": 6,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.74,\n \"volume_mwh\": 116.1,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 537\n }\n },\n {\n \"auction_id\": 6,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 40.82,\n \"volume_mwh\": 60.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 183\n }\n },\n {\n \"auction_id\": 7,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 44.25,\n \"volume_mwh\": 71.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 411\n }\n },\n {\n \"auction_id\": 7,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.87,\n \"volume_mwh\": 103.4,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 294\n }\n },\n {\n \"auction_id\": 8,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.75,\n \"volume_mwh\": 142.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 450\n }\n },\n {\n \"auction_id\": 8,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 43.11,\n \"volume_mwh\": 126.8,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 512\n }\n },\n {\n \"auction_id\": 9,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 42.89,\n \"volume_mwh\": 76.8,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 548\n }\n },\n {\n \"auction_id\": 9,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.95,\n \"volume_mwh\": 71.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 173\n }\n },\n {\n \"auction_id\": 10,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.62,\n \"volume_mwh\": 74.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 482\n }\n },\n {\n \"auction_id\": 10,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 49.61,\n \"volume_mwh\": 139.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 170\n }\n },\n {\n \"auction_id\": 11,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.74,\n \"volume_mwh\": 59.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 178\n }\n },\n {\n \"auction_id\": 11,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.69,\n \"volume_mwh\": 129.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 297\n }\n },\n {\n \"auction_id\": 12,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 44.54,\n \"volume_mwh\": 149.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 540\n }\n },\n {\n \"auction_id\": 12,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.56,\n \"volume_mwh\": 118.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 484\n }\n },\n {\n \"auction_id\": 13,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.67,\n \"volume_mwh\": 114.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 180\n }\n },\n {\n \"auction_id\": 13,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 46.34,\n \"volume_mwh\": 145.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 597\n }\n },\n {\n \"auction_id\": 14,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.51,\n \"volume_mwh\": 67.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 530\n }\n },\n {\n \"auction_id\": 14,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.97,\n \"volume_mwh\": 110.9,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 376\n }\n },\n {\n \"auction_id\": 15,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.84,\n \"volume_mwh\": 142.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 265\n }\n },\n {\n \"auction_id\": 15,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.22,\n \"volume_mwh\": 61.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 222\n }\n },\n {\n \"auction_id\": 16,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.45,\n \"volume_mwh\": 137.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 474\n }\n },\n {\n \"auction_id\": 16,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.15,\n \"volume_mwh\": 56.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 165\n }\n },\n {\n \"auction_id\": 17,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.49,\n \"volume_mwh\": 144.7,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 546\n }\n },\n {\n \"auction_id\": 17,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.16,\n \"volume_mwh\": 146.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 465\n }\n },\n {\n \"auction_id\": 18,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 42.79,\n \"volume_mwh\": 149.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 560\n }\n },\n {\n \"auction_id\": 18,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.94,\n \"volume_mwh\": 62.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 110\n }\n },\n {\n \"auction_id\": 19,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 49.71,\n \"volume_mwh\": 73.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 462\n }\n },\n {\n \"auction_id\": 19,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.78,\n \"volume_mwh\": 72.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 136\n }\n },\n {\n \"auction_id\": 20,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.85,\n \"volume_mwh\": 77.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 167\n }\n },\n {\n \"auction_id\": 20,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.08,\n \"volume_mwh\": 138.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 501\n }\n },\n {\n \"auction_id\": 21,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 46.82,\n \"volume_mwh\": 90.7,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 281\n }\n },\n {\n \"auction_id\": 21,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.59,\n \"volume_mwh\": 96.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 131\n }\n },\n {\n \"auction_id\": 22,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.07,\n \"volume_mwh\": 83.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 198\n }\n },\n {\n \"auction_id\": 22,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.76,\n \"volume_mwh\": 94.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 336\n }\n },\n {\n \"auction_id\": 23,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.25,\n \"volume_mwh\": 142.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 150\n }\n },\n {\n \"auction_id\": 23,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.26,\n \"volume_mwh\": 149.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 574\n }\n },\n {\n \"auction_id\": 24,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.84,\n \"volume_mwh\": 73.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 209\n }\n },\n {\n \"auction_id\": 24,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.62,\n \"volume_mwh\": 140.2,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 299\n }\n },\n {\n \"auction_id\": 25,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.63,\n \"volume_mwh\": 128.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 456\n }\n },\n {\n \"auction_id\": 25,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.93,\n \"volume_mwh\": 149.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 179\n }\n },\n {\n \"auction_id\": 26,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.75,\n \"volume_mwh\": 71.8,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 131\n }\n },\n {\n \"auction_id\": 26,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.7,\n \"volume_mwh\": 55.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 570\n }\n },\n {\n \"auction_id\": 27,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.32,\n \"volume_mwh\": 65.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 535\n }\n },\n {\n \"auction_id\": 27,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.65,\n \"volume_mwh\": 109.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 306\n }\n },\n {\n \"auction_id\": 28,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 38.0,\n \"volume_mwh\": 139.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 120\n }\n },\n {\n \"auction_id\": 28,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.48,\n \"volume_mwh\": 91.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 261\n }\n },\n {\n \"auction_id\": 29,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.37,\n \"volume_mwh\": 70.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 235\n }\n },\n {\n \"auction_id\": 29,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 44.89,\n \"volume_mwh\": 117.2,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 575\n }\n },\n {\n \"auction_id\": 30,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.8,\n \"volume_mwh\": 57.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 151\n }\n },\n {\n \"auction_id\": 30,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.83,\n \"volume_mwh\": 71.3,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 551\n }\n },\n {\n \"auction_id\": 31,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.72,\n \"volume_mwh\": 74.4,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 526\n }\n },\n {\n \"auction_id\": 31,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 48.58,\n \"volume_mwh\": 80.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 104\n }\n },\n {\n \"auction_id\": 32,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.7,\n \"volume_mwh\": 105.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 580\n }\n },\n {\n \"auction_id\": 32,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.95,\n \"volume_mwh\": 76.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 179\n }\n },\n {\n \"auction_id\": 33,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.81,\n \"volume_mwh\": 110.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 451\n }\n },\n {\n \"auction_id\": 33,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.86,\n \"volume_mwh\": 76.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 147\n }\n },\n {\n \"auction_id\": 34,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.86,\n \"volume_mwh\": 132.9,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 494\n }\n },\n {\n \"auction_id\": 34,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 38.27,\n \"volume_mwh\": 148.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 382\n }\n },\n {\n \"auction_id\": 35,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.64,\n \"volume_mwh\": 106.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 379\n }\n },\n {\n \"auction_id\": 35,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 35.9,\n \"volume_mwh\": 86.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 165\n }\n },\n {\n \"auction_id\": 36,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.05,\n \"volume_mwh\": 86.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 560\n }\n },\n {\n \"auction_id\": 36,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 43.95,\n \"volume_mwh\": 118.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 499\n }\n },\n {\n \"auction_id\": 37,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 49.0,\n \"volume_mwh\": 137.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 573\n }\n },\n {\n \"auction_id\": 37,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.25,\n \"volume_mwh\": 136.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 551\n }\n },\n {\n \"auction_id\": 38,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.31,\n \"volume_mwh\": 67.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 510\n }\n },\n {\n \"auction_id\": 38,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.75,\n \"volume_mwh\": 123.5,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 503\n }\n },\n {\n \"auction_id\": 39,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.53,\n \"volume_mwh\": 88.3,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 213\n }\n },\n {\n \"auction_id\": 39,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.99,\n \"volume_mwh\": 141.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 214\n }\n },\n {\n \"auction_id\": 40,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 35.59,\n \"volume_mwh\": 69.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 595\n }\n },\n {\n \"auction_id\": 40,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 54.33,\n \"volume_mwh\": 85.1,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 269\n }\n },\n {\n \"auction_id\": 41,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.48,\n \"volume_mwh\": 61.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 397\n }\n },\n {\n \"auction_id\": 41,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 59.06,\n \"volume_mwh\": 76.5,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 276\n }\n },\n {\n \"auction_id\": 42,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.21,\n \"volume_mwh\": 81.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 159\n }\n },\n {\n \"auction_id\": 42,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 44.63,\n \"volume_mwh\": 107.7,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 100\n }\n },\n {\n \"auction_id\": 43,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 48.0,\n \"volume_mwh\": 130.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 286\n }\n },\n {\n \"auction_id\": 43,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.78,\n \"volume_mwh\": 144.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 260\n }\n },\n {\n \"auction_id\": 44,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.58,\n \"volume_mwh\": 62.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 258\n }\n },\n {\n \"auction_id\": 44,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.67,\n \"volume_mwh\": 82.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 165\n }\n },\n {\n \"auction_id\": 45,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.8,\n \"volume_mwh\": 116.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 415\n }\n },\n {\n \"auction_id\": 45,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 49.23,\n \"volume_mwh\": 90.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 246\n }\n },\n {\n \"auction_id\": 46,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 40.25,\n \"volume_mwh\": 128.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 338\n }\n },\n {\n \"auction_id\": 46,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 46.04,\n \"volume_mwh\": 117.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 437\n }\n },\n {\n \"auction_id\": 47,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 37.12,\n \"volume_mwh\": 101.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 147\n }\n },\n {\n \"auction_id\": 47,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 55.46,\n \"volume_mwh\": 125.1,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 512\n }\n },\n {\n \"auction_id\": 48,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.98,\n \"volume_mwh\": 52.4,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 412\n }\n },\n {\n \"auction_id\": 48,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.24,\n \"volume_mwh\": 57.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 199\n }\n },\n {\n \"auction_id\": 49,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.96,\n \"volume_mwh\": 88.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 435\n }\n },\n {\n \"auction_id\": 49,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.19,\n \"volume_mwh\": 139.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 498\n }\n },\n {\n \"auction_id\": 50,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.63,\n \"volume_mwh\": 67.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 337\n }\n },\n {\n \"auction_id\": 50,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.26,\n \"volume_mwh\": 74.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 168\n }\n },\n {\n \"auction_id\": 51,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 55.04,\n \"volume_mwh\": 116.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 262\n }\n },\n {\n \"auction_id\": 51,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.75,\n \"volume_mwh\": 139.1,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 318\n }\n },\n {\n \"auction_id\": 52,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 55.76,\n \"volume_mwh\": 104.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 330\n }\n },\n {\n \"auction_id\": 52,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.48,\n \"volume_mwh\": 74.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 348\n }\n },\n {\n \"auction_id\": 53,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-10T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.67,\n \"volume_mwh\": 77.5,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 220\n }\n },\n {\n \"auction_id\": 53,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-10T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.79,\n \"volume_mwh\": 82.0,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 177\n }\n },\n {\n \"auction_id\": 54,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-10T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 40.78,\n \"volume_mwh\": 119.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 312\n }\n },\n {\n \"auction_id\": 54,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-10T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.19,\n \"volume_mwh\": 104.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 526\n }\n },\n {\n \"auction_id\": 55,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-10T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.5,\n \"volume_mwh\": 140.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 538\n }\n },\n {\n \"auction_id\": 55,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-10T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 57.02,\n \"volume_mwh\": 107.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 252\n }\n }\n ]\n}", + "raw": "{\n \"resource_id\": \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\",\n \"fields\": [\n {\n \"id\": \"auction_id\",\n \"type\": \"int4\",\n \"info\": {\n \"title\": \"Auction ID\",\n \"description\": \"Unique auction identifier. Stable across all products auctioned in the same market window.\",\n \"comment\": \"Primary-key component; pairs with product_code to uniquely reference a cleared bid.\",\n \"example\": \"144\",\n \"unit\": null\n }\n },\n {\n \"id\": \"product_code\",\n \"type\": \"text\",\n \"info\": {\n \"title\": \"Product Code\",\n \"description\": \"Product mnemonic for the balancing service (e.g. DCL, DCH, FFR).\",\n \"comment\": \"Primary-key component; controlled vocabulary maintained by the market operator.\",\n \"example\": \"DCL\",\n \"unit\": null\n }\n },\n {\n \"id\": \"delivery_start\",\n \"type\": \"timestamptz\",\n \"info\": {\n \"title\": \"Delivery Start (UTC)\",\n \"description\": \"First instant of the delivery window. Stored as UTC; clients render local time.\",\n \"comment\": \"Always persisted as UTC \\u2014 do not assume the caller's timezone.\",\n \"example\": \"2025-11-04T16:00:00Z\",\n \"unit\": null\n }\n },\n {\n \"id\": \"duration_minutes\",\n \"type\": \"int4\",\n \"info\": {\n \"title\": \"Delivery Duration\",\n \"description\": \"Length of the delivery window.\",\n \"comment\": \"Add to delivery_start to derive the delivery end instant.\",\n \"example\": \"30\",\n \"unit\": \"minutes\"\n }\n },\n {\n \"id\": \"clearing_price_gbp_per_mwh\",\n \"type\": \"float8\",\n \"info\": {\n \"title\": \"Clearing Price\",\n \"description\": \"Pay-as-cleared price for the auction. Negative values are possible during oversupply.\",\n \"comment\": \"Pay-as-cleared: every accepted bid settles at this price, not its own offer.\",\n \"example\": \"47.82\",\n \"unit\": \"GBP/MWh\"\n }\n },\n {\n \"id\": \"volume_mwh\",\n \"type\": \"float8\",\n \"info\": {\n \"title\": \"Cleared Volume\",\n \"description\": \"Total volume cleared in this auction.\",\n \"comment\": \"Sum across accepted bids for the (auction_id, product_code) pair.\",\n \"example\": \"120.0\",\n \"unit\": \"MWh\"\n }\n },\n {\n \"id\": \"accepted\",\n \"type\": \"bool\",\n \"info\": {\n \"title\": \"Accepted\",\n \"description\": \"Whether the bid cleared (true) or was rejected (false).\",\n \"comment\": \"Rejected rows are retained for audit; filter accepted=true for settlement.\",\n \"example\": \"true\",\n \"unit\": null\n }\n },\n {\n \"id\": \"bidder_metadata\",\n \"type\": \"jsonb\",\n \"info\": {\n \"title\": \"Bidder Metadata\",\n \"description\": \"Free-form provider-specific metadata captured at submission time.\",\n \"comment\": \"Schema not enforced; kept opaque for downstream analytics.\",\n \"example\": \"{\\\"unit_id\\\": \\\"DRAX-1\\\", \\\"submission_lag_ms\\\": 412}\",\n \"unit\": null\n }\n }\n ],\n \"primary_key\": [\n \"auction_id\",\n \"product_code\"\n ],\n \"records\": [\n {\n \"auction_id\": 1,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-01T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.99,\n \"volume_mwh\": 52.5,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 171\n }\n },\n {\n \"auction_id\": 2,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-01T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.55,\n \"volume_mwh\": 53.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 408\n }\n },\n {\n \"auction_id\": 2,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-01T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 35.66,\n \"volume_mwh\": 69.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 314\n }\n },\n {\n \"auction_id\": 3,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-01T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 40.51,\n \"volume_mwh\": 108.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 488\n }\n },\n {\n \"auction_id\": 3,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-01T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 55.15,\n \"volume_mwh\": 119.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 210\n }\n },\n {\n \"auction_id\": 4,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-01T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.93,\n \"volume_mwh\": 83.7,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 283\n }\n },\n {\n \"auction_id\": 4,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-01T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.19,\n \"volume_mwh\": 110.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 374\n }\n },\n {\n \"auction_id\": 5,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 38.12,\n \"volume_mwh\": 142.2,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 524\n }\n },\n {\n \"auction_id\": 5,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.72,\n \"volume_mwh\": 138.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 460\n }\n },\n {\n \"auction_id\": 6,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.74,\n \"volume_mwh\": 116.1,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 537\n }\n },\n {\n \"auction_id\": 6,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 40.82,\n \"volume_mwh\": 60.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 183\n }\n },\n {\n \"auction_id\": 7,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 44.25,\n \"volume_mwh\": 71.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 411\n }\n },\n {\n \"auction_id\": 7,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.87,\n \"volume_mwh\": 103.4,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 294\n }\n },\n {\n \"auction_id\": 8,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.75,\n \"volume_mwh\": 142.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 450\n }\n },\n {\n \"auction_id\": 8,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 43.11,\n \"volume_mwh\": 126.8,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 512\n }\n },\n {\n \"auction_id\": 9,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 42.89,\n \"volume_mwh\": 76.8,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 548\n }\n },\n {\n \"auction_id\": 9,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.95,\n \"volume_mwh\": 71.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 173\n }\n },\n {\n \"auction_id\": 10,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.62,\n \"volume_mwh\": 74.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 482\n }\n },\n {\n \"auction_id\": 10,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 49.61,\n \"volume_mwh\": 139.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 170\n }\n },\n {\n \"auction_id\": 11,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.74,\n \"volume_mwh\": 59.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 178\n }\n },\n {\n \"auction_id\": 11,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.69,\n \"volume_mwh\": 129.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 297\n }\n },\n {\n \"auction_id\": 12,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 44.54,\n \"volume_mwh\": 149.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 540\n }\n },\n {\n \"auction_id\": 12,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.56,\n \"volume_mwh\": 118.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 484\n }\n },\n {\n \"auction_id\": 13,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.67,\n \"volume_mwh\": 114.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 180\n }\n },\n {\n \"auction_id\": 13,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 46.34,\n \"volume_mwh\": 145.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 597\n }\n },\n {\n \"auction_id\": 14,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.51,\n \"volume_mwh\": 67.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 530\n }\n },\n {\n \"auction_id\": 14,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.97,\n \"volume_mwh\": 110.9,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 376\n }\n },\n {\n \"auction_id\": 15,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.84,\n \"volume_mwh\": 142.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 265\n }\n },\n {\n \"auction_id\": 15,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.22,\n \"volume_mwh\": 61.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 222\n }\n },\n {\n \"auction_id\": 16,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.45,\n \"volume_mwh\": 137.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 474\n }\n },\n {\n \"auction_id\": 16,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.15,\n \"volume_mwh\": 56.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 165\n }\n },\n {\n \"auction_id\": 17,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.49,\n \"volume_mwh\": 144.7,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 546\n }\n },\n {\n \"auction_id\": 17,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.16,\n \"volume_mwh\": 146.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 465\n }\n },\n {\n \"auction_id\": 18,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 42.79,\n \"volume_mwh\": 149.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 560\n }\n },\n {\n \"auction_id\": 18,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.94,\n \"volume_mwh\": 62.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 110\n }\n },\n {\n \"auction_id\": 19,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 49.71,\n \"volume_mwh\": 73.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 462\n }\n },\n {\n \"auction_id\": 19,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.78,\n \"volume_mwh\": 72.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 136\n }\n },\n {\n \"auction_id\": 20,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.85,\n \"volume_mwh\": 77.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 167\n }\n },\n {\n \"auction_id\": 20,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.08,\n \"volume_mwh\": 138.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 501\n }\n },\n {\n \"auction_id\": 21,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 46.82,\n \"volume_mwh\": 90.7,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 281\n }\n },\n {\n \"auction_id\": 21,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.59,\n \"volume_mwh\": 96.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 131\n }\n },\n {\n \"auction_id\": 22,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.07,\n \"volume_mwh\": 83.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 198\n }\n },\n {\n \"auction_id\": 22,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.76,\n \"volume_mwh\": 94.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 336\n }\n },\n {\n \"auction_id\": 23,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.25,\n \"volume_mwh\": 142.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 150\n }\n },\n {\n \"auction_id\": 23,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.26,\n \"volume_mwh\": 149.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 574\n }\n },\n {\n \"auction_id\": 24,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.84,\n \"volume_mwh\": 73.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 209\n }\n },\n {\n \"auction_id\": 24,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.62,\n \"volume_mwh\": 140.2,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 299\n }\n },\n {\n \"auction_id\": 25,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.63,\n \"volume_mwh\": 128.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 456\n }\n },\n {\n \"auction_id\": 25,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.93,\n \"volume_mwh\": 149.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 179\n }\n },\n {\n \"auction_id\": 26,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.75,\n \"volume_mwh\": 71.8,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 131\n }\n },\n {\n \"auction_id\": 26,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.7,\n \"volume_mwh\": 55.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 570\n }\n },\n {\n \"auction_id\": 27,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.32,\n \"volume_mwh\": 65.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 535\n }\n },\n {\n \"auction_id\": 27,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.65,\n \"volume_mwh\": 109.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 306\n }\n },\n {\n \"auction_id\": 28,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 38.0,\n \"volume_mwh\": 139.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 120\n }\n },\n {\n \"auction_id\": 28,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.48,\n \"volume_mwh\": 91.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 261\n }\n },\n {\n \"auction_id\": 29,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.37,\n \"volume_mwh\": 70.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 235\n }\n },\n {\n \"auction_id\": 29,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 44.89,\n \"volume_mwh\": 117.2,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 575\n }\n },\n {\n \"auction_id\": 30,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.8,\n \"volume_mwh\": 57.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 151\n }\n },\n {\n \"auction_id\": 30,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.83,\n \"volume_mwh\": 71.3,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 551\n }\n },\n {\n \"auction_id\": 31,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.72,\n \"volume_mwh\": 74.4,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 526\n }\n },\n {\n \"auction_id\": 31,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 48.58,\n \"volume_mwh\": 80.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 104\n }\n },\n {\n \"auction_id\": 32,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.7,\n \"volume_mwh\": 105.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 580\n }\n },\n {\n \"auction_id\": 32,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.95,\n \"volume_mwh\": 76.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 179\n }\n },\n {\n \"auction_id\": 33,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.81,\n \"volume_mwh\": 110.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 451\n }\n },\n {\n \"auction_id\": 33,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.86,\n \"volume_mwh\": 76.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 147\n }\n },\n {\n \"auction_id\": 34,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.86,\n \"volume_mwh\": 132.9,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 494\n }\n },\n {\n \"auction_id\": 34,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 38.27,\n \"volume_mwh\": 148.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 382\n }\n },\n {\n \"auction_id\": 35,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.64,\n \"volume_mwh\": 106.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 379\n }\n },\n {\n \"auction_id\": 35,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 35.9,\n \"volume_mwh\": 86.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 165\n }\n },\n {\n \"auction_id\": 36,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.05,\n \"volume_mwh\": 86.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 560\n }\n },\n {\n \"auction_id\": 36,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 43.95,\n \"volume_mwh\": 118.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 499\n }\n },\n {\n \"auction_id\": 37,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 49.0,\n \"volume_mwh\": 137.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 573\n }\n },\n {\n \"auction_id\": 37,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.25,\n \"volume_mwh\": 136.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 551\n }\n },\n {\n \"auction_id\": 38,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.31,\n \"volume_mwh\": 67.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 510\n }\n },\n {\n \"auction_id\": 38,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.75,\n \"volume_mwh\": 123.5,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 503\n }\n },\n {\n \"auction_id\": 39,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.53,\n \"volume_mwh\": 88.3,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 213\n }\n },\n {\n \"auction_id\": 39,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.99,\n \"volume_mwh\": 141.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 214\n }\n },\n {\n \"auction_id\": 40,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 35.59,\n \"volume_mwh\": 69.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 595\n }\n },\n {\n \"auction_id\": 40,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 54.33,\n \"volume_mwh\": 85.1,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 269\n }\n },\n {\n \"auction_id\": 41,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.48,\n \"volume_mwh\": 61.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 397\n }\n },\n {\n \"auction_id\": 41,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 59.06,\n \"volume_mwh\": 76.5,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 276\n }\n },\n {\n \"auction_id\": 42,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.21,\n \"volume_mwh\": 81.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 159\n }\n },\n {\n \"auction_id\": 42,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 44.63,\n \"volume_mwh\": 107.7,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 100\n }\n },\n {\n \"auction_id\": 43,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 48.0,\n \"volume_mwh\": 130.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 286\n }\n },\n {\n \"auction_id\": 43,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.78,\n \"volume_mwh\": 144.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 260\n }\n },\n {\n \"auction_id\": 44,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.58,\n \"volume_mwh\": 62.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 258\n }\n },\n {\n \"auction_id\": 44,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.67,\n \"volume_mwh\": 82.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 165\n }\n },\n {\n \"auction_id\": 45,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.8,\n \"volume_mwh\": 116.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 415\n }\n },\n {\n \"auction_id\": 45,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 49.23,\n \"volume_mwh\": 90.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 246\n }\n },\n {\n \"auction_id\": 46,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 40.25,\n \"volume_mwh\": 128.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 338\n }\n },\n {\n \"auction_id\": 46,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 46.04,\n \"volume_mwh\": 117.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 437\n }\n },\n {\n \"auction_id\": 47,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 37.12,\n \"volume_mwh\": 101.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 147\n }\n },\n {\n \"auction_id\": 47,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 55.46,\n \"volume_mwh\": 125.1,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 512\n }\n },\n {\n \"auction_id\": 48,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.98,\n \"volume_mwh\": 52.4,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 412\n }\n },\n {\n \"auction_id\": 48,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.24,\n \"volume_mwh\": 57.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 199\n }\n },\n {\n \"auction_id\": 49,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.96,\n \"volume_mwh\": 88.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 435\n }\n },\n {\n \"auction_id\": 49,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.19,\n \"volume_mwh\": 139.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 498\n }\n },\n {\n \"auction_id\": 50,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.63,\n \"volume_mwh\": 67.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 337\n }\n },\n {\n \"auction_id\": 50,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.26,\n \"volume_mwh\": 74.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 168\n }\n },\n {\n \"auction_id\": 51,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 55.04,\n \"volume_mwh\": 116.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 262\n }\n },\n {\n \"auction_id\": 51,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.75,\n \"volume_mwh\": 139.1,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 318\n }\n },\n {\n \"auction_id\": 52,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 55.76,\n \"volume_mwh\": 104.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 330\n }\n },\n {\n \"auction_id\": 52,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.48,\n \"volume_mwh\": 74.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 348\n }\n },\n {\n \"auction_id\": 53,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-10T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.67,\n \"volume_mwh\": 77.5,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 220\n }\n },\n {\n \"auction_id\": 53,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-10T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.79,\n \"volume_mwh\": 82.0,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 177\n }\n },\n {\n \"auction_id\": 54,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-10T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 40.78,\n \"volume_mwh\": 119.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 312\n }\n },\n {\n \"auction_id\": 54,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-10T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.19,\n \"volume_mwh\": 104.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 526\n }\n },\n {\n \"auction_id\": 55,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-10T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.5,\n \"volume_mwh\": 140.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 538\n }\n },\n {\n \"auction_id\": 55,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-10T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 57.02,\n \"volume_mwh\": 107.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 252\n }\n }\n ]\n}", "options": { "raw": { "language": "json" @@ -161,7 +161,53 @@ }, "description": "Back-compat path: the legacy `fields` + `primary_key` shape against a resource id that already exists in CKAN." }, - "response": [] + "response": [ + { + "name": "Create - legacy fields, existing resource \u2014 example response", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"resource_id\": \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\",\n \"fields\": [\n {\n \"id\": \"auction_id\",\n \"type\": \"int4\",\n \"info\": {\n \"title\": \"Auction ID\",\n \"description\": \"Unique auction identifier. Stable across all products auctioned in the same market window.\",\n \"comment\": \"Primary-key component; pairs with product_code to uniquely reference a cleared bid.\",\n \"example\": \"144\",\n \"unit\": null\n }\n },\n {\n \"id\": \"product_code\",\n \"type\": \"text\",\n \"info\": {\n \"title\": \"Product Code\",\n \"description\": \"Product mnemonic for the balancing service (e.g. DCL, DCH, FFR).\",\n \"comment\": \"Primary-key component; controlled vocabulary maintained by the market operator.\",\n \"example\": \"DCL\",\n \"unit\": null\n }\n },\n {\n \"id\": \"delivery_start\",\n \"type\": \"timestamptz\",\n \"info\": {\n \"title\": \"Delivery Start (UTC)\",\n \"description\": \"First instant of the delivery window. Stored as UTC; clients render local time.\",\n \"comment\": \"Always persisted as UTC \\u2014 do not assume the caller's timezone.\",\n \"example\": \"2025-11-04T16:00:00Z\",\n \"unit\": null\n }\n },\n {\n \"id\": \"duration_minutes\",\n \"type\": \"int4\",\n \"info\": {\n \"title\": \"Delivery Duration\",\n \"description\": \"Length of the delivery window.\",\n \"comment\": \"Add to delivery_start to derive the delivery end instant.\",\n \"example\": \"30\",\n \"unit\": \"minutes\"\n }\n },\n {\n \"id\": \"clearing_price_gbp_per_mwh\",\n \"type\": \"float8\",\n \"info\": {\n \"title\": \"Clearing Price\",\n \"description\": \"Pay-as-cleared price for the auction. Negative values are possible during oversupply.\",\n \"comment\": \"Pay-as-cleared: every accepted bid settles at this price, not its own offer.\",\n \"example\": \"47.82\",\n \"unit\": \"GBP/MWh\"\n }\n },\n {\n \"id\": \"volume_mwh\",\n \"type\": \"float8\",\n \"info\": {\n \"title\": \"Cleared Volume\",\n \"description\": \"Total volume cleared in this auction.\",\n \"comment\": \"Sum across accepted bids for the (auction_id, product_code) pair.\",\n \"example\": \"120.0\",\n \"unit\": \"MWh\"\n }\n },\n {\n \"id\": \"accepted\",\n \"type\": \"bool\",\n \"info\": {\n \"title\": \"Accepted\",\n \"description\": \"Whether the bid cleared (true) or was rejected (false).\",\n \"comment\": \"Rejected rows are retained for audit; filter accepted=true for settlement.\",\n \"example\": \"true\",\n \"unit\": null\n }\n },\n {\n \"id\": \"bidder_metadata\",\n \"type\": \"jsonb\",\n \"info\": {\n \"title\": \"Bidder Metadata\",\n \"description\": \"Free-form provider-specific metadata captured at submission time.\",\n \"comment\": \"Schema not enforced; kept opaque for downstream analytics.\",\n \"example\": \"{\\\"unit_id\\\": \\\"DRAX-1\\\", \\\"submission_lag_ms\\\": 412}\",\n \"unit\": null\n }\n }\n ],\n \"primary_key\": [\n \"auction_id\",\n \"product_code\"\n ],\n \"records\": [\n {\n \"auction_id\": 1,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-01T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.99,\n \"volume_mwh\": 52.5,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 171\n }\n },\n {\n \"auction_id\": 2,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-01T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.55,\n \"volume_mwh\": 53.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 408\n }\n },\n {\n \"auction_id\": 2,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-01T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 35.66,\n \"volume_mwh\": 69.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 314\n }\n },\n {\n \"auction_id\": 3,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-01T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 40.51,\n \"volume_mwh\": 108.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 488\n }\n },\n {\n \"auction_id\": 3,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-01T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 55.15,\n \"volume_mwh\": 119.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 210\n }\n },\n {\n \"auction_id\": 4,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-01T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.93,\n \"volume_mwh\": 83.7,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 283\n }\n },\n {\n \"auction_id\": 4,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-01T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.19,\n \"volume_mwh\": 110.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 374\n }\n },\n {\n \"auction_id\": 5,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 38.12,\n \"volume_mwh\": 142.2,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 524\n }\n },\n {\n \"auction_id\": 5,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.72,\n \"volume_mwh\": 138.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 460\n }\n },\n {\n \"auction_id\": 6,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.74,\n \"volume_mwh\": 116.1,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 537\n }\n },\n {\n \"auction_id\": 6,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 40.82,\n \"volume_mwh\": 60.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 183\n }\n },\n {\n \"auction_id\": 7,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 44.25,\n \"volume_mwh\": 71.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 411\n }\n },\n {\n \"auction_id\": 7,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.87,\n \"volume_mwh\": 103.4,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 294\n }\n },\n {\n \"auction_id\": 8,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.75,\n \"volume_mwh\": 142.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 450\n }\n },\n {\n \"auction_id\": 8,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 43.11,\n \"volume_mwh\": 126.8,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 512\n }\n },\n {\n \"auction_id\": 9,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 42.89,\n \"volume_mwh\": 76.8,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 548\n }\n },\n {\n \"auction_id\": 9,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.95,\n \"volume_mwh\": 71.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 173\n }\n },\n {\n \"auction_id\": 10,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.62,\n \"volume_mwh\": 74.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 482\n }\n },\n {\n \"auction_id\": 10,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 49.61,\n \"volume_mwh\": 139.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 170\n }\n },\n {\n \"auction_id\": 11,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.74,\n \"volume_mwh\": 59.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 178\n }\n },\n {\n \"auction_id\": 11,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.69,\n \"volume_mwh\": 129.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 297\n }\n },\n {\n \"auction_id\": 12,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 44.54,\n \"volume_mwh\": 149.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 540\n }\n },\n {\n \"auction_id\": 12,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.56,\n \"volume_mwh\": 118.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 484\n }\n },\n {\n \"auction_id\": 13,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.67,\n \"volume_mwh\": 114.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 180\n }\n },\n {\n \"auction_id\": 13,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 46.34,\n \"volume_mwh\": 145.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 597\n }\n },\n {\n \"auction_id\": 14,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.51,\n \"volume_mwh\": 67.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 530\n }\n },\n {\n \"auction_id\": 14,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.97,\n \"volume_mwh\": 110.9,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 376\n }\n },\n {\n \"auction_id\": 15,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.84,\n \"volume_mwh\": 142.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 265\n }\n },\n {\n \"auction_id\": 15,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.22,\n \"volume_mwh\": 61.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 222\n }\n },\n {\n \"auction_id\": 16,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.45,\n \"volume_mwh\": 137.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 474\n }\n },\n {\n \"auction_id\": 16,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.15,\n \"volume_mwh\": 56.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 165\n }\n },\n {\n \"auction_id\": 17,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.49,\n \"volume_mwh\": 144.7,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 546\n }\n },\n {\n \"auction_id\": 17,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.16,\n \"volume_mwh\": 146.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 465\n }\n },\n {\n \"auction_id\": 18,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 42.79,\n \"volume_mwh\": 149.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 560\n }\n },\n {\n \"auction_id\": 18,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.94,\n \"volume_mwh\": 62.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 110\n }\n },\n {\n \"auction_id\": 19,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 49.71,\n \"volume_mwh\": 73.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 462\n }\n },\n {\n \"auction_id\": 19,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.78,\n \"volume_mwh\": 72.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 136\n }\n },\n {\n \"auction_id\": 20,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.85,\n \"volume_mwh\": 77.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 167\n }\n },\n {\n \"auction_id\": 20,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.08,\n \"volume_mwh\": 138.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 501\n }\n },\n {\n \"auction_id\": 21,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 46.82,\n \"volume_mwh\": 90.7,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 281\n }\n },\n {\n \"auction_id\": 21,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.59,\n \"volume_mwh\": 96.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 131\n }\n },\n {\n \"auction_id\": 22,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.07,\n \"volume_mwh\": 83.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 198\n }\n },\n {\n \"auction_id\": 22,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.76,\n \"volume_mwh\": 94.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 336\n }\n },\n {\n \"auction_id\": 23,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.25,\n \"volume_mwh\": 142.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 150\n }\n },\n {\n \"auction_id\": 23,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.26,\n \"volume_mwh\": 149.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 574\n }\n },\n {\n \"auction_id\": 24,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.84,\n \"volume_mwh\": 73.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 209\n }\n },\n {\n \"auction_id\": 24,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.62,\n \"volume_mwh\": 140.2,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 299\n }\n },\n {\n \"auction_id\": 25,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.63,\n \"volume_mwh\": 128.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 456\n }\n },\n {\n \"auction_id\": 25,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.93,\n \"volume_mwh\": 149.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 179\n }\n },\n {\n \"auction_id\": 26,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.75,\n \"volume_mwh\": 71.8,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 131\n }\n },\n {\n \"auction_id\": 26,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.7,\n \"volume_mwh\": 55.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 570\n }\n },\n {\n \"auction_id\": 27,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.32,\n \"volume_mwh\": 65.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 535\n }\n },\n {\n \"auction_id\": 27,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.65,\n \"volume_mwh\": 109.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 306\n }\n },\n {\n \"auction_id\": 28,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 38.0,\n \"volume_mwh\": 139.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 120\n }\n },\n {\n \"auction_id\": 28,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.48,\n \"volume_mwh\": 91.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 261\n }\n },\n {\n \"auction_id\": 29,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.37,\n \"volume_mwh\": 70.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 235\n }\n },\n {\n \"auction_id\": 29,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 44.89,\n \"volume_mwh\": 117.2,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 575\n }\n },\n {\n \"auction_id\": 30,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.8,\n \"volume_mwh\": 57.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 151\n }\n },\n {\n \"auction_id\": 30,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.83,\n \"volume_mwh\": 71.3,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 551\n }\n },\n {\n \"auction_id\": 31,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.72,\n \"volume_mwh\": 74.4,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 526\n }\n },\n {\n \"auction_id\": 31,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 48.58,\n \"volume_mwh\": 80.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 104\n }\n },\n {\n \"auction_id\": 32,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.7,\n \"volume_mwh\": 105.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 580\n }\n },\n {\n \"auction_id\": 32,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.95,\n \"volume_mwh\": 76.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 179\n }\n },\n {\n \"auction_id\": 33,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.81,\n \"volume_mwh\": 110.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 451\n }\n },\n {\n \"auction_id\": 33,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.86,\n \"volume_mwh\": 76.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 147\n }\n },\n {\n \"auction_id\": 34,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.86,\n \"volume_mwh\": 132.9,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 494\n }\n },\n {\n \"auction_id\": 34,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 38.27,\n \"volume_mwh\": 148.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 382\n }\n },\n {\n \"auction_id\": 35,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.64,\n \"volume_mwh\": 106.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 379\n }\n },\n {\n \"auction_id\": 35,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 35.9,\n \"volume_mwh\": 86.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 165\n }\n },\n {\n \"auction_id\": 36,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.05,\n \"volume_mwh\": 86.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 560\n }\n },\n {\n \"auction_id\": 36,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 43.95,\n \"volume_mwh\": 118.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 499\n }\n },\n {\n \"auction_id\": 37,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 49.0,\n \"volume_mwh\": 137.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 573\n }\n },\n {\n \"auction_id\": 37,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.25,\n \"volume_mwh\": 136.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 551\n }\n },\n {\n \"auction_id\": 38,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.31,\n \"volume_mwh\": 67.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 510\n }\n },\n {\n \"auction_id\": 38,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.75,\n \"volume_mwh\": 123.5,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 503\n }\n },\n {\n \"auction_id\": 39,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.53,\n \"volume_mwh\": 88.3,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 213\n }\n },\n {\n \"auction_id\": 39,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.99,\n \"volume_mwh\": 141.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 214\n }\n },\n {\n \"auction_id\": 40,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 35.59,\n \"volume_mwh\": 69.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 595\n }\n },\n {\n \"auction_id\": 40,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 54.33,\n \"volume_mwh\": 85.1,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 269\n }\n },\n {\n \"auction_id\": 41,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.48,\n \"volume_mwh\": 61.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 397\n }\n },\n {\n \"auction_id\": 41,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 59.06,\n \"volume_mwh\": 76.5,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 276\n }\n },\n {\n \"auction_id\": 42,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.21,\n \"volume_mwh\": 81.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 159\n }\n },\n {\n \"auction_id\": 42,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 44.63,\n \"volume_mwh\": 107.7,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 100\n }\n },\n {\n \"auction_id\": 43,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 48.0,\n \"volume_mwh\": 130.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 286\n }\n },\n {\n \"auction_id\": 43,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.78,\n \"volume_mwh\": 144.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 260\n }\n },\n {\n \"auction_id\": 44,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.58,\n \"volume_mwh\": 62.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 258\n }\n },\n {\n \"auction_id\": 44,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.67,\n \"volume_mwh\": 82.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 165\n }\n },\n {\n \"auction_id\": 45,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.8,\n \"volume_mwh\": 116.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 415\n }\n },\n {\n \"auction_id\": 45,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 49.23,\n \"volume_mwh\": 90.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 246\n }\n },\n {\n \"auction_id\": 46,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 40.25,\n \"volume_mwh\": 128.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 338\n }\n },\n {\n \"auction_id\": 46,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 46.04,\n \"volume_mwh\": 117.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 437\n }\n },\n {\n \"auction_id\": 47,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 37.12,\n \"volume_mwh\": 101.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 147\n }\n },\n {\n \"auction_id\": 47,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 55.46,\n \"volume_mwh\": 125.1,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 512\n }\n },\n {\n \"auction_id\": 48,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.98,\n \"volume_mwh\": 52.4,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 412\n }\n },\n {\n \"auction_id\": 48,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.24,\n \"volume_mwh\": 57.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 199\n }\n },\n {\n \"auction_id\": 49,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.96,\n \"volume_mwh\": 88.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 435\n }\n },\n {\n \"auction_id\": 49,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.19,\n \"volume_mwh\": 139.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 498\n }\n },\n {\n \"auction_id\": 50,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.63,\n \"volume_mwh\": 67.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 337\n }\n },\n {\n \"auction_id\": 50,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.26,\n \"volume_mwh\": 74.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 168\n }\n },\n {\n \"auction_id\": 51,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 55.04,\n \"volume_mwh\": 116.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 262\n }\n },\n {\n \"auction_id\": 51,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.75,\n \"volume_mwh\": 139.1,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 318\n }\n },\n {\n \"auction_id\": 52,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 55.76,\n \"volume_mwh\": 104.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 330\n }\n },\n {\n \"auction_id\": 52,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.48,\n \"volume_mwh\": 74.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 348\n }\n },\n {\n \"auction_id\": 53,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-10T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.67,\n \"volume_mwh\": 77.5,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 220\n }\n },\n {\n \"auction_id\": 53,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-10T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.79,\n \"volume_mwh\": 82.0,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 177\n }\n },\n {\n \"auction_id\": 54,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-10T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 40.78,\n \"volume_mwh\": 119.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 312\n }\n },\n {\n \"auction_id\": 54,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-10T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.19,\n \"volume_mwh\": 104.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 526\n }\n },\n {\n \"auction_id\": 55,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-10T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.5,\n \"volume_mwh\": 140.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 538\n }\n },\n {\n \"auction_id\": 55,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-10T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 57.02,\n \"volume_mwh\": 107.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 252\n }\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{baseUrl}}/datastore/api/v2/datastore_create", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "datastore", + "api", + "v2", + "datastore_create" + ] + }, + "description": "Back-compat path: the legacy `fields` + `primary_key` shape against a resource id that already exists in CKAN." + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": "{\n \"help\": \"http://localhost:8000/datastore/api/v2/docs#/Datastore/datastore_create\",\n \"success\": true,\n \"result\": {\n \"resource_id\": \"balancing_auction_results_2025\",\n \"fields\": [\n {\n \"id\": \"_id\",\n \"type\": \"int8\",\n \"info\": {\n \"title\": \"Record Identifier\",\n \"description\": \"System-generated unique identifier assigned to each record in the database. Used to uniquely identify a row within a resource table.\",\n \"comment\": \"Automatically populated. Not publisher-maintained. Values are unique within a resource and may be used for record retrieval and updates.\",\n \"example\": \"12345\",\n \"unit\": \"N/A\"\n }\n },\n {\n \"id\": \"_updated_at\",\n \"type\": \"timestamp\",\n \"info\": {\n \"title\": \"Last Updated Timestamp\",\n \"description\": \"System-generated timestamp indicating when the record was last created or modified in the database.\",\n \"comment\": \"Automatically maintained. Useful for identifying recently changed records. Stored in UTC.\",\n \"example\": \"2026-08-27T12:45:30\",\n \"unit\": \"ISO 8601 UTC Timestamp\"\n }\n },\n {\n \"id\": \"auction_id\",\n \"type\": \"int4\",\n \"info\": {\n \"title\": \"Auction ID\",\n \"description\": \"Unique auction identifier. Stable across all products auctioned in the same market window.\",\n \"comment\": \"Primary-key component; pairs with product_code to uniquely reference a cleared bid.\",\n \"example\": \"144\",\n \"unit\": null\n }\n },\n {\n \"id\": \"product_code\",\n \"type\": \"text\",\n \"info\": {\n \"title\": \"Product Code\",\n \"description\": \"Product mnemonic for the balancing service (e.g. DCL, DCH, FFR).\",\n \"comment\": \"Primary-key component; controlled vocabulary maintained by the market operator.\",\n \"example\": \"DCL\",\n \"unit\": null\n }\n },\n {\n \"id\": \"delivery_start\",\n \"type\": \"timestamptz\",\n \"info\": {\n \"title\": \"Delivery Start (UTC)\",\n \"description\": \"First instant of the delivery window. Stored as UTC; clients render local time.\",\n \"comment\": \"Always persisted as UTC \\u2014 do not assume the caller's timezone.\",\n \"example\": \"2025-11-04T16:00:00Z\",\n \"unit\": null\n }\n },\n {\n \"id\": \"duration_minutes\",\n \"type\": \"int4\",\n \"info\": {\n \"title\": \"Delivery Duration\",\n \"description\": \"Length of the delivery window.\",\n \"comment\": \"Add to delivery_start to derive the delivery end instant.\",\n \"example\": \"30\",\n \"unit\": \"minutes\"\n }\n },\n {\n \"id\": \"clearing_price_gbp_per_mwh\",\n \"type\": \"float8\",\n \"info\": {\n \"title\": \"Clearing Price\",\n \"description\": \"Pay-as-cleared price for the auction. Negative values are possible during oversupply.\",\n \"comment\": \"Pay-as-cleared: every accepted bid settles at this price, not its own offer.\",\n \"example\": \"47.82\",\n \"unit\": \"GBP/MWh\"\n }\n },\n {\n \"id\": \"volume_mwh\",\n \"type\": \"float8\",\n \"info\": {\n \"title\": \"Cleared Volume\",\n \"description\": \"Total volume cleared in this auction.\",\n \"comment\": \"Sum across accepted bids for the (auction_id, product_code) pair.\",\n \"example\": \"120.0\",\n \"unit\": \"MWh\"\n }\n },\n {\n \"id\": \"accepted\",\n \"type\": \"bool\",\n \"info\": {\n \"title\": \"Accepted\",\n \"description\": \"Whether the bid cleared (true) or was rejected (false).\",\n \"comment\": \"Rejected rows are retained for audit; filter accepted=true for settlement.\",\n \"example\": \"true\",\n \"unit\": null\n }\n },\n {\n \"id\": \"bidder_metadata\",\n \"type\": \"jsonb\",\n \"info\": {\n \"title\": \"Bidder Metadata\",\n \"description\": \"Free-form provider-specific metadata captured at submission time.\",\n \"comment\": \"Schema not enforced; kept opaque for downstream analytics.\",\n \"example\": \"{\\\"unit_id\\\": \\\"DRAX-1\\\", \\\"submission_lag_ms\\\": 412}\",\n \"unit\": null\n }\n }\n ],\n \"primary_key\": [\n \"auction_id\",\n \"product_code\"\n ],\n \"unique_key\": [\n \"auction_id\",\n \"product_code\"\n ]\n }\n}", + "cookie": [] + } + ] }, { "name": "Create - legacy fields, new resource", @@ -175,7 +221,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"resource\": {\n \"name\": \"Balancing Auction Results 2025\",\n \"package_id\": \"6475b49d-4340-4289-94c9-0404adce912e\",\n \"format\": \"csv\",\n \"mimetype\": \"text/csv\",\n \"description\": \"Cleared bids from the balancing market, one row per auction \\u00d7 product.\"\n },\n \"fields\": [\n {\n \"id\": \"auction_id\",\n \"type\": \"int4\",\n \"info\": {\n \"title\": \"Auction ID\",\n \"description\": \"Unique auction identifier. Stable across all products auctioned in the same market window.\",\n \"comment\": \"Primary-key component; pairs with product_code to uniquely reference a cleared bid.\",\n \"unit\": null\n }\n },\n {\n \"id\": \"product_code\",\n \"type\": \"text\",\n \"info\": {\n \"title\": \"Product Code\",\n \"description\": \"Product mnemonic for the balancing service (e.g. DCL, DCH, FFR).\",\n \"comment\": \"Primary-key component; controlled vocabulary maintained by the market operator.\",\n \"unit\": null\n }\n },\n {\n \"id\": \"delivery_start\",\n \"type\": \"timestamptz\",\n \"info\": {\n \"title\": \"Delivery Start (UTC)\",\n \"description\": \"First instant of the delivery window. Stored as UTC; clients render local time.\",\n \"comment\": \"Always persisted as UTC \\u2014 do not assume the caller's timezone.\",\n \"unit\": null\n }\n },\n {\n \"id\": \"duration_minutes\",\n \"type\": \"int4\",\n \"info\": {\n \"title\": \"Delivery Duration\",\n \"description\": \"Length of the delivery window.\",\n \"comment\": \"Add to delivery_start to derive the delivery end instant.\",\n \"unit\": \"minutes\"\n }\n },\n {\n \"id\": \"clearing_price_gbp_per_mwh\",\n \"type\": \"float8\",\n \"info\": {\n \"title\": \"Clearing Price\",\n \"description\": \"Pay-as-cleared price for the auction. Negative values are possible during oversupply.\",\n \"comment\": \"Pay-as-cleared: every accepted bid settles at this price, not its own offer.\",\n \"unit\": \"GBP/MWh\"\n }\n },\n {\n \"id\": \"volume_mwh\",\n \"type\": \"float8\",\n \"info\": {\n \"title\": \"Cleared Volume\",\n \"description\": \"Total volume cleared in this auction.\",\n \"comment\": \"Sum across accepted bids for the (auction_id, product_code) pair.\",\n \"unit\": \"MWh\"\n }\n },\n {\n \"id\": \"accepted\",\n \"type\": \"bool\",\n \"info\": {\n \"title\": \"Accepted\",\n \"description\": \"Whether the bid cleared (true) or was rejected (false).\",\n \"comment\": \"Rejected rows are retained for audit; filter accepted=true for settlement.\",\n \"unit\": null\n }\n },\n {\n \"id\": \"bidder_metadata\",\n \"type\": \"jsonb\",\n \"info\": {\n \"title\": \"Bidder Metadata\",\n \"description\": \"Free-form provider-specific metadata captured at submission time.\",\n \"comment\": \"Schema not enforced; kept opaque for downstream analytics.\",\n \"unit\": null\n }\n }\n ],\n \"primary_key\": [\n \"auction_id\",\n \"product_code\"\n ],\n \"records\": [\n {\n \"auction_id\": 1,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-01T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.99,\n \"volume_mwh\": 52.5,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 171\n }\n },\n {\n \"auction_id\": 1,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-01T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.41,\n \"volume_mwh\": 117.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 402\n }\n },\n {\n \"auction_id\": 2,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-01T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.55,\n \"volume_mwh\": 53.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 408\n }\n },\n {\n \"auction_id\": 2,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-01T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 35.66,\n \"volume_mwh\": 69.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 314\n }\n },\n {\n \"auction_id\": 3,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-01T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 40.51,\n \"volume_mwh\": 108.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 488\n }\n },\n {\n \"auction_id\": 3,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-01T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 55.15,\n \"volume_mwh\": 119.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 210\n }\n },\n {\n \"auction_id\": 4,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-01T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.93,\n \"volume_mwh\": 83.7,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 283\n }\n },\n {\n \"auction_id\": 4,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-01T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.19,\n \"volume_mwh\": 110.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 374\n }\n },\n {\n \"auction_id\": 5,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 38.12,\n \"volume_mwh\": 142.2,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 524\n }\n },\n {\n \"auction_id\": 5,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.72,\n \"volume_mwh\": 138.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 460\n }\n },\n {\n \"auction_id\": 6,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.74,\n \"volume_mwh\": 116.1,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 537\n }\n },\n {\n \"auction_id\": 6,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 40.82,\n \"volume_mwh\": 60.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 183\n }\n },\n {\n \"auction_id\": 7,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 44.25,\n \"volume_mwh\": 71.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 411\n }\n },\n {\n \"auction_id\": 7,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.87,\n \"volume_mwh\": 103.4,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 294\n }\n },\n {\n \"auction_id\": 8,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.75,\n \"volume_mwh\": 142.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 450\n }\n },\n {\n \"auction_id\": 8,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 43.11,\n \"volume_mwh\": 126.8,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 512\n }\n },\n {\n \"auction_id\": 9,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 42.89,\n \"volume_mwh\": 76.8,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 548\n }\n },\n {\n \"auction_id\": 9,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.95,\n \"volume_mwh\": 71.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 173\n }\n },\n {\n \"auction_id\": 10,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.62,\n \"volume_mwh\": 74.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 482\n }\n },\n {\n \"auction_id\": 10,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 49.61,\n \"volume_mwh\": 139.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 170\n }\n },\n {\n \"auction_id\": 11,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.74,\n \"volume_mwh\": 59.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 178\n }\n },\n {\n \"auction_id\": 11,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.69,\n \"volume_mwh\": 129.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 297\n }\n },\n {\n \"auction_id\": 12,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 44.54,\n \"volume_mwh\": 149.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 540\n }\n },\n {\n \"auction_id\": 12,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.56,\n \"volume_mwh\": 118.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 484\n }\n },\n {\n \"auction_id\": 13,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.67,\n \"volume_mwh\": 114.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 180\n }\n },\n {\n \"auction_id\": 13,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 46.34,\n \"volume_mwh\": 145.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 597\n }\n },\n {\n \"auction_id\": 14,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.51,\n \"volume_mwh\": 67.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 530\n }\n },\n {\n \"auction_id\": 14,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.97,\n \"volume_mwh\": 110.9,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 376\n }\n },\n {\n \"auction_id\": 15,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.84,\n \"volume_mwh\": 142.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 265\n }\n },\n {\n \"auction_id\": 15,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.22,\n \"volume_mwh\": 61.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 222\n }\n },\n {\n \"auction_id\": 16,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.45,\n \"volume_mwh\": 137.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 474\n }\n },\n {\n \"auction_id\": 16,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.15,\n \"volume_mwh\": 56.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 165\n }\n },\n {\n \"auction_id\": 17,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.49,\n \"volume_mwh\": 144.7,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 546\n }\n },\n {\n \"auction_id\": 17,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.16,\n \"volume_mwh\": 146.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 465\n }\n },\n {\n \"auction_id\": 18,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 42.79,\n \"volume_mwh\": 149.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 560\n }\n },\n {\n \"auction_id\": 18,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.94,\n \"volume_mwh\": 62.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 110\n }\n },\n {\n \"auction_id\": 19,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 49.71,\n \"volume_mwh\": 73.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 462\n }\n },\n {\n \"auction_id\": 19,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.78,\n \"volume_mwh\": 72.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 136\n }\n },\n {\n \"auction_id\": 20,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.85,\n \"volume_mwh\": 77.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 167\n }\n },\n {\n \"auction_id\": 20,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.08,\n \"volume_mwh\": 138.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 501\n }\n },\n {\n \"auction_id\": 21,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 46.82,\n \"volume_mwh\": 90.7,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 281\n }\n },\n {\n \"auction_id\": 21,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.59,\n \"volume_mwh\": 96.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 131\n }\n },\n {\n \"auction_id\": 22,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.07,\n \"volume_mwh\": 83.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 198\n }\n },\n {\n \"auction_id\": 22,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.76,\n \"volume_mwh\": 94.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 336\n }\n },\n {\n \"auction_id\": 23,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.25,\n \"volume_mwh\": 142.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 150\n }\n },\n {\n \"auction_id\": 23,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.26,\n \"volume_mwh\": 149.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 574\n }\n },\n {\n \"auction_id\": 24,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.84,\n \"volume_mwh\": 73.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 209\n }\n },\n {\n \"auction_id\": 24,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.62,\n \"volume_mwh\": 140.2,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 299\n }\n },\n {\n \"auction_id\": 25,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.63,\n \"volume_mwh\": 128.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 456\n }\n },\n {\n \"auction_id\": 25,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.93,\n \"volume_mwh\": 149.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 179\n }\n },\n {\n \"auction_id\": 26,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.75,\n \"volume_mwh\": 71.8,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 131\n }\n },\n {\n \"auction_id\": 26,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.7,\n \"volume_mwh\": 55.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 570\n }\n },\n {\n \"auction_id\": 27,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.32,\n \"volume_mwh\": 65.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 535\n }\n },\n {\n \"auction_id\": 27,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.65,\n \"volume_mwh\": 109.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 306\n }\n },\n {\n \"auction_id\": 28,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 38.0,\n \"volume_mwh\": 139.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 120\n }\n },\n {\n \"auction_id\": 28,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.48,\n \"volume_mwh\": 91.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 261\n }\n },\n {\n \"auction_id\": 29,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.37,\n \"volume_mwh\": 70.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 235\n }\n },\n {\n \"auction_id\": 29,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 44.89,\n \"volume_mwh\": 117.2,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 575\n }\n },\n {\n \"auction_id\": 30,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.8,\n \"volume_mwh\": 57.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 151\n }\n },\n {\n \"auction_id\": 30,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.83,\n \"volume_mwh\": 71.3,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 551\n }\n },\n {\n \"auction_id\": 31,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.72,\n \"volume_mwh\": 74.4,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 526\n }\n },\n {\n \"auction_id\": 31,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 48.58,\n \"volume_mwh\": 80.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 104\n }\n },\n {\n \"auction_id\": 32,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.7,\n \"volume_mwh\": 105.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 580\n }\n },\n {\n \"auction_id\": 32,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.95,\n \"volume_mwh\": 76.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 179\n }\n },\n {\n \"auction_id\": 33,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.81,\n \"volume_mwh\": 110.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 451\n }\n },\n {\n \"auction_id\": 33,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.86,\n \"volume_mwh\": 76.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 147\n }\n },\n {\n \"auction_id\": 34,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.86,\n \"volume_mwh\": 132.9,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 494\n }\n },\n {\n \"auction_id\": 34,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 38.27,\n \"volume_mwh\": 148.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 382\n }\n },\n {\n \"auction_id\": 35,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.64,\n \"volume_mwh\": 106.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 379\n }\n },\n {\n \"auction_id\": 35,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 35.9,\n \"volume_mwh\": 86.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 165\n }\n },\n {\n \"auction_id\": 36,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.05,\n \"volume_mwh\": 86.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 560\n }\n },\n {\n \"auction_id\": 36,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 43.95,\n \"volume_mwh\": 118.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 499\n }\n },\n {\n \"auction_id\": 37,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 49.0,\n \"volume_mwh\": 137.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 573\n }\n },\n {\n \"auction_id\": 37,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.25,\n \"volume_mwh\": 136.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 551\n }\n },\n {\n \"auction_id\": 38,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.31,\n \"volume_mwh\": 67.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 510\n }\n },\n {\n \"auction_id\": 38,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.75,\n \"volume_mwh\": 123.5,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 503\n }\n },\n {\n \"auction_id\": 39,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.53,\n \"volume_mwh\": 88.3,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 213\n }\n },\n {\n \"auction_id\": 39,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.99,\n \"volume_mwh\": 141.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 214\n }\n },\n {\n \"auction_id\": 40,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 35.59,\n \"volume_mwh\": 69.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 595\n }\n },\n {\n \"auction_id\": 40,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 54.33,\n \"volume_mwh\": 85.1,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 269\n }\n },\n {\n \"auction_id\": 41,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.48,\n \"volume_mwh\": 61.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 397\n }\n },\n {\n \"auction_id\": 41,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 59.06,\n \"volume_mwh\": 76.5,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 276\n }\n },\n {\n \"auction_id\": 42,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.21,\n \"volume_mwh\": 81.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 159\n }\n },\n {\n \"auction_id\": 42,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 44.63,\n \"volume_mwh\": 107.7,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 100\n }\n },\n {\n \"auction_id\": 43,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 48.0,\n \"volume_mwh\": 130.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 286\n }\n },\n {\n \"auction_id\": 43,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.78,\n \"volume_mwh\": 144.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 260\n }\n },\n {\n \"auction_id\": 44,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.58,\n \"volume_mwh\": 62.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 258\n }\n },\n {\n \"auction_id\": 44,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.67,\n \"volume_mwh\": 82.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 165\n }\n },\n {\n \"auction_id\": 45,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.8,\n \"volume_mwh\": 116.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 415\n }\n },\n {\n \"auction_id\": 45,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 49.23,\n \"volume_mwh\": 90.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 246\n }\n },\n {\n \"auction_id\": 46,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 40.25,\n \"volume_mwh\": 128.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 338\n }\n },\n {\n \"auction_id\": 46,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 46.04,\n \"volume_mwh\": 117.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 437\n }\n },\n {\n \"auction_id\": 47,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 37.12,\n \"volume_mwh\": 101.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 147\n }\n },\n {\n \"auction_id\": 47,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 55.46,\n \"volume_mwh\": 125.1,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 512\n }\n },\n {\n \"auction_id\": 48,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.98,\n \"volume_mwh\": 52.4,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 412\n }\n },\n {\n \"auction_id\": 48,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.24,\n \"volume_mwh\": 57.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 199\n }\n },\n {\n \"auction_id\": 49,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.96,\n \"volume_mwh\": 88.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 435\n }\n },\n {\n \"auction_id\": 49,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.19,\n \"volume_mwh\": 139.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 498\n }\n },\n {\n \"auction_id\": 50,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.63,\n \"volume_mwh\": 67.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 337\n }\n },\n {\n \"auction_id\": 50,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.26,\n \"volume_mwh\": 74.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 168\n }\n },\n {\n \"auction_id\": 51,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 55.04,\n \"volume_mwh\": 116.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 262\n }\n },\n {\n \"auction_id\": 51,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.75,\n \"volume_mwh\": 139.1,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 318\n }\n },\n {\n \"auction_id\": 52,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 55.76,\n \"volume_mwh\": 104.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 330\n }\n },\n {\n \"auction_id\": 52,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.48,\n \"volume_mwh\": 74.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 348\n }\n },\n {\n \"auction_id\": 53,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-10T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.67,\n \"volume_mwh\": 77.5,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 220\n }\n },\n {\n \"auction_id\": 53,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-10T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.79,\n \"volume_mwh\": 82.0,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 177\n }\n },\n {\n \"auction_id\": 54,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-10T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 40.78,\n \"volume_mwh\": 119.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 312\n }\n },\n {\n \"auction_id\": 54,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-10T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.19,\n \"volume_mwh\": 104.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 526\n }\n },\n {\n \"auction_id\": 55,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-10T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.5,\n \"volume_mwh\": 140.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 538\n }\n },\n {\n \"auction_id\": 55,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-10T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 57.02,\n \"volume_mwh\": 107.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 252\n }\n }\n ]\n}", + "raw": "{\n \"resource\": {\n \"name\": \"Balancing Auction Results 2025\",\n \"package_id\": \"6475b49d-4340-4289-94c9-0404adce912e\",\n \"format\": \"csv\",\n \"mimetype\": \"text/csv\",\n \"description\": \"Cleared bids from the balancing market, one row per auction \\u00d7 product.\"\n },\n \"fields\": [\n {\n \"id\": \"auction_id\",\n \"type\": \"int4\",\n \"info\": {\n \"title\": \"Auction ID\",\n \"description\": \"Unique auction identifier. Stable across all products auctioned in the same market window.\",\n \"comment\": \"Primary-key component; pairs with product_code to uniquely reference a cleared bid.\",\n \"example\": \"144\",\n \"unit\": null\n }\n },\n {\n \"id\": \"product_code\",\n \"type\": \"text\",\n \"info\": {\n \"title\": \"Product Code\",\n \"description\": \"Product mnemonic for the balancing service (e.g. DCL, DCH, FFR).\",\n \"comment\": \"Primary-key component; controlled vocabulary maintained by the market operator.\",\n \"example\": \"DCL\",\n \"unit\": null\n }\n },\n {\n \"id\": \"delivery_start\",\n \"type\": \"timestamptz\",\n \"info\": {\n \"title\": \"Delivery Start (UTC)\",\n \"description\": \"First instant of the delivery window. Stored as UTC; clients render local time.\",\n \"comment\": \"Always persisted as UTC \\u2014 do not assume the caller's timezone.\",\n \"example\": \"2025-11-04T16:00:00Z\",\n \"unit\": null\n }\n },\n {\n \"id\": \"duration_minutes\",\n \"type\": \"int4\",\n \"info\": {\n \"title\": \"Delivery Duration\",\n \"description\": \"Length of the delivery window.\",\n \"comment\": \"Add to delivery_start to derive the delivery end instant.\",\n \"example\": \"30\",\n \"unit\": \"minutes\"\n }\n },\n {\n \"id\": \"clearing_price_gbp_per_mwh\",\n \"type\": \"float8\",\n \"info\": {\n \"title\": \"Clearing Price\",\n \"description\": \"Pay-as-cleared price for the auction. Negative values are possible during oversupply.\",\n \"comment\": \"Pay-as-cleared: every accepted bid settles at this price, not its own offer.\",\n \"example\": \"47.82\",\n \"unit\": \"GBP/MWh\"\n }\n },\n {\n \"id\": \"volume_mwh\",\n \"type\": \"float8\",\n \"info\": {\n \"title\": \"Cleared Volume\",\n \"description\": \"Total volume cleared in this auction.\",\n \"comment\": \"Sum across accepted bids for the (auction_id, product_code) pair.\",\n \"example\": \"120.0\",\n \"unit\": \"MWh\"\n }\n },\n {\n \"id\": \"accepted\",\n \"type\": \"bool\",\n \"info\": {\n \"title\": \"Accepted\",\n \"description\": \"Whether the bid cleared (true) or was rejected (false).\",\n \"comment\": \"Rejected rows are retained for audit; filter accepted=true for settlement.\",\n \"example\": \"true\",\n \"unit\": null\n }\n },\n {\n \"id\": \"bidder_metadata\",\n \"type\": \"jsonb\",\n \"info\": {\n \"title\": \"Bidder Metadata\",\n \"description\": \"Free-form provider-specific metadata captured at submission time.\",\n \"comment\": \"Schema not enforced; kept opaque for downstream analytics.\",\n \"example\": \"{\\\"unit_id\\\": \\\"DRAX-1\\\", \\\"submission_lag_ms\\\": 412}\",\n \"unit\": null\n }\n }\n ],\n \"primary_key\": [\n \"auction_id\",\n \"product_code\"\n ],\n \"records\": [\n {\n \"auction_id\": 1,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-01T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.99,\n \"volume_mwh\": 52.5,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 171\n }\n },\n {\n \"auction_id\": 1,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-01T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.41,\n \"volume_mwh\": 117.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 402\n }\n },\n {\n \"auction_id\": 2,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-01T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.55,\n \"volume_mwh\": 53.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 408\n }\n },\n {\n \"auction_id\": 2,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-01T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 35.66,\n \"volume_mwh\": 69.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 314\n }\n },\n {\n \"auction_id\": 3,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-01T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 40.51,\n \"volume_mwh\": 108.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 488\n }\n },\n {\n \"auction_id\": 3,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-01T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 55.15,\n \"volume_mwh\": 119.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 210\n }\n },\n {\n \"auction_id\": 4,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-01T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.93,\n \"volume_mwh\": 83.7,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 283\n }\n },\n {\n \"auction_id\": 4,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-01T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.19,\n \"volume_mwh\": 110.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 374\n }\n },\n {\n \"auction_id\": 5,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 38.12,\n \"volume_mwh\": 142.2,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 524\n }\n },\n {\n \"auction_id\": 5,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.72,\n \"volume_mwh\": 138.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 460\n }\n },\n {\n \"auction_id\": 6,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.74,\n \"volume_mwh\": 116.1,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 537\n }\n },\n {\n \"auction_id\": 6,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 40.82,\n \"volume_mwh\": 60.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 183\n }\n },\n {\n \"auction_id\": 7,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 44.25,\n \"volume_mwh\": 71.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 411\n }\n },\n {\n \"auction_id\": 7,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.87,\n \"volume_mwh\": 103.4,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 294\n }\n },\n {\n \"auction_id\": 8,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.75,\n \"volume_mwh\": 142.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 450\n }\n },\n {\n \"auction_id\": 8,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 43.11,\n \"volume_mwh\": 126.8,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 512\n }\n },\n {\n \"auction_id\": 9,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 42.89,\n \"volume_mwh\": 76.8,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 548\n }\n },\n {\n \"auction_id\": 9,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.95,\n \"volume_mwh\": 71.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 173\n }\n },\n {\n \"auction_id\": 10,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-02T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.62,\n \"volume_mwh\": 74.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 482\n }\n },\n {\n \"auction_id\": 10,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-02T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 49.61,\n \"volume_mwh\": 139.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 170\n }\n },\n {\n \"auction_id\": 11,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.74,\n \"volume_mwh\": 59.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 178\n }\n },\n {\n \"auction_id\": 11,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.69,\n \"volume_mwh\": 129.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 297\n }\n },\n {\n \"auction_id\": 12,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 44.54,\n \"volume_mwh\": 149.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 540\n }\n },\n {\n \"auction_id\": 12,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.56,\n \"volume_mwh\": 118.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 484\n }\n },\n {\n \"auction_id\": 13,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.67,\n \"volume_mwh\": 114.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 180\n }\n },\n {\n \"auction_id\": 13,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 46.34,\n \"volume_mwh\": 145.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 597\n }\n },\n {\n \"auction_id\": 14,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.51,\n \"volume_mwh\": 67.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 530\n }\n },\n {\n \"auction_id\": 14,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.97,\n \"volume_mwh\": 110.9,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 376\n }\n },\n {\n \"auction_id\": 15,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.84,\n \"volume_mwh\": 142.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 265\n }\n },\n {\n \"auction_id\": 15,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.22,\n \"volume_mwh\": 61.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 222\n }\n },\n {\n \"auction_id\": 16,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-03T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.45,\n \"volume_mwh\": 137.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 474\n }\n },\n {\n \"auction_id\": 16,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-03T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.15,\n \"volume_mwh\": 56.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 165\n }\n },\n {\n \"auction_id\": 17,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.49,\n \"volume_mwh\": 144.7,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 546\n }\n },\n {\n \"auction_id\": 17,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.16,\n \"volume_mwh\": 146.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 465\n }\n },\n {\n \"auction_id\": 18,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 42.79,\n \"volume_mwh\": 149.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 560\n }\n },\n {\n \"auction_id\": 18,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.94,\n \"volume_mwh\": 62.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 110\n }\n },\n {\n \"auction_id\": 19,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 49.71,\n \"volume_mwh\": 73.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 462\n }\n },\n {\n \"auction_id\": 19,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.78,\n \"volume_mwh\": 72.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 136\n }\n },\n {\n \"auction_id\": 20,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 47.85,\n \"volume_mwh\": 77.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 167\n }\n },\n {\n \"auction_id\": 20,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.08,\n \"volume_mwh\": 138.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 501\n }\n },\n {\n \"auction_id\": 21,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 46.82,\n \"volume_mwh\": 90.7,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 281\n }\n },\n {\n \"auction_id\": 21,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.59,\n \"volume_mwh\": 96.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 131\n }\n },\n {\n \"auction_id\": 22,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-04T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.07,\n \"volume_mwh\": 83.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 198\n }\n },\n {\n \"auction_id\": 22,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-04T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.76,\n \"volume_mwh\": 94.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 336\n }\n },\n {\n \"auction_id\": 23,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.25,\n \"volume_mwh\": 142.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 150\n }\n },\n {\n \"auction_id\": 23,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.26,\n \"volume_mwh\": 149.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 574\n }\n },\n {\n \"auction_id\": 24,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.84,\n \"volume_mwh\": 73.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 209\n }\n },\n {\n \"auction_id\": 24,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.62,\n \"volume_mwh\": 140.2,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 299\n }\n },\n {\n \"auction_id\": 25,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.63,\n \"volume_mwh\": 128.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 456\n }\n },\n {\n \"auction_id\": 25,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.93,\n \"volume_mwh\": 149.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 179\n }\n },\n {\n \"auction_id\": 26,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.75,\n \"volume_mwh\": 71.8,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 131\n }\n },\n {\n \"auction_id\": 26,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.7,\n \"volume_mwh\": 55.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 570\n }\n },\n {\n \"auction_id\": 27,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.32,\n \"volume_mwh\": 65.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 535\n }\n },\n {\n \"auction_id\": 27,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.65,\n \"volume_mwh\": 109.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 306\n }\n },\n {\n \"auction_id\": 28,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-05T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 38.0,\n \"volume_mwh\": 139.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 120\n }\n },\n {\n \"auction_id\": 28,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-05T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.48,\n \"volume_mwh\": 91.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 261\n }\n },\n {\n \"auction_id\": 29,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.37,\n \"volume_mwh\": 70.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 235\n }\n },\n {\n \"auction_id\": 29,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 44.89,\n \"volume_mwh\": 117.2,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 575\n }\n },\n {\n \"auction_id\": 30,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.8,\n \"volume_mwh\": 57.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 151\n }\n },\n {\n \"auction_id\": 30,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.83,\n \"volume_mwh\": 71.3,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 551\n }\n },\n {\n \"auction_id\": 31,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.72,\n \"volume_mwh\": 74.4,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 526\n }\n },\n {\n \"auction_id\": 31,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 48.58,\n \"volume_mwh\": 80.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 104\n }\n },\n {\n \"auction_id\": 32,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.7,\n \"volume_mwh\": 105.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 580\n }\n },\n {\n \"auction_id\": 32,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.95,\n \"volume_mwh\": 76.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 179\n }\n },\n {\n \"auction_id\": 33,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.81,\n \"volume_mwh\": 110.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 451\n }\n },\n {\n \"auction_id\": 33,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.86,\n \"volume_mwh\": 76.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 147\n }\n },\n {\n \"auction_id\": 34,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-06T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.86,\n \"volume_mwh\": 132.9,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 494\n }\n },\n {\n \"auction_id\": 34,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-06T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 38.27,\n \"volume_mwh\": 148.0,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 382\n }\n },\n {\n \"auction_id\": 35,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.64,\n \"volume_mwh\": 106.1,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 379\n }\n },\n {\n \"auction_id\": 35,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 35.9,\n \"volume_mwh\": 86.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 165\n }\n },\n {\n \"auction_id\": 36,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.05,\n \"volume_mwh\": 86.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 560\n }\n },\n {\n \"auction_id\": 36,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 43.95,\n \"volume_mwh\": 118.2,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 499\n }\n },\n {\n \"auction_id\": 37,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 49.0,\n \"volume_mwh\": 137.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 573\n }\n },\n {\n \"auction_id\": 37,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.25,\n \"volume_mwh\": 136.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 551\n }\n },\n {\n \"auction_id\": 38,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.31,\n \"volume_mwh\": 67.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 510\n }\n },\n {\n \"auction_id\": 38,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.75,\n \"volume_mwh\": 123.5,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 503\n }\n },\n {\n \"auction_id\": 39,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.53,\n \"volume_mwh\": 88.3,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 213\n }\n },\n {\n \"auction_id\": 39,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.99,\n \"volume_mwh\": 141.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 214\n }\n },\n {\n \"auction_id\": 40,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-07T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 35.59,\n \"volume_mwh\": 69.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 595\n }\n },\n {\n \"auction_id\": 40,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-07T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 54.33,\n \"volume_mwh\": 85.1,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 269\n }\n },\n {\n \"auction_id\": 41,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.48,\n \"volume_mwh\": 61.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 397\n }\n },\n {\n \"auction_id\": 41,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 59.06,\n \"volume_mwh\": 76.5,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 276\n }\n },\n {\n \"auction_id\": 42,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 53.21,\n \"volume_mwh\": 81.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 159\n }\n },\n {\n \"auction_id\": 42,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 44.63,\n \"volume_mwh\": 107.7,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 100\n }\n },\n {\n \"auction_id\": 43,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 48.0,\n \"volume_mwh\": 130.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 286\n }\n },\n {\n \"auction_id\": 43,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.78,\n \"volume_mwh\": 144.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 260\n }\n },\n {\n \"auction_id\": 44,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.58,\n \"volume_mwh\": 62.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 258\n }\n },\n {\n \"auction_id\": 44,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 51.67,\n \"volume_mwh\": 82.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 165\n }\n },\n {\n \"auction_id\": 45,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.8,\n \"volume_mwh\": 116.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 415\n }\n },\n {\n \"auction_id\": 45,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 49.23,\n \"volume_mwh\": 90.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 246\n }\n },\n {\n \"auction_id\": 46,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-08T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 40.25,\n \"volume_mwh\": 128.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 338\n }\n },\n {\n \"auction_id\": 46,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-08T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 46.04,\n \"volume_mwh\": 117.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 437\n }\n },\n {\n \"auction_id\": 47,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 37.12,\n \"volume_mwh\": 101.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 147\n }\n },\n {\n \"auction_id\": 47,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 55.46,\n \"volume_mwh\": 125.1,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 512\n }\n },\n {\n \"auction_id\": 48,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 39.98,\n \"volume_mwh\": 52.4,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 412\n }\n },\n {\n \"auction_id\": 48,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 56.24,\n \"volume_mwh\": 57.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 199\n }\n },\n {\n \"auction_id\": 49,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.96,\n \"volume_mwh\": 88.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 435\n }\n },\n {\n \"auction_id\": 49,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 52.19,\n \"volume_mwh\": 139.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 498\n }\n },\n {\n \"auction_id\": 50,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.63,\n \"volume_mwh\": 67.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 337\n }\n },\n {\n \"auction_id\": 50,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T12:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 36.26,\n \"volume_mwh\": 74.9,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 168\n }\n },\n {\n \"auction_id\": 51,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 55.04,\n \"volume_mwh\": 116.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 262\n }\n },\n {\n \"auction_id\": 51,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T16:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 58.75,\n \"volume_mwh\": 139.1,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 318\n }\n },\n {\n \"auction_id\": 52,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-09T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 55.76,\n \"volume_mwh\": 104.8,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-3\",\n \"submission_lag_ms\": 330\n }\n },\n {\n \"auction_id\": 52,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-09T20:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.48,\n \"volume_mwh\": 74.7,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"SSE-PEH-3\",\n \"submission_lag_ms\": 348\n }\n },\n {\n \"auction_id\": 53,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-10T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 50.67,\n \"volume_mwh\": 77.5,\n \"accepted\": false,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 220\n }\n },\n {\n \"auction_id\": 53,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-10T00:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 41.79,\n \"volume_mwh\": 82.0,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 177\n }\n },\n {\n \"auction_id\": 54,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-10T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 40.78,\n \"volume_mwh\": 119.4,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 312\n }\n },\n {\n \"auction_id\": 54,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-10T04:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.19,\n \"volume_mwh\": 104.3,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-2\",\n \"submission_lag_ms\": 526\n }\n },\n {\n \"auction_id\": 55,\n \"product_code\": \"DCL\",\n \"delivery_start\": \"2025-11-10T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 45.5,\n \"volume_mwh\": 140.5,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"DRAX-1\",\n \"submission_lag_ms\": 538\n }\n },\n {\n \"auction_id\": 55,\n \"product_code\": \"DCH\",\n \"delivery_start\": \"2025-11-10T08:00:00Z\",\n \"duration_minutes\": 30,\n \"clearing_price_gbp_per_mwh\": 57.02,\n \"volume_mwh\": 107.6,\n \"accepted\": true,\n \"bidder_metadata\": {\n \"unit_id\": \"EDF-COTT-2\",\n \"submission_lag_ms\": 252\n }\n }\n ]\n}", "options": { "raw": { "language": "json" @@ -340,7 +386,45 @@ }, "description": "Return the stored Frictionless schema + row count via `INFORMATION_SCHEMA.TABLE_STORAGE`." }, - "response": [] + "response": [ + { + "name": "Info - by resource_id \u2014 example response", + "originalRequest": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/datastore/api/v2/datastore_info?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "datastore", + "api", + "v2", + "datastore_info" + ], + "query": [ + { + "key": "resource_id", + "value": "7a10def4-8e95-46f9-96c7-9f61bdfd1a09" + } + ] + }, + "description": "Return the stored Frictionless schema + row count via `INFORMATION_SCHEMA.TABLE_STORAGE`." + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": "{\n \"help\": \"http://localhost:8000/datastore/api/v2/docs#/Datastore/datastore_info\",\n \"success\": true,\n \"result\": {\n \"resource_id\": \"balancing_auction_results_2025\",\n \"fields\": [\n {\n \"id\": \"_id\",\n \"type\": \"int8\",\n \"info\": {\n \"title\": \"Record Identifier\",\n \"description\": \"System-generated unique identifier assigned to each record in the database. Used to uniquely identify a row within a resource table.\",\n \"comment\": \"Automatically populated. Not publisher-maintained. Values are unique within a resource and may be used for record retrieval and updates.\",\n \"example\": \"12345\",\n \"unit\": \"N/A\"\n }\n },\n {\n \"id\": \"_updated_at\",\n \"type\": \"timestamp\",\n \"info\": {\n \"title\": \"Last Updated Timestamp\",\n \"description\": \"System-generated timestamp indicating when the record was last created or modified in the database.\",\n \"comment\": \"Automatically maintained. Useful for identifying recently changed records. Stored in UTC.\",\n \"example\": \"2026-08-27T12:45:30\",\n \"unit\": \"ISO 8601 UTC Timestamp\"\n }\n },\n {\n \"id\": \"auction_id\",\n \"type\": \"int4\",\n \"info\": {\n \"title\": \"Auction ID\",\n \"description\": \"Unique auction identifier. Stable across all products auctioned in the same market window.\",\n \"comment\": \"Primary-key component; pairs with product_code to uniquely reference a cleared bid.\",\n \"example\": \"144\",\n \"unit\": \"N/A\"\n }\n },\n {\n \"id\": \"product_code\",\n \"type\": \"text\",\n \"info\": {\n \"title\": \"Product Code\",\n \"description\": \"Product mnemonic for the balancing service (e.g. DCL, DCH, FFR).\",\n \"comment\": \"Primary-key component; controlled vocabulary maintained by the market operator.\",\n \"example\": \"DCL\",\n \"unit\": \"N/A\"\n }\n },\n {\n \"id\": \"delivery_start\",\n \"type\": \"timestamptz\",\n \"info\": {\n \"title\": \"Delivery Start (UTC)\",\n \"description\": \"First instant of the delivery window. Stored as UTC; clients render local time.\",\n \"example\": \"2025-11-04T16:00:00Z\"\n }\n },\n {\n \"id\": \"duration_minutes\",\n \"type\": \"int4\",\n \"info\": {\n \"title\": \"Delivery Duration\",\n \"example\": \"30\",\n \"unit\": \"minutes\"\n }\n },\n {\n \"id\": \"clearing_price_gbp_per_mwh\",\n \"type\": \"float8\",\n \"info\": {\n \"title\": \"Clearing Price\",\n \"example\": \"47.82\",\n \"unit\": \"GBP/MWh\"\n }\n },\n {\n \"id\": \"volume_mwh\",\n \"type\": \"float8\",\n \"info\": {\n \"title\": \"Cleared Volume\",\n \"example\": \"120.0\",\n \"unit\": \"MWh\"\n }\n },\n {\n \"id\": \"accepted\",\n \"type\": \"bool\",\n \"info\": {\n \"title\": \"Accepted\",\n \"example\": \"true\"\n }\n },\n {\n \"id\": \"bidder_metadata\",\n \"type\": \"jsonb\",\n \"info\": {\n \"title\": \"Bidder Metadata\",\n \"example\": \"{\\\"unit_id\\\": \\\"DRAX-1\\\", \\\"submission_lag_ms\\\": 412}\"\n }\n }\n ],\n \"unique_key\": [\n \"auction_id\",\n \"product_code\"\n ],\n \"primary_key\": [\n \"auction_id\",\n \"product_code\"\n ],\n \"total\": 110\n }\n}", + "cookie": [] + } + ] }, { "name": "Info - `id` alias", @@ -400,7 +484,45 @@ }, "description": "Default page (limit=100, offset=0, include_total=true)." }, - "response": [] + "response": [ + { + "name": "Search - minimal (just resource_id) \u2014 example response", + "originalRequest": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/datastore/api/v2/datastore_search?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "datastore", + "api", + "v2", + "datastore_search" + ], + "query": [ + { + "key": "resource_id", + "value": "7a10def4-8e95-46f9-96c7-9f61bdfd1a09" + } + ] + }, + "description": "Default page (limit=100, offset=0, include_total=true)." + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": "{\n \"help\": \"http://localhost:8000/datastore/api/v2/docs#/Datastore/datastore_search\",\n \"success\": true,\n \"result\": {\n \"include_total\": true,\n \"resource_id\": \"balancing_auction_results_2025\",\n \"fields\": [\n {\n \"id\": \"_id\",\n \"type\": \"int8\",\n \"info\": {\n \"title\": \"Record Identifier\",\n \"description\": \"System-generated unique identifier assigned to each record in the database. Used to uniquely identify a row within a resource table.\",\n \"comment\": \"Automatically populated. Not publisher-maintained. Values are unique within a resource and may be used for record retrieval and updates.\",\n \"example\": \"12345\",\n \"unit\": \"N/A\"\n }\n },\n {\n \"id\": \"_updated_at\",\n \"type\": \"timestamp\",\n \"info\": {\n \"title\": \"Last Updated Timestamp\",\n \"description\": \"System-generated timestamp indicating when the record was last created or modified in the database.\",\n \"comment\": \"Automatically maintained. Useful for identifying recently changed records. Stored in UTC.\",\n \"example\": \"2026-08-27T12:45:30\",\n \"unit\": \"ISO 8601 UTC Timestamp\"\n }\n },\n {\n \"id\": \"auction_id\",\n \"type\": \"int4\",\n \"info\": {\n \"title\": \"Auction ID\",\n \"example\": \"144\",\n \"unit\": \"N/A\"\n }\n },\n {\n \"id\": \"product_code\",\n \"type\": \"text\",\n \"info\": {\n \"title\": \"Product Code\",\n \"example\": \"DCL\"\n }\n },\n {\n \"id\": \"clearing_price_gbp_per_mwh\",\n \"type\": \"float8\",\n \"info\": {\n \"title\": \"Clearing Price\",\n \"example\": \"47.82\",\n \"unit\": \"GBP/MWh\"\n }\n }\n ],\n \"records_format\": \"objects\",\n \"records\": [\n {\n \"_id\": 1,\n \"_updated_at\": \"2026-08-27T12:45:30\",\n \"auction_id\": 1,\n \"product_code\": \"DCL\",\n \"clearing_price_gbp_per_mwh\": 50.99\n },\n {\n \"_id\": 2,\n \"_updated_at\": \"2026-08-27T12:45:30\",\n \"auction_id\": 2,\n \"product_code\": \"DCL\",\n \"clearing_price_gbp_per_mwh\": 45.55\n }\n ],\n \"limit\": 100,\n \"offset\": 0,\n \"total\": 110,\n \"_links\": {\n \"start\": \"http://localhost:8000/datastore/api/v2/datastore_search?resource_id=balancing_auction_results_2025&limit=100\",\n \"next\": \"http://localhost:8000/datastore/api/v2/datastore_search?resource_id=balancing_auction_results_2025&limit=100&offset=100\"\n }\n }\n}", + "cookie": [] + } + ] }, { "name": "Search - with filters", @@ -636,7 +758,45 @@ }, "description": "Plain SELECT with WHERE + LIMIT. Total comes from `INFORMATION_SCHEMA.TABLE_STORAGE` since there's no aggregate." }, - "response": [] + "response": [ + { + "name": "SQL - basic SELECT \u2014 example response", + "originalRequest": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/datastore/api/v2/datastore_search_sql?sql=SELECT auction_id, product_code, clearing_price_gbp_per_mwh FROM \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\" WHERE accepted = true LIMIT 100", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "datastore", + "api", + "v2", + "datastore_search_sql" + ], + "query": [ + { + "key": "sql", + "value": "SELECT auction_id, product_code, clearing_price_gbp_per_mwh FROM \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\" WHERE accepted = true LIMIT 100" + } + ] + }, + "description": "Plain SELECT with WHERE + LIMIT. Total comes from `INFORMATION_SCHEMA.TABLE_STORAGE` since there's no aggregate." + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": "{\n \"help\": \"http://localhost:8000/datastore/api/v2/docs#/Datastore/datastore_search_sql\",\n \"success\": true,\n \"result\": {\n \"sql\": \"SELECT auction_id, product_code, clearing_price_gbp_per_mwh FROM \\\"balancing_auction_results_2025\\\" WHERE accepted = true LIMIT 100\",\n \"fields\": [\n {\n \"id\": \"auction_id\",\n \"type\": \"int4\",\n \"info\": {\n \"title\": \"Auction ID\",\n \"example\": \"144\",\n \"unit\": \"N/A\"\n }\n },\n {\n \"id\": \"product_code\",\n \"type\": \"text\",\n \"info\": {\n \"title\": \"Product Code\",\n \"example\": \"DCL\"\n }\n },\n {\n \"id\": \"clearing_price_gbp_per_mwh\",\n \"type\": \"float8\",\n \"info\": {\n \"title\": \"Clearing Price\",\n \"example\": \"47.82\",\n \"unit\": \"GBP/MWh\"\n }\n }\n ],\n \"records_format\": \"objects\",\n \"records\": [\n {\n \"auction_id\": 2,\n \"product_code\": \"DCH\",\n \"clearing_price_gbp_per_mwh\": 35.66\n },\n {\n \"auction_id\": 3,\n \"product_code\": \"DCL\",\n \"clearing_price_gbp_per_mwh\": 40.51\n }\n ],\n \"records_truncated\": false\n }\n}", + "cookie": [] + } + ] }, { "name": "SQL - GROUP BY + aggregates", diff --git a/postman/generate_postman.py b/postman/generate_postman.py index bf31da5..8565c2b 100644 --- a/postman/generate_postman.py +++ b/postman/generate_postman.py @@ -374,18 +374,19 @@ def _build_endpoint_folder( if not dir_path.is_dir(): return {"name": action, "description": description, "item": []} for payload_file in _sorted_payloads(action, dir_path): - if "response" in payload_file.stem: - # Sample server responses live next to requests for docs; - # they don't belong in a Postman *request* collection. + if payload_file.stem.endswith("_response"): + # Sample server responses are attached to their sibling + # request (stem without `_response`), not shown as items. continue with payload_file.open() as f: body = json.load(f) name, scenario_desc = _scenario(action, payload_file) builder = _post_request if method == "POST" else _get_request + request = builder(action, body, scenario_desc) folder_items.append({ "name": name, - "request": builder(action, body, scenario_desc), - "response": [], + "request": request, + "response": _collect_responses(dir_path, payload_file.stem, request, name), }) return { "name": action, @@ -394,6 +395,27 @@ def _build_endpoint_folder( } +def _collect_responses( + dir_path: Path, request_stem: str, request: dict[str, Any], request_name: str, +) -> list[dict[str, Any]]: + """Attach a sibling `_response.json` (if present) as a Postman example.""" + response_file = dir_path / f"{request_stem}_response.json" + if not response_file.is_file(): + return [] + with response_file.open() as f: + body = json.load(f) + return [{ + "name": f"{request_name} — example response", + "originalRequest": request, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [{"key": "Content-Type", "value": "application/json"}], + "body": json.dumps(body, indent=2), + "cookie": [], + }] + + def _build_health_folder() -> dict[str, Any]: items = [] for name, sub, desc in HEALTH_REQUESTS: diff --git a/pyproject.toml b/pyproject.toml index 86dcbce..29ef3d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,11 @@ dependencies = [ # this package imports it directly. "jinja2>=3.1", "google-cloud-storage>=2.14", + # Byte-for-byte CKAN parity on the legacy `_full_text` tsvector field: + # Postgres uses Snowball under the hood (`english_stem` dictionary), so + # `restrictions` lemmatises to `restrict`, `networks` to `network`, etc. + # ~500KB pure Python, no transitive deps. + "snowballstemmer>=3.0", ] [project.optional-dependencies] diff --git a/tests/engines/bigquery/test_tables.py b/tests/engines/bigquery/test_tables.py index 6ad3e24..baa2e64 100644 --- a/tests/engines/bigquery/test_tables.py +++ b/tests/engines/bigquery/test_tables.py @@ -128,7 +128,7 @@ def test_alter_adds_new_columns_widens_types_then_refreshes_options( old = {"fields": [{"name": "a", "type": "integer"}]} new = { "fields": [ - {"name": "a", "type": "number"}, # widen INT64 → FLOAT64 + {"name": "a", "type": "number"}, # widen INT64 → BIGNUMERIC {"name": "b", "type": "string"}, # add ] } @@ -140,7 +140,7 @@ def test_alter_adds_new_columns_widens_types_then_refreshes_options( opts_sql = mock_client.query.call_args_list[1].args[0] assert "ALTER TABLE `proj-1.ds-1.res-1`" in col_sql assert "ADD COLUMN IF NOT EXISTS `b` STRING" in col_sql - assert "ALTER COLUMN `a` SET DATA TYPE FLOAT64" in col_sql + assert "ALTER COLUMN `a` SET DATA TYPE BIGNUMERIC" in col_sql assert opts_sql.startswith("ALTER TABLE `proj-1.ds-1.res-1` SET OPTIONS(") # Refreshed table OPTIONS carries the new schema verbatim. assert '"name":"a","type":"number"' in opts_sql @@ -1295,13 +1295,13 @@ def test_build_search_renders_full_param_set() -> None: offset=25, ) - assert sql.startswith("SELECT `auction_id`, `product_code` FROM `p.d.r` AS t") + assert sql.startswith("SELECT `auction_id`, `product_code` FROM `p.d.r` AS _src") assert "WHERE `product_code` = @f0 AND `accepted` = @f1" in sql - assert "SEARCH(t, @f2)" in sql - # ORDER BY references the underlying table column (`t.`) so the + assert "SEARCH(_src, @f2)" in sql + # ORDER BY references the underlying table column (`_src.`) so the # FORMAT_TIMESTAMP projection alias for datetime columns doesn't shadow # the native value at sort time. - assert "ORDER BY t.`auction_id` DESC" in sql + assert "ORDER BY _src.`auction_id` DESC" in sql assert sql.rstrip().endswith("LIMIT 100 OFFSET 25") # Parameter types track the schema (STRING / BOOL / STRING for q). by_name = {p.name: p for p in params} @@ -1358,7 +1358,7 @@ def test_build_search_default_sort_is_id_asc() -> None: limit=10, offset=0, ) - assert "ORDER BY t.`_id` ASC" in sql + assert "ORDER BY _src.`_id` ASC" in sql def test_build_search_rejects_unknown_columns() -> None: @@ -1476,7 +1476,7 @@ def test_search_returns_projection_schema_and_lazy_rows( search_sql = mock_client.query.call_args_list[1][0][0] assert count_sql.startswith("SELECT COUNT(*) AS n FROM (") assert search_sql.startswith( - "SELECT `auction_id`, `product_code` FROM `proj-1.ds-1.res-1` AS t" + "SELECT `auction_id`, `product_code` FROM `proj-1.ds-1.res-1` AS _src" ) assert result.total == 1 # `records` is a generator — assert lazy by exhausting it once. @@ -1533,7 +1533,7 @@ def test_search_unfiltered_uses_cheap_row_count( assert sqls[0].startswith( "SELECT `_id`, `a`, " "FORMAT_TIMESTAMP('%Y-%m-%dT%H:%M:%S', `_updated_at`, 'UTC') " - "AS `_updated_at` FROM `proj-1.ds-1.res-1` AS t" + "AS `_updated_at` FROM `proj-1.ds-1.res-1` AS _src" ) assert sqls[1] == ("SELECT COUNT(*) AS n FROM `proj-1.ds-1.res-1`") # No filtered count subquery anywhere. diff --git a/tests/engines/bigquery/test_translate.py b/tests/engines/bigquery/test_translate.py new file mode 100644 index 0000000..77141c9 --- /dev/null +++ b/tests/engines/bigquery/test_translate.py @@ -0,0 +1,498 @@ +"""Unit tests for the Postgres→BigQuery SQL translator. + +Covers every pg-ism we have seen in real customer traffic — the test +cases are deliberately verbatim copies of queries pulled from a +production job log. See the dialect notes in +`datastore/infrastructure/engines/bigquery/translate.py`. + +The translator is the one hook that lets customer scripts carried over +from the Postgres-backed CKAN datastore keep working on BigQuery, so +regressions here would break real `datastore_search_sql` and +`dump/query` calls under both the v2 and legacy API surfaces. +""" + +from __future__ import annotations + +import pytest +import sqlglot +from datastore.infrastructure.engines.bigquery.lib import qualify_table_refs +from datastore.infrastructure.engines.bigquery.translate import translate_pg_to_bq + + +def _translate(sql: str) -> tuple[str, list[str]]: + """Convenience wrapper: enabled by default.""" + return translate_pg_to_bq(sql, enabled=True) + + +# --- identifier quoting ---------------------------------------------------- + + +def test_double_quoted_identifiers_rewritten_to_backticks() -> None: + sql = 'SELECT * FROM "my_table" ORDER BY "_id" ASC' + out, notes = _translate(sql) + + assert "`my_table`" in out + assert "`_id`" in out + assert '"my_table"' not in out + assert any("backticks" in n for n in notes) + + +def test_bare_identifiers_unchanged() -> None: + sql = "SELECT * FROM my_table" + out, notes = _translate(sql) + + assert "my_table" in out + assert notes == [] + + +# --- ::type cast operator -------------------------------------------------- + + +def test_pg_cast_operator_rewritten_to_cast_function() -> None: + sql = 'SELECT "MW"::numeric FROM "t"' + out, notes = _translate(sql) + + assert "CAST(`MW` AS NUMERIC)" in out + assert "::" not in out + assert any("::type" in n for n in notes) + + +# --- to_char --------------------------------------------------------------- + + +def test_to_char_rewritten_to_format_date() -> None: + sql = "SELECT to_char(\"delivery_start\", 'YYYY-MM-DD') AS day FROM \"t\"" + out, _ = _translate(sql) + + # sqlglot picks the right FORMAT_* variant for the format mask. + assert "FORMAT_DATE('%F'" in out or "FORMAT_TIMESTAMP('%F'" in out + assert "to_char" not in out.lower() + + +# --- INTERVAL '10 days' ---------------------------------------------------- + + +def test_pg_interval_literal_rewritten() -> None: + sql = "SELECT * FROM t WHERE d > CURRENT_DATE - INTERVAL '10 days'" + out, _ = _translate(sql) + + # BigQuery wants `INTERVAL '10' DAY` (quoted value + unit keyword). + assert "INTERVAL '10' DAY" in out + + +# --- CURRENT_DATE ± integer-literal arithmetic ----------------------------- + + +def test_current_date_minus_integer_rewritten_to_date_sub() -> None: + sql = 'SELECT * FROM "t" WHERE "d" > CURRENT_DATE - 1' + out, notes = _translate(sql) + + assert "DATE_SUB(CURRENT_DATE, INTERVAL 1 DAY)" in out + assert any("date±integer" in n for n in notes) + + +def test_current_date_plus_integer_rewritten_to_date_add() -> None: + sql = "SELECT * FROM t WHERE d < CURRENT_DATE + 2" + out, notes = _translate(sql) + + assert "DATE_ADD(CURRENT_DATE, INTERVAL 2 DAY)" in out + assert any("date±integer" in n for n in notes) + + +def test_integer_column_minus_integer_not_treated_as_date_arithmetic() -> None: + # Pure integer arithmetic on a bare column stays untouched: we only + # want to rewrite date-returning expressions, not every `- 1`. + sql = "SELECT * FROM t WHERE col > 5" + out, notes = _translate(sql) + + assert "col > 5" in out + assert not any("date±integer" in n for n in notes) + + +# --- combined / real-world queries from the production log ----------------- + + +def test_production_window_function_with_current_date_minus_one() -> None: + sql = ( + 'SELECT COUNT(*) OVER () AS _count, * FROM "a63ab354" ' + 'WHERE "deliveryStart" > Current_Date-1 ' + 'AND "auctionUnit" = (\'NONE-0\') ' + 'ORDER BY "unitResultID" ASC' + ) + out, _ = _translate(sql) + + # Pg-isms gone, BigQuery-shaped SQL out. + assert "DATE_SUB(CURRENT_DATE, INTERVAL 1 DAY)" in out + assert "`deliveryStart`" in out + assert "`a63ab354`" in out + assert '"' not in out + + +def test_production_case_with_pg_cast_round_trips() -> None: + sql = ( + 'SELECT COUNT(*) AS total, ' + 'SUM(CASE WHEN "Status" = \'Accepted\' THEN "MW"::numeric ELSE 0 END) ' + 'FROM "t"' + ) + out, _ = _translate(sql) + + assert "CAST(`MW` AS NUMERIC)" in out + assert "::" not in out + + +# --- kill switch ----------------------------------------------------------- + + +def test_disabled_returns_sql_unchanged() -> None: + sql = 'SELECT * FROM "t" WHERE "d" > CURRENT_DATE - 1' + out, notes = translate_pg_to_bq(sql, enabled=False) + + assert out == sql + assert notes == [] + + +# --- fail-soft on garbage input ------------------------------------------- + + +def test_unparseable_sql_returns_input_unchanged() -> None: + sql = "::: not actually sql :::" + out, notes = _translate(sql) + + assert out == sql + assert notes == [] + + +def test_empty_sql_returns_input_unchanged() -> None: + for sql in ("", " ", "\n\t "): + out, notes = translate_pg_to_bq(sql) + assert out == sql + assert notes == [] + + +# --- production-log regression suite -------------------------------------- +# +# Every entry below is a real query pulled verbatim from a customer-traffic +# job log; the top-end entries account for 55M+ requests. The suite runs +# each one through the full `qualify_table_refs` entry point (same code +# path as v2 `datastore_search_sql` and legacy `/api/3/action/datastore_search_sql`) +# and asserts the output parses as BigQuery dialect. If the translator +# ever regresses on one of these shapes, a real customer breaks. + +PRODUCTION_SAMPLES: list[tuple[str, str]] = [ + ( + "current_date minus integer (1.9M calls)", + 'select count(*) over () as _count, * from ' + '"a63ab354-7e68-44c2-ad96-c6f920c30e85" ' + 'where "deliveryStart" > Current_Date-1 ' + 'and "auctionUnit" = (\'NONE-0\') ' + 'ORDER BY "unitResultID" ASC', + ), + ( + "pg INTERVAL '10 days' literal", + 'SELECT * FROM "13b511df-d6ec-4143-afb1-0ecc6fd19810" ' + "WHERE \"deliveryStart\" > CURRENT_DATE - INTERVAL '10 days'", + ), + ( + "current_date + INTERVAL '2 day'", + 'SELECT * FROM "t" ' + 'WHERE "deliveryStart" >= CURRENT_DATE ' + "AND \"deliveryStart\" < CURRENT_DATE + INTERVAL '2 day'", + ), + ( + "::numeric cast inside CASE (271K calls)", + 'SELECT COUNT(*) AS total, ' + 'SUM(CASE WHEN "Status" = \'Accepted\' ' + 'THEN "DFS Procured MW"::numeric ELSE 0 END) AS accepted_mw ' + 'FROM "3ebf77d7-05df-466e-a023-dc45a90efeea" ' + "WHERE \"Zone\" = '10'", + ), + ( + "to_char with GROUP BY alias", + "SELECT to_char(\"deliveryStart\", 'YYYY-MM-DD') AS day, " + '"auctionUnit" AS unit, ' + 'sum("executedQuantity" * "clearingPrice") AS gbp ' + 'FROM "c312358e-87da-480e-88cc-d45c1dce1b41" ' + "WHERE \"technologyType\" = 'Batteries' " + 'GROUP BY day, "auctionUnit"', + ), + ( + "string_agg with ORDER BY (sqlglot warns, output still valid)", + 'SELECT COUNT(*) AS event_count, MIN("From_Local") AS earliest, ' + "string_agg(DISTINCT \"DFS Submission Time_Local\", ',' " + 'ORDER BY "DFS Submission Time_Local") AS submissions ' + 'FROM "3635fd80" ' + "WHERE \"Delivery Date\" = '2026-10-13'", + ), + ( + "trailing semicolon", + 'SELECT "ID", "Date", "Volume" ' + 'from "0b7e84ec-eded-4458-b111-d29ba4508e85";', + ), + ( + "MAX aggregate with aliased quoted column", + 'SELECT MAX("Forecast_Datetime") AS max_ts ' + 'FROM "31861619-0b86-47ba-bac2-d008a760af54"', + ), + ( + "window + ORDER BY + WHERE on timestamp literal", + 'SELECT COUNT(*) OVER () AS _count, * ' + 'FROM "832e6702-3007-4424-9146-8d35db8b2fe4" ' + "WHERE \"CreatedTime\" > '2026-04-17 21:01:00' " + 'order by "CreatedTime" DESC', + ), + ( + "LIMIT + OFFSET with multi-column projection", + 'SELECT "_id", "EntryID", "ServiceType", "AssetID", "MW", ' + '"IndicativePrice", "DispatchDateTimeStamp" ' + 'FROM "099bd868-d0af-4157-85c6-4508ddbd8a64" ' + "WHERE \"_id\" >= '944' " + 'ORDER BY "_id" ASC LIMIT 10000 OFFSET 0', + ), + ( + "date range on datetime column", + 'SELECT * FROM "6a928369-bed3-445f-af8a-69cdb2cc5089" ' + "WHERE \"Published DateTime\" >= '2026-10-01T00:00:00' " + "AND \"Published DateTime\" <= '2026-10-01T22:00:00.999Z' " + 'ORDER BY "_id" ASC', + ), + ( + "SELECT DATE() with GROUP BY on alias", + "SELECT DATE(delivery_start) AS delivery_date, product_code, " + "AVG(clearing_price_gbp_per_mwh) AS avg_price " + "FROM balancing_auction_results_2025 " + "WHERE accepted = true " + "GROUP BY delivery_date, product_code", + ), +] + + +@pytest.mark.parametrize( + "label,sql", + PRODUCTION_SAMPLES, + ids=[label for label, _ in PRODUCTION_SAMPLES], +) +def test_production_query_translates_and_parses_as_bigquery(label: str, sql: str) -> None: + """Each real-world query must survive the full qualify_table_refs + pipeline AND parse under the BigQuery dialect — i.e. BigQuery itself + would at least accept the SQL (modulo table existence).""" + qualified = qualify_table_refs(sql, project="p", dataset="d") + # Round-trip under the BigQuery dialect — this is the final check + # that BigQuery's parser would at least recognise the shape. + sqlglot.parse_one(qualified, dialect="bigquery") + + # Fully-qualified table refs are the whole point of qualify_table_refs; + # if these disappear the function has been accidentally short-circuited. + assert "`p`.`d`." in qualified + # And none of the pg-ism shapes should survive — specifically no + # `::` cast operator anywhere outside a comment and no `"ident"` + # (backticks only). + assert "::" not in qualified + assert '"' not in qualified + + +# --- spec-driven acceptance tests ------------------------------------------ +# +# One test per compatibility area in the migration spec. These are the +# shapes CKAN clients ship today and need to keep working against the +# BigQuery datastore. + + +def test_spec_ilike_rewritten_to_lower_like() -> None: + # Spec #1: ILIKE case-insensitive match. + sql = ( + 'SELECT * FROM "3ebf77d7-05df-466e-a023-dc45a90efeea" ' + "WHERE \"Registered DFS Participant\" ILIKE '%Octopus%'" + ) + out, _ = _translate(sql) + # The pattern-matching stays logically equivalent — both sides lower-cased. + assert "LOWER(`Registered DFS Participant`)" in out + assert "LIKE LOWER('%Octopus%')" in out + assert "ILIKE" not in out + + +def test_spec_current_date_minus_integer_wrapped_in_timestamp() -> None: + # Spec #2: `col > CURRENT_DATE - 1` has to become a TIMESTAMP comparison. + sql = ( + 'SELECT COUNT(*) OVER () AS _count, * ' + 'FROM "a63ab354-7e68-44c2-ad96-c6f920c30e85" ' + 'WHERE "deliveryStart" > CURRENT_DATE - 1' + ) + out, notes = _translate(sql) + # Exactly the shape the spec cites. + assert "TIMESTAMP(DATE_SUB(CURRENT_DATE, INTERVAL 1 DAY))" in out + # The two cooperating rewrites both fire. + assert any("date±integer" in n for n in notes) + assert any("TIMESTAMP(...)" in n for n in notes) + + +def test_spec_to_timestamp_epoch_to_timestamp_seconds() -> None: + # Spec #3: `to_timestamp(epoch_ms/1000)`. The spec cites TIMESTAMP_SECONDS + # as the shape; in practice BigQuery's `/` returns FLOAT64 while + # TIMESTAMP_SECONDS requires INT64, so we collapse `/1000` to the direct + # `TIMESTAMP_MILLIS(epoch_ms)` equivalent (semantically identical, no + # type coercion needed). + sql = ( + 'SELECT COUNT(*) OVER () AS _count, * ' + 'FROM "6a928369-bed3-445f-af8a-69cdb2cc5089" ' + 'WHERE "Published DateTime" >= to_timestamp(1791939450000/1000) ' + 'AND "Published DateTime" <= to_timestamp(1791939750000/1000) ' + 'ORDER BY "_id" ASC LIMIT 100' + ) + out, _ = _translate(sql) + assert "TIMESTAMP_MILLIS(1791939450000)" in out + assert "TIMESTAMP_MILLIS(1791939750000)" in out + assert "to_timestamp" not in out.lower() + + +def test_spec_pg_timestamp_cast_targets_bq_timestamp() -> None: + # Spec #4: `::timestamp` has to land on `CAST(... AS TIMESTAMP)`, not + # DATETIME — the backend creates `timestamp` columns as BQ TIMESTAMP + # and a DATETIME literal would make the comparison a type mismatch. + sql = ( + 'SELECT * FROM "13b511df-d6ec-4143-afb1-0ecc6fd19810" ' + "WHERE \"deliveryStart\" BETWEEN " + "'2026-08-29T22:00:00'::timestamp AND '2026-08-30T21:30:00'::timestamp" + ) + out, notes = _translate(sql) + assert "CAST('2026-08-29T22:00:00' AS TIMESTAMP)" in out + assert "CAST('2026-08-30T21:30:00' AS TIMESTAMP)" in out + # And the DATETIME intermediate from sqlglot is nowhere to be seen. + assert "AS DATETIME)" not in out + assert any("CAST(... AS TIMESTAMP)" in n for n in notes) + + +def test_spec_extract_epoch_from_interval_to_timestamp_diff() -> None: + # Spec #5: `EXTRACT(EPOCH FROM (a - b))` → `TIMESTAMP_DIFF(a, b, SECOND)`. + sql = ( + 'SELECT "auctionProduct" AS p, "serviceType" AS s, COUNT(*) AS n, ' + 'SUM("clearedVolume" * "clearingPrice" * ' + 'EXTRACT(EPOCH FROM ("deliveryEnd"::timestamp - "deliveryStart"::timestamp)) ' + '/ 3600) AS spend ' + 'FROM "be55ee51-b79e-47da-b71e-a0f8865d9d66" ' + 'GROUP BY 1, 2 ORDER BY spend DESC' + ) + out, notes = _translate(sql) + # The SECOND unit and the two column arguments survive, in order. + assert "TIMESTAMP_DIFF(" in out + assert "SECOND)" in out + # The source operands keep their TIMESTAMP cast too (spec #4 cooperates). + assert "CAST(`deliveryEnd` AS TIMESTAMP)" in out + assert "CAST(`deliveryStart` AS TIMESTAMP)" in out + # No EXTRACT(EPOCH ...) leaks through. + assert "EPOCH" not in out.upper() + assert any("EXTRACT(EPOCH" in n for n in notes) + + +def test_spec_extract_epoch_from_bare_column_uses_unix_seconds() -> None: + # Companion to spec #5: `EXTRACT(EPOCH FROM col)` with no subtraction + # is the "seconds since epoch for this timestamp" idiom, which BQ + # spells `UNIX_SECONDS(col)`. + sql = 'SELECT EXTRACT(EPOCH FROM "ts") FROM "t"' + out, _ = _translate(sql) + assert "UNIX_SECONDS(`ts`)" in out + assert "EXTRACT" not in out + + +def test_spec_identifier_quoting_preserves_names() -> None: + # Spec #6: identifier quoting switches from "ident" to `ident` + # without mangling the inner name (spaces, UUIDs, mixed case). + sql = ( + 'SELECT "Registered DFS Participant" ' + 'FROM "3ebf77d7-05df-466e-a023-dc45a90efeea"' + ) + out, _ = _translate(sql) + assert "`Registered DFS Participant`" in out + assert "`3ebf77d7-05df-466e-a023-dc45a90efeea`" in out + assert '"' not in out + + +# --- surgical safety-net tests for the new TIMESTAMP-wrap rewrite ---------- + + +def test_date_wrap_non_date_rhs_untouched() -> None: + # The wrap must not fire on a column vs integer / string / column + # comparison — nothing to coerce there. + for sql in ( + "SELECT * FROM t WHERE col > 5", + "SELECT * FROM t WHERE col = 'hello'", + "SELECT * FROM t WHERE a = b", + ): + out, notes = _translate(sql) + assert "TIMESTAMP(" not in out + assert not any("TIMESTAMP(...)" in n for n in notes) + + +def test_date_wrap_already_timestamp_rhs_untouched() -> None: + # CURRENT_TIMESTAMP is already TIMESTAMP-typed on BQ, so no wrap needed. + sql = "SELECT * FROM t WHERE col > CURRENT_TIMESTAMP" + out, notes = _translate(sql) + assert "TIMESTAMP(CURRENT_TIMESTAMP" not in out + assert not any("TIMESTAMP(...)" in n for n in notes) + + +def test_date_wrap_between_wraps_both_bounds() -> None: + sql = ( + "SELECT * FROM t WHERE col BETWEEN " + "CURRENT_DATE AND CURRENT_DATE + INTERVAL '2 day'" + ) + out, notes = _translate(sql) + assert "TIMESTAMP(CURRENT_DATE)" in out + assert "TIMESTAMP(CURRENT_DATE + INTERVAL '2' DAY)" in out + assert any("wrapped 2" in n for n in notes) + + +def test_extract_non_epoch_unit_untouched() -> None: + # Only the EPOCH unit is rewritten; YEAR, DAY, etc. are legitimate on BQ. + sql = 'SELECT EXTRACT(YEAR FROM "ts") FROM t' + out, notes = _translate(sql) + assert "EXTRACT(YEAR FROM `ts`)" in out + assert not any("EXTRACT(EPOCH" in n for n in notes) + + +def test_non_datetime_cast_untouched_by_retarget() -> None: + # The DATETIME→TIMESTAMP retarget must not touch INT64, STRING, … + sql = 'SELECT CAST("x" AS INT64), CAST("y" AS STRING) FROM t' + out, notes = _translate(sql) + assert "CAST(`x` AS INT64)" in out + assert "CAST(`y` AS STRING)" in out + assert not any("DATETIME" in n for n in notes) + + +# --- TIMESTAMP_SECONDS(x/1000) → TIMESTAMP_MILLIS(x) ----------------------- + + +def test_to_timestamp_ms_div_1000_collapsed_to_timestamp_millis() -> None: + # Pg's `to_timestamp(epoch_ms/1000)` would land on `TIMESTAMP_SECONDS(x/1000)` + # which BQ rejects (FLOAT64 argument, needs INT64). Collapse to the + # millisecond equivalent so the type is INT64 and semantics match. + sql = 'SELECT * FROM "t" WHERE "ts" >= to_timestamp(1790805660000/1000)' + out, notes = _translate(sql) + assert "TIMESTAMP_MILLIS(1790805660000)" in out + assert "TIMESTAMP_SECONDS" not in out + assert any("TIMESTAMP_MILLIS" in n for n in notes) + + +def test_to_timestamp_bare_seconds_stays_timestamp_seconds() -> None: + # `to_timestamp(bare_int)` is already INT64-safe; keep it as + # TIMESTAMP_SECONDS without the millis collapse. + sql = 'SELECT * FROM "t" WHERE "ts" >= to_timestamp(1790805660)' + out, notes = _translate(sql) + assert "TIMESTAMP_SECONDS(1790805660)" in out + assert not any("TIMESTAMP_MILLIS" in n for n in notes) + + +def test_timestamp_seconds_non_1000_divisor_untouched() -> None: + # Only the `/1000` idiom collapses; other divisors stay as-is (and + # become the caller's problem to make INT64-safe). + sql = "SELECT TIMESTAMP_SECONDS(x / 60) FROM t" + out, _ = _translate(sql) + assert "TIMESTAMP_SECONDS(x / 60)" in out + + +def test_timestamp_seconds_fix_skips_string_literals() -> None: + # A string literal that happens to contain the TIMESTAMP_SECONDS(x/1000) + # text must not be rewritten. + sql = "SELECT 'TIMESTAMP_SECONDS(x / 1000)' AS label FROM t" + out, notes = _translate(sql) + assert "'TIMESTAMP_SECONDS(x / 1000)'" in out + assert not any("TIMESTAMP_MILLIS" in n for n in notes) diff --git a/tests/test_analytics.py b/tests/test_analytics.py index 2ca1fbb..982d704 100644 --- a/tests/test_analytics.py +++ b/tests/test_analytics.py @@ -23,6 +23,7 @@ from datastore.core.constants import API_BASE_PREFIX, API_PREFIX, DUMP_PREFIX from datastore.infrastructure.cache import InMemoryCache from datastore.infrastructure.engines.bigquery import BigQueryBackend +from datastore.legacy.constants import LEGACY_API_PREFIX, LEGACY_DUMP_PREFIX from datastore.main import create_app from fastapi.testclient import TestClient @@ -559,6 +560,15 @@ def test_the_docs_surface_is_not_recorded(suffix: str) -> None: (f"{DUMP_PREFIX}/res-1", "datastore_dump"), (f"{API_BASE_PREFIX}/health", None), (f"{API_BASE_PREFIX}/ready", None), + # Legacy CKAN surface — recorded with the same action names + # as v2 so one event stream covers both API versions. + (f"{LEGACY_API_PREFIX}/datastore_search", "datastore_search"), + (f"{LEGACY_API_PREFIX}/datastore_search_sql", "datastore_search_sql"), + (f"{LEGACY_API_PREFIX}/datastore_info", "datastore_info"), + (f"{LEGACY_DUMP_PREFIX}/res-1", "datastore_dump"), + # Not under any tracked prefix. + ("/robots.txt", None), + ("/", None), ], ) def test_action_name_matches_the_live_routes(path: str, expected: str | None) -> None: diff --git a/tests/test_datastore_create.py b/tests/test_datastore_create.py index 96d7f5b..eb7a6a0 100644 --- a/tests/test_datastore_create.py +++ b/tests/test_datastore_create.py @@ -328,7 +328,7 @@ def test_create_with_schema_succeeds_and_returns_both_shapes( assert result["primary_key"] == ["auction_id", "product_code"] # Legacy `fields` derived from the schema, with Postgres types. assert [f["id"] for f in result["fields"]] == ["auction_id", "product_code"] - assert result["fields"][0]["type"] == "int8" + assert result["fields"][0]["type"] == "int4" assert result["fields"][1]["type"] == "text" # Schema returned verbatim (Frictionless shape). assert result["schema"]["primaryKey"] == ["auction_id", "product_code"] diff --git a/tests/test_datastore_search.py b/tests/test_datastore_search.py index 9920946..70d5771 100644 --- a/tests/test_datastore_search.py +++ b/tests/test_datastore_search.py @@ -127,23 +127,29 @@ def test_search_with_distinct(client: TestClient) -> None: # 3. Pagination + include_total -------------------------------------------- -def test_include_total_returns_total(client: TestClient) -> None: +def test_include_total_returns_total_records(client: TestClient) -> None: + """`include_total=true` surfaces the row count under the `total_records` + wire key (renamed from legacy `total` on v2 to disambiguate from + pagination's `total_pages`).""" response = client.get(SEARCH_URL, params=_params(include_total=True)) assert response.status_code == 200 result = response.json()["result"] assert "total_records" in result + assert "total" not in result # old key must not sneak back in # Placeholder engine returns 0 when include_total=True. assert result["total_records"] == 0 -def test_include_total_false_omits_total(client: TestClient) -> None: - """exclude_none keeps `total_records` off the wire when the engine returns None.""" +def test_include_total_false_omits_total_records(client: TestClient) -> None: + """exclude_none keeps `total_records` off the wire when the engine + returns None.""" response = client.get(SEARCH_URL, params=_params(include_total=False)) assert response.status_code == 200 result = response.json()["result"] assert "total_records" not in result + assert "total" not in result def test_pagination_echoes_limit_offset(client: TestClient) -> None: diff --git a/tests/test_datastore_upsert.py b/tests/test_datastore_upsert.py index ed3deba..ae44bb7 100644 --- a/tests/test_datastore_upsert.py +++ b/tests/test_datastore_upsert.py @@ -86,24 +86,28 @@ def test_include_records_echoes_records(client: TestClient) -> None: assert response.json()["result"]["records"] == payload["records"] -def test_include_total_returns_total(client: TestClient) -> None: +def test_include_total_returns_total_records(client: TestClient) -> None: + """v2 wire key is `total_records` (renamed from legacy `total`).""" response = client.post(UPSERT_URL, json=_payload(include_total=True)) assert response.status_code == 200 result = response.json()["result"] assert "total_records" in result + assert "total" not in result # BigQuery placeholder returns len(records); real engine will COUNT(*). assert result["total_records"] == 2 def test_default_omits_optional_fields(client: TestClient) -> None: - """`records` and `total_records` should not appear when their flags are False.""" + """`records` and `total_records` should not appear when their flags + are False.""" response = client.post(UPSERT_URL, json=_payload()) assert response.status_code == 200 result = response.json()["result"] assert "records" not in result assert "total_records" not in result + assert "total" not in result # 3. Records optional -------------------------------------------------------- diff --git a/tests/test_legacy_api.py b/tests/test_legacy_api.py new file mode 100644 index 0000000..6c45822 --- /dev/null +++ b/tests/test_legacy_api.py @@ -0,0 +1,894 @@ +"""End-to-end tests for the legacy CKAN compatibility endpoints. + +Mounted paths: + GET/POST /api/3/action/datastore_search + GET/POST /api/3/action/datastore_search_sql + GET/POST /api/3/action/datastore_info + GET /datastore/dump/{resource_id} + +The engine is stubbed (same pattern as `tests/test_datastore_search.py`) +with rows that *include* `_id` and `_updated_at`. Legacy response rules: + + - `_updated_at` is stripped (datastore-internal; CKAN never surfaces it). + - `_id` passes through — real CKAN exposes it and scripts filter on it. + - On `datastore_search_sql` only, a `_full_text` column is synthesised + (space-joined concat of non-null text values) so clients that read + `record["_full_text"]` on a `SELECT *` get a value in the expected + shape even though BigQuery has no Postgres tsvector equivalent. + `datastore_search` and the dump endpoints do NOT emit `_full_text`. +""" + +from __future__ import annotations + +import json +from collections.abc import Iterator +from typing import Any + +import pytest +from datastore.infrastructure.engines.base import InfoResult, SearchResult +from datastore.infrastructure.engines.bigquery import BigQueryBackend +from fastapi.testclient import TestClient + +from tests.conftest import FakeCKAN + +SEARCH_URL = "/api/3/action/datastore_search" +SEARCH_SQL_URL = "/api/3/action/datastore_search_sql" +INFO_URL = "/api/3/action/datastore_info" +DUMP_URL_TMPL = "/datastore/dump/{rid}" + +_RESOURCE_ID = "balancing_auction_results_2025" + + +# Engine side: a Frictionless schema that INCLUDES the two system columns +# so the stripping logic actually has something to strip. +_FAKE_SCHEMA: dict[str, Any] = { + "fields": [ + {"name": "_id", "type": "integer"}, + {"name": "auction_id", "type": "integer"}, + {"name": "product_code", "type": "string"}, + {"name": "clearing_price_gbp_per_mwh", "type": "number"}, + {"name": "_updated_at", "type": "datetime"}, + ] +} + +_FAKE_ROWS: list[tuple] = [ + (1, 144, "DCL", 47.82, "2026-10-20T00:00:00Z"), + (2, 145, "DCH", 51.10, "2026-10-20T00:00:01Z"), + (3, 146, "FFR", 32.40, "2026-10-20T00:00:02Z"), +] + + +def _install_mock_search( + monkeypatch: pytest.MonkeyPatch, + *, + schema: dict[str, Any] | None = None, + rows: list[tuple] | None = None, +) -> list[tuple]: + """Patch `BigQueryBackend.search` + `.search_sql` so legacy tests + exercise the stripping writers against known data.""" + schema = schema if schema is not None else _FAKE_SCHEMA + rows = rows if rows is not None else _FAKE_ROWS + consumed: list[tuple] = [] + + def lazy_records() -> Iterator[tuple]: + for r in rows: + consumed.append(r) + yield r + + def fake_search(self: BigQueryBackend, **kwargs: Any) -> SearchResult: + return SearchResult( + schema=schema, + records=lazy_records(), + total=len(rows), + records_truncated=False, + ) + + def fake_search_sql(self: BigQueryBackend, sql: str, limit: int) -> SearchResult: + return SearchResult( + schema=schema, + records=lazy_records(), + total=None, + records_truncated=False, + ) + + monkeypatch.setattr(BigQueryBackend, "search", fake_search) + monkeypatch.setattr(BigQueryBackend, "search_sql", fake_search_sql) + return consumed + + +def _install_mock_info(monkeypatch: pytest.MonkeyPatch) -> None: + def fake_info(self: BigQueryBackend, resource_id: str) -> InfoResult: + return InfoResult( + schema=_FAKE_SCHEMA, + meta={"resource_id": resource_id, "total_records": 3, "primary_key": []}, + ) + + monkeypatch.setattr(BigQueryBackend, "info", fake_info) + + +# --- datastore_search ------------------------------------------------------ + + +def test_legacy_search_basic_envelope( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + _install_mock_search(monkeypatch) + response = client.get(SEARCH_URL, params={"resource_id": _RESOURCE_ID, "limit": 10}) + + assert response.status_code == 200 + body = response.json() + assert body["success"] is True + result = body["result"] + assert result["resource_id"] == _RESOURCE_ID + assert result["limit"] == 10 + assert result["offset"] == 0 + assert result["records_format"] == "objects" + assert "_links" in result + + +def test_legacy_search_fields_shape_matches_ckan( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """`_updated_at` stripped; `_id` passes through. `_full_text` is + NOT emitted on `datastore_search` — CKAN only surfaces it via + `datastore_search_sql`'s `SELECT *` path. Each field entry carries + `id` + `type`, and (when the resource schema set one) `info` — the + same per-column metadata `datastore_info` returns, so clients don't + need a second round-trip to learn a column's title / unit / etc. + """ + _install_mock_search(monkeypatch) + response = client.get(SEARCH_URL, params={"resource_id": _RESOURCE_ID}) + assert response.status_code == 200 + + result = response.json()["result"] + field_ids = [f["id"] for f in result["fields"]] + assert "_updated_at" not in field_ids + assert "_full_text" not in field_ids + assert field_ids == [ + "_id", + "auction_id", + "product_code", + "clearing_price_gbp_per_mwh", + ] + for f in result["fields"]: + assert set(f.keys()).issubset({"id", "type", "info"}), ( + "legacy fields entries carry id, type, and optional info only" + ) + assert "id" in f and "type" in f + + +def test_legacy_search_fields_carry_info_metadata( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """`info` set at `datastore_create` time rides on each field in the + search response — same shape `datastore_info` returns, no second + round-trip needed. + """ + schema = { + "fields": [ + {"name": "_id", "type": "integer"}, + { + "name": "clearing_price_gbp_per_mwh", + "type": "number", + "title": "Clearing Price", + "unit": "GBP/MWh", + "description": "Pay-as-cleared price.", + }, + ] + } + rows: list[tuple[Any, ...]] = [(1, 47.82)] + _install_mock_search(monkeypatch, schema=schema, rows=rows) + response = client.get(SEARCH_URL, params={"resource_id": _RESOURCE_ID}) + assert response.status_code == 200 + + fields = response.json()["result"]["fields"] + priced = next(f for f in fields if f["id"] == "clearing_price_gbp_per_mwh") + assert "info" in priced + assert priced["info"]["title"] == "Clearing Price" + assert priced["info"]["unit"] == "GBP/MWh" + assert priced["info"]["description"] == "Pay-as-cleared price." + + +def test_legacy_search_records_strip_updated_at_keep_id( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """Each record keeps `_id` and drops `_updated_at`. No `_full_text` + on `datastore_search` records (search_sql only).""" + _install_mock_search(monkeypatch) + response = client.get(SEARCH_URL, params={"resource_id": _RESOURCE_ID}) + + result = response.json()["result"] + assert len(result["records"]) == 3 + for row in result["records"]: + assert "_updated_at" not in row + assert "_id" in row + assert "_full_text" not in row + assert result["records"][0] == { + "_id": 1, + "auction_id": 144, + "product_code": "DCL", + "clearing_price_gbp_per_mwh": 47.82, + } + + +def test_legacy_search_no_schema_block( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """V2 emits a `schema` sibling of `fields`; legacy must not.""" + _install_mock_search(monkeypatch) + response = client.get(SEARCH_URL, params={"resource_id": _RESOURCE_ID}) + assert "schema" not in response.json()["result"] + + +def test_legacy_search_post_matches_get( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + _install_mock_search(monkeypatch) + get_body = client.get(SEARCH_URL, params={"resource_id": _RESOURCE_ID, "limit": 5}).json() + + _install_mock_search(monkeypatch) + post_body = client.post( + SEARCH_URL, json={"resource_id": _RESOURCE_ID, "limit": 5} + ).json() + + # help URL + _links differ by HTTP method presence; blank them for the diff. + for b in (get_body, post_body): + b["help"] = "" + b["result"]["_links"] = {} + assert get_body == post_body + + +def test_legacy_search_total_included_by_default( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + _install_mock_search(monkeypatch) + result = client.get(SEARCH_URL, params={"resource_id": _RESOURCE_ID}).json()["result"] + assert result["total"] == 3 + assert result["include_total"] is True + # Real CKAN does NOT emit `total_was_truncated` — keep parity. + assert "total_was_truncated" not in result + + +# --- datastore_search_sql -------------------------------------------------- + + +def test_legacy_search_sql_no_limit_accepted( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """Real CKAN accepts SQL without LIMIT; v2 doesn't. Legacy must.""" + _install_mock_search(monkeypatch) + response = client.get( + SEARCH_SQL_URL, params={"sql": f'SELECT * FROM "{_RESOURCE_ID}"'} + ) + assert response.status_code == 200 + + +def test_legacy_search_sql_limit_is_not_capped( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """`datastore_search_sql` on legacy has NO row cap — a caller is + allowed to request `LIMIT 500000` even though + `SEARCH_RESULT_ROWS_MAX` is 32000 (which still gates + `datastore_search`). Legacy CKAN scripts assume the SQL path + returns whatever the query asks for; the engine streams row-by-row + so there's no memory risk. + """ + captured: dict[str, Any] = {} + + def fake_search_sql(self: BigQueryBackend, sql: str, limit: int) -> SearchResult: + captured["limit"] = limit + return SearchResult( + schema=_FAKE_SCHEMA, records=iter([]), total=None, records_truncated=False + ) + + monkeypatch.setattr(BigQueryBackend, "search_sql", fake_search_sql) + + response = client.get( + SEARCH_SQL_URL, + params={"sql": f'SELECT * FROM "{_RESOURCE_ID}" LIMIT 500000'}, + ) + assert response.status_code == 200 + # The engine saw the caller's LIMIT verbatim — not clamped to + # SEARCH_RESULT_ROWS_MAX (32000). + assert captured["limit"] == 500000 + + +def test_legacy_search_sql_no_limit_passes_unbounded_sentinel( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """When the caller omits LIMIT, the engine gets int32-max — not + the `SEARCH_RESULT_ROWS_MAX` cap. Confirms the omit-LIMIT path + drains the full result set rather than silently clamping.""" + captured: dict[str, Any] = {} + + def fake_search_sql(self: BigQueryBackend, sql: str, limit: int) -> SearchResult: + captured["limit"] = limit + return SearchResult( + schema=_FAKE_SCHEMA, records=iter([]), total=None, records_truncated=False + ) + + monkeypatch.setattr(BigQueryBackend, "search_sql", fake_search_sql) + + response = client.get( + SEARCH_SQL_URL, params={"sql": f'SELECT * FROM "{_RESOURCE_ID}"'} + ) + assert response.status_code == 200 + assert captured["limit"] == 2**31 - 1 + + +def test_legacy_search_sql_system_column_handling( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """SQL path: `_updated_at` stripped; `_id` passes through; and + `_full_text` is synthesised per row. Matches real CKAN's + `datastore_search_sql` which exposes the tsvector column via + `SELECT *` on a Postgres datastore.""" + _install_mock_search(monkeypatch) + response = client.get( + SEARCH_SQL_URL, + params={"sql": f'SELECT * FROM "{_RESOURCE_ID}" LIMIT 10'}, + ) + assert response.status_code == 200 + + result = response.json()["result"] + assert result["sql"].startswith("SELECT") + for row in result["records"]: + assert "_updated_at" not in row + assert "_id" in row + assert "_full_text" in row + field_ids = [f["id"] for f in result["fields"]] + assert "_updated_at" not in field_ids + assert field_ids[:2] == ["_id", "_full_text"] + full_text_field = next(f for f in result["fields"] if f["id"] == "_full_text") + assert full_text_field["type"] == "tsvector" + + +def test_legacy_search_sql_full_text_is_tsvector_shape( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """`_full_text` is a Postgres tsvector lexeme string, matching + `to_tsvector('english', …)` byte-for-byte: lowercased + stopword- + filtered + Snowball-stemmed, 1-indexed walk-order positions + (positions advance for stopwords too), alphabetically sorted. + + Row: _id=1, auction_id=144, product_code=DCL, + clearing_price_gbp_per_mwh=47.82 + After `_id` is excluded, tokens are: 144, DCL, 47, 82 — none + stopwords, none stemmable, so the output is unchanged by the + linguistic pipeline: + """ + _install_mock_search(monkeypatch) + response = client.get( + SEARCH_SQL_URL, + params={"sql": f'SELECT * FROM "{_RESOURCE_ID}" LIMIT 10'}, + ) + row = response.json()["result"]["records"][0] + assert row["_full_text"] == "'144':1 '47':3 '82':4 'dcl':2" + + +def test_legacy_full_text_matches_postgres_tsvector_byte_for_byte( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """Round-trip an actual CKAN sample from the legacy CKAN API to + prove our tsvector output is byte-identical to Postgres's.""" + schema: dict[str, Any] = { + "fields": [ + {"name": "_id", "type": "integer"}, + {"name": "Operational Date", "type": "string"}, + {"name": "Auction Type", "type": "string"}, + {"name": "Version", "type": "integer"}, + {"name": "Hourly Time Period GMT", "type": "string"}, + {"name": "Flow in MW To GB", "type": "integer"}, + {"name": "Flow in MW From GB", "type": "integer"}, + {"name": "Reason For Reduction", "type": "string"}, + ] + } + rows: list[tuple] = [ + (1, "2024-09-24", "DayAhead", 1, "22:00 to 23:00", 999, 1029, "No Restriction"), + ] + _install_mock_search(monkeypatch, schema=schema, rows=rows) + response = client.get( + SEARCH_SQL_URL, + params={"sql": f'SELECT * FROM "{_RESOURCE_ID}" LIMIT 1'}, + ) + row = response.json()["result"]["records"][0] + # Verbatim from the legacy CKAN API's `datastore_search_sql`. + assert row["_full_text"] == ( + "'-09':2 '-24':3 '00':7,10 '1':5 '1029':12 '2024':1 " + "'22':6 '23':9 '999':11 'dayahead':4 'restrict':14" + ) + + +def test_legacy_search_sql_strips_trailing_whitespace_in_echo( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """A trailing `%20` in the query string must not survive to the + echoed `sql`. Real CKAN emits the SQL with surrounding whitespace + stripped: `?sql=SELECT…%20` → `"sql": "SELECT…"`. + """ + _install_mock_search(monkeypatch) + response = client.get( + SEARCH_SQL_URL, + # URL-encoded trailing space (`%20`) — mimics the bug report. + params={"sql": f'SELECT * FROM "{_RESOURCE_ID}" '}, + ) + assert response.status_code == 200 + sql = response.json()["result"]["sql"] + assert sql == f'SELECT * FROM "{_RESOURCE_ID}"', ( + f"sql echo retained whitespace: {sql!r}" + ) + + +def test_legacy_search_sql_omits_records_truncated( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """Real CKAN's `datastore_search_sql` doesn't emit `records_truncated`.""" + _install_mock_search(monkeypatch) + response = client.get( + SEARCH_SQL_URL, + params={"sql": f'SELECT * FROM "{_RESOURCE_ID}" LIMIT 10'}, + ) + assert "records_truncated" not in response.json()["result"] + + +def test_legacy_search_sql_post_matches_get( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + sql = f'SELECT * FROM "{_RESOURCE_ID}" LIMIT 5' + + _install_mock_search(monkeypatch) + get_body = client.get(SEARCH_SQL_URL, params={"sql": sql}).json() + + _install_mock_search(monkeypatch) + post_body = client.post(SEARCH_SQL_URL, json={"sql": sql}).json() + + for b in (get_body, post_body): + b["help"] = "" + assert get_body == post_body + + +# --- datastore_info -------------------------------------------------------- + + +def test_legacy_info_ckan_shape( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """Shape matches CKAN 2.8 keys (flat `schema`, `meta.count`, + `primary_keys` list, no `fields` array, no `_id` / `_updated_at`), + but column types are the Postgres names `datastore_search` reports + (`int4`, `numeric`, `text`, ...) instead of CKAN's coarsened + `"number"` / `"date"` / `"text"` — so both endpoints share one + vocabulary.""" + _install_mock_info(monkeypatch) + response = client.get(INFO_URL, params={"resource_id": _RESOURCE_ID}) + assert response.status_code == 200 + + result = response.json()["result"] + assert set(result.keys()) == {"meta", "primary_keys", "schema"} + assert result["meta"] == {"count": 3} + assert result["primary_keys"] == [] + assert "_id" not in result["schema"] + assert "_updated_at" not in result["schema"] + assert result["schema"]["auction_id"] == "int4" + assert result["schema"]["product_code"] == "text" + assert result["schema"]["clearing_price_gbp_per_mwh"] == "numeric" + + +def test_legacy_info_post_accepts_body( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + _install_mock_info(monkeypatch) + response = client.post(INFO_URL, json={"resource_id": _RESOURCE_ID}) + assert response.status_code == 200 + assert set(response.json()["result"].keys()) == {"meta", "primary_keys", "schema"} + + +def test_legacy_info_id_alias( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """CKAN callers pass `?id=`; the schema already normalises it.""" + _install_mock_info(monkeypatch) + response = client.get(INFO_URL, params={"id": _RESOURCE_ID}) + assert response.status_code == 200 + assert set(response.json()["result"].keys()) == {"meta", "primary_keys", "schema"} + + +# --- auth / errors (legacy CKAN API status-code parity) ------------------- +# +# The legacy CKAN API promotes logical errors from the classic CKAN +# "all 200" convention back to real HTTP codes: +# +# - Not Found Error → 404 +# - Validation Error → 409 +# - Authorization Error → 403 +# +# Pinned here so a regression that masks everything as 200 (or picks a +# different code per type) fails loudly. + + +def test_legacy_search_unknown_resource_returns_404_with_error_envelope( + client: TestClient, +) -> None: + """No-field-context error: `message` + `__type` under `error`. + No `fields` sub-dict (CKAN flattens field details onto `error` itself). + """ + response = client.get(SEARCH_URL, params={"resource_id": "does-not-exist"}) + assert response.status_code == 404 + body = response.json() + assert body["success"] is False + assert body["error"]["__type"] == "Not Found Error" + assert "message" in body["error"] + assert "fields" not in body["error"] + assert "help" in body + + +def test_legacy_search_denied_key_returns_403_with_error_envelope( + client: TestClient, fake_ckan: FakeCKAN +) -> None: + fake_ckan.deny("test-token") + response = client.get(SEARCH_URL, params={"resource_id": _RESOURCE_ID}) + assert response.status_code == 403 + body = response.json() + assert body["success"] is False + assert body["error"]["__type"] == "Authorization Error" + + +def test_legacy_search_bad_query_param_returns_409_validation_error( + client: TestClient, +) -> None: + """CKAN-shape validation error: per-field details are AT THE ROOT + of the `error` dict (not nested under `fields`), matching real + CKAN's `{"error": {"": ["msg"], "__type": "Validation Error"}}`. + """ + response = client.get(SEARCH_URL, params={"resource_id": _RESOURCE_ID, "limit": "notanint"}) + assert response.status_code == 409 + body = response.json() + assert body["success"] is False + error = body["error"] + assert error["__type"] == "Validation Error" + # Per-field keys live at the error root (CKAN parity). + assert "limit" in error + assert isinstance(error["limit"], list) + # No nested `fields` sub-dict — v2 shape, not legacy. + assert "fields" not in error + + +def test_legacy_search_sql_bad_sql_returns_409( + client: TestClient, +) -> None: + """A disallowed statement is a validation error — 409 under prod. + + SQL-content errors land under `error.query` to match CKAN; a + missing `sql` param stays under `error.sql`. + """ + response = client.get(SEARCH_SQL_URL, params={"sql": "DELETE FROM foo"}) + assert response.status_code == 409 + body = response.json() + assert body["success"] is False + error = body["error"] + assert error["__type"] == "Validation Error" + assert "query" in error + assert "sql" not in error + assert "fields" not in error + + +def test_legacy_info_unknown_resource_returns_404( + client: TestClient, +) -> None: + response = client.get(INFO_URL, params={"resource_id": "does-not-exist"}) + assert response.status_code == 404 + body = response.json() + assert body["success"] is False + assert body["error"]["__type"] == "Not Found Error" + + +def test_legacy_search_post_error_returns_404( + client: TestClient, +) -> None: + """POST errors get the same status-code treatment as GET errors.""" + response = client.post(SEARCH_URL, json={"resource_id": "does-not-exist"}) + assert response.status_code == 404 + assert response.json()["error"]["__type"] == "Not Found Error" + + +def test_legacy_content_type_is_json( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """Successful responses must advertise application/json (CKAN parity).""" + _install_mock_search(monkeypatch) + response = client.get(SEARCH_URL, params={"resource_id": _RESOURCE_ID}) + assert response.headers["content-type"].startswith("application/json") + + +def test_legacy_error_content_type_is_json( + client: TestClient, +) -> None: + response = client.get(SEARCH_URL, params={"resource_id": "does-not-exist"}) + assert response.headers["content-type"].startswith("application/json") + + +# --- dump ------------------------------------------------------------------ + + +def test_legacy_dump_csv_default( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """Default (no ?format) is CSV. Dumps hide BOTH `_updated_at` and + `_id` — a published download is meant to look like the "clean" + dataset the publisher would hand over; the datastore's row-id + serial is an implementation detail that shouldn't leak into CSVs + people archive and share. `_full_text` is also absent (synthesis + is for the JSON action API only).""" + _install_mock_search(monkeypatch) + response = client.get(DUMP_URL_TMPL.format(rid=_RESOURCE_ID)) + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/csv") + assert "attachment" in response.headers["content-disposition"] + + text = response.text + lines = text.strip().split("\r\n") + header = lines[0].split(",") + assert "_updated_at" not in header + assert "_id" not in header + assert "_full_text" not in header + assert header == ["auction_id", "product_code", "clearing_price_gbp_per_mwh"] + assert len(lines) == 1 + len(_FAKE_ROWS) + assert lines[1] == "144,DCL,47.82" + + +def test_legacy_dump_tsv_format( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + _install_mock_search(monkeypatch) + response = client.get(DUMP_URL_TMPL.format(rid=_RESOURCE_ID), params={"format": "tsv"}) + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/tab-separated-values") + text = response.text + lines = text.strip().split("\r\n") + assert lines[0] == "auction_id\tproduct_code\tclearing_price_gbp_per_mwh" + assert lines[1] == "144\tDCL\t47.82" + + +def test_legacy_dump_json_format( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """JSON format is a JSON ARRAY of objects (not NDJSON). `_id` and + `_updated_at` both absent — dump-only hidden-column rule.""" + _install_mock_search(monkeypatch) + response = client.get(DUMP_URL_TMPL.format(rid=_RESOURCE_ID), params={"format": "json"}) + assert response.status_code == 200 + body = json.loads(response.content) + assert isinstance(body, list) + assert len(body) == 3 + for row in body: + assert "_updated_at" not in row + assert "_id" not in row + assert body[0] == { + "auction_id": 144, + "product_code": "DCL", + "clearing_price_gbp_per_mwh": 47.82, + } + + +def test_legacy_dump_xml_format( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + _install_mock_search(monkeypatch) + response = client.get(DUMP_URL_TMPL.format(rid=_RESOURCE_ID), params={"format": "xml"}) + assert response.status_code == 200 + assert response.headers["content-type"].startswith("application/xml") + text = response.text + assert text.startswith('') + assert "" in text and "" in text + assert text.count("") == 3 + assert "144" in text + assert "DCL" in text + # Dumps hide both `_id` and `_updated_at` — see test_legacy_dump_csv_default. + assert "<_id>" not in text + assert "<_updated_at>" not in text + + +def test_legacy_dump_bom_flag( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """`?bom=true` prepends the UTF-8 BOM — Excel-on-Windows parity.""" + _install_mock_search(monkeypatch) + without = client.get(DUMP_URL_TMPL.format(rid=_RESOURCE_ID)) + _install_mock_search(monkeypatch) + with_bom = client.get(DUMP_URL_TMPL.format(rid=_RESOURCE_ID), params={"bom": "true"}) + + assert without.content[:3] != b"\xef\xbb\xbf" + assert with_bom.content[:3] == b"\xef\xbb\xbf" + # Content after the BOM is identical byte-for-byte. + assert with_bom.content[3:] == without.content + + +def test_legacy_dump_rejects_unknown_format( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + _install_mock_search(monkeypatch) + response = client.get(DUMP_URL_TMPL.format(rid=_RESOURCE_ID), params={"format": "parquet"}) + # `parquet` is a v2 format but not a legacy one — FastAPI rejects it + # at the Literal boundary with a 422 (unprocessable entity). + assert response.status_code in (400, 422) + + +def test_legacy_dump_passes_filters_through_to_engine( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """`filters` / `q` / `sort` / `limit` land on the engine call so + `curl /datastore/dump/?filters=...` keeps working.""" + captured: dict[str, Any] = {} + + def fake_search(self: BigQueryBackend, **kwargs: Any) -> SearchResult: + captured.update(kwargs) + return SearchResult( + schema=_FAKE_SCHEMA, + records=iter(_FAKE_ROWS), + total=None, + records_truncated=False, + ) + + monkeypatch.setattr(BigQueryBackend, "search", fake_search) + + response = client.get( + DUMP_URL_TMPL.format(rid=_RESOURCE_ID), + params={ + "filters": json.dumps({"product_code": "DCL"}), + "q": "DCL", + "sort": "auction_id desc", + "limit": 50, + "offset": 10, + "distinct": "true", + }, + ) + assert response.status_code == 200 + assert captured["filters"] == {"product_code": "DCL"} + assert captured["q"] == "DCL" + assert captured["sort"] == "auction_id desc" + assert captured["limit"] == 50 + assert captured["offset"] == 10 + assert captured["distinct"] is True + + +def test_legacy_dump_unbounded_without_limit( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """Omitting `limit` passes the int32-max sentinel to the engine so + a whole-table dump drains without an explicit cap.""" + captured: dict[str, Any] = {} + + def fake_search(self: BigQueryBackend, **kwargs: Any) -> SearchResult: + captured.update(kwargs) + return SearchResult( + schema=_FAKE_SCHEMA, records=iter([]), total=None, records_truncated=False + ) + + monkeypatch.setattr(BigQueryBackend, "search", fake_search) + + client.get(DUMP_URL_TMPL.format(rid=_RESOURCE_ID)) + assert captured["limit"] == 2**31 - 1 + + +def test_legacy_dump_unknown_resource_returns_404( + client: TestClient, +) -> None: + response = client.get(DUMP_URL_TMPL.format(rid="does-not-exist")) + assert response.status_code == 404 + + +def test_legacy_dump_denied_key_returns_403( + client: TestClient, fake_ckan: FakeCKAN +) -> None: + fake_ckan.deny("test-token") + response = client.get(DUMP_URL_TMPL.format(rid=_RESOURCE_ID)) + assert response.status_code == 403 + + +# --- parity with v2 shape sanity ------------------------------------------- + + +def test_legacy_search_record_values_match_input_tuple_order( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """Row tuples are engine-column-ordered; stripping must project by + index, not by name, or values and columns would misalign.""" + _install_mock_search(monkeypatch) + result = client.get(SEARCH_URL, params={"resource_id": _RESOURCE_ID}).json()["result"] + + assert result["records"] == [ + {"_id": 1, "auction_id": 144, "product_code": "DCL", "clearing_price_gbp_per_mwh": 47.82}, + {"_id": 2, "auction_id": 145, "product_code": "DCH", "clearing_price_gbp_per_mwh": 51.10}, + {"_id": 3, "auction_id": 146, "product_code": "FFR", "clearing_price_gbp_per_mwh": 32.40}, + ] + + +def test_legacy_search_links_shape_matches_ckan( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """`_links` must be the CKAN-legacy shape: only URL strings, no + page counters. Real CKAN emits something like + {"start": "/api/3/action/datastore_search?limit=5&resource_id=...", + "next": "/api/3/action/datastore_search?offset=5&limit=5&resource_id=..."} + v2's `_links` adds `page_size`, `page`, `total_pages` — these must + not leak into the legacy envelope or old clients that treat + `_links` as a `{name: href}` map will fail. + """ + _install_mock_search(monkeypatch) + links = ( + client.get(SEARCH_URL, params={"resource_id": _RESOURCE_ID, "limit": 2}) + .json()["result"]["_links"] + ) + assert "start" in links + # total=3, limit=2 → next page exists at offset=2 + assert "next" in links + + # Only URL strings, no integer counters. + assert "page_size" not in links + assert "page" not in links + assert "total_pages" not in links + for key, val in links.items(): + assert isinstance(val, str), f"_links[{key!r}] is {type(val).__name__}, want str" + + +def test_legacy_search_links_are_path_only( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """CKAN emits `_links` as path+query only, no scheme/host — the + client appends them to the API host it already hit.""" + _install_mock_search(monkeypatch) + links = ( + client.get(SEARCH_URL, params={"resource_id": _RESOURCE_ID, "limit": 2}) + .json()["result"]["_links"] + ) + for key, href in links.items(): + assert href.startswith("/api/3/action/datastore_search?"), ( + f"_links[{key!r}] = {href!r}; CKAN emits path-only URLs" + ) + assert "://" not in href, ( + f"_links[{key!r}] = {href!r} carries a scheme; CKAN strips it" + ) + + +def test_legacy_search_links_next_always_emitted( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """`next` is present on every page, including the last.""" + _install_mock_search(monkeypatch) + links = ( + client.get(SEARCH_URL, params={"resource_id": _RESOURCE_ID, "limit": 10}) + .json()["result"]["_links"] + ) + assert "start" in links + assert "next" in links + assert "offset=10" in links["next"] + + +def test_legacy_search_links_prev_emitted_when_paging( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """`prev` appears once we've paged past offset 0.""" + _install_mock_search(monkeypatch) + links = ( + client.get( + SEARCH_URL, + params={"resource_id": _RESOURCE_ID, "limit": 2, "offset": 2}, + ) + .json()["result"]["_links"] + ) + assert "prev" in links + assert "offset=0" in links["prev"] + + +def test_legacy_search_post_json_body_round_trips_filters( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """POST bodies take `filters` as a real JSON object (not a string). + FastAPI's `Body(...)` parses it directly; the field validator is + tolerant of both — a dict passes `to_json_object` as-is.""" + _install_mock_search(monkeypatch) + response = client.post( + SEARCH_URL, + json={"resource_id": _RESOURCE_ID, "filters": json.dumps({"product_code": "DCL"})}, + ) + assert response.status_code == 200 diff --git a/tests/test_read_service.py b/tests/test_read_service.py index 3fba8fe..798ddb3 100644 --- a/tests/test_read_service.py +++ b/tests/test_read_service.py @@ -104,14 +104,18 @@ def test_limit_and_offset_round_trip() -> None: def test_include_total_true_emits_total_records() -> None: + """`include_total=True` surfaces the row count under the `total_records` + wire key (v2 rename from legacy `total`).""" body = _call(data_dict_overrides={"include_total": True}) # Placeholder engine yields total=0 when include_total=True. assert body["result"]["total_records"] == 0 + assert "total" not in body["result"] # old key must not leak back in def test_include_total_false_omits_total_records() -> None: body = _call(data_dict_overrides={"include_total": False}) assert "total_records" not in body["result"] + assert "total" not in body["result"] # --- search_datastore: format dispatch -------------------------------------