Drop /metadata download from dump API (moved to CKAN)

API.md

@@ -79,7 +79,6 @@ token (except under `anonymous`).
 | GET | `/datastore/api/v2/datastore_info` | Schema + row stats for a resource |
 | GET | `/datastore/api/v2/dump/{resource_id}` | Download a whole resource (CSV/NDJSON/Parquet) |
 | GET | `/datastore/api/v2/dump/{resource_id}/dictionary` | Download a resource's data dictionary (CSV/JSON) |
-| GET | `/datastore/api/v2/dump/{resource_id}/metadata` | Download a resource's + its dataset's metadata (CSV/JSON) |
 | GET | `/datastore/api/health` · `/datastore/api/ready` | Liveness / readiness |
 
 ---
@@ -443,35 +442,6 @@ id). Requires `read` permission on the resource.
 
 ---
 
-## `GET /datastore/api/v2/dump/{resource_id}/metadata`
-
-Download the resource's metadata together with its dataset's. `?format=csv`
-(default) or `json`. Served directly (`200`). **Requires `AUTH_TYPE=ckan`**
-(`Validation Error` otherwise) and `read` permission on the resource.
-
-| Field | Source |
-|---|---|
-| Dataset name / id / description | package `title` / `id` / `notes` |
-| Data group | package `organization.title` |
-| Dataset update frequency | package `update_frequency` |
-| Dataset created / last updated | package `metadata_created` / `metadata_modified` |
-| Licence | package `license_title` (else `license_id`) |
-| Tags | package tag names |
-| Resource name / id / format | resource `name` / `id` / `format` |
-| Resource created | resource `created` |
-| Resource last updated | resource `metadata_modified`, else `last_modified` (what the portal shows) |
-| Download URL | this service's `dump/{resource_id}` URL |
-| Total record count | datastore row count; empty when the resource has no table |
-| Resource update frequency | resource `update_frequency` |
-| Temporal coverage from / to | resource `temporal_coverage`, split on `/` |
-| Spatial coverage | resource `spatial_coverage` |
-
-The dataset is read with CKAN `package_show`. **json** is one object keyed by
-the labels above (`Tags` is a list); **csv** is a header row and one value
-row, with tags comma-joined. Named `<name>_metadata.<format>`.
-
----
-
 ## `GET /datastore/api/v2/dump/query`
 
 Download the result of a **SQL `SELECT`** as a single file — filtered

CLAUDE.md

@@ -434,7 +434,6 @@ Each endpoint takes a single `ContextDep`. The handler calls `context.authorize(
 | GET  | `/datastore/api/v2/datastore_info` | **implemented** | `DatastoreInfoRequest` | `DatastoreInfoResponse` |
 | GET  | `/datastore/api/v2/dump/{resource_id}` | **implemented** | `format=csv\|ndjson\|parquet` | 302 → GCS *or* streaming body (see §5.3) |
 | GET  | `/datastore/api/v2/dump/{resource_id}/dictionary` | **implemented** | `format=csv\|json` | 200 file — the resource's schema (`services/metadata.py`) |
-| GET  | `/datastore/api/v2/dump/{resource_id}/metadata` | **implemented** (`AUTH_TYPE=ckan`) | `format=csv\|json` | 200 file — resource + dataset metadata via CKAN `package_show` |
 
 The BigQuery engine is wired end-to-end: DDL, MERGE-based upsert, DML delete, parameterised search, native table-level metadata (the Frictionless schema + unique_key are JSON-encoded into the table's own `description` OPTION) for the schema round-trip, a row-count fast path via `INFORMATION_SCHEMA.TABLE_STORAGE`, and `EXPORT DATA`-backed dump with `table.modified`-keyed GCS caching. The DuckLake engine is the next concrete adapter — see §7.
 

datastore/analytics.py

@@ -77,7 +77,7 @@ def action_name(path: str) -> str | None:
         return "datastore_dump_query"
     if path.startswith(DUMP_PATH_PREFIX):
         document = path[len(DUMP_PATH_PREFIX) :].strip("/").split("/")[1:]
-        if document in (["dictionary"], ["metadata"]):
+        if document == ["dictionary"]:
             return f"datastore_dump_{document[0]}"
         return "datastore_dump"
     if path.startswith(ACTION_PREFIX):

datastore/api/endpoints/dump.py

@@ -1,6 +1,6 @@
 """Download endpoints: `<DUMP_PREFIX>/{resource_id}` + `<DUMP_PREFIX>/query`,
-plus a resource's data dictionary and metadata at
-`<DUMP_PREFIX>/{resource_id}/dictionary` and `…/metadata`.
+plus a resource's data dictionary at
+`<DUMP_PREFIX>/{resource_id}/dictionary`.
 
 csv / gzip / ndjson shards are composed into one GCS object, so those
 always redirect — the server never touches the bytes. Parquet can't be
@@ -35,8 +35,6 @@
     dictionary_csv,
     dictionary_document,
     json_bytes,
-    metadata_csv,
-    metadata_record,
 )
 from datastore.services.read import dump_sql_datastore
 from datastore.services.streaming import zip_archive_writer
@@ -182,7 +180,7 @@ async def dump(
     return download_response(request, urls, fmt, filename_base)
 
 
-# What a dictionary / metadata download returns, for OpenAPI: the file
+# What a dictionary download returns, for OpenAPI: the file
 # itself, served by this API (no redirect — the documents are small).
 _DOCUMENT_RESPONSES: dict[int | str, dict[str, Any]] = {
     200: {
@@ -235,28 +233,3 @@ async def dump_dictionary(
     )
     body = dictionary_csv(schema) if fmt == "csv" else json_bytes(schema)
     return document_response(body, fmt, authorized, resource_id, "data_dictionary")
-
-
-@router.get(
-    "/{resource_id}/metadata",
-    summary="Download a resource's and its dataset's metadata",
-    response_class=Response,
-    responses=_DOCUMENT_RESPONSES,
-)
-async def dump_metadata(
-    context: Context,
-    resource_id: ResourceId,
-    fmt: Annotated[DocumentFormat, Query(alias="format")] = "csv",
-):
-    """Download the resource's metadata alongside its dataset's: names,
-    ids, description, organization, update frequencies, timestamps,
-    licence, tags, format, download URL, record count, temporal and
-    spatial coverage.
-
-    `json` is one object; `csv` has a `Field` and a `Value` column with
-    one row per metadata key, tags comma-joined. Requires `AUTH_TYPE=ckan`."""
-    authorized = await context.authorize(resource_id=resource_id, permission="read")
-    authorized["resource"] = await context.fresh_resource(resource_id, authorized.get("resource"))
-    record = await metadata_record(context, resource_id, authorized)
-    body = metadata_csv(record) if fmt == "csv" else json_bytes(record)
-    return document_response(body, fmt, authorized, resource_id, "metadata")

datastore/core/constants.py

@@ -57,9 +57,8 @@
     "parquet": "application/vnd.apache.parquet",
 }
 
-# Formats for the data dictionary + metadata downloads
-# (`<DUMP_PREFIX>/{rid}/dictionary`, `<DUMP_PREFIX>/{rid}/metadata`). Small
-# documents built in memory, so the API serves the bytes itself.
+# Format for the data dictionary download (`<DUMP_PREFIX>/{rid}/dictionary`).
+# A small document built in memory, so the API serves the bytes itself.
 DocumentFormat = Literal["csv", "json"]
 DOCUMENT_MEDIA_TYPES: dict[str, str] = {
     "csv": "text/csv; charset=utf-8",

datastore/infrastructure/ckan_client.py

@@ -78,11 +78,6 @@ async def resource_show(self, *, resource_id: str) -> dict[str, Any]:
         """`/api/3/action/resource_show`. The live resource dict."""
         return await self._post_action("resource_show", {"id": resource_id})
 
-    async def package_show(self, *, package_id: str) -> dict[str, Any]:
-        """`/api/3/action/package_show`. The full dataset dict, with its
-        resources, organization, tags and scheming extras."""
-        return await self._post_action("package_show", {"id": package_id})
-
     async def resource_create(self, *, resource: dict[str, Any]) -> dict[str, Any]:
         """`/api/3/action/resource_create`. `resource` must include `package_id`."""
         if not resource.get("package_id"):

datastore/services/metadata.py

@@ -1,23 +1,20 @@
-"""Data dictionary + metadata documents for `<DUMP_PREFIX>/{rid}/…`.
+"""Data dictionary document for `<DUMP_PREFIX>/{rid}/dictionary`.
 
-Both are small, so they're built in memory and handed to the endpoint as
-bytes — no export pipeline, no GCS. CSV and JSON carry the same content;
-CSV flattens anything nested into a single cell.
+Small, so it's built in memory and handed to the endpoint as bytes — no
+export pipeline, no GCS. CSV and JSON carry the same content; CSV
+flattens anything nested into a single cell. (Resource/dataset metadata
+is not a datastore concern: it is served by CKAN.)
 """
 
 from __future__ import annotations
 
-import asyncio
 import csv
 import io
 from typing import TYPE_CHECKING, Any
 
 import orjson
 
 from datastore.core.constants import RESERVED_SYSTEM_COLUMN_NAMES
-from datastore.core.exceptions import NotFoundError, ValidationError
-from datastore.core.helper import dump_url
-from datastore.infrastructure.engines import get_datastore_engine
 from datastore.services.read import info_datastore
 
 if TYPE_CHECKING:  # type-only — no runtime import from api/
@@ -62,83 +59,10 @@ def dictionary_csv(schema: dict[str, Any]) -> bytes:
     return _csv_bytes(columns, fields)
 
 
-async def metadata_record(
-    context: RequestContext, resource_id: str, authorized: dict[str, Any]
-) -> dict[str, Any]:
-    """The resource's metadata plus its dataset's, one flat record keyed
-    by the labels the website shows.
-
-    Reads the dataset through `package_show` rather than the dict
-    `datastore_authorize` returned: that one is cached by the auth
-    provider and isn't guaranteed to carry the organization, tags or
-    scheming extras.
-    """
-    if context.config.AUTH_TYPE != "ckan" or context.ckan is None:
-        raise ValidationError("Metadata download requires AUTH_TYPE=ckan")
-
-    authorized_resource = authorized.get("resource") or {}
-    package_id = authorized_resource.get("package_id") or (authorized.get("package") or {}).get(
-        "id"
-    )
-    if not package_id:
-        raise NotFoundError(f"resource {resource_id!r} has no dataset")
-
-    package = await context.ckan.package_show(package_id=str(package_id))
-    resource = next(
-        (r for r in package.get("resources") or [] if r.get("id") == resource_id),
-        authorized_resource,
-    )
-    temporal_from, temporal_to = _split_temporal_coverage(resource.get("temporal_coverage"))
-
-    return {
-        "Dataset name": package.get("title"),
-        "Dataset id": package.get("id"),
-        "Dataset description": package.get("notes"),
-        "Data group": (package.get("organization") or {}).get("title"),
-        "Dataset update frequency": package.get("update_frequency"),
-        "Dataset created": package.get("metadata_created"),
-        "Dataset last updated": package.get("metadata_modified"),
-        "Licence": package.get("license_title") or package.get("license_id"),
-        "Tags": [t["name"] for t in package.get("tags") or [] if t.get("name")],
-        "Resource name": resource.get("name"),
-        "Resource id": resource_id,
-        "Format": resource.get("format"),
-        "Resource created": resource.get("created"),
-        # Same fallback the frontend shows as "updated", so the file and
-        # the page agree.
-        "Resource last updated": resource.get("metadata_modified") or resource.get("last_modified"),
-        "Download URL": dump_url(context.config, resource_id),
-        "Total record count": await _record_count(context, resource_id),
-        "Resource update frequency": resource.get("update_frequency"),
-        "Temporal coverage from": temporal_from,
-        "Temporal coverage to": temporal_to,
-        "Spatial coverage": resource.get("spatial_coverage"),
-    }
-
-
-def metadata_csv(record: dict[str, Any]) -> bytes:
-    """One row per metadata key: the label in the first column, its
-    value in the second."""
-    rows = [{"Field": key, "Value": value} for key, value in record.items()]
-    return _csv_bytes(["Field", "Value"], rows)
-
-
 def json_bytes(document: Any) -> bytes:
     return orjson.dumps(document, option=orjson.OPT_INDENT_2)
 
 
-async def _record_count(context: RequestContext, resource_id: str) -> int | None:
-    """Row count from the engine; None when the resource has no table
-    yet — the rest of the metadata is still worth downloading."""
-    engine = get_datastore_engine(context, mode="ro")
-    try:
-        info = await asyncio.to_thread(engine.info, resource_id=resource_id)
-    except NotFoundError:
-        return None
-    total = info.meta.get("total_records")
-    return total if isinstance(total, int) else None
-
-
 def _with_system_fields(schema: dict[str, Any], include_updated_at: bool) -> dict[str, Any]:
     """`schema` with `_id` prepended and `_updated_at` (when the engine
     adds it) appended. A schema with no fields means no table to
@@ -151,15 +75,6 @@ def _with_system_fields(schema: dict[str, Any], include_updated_at: bool) -> dic
     return {**schema, "fields": [_ID_FIELD, *user, *system_tail]}
 
 
-def _split_temporal_coverage(value: Any) -> tuple[str | None, str | None]:
-    """`YYYY-MM-DD/YYYY-MM-DD` → (from, to). The validator enforces that
-    shape, but a legacy value without the `/` lands whole in `from`."""
-    if not isinstance(value, str) or not value.strip():
-        return None, None
-    start, sep, end = value.partition("/")
-    return start.strip() or None, (end.strip() or None) if sep else None
-
-
 def _csv_bytes(columns: list[str], rows: list[dict[str, Any]]) -> bytes:
     buf = io.StringIO()
     writer = csv.writer(buf, lineterminator="\r\n")

tests/conftest.py

@@ -103,7 +103,6 @@ def __init__(self) -> None:
         self.resources: dict[str, dict[str, Any]] = {}
         self.packages: dict[str, dict[str, Any]] = {}
         self.authorize_calls = 0
-        self.package_show_calls = 0
         self.deny_keys: set[str] = set()
         # What `datastore_authorize` still reports after a resource changed,
         # i.e. the entry the provider's TTL cache would hand back.
@@ -166,17 +165,6 @@ async def resource_show(self, *, resource_id: str) -> dict[str, Any]:
             raise NotFoundError(f"resource '{resource_id}' not found")
         return existing
 
-    async def package_show(self, *, package_id: str) -> dict[str, Any]:
-        """Mirrors CKAN: the package with its resources embedded."""
-        self.package_show_calls += 1
-        if self._api_key and self._api_key in self.deny_keys:
-            raise AuthorizationError(f"key '{self._api_key}' is not allowed")
-        package = self.packages.get(package_id)
-        if package is None:
-            raise NotFoundError(f"package '{package_id}' not found")
-        resources = [r for r in self.resources.values() if r.get("package_id") == package_id]
-        return {**package, "resources": resources}
-
     async def resource_create(self, *, resource: dict[str, Any]) -> dict[str, Any]:
         self._guard()
         package_id = str(resource.get("package_id") or "")