fix(batch): a response enum value the SDK did not know yet failed the whole response, so response enums now accept unknown values

.github/workflows/ci.yml

@@ -30,5 +30,11 @@ jobs:
       - name: "Lint, format check, and typecheck"
         run: make check
 
+      - name: "Generated models match the spec"
+        # Offline: codegen reads docs/openapi.yaml. Ignores the timestamp header.
+        run: |
+          make generate
+          git diff --exit-code -I '^#   timestamp:' -- src/zenrows/batch/models.py
+
       - name: "Unit tests"
         run: make test

DEVELOPMENT.md

@@ -67,6 +67,27 @@ reads it to emit the models. To refresh after a backend spec change:
 4. If the wire shape changed, update `src/zenrows/batch/client.py`
    so the facade method signatures still typecheck.
 
+## Open enums (generated)
+
+docs/openapi.yaml marks the response enums `x-extensible-enum: true`: the
+server may add values at any time. After datamodel-codegen runs,
+`make generate` calls `scripts/open_extensible_enums.py`, which matches each
+extensible schema in the spec to its generated Enum (by value set) and adds
+
+```python
+_missing_ = classmethod(open_enum_missing)
+```
+
+from `src/zenrows/batch/_open_enum.py`. An unknown value becomes a cached
+`UNKNOWN` pseudo-member keeping the raw value (serializes back verbatim,
+hashable, picklable, absent from iteration). Enums without the flag
+(request side, e.g. `JobType`) stay strict, so a typo still fails locally.
+The step is a script because datamodel-codegen does not expose schema
+extensions to enum templates.
+
+`tests/test_open_enums.py` derives both sets from the spec and fails if a
+regeneration drops the hook or applies it to a strict enum.
+
 ## Publishing
 
 ```bash

Makefile

@@ -27,6 +27,11 @@ format:
 # Regenerate the pydantic v2 models from the backend's canonical spec.
 # docs/openapi.yaml is the SDK-local copy of the spec; refresh it from the
 # backend when the API changes.
+# Open enums: scripts/open_extensible_enums.py then adds a `_missing_` hook
+# (src/zenrows/batch/_open_enum.py) to every enum the spec marks
+# `x-extensible-enum: true`, so a value the server adds later parses as an
+# UNKNOWN member keeping the raw value. Other (request-side) enums stay strict.
+# tests/test_open_enums.py fails if a regeneration drops the hook.
 # The HTTP client + facade are HAND-WRITTEN in src/zenrows/batch/client.py;
 # only the type definitions come from this command.
 generate:
@@ -48,6 +53,7 @@ generate:
 		--capitalise-enum-members \
 		--reuse-model \
 		--use-default
+	uv run python scripts/open_extensible_enums.py docs/openapi.yaml src/zenrows/batch/models.py
 
 # Regenerate the markdown API reference (docs/batch-client-reference.md) from
 # the SDK's docstrings via pydoc-markdown (ephemeral — no permanent dep). The

docs/openapi.yaml

@@ -1154,6 +1154,7 @@ components:
 
     JobStatus:
       type: string
+      x-extensible-enum: true
       enum: [open, closed, deleted]
       description: |
         - `open` — initial run still accepting `addTasks`. Only
@@ -1179,6 +1180,7 @@ components:
 
     RunTrigger:
       type: string
+      x-extensible-enum: true
       enum: [manual, scheduled]
       description: |
         What set this run in motion. Always set.
@@ -1190,6 +1192,7 @@ components:
 
     RunStatus:
       type: string
+      x-extensible-enum: true
       enum: [running, pending, completed, stopped, failed, deleted]
       description: |
         In-flight:
@@ -1214,15 +1217,18 @@ components:
 
     TaskStatus:
       type: string
+      x-extensible-enum: true
       enum: [pending, processing, successful, failed]
 
     ResultType:
       type: string
+      x-extensible-enum: true
       enum: [html, json, markdown, plaintext, pdf]
       description: Body format of a successful task result; matches the job's `format` 1:1.
 
     Format:
       type: string
+      x-extensible-enum: true
       enum: [html, json, markdown, plaintext, pdf]
       description: |
         Derived server-side from `zenrows_params` at submit time.
@@ -1567,6 +1573,7 @@ components:
             `/resume`.
         ingest_status:
           type: string
+          x-extensible-enum: true
           enum: [pending, done]
           description: |
             Present only on runs created by a large (202) submission
@@ -1580,6 +1587,7 @@ components:
         updated_at: { type: string, format: date-time }
         failure_reason:
           type: string
+          x-extensible-enum: true
           enum: [insufficient_credits, subscription_inactive]
           description: |
             Present only when `status == failed`: the account-level
@@ -2112,6 +2120,7 @@ components:
               index: { type: integer }
               reason:
                 type: string
+                x-extensible-enum: true
                 enum:
                   - malformed_url
                   - unsupported_scheme
@@ -2133,6 +2142,7 @@ components:
 
     ExportStatus:
       type: string
+      x-extensible-enum: true
       enum: [pending, running, completed, failed]
       description: |
         Lifecycle state of a results export.

pyproject.toml

@@ -67,6 +67,8 @@ dev-dependencies = [
     # OpenAPI → pydantic-v2 models. Run via `uv run make-models`
     # (Makefile recipe). Output lands at src/zenrows/batch/models.py.
     "datamodel-code-generator>=0.26",
+    # scripts/open_extensible_enums.py reads the spec (make generate).
+    "pyyaml>=6",
 ]
 
 # --- ruff ---

scripts/open_extensible_enums.py

@@ -0,0 +1,88 @@
+"""Post-generation step for `make generate`: open the extensible enums.
+
+datamodel-codegen does not pass schema extensions to enum templates, so
+this step reads the spec itself. Every schema with an `enum` list and
+`x-extensible-enum: true` (the response side) is matched to its generated
+Enum class by its set of values, and that class gets
+
+    _missing_ = classmethod(open_enum_missing)
+
+(see src/zenrows/batch/_open_enum.py). Enums without the flag (request
+side) stay strict, so a typo still fails locally. Deterministic; exits
+non-zero if a value set is both extensible and strict, or an extensible
+schema has no generated class.
+
+Usage: python scripts/open_extensible_enums.py SPEC MODELS_PY
+"""
+
+import ast
+import sys
+from pathlib import Path
+
+import yaml
+
+HOOK = (
+    "    # Open enum (x-extensible-enum): unknown values parse as UNKNOWN.\n"
+    "    _missing_ = classmethod(open_enum_missing)\n"
+)
+IMPORT = "from zenrows.batch._open_enum import open_enum_missing\n"
+
+
+def collect(node, extensible: set, strict: set) -> None:
+    if isinstance(node, dict):
+        if isinstance(node.get("enum"), list):
+            values = frozenset(node["enum"])
+            (extensible if node.get("x-extensible-enum") is True else strict).add(values)
+        for v in node.values():
+            collect(v, extensible, strict)
+    elif isinstance(node, list):
+        for v in node:
+            collect(v, extensible, strict)
+
+
+def main(spec_path: str, models_path: str) -> int:
+    extensible: set = set()
+    strict: set = set()
+    collect(yaml.safe_load(Path(spec_path).read_text()), extensible, strict)
+    if clash := extensible & strict:
+        print(f"ambiguous enum value sets (extensible and strict): {clash}", file=sys.stderr)
+        return 1
+
+    src = Path(models_path).read_text()
+    if "open_enum_missing" in src:
+        print("models already processed; regenerate first", file=sys.stderr)
+        return 1
+    lines = src.splitlines(keepends=True)
+    inserts: list[int] = []
+    matched: set = set()
+    for node in ast.parse(src).body:
+        if not isinstance(node, ast.ClassDef):
+            continue
+        if not any(isinstance(b, ast.Name) and b.id == "Enum" for b in node.bases):
+            continue
+        values = frozenset(
+            stmt.value.value
+            for stmt in node.body
+            if isinstance(stmt, ast.Assign) and isinstance(stmt.value, ast.Constant)
+        )
+        if values in extensible:
+            matched.add(values)
+            inserts.append(node.end_lineno or 0)
+    if missing := extensible - matched:
+        print(f"extensible enums with no generated class: {missing}", file=sys.stderr)
+        return 1
+
+    for idx in sorted(inserts, reverse=True):
+        lines.insert(idx, "\n" + HOOK)
+    future = next((i for i, ln in enumerate(lines) if ln.startswith("from __future__")), None)
+    if future is None:
+        print(f"{models_path}: no `from __future__` import to anchor on", file=sys.stderr)
+        return 1
+    lines.insert(future + 1, IMPORT)
+    Path(models_path).write_text("".join(lines))
+    print(f"opened {len(inserts)} extensible enums")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(*sys.argv[1:3]))

src/zenrows/batch/_open_enum.py

@@ -0,0 +1,50 @@
+"""Open (extensible) enums for the generated Batch models.
+
+The Batch API marks its response enums `x-extensible-enum: true`:
+clients must accept values the spec does not list yet. A plain `Enum`
+raises on an unknown value, so pydantic would reject the whole
+response the day the server adds one.
+
+`make generate` renders every enum in `models.py` through
+`codegen/templates/Enum.jinja2`, which wires this module's
+`open_enum_missing` in as the enum's `_missing_` hook. An unknown value
+then resolves to a pseudo-member that:
+
+- keeps the raw wire value in `.value` (so JSON serialization emits it);
+- is named `UNKNOWN` and is not part of iteration / `len()`;
+- is cached per (enum, value), so equal raw values give the same object,
+  and it compares equal only to itself;
+- is hashable and pickles back through the same lookup.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any
+
+UNKNOWN_NAME = "UNKNOWN"
+
+
+def open_enum_missing(cls: type[Enum], value: Any) -> Enum | None:
+    """`Enum._missing_` hook: return an `UNKNOWN` pseudo-member for `value`."""
+    if value is None:
+        # pydantic-core's JSON path probes `_missing_(None)` before retrying
+        # with the real input; answering it would swallow the raw value.
+        # None is never a wire enum value (nullable fields are `X | None`).
+        return None
+    try:
+        hash(value)
+    except TypeError:
+        return None  # unhashable — let Enum raise its normal ValueError
+    member = object.__new__(cls)
+    member._name_ = UNKNOWN_NAME
+    member._value_ = value
+    # Cache so `cls(value) is cls(value)`. Not added to `_member_map_`,
+    # so the pseudo-member never shows up when iterating the enum.
+    cls._value2member_map_.setdefault(value, member)
+    return cls._value2member_map_[value]
+
+
+def is_unknown(member: Enum) -> bool:
+    """True when `member` is a value the SDK's spec did not list."""
+    return member._name_ == UNKNOWN_NAME and member not in type(member).__members__.values()

src/zenrows/batch/models.py

@@ -1,8 +1,9 @@
 # generated by datamodel-codegen:
 #   filename:  openapi.yaml
-#   timestamp: 2026-08-25T14:35:19+00:00
+#   timestamp: 2026-09-30T11:15:03+00:00
 
 from __future__ import annotations
+from zenrows.batch._open_enum import open_enum_missing
 
 from enum import Enum
 from typing import Annotated, Any, Literal
@@ -41,6 +42,9 @@ class JobStatus(Enum):
     CLOSED = "closed"
     DELETED = "deleted"
 
+    # Open enum (x-extensible-enum): unknown values parse as UNKNOWN.
+    _missing_ = classmethod(open_enum_missing)
+
 
 class ScheduleState(Enum):
     """
@@ -72,6 +76,9 @@ class RunTrigger(Enum):
     MANUAL = "manual"
     SCHEDULED = "scheduled"
 
+    # Open enum (x-extensible-enum): unknown values parse as UNKNOWN.
+    _missing_ = classmethod(open_enum_missing)
+
 
 class RunStatus(Enum):
     """
@@ -104,13 +111,19 @@ class RunStatus(Enum):
     FAILED = "failed"
     DELETED = "deleted"
 
+    # Open enum (x-extensible-enum): unknown values parse as UNKNOWN.
+    _missing_ = classmethod(open_enum_missing)
+
 
 class TaskStatus(Enum):
     PENDING = "pending"
     PROCESSING = "processing"
     SUCCESSFUL = "successful"
     FAILED = "failed"
 
+    # Open enum (x-extensible-enum): unknown values parse as UNKNOWN.
+    _missing_ = classmethod(open_enum_missing)
+
 
 class ResultType(Enum):
     """
@@ -123,6 +136,9 @@ class ResultType(Enum):
     PLAINTEXT = "plaintext"
     PDF = "pdf"
 
+    # Open enum (x-extensible-enum): unknown values parse as UNKNOWN.
+    _missing_ = classmethod(open_enum_missing)
+
 
 class Format(Enum):
     """
@@ -143,6 +159,9 @@ class Format(Enum):
     PLAINTEXT = "plaintext"
     PDF = "pdf"
 
+    # Open enum (x-extensible-enum): unknown values parse as UNKNOWN.
+    _missing_ = classmethod(open_enum_missing)
+
 
 class Method(Enum):
     """
@@ -331,6 +350,9 @@ class IngestStatus(Enum):
     PENDING = "pending"
     DONE = "done"
 
+    # Open enum (x-extensible-enum): unknown values parse as UNKNOWN.
+    _missing_ = classmethod(open_enum_missing)
+
 
 class FailureReason(Enum):
     """
@@ -345,6 +367,9 @@ class FailureReason(Enum):
     INSUFFICIENT_CREDITS = "insufficient_credits"
     SUBSCRIPTION_INACTIVE = "subscription_inactive"
 
+    # Open enum (x-extensible-enum): unknown values parse as UNKNOWN.
+    _missing_ = classmethod(open_enum_missing)
+
 
 class Run(BaseModel):
     run_id: str
@@ -730,6 +755,9 @@ class Reason(Enum):
     UNKNOWN_PARAM = "unknown_param"
     INVALID_PARAM_VALUE = "invalid_param_value"
 
+    # Open enum (x-extensible-enum): unknown values parse as UNKNOWN.
+    _missing_ = classmethod(open_enum_missing)
+
 
 class InvalidTask(BaseModel):
     index: int
@@ -780,6 +808,9 @@ class ExportStatus(Enum):
     COMPLETED = "completed"
     FAILED = "failed"
 
+    # Open enum (x-extensible-enum): unknown values parse as UNKNOWN.
+    _missing_ = classmethod(open_enum_missing)
+
 
 class StartExportResponse(BaseModel):
     """