diff --git a/.gts-spec b/.gts-spec index 1bffb45..98a16f3 160000 --- a/.gts-spec +++ b/.gts-spec @@ -1 +1 @@ -Subproject commit 1bffb45de77982a0852a351cceba5787e36bd810 +Subproject commit 98a16f3f9815dd4b852484ae7e52713d797d4dbb diff --git a/.gts-spec-version b/.gts-spec-version new file mode 100644 index 0000000..818cabd --- /dev/null +++ b/.gts-spec-version @@ -0,0 +1 @@ +v0.14.3 diff --git a/Makefile b/Makefile index 8ae4e27..1b5780b 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ $(error PYTHON must be set for local package targets (examples: venv: PYTHON=.ve endif endif -.PHONY: help py-env install build install-local uninstall-local clean dev-fmt all check fmt lint clippy mypy test security update-spec e2e coverage gts-server +.PHONY: help py-env install build install-local uninstall-local clean dev-fmt all check fmt lint clippy mypy test security update-spec verify-spec-version e2e coverage gts-server # Default target - show help .DEFAULT_GOAL := help @@ -125,8 +125,8 @@ PORT ?= 8000 gts-server: install $(PYTHON) -m gts server --host 127.0.0.1 --port $(PORT) -# Run end-to-end tests against gts-spec -e2e: install +# Run end-to-end tests against gts-spec (pinned via .gts-spec-version) +e2e: install verify-spec-version @echo "Starting server in background..." @$(PYTHON) -m gts server --port 8000 & echo $$! > .server.pid @sleep 2 @@ -144,12 +144,29 @@ security: py-env $(PYTHON) -m pip install pip-audit $(PYTHON) -m pip_audit -# Update gts-spec submodule to latest +# Spec conformance suite is pinned in .gts-spec-version (format vMAJOR.MINOR.PATCH) +# so every checkout reproduces the same e2e run, mirroring the Rust reference. +GTS_SPEC_VERSION ?= $(shell cat .gts-spec-version 2>/dev/null) + +# Check out the gts-spec submodule at the pinned .gts-spec-version tag update-spec: - git submodule update --remote .gts-spec + git submodule update --init .gts-spec + git -C .gts-spec fetch --tags --quiet origin + git -C .gts-spec checkout --quiet "$(GTS_SPEC_VERSION)" + @echo "gts-spec pinned to $(GTS_SPEC_VERSION)" + +# Fail if the checked-out gts-spec submodule does not match the pinned version +verify-spec-version: + @current="$$(git -C .gts-spec describe --tags 2>/dev/null)"; \ + if [ "$$current" != "$(GTS_SPEC_VERSION)" ]; then \ + echo "gts-spec is at '$$current' but .gts-spec-version pins '$(GTS_SPEC_VERSION)'"; \ + echo "run 'make update-spec' to sync"; \ + exit 1; \ + fi; \ + echo "gts-spec matches pinned $(GTS_SPEC_VERSION)" # Run all checks and build all: check build # Run all quality checks -check: fmt lint test e2e +check: fmt lint mypy test e2e diff --git a/README.md b/README.md index a356620..c79e83e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ A minimal, idiomatic Python library for working with **GTS** ([Global Type System](https://github.com/gts-spec/gts-spec)) identifiers and JSON/JSON Schema artifacts. -Current supported GTS spec version: `0.14.0` +Current supported GTS spec version: `0.14.3` ## Roadmap diff --git a/gts/README.md b/gts/README.md index 43f66ff..6e18b3a 100644 --- a/gts/README.md +++ b/gts/README.md @@ -2,7 +2,7 @@ Python helpers and a reference HTTP service for the [Global Type System (GTS)](https://github.com/globaltypesystem/gts-spec). The package supports GTS identifier parsing, JSON Schema-backed validation, schema compatibility and derivation checks, traits, casting, queries, file loading, a CLI, and a FastAPI application. -The package targets GTS specification v0.14.0 and requires Python 3.9 or later. +The package targets GTS specification v0.14.3 and requires Python 3.9 or later. ## Installation @@ -185,7 +185,7 @@ ops.reload_from_path("replacement-directory") ops.add_entity(content, validate=False) ops.add_entities([content_a, content_b]) -ops.add_schema(type_id, schema) +ops.add_schemas([schema_a, schema_b]) ops.extract_id(content) ops.validate_id(gts_id) ops.parse_id(gts_id) @@ -268,7 +268,7 @@ app = GtsHttpServer(ops=GtsOps()).app | `/entities/{gts_id}` | `GET` | Retrieves one entity. | | `/entities` | `POST` | Entity/schema body; optional `validate=true` runs full validation. Failed registration returns 422 and is rolled back. | | `/entities/bulk` | `POST` | JSON array of entity/schema objects. | -| `/type-schemas` | `POST` | `{"type_id": "...~", "type_schema": {...}}`. | +| `/type-schemas` | `POST` | JSON array of GTS Type Schema objects (batch); each entry's `type_id` is derived from its embedded `$id`. Returns `{"ok": ..., "results": [{"ok": ..., "type_id": ..., "error": ...}]}`. | | `/validate-id` | `GET` | `gts_id` query parameter. | | `/extract-id` | `POST` | JSON entity/schema object. | | `/parse-id` | `GET` | `gts_id` query parameter. | diff --git a/gts/openapi.json b/gts/openapi.json index e4a2b05..872e139 100644 --- a/gts/openapi.json +++ b/gts/openapi.json @@ -2,7 +2,7 @@ "openapi": "3.1.0", "info": { "title": "GTS Server", - "version": "0.14.0" + "version": "0.15.0" }, "paths": { "/entities": { @@ -11,16 +11,16 @@ "operationId": "get_entities_entities_get", "parameters": [ { + "name": "limit", + "in": "query", "required": false, "schema": { "type": "integer", - "maximum": 1000.0, - "minimum": 1.0, - "title": "Limit", - "default": 100 - }, - "name": "limit", - "in": "query" + "maximum": 1000, + "minimum": 1, + "default": 100, + "title": "Limit" + } } ], "responses": { @@ -30,6 +30,7 @@ "application/json": { "schema": { "type": "object", + "additionalProperties": true, "title": "Response Get Entities Entities Get" } } @@ -52,26 +53,46 @@ "operationId": "add_entity_entities_post", "parameters": [ { + "name": "validate", + "in": "query", "required": false, "schema": { "type": "boolean", - "title": "Validate", - "default": false - }, - "name": "validate", - "in": "query" + "default": false, + "title": "Validate" + } + }, + { + "name": "validation", + "in": "query", + "required": false, + "schema": { + "type": "boolean", + "default": false, + "title": "Validation" + } + }, + { + "name": "gts-ref-validation", + "in": "query", + "required": false, + "schema": { + "$ref": "#/components/schemas/GtsRefValidationMode", + "default": "any-valid" + } } ], "requestBody": { + "required": true, "content": { "application/json": { "schema": { "type": "object", + "additionalProperties": true, "title": "Body" } } - }, - "required": true + } }, "responses": { "200": { @@ -101,13 +122,13 @@ "operationId": "get_entity_entities__gts_id__get", "parameters": [ { + "name": "gts_id", + "in": "path", "required": true, "schema": { "type": "string", "title": "Gts Id" - }, - "name": "gts_id", - "in": "path" + } } ], "responses": { @@ -117,6 +138,7 @@ "application/json": { "schema": { "type": "object", + "additionalProperties": true, "title": "Response Get Entity Entities Gts Id Get" } } @@ -144,6 +166,7 @@ "application/json": { "schema": { "items": { + "additionalProperties": true, "type": "object" }, "type": "array", @@ -175,19 +198,19 @@ } } }, - "/schemas": { + "/type-schemas": { "post": { - "summary": "Register schema by explicit type_id", - "operationId": "add_schema_schemas_post", + "summary": "Register a batch of GTS Type Schemas", + "operationId": "add_schemas_type_schemas_post", "requestBody": { "content": { "application/json": { "schema": { - "allOf": [ - { - "$ref": "#/components/schemas/SchemaRegister" - } - ], + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array", "title": "Body" } } @@ -222,13 +245,13 @@ "operationId": "validate_id_validate_id_get", "parameters": [ { + "name": "gts_id", + "in": "query", "required": true, "schema": { "type": "string", "title": "Gts Id" - }, - "name": "gts_id", - "in": "query" + } } ], "responses": { @@ -238,6 +261,7 @@ "application/json": { "schema": { "type": "object", + "additionalProperties": true, "title": "Response Validate Id Validate Id Get" } } @@ -264,6 +288,7 @@ "content": { "application/json": { "schema": { + "additionalProperties": true, "type": "object", "title": "Body" } @@ -277,6 +302,7 @@ "content": { "application/json": { "schema": { + "additionalProperties": true, "type": "object", "title": "Response Extract Id Extract Id Post" } @@ -302,13 +328,13 @@ "operationId": "parse_parse_id_get", "parameters": [ { + "name": "gts_id", + "in": "query", "required": true, "schema": { "type": "string", "title": "Gts Id" - }, - "name": "gts_id", - "in": "query" + } } ], "responses": { @@ -318,6 +344,7 @@ "application/json": { "schema": { "type": "object", + "additionalProperties": true, "title": "Response Parse Parse Id Get" } } @@ -342,22 +369,22 @@ "operationId": "match_id_pattern_match_id_pattern_get", "parameters": [ { + "name": "candidate", + "in": "query", "required": true, "schema": { "type": "string", "title": "Candidate" - }, - "name": "candidate", - "in": "query" + } }, { + "name": "pattern", + "in": "query", "required": true, "schema": { "type": "string", "title": "Pattern" - }, - "name": "pattern", - "in": "query" + } } ], "responses": { @@ -367,6 +394,7 @@ "application/json": { "schema": { "type": "object", + "additionalProperties": true, "title": "Response Match Id Pattern Match Id Pattern Get" } } @@ -391,13 +419,13 @@ "operationId": "id_to_uuid_uuid_get", "parameters": [ { + "name": "gts_id", + "in": "query", "required": true, "schema": { "type": "string", "title": "Gts Id" - }, - "name": "gts_id", - "in": "query" + } } ], "responses": { @@ -407,6 +435,7 @@ "application/json": { "schema": { "type": "object", + "additionalProperties": true, "title": "Response Id To Uuid Uuid Get" } } @@ -429,15 +458,63 @@ "post": { "summary": "Validate instance by GTS ID", "operationId": "validate_instance_validate_instance_post", + "parameters": [ + { + "name": "gts-ref-validation", + "in": "query", + "required": false, + "schema": { + "$ref": "#/components/schemas/GtsRefValidationMode", + "default": "any-valid" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ValidateInstanceRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Validate Instance Validate Instance Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/validate-json": { + "post": { + "summary": "Validate unregistered JSON entity", + "operationId": "validate_json_validate_json_post", "requestBody": { "content": { "application/json": { "schema": { - "allOf": [ - { - "$ref": "#/components/schemas/ValidateInstanceRequest" - } - ], + "additionalProperties": true, + "type": "object", "title": "Body" } } @@ -450,8 +527,164 @@ "content": { "application/json": { "schema": { + "additionalProperties": true, "type": "object", - "title": "Response Validate Instance Validate Instance Post" + "title": "Response Validate Json Validate Json Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/validate-json/{gts_type}": { + "post": { + "summary": "Validate unregistered JSON instance against a type", + "operationId": "validate_json_as_type_validate_json__gts_type__post", + "parameters": [ + { + "name": "gts_type", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Gts Type" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Body" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Validate Json As Type Validate Json Gts Type Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/validate-type-schema": { + "post": { + "summary": "Validate that a derived GTS Type Schema correctly extends its base chain", + "operationId": "validate_type_schema_validate_type_schema_post", + "parameters": [ + { + "name": "gts-ref-validation", + "in": "query", + "required": false, + "schema": { + "$ref": "#/components/schemas/GtsRefValidationMode", + "default": "any-valid" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ValidateTypeSchemaRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Validate Type Schema Validate Type Schema Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/validate-entity": { + "post": { + "summary": "Validate entity (instance or type schema) by GTS Identifier", + "operationId": "validate_entity_validate_entity_post", + "parameters": [ + { + "name": "gts-ref-validation", + "in": "query", + "required": false, + "schema": { + "$ref": "#/components/schemas/GtsRefValidationMode", + "default": "any-valid" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ValidateEntityRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Validate Entity Validate Entity Post" } } } @@ -475,13 +708,13 @@ "operationId": "schema_graph_resolve_relationships_get", "parameters": [ { + "name": "gts_id", + "in": "query", "required": true, "schema": { "type": "string", "title": "Gts Id" - }, - "name": "gts_id", - "in": "query" + } } ], "responses": { @@ -491,6 +724,7 @@ "application/json": { "schema": { "type": "object", + "additionalProperties": true, "title": "Response Schema Graph Resolve Relationships Get" } } @@ -511,26 +745,26 @@ }, "/compatibility": { "get": { - "summary": "Check minor version compatibility", + "summary": "Check Type Schema evolution compatibility", "operationId": "compatibility_compatibility_get", "parameters": [ { + "name": "old_type_id", + "in": "query", "required": true, "schema": { "type": "string", - "title": "Old Schema Id" - }, - "name": "old_schema_id", - "in": "query" + "title": "Old Type Id" + } }, { + "name": "new_type_id", + "in": "query", "required": true, "schema": { "type": "string", - "title": "New Schema Id" - }, - "name": "new_schema_id", - "in": "query" + "title": "New Type Id" + } } ], "responses": { @@ -540,6 +774,7 @@ "application/json": { "schema": { "type": "object", + "additionalProperties": true, "title": "Response Compatibility Compatibility Get" } } @@ -566,12 +801,7 @@ "content": { "application/json": { "schema": { - "allOf": [ - { - "$ref": "#/components/schemas/CastRequest" - } - ], - "title": "Body" + "$ref": "#/components/schemas/CastRequest" } } }, @@ -583,6 +813,7 @@ "content": { "application/json": { "schema": { + "additionalProperties": true, "type": "object", "title": "Response Cast Cast Post" } @@ -608,25 +839,25 @@ "operationId": "query_query_get", "parameters": [ { + "name": "expr", + "in": "query", "required": true, "schema": { "type": "string", "title": "Expr" - }, - "name": "expr", - "in": "query" + } }, { + "name": "limit", + "in": "query", "required": false, "schema": { "type": "integer", - "maximum": 1000.0, - "minimum": 1.0, - "title": "Limit", - "default": 100 - }, - "name": "limit", - "in": "query" + "maximum": 1000, + "minimum": 1, + "default": 100, + "title": "Limit" + } } ], "responses": { @@ -636,6 +867,7 @@ "application/json": { "schema": { "type": "object", + "additionalProperties": true, "title": "Response Query Query Get" } } @@ -660,13 +892,13 @@ "operationId": "attr_attr_get", "parameters": [ { + "name": "gts_with_path", + "in": "query", "required": true, "schema": { "type": "string", "title": "Gts With Path" - }, - "name": "gts_with_path", - "in": "query" + } } ], "responses": { @@ -676,6 +908,7 @@ "application/json": { "schema": { "type": "object", + "additionalProperties": true, "title": "Response Attr Attr Get" } } @@ -703,18 +936,27 @@ "type": "string", "title": "Instance Id" }, - "to_schema_id": { + "to_type_id": { "type": "string", - "title": "To Schema Id" + "title": "To Type Id" } }, "type": "object", "required": [ "instance_id", - "to_schema_id" + "to_type_id" ], "title": "CastRequest" }, + "GtsRefValidationMode": { + "type": "string", + "enum": [ + "none", + "any-present", + "any-valid" + ], + "title": "GtsRefValidationMode" + }, "HTTPValidationError": { "properties": { "detail": { @@ -728,23 +970,33 @@ "type": "object", "title": "HTTPValidationError" }, - "SchemaRegister": { + "ValidateEntityRequest": { "properties": { - "type_id": { - "type": "string", - "title": "Type Id" + "entity_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Entity Id" }, - "schema": { - "type": "object", - "title": "Schema" + "gts_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Gts Id" } }, "type": "object", - "required": [ - "type_id", - "schema" - ], - "title": "SchemaRegister" + "title": "ValidateEntityRequest" }, "ValidateInstanceRequest": { "properties": { @@ -759,6 +1011,19 @@ ], "title": "ValidateInstanceRequest" }, + "ValidateTypeSchemaRequest": { + "properties": { + "type_id": { + "type": "string", + "title": "Type Id" + } + }, + "type": "object", + "required": [ + "type_id" + ], + "title": "ValidateTypeSchemaRequest" + }, "ValidationError": { "properties": { "loc": { @@ -794,4 +1059,4 @@ } } } -} +} \ No newline at end of file diff --git a/gts/pyproject.toml b/gts/pyproject.toml index 2d5a04a..f7116e0 100644 --- a/gts/pyproject.toml +++ b/gts/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "gts" -version = "0.14.0" +version = "0.15.0" description = "Global Type System (GTS) helpers: identifiers, parsing, validation, and operations" readme = "README.md" authors = [{ name = "GTS Community" }] diff --git a/gts/src/gts/__init__.py b/gts/src/gts/__init__.py index 3bc3e95..7f9a496 100644 --- a/gts/src/gts/__init__.py +++ b/gts/src/gts/__init__.py @@ -1,3 +1,10 @@ +from ._errors import ( + GtsConflictError, + GtsError, + GtsNotFoundError, + GtsUnresolvedRefError, + GtsValidationError, +) from .entities import ( DEFAULT_GTS_CONFIG, GtsConfig, @@ -15,6 +22,25 @@ GtsWildcard, ) from .gts_ref_validation import GtsRefValidationMode +from .ops import ( + GtsAddEntitiesResult, + GtsAddEntityResult, + GtsAddSchemaResult, + GtsAddSchemasResult, + GtsEntitiesListResult, + GtsEntityInfo, + GtsEntityValidationResult, + GtsExtractIdResult, + GtsGetEntityResult, + GtsIdMatchResult, + GtsIdParseResult, + GtsIdValidationResult, + GtsJsonValidationResult, + GtsOps, + GtsSchemaGraphResult, + GtsUuidResult, + GtsValidationResult, +) from .path_resolver import GtsPathResolver from .store import ( GtsReader, @@ -23,16 +49,38 @@ __all__ = [ "DEFAULT_GTS_CONFIG", + "GtsAddEntitiesResult", + "GtsAddEntityResult", + "GtsAddSchemaResult", + "GtsAddSchemasResult", "GtsConfig", + "GtsConflictError", + "GtsEntitiesListResult", "GtsEntity", + "GtsEntityInfo", + "GtsEntityValidationResult", + "GtsError", + "GtsExtractIdResult", "GtsFile", "GtsFileReader", + "GtsGetEntityResult", "GtsID", + "GtsIdMatchResult", + "GtsIdParseResult", "GtsIdSegment", + "GtsIdValidationResult", + "GtsJsonValidationResult", + "GtsNotFoundError", + "GtsOps", "GtsPathResolver", "GtsReader", "GtsRefValidationMode", + "GtsSchemaGraphResult", "GtsStore", + "GtsUnresolvedRefError", + "GtsUuidResult", + "GtsValidationError", + "GtsValidationResult", "GtsWildcard", "JsonEntity", # Backward compatibility aliases diff --git a/gts/src/gts/_errors.py b/gts/src/gts/_errors.py new file mode 100644 index 0000000..d8ad218 --- /dev/null +++ b/gts/src/gts/_errors.py @@ -0,0 +1,45 @@ +"""Structured GTS exception hierarchy. + +The Rust reference models store failures as a single ``StoreError`` enum whose +variants (not-found, invalid, conflict, unresolved-ref, ...) let callers react +by category rather than by matching on message text. The idiomatic Python +analogue is a small exception hierarchy rooted at :class:`GtsError`. + +:class:`GtsValidationError` also derives from the builtin :class:`ValueError`: +the library historically raised bare ``ValueError`` for validation failures and +callers/tests catch it, so keeping that base preserves backward compatibility +while letting new code catch :class:`GtsError` / :class:`GtsValidationError`. +""" + +from __future__ import annotations + + +class GtsError(Exception): + """Base class for every error raised by the GTS library.""" + + +class GtsNotFoundError(GtsError): + """An entity, schema or instance was not present in the store.""" + + +class GtsValidationError(GtsError, ValueError): + """An entity failed GTS validation. + + Subclasses :class:`ValueError` so existing ``except ValueError`` callers + keep working while new code can catch the narrower GTS categories. + """ + + +class GtsConflictError(GtsError): + """An entity id is already registered with different content.""" + + def __init__(self, entity_id: str, message: str | None = None) -> None: + self.entity_id = entity_id + super().__init__( + message + or f"Entity '{entity_id}' is already registered with different content" + ) + + +class GtsUnresolvedRefError(GtsValidationError): + """A ``$ref`` or ``x-gts-ref`` target could not be resolved.""" diff --git a/gts/src/gts/_json_validation.py b/gts/src/gts/_json_validation.py index c7e23df..b1bb05c 100644 --- a/gts/src/gts/_json_validation.py +++ b/gts/src/gts/_json_validation.py @@ -256,7 +256,8 @@ def _validate_schemas(self, store: GtsStore) -> None: if not entity.is_schema or not entity.gts_id: continue gid = entity.gts_id - if store.get(gid.id) is not entity: + stored = store.get(gid.id) + if stored is None or stored.content != entity.content: continue depth = len(gid.gts_id_segments) file = entity.file.path if entity.file else entity.label @@ -290,8 +291,9 @@ def _validate_instances(self, store: GtsStore) -> None: key = self._registry_key(entity) if key is None: continue - # Skip rejected duplicates: only validate the registered entity - if store.get(key) is not entity: + # Skip rejected duplicates: only validate content retained by the registry. + stored = store.get(key) + if stored is None or stored.content != entity.content: continue depth = self._entity_depth(entity) gts_id_str = entity.gts_id.id if entity.gts_id else "" diff --git a/gts/src/gts/_server.py b/gts/src/gts/_server.py index 5f13b89..f406341 100644 --- a/gts/src/gts/_server.py +++ b/gts/src/gts/_server.py @@ -45,32 +45,13 @@ def __init__(self, app: FastAPI, verbose: int) -> None: self.verbose = verbose async def dispatch(self, request, call_next): - if not self.verbose: - response = await call_next(request) - response.headers["connection"] = "close" - return response - start = time.time() - - # Cache request body for DEBUG logging (verbose >= 2) - cached_body = None - if self.verbose >= 2: - # Read and cache the request body - cached_body = await request.body() - - # Create a new request with the cached body - from starlette.requests import Request - - async def receive(): - return {"type": "http.request", "body": cached_body} - - request = Request(request.scope, receive) - response = await call_next(request) response.headers["connection"] = "close" - dur = (time.time() - start) * 1000.0 + if not self.verbose: + return response - # Determine status color + dur = (time.time() - start) * 1000.0 if 200 <= response.status_code < 300: status_color = Colors.GREEN elif 300 <= response.status_code < 400: @@ -78,10 +59,6 @@ async def receive(): else: status_color = Colors.RED - # Log response at INFO level (verbose >= 1). - # Neutralize CR/LF in the request-derived path to prevent log forging - # (CWE-117); ASGI percent-decodes scope["path"], so it may contain - # newlines that would otherwise inject forged log records. safe_path = request.url.path.replace("\r", "\\r").replace("\n", "\\n") logger.info( f"{Colors.CYAN}{request.method}{Colors.RESET} " @@ -89,68 +66,9 @@ async def receive(): f"{status_color}{response.status_code}{Colors.RESET} " f"in {Colors.MAGENTA}{dur:.1f}ms{Colors.RESET}" ) - - # Log request body at DEBUG level (verbose >= 2) - if self.verbose >= 2 and cached_body: - try: - import json - - body_json = json.loads(cached_body.decode("utf-8")) - body_str = json.dumps(body_json, indent=2) - logger.debug( - f"{Colors.DIM}Request body:{Colors.RESET}\n" - f"{Colors.GRAY}{body_str}{Colors.RESET}" - ) - except Exception: # noqa: BLE001 - best-effort debug logging - body_str = cached_body.decode("utf-8", errors="replace") - logger.debug( - f"{Colors.DIM}Request body (raw):{Colors.RESET}\n" - f"{Colors.GRAY}{body_str}{Colors.RESET}" - ) - - # Log response body at DEBUG level (verbose >= 2) - if self.verbose >= 2: - # Read response body - from starlette.responses import Response, StreamingResponse - - if isinstance(response, (Response, StreamingResponse)): - response_body = b"" - async for chunk in response.body_iterator: - response_body += chunk - - if response_body: - try: - import json - - body_json = json.loads(response_body.decode("utf-8")) - body_str = json.dumps(body_json, indent=2) - logger.debug( - f"{Colors.DIM}Response body:{Colors.RESET}\n" - f"{Colors.GRAY}{body_str}{Colors.RESET}" - ) - except Exception: # noqa: BLE001 - best-effort debug logging - body_str = response_body.decode("utf-8", errors="replace") - logger.debug( - f"{Colors.DIM}Response body (raw):{Colors.RESET}\n" - f"{Colors.GRAY}{body_str}{Colors.RESET}" - ) - - # Recreate response with the body - return Response( - content=response_body, - status_code=response.status_code, - headers=dict(response.headers), - media_type=response.media_type, - ) - return response -class SchemaRegister(BaseModel): - type_id: str - type_schema: dict[str, Any] - - class CastRequest(BaseModel): instance_id: str to_type_id: str @@ -193,9 +111,9 @@ def __init__( self.host = host self.port = port self.base_url = f"http://{self.host}:{self.port}" - self.app = FastAPI(title="GTS Server", version="0.14.0") + self.app = FastAPI(title="GTS Server", version="0.15.0") self.app.add_middleware( - _RequestLoggingMiddleware, + _RequestLoggingMiddleware, # type: ignore[arg-type] verbose=self.ops.verbose, ) self._register_routes() @@ -235,9 +153,9 @@ def _register_routes(self) -> None: ) app.add_api_route( "/type-schemas", - self.add_schema, + self.add_schemas, methods=["POST"], - summary="Register a GTS Type Schema under an explicit type_id", + summary="Register a batch of GTS Type Schemas", response_class=JSONResponse, ) @@ -367,10 +285,19 @@ async def add_entities( ) -> JSONResponse: return JSONResponse(self.ops.add_entities(body).to_dict()) - async def add_schema(self, body: SchemaRegister) -> JSONResponse: - result = self.ops.add_schema(body.type_id, body.type_schema) + async def add_schemas( + self, + body: list[dict[str, Any]] = Body(...), + validate: bool = Query(False), + validation: bool = Query(False), + gts_ref_validation: GtsRefValidationMode = GTS_REF_VALIDATION_QUERY, + ) -> JSONResponse: return JSONResponse( - result.to_dict(), status_code=409 if result.conflict else 200 + self.ops.add_schemas( + body, + validate=validate is True or validation is True, + gts_ref_validation=gts_ref_validation, + ).to_dict() ) async def validate_id(self, id: str = Query(..., alias="gts_id")) -> dict[str, Any]: diff --git a/gts/src/gts/compatibility.py b/gts/src/gts/compatibility.py index 76a1270..5feb5b4 100644 --- a/gts/src/gts/compatibility.py +++ b/gts/src/gts/compatibility.py @@ -244,3 +244,174 @@ def full_verdict(backward: str, forward: str) -> str: def check_accepted_set_inclusion(subset: Any, superset: Any) -> bool | None: """Shared inclusion primitive used by OP#12 derivation admission.""" return _is_subschema(subset, superset) + + +# --- diagnostics (spec sec 4.4) ------------------------------------------- +# +# The inclusion verdict tells you *whether* two definitions are compatible; a +# caller admitting a new version also wants to know *why* a direction failed and +# whether a level can still gain optional properties later. The Rust reference +# surfaces both (``SchemaComparison`` carries diagnostics plus the content model +# of every object level). We keep ``jsonsubschema`` as the inclusion primitive +# and add the same reporting on top of it. + +OPEN = "open" +CLOSED = "closed" +PARTIAL = "partially_open" + +_APPLICATOR_OBJECT_KEYWORDS = ( + "properties", + "patternProperties", + "$defs", + "definitions", +) + + +def _level_content_model(node: dict[str, Any]) -> str: + """Classify how one object level treats undeclared properties. + + - ``open``: accepts an undeclared property with any value; + - ``closed``: rejects every undeclared property; + - ``partially_open``: accepts some undeclared names or constrains their + values (schema-valued ``additionalProperties``, ``patternProperties`` or + ``propertyNames``). + """ + has_pattern = bool(node.get("patternProperties")) + has_property_names = "propertyNames" in node + additional = node.get("additionalProperties") + if additional is None: + additional_kind = "open" + else: + boolean = boolean_schema_value(additional) + if boolean is True: + additional_kind = "open" + elif boolean is False: + additional_kind = "closed" + else: + additional_kind = "schema" + + if additional_kind == "closed" and not has_pattern and not has_property_names: + return CLOSED + if additional_kind == "open" and not has_pattern and not has_property_names: + return OPEN + return PARTIAL + + +def _is_object_level(node: Any) -> bool: + if not isinstance(node, dict): + return False + if node.get("type") == "object" or "properties" in node: + return True + return any( + key in node + for key in ("additionalProperties", "patternProperties", "propertyNames") + ) + + +def classify_object_levels(schema: Any) -> list[dict[str, str]]: + """Content model of every object level of a (resolved) schema. + + Returns one entry per object level, e.g. + ``[{"path": "$", "content_model": "closed"}, ...]``. Callers use it to + report, per level, whether a later definition can add an optional property + there (only a ``closed`` level can, per spec sec 4.4). + """ + levels: list[dict[str, str]] = [] + seen: set[int] = set() + + def walk(node: Any, path: str, depth: int) -> None: + if depth > 64 or not isinstance(node, dict): + return + marker = id(node) + if marker in seen: + return + seen.add(marker) + + if _is_object_level(node): + levels.append({"path": path, "content_model": _level_content_model(node)}) + + properties = node.get("properties") + if isinstance(properties, dict): + for name, child in properties.items(): + walk(child, f"{path}.{name}", depth + 1) + + # Undeclared-property and pattern-property values are object levels of + # the instance too, so classify their schemas (a bare boolean/absent + # additionalProperties carries no nested level). + pattern_properties = node.get("patternProperties") + if isinstance(pattern_properties, dict): + for pattern, child in pattern_properties.items(): + walk(child, f"{path}.patternProperties[{pattern}]", depth + 1) + additional = node.get("additionalProperties") + if isinstance(additional, dict): + walk(additional, f"{path}.additionalProperties", depth + 1) + + # Array element schemas: `items` as a single schema, and the tuple forms + # (`prefixItems`, or `items`/`additionalItems` as a list). + items = node.get("items") + if isinstance(items, dict): + walk(items, f"{path}[]", depth + 1) + elif isinstance(items, list): + for index, child in enumerate(items): + walk(child, f"{path}[{index}]", depth + 1) + prefix_items = node.get("prefixItems") + if isinstance(prefix_items, list): + for index, child in enumerate(prefix_items): + walk(child, f"{path}[{index}]", depth + 1) + additional_items = node.get("additionalItems") + if isinstance(additional_items, dict): + walk(additional_items, f"{path}[].additionalItems", depth + 1) + + for combinator in ("allOf", "anyOf", "oneOf"): + branches = node.get(combinator) + if isinstance(branches, list): + for branch in branches: + walk(branch, path, depth + 1) + + walk(schema, "$", 0) + return levels + + +def explain_verdict( + verdict: str, *, backward: bool, differing_dialects: bool = False +) -> list[str]: + """Human-readable reasons for a non-``compatible`` directional verdict. + + Pure function of an already-computed ``verdict`` (``compatible`` / + ``incompatible`` / ``unknown``); it does not re-run the inclusion check, so + the caller pays for the accepted-instance-set comparison exactly once. + ``backward`` selects the direction (backward is ``Valid(old) subset-of + Valid(new)``, forward the reverse) for message wording; ``differing_dialects`` + distinguishes an ``unknown`` caused by incomparable dialects from one the + checker could not decide. Returns ``[]`` for a compatible verdict. + """ + if verdict == COMPATIBLE: + return [] + if verdict == UNKNOWN: + if differing_dialects: + return [ + ( + "compatibility is unknown: the two definitions declare " + "different JSON Schema dialects, so their accepted-instance " + "sets are not comparable" + ) + ] + return [ + ( + "compatibility is unknown: the accepted-instance-set inclusion " + "could not be proved or disproved for this direction" + ) + ] + if backward: + return [ + ( + "backward incompatible: Valid(old) is not a subset of Valid(new); " + "the new definition rejects instances the old definition accepts" + ) + ] + return [ + ( + "forward incompatible: Valid(new) is not a subset of Valid(old); the " + "old definition rejects instances the new definition accepts" + ) + ] diff --git a/gts/src/gts/entities.py b/gts/src/gts/entities.py index 94b61c3..5223d8d 100644 --- a/gts/src/gts/entities.py +++ b/gts/src/gts/entities.py @@ -184,6 +184,10 @@ def cast( from_schema: GtsEntity, resolver: Any | None = None, ) -> GtsEntityCastResult: + if self.gts_id is None: + raise SchemaCastError("source entity has no GTS identifier") + if to_schema.gts_id is None: + raise SchemaCastError("target schema has no GTS identifier") if ( self.is_schema and from_schema.gts_id @@ -386,8 +390,12 @@ def _calc_json_schema_id(self, cfg: GtsConfig) -> str | None: # No schema reference found for instance return None - def get_graph(self) -> dict[str, set[str]]: - refs = {} + def get_graph(self) -> dict[str, Any]: + refs: dict[str, str] = {} for r in self.gts_refs: refs[r["sourcePath"]] = r["id"] - return {"id": self.gts_id.id, "type_id": self.type_id, "refs": refs} + return { + "id": self.gts_id.id if self.gts_id else None, + "type_id": self.type_id, + "refs": refs, + } diff --git a/gts/src/gts/gts.py b/gts/src/gts/gts.py index 6a21b00..9303a4e 100644 --- a/gts/src/gts/gts.py +++ b/gts/src/gts/gts.py @@ -81,10 +81,12 @@ def __init__(self, num: int, offset: int, segment: str): self.package: str = "" self.namespace: str = "" self.type: str = "" - self.ver_major: int = 0 + self.ver_major: int | None = 0 self.ver_minor: int | None = None self.is_type: bool = False self.is_wildcard: bool = False + # Marks the synthetic UUID tail of a combined anonymous instance id. + self._is_uuid_tail: bool = False self._parse_segment_id(num, offset, segment) @@ -206,8 +208,12 @@ class GtsID: def __init__(self, id: str): raw = id.strip() - # Normalize to the canonical bare form at this boundary. - raw = _strip_scheme(raw) + # A GtsID is always the bare canonical form ("gts.…"). The "gts://" URI + # form is a JSON Schema serialization detail ($id/$ref) and is stripped + # by those URI-specific callers (e.g. entity extraction via + # ``strip_scheme``) before reaching here, mirroring the gts-rust/gts-go + # reference implementations. Accepting it here would let URI-form values + # pass validate-id/parse-id and disagree with the canonical ``id``. # Validate it's lower case if raw != raw.lower(): @@ -335,7 +341,9 @@ def to_uuid(self) -> uuid.UUID: @classmethod def is_valid(cls, s: str) -> bool: - if not _strip_scheme(s).startswith(GTS_PREFIX): + # Only the bare canonical form is a valid id; the "gts://" URI form is + # stripped by URI-specific callers before validation (see GtsID.__init__). + if not s.startswith(GTS_PREFIX): return False try: _ = cls(s) diff --git a/gts/src/gts/ops.py b/gts/src/gts/ops.py index f9709b9..500a6f9 100644 --- a/gts/src/gts/ops.py +++ b/gts/src/gts/ops.py @@ -6,7 +6,7 @@ from pathlib import Path as SysPath from typing import Any -from ._naming import looks_like_gts +from ._naming import GTS_PREFIX, GTS_URI_PREFIX, looks_like_gts, strip_scheme from .entities import DEFAULT_GTS_CONFIG, GtsConfig, GtsEntity from .files_reader import GtsFileReader from .gts import GtsID, GtsWildcard @@ -291,22 +291,34 @@ def to_dict(self) -> dict[str, Any]: @dataclass class GtsAddSchemaResult: - """Result of adding a schema to the store.""" + """Result of adding a single GTS Type Schema to the store.""" ok: bool - id: str = "" + type_id: str | None = None error: str = "" conflict: bool = False def to_dict(self) -> dict[str, Any]: - result: dict[str, Any] = {"ok": self.ok} - if self.ok: - result["id"] = self.id - else: + result: dict[str, Any] = {"ok": self.ok, "type_id": self.type_id} + if not self.ok: result["error"] = self.error return result +@dataclass +class GtsAddSchemasResult: + """Result of registering a batch of GTS Type Schemas.""" + + ok: bool + results: list[GtsAddSchemaResult] + + def to_dict(self) -> dict[str, Any]: + return { + "ok": self.ok, + "results": [r.to_dict() for r in self.results], + } + + @dataclass class GtsExtractIdResult: """Result of extracting ID information from content.""" @@ -424,39 +436,45 @@ def add_entity( is_type_schema=entity.is_schema, ) - store_key = entity.gts_id.id if entity.is_schema else entity.raw_id - previous = self.store.get(store_key) - if ( - previous - and not self.allow_entity_updates - and previous.content != entity.content - ): - return GtsAddEntityResult( - ok=False, - error=f"Entity '{store_key}' is already registered with different content", - is_type_schema=entity.is_schema, - conflict=True, - ) - self.store.register(entity) + # Both branches are guaranteed non-None by the guards above: a schema + # without gts_id and an instance without raw_id have already returned. + if entity.is_schema: + assert entity.gts_id is not None + store_key = entity.gts_id.id + else: + assert entity.raw_id is not None + store_key = entity.raw_id + with self.store.transaction(): + previous = self.store.get(store_key) + if ( + previous + and not self.allow_entity_updates + and previous.content != entity.content + ): + return GtsAddEntityResult( + ok=False, + error=f"Entity '{store_key}' is already registered with different content", + is_type_schema=entity.is_schema, + conflict=True, + ) + self.store.register(entity) - try: - if entity.is_schema: - self.store.validate_schema_basic(entity.gts_id.id) - if validate: - self.store.validate_schema(entity.gts_id.id, gts_ref_validation) - elif validate: - self.store.validate_instance( - entity.raw_id or entity.gts_id.id, gts_ref_validation + try: + if entity.is_schema: + self.store.validate_schema_basic(store_key) + if validate: + self.store.validate_schema(store_key, gts_ref_validation) + elif validate: + self.store.validate_instance(store_key, gts_ref_validation) + except Exception as e: # noqa: BLE001 - converted to a result object at API boundary + self.store.unregister(store_key) + if previous: + self.store.register(previous) + return GtsAddEntityResult( + ok=False, + error=f"Validation failed: {e!s}", + is_type_schema=entity.is_schema, ) - except Exception as e: # noqa: BLE001 - converted to a result object at API boundary - self.store.unregister(store_key) - if previous: - self.store.register(previous) - return GtsAddEntityResult( - ok=False, - error=f"Validation failed: {e!s}", - is_type_schema=entity.is_schema, - ) # Return gts_id if available, otherwise raw_id entity_id = entity.gts_id.id if entity.gts_id else (entity.raw_id or "") @@ -476,23 +494,82 @@ def add_entities( ok = all(r.ok for r in results) return GtsAddEntitiesResult(ok=ok, results=results) - def add_schema(self, type_id: str, schema: dict[str, Any]) -> GtsAddSchemaResult: + def add_schemas( + self, + schemas: builtins.list[dict[str, Any]], + validate: bool = False, + gts_ref_validation: GtsRefValidationMode = GtsRefValidationMode.ANY_VALID, + ) -> GtsAddSchemasResult: + """Register a batch of GTS Type Schemas. + + Each entry's GTS Type Identifier is derived from its embedded ``$id``; + the aggregate ``ok`` is ``True`` only when every entry registered. + ``validate`` / ``gts_ref_validation`` apply to every entry exactly as + they do on ``POST /entities``. + """ + results = [ + self.add_schema(schema, validate=validate, gts_ref_validation=gts_ref_validation) + for schema in schemas + ] + ok = all(r.ok for r in results) + return GtsAddSchemasResult(ok=ok, results=results) + + def add_schema( + self, + schema: dict[str, Any], + validate: bool = False, + gts_ref_validation: GtsRefValidationMode = GtsRefValidationMode.ANY_VALID, + ) -> GtsAddSchemaResult: + """Register a single GTS Type Schema, deriving its type_id from ``$id``. + + The embedded ``$schema`` / ``$id`` presence checks are batch-specific; + the actual registration and (when requested) validation reuse the + single-entity :meth:`add_entity` path, so each entry honors ``validate`` + / ``gts_ref_validation`` exactly like a ``POST /entities`` call. + """ + if not isinstance(schema, dict): + return GtsAddSchemaResult( + ok=False, + type_id=None, + error="GTS Type Schema entry must be a JSON object", + ) + dialect = schema.get("$schema") + if not isinstance(dialect, str) or not dialect: + return GtsAddSchemaResult( + ok=False, + type_id=None, + error="GTS Type Schema must contain a top-level $schema field", + ) + embedded_id = schema.get("$id") + if not isinstance(embedded_id, str) or not embedded_id.startswith( + GTS_URI_PREFIX + GTS_PREFIX + ): + return GtsAddSchemaResult( + ok=False, + type_id=None, + error="GTS Type Schema must contain a top-level $id in gts:// form", + ) + type_id = strip_scheme(embedded_id) try: - previous = self.store.get(type_id) - if ( - previous - and not self.allow_entity_updates - and previous.content != schema - ): - return GtsAddSchemaResult( - ok=False, - error=f"Entity '{type_id}' is already registered with different content", - conflict=True, - ) - self.store.register_schema(type_id, schema) - return GtsAddSchemaResult(ok=True, id=type_id) - except Exception as e: # noqa: BLE001 - converted to a result object at API boundary - return GtsAddSchemaResult(ok=False, error=str(e)) + GtsID.parse_type(type_id) + except ValueError: + return GtsAddSchemaResult( + ok=False, + type_id=type_id, + error=f"Invalid GTS Type Schema $id: {embedded_id}", + ) + + result = self.add_entity( + schema, validate=validate, gts_ref_validation=gts_ref_validation + ) + if result.ok: + return GtsAddSchemaResult(ok=True, type_id=type_id) + return GtsAddSchemaResult( + ok=False, + type_id=type_id, + error=result.error, + conflict=result.conflict, + ) def validate_id(self, gts_id: str) -> GtsIdValidationResult: # Check if it's a wildcard pattern (contains *) @@ -522,6 +599,7 @@ def parse_id(self, gts_id: str) -> GtsIdParseResult: # Check if it's a wildcard pattern (contains *) is_wildcard = "*" in gts_id try: + parsed: GtsID if is_wildcard: parsed = GtsWildcard(gts_id) segs = parsed.gts_id_segments @@ -644,8 +722,10 @@ def validate_json( try: if entity.is_schema: - self.store.validate_schema_content(entity.gts_id.id, content) # type: ignore[union-attr] + assert entity.gts_id is not None + self.store.validate_schema_content(entity.gts_id.id, content) else: + assert entity.type_id is not None self.store.validate_instance_content(content, entity.type_id) except Exception as error: # noqa: BLE001 - converted to a result object at API boundary error_message = str(error) diff --git a/gts/src/gts/schema_cast.py b/gts/src/gts/schema_cast.py index b197d9c..0d70bff 100644 --- a/gts/src/gts/schema_cast.py +++ b/gts/src/gts/schema_cast.py @@ -2,10 +2,11 @@ import copy import logging -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any from jsonschema import exceptions as js_exceptions +from referencing import Registry from .compatibility import UNKNOWN, dialects_differ from .gts import GtsID @@ -23,15 +24,15 @@ class GtsEntityCastResult: from_id: str = "" to_id: str = "" direction: str = "unknown" - added_properties: list[str] = None # type: ignore - removed_properties: list[str] = None # type: ignore - changed_properties: list[dict[str, str]] = None # type: ignore + added_properties: list[str] = field(default_factory=list) + removed_properties: list[str] = field(default_factory=list) + changed_properties: list[dict[str, str]] = field(default_factory=list) is_fully_compatible: bool | None = False is_backward_compatible: bool | None = False is_forward_compatible: bool | None = False - incompatibility_reasons: list[str] = None # type: ignore - backward_errors: list[str] = None # type: ignore - forward_errors: list[str] = None # type: ignore + incompatibility_reasons: list[str] = field(default_factory=list) + backward_errors: list[str] = field(default_factory=list) + forward_errors: list[str] = field(default_factory=list) casted_entity: dict[str, Any] | None = None error: str = "" # Optional explicit verdict strings ("compatible"/"incompatible"/"unknown"). @@ -39,21 +40,9 @@ class GtsEntityCastResult: backward_verdict: str | None = None forward_verdict: str | None = None full_verdict: str | None = None - - def __post_init__(self): - # Initialize list fields if None - if self.added_properties is None: - self.added_properties = [] - if self.removed_properties is None: - self.removed_properties = [] - if self.changed_properties is None: - self.changed_properties = [] - if self.incompatibility_reasons is None: - self.incompatibility_reasons = [] - if self.backward_errors is None: - self.backward_errors = [] - if self.forward_errors is None: - self.forward_errors = [] + # Content model (open/closed/partially_open) of every object level of the + # candidate ("new") schema, per spec sec 4.4. Empty for the cast op. + candidate_object_levels: list[dict[str, str]] = field(default_factory=list) def to_dict(self) -> dict[str, Any]: def _compat_str(val: bool | None) -> str: @@ -84,6 +73,7 @@ def _compat_str(val: bool | None) -> str: "backward_errors": self.backward_errors, "forward_errors": self.forward_errors, "casted_entity": self.casted_entity, + "candidate_object_levels": self.candidate_object_levels, } if self.error: result["error"] = self.error @@ -255,7 +245,7 @@ def _cast_instance_to_schema( incompatibility_reasons = [] added: list[str] = [] removed: list[str] = [] - incompatibility_reasons: list[str] = [] + incompatibility_reasons = [] if not isinstance(instance, dict): raise SchemaCastError("Instance must be an object for casting") @@ -399,7 +389,9 @@ def _validate_with_gts_id_tolerance( modified_schema = GtsEntityCastResult._remove_gts_const_constraints(schema) validator_class = validator_for(modified_schema) - if resolver is not None: + if isinstance(resolver, Registry): + validator = validator_class(modified_schema, registry=resolver) + elif resolver is not None: validator = validator_class(modified_schema, resolver=resolver) else: validator = validator_class(modified_schema) @@ -413,7 +405,7 @@ def _remove_gts_const_constraints(schema: Any) -> Any: if not isinstance(schema, dict): return schema - result = {} + result: dict[str, Any] = {} for key, value in schema.items(): if key == "const" and isinstance(value, str) and GtsID.is_valid(value): # Replace const with a type constraint instead diff --git a/gts/src/gts/schema_dialect.py b/gts/src/gts/schema_dialect.py new file mode 100644 index 0000000..70acac2 --- /dev/null +++ b/gts/src/gts/schema_dialect.py @@ -0,0 +1,99 @@ +"""JSON Schema dialects a GTS Type Schema may declare (spec sec 2.4 / 11). + +GTS admits Draft-07, Draft 2019-09 and Draft 2020-12 and nothing else. This +module is the single home for detecting the dialect a document declares and +mapping it to its canonical meta-schema URI, mirroring the Rust reference's +``schema_dialect`` module. Keeping it out of :class:`~gts.store.GtsStore` makes +the dialect rules unit-testable on their own. +""" + +from __future__ import annotations + +from typing import Any + +# Canonical (http, unfragmented) meta-schema -> short dialect label. +_SUPPORTED = { + "http://json-schema.org/draft-07/schema": "draft-07", + "http://json-schema.org/draft/2019-09/schema": "2019-09", + "http://json-schema.org/draft/2020-12/schema": "2020-12", +} + +_DIALECT_URI = { + "draft-07": "http://json-schema.org/draft-07/schema#", + "2019-09": "https://json-schema.org/draft/2019-09/schema", + "2020-12": "https://json-schema.org/draft/2020-12/schema", +} + + +def document_dialect(schema: dict[str, Any]) -> str: + """Short label of the dialect ``schema`` declares in its ``$schema``. + + The ``http``/``https`` spellings and a trailing ``#`` fragment are treated + as equivalent; anything outside the supported set is refused. + + Raises: + ValueError: if no supported dialect is declared. + """ + dialect = schema.get("$schema") + if not isinstance(dialect, str) or not dialect: + raise ValueError("$schema must declare a supported JSON Schema dialect") + normalized = dialect.removesuffix("#").lower().replace("https://", "http://", 1) + try: + return _SUPPORTED[normalized] + except KeyError as error: + raise ValueError(f"Unsupported JSON Schema dialect: {dialect}") from error + + +def dialect_uri(schema: dict[str, Any]) -> str: + """Canonical meta-schema URI for the dialect ``schema`` declares.""" + return _DIALECT_URI[document_dialect(schema)] + + +def supports_ref_siblings(schema: Any) -> bool: + """Whether the declared dialect evaluates keywords alongside ``$ref``. + + Draft 2019-09 and 2020-12 do; Draft-07 ignores ``$ref`` siblings. + """ + dialect = schema.get("$schema") if isinstance(schema, dict) else None + return isinstance(dialect, str) and ( + "/draft/2019-09/" in dialect or "/draft/2020-12/" in dialect + ) + + +def check_subschemas(schema: dict[str, Any]) -> None: + """Reject a nested subschema that switches JSON Schema dialect. + + JSON Schema lets any subschema restate ``$schema`` and switch dialect, but + GTS reads every part of a type under the single dialect its top-level + ``$schema`` selects (spec sec 11). This mirrors the Rust reference's + ``schema_dialect::check_subschemas``: a nested ``$schema`` may only restate + the document's own dialect. An unrecognized nested ``$schema`` is left to + the meta-schema check rather than reported here. + + Raises: + ValueError: on the first subschema that changes dialect, by location. + """ + # Imported lazily to avoid any import-order coupling with schema_validation. + from .schema_validation import iter_schema_nodes + + root_dialect = document_dialect(schema) + for node, path in iter_schema_nodes(schema): + if not path: + continue # the document root defines the dialect + # Trait schemas are dialect-checked by the traits subsystem, which + # reports a more specific "differs from host dialect" message. + if any(segment == "x-gts-traits-schema" for segment in path.split("/")): + continue + declared = node.get("$schema") + if not isinstance(declared, str) or not declared: + continue + try: + nested_dialect = document_dialect(node) + except ValueError: + continue # not a recognized dialect; the meta-schema check owns it + if nested_dialect != root_dialect: + raise ValueError( + f"subschema at '{path}' declares dialect {nested_dialect} but the " + f"type is read under {root_dialect}; a subschema must not change " + "JSON Schema dialect" + ) diff --git a/gts/src/gts/schema_resolver.py b/gts/src/gts/schema_resolver.py new file mode 100644 index 0000000..594e85a --- /dev/null +++ b/gts/src/gts/schema_resolver.py @@ -0,0 +1,103 @@ +"""Inlining of GTS ``$ref`` targets (spec sec 4.3). + +Extracted from :class:`~gts.store.GtsStore` so reference resolution is testable +without a full store. Resolution takes a ``provider`` callable that returns a +schema's content by GTS id (or raises :class:`KeyError` when absent) — the +Python analogue of the Rust reference's ``SchemaProvider`` trait. + +References are inlined recursively so a schema reached through an intermediate +is fully expanded. Cyclic references are left unresolved on purpose: the +surviving ``$ref`` makes the effective schema unprovable, which is the intended +admission failure. +""" + +from __future__ import annotations + +import copy +from collections.abc import Callable +from typing import Any + +from .gts import GtsRef +from .schema_dialect import supports_ref_siblings + +# A ``$ref`` provider maps a bare GTS id to its schema content, raising +# ``KeyError`` when the id is unknown. +SchemaProvider = Callable[[str], dict] + +MAX_SCHEMA_REF_EXPANSIONS = 10_000 + + +def resolve_schema_refs(schema: Any, provider: SchemaProvider) -> Any: + """Return ``schema`` with external ``$ref`` targets inlined via ``provider``.""" + return inline_refs( + copy.deepcopy(schema), set(), supports_ref_siblings(schema), provider, [0] + ) + + +def inline_refs( + node: Any, + seen: set[str], + supports_ref_siblings_flag: bool, + provider: SchemaProvider, + expansions: list[int] | None = None, +) -> Any: + """Recursively inline ``$ref`` references, guarding against cycles.""" + expansions = expansions if expansions is not None else [0] + if isinstance(node, dict): + ref_uri = node.get("$ref") + if isinstance(ref_uri, str): + ref = GtsRef.parse(ref_uri) + # Local (#/...) refs are resolved by JSON Schema itself; only + # external targets are inlined from the provider. + ref_id = None if ref.is_local else ref.target_id + if ref_id is not None: + if ref_id in seen: + # Cycle detected: leave the $ref unresolved. + return node + expansions[0] += 1 + if expansions[0] > MAX_SCHEMA_REF_EXPANSIONS: + raise ValueError( + f"schema reference expansion exceeds limit of " + f"{MAX_SCHEMA_REF_EXPANSIONS}" + ) + try: + ref_schema = provider(ref_id) + except KeyError: + return node # Leave unresolved + + resolved = inline_refs( + copy.deepcopy(ref_schema), + seen | {ref_id}, + supports_ref_siblings(ref_schema), + provider, + expansions, + ) + if supports_ref_siblings_flag and len(node) > 1: + siblings = { + key: value for key, value in node.items() if key != "$ref" + } + return { + "allOf": [ + resolved, + inline_refs( + siblings, + seen, + supports_ref_siblings_flag, + provider, + expansions, + ), + ] + } + return resolved + return { + key: inline_refs( + value, seen, supports_ref_siblings_flag, provider, expansions + ) + for key, value in node.items() + } + if isinstance(node, list): + return [ + inline_refs(item, seen, supports_ref_siblings_flag, provider, expansions) + for item in node + ] + return node diff --git a/gts/src/gts/store.py b/gts/src/gts/store.py index f8c601e..4eb24ad 100644 --- a/gts/src/gts/store.py +++ b/gts/src/gts/store.py @@ -1,23 +1,41 @@ from __future__ import annotations +import copy import logging +import threading import uuid from abc import ABC, abstractmethod +from collections import deque from collections.abc import Iterator +from contextlib import contextmanager from typing import Any -from jsonschema import RefResolver from referencing import Registry, Resource from referencing.jsonschema import DRAFT202012 -from . import compatibility, derivation, traits -from ._naming import looks_like_gts, strip_scheme, with_scheme +from . import compatibility, derivation, schema_dialect, schema_resolver, traits +from ._errors import ( + GtsNotFoundError, + GtsUnresolvedRefError, + GtsValidationError, +) +from ._json_pointer import resolve as resolve_json_pointer +from ._naming import ( + GTS_PREFIX, + GTS_URI_PREFIX, + looks_like_gts, + strip_scheme, + with_scheme, +) from .entities import GtsEntity from .gts import GtsID, GtsRef, GtsWildcard from .gts_ref_validation import GtsRefValidationMode from .schema_cast import GtsEntityCastResult + +# Re-exported for backward compatibility; the limit now lives in schema_resolver. +from .schema_resolver import MAX_SCHEMA_REF_EXPANSIONS # noqa: F401 from .schema_validation import FORMAT_CHECKER, iter_schema_nodes, validator_for -from .x_gts_ref import XGtsRefValidator, _without_x_gts_ref +from .x_gts_ref import XGtsRefValidator, _without_x_gts_ref, extended_validator_for logger = logging.getLogger(__name__) @@ -29,7 +47,7 @@ def _require_schema_id(value: str) -> GtsID: raise ValueError(f"ID '{value}' is not a schema (must end with '~')") from error -class StoreGtsObjectNotFound(Exception): +class StoreGtsObjectNotFound(GtsNotFoundError): """Exception raised when a GTS entity is not found in the store.""" def __init__(self, entity_id: str): @@ -37,7 +55,7 @@ def __init__(self, entity_id: str): self.entity_id = entity_id -class StoreGtsSchemaNotFound(Exception): +class StoreGtsSchemaNotFound(GtsNotFoundError): """Exception raised when a GTS schema is not found in the store.""" def __init__(self, entity_id: str): @@ -45,7 +63,7 @@ def __init__(self, entity_id: str): self.entity_id = entity_id -class StoreGtsEntityNotFound(Exception): +class StoreGtsEntityNotFound(GtsNotFoundError): """Exception raised when a GTS entity is not found in the store.""" def __init__(self, entity_id: str): @@ -53,7 +71,7 @@ def __init__(self, entity_id: str): self.entity_id = entity_id -class StoreGtsSchemaForInstanceNotFound(Exception): +class StoreGtsSchemaForInstanceNotFound(GtsNotFoundError): """Exception raised when a GTS schema for an instance is not found in the store.""" def __init__(self, entity_id: str): @@ -63,7 +81,7 @@ def __init__(self, entity_id: str): self.entity_id = entity_id -class StoreGtsCastFromSchemaNotAllowed(Exception): +class StoreGtsCastFromSchemaNotAllowed(GtsValidationError): """Exception raised when attempting to cast from a schema ID.""" def __init__(self, from_id: str): @@ -113,7 +131,7 @@ def to_dict(self) -> dict[str, Any]: class GtsStore: - def __init__(self, reader: GtsReader) -> None: + def __init__(self, reader: GtsReader | None = None) -> None: """ Initialize GtsStore with an optional GtsReader. @@ -122,6 +140,8 @@ def __init__(self, reader: GtsReader) -> None: """ self._by_id: dict[str, GtsEntity] = {} self._reader = reader + self._lock = threading.RLock() + self._reference_registry: Registry | None = None # Populate entities from reader if provided if self._reader: @@ -136,7 +156,11 @@ def _populate_from_reader(self) -> None: for entity in self._reader: if entity.gts_id and entity.gts_id.id: - self._by_id[entity.gts_id.id] = entity + self._by_id[entity.gts_id.id] = copy.deepcopy(entity) + self._reference_registry = None + + def _invalidate_reference_registry(self) -> None: + self._reference_registry = None def register(self, entity: GtsEntity) -> None: """Register a GtsEntity in the store. @@ -147,21 +171,33 @@ def register(self, entity: GtsEntity) -> None: # Instances should remain addressable by the id value they carry. # For plain UUID anonymous instances, `gts_id` may be inferred from # the `type` field while `raw_id` is the UUID we must look up by. - if not entity.is_schema and entity.raw_id: - self._by_id[entity.raw_id] = entity - return + stored = copy.deepcopy(entity) + with self._lock: + if not stored.is_schema and stored.raw_id: + self._by_id[stored.raw_id] = stored + return - if entity.gts_id and entity.gts_id.id: - self._by_id[entity.gts_id.id] = entity - elif entity.raw_id: - # Allow non-GTS entities with raw_id (e.g., UUIDs or simple strings) - self._by_id[entity.raw_id] = entity - else: - raise ValueError("Entity must have a valid gts_id or raw_id") + if stored.gts_id and stored.gts_id.id: + self._by_id[stored.gts_id.id] = stored + elif stored.raw_id: + # Allow non-GTS entities with raw_id (e.g., UUIDs or simple strings) + self._by_id[stored.raw_id] = stored + else: + raise ValueError("Entity must have a valid gts_id or raw_id") + if stored.is_schema: + self._invalidate_reference_registry() def unregister(self, entity_id: str) -> None: """Remove an entity from the in-memory registry if it is present.""" - self._by_id.pop(entity_id, None) + with self._lock: + removed = self._by_id.pop(entity_id, None) + if removed is not None and removed.is_schema: + self._invalidate_reference_registry() + + @contextmanager + def transaction(self): + with self._lock: + yield def register_schema(self, type_id: str, schema: dict[str, Any]) -> None: """ @@ -169,8 +205,26 @@ def register_schema(self, type_id: str, schema: dict[str, Any]) -> None: Creates a JsonEntity from the schema dict. """ gts_id = GtsID.parse_type(type_id) - entity = GtsEntity(content=schema, gts_id=gts_id, is_schema=True) - self._by_id[gts_id.id] = entity + dialect = schema.get("$schema") + if not isinstance(dialect, str) or not dialect: + raise ValueError("Type Schema must contain a top-level $schema field") + embedded_id = schema.get("$id") + if not isinstance(embedded_id, str) or not embedded_id.startswith( + GTS_URI_PREFIX + GTS_PREFIX + ): + raise ValueError("Type Schema must contain a top-level $id in gts:// form") + try: + normalized_id = GtsID.parse_type(strip_scheme(embedded_id)) + except ValueError as error: + raise ValueError(f"Invalid GTS Type Schema $id: {embedded_id}") from error + if normalized_id.id != gts_id.id: + raise ValueError( + f"Embedded $id '{embedded_id}' must match external type_id '{type_id}'" + ) + entity = GtsEntity(content=copy.deepcopy(schema), gts_id=gts_id, is_schema=True) + with self._lock: + self._by_id[gts_id.id] = entity + self._invalidate_reference_registry() def get(self, entity_id: str) -> GtsEntity | None: """ @@ -183,18 +237,21 @@ def get(self, entity_id: str) -> GtsEntity | None: scheme themselves. """ entity_id = strip_scheme(entity_id) - # Check cache first - if entity_id in self._by_id: - return self._by_id[entity_id] - - # Try to fetch from reader - if self._reader: - entity = self._reader.read_by_id(entity_id) - if entity: - self._by_id[entity_id] = entity - return entity - - return None + with self._lock: + entity = self._by_id.get(entity_id) + if entity is not None: + return copy.deepcopy(entity) + + if self._reader: + entity = self._reader.read_by_id(entity_id) + if entity: + stored = copy.deepcopy(entity) + self._by_id[entity_id] = stored + if stored.is_schema: + self._invalidate_reference_registry() + return copy.deepcopy(stored) + + return None def get_schema_content(self, type_id: str) -> dict[str, Any]: """Get schema content as dict (legacy method for backward compatibility).""" @@ -203,45 +260,36 @@ def get_schema_content(self, type_id: str) -> dict[str, Any]: return entity.content raise KeyError(f"Schema not found: {type_id}") - def _create_ref_resolver(self, schema: dict[str, Any]) -> RefResolver: - """Create a custom RefResolver that can resolve GTS ID references from the store.""" - - def resolve_gts_ref(uri: str) -> dict[str, Any]: - """Resolve a GTS ID reference to its schema content. - - ``get_schema_content`` normalizes the ``gts://`` scheme internally. - """ - try: - return self.get_schema_content(uri) - except KeyError as e: - raise ValueError(f"Unresolvable: {strip_scheme(uri)}") from e - - # Create a store dict that maps GTS IDs to their schema content - store = {} - for entity_id, entity in self._by_id.items(): - if entity.is_schema and isinstance(entity.content, dict): - store[entity_id] = entity.content - - # Create RefResolver with custom handlers - # Issue #32: Support "gts" scheme - handlers = {"": resolve_gts_ref, "gts": resolve_gts_ref} - resolver = RefResolver.from_schema(schema, store=store, handlers=handlers) - return resolver - def _create_reference_registry(self) -> Registry: - registry = Registry() - for entity_id, entity in self._by_id.items(): - if entity.is_schema and isinstance(entity.content, dict): - resource = Resource.from_contents( - _without_x_gts_ref(entity.content), - default_specification=DRAFT202012, - ) - registry = registry.with_resource(with_scheme(entity_id), resource) - return registry + with self._lock: + if self._reference_registry is not None: + return self._reference_registry + registry = Registry() + for entity_id, entity in self._by_id.items(): + if entity.is_schema and isinstance(entity.content, dict): + resource = Resource.from_contents( + _without_x_gts_ref(entity.content), + default_specification=DRAFT202012, + ) + registry = registry.with_resource(with_scheme(entity_id), resource) + self._reference_registry = registry + return registry def items(self): """Return all entity ID and entity pairs.""" - return self._by_id.items() + with self._lock: + return tuple( + (entity_id, copy.deepcopy(entity)) + for entity_id, entity in self._by_id.items() + ) + + def keys(self) -> tuple[str, ...]: + with self._lock: + return tuple(self._by_id) + + def values(self) -> tuple[GtsEntity, ...]: + with self._lock: + return tuple(copy.deepcopy(entity) for entity in self._by_id.values()) @staticmethod def _validate_schema_refs(schema: dict[str, Any], path: str = "") -> None: @@ -309,7 +357,7 @@ def _validate_schema_ref_targets( try: target = self.get_schema_content(ref.target_id) except KeyError as error: - raise ValueError( + raise GtsUnresolvedRefError( f"Unresolvable $ref at '{current_path}': '{ref_uri}'" ) from error if ref.target_id not in visited: @@ -410,6 +458,108 @@ def _content_is_abstract(content: dict[str, Any]) -> bool: def _content_is_final(content: dict[str, Any]) -> bool: return content.get("x-gts-final") is True + @staticmethod + def _schema_dialect(schema: dict[str, Any]) -> str: + return schema_dialect.document_dialect(schema) + + @staticmethod + def _schema_dialect_uri(schema: dict[str, Any]) -> str: + return schema_dialect.dialect_uri(schema) + + def _validate_local_ref_dialects( + self, schema: dict[str, Any], root_id: str, root_dialect: str + ) -> None: + for subschema, _schema_path in iter_schema_nodes(schema): + ref = subschema.get("$ref") + if not isinstance(ref, str) or not ref.startswith("#"): + continue + target = resolve_json_pointer(schema, ref) + if not isinstance(target, dict) or "$schema" not in target: + continue + target_dialect = self._schema_dialect(target) + if target_dialect != root_dialect: + raise ValueError( + "GTS schema reference graph mixes JSON Schema dialects: " + f"root type '{root_id}' uses {root_dialect} but local $ref " + f"target '{ref}' uses {target_dialect}" + ) + + def _validate_chain_dialect( + self, + gts_id: str, + chain_ids: list[str], + transient_schema: dict[str, Any] | None, + ) -> None: + root_entity = self.get(chain_ids[0]) + root_content = ( + transient_schema + if chain_ids[0] == gts_id and transient_schema is not None + else root_entity.content + if root_entity + else None + ) + if not isinstance(root_content, dict): + return + root_dialect = self._schema_dialect(root_content) + + visited: set[str] = set() + queue: deque[tuple[str, dict[str, Any]]] = deque() + for chain_id in chain_ids: + entity = self.get(chain_id) + content = ( + transient_schema + if chain_id == gts_id and transient_schema is not None + else entity.content + if entity + else None + ) + if isinstance(content, dict): + chain_dialect = self._schema_dialect(content) + if chain_dialect != root_dialect: + raise ValueError( + "GTS derivation chain mixes JSON Schema dialects: " + f"root type '{chain_ids[0]}' uses {root_dialect} but " + f"'{chain_id}' uses {chain_dialect}; every type in a chained " + "$id hierarchy must use the root type's dialect" + ) + self._validate_local_ref_dialects(content, chain_ids[0], root_dialect) + # Seed the reference walk from every chain member, not just the + # leaf, so a cross-dialect gts:// $ref on an ancestor is caught + # even when the leaf does not reference it (spec §11.0/§12; + # matches the Rust reference which validates each related type in + # the closure). + queue.append((chain_id, content)) + while queue: + current_id, content = queue.popleft() + if current_id in visited: + continue + visited.add(current_id) + for dependency_id, dependency_is_type in self._schema_dependencies( + content, include_gts_refs=False + ): + if not dependency_is_type or dependency_id == current_id: + continue + target = self.get(dependency_id) + if ( + not target + or not target.is_schema + or not isinstance(target.content, dict) + ): + continue + target_dialect = self._schema_dialect(target.content) + if target_dialect != root_dialect: + raise ValueError( + "GTS derivation mixes JSON Schema dialects: " + f"root type '{chain_ids[0]}' uses {root_dialect} but $ref " + f"target '{dependency_id}' uses {target_dialect}; every type " + "in the chain and its transitive gts:// $ref targets must use " + "the root type's dialect" + ) + self._validate_local_ref_dialects( + target.content, chain_ids[0], root_dialect + ) + queue.append((dependency_id, target.content)) + def _validate_schema_chain( self, gts_id: str, transient_schema: dict[str, Any] | None = None ) -> None: @@ -417,17 +567,18 @@ def _validate_schema_chain( gid = GtsID(gts_id) segments = gid.gts_id_segments - # Single-segment schemas have no parent to validate against - if len(segments) < 2: - return - - # Build chain IDs chain_ids = [] - prefix = "gts." + prefix = GTS_PREFIX for seg in segments: chain_ids.append(prefix + seg.segment) prefix = prefix + seg.segment + self._validate_chain_dialect(gts_id, chain_ids, transient_schema) + + # Single-segment schemas have no parent to validate against + if len(segments) < 2: + return + # Validate each adjacent pair for i in range(len(chain_ids) - 1): base_id = chain_ids[i] @@ -482,74 +633,28 @@ def _validate_schema_chain( ) def _resolve_schema_refs(self, schema: Any) -> Any: - """Resolve $ref references in a schema by inlining referenced schemas. + """Resolve external ``$ref`` targets by inlining them from the store. - References are inlined recursively so that a schema reached through an - intermediate (A referenced via A~B) is fully expanded. Cyclic - references are left unresolved: the surviving $ref makes the effective - schema unprovable, which is the intended admission failure. + Thin adapter over :func:`schema_resolver.resolve_schema_refs`, passing + the store's :meth:`get_schema_content` as the reference provider. """ - import copy - - return self._inline_refs( - copy.deepcopy(schema), set(), self._supports_ref_siblings(schema) - ) + return schema_resolver.resolve_schema_refs(schema, self.get_schema_content) @staticmethod def _supports_ref_siblings(schema: Any) -> bool: - dialect = schema.get("$schema") if isinstance(schema, dict) else None - return isinstance(dialect, str) and ( - "/draft/2019-09/" in dialect or "/draft/2020-12/" in dialect - ) + return schema_dialect.supports_ref_siblings(schema) def _inline_refs( - self, node: Any, seen: set[str], supports_ref_siblings: bool + self, + node: Any, + seen: set[str], + supports_ref_siblings: bool, + expansions: list[int] | None = None, ) -> Any: - """Recursively inline $ref references, guarding against cycles.""" - if isinstance(node, dict): - ref_uri = node.get("$ref") - if isinstance(ref_uri, str): - ref = GtsRef.parse(ref_uri) - # Local (#/...) refs are resolved by JSON Schema itself; only - # external targets are inlined from the store. - ref_id = None if ref.is_local else ref.target_id - if ref_id is not None: - if ref_id in seen: - # Cycle detected: leave the $ref unresolved. - return node - try: - ref_schema = self.get_schema_content(ref_id) - except KeyError: - return node # Leave unresolved - import copy - - resolved = self._inline_refs( - copy.deepcopy(ref_schema), - seen | {ref_id}, - self._supports_ref_siblings(ref_schema), - ) - if supports_ref_siblings and len(node) > 1: - siblings = { - key: value for key, value in node.items() if key != "$ref" - } - return { - "allOf": [ - resolved, - self._inline_refs( - siblings, seen, supports_ref_siblings - ), - ] - } - return resolved - return { - key: self._inline_refs(value, seen, supports_ref_siblings) - for key, value in node.items() - } - if isinstance(node, list): - return [ - self._inline_refs(item, seen, supports_ref_siblings) for item in node - ] - return node + """Recursively inline ``$ref`` references, guarding against cycles.""" + return schema_resolver.inline_refs( + node, seen, supports_ref_siblings, self.get_schema_content, expansions + ) def _build_effective_traits( self, gts_id: str, transient_schema: dict[str, Any] | None = None @@ -559,7 +664,7 @@ def _build_effective_traits( segments = gid.gts_id_segments chain_ids: list[str] = [] - prefix = "gts." + prefix = GTS_PREFIX for seg in segments: chain_ids.append(prefix + seg.segment) prefix = prefix + seg.segment @@ -582,6 +687,13 @@ def _build_effective_traits( level_schemas: list[Any] = [] traits.collect_trait_schema_from_value(content, level_schemas) for ts in level_schemas: + if isinstance(ts, dict) and "$schema" in ts: + trait_dialect = self._schema_dialect(ts) + host_dialect = self._schema_dialect(content) + if trait_dialect != host_dialect: + raise ValueError( + f"trait schema dialect {trait_dialect} differs from host dialect {host_dialect}" + ) # Inline local JSON Pointer refs against the host document, then # resolve any gts:// refs so the composed schema is self-contained. inlined = traits.inline_local_pointers(ts, content) @@ -599,11 +711,11 @@ def _build_effective_traits( if leaf else None ) - dialect = None - if isinstance(leaf_content, dict): - ds = leaf_content.get("$schema") - if isinstance(ds, str): - dialect = ds + dialect = ( + self._schema_dialect_uri(leaf_content) + if isinstance(leaf_content, dict) + else None + ) return traits.build_effective_traits(trait_schemas, merged_traits, dialect) @@ -688,6 +800,10 @@ def validate_schema_content( f"Invalid $schema URL '{meta_schema_url}': must be a standard JSON Schema URL, not a GTS ID" ) + self._schema_dialect(schema_content) + # A subschema may not switch JSON Schema dialect (spec sec 11); the whole + # type is read under the dialect its top-level $schema selects. + schema_dialect.check_subschemas(schema_content) logger.info(f"Validating schema {schema_id.id}") self._validate_schema_refs(schema_content, "") self._validate_schema_ref_targets(schema_content) @@ -696,13 +812,10 @@ def validate_schema_content( self._validate_schema_chain(schema_id.id, schema_content) try: - from jsonschema import Draft7Validator - - if meta_schema_url: - validator_class = validator_for({"$schema": meta_schema_url}) - validator_class.check_schema(schema_content) - else: - Draft7Validator.check_schema(schema_content) + validator_class = validator_for( + {"$schema": self._schema_dialect_uri(schema_content)} + ) + validator_class.check_schema(schema_content) logger.info( f"Schema {schema_id.id} passed JSON Schema meta-schema validation" @@ -801,7 +914,7 @@ def _validate_schema_transitive( ) chain_ids: list[str] = [] - prefix = "gts." + prefix = GTS_PREFIX for segment in schema_id.gts_id_segments: chain_ids.append(prefix + segment.segment) prefix += segment.segment @@ -858,7 +971,7 @@ def _schema_dependencies( ) if ( isinstance(x_gts_ref, str) - and x_gts_ref.startswith("gts.") + and x_gts_ref.startswith(GTS_PREFIX) and "*" not in x_gts_ref ): yield x_gts_ref, True @@ -890,7 +1003,7 @@ def _has_valid_wildcard_match( gts_ref_validation: GtsRefValidationMode, ) -> bool: wildcard = GtsWildcard(pattern) - for entity_id in self._by_id: + for entity_id in self.keys(): try: if not GtsID(entity_id).wildcard_match(wildcard): continue @@ -916,13 +1029,24 @@ def validate_instance_content( except KeyError as error: raise StoreGtsSchemaNotFound(schema_type.id) from error + self._validate_schema_chain(schema_type.id) + self._schema_dialect(schema) if isinstance(schema, dict) and self._content_is_abstract(schema): raise ValueError( f"type '{schema_type.id}' is abstract and cannot have direct instances" ) - schema_for_validation = _without_x_gts_ref(schema) - validator_class = validator_for(schema_for_validation) + # Keep x-gts-ref in the schema and evaluate it with the extended + # validator so the engine resolves oneOf/anyOf/allOf correctly (branches + # that differ only by x-gts-ref stay distinct). Existence and /$id are + # still enforced by the XGtsRefValidator walker below. + schema_for_validation = { + **schema, + "$schema": self._schema_dialect_uri(schema), + } + validator_class = extended_validator_for( + schema_for_validation, selected_type_id=schema_type.id + ) validator = validator_class( schema_for_validation, registry=self._create_reference_registry(), @@ -1063,9 +1187,9 @@ def cast( raise StoreGtsObjectNotFound(from_schema_id) # Create a resolver to handle $ref in schemas - resolver = self._create_ref_resolver(to_schema.content) + registry = self._create_reference_registry() - return from_entity.cast(to_schema, from_schema, resolver=resolver) + return from_entity.cast(to_schema, from_schema, resolver=registry) def is_minor_compatible( self, @@ -1120,6 +1244,22 @@ def is_minor_compatible( # Determine direction direction = GtsEntityCastResult._infer_direction(old_schema_id, new_schema_id) + # Surface *why* a direction failed and classify the candidate's object + # levels so a caller can see whether a level can still gain optional + # properties later (spec sec 4.4). Reasons are derived from the verdicts + # already computed above (no second inclusion pass), so they never + # contradict the verdict. + differing_dialects = compatibility.dialects_differ(old_resolved, new_resolved) + backward_errors = compatibility.explain_verdict( + backward, backward=True, differing_dialects=differing_dialects + ) + forward_errors = compatibility.explain_verdict( + forward, backward=False, differing_dialects=differing_dialects + ) + # Union the directional reasons for the summary, dropping the duplicate + # a single shared cause (e.g. differing dialects) would produce. + incompatibility_reasons = list(dict.fromkeys(backward_errors + forward_errors)) + return GtsEntityCastResult( from_id=old_schema_id, to_id=new_schema_id, @@ -1130,20 +1270,21 @@ def is_minor_compatible( is_fully_compatible=full == compatibility.COMPATIBLE, is_backward_compatible=backward == compatibility.COMPATIBLE, is_forward_compatible=forward == compatibility.COMPATIBLE, - incompatibility_reasons=[], - backward_errors=[], - forward_errors=[], + incompatibility_reasons=incompatibility_reasons, + backward_errors=backward_errors, + forward_errors=forward_errors, casted_entity=None, backward_verdict=backward, forward_verdict=forward, full_verdict=full, + candidate_object_levels=compatibility.classify_object_levels(new_resolved), ) - def build_schema_graph(self, gts_id: str) -> tuple[dict[str, set[str]], list[str]]: - seen_gts_ids = set() + def build_schema_graph(self, gts_id: str) -> dict[str, Any]: + seen_gts_ids: set[str] = set() - def gts2node(gts_id: str, seen_gts_ids: set[str]) -> str: - ret = {"id": gts_id} + def gts2node(gts_id: str, seen_gts_ids: set[str]) -> dict[str, Any]: + ret: dict[str, Any] = {"id": gts_id} if gts_id in seen_gts_ids: return ret @@ -1152,7 +1293,7 @@ def gts2node(gts_id: str, seen_gts_ids: set[str]) -> str: entity = self.get(gts_id) if entity: - refs = {} + refs: dict[str, Any] = {} for r in entity.gts_refs: if r["id"] == gts_id: continue @@ -1336,7 +1477,7 @@ def query(self, expr: str, limit: int = 100) -> GtsStoreQueryResult: return result # Filter entities - for entity in self._by_id.values(): + for entity in self.values(): if len(result.results) >= limit: break if not isinstance(entity.content, dict) or not entity.gts_id: diff --git a/gts/src/gts/traits.py b/gts/src/gts/traits.py index 36c40c5..2ca9571 100644 --- a/gts/src/gts/traits.py +++ b/gts/src/gts/traits.py @@ -41,11 +41,13 @@ def __init__( values: Any, resolved_trait_schemas: list[Any], merged_traits: dict[str, Any], + dialect: str | None, ) -> None: self.schema = schema self.values = values self.resolved_trait_schemas = resolved_trait_schemas self.merged_traits = merged_traits + self.dialect = dialect def _has_schema(self) -> bool: return len(self.resolved_trait_schemas) > 0 @@ -61,7 +63,9 @@ def validate( selected_type_id: str | None = None, ) -> list[str]: """Return a list of error strings (empty means valid).""" - errors = _validate_trait_schema_integrity(self.resolved_trait_schemas) + errors = _validate_trait_schema_integrity( + self.resolved_trait_schemas, self.dialect + ) if errors: return errors errors = _validate_trait_schema_compatibility(self.resolved_trait_schemas) @@ -191,6 +195,7 @@ def build_effective_traits( values=values, resolved_trait_schemas=list(resolved_trait_schemas), merged_traits=copy.deepcopy(merged_traits), + dialect=dialect, ) @@ -298,14 +303,19 @@ def strip(node: Any) -> Any: return map_schema_nodes(copy.deepcopy(schema), strip) -def _validate_trait_schema_integrity(resolved_trait_schemas: list[Any]) -> list[str]: +def _validate_trait_schema_integrity( + resolved_trait_schemas: list[Any], dialect: str | None +) -> list[str]: for i, ts in enumerate(resolved_trait_schemas): if isinstance(ts, bool): continue if isinstance(ts, dict): + schema = copy.deepcopy(ts) + if dialect: + schema["$schema"] = dialect try: - cls = validator_for(ts) - cls.check_schema(ts) + cls = validator_for(schema) + cls.check_schema(schema) except Exception as e: # noqa: BLE001 - surfaced as validation error message return [f"{X_GTS_TRAITS_SCHEMA}[{i}] is not a valid JSON Schema: {e}"] else: diff --git a/gts/src/gts/x_gts_ref.py b/gts/src/gts/x_gts_ref.py index ae7595e..90ff898 100644 --- a/gts/src/gts/x_gts_ref.py +++ b/gts/src/gts/x_gts_ref.py @@ -14,7 +14,8 @@ from typing import Any -from jsonschema.validators import validator_for +from jsonschema.exceptions import ValidationError +from jsonschema.validators import extend, validator_for from ._json_pointer import resolve as resolve_json_pointer from ._naming import GTS_PREFIX, strip_scheme @@ -25,7 +26,96 @@ X_GTS_REF_SELF = "/$id" +def _gts_pattern_violation(value: str, pattern: str) -> str | None: + """Return why the string ``value`` does not satisfy ``pattern``, or None. + + Pure pattern matching (concrete id, type prefix, or single trailing + wildcard). Registry existence and the ``/$id`` self-reference are NOT decided + here — they need the store and the selected-type context, which the + :class:`XGtsRefValidator` walker owns. This helper is shared by that walker + and by the structural ``x-gts-ref`` keyword so both agree on matching. + """ + if not GtsID.is_valid(value): + return f"Value '{value}' is not a valid GTS identifier" + if pattern == GTS_PREFIX + "*": + return None + if pattern.endswith("*"): + if not value.startswith(pattern[:-1]): + return f"Value '{value}' does not match pattern '{pattern}'" + return None + # Prefix matching alone ignores segment boundaries: an exact constraint such + # as "gts.a.b.c.d.v1~x.y.z.w.v1" would otherwise also accept "…w.v12"/"…w.v1.5". + # Type patterns (ending with '~') admit derived identifiers; any other (exact) + # pattern requires a full match or a '~' boundary right after the pattern. + if not value.startswith(pattern) or not ( + len(value) == len(pattern) + or pattern.endswith("~") + or value[len(pattern)] == "~" + ): + return f"Value '{value}' does not match pattern '{pattern}'" + return None + + +def _make_x_gts_ref_keyword(selected_type_id: str | None): + """Build a ``jsonschema`` keyword handler that makes ``x-gts-ref`` a + first-class assertion, so ``oneOf``/``anyOf``/``allOf`` resolve correctly: + two branches that differ only by ``x-gts-ref`` are genuinely different + schemas rather than identical match-all schemas. This is the same design + gts-go and gts-rust use (a registered keyword/vocabulary) and removes the + need to strip x-gts-ref and rewrite ``oneOf``→``anyOf``. + + ``selected_type_id`` (the type being validated) lets the ``/$id`` + self-reference resolve here so a ``/$id`` branch matches only that type + instead of matching unconditionally; without it, ``/$id`` is deferred to + XGtsRefValidator. Registry existence always stays with XGtsRefValidator. + """ + + def _keyword(validator, ref_pattern, instance, schema): + if not isinstance(ref_pattern, str): + return + if not isinstance(instance, str): + return + pattern = ref_pattern + if ref_pattern == X_GTS_REF_SELF: + if not selected_type_id: + return + pattern = selected_type_id + reason = _gts_pattern_violation(instance, strip_scheme(pattern)) + if reason is not None: + yield ValidationError(reason) + + return _keyword + + +# Backwards-compatible symbol: the /$id-deferring keyword (no selected type). +_x_gts_ref_keyword = _make_x_gts_ref_keyword(None) + + +_EXTENDED_VALIDATORS: dict[tuple[type, str | None], type] = {} + + +def extended_validator_for(schema: Any, selected_type_id: str | None = None) -> type: + """Return the ``jsonschema`` validator class for ``schema``'s dialect, + extended so ``x-gts-ref`` is evaluated as a real keyword during structural + validation (including inside combinators). ``selected_type_id`` is threaded + into the keyword so ``/$id`` resolves during combinator resolution.""" + base = validator_for(schema) + key = (base, selected_type_id) + extended = _EXTENDED_VALIDATORS.get(key) + if extended is None: + extended = extend( + base, {"x-gts-ref": _make_x_gts_ref_keyword(selected_type_id)} + ) + _EXTENDED_VALIDATORS[key] = extended + return extended + + def _without_x_gts_ref(schema: Any) -> Any: + # Plain removal of x-gts-ref for the few places that deliberately need the + # bare structural shape: the walker's per-branch structural check + # (_is_structurally_valid) and the $ref reference registry. Structural + # instance validation does NOT use this — it uses extended_validator_for so + # the engine evaluates x-gts-ref natively (see _x_gts_ref_keyword). def strip(node: Any) -> Any: if not isinstance(node, dict): return node @@ -419,7 +509,7 @@ def _validate_gts_id_or_pattern( self, pattern: str, field_path: str ) -> XGtsRefValidationError | None: """Validate a GTS ID or pattern in schema definition.""" - if pattern == "gts.*": + if pattern == GTS_PREFIX + "*": return None # Valid wildcard if "*" in pattern: @@ -455,34 +545,10 @@ def _validate_gts_pattern( Returns: Error if validation fails, None otherwise """ - # Validate it's a valid GTS ID - if not GtsID.is_valid(value): - return XGtsRefValidationError( - field_path, - value, - pattern, - f"Value '{value}' is not a valid GTS identifier", - ) - - # Check pattern match - if pattern == "gts.*": - pass # Any valid GTS ID matches - elif pattern.endswith("*"): - prefix = pattern[:-1] - if not value.startswith(prefix): - return XGtsRefValidationError( - field_path, - value, - pattern, - f"Value '{value}' does not match pattern '{pattern}'", - ) - elif not value.startswith(pattern): - return XGtsRefValidationError( - field_path, - value, - pattern, - f"Value '{value}' does not match pattern '{pattern}'", - ) + # Shared pattern matching (also used by the structural x-gts-ref keyword). + reason = _gts_pattern_violation(value, pattern) + if reason is not None: + return XGtsRefValidationError(field_path, value, pattern, reason) # Referenced values use exact registry lookup in presence/full modes. if self.store and self.mode != GtsRefValidationMode.NONE: diff --git a/tests/test_compatibility.py b/tests/test_compatibility.py index ed0f355..d7602d9 100644 --- a/tests/test_compatibility.py +++ b/tests/test_compatibility.py @@ -1,18 +1,115 @@ """Tests for gts.compatibility (spec sec 4, OP#8 & OP#12 inclusion primitive).""" from gts.compatibility import ( + CLOSED, COMPATIBLE, INCOMPATIBLE, + OPEN, + PARTIAL, UNKNOWN, boolean_schema_value, sanitize, check_backward_compatibility, check_forward_compatibility, + classify_object_levels, + explain_verdict, full_verdict, check_accepted_set_inclusion, ) +class TestObjectLevelClassification: + def test_open_when_additional_properties_absent(self): + levels = classify_object_levels( + {"type": "object", "properties": {"a": {"type": "string"}}} + ) + assert levels == [{"path": "$", "content_model": OPEN}] + + def test_closed_when_additional_properties_false(self): + levels = classify_object_levels( + {"type": "object", "properties": {}, "additionalProperties": False} + ) + assert levels == [{"path": "$", "content_model": CLOSED}] + + def test_partial_when_additional_properties_is_schema(self): + levels = classify_object_levels( + {"type": "object", "additionalProperties": {"type": "string"}} + ) + assert levels == [{"path": "$", "content_model": PARTIAL}] + + def test_partial_when_pattern_properties_present(self): + levels = classify_object_levels( + { + "type": "object", + "additionalProperties": False, + "patternProperties": {"^x-": {"type": "string"}}, + } + ) + assert levels == [{"path": "$", "content_model": PARTIAL}] + + def test_nested_levels_are_reported(self): + levels = classify_object_levels( + { + "type": "object", + "additionalProperties": False, + "properties": { + "payload": {"type": "object", "properties": {}}, + }, + } + ) + paths = {level["path"]: level["content_model"] for level in levels} + assert paths == {"$": CLOSED, "$.payload": OPEN} + + def test_levels_under_applicator_keywords_are_reported(self): + levels = classify_object_levels( + { + "type": "object", + "additionalProperties": {"type": "object", "properties": {}}, + "properties": { + "items": { + "type": "array", + "items": {"type": "object", "additionalProperties": False}, + }, + "map": { + "type": "object", + "patternProperties": { + "^x-": {"type": "object", "additionalProperties": False} + }, + }, + }, + } + ) + paths = {level["path"]: level["content_model"] for level in levels} + # Object levels nested under additionalProperties, array items, and + # patternProperties are all classified now, not only `properties`. + assert paths["$.additionalProperties"] == OPEN + assert paths["$.items[]"] == CLOSED + assert paths["$.map.patternProperties[^x-]"] == CLOSED + + +class TestExplainVerdict: + def test_compatible_direction_has_no_reasons(self): + assert explain_verdict(COMPATIBLE, backward=True) == [] + + def test_incompatible_backward_explains_relation(self): + reasons = explain_verdict(INCOMPATIBLE, backward=True) + assert reasons + assert "backward incompatible" in reasons[0] + + def test_incompatible_forward_explains_relation(self): + reasons = explain_verdict(INCOMPATIBLE, backward=False) + assert reasons + assert "forward incompatible" in reasons[0] + + def test_unknown_on_differing_dialects(self): + reasons = explain_verdict(UNKNOWN, backward=True, differing_dialects=True) + assert reasons and "different JSON Schema dialects" in reasons[0] + + def test_unknown_without_dialect_difference(self): + reasons = explain_verdict(UNKNOWN, backward=False, differing_dialects=False) + assert reasons and "could not be proved or disproved" in reasons[0] + + class TestBooleanSchemaValue: def test_bool_passthrough(self): assert boolean_schema_value(True) is True diff --git a/tests/test_files_reader_coverage.py b/tests/test_files_reader_coverage.py index 2583029..7d6d3c9 100644 --- a/tests/test_files_reader_coverage.py +++ b/tests/test_files_reader_coverage.py @@ -35,13 +35,16 @@ def test_reader_discovers_json_yaml_and_list_entities_and_skips_invalid_files(tm entities = list(GtsFileReader(str(source))) - assert [entity.gts_id.id for entity in entities] == [ + entities_by_id = {entity.gts_id.id: entity for entity in entities} + assert set(entities_by_id) == { "gts.acme.catalog._.item.v1~acme.catalog._.one.v1", "gts.acme.catalog._.item.v1~", - ] - assert entities[0].label == "entities.json#0" - assert entities[0].file.sequencesCount == 2 - assert entities[1].file.name == "schema.yaml" + } + list_entity = entities_by_id["gts.acme.catalog._.item.v1~acme.catalog._.one.v1"] + schema_entity = entities_by_id["gts.acme.catalog._.item.v1~"] + assert list_entity.label == "entities.json#0" + assert list_entity.file.sequencesCount == 2 + assert schema_entity.file.name == "schema.yaml" def test_reader_accepts_multiple_paths_and_reset_recollects_files(tmp_path): diff --git a/tests/test_gts_id.py b/tests/test_gts_id.py index ba613f5..745b071 100644 --- a/tests/test_gts_id.py +++ b/tests/test_gts_id.py @@ -89,10 +89,12 @@ def test_valid_gts_id_basic(self): assert gts_id.is_type is True assert len(gts_id.gts_id_segments) == 1 - def test_valid_gts_id_with_uri_prefix(self): - """Test GTS ID with URI prefix is normalized.""" - gts_id = GtsID("gts://gts.vendor.package.namespace.type.v1~") - assert gts_id.id == "gts.vendor.package.namespace.type.v1~" + def test_uri_prefix_rejected_by_core_parser(self): + """The core parser accepts only the bare canonical form; the ``gts://`` + URI form is a $id/$ref serialization detail stripped by URI-specific + callers before parsing (mirroring gts-rust/gts-go).""" + with pytest.raises(GtsInvalidId): + GtsID("gts://gts.vendor.package.namespace.type.v1~") def test_valid_gts_id_instance(self): """Test instance GTS ID (not ending with ~).""" @@ -193,7 +195,8 @@ def test_to_uuid(self): def test_is_valid_static_method(self): """Test static is_valid method.""" assert GtsID.is_valid("gts.vendor.package.namespace.type.v1~") is True - assert GtsID.is_valid("gts://gts.vendor.package.namespace.type.v1~") is True + # The URI form is not a canonical id; callers strip the scheme first. + assert GtsID.is_valid("gts://gts.vendor.package.namespace.type.v1~") is False assert GtsID.is_valid("invalid") is False assert GtsID.is_valid("") is False diff --git a/tests/test_ops.py b/tests/test_ops.py index 77e42fb..0887732 100644 --- a/tests/test_ops.py +++ b/tests/test_ops.py @@ -3,6 +3,8 @@ import pytest from gts.ops import GtsOps +from gts import GtsOps as PublicGtsOps + SCHEMA = { "$schema": "http://json-schema.org/draft-07/schema#", "$id": "gts://gts.x.test._.foo.v1~", @@ -24,6 +26,9 @@ def ops(): class TestConstructionAndConfig: + def test_high_level_facade_is_exported(self): + assert PublicGtsOps is GtsOps + def test_default_config_used_when_no_path(self, ops): assert "$id" in ops.cfg.entity_id_fields @@ -153,23 +158,54 @@ def test_add_entities_rejects_unsupported_x_gts_ref_pointer(self, ops): assert "must be a GTS identifier" in result.results[0].error -class TestAddSchemaLegacy: - def test_add_schema_legacy_success(self, ops): - result = ops.add_schema("gts.x.test._.legacy.v1~", {"type": "object"}) +class TestAddSchema: + def test_add_schema_success(self, ops): + schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "gts://gts.x.test._.legacy.v1~", + "type": "object", + } + result = ops.add_schema(schema) assert result.ok is True - assert result.id == "gts.x.test._.legacy.v1~" - - def test_add_schema_legacy_changed_content_is_conflict(self, ops): - assert ops.add_schema("gts.x.test._.legacy.v1~", {"type": "object"}).ok is True - result = ops.add_schema("gts.x.test._.legacy.v1~", {"type": "string"}) + assert result.type_id == "gts.x.test._.legacy.v1~" + + def test_add_schema_changed_content_is_conflict(self, ops): + dialect = "http://json-schema.org/draft-07/schema#" + schema_id = "gts://gts.x.test._.legacy.v1~" + assert ( + ops.add_schema( + {"$schema": dialect, "$id": schema_id, "type": "object"}, + ).ok + is True + ) + result = ops.add_schema( + {"$schema": dialect, "$id": schema_id, "type": "string"}, + ) assert result.ok is False assert result.conflict is True - def test_add_schema_legacy_failure(self, ops): - result = ops.add_schema("gts.x.test._.legacy.v1", {"type": "object"}) + def test_add_schema_missing_id_failure(self, ops): + result = ops.add_schema({"type": "object"}) assert result.ok is False - assert result.error + assert "$id" in result.error + + def test_add_schemas_batch_partial(self, ops): + dialect = "http://json-schema.org/draft-07/schema#" + result = ops.add_schemas( + [ + { + "$schema": dialect, + "$id": "gts://gts.x.test._.batch_ok.v1~", + "type": "object", + }, + {"$schema": dialect, "type": "object"}, + ] + ) + assert result.ok is False + assert result.results[0].ok is True + assert result.results[0].type_id == "gts.x.test._.batch_ok.v1~" + assert result.results[1].ok is False class TestValidateId: @@ -323,6 +359,33 @@ def test_validates_instance_against_explicit_type_without_storing_it(self, ops): assert result.ok is True assert result.type_id == "gts.x.test._.foo.v1~" + def test_rejects_instance_of_mixed_dialect_schema_graph(self, ops): + ops.add_entity( + { + "$id": "gts://gts.x.test._.foreign.v1~", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + } + ) + ops.add_entity( + { + "$id": "gts://gts.x.test._.host.v1~", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "allOf": [{"$ref": "gts://gts.x.test._.foreign.v1~"}], + } + ) + + result = ops.validate_json( + { + "id": "gts.x.test._.host.v1~x.test._.item.v1", + "type": "gts.x.test._.host.v1~", + } + ) + + assert result.ok is False + assert "mixes JSON Schema dialects" in result.error + def test_does_not_mutate_registry(self, ops, monkeypatch): monkeypatch.setattr( ops.store, diff --git a/tests/test_server.py b/tests/test_server.py index d6a6856..0d5c2d3 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -6,10 +6,8 @@ import pytest from fastapi.responses import JSONResponse - -from gts.ops import GtsOps from gts._server import GtsHttpServer, ValidateEntityRequest, _RequestLoggingMiddleware - +from gts.ops import GtsOps SCHEMA = { "$schema": "http://json-schema.org/draft-07/schema#", @@ -105,25 +103,36 @@ def test_add_entities(self, server): resp = run(server.add_entities(body=[SCHEMA, INSTANCE])) assert resp.status_code == 200 - def test_add_schema(self, server): - from gts._server import SchemaRegister - - body = SchemaRegister(type_id="gts.x.test._.bar.v1~", type_schema={"type": "object"}) - resp = run(server.add_schema(body)) + def test_add_schemas(self, server): + import json + + body = [ + { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "gts://gts.x.test._.bar.v1~", + "type": "object", + }, + ] + resp = run(server.add_schemas(body)) assert resp.status_code == 200 + payload = json.loads(resp.body) + assert payload["ok"] is True + assert payload["results"][0]["type_id"] == "gts.x.test._.bar.v1~" - def test_add_schema_changed_content_conflict(self, server): - from gts._server import SchemaRegister + def test_add_schemas_changed_content_conflict(self, server): + import json - initial = SchemaRegister( - type_id="gts.x.test._.bar.v1~", type_schema={"type": "object"} - ) - changed = SchemaRegister( - type_id="gts.x.test._.bar.v1~", type_schema={"type": "string"} - ) + dialect = "http://json-schema.org/draft-07/schema#" + schema_id = "gts://gts.x.test._.bar.v1~" + initial = [{"$schema": dialect, "$id": schema_id, "type": "object"}] + changed = [{"$schema": dialect, "$id": schema_id, "type": "string"}] - assert run(server.add_schema(initial)).status_code == 200 - assert run(server.add_schema(changed)).status_code == 409 + assert run(server.add_schemas(initial)).status_code == 200 + resp = run(server.add_schemas(changed)) + assert resp.status_code == 200 + payload = json.loads(resp.body) + assert payload["ok"] is False + assert payload["results"][0]["ok"] is False def test_validate_id(self, server): result = run(server.validate_id(id="gts.x.test._.foo.v1~")) @@ -226,6 +235,27 @@ def test_get_entities(self, server): class TestRequestLoggingMiddlewareVerboseOff: + def test_verbose_logging_does_not_read_or_log_bodies(self, server, caplog): + class URL: + path = "/entities" + + class Request: + method = "POST" + url = URL() + + async def body(self): + raise AssertionError("request body must not be read by logging middleware") + + middleware = _RequestLoggingMiddleware(server.app, verbose=2) + + async def call_next(request): + return JSONResponse({"secret": "must-not-be-logged"}) + + with caplog.at_level("DEBUG"): + result = run(middleware.dispatch(request=Request(), call_next=call_next)) + assert result.headers["connection"] == "close" + assert "must-not-be-logged" not in caplog.text + def test_dispatch_skips_when_not_verbose(self, server): middleware = _RequestLoggingMiddleware(server.app, verbose=0) diff --git a/tests/test_store.py b/tests/test_store.py index d8ea42c..b942b1a 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -92,6 +92,8 @@ def test_store_register_schema(self): store = GtsStore(reader) schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "gts://gts.vendor.package.namespace.type.v1~", "type": "object", "properties": {"name": {"type": "string"}}, } @@ -101,6 +103,33 @@ def test_store_register_schema(self): assert result is not None assert result.is_schema is True + def test_store_register_schema_requires_schema_marker(self): + """An explicit type ID does not turn an instance document into a schema.""" + store = GtsStore(MockGtsReader([])) + + with pytest.raises(ValueError, match=r"top-level \$schema"): + store.register_schema( + "gts.vendor.package.namespace.type.v1~", {"type": "object"} + ) + + def test_store_register_schema_requires_matching_embedded_id(self): + store = GtsStore(MockGtsReader([])) + type_id = "gts.vendor.package.namespace.type.v1~" + + with pytest.raises(ValueError, match=r"top-level \$id"): + store.register_schema( + type_id, + {"$schema": "http://json-schema.org/draft-07/schema#"}, + ) + with pytest.raises(ValueError, match="must match"): + store.register_schema( + type_id, + { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "gts://gts.vendor.package.namespace.other.v1~", + }, + ) + def test_store_register_schema_invalid_id(self): """Test registering schema with invalid ID (not ending with ~).""" reader = MockGtsReader([]) @@ -307,6 +336,7 @@ def test_validate_instance_content_enforces_standard_formats(self): type_id, { "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "gts://gts.vendor.package.namespace.type.v1~", "type": "object", "properties": { "uuid": {"type": "string", "format": "uuid"}, diff --git a/tests/test_store_extra.py b/tests/test_store_extra.py index 1850bcc..a2b5a13 100644 --- a/tests/test_store_extra.py +++ b/tests/test_store_extra.py @@ -1,5 +1,6 @@ """Additional coverage-focused tests for gts.store.GtsStore.""" +import threading from collections.abc import Iterator from typing import Optional @@ -8,11 +9,13 @@ from gts.gts import GtsID from gts.schema_validation import PATTERN_TIMEOUT_SECONDS, validator_for from gts.store import ( + MAX_SCHEMA_REF_EXPANSIONS, GtsReader, GtsStore, StoreGtsEntityNotFound, StoreGtsObjectNotFound, ) +from jsonschema import ValidationError class MockGtsReader(GtsReader): @@ -87,6 +90,74 @@ def test_unregister_missing_id_noop(self): store = GtsStore(reader=None) store.unregister("gts.x.test._.missing.v1~") # should not raise + def test_store_uses_defensive_copies(self): + store = GtsStore(reader=None) + entity = _schema_entity("gts.x.test._.copy.v1~") + store.register(entity) + entity.content["type"] = "array" + first = store.get("gts.x.test._.copy.v1~") + assert first.content["type"] == "object" + first.content["type"] = "number" + items = dict(store.items()) + items["gts.x.test._.copy.v1~"].content["type"] = "boolean" + assert store.get("gts.x.test._.copy.v1~").content["type"] == "object" + + def test_reference_registry_is_cached_and_invalidated_by_schema_changes(self): + store = GtsStore(reader=None) + store.register(_schema_entity("gts.x.test._.cached.v1~")) + first = store._create_reference_registry() + second = store._create_reference_registry() + assert first is second + + instance = GtsEntity( + content={"id": "gts.x.test._.cached.v1~x.test._.i.v1.0"}, + gts_id=GtsID("gts.x.test._.cached.v1~x.test._.i.v1.0"), + ) + store.register(instance) + assert store._create_reference_registry() is first + + store.register(_schema_entity("gts.x.test._.cached2.v1~")) + assert store._create_reference_registry() is not first + + def test_transaction_serializes_writers(self): + store = GtsStore(reader=None) + entered = threading.Event() + release = threading.Event() + completed = threading.Event() + + def first_writer(): + with store.transaction(): + entered.set() + release.wait() + + def second_writer(): + entered.wait() + store.register(_schema_entity("gts.x.test._.serialized.v1~")) + completed.set() + + first = threading.Thread(target=first_writer) + second = threading.Thread(target=second_writer) + first.start() + second.start() + assert not completed.wait(0.05) + release.set() + first.join() + second.join() + assert completed.is_set() + + def test_reference_expansion_budget(self): + store = GtsStore(reader=None) + target_id = "gts.x.test._.budget.v1~" + store.register(_schema_entity(target_id)) + schema = { + "allOf": [ + {"$ref": f"gts://{target_id}"} + for _ in range(MAX_SCHEMA_REF_EXPANSIONS + 1) + ] + } + with pytest.raises(ValueError, match="expansion exceeds limit"): + store._resolve_schema_refs(schema) + class TestValidateSchemaRefs: def test_local_ref_valid(self): @@ -285,6 +356,175 @@ def test_missing_base_schema_raises(self): with pytest.raises(ValueError, match="not found for chain validation"): store._validate_schema_chain("gts.x.test._.base.v1~x.test._.derived.v1~") + @pytest.mark.parametrize( + "dialect", + [ + "https://example.invalid/not-a-json-schema-dialect", + "https://json-schema.org/draft/2020-21/schema", + ], + ) + def test_unsupported_schema_dialect_raises(self, dialect): + with pytest.raises(ValueError, match="Unsupported JSON Schema dialect"): + GtsStore._schema_dialect({"$schema": dialect}) + + def test_missing_schema_dialect_raises(self): + """A document without $schema is not eligible for schema validation.""" + with pytest.raises(ValueError, match=r"\$schema must declare"): + GtsStore._schema_dialect({}) + + def test_draft7_alias_uses_draft7_validator(self): + """Canonicalize accepted aliases before schema and instance validation.""" + schema_id = "gts.x.test._.draft7_alias.v1~" + schema = _schema_entity( + schema_id, + { + "$schema": "https://json-schema.org/draft-07/schema#", + "required": ["pair"], + "properties": { + "pair": { + "type": "array", + "items": [{"type": "string"}, {"type": "integer"}], + "additionalItems": False, + } + }, + }, + ) + store = GtsStore(reader=None) + store.register(schema) + + store.validate_schema(schema_id) + store.validate_instance_content({"pair": ["ok", 1]}, schema_id) + with pytest.raises(ValidationError): + store.validate_instance_content({"pair": ["ok", "wrong"]}, schema_id) + + def test_local_ref_dialect_mismatch_raises(self): + schema = _schema_entity( + "gts.x.test._.embedded.v1~", + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": {"legacy": {"$ref": "#/$defs/legacy"}}, + "$defs": { + "legacy": { + "$id": "legacy", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "string", + } + }, + }, + ) + store = GtsStore(reader=None) + store.register(schema) + + with pytest.raises(ValueError, match=r"local \$ref target"): + store._validate_schema_chain(schema.gts_id.id) + + def test_mixed_dialect_chain_raises(self): + base = _schema_entity("gts.x.test._.base.v1~") + derived = _schema_entity( + "gts.x.test._.base.v1~x.test._.derived.v1~", + {"$schema": "https://json-schema.org/draft/2020-12/schema"}, + ) + store = GtsStore(reader=None) + store.register(base) + store.register(derived) + + with pytest.raises(ValueError, match="mixes JSON Schema dialects"): + store._validate_schema_chain("gts.x.test._.base.v1~x.test._.derived.v1~") + + def test_transitive_ref_dialect_mismatch_raises(self): + foreign = _schema_entity( + "gts.x.test._.foreign.v1~", + {"$schema": "https://json-schema.org/draft/2020-12/schema"}, + ) + middle = _schema_entity( + "gts.x.test._.middle.v1~", + {"allOf": [{"$ref": "gts://gts.x.test._.foreign.v1~"}]}, + ) + host = _schema_entity( + "gts.x.test._.host.v1~", + {"allOf": [{"$ref": "gts://gts.x.test._.middle.v1~"}]}, + ) + store = GtsStore(reader=None) + store.register(foreign) + store.register(middle) + store.register(host) + + with pytest.raises(ValueError, match="gts.x.test._.foreign.v1~"): + store._validate_schema_chain("gts.x.test._.host.v1~") + + def test_ancestor_ref_dialect_mismatch_raises(self): + # Issue C: the cross-dialect $ref lives on an ancestor; the descendant + # derives by re-declaration and references neither the ancestor nor the + # 2020-12 target. A leaf-only reference walk would accept it; walking the + # whole chain closure must reject it (OP#12 path). + foreign = _schema_entity( + "gts.x.test._.ancforeign.v1~", + {"$schema": "https://json-schema.org/draft/2020-12/schema"}, + ) + base = _schema_entity( + "gts.x.test._.ancbase.v1~", + {"properties": {"ext": {"$ref": "gts://gts.x.test._.ancforeign.v1~"}}}, + ) + child = _schema_entity( + "gts.x.test._.ancbase.v1~x.test._.ancchild.v1~", + {"properties": {"label": {"type": "string"}}}, + ) + store = GtsStore(reader=None) + store.register(foreign) + store.register(base) + store.register(child) + + with pytest.raises(ValueError, match="gts.x.test._.ancforeign.v1~"): + store._validate_schema_chain( + "gts.x.test._.ancbase.v1~x.test._.ancchild.v1~" + ) + + def test_instance_content_rejects_ancestor_cross_dialect_ref(self): + # Issue C on the OP#6 instance path: validating an instance of the + # descendant must reject it because an ancestor references a schema of a + # different dialect. + foreign = _schema_entity( + "gts.x.test._.ancforeign2.v1~", + {"$schema": "https://json-schema.org/draft/2020-12/schema"}, + ) + base = _schema_entity( + "gts.x.test._.ancbase2.v1~", + {"properties": {"ext": {"$ref": "gts://gts.x.test._.ancforeign2.v1~"}}}, + ) + child = _schema_entity( + "gts.x.test._.ancbase2.v1~x.test._.ancchild2.v1~", + {"properties": {"label": {"type": "string"}}}, + ) + store = GtsStore(reader=None) + store.register(foreign) + store.register(base) + store.register(child) + + with pytest.raises(ValueError, match="gts.x.test._.ancforeign2.v1~"): + store.validate_instance_content( + {"label": "ok"}, + "gts.x.test._.ancbase2.v1~x.test._.ancchild2.v1~", + ) + + def test_trait_resource_dialect_mismatch_raises(self): + schema_id = "gts.x.test._.trait_resource.v1~" + schema = _schema_entity( + schema_id, + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "x-gts-traits-schema": { + "$id": "https://example.com/gts/legacy-traits", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + }, + }, + ) + store = GtsStore(reader=None) + store.register(schema) + + with pytest.raises(ValueError, match="differs from host dialect"): + store.validate_schema(schema_id) + def test_incompatible_derivation_raises(self): base = _schema_entity( "gts.x.test._.base.v1~", @@ -554,3 +794,71 @@ def test_validate_instance_invalid_non_uuid_non_gts_raises(self): store = GtsStore(reader=None) with pytest.raises(StoreGtsObjectNotFound): store.validate_instance("totally-not-valid") + + +class TestSubschemaDialect: + def test_nested_schema_switching_dialect_is_rejected(self): + schema_id = "gts.x.test._.nested_dialect.v1~" + schema = _schema_entity( + schema_id, + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "inner": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + } + }, + }, + ) + store = GtsStore(reader=None) + store.register(schema) + with pytest.raises(ValueError, match="must not change JSON Schema dialect"): + store.validate_schema(schema_id) + + def test_nested_schema_restating_dialect_is_allowed(self): + schema_id = "gts.x.test._.nested_same_dialect.v1~" + schema = _schema_entity( + schema_id, + { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "inner": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + } + }, + }, + ) + store = GtsStore(reader=None) + store.register(schema) + store.validate_schema(schema_id) # must not raise + + +class TestExceptionHierarchy: + def test_not_found_errors_share_a_base(self): + from gts import GtsError, GtsNotFoundError + from gts.store import StoreGtsObjectNotFound, StoreGtsSchemaNotFound + + assert issubclass(StoreGtsObjectNotFound, GtsNotFoundError) + assert issubclass(StoreGtsSchemaNotFound, GtsNotFoundError) + assert issubclass(GtsNotFoundError, GtsError) + + def test_validation_error_is_value_error(self): + from gts import GtsError, GtsValidationError + + assert issubclass(GtsValidationError, ValueError) + assert issubclass(GtsValidationError, GtsError) + + def test_unresolvable_ref_is_catchable_as_gts_error(self): + from gts import GtsError + + schema_id = "gts.x.test._.dangling_ref.v1~" + schema = _schema_entity( + schema_id, + {"properties": {"other": {"$ref": "gts://gts.x.test._.missing.v1~"}}}, + ) + store = GtsStore(reader=None) + store.register(schema) + with pytest.raises(GtsError, match="Unresolvable \\$ref"): + store.validate_schema(schema_id) diff --git a/tests/test_traits.py b/tests/test_traits.py index 3b8a6da..9746c53 100644 --- a/tests/test_traits.py +++ b/tests/test_traits.py @@ -308,6 +308,23 @@ def test_invalid_trait_schema_integrity_flagged(self): errors = effective.validate(check_unresolved=True) assert any("not a valid JSON Schema" in e for e in errors) + def test_draft7_tuple_schema_integrity_uses_host_dialect(self): + schema = { + "type": "object", + "properties": { + "pair": { + "type": "array", + "items": [{"type": "string"}], + } + }, + } + effective = build_effective_traits( + [schema], + {"pair": ["ok"]}, + "http://json-schema.org/draft-07/schema#", + ) + assert effective.validate(check_unresolved=True) == [] + def test_dialect_applied_to_effective_schema(self): effective = build_effective_traits( [{"type": "object"}], {}, "https://json-schema.org/draft/2020-12/schema"