From 1596508274efae98a15973983b52fc13e7a799e1 Mon Sep 17 00:00:00 2001 From: eunwoo song Date: Sun, 6 Sep 2026 13:40:53 +0900 Subject: [PATCH] feat: support multi-document GitLab CI files --- CHANGELOG.rst | 1 + docs/precommit_usage.rst | 3 ++ src/check_jsonschema/instance_loader.py | 5 ++- src/check_jsonschema/parsers/__init__.py | 7 +++- src/check_jsonschema/parsers/yaml.py | 15 +++++++- src/check_jsonschema/transforms/base.py | 4 +- src/check_jsonschema/transforms/gitlab.py | 24 +++++++++++- .../hooks/positive/gitlab-ci/spec-inputs.yaml | 8 ++++ tests/unit/test_gitlab_data_transform.py | 6 +++ tests/unit/test_instance_loader.py | 37 +++++++++++++++++++ 10 files changed, 103 insertions(+), 7 deletions(-) create mode 100644 tests/example-files/hooks/positive/gitlab-ci/spec-inputs.yaml diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9d030bcd9..209ddecdf 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,7 @@ Unreleased .. vendor-insert-here +- Support multi-document GitLab CI configuration files. (:issue:`561`) - Update vendored schemas: bitbucket-pipelines, mergify, renovate (2026-08-16) 0.38.0 diff --git a/docs/precommit_usage.rst b/docs/precommit_usage.rst index 02e1ae6a2..1aba4ae11 100644 --- a/docs/precommit_usage.rst +++ b/docs/precommit_usage.rst @@ -286,6 +286,9 @@ Validate GitHub Workflows against the schema provided by SchemaStore Validate GitLab CI config against the schema provided by SchemaStore +The hook supports multi-document configuration files, including a ``spec:inputs`` +header separated from the pipeline configuration by ``---``. + .. code-block:: yaml :caption: example config diff --git a/src/check_jsonschema/instance_loader.py b/src/check_jsonschema/instance_loader.py index 5d76bbfe7..add8c087b 100644 --- a/src/check_jsonschema/instance_loader.py +++ b/src/check_jsonschema/instance_loader.py @@ -25,7 +25,10 @@ def __init__( ) self._parsers = ParserSet( - modify_yaml_implementation=self._data_transform.modify_yaml_implementation + modify_yaml_implementation=self._data_transform.modify_yaml_implementation, + load_multiple_yaml_documents=( + self._data_transform.load_multiple_yaml_documents + ), ) def iter_files(self) -> t.Iterator[tuple[str, ParseError | t.Any]]: diff --git a/src/check_jsonschema/parsers/__init__.py b/src/check_jsonschema/parsers/__init__.py index 5938ce5d7..3472a8538 100644 --- a/src/check_jsonschema/parsers/__init__.py +++ b/src/check_jsonschema/parsers/__init__.py @@ -46,6 +46,7 @@ def __init__( self, *, modify_yaml_implementation: t.Callable[[ruamel.yaml.YAML], None] | None = None, + load_multiple_yaml_documents: bool = False, supported_formats: t.Sequence[str] | None = None, ) -> None: yaml_impl = yaml.construct_yaml_implementation() @@ -54,7 +55,11 @@ def __init__( modify_yaml_implementation(yaml_impl) modify_yaml_implementation(failover_yaml_impl) base_by_tag = { - "yaml": yaml.impl2loader(yaml_impl, failover_yaml_impl), + "yaml": yaml.impl2loader( + yaml_impl, + failover_yaml_impl, + load_multiple_documents=load_multiple_yaml_documents, + ), **DEFAULT_LOAD_FUNC_BY_TAG, } if supported_formats is None: diff --git a/src/check_jsonschema/parsers/yaml.py b/src/check_jsonschema/parsers/yaml.py index d2780d77c..185993d8a 100644 --- a/src/check_jsonschema/parsers/yaml.py +++ b/src/check_jsonschema/parsers/yaml.py @@ -8,6 +8,10 @@ ParseError = ruamel.yaml.YAMLError +class YAMLDocuments(list[t.Any]): + """Distinct collection returned when all YAML documents are requested.""" + + def construct_yaml_implementation( typ: str = "safe", pure: bool = False ) -> ruamel.yaml.YAML: @@ -42,6 +46,8 @@ def _normalize(data: t.Any) -> t.Any: """ if isinstance(data, dict): return {str(k): _normalize(v) for k, v in data.items()} + elif isinstance(data, YAMLDocuments): + return YAMLDocuments(_normalize(x) for x in data) elif isinstance(data, list): return [_normalize(x) for x in data] else: @@ -52,7 +58,9 @@ def _normalize(data: t.Any) -> t.Any: def impl2loader( - primary: ruamel.yaml.YAML, *fallbacks: ruamel.yaml.YAML + primary: ruamel.yaml.YAML, + *fallbacks: ruamel.yaml.YAML, + load_multiple_documents: bool = False, ) -> t.Callable[[t.IO[bytes]], t.Any]: def load(stream: t.IO[bytes]) -> t.Any: stream_bytes = stream.read() @@ -62,7 +70,10 @@ def load(stream: t.IO[bytes]) -> t.Any: warnings.simplefilter("ignore", ruamel.yaml.error.ReusedAnchorWarning) for impl in [primary] + list(fallbacks): try: - data = impl.load(stream_bytes) + if load_multiple_documents: + data = YAMLDocuments(impl.load_all(stream_bytes)) + else: + data = impl.load(stream_bytes) except ruamel.yaml.YAMLError as e: lasterr = e else: diff --git a/src/check_jsonschema/transforms/base.py b/src/check_jsonschema/transforms/base.py index 0ed9a8647..e188cdb7d 100644 --- a/src/check_jsonschema/transforms/base.py +++ b/src/check_jsonschema/transforms/base.py @@ -6,6 +6,8 @@ class Transform: + load_multiple_yaml_documents = False + def __init__( self, *, @@ -16,7 +18,7 @@ def __init__( def modify_yaml_implementation(self, implementation: ruamel.yaml.YAML) -> None: pass - def __call__(self, data: list | dict) -> list | dict: + def __call__(self, data: t.Any) -> t.Any: if self.on_data is not None: return self.on_data(data) return data diff --git a/src/check_jsonschema/transforms/gitlab.py b/src/check_jsonschema/transforms/gitlab.py index 54825a893..d4fdda0a9 100644 --- a/src/check_jsonschema/transforms/gitlab.py +++ b/src/check_jsonschema/transforms/gitlab.py @@ -1,7 +1,10 @@ from __future__ import annotations +import typing as t + import ruamel.yaml +from ..parsers.yaml import YAMLDocuments from .base import Transform @@ -24,11 +27,28 @@ def from_yaml( return [item.value for item in node.value] -# this "transform" is actually a no-op on the data, but it registers the GitLab !reference -# tag with the instance YAML loader +# Register GitLab's !reference tag and combine multi-document CI configuration. class GitLabDataTransform(Transform): + load_multiple_yaml_documents = True + def modify_yaml_implementation(self, implementation: ruamel.yaml.YAML) -> None: implementation.register_class(GitLabReference) + def __call__(self, data: t.Any) -> t.Any: + if not isinstance(data, YAMLDocuments): + return data + documents = [document for document in data if document is not None] + if not documents: + return None + if len(documents) == 1: + return documents[0] + if not all(isinstance(document, dict) for document in documents): + return documents + + merged = {} + for document in documents: + merged.update(document) + return merged + GITLAB_TRANSFORM = GitLabDataTransform() diff --git a/tests/example-files/hooks/positive/gitlab-ci/spec-inputs.yaml b/tests/example-files/hooks/positive/gitlab-ci/spec-inputs.yaml new file mode 100644 index 000000000..cb26cd065 --- /dev/null +++ b/tests/example-files/hooks/positive/gitlab-ci/spec-inputs.yaml @@ -0,0 +1,8 @@ +spec: + inputs: + job-stage: + default: test +--- +scan-website: + stage: $[[ inputs.job-stage ]] + script: echo scan diff --git a/tests/unit/test_gitlab_data_transform.py b/tests/unit/test_gitlab_data_transform.py index 74e1e0fba..eb626715a 100644 --- a/tests/unit/test_gitlab_data_transform.py +++ b/tests/unit/test_gitlab_data_transform.py @@ -23,6 +23,12 @@ def test_can_parse_yaml_with_transform(): assert data == {"a": "b", "c": "d"} +def test_transform_preserves_non_yaml_data(): + data = {"job": {"script": "echo ok"}} + + assert GITLAB_TRANSFORM(data) is data + + def test_can_parse_ok_gitlab_yaml_with_transform(): rawdata = """\ foo: diff --git a/tests/unit/test_instance_loader.py b/tests/unit/test_instance_loader.py index 4835b814a..80948beb0 100644 --- a/tests/unit/test_instance_loader.py +++ b/tests/unit/test_instance_loader.py @@ -3,6 +3,7 @@ from check_jsonschema.instance_loader import InstanceLoader from check_jsonschema.parsers import BadFileTypeError, FailedFileLoadError from check_jsonschema.parsers.json5 import ENABLED as JSON5_ENABLED +from check_jsonschema.transforms.gitlab import GITLAB_TRANSFORM # handy helper for opening multiple files for InstanceLoader @@ -67,6 +68,42 @@ def test_instanceloader_yaml_data(tmp_path, filename, default_filetype, open_wid assert data == [(str(f), {"a": {"b": [1, 2], "c": "d"}})] +def test_instanceloader_gitlab_multidocument_yaml(tmp_path, open_wide): + f = tmp_path / ".gitlab-ci.yml" + f.write_text("""\ +spec: + inputs: + job-stage: + default: test +--- +scan-website: + stage: $[[ inputs.job-stage ]] + script: echo scan +""") + + regular_loader = InstanceLoader(open_wide(f), default_filetype="yaml") + regular_result = list(regular_loader.iter_files()) + assert len(regular_result) == 1 + assert isinstance(regular_result[0][1], FailedFileLoadError) + + loader = InstanceLoader( + open_wide(f), default_filetype="yaml", data_transform=GITLAB_TRANSFORM + ) + + assert list(loader.iter_files()) == [ + ( + str(f), + { + "spec": {"inputs": {"job-stage": {"default": "test"}}}, + "scan-website": { + "stage": "$[[ inputs.job-stage ]]", + "script": "echo scan", + }, + }, + ) + ] + + @pytest.mark.parametrize( "filename, default_filetype", [