diff --git a/.github/workflows/downstream_tests.yml b/.github/workflows/downstream_tests.yml new file mode 100644 index 0000000..7e2d962 --- /dev/null +++ b/.github/workflows/downstream_tests.yml @@ -0,0 +1,73 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +name: Downstream tests + +on: + pull_request: + types: [opened, reopened, synchronize] + push: + branches: [main] + merge_group: + types: [checks_requested] + +concurrency: + group: downstream-tests-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +jobs: + consumer: + name: ${{ matrix.consumer }} + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + consumer: [baselibs, lifecycle] + steps: + - name: Checkout coverage_tool + uses: actions/checkout@v7.0.1 + + - name: Free disk space + uses: eclipse-score/more-disk-space@v1 + with: + level: 4 + + - name: Setup Bazel cache + uses: eclipse-score/cicd-actions/setup-bazel-cache@setup-bazel-cache/v0.1.0 + with: + disk-cache-key: downstream-${{ matrix.consumer }} + + - name: Run the consumer's coverage workflow + run: bazel test --lockfile_mode=error //tools/downstream_tests:${{ matrix.consumer }} + + - name: Publish coverage summary + if: always() + env: + REPORT_ARCHIVE: bazel-testlogs/tools/downstream_tests/${{ matrix.consumer }}/test.outputs/coverage-report.zip + run: | + if [ -f "$REPORT_ARCHIVE" ] && unzip -Z1 "$REPORT_ARCHIVE" | grep -x coverage_summary.md > /dev/null; then + unzip -p "$REPORT_ARCHIVE" coverage_summary.md >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload pytest results + if: always() + uses: actions/upload-artifact@v4 + with: + name: downstream-${{ matrix.consumer }}-results + path: | + bazel-testlogs/tools/downstream_tests/${{ matrix.consumer }}/test.xml + bazel-testlogs/tools/downstream_tests/${{ matrix.consumer }}/test.outputs/coverage-report.zip + if-no-files-found: ignore + retention-days: 3 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2741efe..d5b3473 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -45,7 +45,7 @@ jobs: - name: Build everything run: bazel build --lockfile_mode=error //... - name: Run unit and Starlark analysis tests - run: bazel test --lockfile_mode=error //score_coverage/... //tools/... + run: bazel test --lockfile_mode=error --test_tag_filters=-integration //score_coverage/... //tools/... - name: Static analysis of the Python (ruff, pylint, ty; findings fail the build) run: bazel build --lockfile_mode=error --config=lint //score_coverage/... //tools/... - name: Measure structural coverage of the tool itself (coverage.py) diff --git a/MODULE.bazel b/MODULE.bazel index e1cfd46..00a2f6f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -59,6 +59,7 @@ use_repo(pip, "pip_score_coverage") # Development-only dependencies (repository hygiene: copyright, formatting) ############################################################################### bazel_dep(name = "score_tooling", version = "2.2.0", dev_dependency = True) +bazel_dep(name = "score_tools", version = "0.0.3", dev_dependency = True) # use_format_targets() (from score_tooling) loads these from the ROOT module's # repo mapping, so the root has to declare them itself. diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 939ccf6..650c2d2 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -821,6 +821,8 @@ "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_toolchains_rust/0.10.0/source.json": "8bda773be264da16d2a82a03ebb737421dd4a35855f1e9a5d03d9722d84c1df5", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_tooling/2.2.0/MODULE.bazel": "178ba4862246b6ba2bbcd96b7e9e728299b19fb94bcf23d315dc2299aabf7178", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_tooling/2.2.0/source.json": "a76f2d093cff26d5b256ce531bb2f69b6c667c968f99a156327fd194d4f36e61", + "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_tools/0.0.3/MODULE.bazel": "69f441bf28d938de2b6aef61ec91e65633f8ea908d2527e6f5a424895c9ac388", + "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_tools/0.0.3/source.json": "84fda1c7e1e73811b67c239ce4d10800132c87dbc0af549ab425e66e679dc37f", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/sphinxdocs/2.2.0/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/stardoc/0.5.0/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/stardoc/0.5.1/MODULE.bazel": "not found", diff --git a/tools/downstream_tests/BUILD b/tools/downstream_tests/BUILD new file mode 100644 index 0000000..e17d07e --- /dev/null +++ b/tools/downstream_tests/BUILD @@ -0,0 +1,46 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +load("@score_tools//score_pytest:pytest.bzl", "score_pytest") + +# Each test runs the consumer's full coverage pipeline, which can take many +# minutes on a cold CI runner. The tests need network access to clone current +# consumer main branches and run Bazel in those nested workspaces. +# +# Keep each consumer scenario in its own target so CI and JUnit reports show +# exactly which real downstream workflow failed. +score_pytest( + name = "baselibs", + size = "large", + timeout = "eternal", + srcs = ["test_baselibs.py"], + data = ["_consumer_workspace.py"], + imports = ["."], + tags = [ + "integration", + "local", + ], +) + +score_pytest( + name = "lifecycle", + size = "large", + timeout = "eternal", + srcs = ["test_lifecycle.py"], + data = ["_consumer_workspace.py"], + imports = ["."], + tags = [ + "integration", + "local", + ], +) diff --git a/tools/downstream_tests/_consumer_workspace.py b/tools/downstream_tests/_consumer_workspace.py new file mode 100644 index 0000000..5752890 --- /dev/null +++ b/tools/downstream_tests/_consumer_workspace.py @@ -0,0 +1,162 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Prepare real consumer workspaces for the downstream coverage scenarios.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +from pathlib import Path + +_COVERAGE_TOOL_ROOT = Path(__file__).resolve().parents[2] + + +def clone_consumer(name: str, parent_directory: Path) -> Path: + """Clone a consumer's default branch and point it at this checkout.""" + workspace = parent_directory / name + repository_url = f"https://github.com/eclipse-score/{name}.git" + subprocess.run( + ["git", "clone", "--depth", "1", repository_url, str(workspace)], + check=True, + ) + + tool_link = parent_directory / "coverage_tool" + tool_link.symlink_to(_COVERAGE_TOOL_ROOT, target_is_directory=True) + + module_file = workspace / "MODULE.bazel" + module_contents = module_file.read_text(encoding="utf-8") + module_contents = ( + module_contents.rstrip() + + '\n\nlocal_path_override(\n module_name = "score_coverage",\n path = "../coverage_tool",\n)\n' + ) + module_file.write_text(module_contents, encoding="utf-8") + + # Updating the consumer lockfile for the local override lets its production + # commands keep their normal --lockfile_mode=error setting. + subprocess.run( + ["bazel", "mod", "deps", "--lockfile_mode=update"], + cwd=workspace, + check=True, + ) + return workspace + + +def run_bazel( + workspace: Path, + *arguments: str, + extra_environment: dict[str, str] | None = None, +) -> None: + """Run one consumer-facing Bazel command, keeping failure output concise.""" + environment = os.environ.copy() + if extra_environment: + environment.update(extra_environment) + + command = ["bazel", *arguments] + result = subprocess.run( + command, + cwd=workspace, + env=environment, + capture_output=True, + check=False, + text=True, + ) + if result.returncode: + output = (result.stdout + result.stderr).splitlines() + output_tail = "\n".join(output[-80:]) + raise AssertionError( + f"`{' '.join(command)}` exited with code {result.returncode}.\nLast Bazel output lines:\n{output_tail}" + ) + + +def retain_coverage_results(workspace: Path, archive_directory: str) -> None: + """Keep whatever report files exist, including reports from a failed gate.""" + archive = workspace / archive_directory + html_report = archive / "coverage_linux" + lcov_report = archive / "coverage_report.dat" + outputs_directory = os.environ.get("TEST_UNDECLARED_OUTPUTS_DIR") + if not archive.is_dir() or not outputs_directory: + return + + retained_report = Path(outputs_directory) / "coverage-report" + summary_markdown = workspace / "coverage_summary.md" + try: + retained_report.mkdir(parents=True, exist_ok=True) + if html_report.is_dir(): + shutil.copytree(html_report, retained_report / "coverage_linux") + if lcov_report.is_file(): + shutil.copy2(lcov_report, retained_report / lcov_report.name) + if summary_markdown.is_file(): + shutil.copy2(summary_markdown, retained_report / summary_markdown.name) + + for name in ("justification_report", "unmapped_files.txt"): + result = archive / name + if result.is_dir(): + shutil.copytree(result, retained_report / name) + elif result.is_file(): + shutil.copy2(result, retained_report / name) + except Exception as error: + print(f"Could not copy all coverage report files: {error}", file=sys.stderr) + + try: + report_archive = Path(shutil.make_archive(str(retained_report), "zip", retained_report)) + except Exception as error: + print(f"Could not archive the coverage report: {error}", file=sys.stderr) + return + + try: + shutil.rmtree(retained_report) + except OSError as error: + print(f"Could not remove the unpacked coverage report: {error}", file=sys.stderr) + + print(f"Coverage report retained at {report_archive}.") + + +def verify_coverage_results(workspace: Path, archive_directory: str) -> None: + """Require measurable coverage after retaining all available report files.""" + archive = workspace / archive_directory + html_report = archive / "coverage_linux" + lcov_report = archive / "coverage_report.dat" + summary_markdown = workspace / "coverage_summary.md" + + assert (html_report / "index.html").is_file(), "Coverage HTML index was not generated" + assert lcov_report.is_file(), "LCOV coverage report was not generated" + assert summary_markdown.is_file(), "Markdown coverage summary was not generated" + + source_files = 0 + lines_found = 0 + lines_hit = 0 + for line in lcov_report.read_text(encoding="utf-8", errors="replace").splitlines(): + if line.startswith("SF:"): + source_files += 1 + elif line.startswith("LF:"): + lines_found += int(line[3:]) + elif line.startswith("LH:"): + lines_hit += int(line[3:]) + + assert source_files > 0, "LCOV report contains no source files" + assert lines_found > 0, "LCOV report contains no measurable lines" + assert lines_hit > 0, "LCOV report contains no covered lines" + + outputs_directory = os.environ.get("TEST_UNDECLARED_OUTPUTS_DIR") + assert outputs_directory, "Bazel did not provide TEST_UNDECLARED_OUTPUTS_DIR" + report_archive = Path(outputs_directory) / "coverage-report.zip" + assert report_archive.is_file(), "Coverage report was not retained for Bazel" + + coverage_percent = 100 * lines_hit / lines_found + print( + f"Coverage report contains {source_files} source files and " + f"{lines_hit}/{lines_found} covered lines ({coverage_percent:.2f}%). " + f"Report retained at {report_archive}." + ) diff --git a/tools/downstream_tests/test_baselibs.py b/tools/downstream_tests/test_baselibs.py new file mode 100644 index 0000000..e0e66c9 --- /dev/null +++ b/tools/downstream_tests/test_baselibs.py @@ -0,0 +1,60 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Black-box test for baselibs' production LLVM coverage workflow.""" + +from pathlib import Path + +from _consumer_workspace import clone_consumer, retain_coverage_results, run_bazel, verify_coverage_results + + +def test_baselibs_coverage_workflow_reports_measured_coverage(tmp_path: Path) -> None: + """The checked-in baselibs workflow produces a non-empty coverage report.""" + workspace = clone_consumer("baselibs", tmp_path) + + run_bazel( + workspace, + "coverage", + "--lockfile_mode=error", + "--config=llvm_cov", + "--build_tests_only", + "--", + "//score/...", + # This upstream-main test currently fails under the coverage config + # because its compact-JSON expectations conflict with default pretty printing. + "-//score/json/internal/writer/vajson:vajson_serialize_test", + "-//score/language/safecpp/aborts_upon_exception/...", + "-//score/language/safecpp/safe_math/details:floating_point_environment_test", + "-//score/os/linux/utils/test:network_interface_test", + ) + # The generator assembles artifacts before returning a gate failure. + try: + run_bazel( + workspace, + "run", + "--lockfile_mode=error", + "@score_coverage//:generate_coverage_html", + "--", + "--yaml", + "tools/coverage/coverage_justifications.yaml", + "--summary-md", + "coverage_summary.md", + "--testlogs-subdir", + "score", + "--archive-dir", + "coverage_artifact", + extra_environment={"COVERAGE_THRESHOLD": "0"}, + ) + finally: + retain_coverage_results(workspace, "coverage_artifact") + + verify_coverage_results(workspace, "coverage_artifact") diff --git a/tools/downstream_tests/test_lifecycle.py b/tools/downstream_tests/test_lifecycle.py new file mode 100644 index 0000000..ccbb158 --- /dev/null +++ b/tools/downstream_tests/test_lifecycle.py @@ -0,0 +1,50 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Black-box test for lifecycle's production Linux coverage workflow.""" + +from pathlib import Path + +from _consumer_workspace import clone_consumer, retain_coverage_results, run_bazel, verify_coverage_results + + +def test_lifecycle_coverage_workflow_reports_measured_coverage(tmp_path: Path) -> None: + """The checked-in workflow meets its gate and produces a coverage report.""" + workspace = clone_consumer("lifecycle", tmp_path) + + run_bazel( + workspace, + "coverage", + "--config=llvm_cov", + "//score/...", + "--lockfile_mode=error", + "--build_tests_only", + ) + # The generator assembles artifacts before returning a gate failure. + try: + run_bazel( + workspace, + "run", + "@score_coverage//:generate_coverage_html", + "--", + "--yaml", + "quality/coverage/coverage_justifications.yaml", + "--summary-md", + "coverage_summary.md", + "--archive-dir", + "coverage_artifacts", + extra_environment={"COVERAGE_THRESHOLD": "66"}, + ) + finally: + retain_coverage_results(workspace, "coverage_artifacts") + + verify_coverage_results(workspace, "coverage_artifacts")