From 7657327ef1a8f4261664c39fbaaf7cf1ea53a57f Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sat, 14 Nov 2026 09:12:11 -0800 Subject: [PATCH] tests: restore MODULE.bazel.lock after bzlmod_lockfile_test failure `bzlmod_lockfile_test` runs with `no-sandbox` and invokes `bazel mod deps --lockfile_mode=update` in place when `bazel test //...` fails. Leaving `MODULE.bazel.lock` modified on disk lets `--flaky_test_attempts` retries pass on CI, masking stale lockfile failures, and dirties the local working tree. Wrap the lockfile update in a context manager that restores the original bytes and `mtime` on exit after reading the updated contents for diff and undeclared-output reporting, and add unit tests covering retry failure and restoration. --- tests/integration/BUILD.bazel | 14 ++ .../bzlmod_lockfile_runner_test.py | 213 ++++++++++++++++++ tests/integration/bzlmod_lockfile_test.py | 49 +++- 3 files changed, 270 insertions(+), 6 deletions(-) create mode 100644 tests/integration/bzlmod_lockfile_runner_test.py diff --git a/tests/integration/BUILD.bazel b/tests/integration/BUILD.bazel index 950de14368..3e2e859f3b 100644 --- a/tests/integration/BUILD.bazel +++ b/tests/integration/BUILD.bazel @@ -15,6 +15,7 @@ load("@rules_bazel_integration_test//bazel_integration_test:defs.bzl", "default_test_runner") load("//python:py_binary.bzl", "py_binary") load("//python:py_library.bzl", "py_library") +load("//python:pytest_test.bzl", "pytest_test") load("//tests/support:support.bzl", "NOT_WINDOWS", "SUPPORTS_BZLMOD") load(":integration_test.bzl", "rules_python_integration_test") @@ -169,6 +170,19 @@ rules_python_integration_test( py_main = "whl_library_test.py", ) +py_library( + name = "bzlmod_lockfile_test_lib", + srcs = ["bzlmod_lockfile_test.py"], + deps = [":runner_lib"], +) + +pytest_test( + name = "bzlmod_lockfile_runner_test", + size = "small", + srcs = ["bzlmod_lockfile_runner_test.py"], + deps = [":bzlmod_lockfile_test_lib"], +) + py_library( name = "runner_lib", srcs = ["runner.py"], diff --git a/tests/integration/bzlmod_lockfile_runner_test.py b/tests/integration/bzlmod_lockfile_runner_test.py new file mode 100644 index 0000000000..5375d11baf --- /dev/null +++ b/tests/integration/bzlmod_lockfile_runner_test.py @@ -0,0 +1,213 @@ +"""Unit tests for BzlmodLockfileTest lockfile preservation and reporting.""" + +from __future__ import annotations + +import os +import pathlib +import subprocess +from collections.abc import Callable, Sequence +from unittest import mock + +import pytest + +from tests.integration import bzlmod_lockfile_test + +_STALE_LOCKFILE = b'{\n "lockFileVersion": 1,\n "digest": "aaaa"\n}\n' +_UPDATED_LOCKFILE = b'{\n "lockFileVersion": 1,\n "digest": "bbbb"\n}\n' +_EXPECTED_DIFF = ( + "--- MODULE.bazel.lock (checked-in)\n" + "+++ MODULE.bazel.lock (expected/updated)\n" + "@@ -1,4 +1,4 @@\n" + " {\n" + ' "lockFileVersion": 1,\n' + '- "digest": "aaaa"\n' + '+ "digest": "bbbb"\n' + " }\n" +) +# Distinct historical timestamp so tests verify mtime restoration in addition to +# byte contents. +_ORIGINAL_MTIME_NS = 1_700_000_000_000_000_000 + + +@pytest.fixture(name="lockfile_env") +def fixture_lockfile_env( + tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch +) -> tuple[pathlib.Path, pathlib.Path]: + """Sets up `runner.TestCase` environment variables and workspace files. + + Args: + tmp_path: Per-test temporary directory. + monkeypatch: Pytest monkeypatch helper. + + Returns: + Tuple of `(lockfile_path, undeclared_outputs_dir)` inside `tmp_path`. + """ + workspace_dir = tmp_path / "workspace" + workspace_dir.mkdir() + lockfile_path = workspace_dir / "MODULE.bazel.lock" + lockfile_path.write_bytes(_STALE_LOCKFILE) + os.utime(lockfile_path, ns=(_ORIGINAL_MTIME_NS, _ORIGINAL_MTIME_NS)) + + # runner.TestCase.setUp opens BIT_BAZEL_BINARY on Windows to inspect any + # shebang line, so point it at an existing file. + fake_bazel = tmp_path / "fake_bazel" + fake_bazel.write_bytes(b"") + + test_tmpdir = tmp_path / "test_tmpdir" + test_tmpdir.mkdir() + runfiles_dir = tmp_path / "runfiles" + runfiles_dir.mkdir() + undeclared_outputs_dir = tmp_path / "undeclared_outputs" + undeclared_outputs_dir.mkdir() + + monkeypatch.setenv("BIT_WORKSPACE_DIR", str(workspace_dir)) + monkeypatch.setenv("BIT_BAZEL_BINARY", str(fake_bazel)) + monkeypatch.setenv("TEST_TMPDIR", str(test_tmpdir)) + monkeypatch.setenv("TEST_SRCDIR", str(runfiles_dir)) + monkeypatch.setenv("TEST_UNDECLARED_OUTPUTS_DIR", str(undeclared_outputs_dir)) + monkeypatch.setenv("RULES_PYTHON_INTEGRATION_TEST_BZLMOD", "1") + return lockfile_path, undeclared_outputs_dir + + +def _run_case() -> None: + case = bzlmod_lockfile_test.BzlmodLockfileTest("test_bzlmod_lockfile") + case.setUp() + try: + case.test_bzlmod_lockfile() + finally: + case.tearDown() + + +def _assert_lockfile_preserved( + lockfile_path: pathlib.Path, expected_bytes: bytes +) -> None: + assert lockfile_path.read_bytes() == expected_bytes + assert lockfile_path.stat().st_mtime_ns == _ORIGINAL_MTIME_NS + + +def _make_bazel_simulator( + lockfile_path: pathlib.Path, + *, + force_test_failure_stderr: str | None = None, +) -> Callable[..., subprocess.CompletedProcess[str]]: + """Builds a `subprocess.run` side effect simulating inner Bazel commands.""" + + def _fake_run( + args: Sequence[str], **_kwargs: object + ) -> subprocess.CompletedProcess[str]: + cmd_list = list(args) + subcommand = cmd_list[1] + if subcommand == "test": + if force_test_failure_stderr is not None: + return subprocess.CompletedProcess( + cmd_list, 1, stdout="", stderr=force_test_failure_stderr + ) + if lockfile_path.read_bytes() == _UPDATED_LOCKFILE: + return subprocess.CompletedProcess(cmd_list, 0, stdout="", stderr="") + return subprocess.CompletedProcess( + cmd_list, 1, stdout="", stderr="ERROR: Lockfile is out of date\n" + ) + if subcommand == "mod": + lockfile_path.write_bytes(_UPDATED_LOCKFILE) + return subprocess.CompletedProcess(cmd_list, 0, stdout="", stderr="") + raise AssertionError(f"Unexpected bazel subcommand: {cmd_list}") + + return _fake_run + + +def test_passes_when_up_to_date( + lockfile_env: tuple[pathlib.Path, pathlib.Path], +) -> None: + lockfile_path, undeclared_outputs_dir = lockfile_env + lockfile_path.write_bytes(_UPDATED_LOCKFILE) + os.utime(lockfile_path, ns=(_ORIGINAL_MTIME_NS, _ORIGINAL_MTIME_NS)) + + with mock.patch.object( + subprocess, + "run", + autospec=True, + side_effect=_make_bazel_simulator(lockfile_path), + ) as mock_run: + _run_case() + + assert mock_run.call_count == 1 + _assert_lockfile_preserved(lockfile_path, _UPDATED_LOCKFILE) + assert list(undeclared_outputs_dir.iterdir()) == [] + + +def test_restores_lockfile_across_retries_on_mismatch( + lockfile_env: tuple[pathlib.Path, pathlib.Path], +) -> None: + lockfile_path, undeclared_outputs_dir = lockfile_env + + with mock.patch.object( + subprocess, + "run", + autospec=True, + side_effect=_make_bazel_simulator(lockfile_path), + ): + # Simulate `--flaky_test_attempts=2`: both attempt 1 and attempt 2 must + # fail identically because attempt 1 restores the stale lockfile. + for _ in range(2): + with pytest.raises( + AssertionError, match=r"MODULE\.bazel\.lock is out of date\." + ) as exc_info: + _run_case() + assert _EXPECTED_DIFF in str(exc_info.value) + _assert_lockfile_preserved(lockfile_path, _STALE_LOCKFILE) + + assert ( + undeclared_outputs_dir / "MODULE.bazel.lock" + ).read_bytes() == _UPDATED_LOCKFILE + assert (undeclared_outputs_dir / "MODULE.bazel.lock.diff").read_text( + encoding="utf-8" + ) == _EXPECTED_DIFF + + +def test_reports_test_failure_when_lockfile_matches( + lockfile_env: tuple[pathlib.Path, pathlib.Path], +) -> None: + lockfile_path, undeclared_outputs_dir = lockfile_env + lockfile_path.write_bytes(_UPDATED_LOCKFILE) + os.utime(lockfile_path, ns=(_ORIGINAL_MTIME_NS, _ORIGINAL_MTIME_NS)) + + with mock.patch.object( + subprocess, + "run", + autospec=True, + side_effect=_make_bazel_simulator( + lockfile_path, + force_test_failure_stderr="FAIL: //:test_dummy\n", + ), + ): + with pytest.raises( + AssertionError, match=r"bazel test //\.\.\. failed:" + ) as exc_info: + _run_case() + + assert "FAIL: //:test_dummy" in str(exc_info.value) + _assert_lockfile_preserved(lockfile_path, _UPDATED_LOCKFILE) + assert not (undeclared_outputs_dir / "MODULE.bazel.lock.diff").exists() + + +def test_restores_lockfile_when_update_raises( + lockfile_env: tuple[pathlib.Path, pathlib.Path], +) -> None: + lockfile_path, _ = lockfile_env + + def _mutating_crash( + args: Sequence[str], **_kwargs: object + ) -> subprocess.CompletedProcess[str]: + cmd_list = list(args) + if cmd_list[1] == "test": + return subprocess.CompletedProcess(cmd_list, 1, stdout="", stderr="") + lockfile_path.write_bytes(_UPDATED_LOCKFILE) + raise OSError("simulated bazel crash after partial lockfile write") + + with mock.patch.object( + subprocess, "run", autospec=True, side_effect=_mutating_crash + ): + with pytest.raises(OSError, match="simulated bazel crash"): + _run_case() + + _assert_lockfile_preserved(lockfile_path, _STALE_LOCKFILE) diff --git a/tests/integration/bzlmod_lockfile_test.py b/tests/integration/bzlmod_lockfile_test.py index 650f132f0f..0b50010370 100644 --- a/tests/integration/bzlmod_lockfile_test.py +++ b/tests/integration/bzlmod_lockfile_test.py @@ -1,11 +1,39 @@ +from __future__ import annotations + +import contextlib import difflib import os import pathlib import unittest +from collections.abc import Iterator from tests.integration import runner +@contextlib.contextmanager +def _preserve_file(path: pathlib.Path) -> Iterator[bytes]: + """Saves `path`'s contents and timestamps and restores them on exit. + + Integration tests run with `no-sandbox`, so `path` resolves into the source + workspace. Restoring both bytes and `mtime` ensures `--flaky_test_attempts` + retries do not observe in-place updates (or hit Skyframe's `mtime` cache) + and local runs do not dirty the working tree. + + Args: + path: File whose bytes and timestamps should be preserved. + + Yields: + The original raw bytes of `path`. + """ + original_stat = path.stat() + original_bytes = path.read_bytes() + try: + yield original_bytes + finally: + path.write_bytes(original_bytes) + os.utime(path, ns=(original_stat.st_atime_ns, original_stat.st_mtime_ns)) + + class BzlmodLockfileTest(runner.TestCase): def test_bzlmod_lockfile(self): lockfile_path = self.repo_root / "MODULE.bazel.lock" @@ -13,20 +41,27 @@ def test_bzlmod_lockfile(self): lockfile_path.exists(), f"Expected lockfile at {lockfile_path}", ) - original_lockfile = lockfile_path.read_text() res = self.run_bazel("test", "//...", check=False) if res.exit_code == 0: return # Generate updated lockfile to compare diff and export artifact - self.run_bazel("mod", "deps", "--lockfile_mode=update", check=False) - updated_lockfile = lockfile_path.read_text() if lockfile_path.exists() else "" + with _preserve_file(lockfile_path) as original_bytes: + self.run_bazel("mod", "deps", "--lockfile_mode=update", check=False) + if lockfile_path.exists(): + updated_bytes = lockfile_path.read_bytes() + else: + updated_bytes = b"" + original_lockfile = original_bytes.decode("utf-8") + updated_lockfile = updated_bytes.decode("utf-8") + + out_dir: pathlib.Path | None = None undeclared_outputs_dir = os.environ.get("TEST_UNDECLARED_OUTPUTS_DIR") if undeclared_outputs_dir: out_dir = pathlib.Path(undeclared_outputs_dir) - (out_dir / "MODULE.bazel.lock").write_text(updated_lockfile) + (out_dir / "MODULE.bazel.lock").write_bytes(updated_bytes) if original_lockfile != updated_lockfile: diff_lines = list( @@ -39,8 +74,10 @@ def test_bzlmod_lockfile(self): ) diff_str = "".join(diff_lines) - if undeclared_outputs_dir: - (out_dir / "MODULE.bazel.lock.diff").write_text(diff_str) + if out_dir is not None: + (out_dir / "MODULE.bazel.lock.diff").write_text( + diff_str, encoding="utf-8" + ) msg = ( f"MODULE.bazel.lock is out of date.\n\n"