From 88f5e3a7e3fd93fd5fd393f3ab05c3066f41ea60 Mon Sep 17 00:00:00 2001 From: Ali Date: Thu, 22 Oct 2026 16:06:44 +0100 Subject: [PATCH] Make the output folder the RO-Crate itself Rename the default output folder from fl_crate_out to flwrcrate-ro-crate and write the crate directly into it, instead of an ro-crate/ sub-folder. captured_metadata.json is deleted once the crate is written (kept if writing fails), and flwrCrate warns about files in the folder that it didn't write, since they would travel with the crate. This changes the crate's path from /ro-crate/ to /. --- .github/workflows/realapps.yml | 8 ++--- .gitignore | 2 +- README.md | 25 +++++++++------- examples/quickstart-pytorch/server_app.py | 4 +-- flwrcrate/tracker.py | 29 +++++++++++++++--- tests/e2e/sklearn_ci_server_app.py | 2 +- tests/test_integration.py | 36 ++++++++++++++++++++++- 7 files changed, 83 insertions(+), 23 deletions(-) diff --git a/.github/workflows/realapps.yml b/.github/workflows/realapps.yml index 2de63d7..9face24 100644 --- a/.github/workflows/realapps.yml +++ b/.github/workflows/realapps.yml @@ -58,13 +58,13 @@ jobs: - name: Run the federation working-directory: ${{ runner.temp }}/${{ matrix.app }} run: | - rm -rf /tmp/flcrate_out + rm -rf /tmp/flwrcrate-ro-crate # flwr run detaches the simulation when there's no TTY, so we kick it # off and then poll for the crate within this same step (keeping the # step alive so the background sim isn't killed). flwr run . --run-config "num-server-rounds=2" || true for i in $(seq 1 90); do - if [ -f /tmp/flcrate_out/ro-crate/ro-crate-metadata.json ]; then + if [ -f /tmp/flwrcrate-ro-crate/ro-crate-metadata.json ]; then echo "crate appeared after ~$((i*5))s"; break fi sleep 5 @@ -73,7 +73,7 @@ jobs: - name: Validate the generated crate run: | python tests/e2e/validate_crate.py \ - /tmp/flcrate_out/ro-crate/ro-crate-metadata.json \ + /tmp/flwrcrate-ro-crate/ro-crate-metadata.json \ "${{ matrix.strategy }}" "${{ matrix.framework }}" - name: Upload the crate as an artifact @@ -81,5 +81,5 @@ jobs: uses: actions/upload-artifact@v4 with: name: crate-${{ matrix.app }} - path: /tmp/flcrate_out/ro-crate/ + path: /tmp/flwrcrate-ro-crate/ if-no-files-found: warn diff --git a/.gitignore b/.gitignore index 3d22253..c06a8c7 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,7 @@ dist/ .eggs/ # Tool output -fl_crate_out/ +flwrcrate-ro-crate/ *.tmp # Test / coverage artifacts diff --git a/README.md b/README.md index 6b85616..e97be72 100644 --- a/README.md +++ b/README.md @@ -127,7 +127,7 @@ def main(grid: Grid, context: Context) -> None: with FLCrateTracker( context, strategy, - output_dir="/absolute/path/to/your-app/fl_crate_out", # absolute! + output_dir="/absolute/path/to/your-app/flwrcrate-ro-crate", # absolute! pyproject_path="/absolute/path/to/your-app/pyproject.toml", # absolute! app_name="My federated run", author={"name": "Your Name", "orcid": "https://orcid.org/0000-0000-0000-0000"}, @@ -142,7 +142,7 @@ def main(grid: Grid, context: Context) -> None: ) torch.save(result.arrays.to_torch_state_dict(), "final_model.pt") tracker.record_result(result, model_path="final_model.pt") - # On clean exit the RO-Crate is written to /ro-crate/ + # On clean exit, is the RO-Crate ``` That is the whole integration. On exit — **including on failure** — the crate @@ -186,7 +186,7 @@ as absolute paths. |---|---|---| | `context` | yes | The Flower `Context` (gives access to `run_config`) | | `strategy` | yes | Your strategy instance (FedAvg, FedProx, …) — class, module, and hyperparameters are captured | -| `output_dir` | recommended | Where to write outputs. **Use an absolute path.** Default: `./fl_crate_out` | +| `output_dir` | recommended | Where to write outputs. **Use an absolute path.** Default: `./flwrcrate-ro-crate` | | `pyproject_path` | recommended | Path to your app's `pyproject.toml`. **Use an absolute path.** Default: `"pyproject.toml"` | | `app_name` | no | Human-readable name for the crate's root dataset | | `author` | no | `"Name"` or `{"name": ..., "orcid": ..., "affiliation": ...}` — becomes a `Person` entity (ORCID as `@id`) | @@ -211,16 +211,21 @@ Mapped metrics get a `propertyID`; unmapped metrics still emit a plain ## Output +The output folder is the RO-Crate itself: + ``` -/ -├── captured_metadata.json # full capture: config, frameworks, strategy, timing, final metrics -├── metrics_log.json # per-round metrics + federation details (referenced by the crate) -└── ro-crate/ - ├── ro-crate-metadata.json - ├── final_model.pt # if a model_path was given - └── metrics_log.json +/ # default: ./flwrcrate-ro-crate +├── ro-crate-metadata.json +├── metrics_log.json # per-round metrics + federation details +└── final_model.pt # if a model_path was given ``` +During the run, flwrCrate also keeps `captured_metadata.json` there, a record of +everything captured so far. It is deleted once the crate is written, and kept if +writing the crate fails. If the folder contains files flwrCrate didn't write, it +logs a warning: they aren't described in the crate, but would travel with it if +the folder is shared. + ### What's inside `ro-crate-metadata.json` The crate's `@graph` contains, linked together: diff --git a/examples/quickstart-pytorch/server_app.py b/examples/quickstart-pytorch/server_app.py index 0b618ab..15a078e 100644 --- a/examples/quickstart-pytorch/server_app.py +++ b/examples/quickstart-pytorch/server_app.py @@ -45,7 +45,7 @@ def main(grid: Grid, context: Context) -> None: # relative paths resolve there, not in your project. with FLCrateTracker( context, strategy, - output_dir="/absolute/path/to/quickstart-pytorch/fl_crate_out", + output_dir="/absolute/path/to/quickstart-pytorch/flwrcrate-ro-crate", pyproject_path="/absolute/path/to/quickstart-pytorch/pyproject.toml", app_name="Quickstart PyTorch (demo)", author={"name": "Ali Faizollah", "orcid": "https://orcid.org/0009-0000-0000-0000"}, @@ -60,7 +60,7 @@ def main(grid: Grid, context: Context) -> None: ) torch.save(result.arrays.to_torch_state_dict(), "final_model.pt") tracker.record_result(result, model_path="final_model.pt") # flwrCrate (3/3b) - # On clean exit the RO-Crate is written to /ro-crate/ + # On clean exit, is the RO-Crate def global_evaluate(server_round: int, arrays: ArrayRecord) -> MetricRecord: diff --git a/flwrcrate/tracker.py b/flwrcrate/tracker.py index 2d7c4eb..b40fdcb 100755 --- a/flwrcrate/tracker.py +++ b/flwrcrate/tracker.py @@ -6,7 +6,7 @@ with FLCrateTracker( context, strategy, - output_dir="fl_crate_out", + output_dir="flwrcrate-ro-crate", author="Ali", license="http://spdx.org/licenses/MIT", ) as tracker: @@ -19,7 +19,7 @@ ) torch.save(result.arrays.to_torch_state_dict(), "final_model.pt") tracker.record_result(result, model_path="final_model.pt") - # On clean exit, the RO-Crate is written to output_dir/ro-crate/. + # On clean exit, output_dir itself becomes the RO-Crate. """ import json @@ -47,12 +47,12 @@ class FLCrateTracker: def __init__(self, context, strategy, output_dir=None, pyproject_path="pyproject.toml", app_name=None, author=None, license=None, agent=None): - self.output_dir = Path(output_dir) if output_dir else (Path.cwd() / "fl_crate_out") + self.output_dir = Path(output_dir) if output_dir else (Path.cwd() / "flwrcrate-ro-crate") self.output_dir.mkdir(parents=True, exist_ok=True) self.pyproject_path = pyproject_path self.capture_path = self.output_dir / "captured_metadata.json" self.metrics_log_path = self.output_dir / "metrics_log.json" - self.crate_dir = self.output_dir / "ro-crate" + self.crate_dir = self.output_dir # the output folder is the crate itself self.model_path = None self._built = False @@ -200,6 +200,24 @@ def record_result(self, result, model_path=None): self._capture["final_metrics"] = self._final_round_metrics() self._save_capture() + def _warn_about_unrelated_files(self): + """Warn about files in the crate folder that flwrCrate didn't write. + + They aren't described in the crate, but they travel with it whenever the + folder is shared. Hidden files such as .DS_Store are ignored. + """ + ours = {"ro-crate-metadata.json", self.metrics_log_path.name} + if self.model_path: + ours.add(self.model_path.name) + others = sorted(p.name for p in self.crate_dir.iterdir() + if p.name not in ours and not p.name.startswith(".")) + if others: + logger.warning( + "The RO-Crate folder %s also contains files flwrCrate did not write: %s. " + "They are not described in the crate, but would travel with it if the " + "folder is shared.", self.crate_dir, ", ".join(others) + ) + def build(self): """Write the RO-Crate. Called automatically on clean exit.""" if self._built: @@ -216,6 +234,9 @@ def build(self): agent=self.agent, ) self._built = True + # The capture is only a fallback record for runs whose crate can't be written. + self.capture_path.unlink(missing_ok=True) + self._warn_about_unrelated_files() logger.info("RO-Crate written to %s", self.crate_dir) return self.crate_dir diff --git a/tests/e2e/sklearn_ci_server_app.py b/tests/e2e/sklearn_ci_server_app.py index 93a464c..4ff80ba 100644 --- a/tests/e2e/sklearn_ci_server_app.py +++ b/tests/e2e/sklearn_ci_server_app.py @@ -38,7 +38,7 @@ def main(grid: Grid, context: Context) -> None: with FLCrateTracker( context, strategy, - output_dir="/tmp/flcrate_out", + output_dir="/tmp/flwrcrate-ro-crate", pyproject_path="/tmp/flcrate_pyproject.toml", app_name="Quickstart scikit-learn (CI e2e)", author={"name": "flwrCrate CI", "orcid": "https://orcid.org/0000-0000-0000-0000"}, diff --git a/tests/test_integration.py b/tests/test_integration.py index feb3d9a..c2a7495 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -15,7 +15,7 @@ def _graph(out_dir): - data = json.loads((out_dir / "ro-crate" / "ro-crate-metadata.json").read_text()) + data = json.loads((out_dir / "ro-crate-metadata.json").read_text()) return {e["@id"]: e for e in data["@graph"]} @@ -40,6 +40,9 @@ def test_full_lifecycle_produces_complete_crate( tracker.record_result(fake_result, model_path=str(model)) g = _graph(out) + # the output folder is the crate itself: no sub-folder, and the capture file is gone + assert sorted(p.name for p in out.iterdir()) == [ + "final_model.pt", "metrics_log.json", "ro-crate-metadata.json"] # all five profile fixes present assert g["./"]["mentions"] == [{"@id": "#fl-run"}] # #1 @@ -119,3 +122,34 @@ def test_federation_summary_picks_up_config_hints( assert "fraction-evaluate" in fed assert "min-available-clients" in fed assert "learning-rate" not in fed # not a federation key + + +def test_capture_kept_when_crate_cannot_be_written( + tmp_path, fake_context, fake_strategy, app_pyproject, monkeypatch +): + """captured_metadata.json is deleted after a successful build, so it must + survive as the run's only record when the build fails.""" + def broken_build(*args, **kwargs): + raise OSError("disk full") + + monkeypatch.setattr("flwrcrate.tracker.build_crate", broken_build) + out = tmp_path / "out" + with FLCrateTracker(fake_context, fake_strategy, output_dir=str(out), + pyproject_path=str(app_pyproject)): + pass + + assert (out / "captured_metadata.json").exists() + + +def test_warns_about_files_flwrcrate_did_not_write( + tmp_path, fake_context, fake_strategy, app_pyproject, caplog +): + """Files already in the output folder would travel with the crate, so say so.""" + out = tmp_path / "out" + out.mkdir() + (out / "patients.csv").write_text("id,age\n1,42\n") + with FLCrateTracker(fake_context, fake_strategy, output_dir=str(out), + pyproject_path=str(app_pyproject)): + pass + + assert "patients.csv" in caplog.text