diff --git a/.github/workflows/realapps.yml b/.github/workflows/realapps.yml index e98e35c..87afe39 100644 --- a/.github/workflows/realapps.yml +++ b/.github/workflows/realapps.yml @@ -58,13 +58,13 @@ jobs: - name: Run the federation working-directory: ${{ runner.temp }}/${{ matrix.app }} run: | - rm -rf /tmp/flcrate_out + rm -rf /tmp/flwrcrate-ro-crate # flwr run detaches the simulation when there's no TTY, so we kick it # off and then poll for the crate within this same step (keeping the # step alive so the background sim isn't killed). flwr run . --run-config "num-server-rounds=2" || true for i in $(seq 1 90); do - if [ -f /tmp/flcrate_out/ro-crate/ro-crate-metadata.json ]; then + if [ -f /tmp/flwrcrate-ro-crate/ro-crate-metadata.json ]; then echo "crate appeared after ~$((i*5))s"; break fi sleep 5 @@ -82,7 +82,7 @@ jobs: /tmp/validator/bin/pip install -e . --no-deps /tmp/validator/bin/rocrate-validator cache warm -p ro-crate-1.2 -p process-run-crate-0.5 /tmp/validator/bin/python tests/e2e/validate_crate.py \ - /tmp/flcrate_out/ro-crate/ro-crate-metadata.json \ + /tmp/flwrcrate-ro-crate/ro-crate-metadata.json \ "${{ matrix.strategy }}" "${{ matrix.framework }}" - name: Upload the crate as an artifact @@ -90,5 +90,5 @@ jobs: uses: actions/upload-artifact@v4 with: name: crate-${{ matrix.app }} - path: /tmp/flcrate_out/ro-crate/ + path: /tmp/flwrcrate-ro-crate/ if-no-files-found: warn diff --git a/.gitignore b/.gitignore index 3d22253..c06a8c7 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,7 @@ dist/ .eggs/ # Tool output -fl_crate_out/ +flwrcrate-ro-crate/ *.tmp # Test / coverage artifacts diff --git a/README.md b/README.md index d479c1b..4d6369d 100644 --- a/README.md +++ b/README.md @@ -127,7 +127,7 @@ def main(grid: Grid, context: Context) -> None: with FLCrateTracker( context, strategy, - output_dir="/absolute/path/to/your-app/fl_crate_out", # absolute! + output_dir="/absolute/path/to/your-app/flwrcrate-ro-crate", # absolute! pyproject_path="/absolute/path/to/your-app/pyproject.toml", # absolute! app_name="My federated run", author={"name": "Your Name", "orcid": "https://orcid.org/0000-0000-0000-0000"}, @@ -142,7 +142,7 @@ def main(grid: Grid, context: Context) -> None: ) torch.save(result.arrays.to_torch_state_dict(), "final_model.pt") tracker.record_result(result, model_path="final_model.pt") - # On clean exit the RO-Crate is written to /ro-crate/ + # On clean exit, is the RO-Crate ``` That is the whole integration. On exit — **including on failure** — the crate @@ -186,7 +186,7 @@ as absolute paths. |---|---|---| | `context` | yes | The Flower `Context` (gives access to `run_config`) | | `strategy` | yes | Your strategy instance (FedAvg, FedProx, …) — class, module, and hyperparameters are captured | -| `output_dir` | recommended | Where to write outputs. **Use an absolute path.** Default: `./fl_crate_out` | +| `output_dir` | recommended | Where to write outputs. **Use an absolute path.** Default: `./flwrcrate-ro-crate` | | `pyproject_path` | recommended | Path to your app's `pyproject.toml`. **Use an absolute path.** Default: `"pyproject.toml"` | | `app_name` | no | Human-readable name for the crate's root dataset | | `author` | no | `"Name"` or `{"name": ..., "orcid": ..., "affiliation": ...}` — becomes a `Person` entity (ORCID as `@id`) | @@ -211,16 +211,21 @@ Mapped metrics get a `propertyID`; unmapped metrics still emit a plain ## Output +The output folder is the RO-Crate itself: + ``` -/ -├── captured_metadata.json # full capture: config, frameworks, strategy, timing, final metrics -├── metrics_log.json # per-round metrics + federation details (referenced by the crate) -└── ro-crate/ - ├── ro-crate-metadata.json - ├── final_model.pt # if a model_path was given - └── metrics_log.json +/ # default: ./flwrcrate-ro-crate +├── ro-crate-metadata.json +├── metrics_log.json # per-round metrics + federation details +└── final_model.pt # if a model_path was given ``` +During the run, flwrCrate also keeps `captured_metadata.json` there, a record of +everything captured so far. It is deleted once the crate is written, and kept if +writing the crate fails. If the folder contains files flwrCrate didn't write, it +logs a warning: they aren't described in the crate, but would travel with it if +the folder is shared. + ### What's inside `ro-crate-metadata.json` The crate's `@graph` contains, linked together: diff --git a/examples/quickstart-pytorch/server_app.py b/examples/quickstart-pytorch/server_app.py index 0b618ab..15a078e 100644 --- a/examples/quickstart-pytorch/server_app.py +++ b/examples/quickstart-pytorch/server_app.py @@ -45,7 +45,7 @@ def main(grid: Grid, context: Context) -> None: # relative paths resolve there, not in your project. with FLCrateTracker( context, strategy, - output_dir="/absolute/path/to/quickstart-pytorch/fl_crate_out", + output_dir="/absolute/path/to/quickstart-pytorch/flwrcrate-ro-crate", pyproject_path="/absolute/path/to/quickstart-pytorch/pyproject.toml", app_name="Quickstart PyTorch (demo)", author={"name": "Ali Faizollah", "orcid": "https://orcid.org/0009-0000-0000-0000"}, @@ -60,7 +60,7 @@ def main(grid: Grid, context: Context) -> None: ) torch.save(result.arrays.to_torch_state_dict(), "final_model.pt") tracker.record_result(result, model_path="final_model.pt") # flwrCrate (3/3b) - # On clean exit the RO-Crate is written to /ro-crate/ + # On clean exit, is the RO-Crate def global_evaluate(server_round: int, arrays: ArrayRecord) -> MetricRecord: diff --git a/flwrcrate/tracker.py b/flwrcrate/tracker.py index 2d7c4eb..b40fdcb 100755 --- a/flwrcrate/tracker.py +++ b/flwrcrate/tracker.py @@ -6,7 +6,7 @@ with FLCrateTracker( context, strategy, - output_dir="fl_crate_out", + output_dir="flwrcrate-ro-crate", author="Ali", license="http://spdx.org/licenses/MIT", ) as tracker: @@ -19,7 +19,7 @@ ) torch.save(result.arrays.to_torch_state_dict(), "final_model.pt") tracker.record_result(result, model_path="final_model.pt") - # On clean exit, the RO-Crate is written to output_dir/ro-crate/. + # On clean exit, output_dir itself becomes the RO-Crate. """ import json @@ -47,12 +47,12 @@ class FLCrateTracker: def __init__(self, context, strategy, output_dir=None, pyproject_path="pyproject.toml", app_name=None, author=None, license=None, agent=None): - self.output_dir = Path(output_dir) if output_dir else (Path.cwd() / "fl_crate_out") + self.output_dir = Path(output_dir) if output_dir else (Path.cwd() / "flwrcrate-ro-crate") self.output_dir.mkdir(parents=True, exist_ok=True) self.pyproject_path = pyproject_path self.capture_path = self.output_dir / "captured_metadata.json" self.metrics_log_path = self.output_dir / "metrics_log.json" - self.crate_dir = self.output_dir / "ro-crate" + self.crate_dir = self.output_dir # the output folder is the crate itself self.model_path = None self._built = False @@ -200,6 +200,24 @@ def record_result(self, result, model_path=None): self._capture["final_metrics"] = self._final_round_metrics() self._save_capture() + def _warn_about_unrelated_files(self): + """Warn about files in the crate folder that flwrCrate didn't write. + + They aren't described in the crate, but they travel with it whenever the + folder is shared. Hidden files such as .DS_Store are ignored. + """ + ours = {"ro-crate-metadata.json", self.metrics_log_path.name} + if self.model_path: + ours.add(self.model_path.name) + others = sorted(p.name for p in self.crate_dir.iterdir() + if p.name not in ours and not p.name.startswith(".")) + if others: + logger.warning( + "The RO-Crate folder %s also contains files flwrCrate did not write: %s. " + "They are not described in the crate, but would travel with it if the " + "folder is shared.", self.crate_dir, ", ".join(others) + ) + def build(self): """Write the RO-Crate. Called automatically on clean exit.""" if self._built: @@ -216,6 +234,9 @@ def build(self): agent=self.agent, ) self._built = True + # The capture is only a fallback record for runs whose crate can't be written. + self.capture_path.unlink(missing_ok=True) + self._warn_about_unrelated_files() logger.info("RO-Crate written to %s", self.crate_dir) return self.crate_dir diff --git a/tests/e2e/sklearn_ci_server_app.py b/tests/e2e/sklearn_ci_server_app.py index 93a464c..4ff80ba 100644 --- a/tests/e2e/sklearn_ci_server_app.py +++ b/tests/e2e/sklearn_ci_server_app.py @@ -38,7 +38,7 @@ def main(grid: Grid, context: Context) -> None: with FLCrateTracker( context, strategy, - output_dir="/tmp/flcrate_out", + output_dir="/tmp/flwrcrate-ro-crate", pyproject_path="/tmp/flcrate_pyproject.toml", app_name="Quickstart scikit-learn (CI e2e)", author={"name": "flwrCrate CI", "orcid": "https://orcid.org/0000-0000-0000-0000"}, diff --git a/tests/test_integration.py b/tests/test_integration.py index af16ef4..6702b23 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -15,7 +15,7 @@ def _graph(out_dir): - data = json.loads((out_dir / "ro-crate" / "ro-crate-metadata.json").read_text()) + data = json.loads((out_dir / "ro-crate-metadata.json").read_text()) return {e["@id"]: e for e in data["@graph"]} @@ -40,6 +40,9 @@ def test_full_lifecycle_produces_complete_crate( tracker.record_result(fake_result, model_path=str(model)) g = _graph(out) + # the output folder is the crate itself: no sub-folder, and the capture file is gone + assert sorted(p.name for p in out.iterdir()) == [ + "final_model.pt", "metrics_log.json", "ro-crate-metadata.json"] # all five profile fixes present assert g["./"]["mentions"] == [{"@id": "#fl-run"}] # #1 @@ -119,3 +122,34 @@ def test_federation_summary_picks_up_config_hints( assert "fraction-evaluate" in fed assert "min-available-clients" in fed assert "learning-rate" not in fed # not a federation key + + +def test_capture_kept_when_crate_cannot_be_written( + tmp_path, fake_context, fake_strategy, app_pyproject, monkeypatch +): + """captured_metadata.json is deleted after a successful build, so it must + survive as the run's only record when the build fails.""" + def broken_build(*args, **kwargs): + raise OSError("disk full") + + monkeypatch.setattr("flwrcrate.tracker.build_crate", broken_build) + out = tmp_path / "out" + with FLCrateTracker(fake_context, fake_strategy, output_dir=str(out), + pyproject_path=str(app_pyproject)): + pass + + assert (out / "captured_metadata.json").exists() + + +def test_warns_about_files_flwrcrate_did_not_write( + tmp_path, fake_context, fake_strategy, app_pyproject, caplog +): + """Files already in the output folder would travel with the crate, so say so.""" + out = tmp_path / "out" + out.mkdir() + (out / "patients.csv").write_text("id,age\n1,42\n") + with FLCrateTracker(fake_context, fake_strategy, output_dir=str(out), + pyproject_path=str(app_pyproject)): + pass + + assert "patients.csv" in caplog.text