From 5ef958fb801d7e5c385457aefdd9954a27076cee Mon Sep 17 00:00:00 2001 From: galargh Date: Thu, 13 Aug 2026 15:44:37 +0200 Subject: [PATCH] feat: add top-level dependency manifest --- .github/workflows/ci_run.yml | 6 +- README_ADVANCED.md | 12 +- build.rs | 1 + ci/README.md | 133 +++++------- ci/dependency-profiles.json | 130 ------------ dependencies.toml | 71 +++++++ renovate.json | 12 +- scenarios/dependencies.py | 32 +-- ...ependencies.py => resolve-dependencies.py} | 155 +++++++++++--- ...encies.py => test_resolve_dependencies.py} | 192 ++++++++++++------ src/commands/config.rs | 14 +- src/commands/init/config.rs | 40 +++- src/config.rs | 139 +++++++++++-- 13 files changed, 566 insertions(+), 371 deletions(-) delete mode 100644 ci/dependency-profiles.json create mode 100644 dependencies.toml rename scripts/{resolve-ci-dependencies.py => resolve-dependencies.py} (76%) rename scripts/tests/{test_resolve_ci_dependencies.py => test_resolve_dependencies.py} (83%) diff --git a/.github/workflows/ci_run.yml b/.github/workflows/ci_run.yml index c1cf06d4..e4f77b7a 100644 --- a/.github/workflows/ci_run.yml +++ b/.github/workflows/ci_run.yml @@ -17,7 +17,7 @@ on: required: true type: string profile: - description: 'Dependency profile declared in ci/dependency-profiles.json' + description: 'Dependency profile declared in dependencies.toml' required: true type: string enable_reporting: @@ -133,7 +133,7 @@ jobs: id: dependencies run: | METADATA="$RUNNER_TEMP/ci-dependencies.json" - python3 scripts/resolve-ci-dependencies.py resolve \ + python3 scripts/resolve-dependencies.py resolve \ --profile '${{ inputs.profile }}' \ --output "$METADATA" \ --github-output "$GITHUB_OUTPUT" \ @@ -214,7 +214,7 @@ jobs: - name: "CHECK: {Verify dependency checkouts}" id: verified-dependencies run: | - python3 scripts/resolve-ci-dependencies.py verify \ + python3 scripts/resolve-dependencies.py verify \ --metadata "$CI_DEPENDENCY_METADATA" \ --github-output "$GITHUB_OUTPUT" diff --git a/README_ADVANCED.md b/README_ADVANCED.md index b02c5f2b..274467cd 100644 --- a/README_ADVANCED.md +++ b/README_ADVANCED.md @@ -284,12 +284,14 @@ branch = "main" ### How Defaults Work -Defaults are defined in code (see [`src/config.rs`](src/config.rs) `Config::default()`) and written to `config.toml` during `init`. This means: +Dependency defaults are defined in [`dependencies.toml`](dependencies.toml) and +embedded into the binary at build time. They are written to `config.toml` during +`init`. This means: -- **First-time setup:** Running `foc-devnet init` creates `config.toml` with current defaults from code +- **First-time setup:** Running `foc-devnet init` creates `config.toml` with current defaults from the embedded manifest - **Updating defaults:** When a new version of `foc-devnet` includes updated defaults (e.g., newer Lotus version), run `foc-devnet clean --all` then `foc-devnet init` to regenerate `config.toml` with the new defaults - **Preserving config across re-init:** Running `foc-devnet clean` (without `--all`) preserves your `config.toml`, so a subsequent `init` reuses your existing settings -- **Source of truth:** The code defines what defaults are available; `config.toml` stores your specific configuration +- **Source of truth:** `dependencies.toml` defines dependency defaults; `config.toml` stores your specific configuration ### Editing Config @@ -820,7 +822,7 @@ port_range_count = 100 - **[multicall3](https://github.com/mds1/multicall3)** - Multicall3 contract ### Dependent Version Strategy -Default versions for these repositories are defined in code (see [`src/config.rs`](src/config.rs) `Config::default()`). +Default versions for these repositories are defined in [`dependencies.toml`](dependencies.toml). **Version specification methods:** - **Latest tag** (`latesttag`, `latesttag:`, `latesttag::`): Resolved once at `init` time via `git ls-remote` and pinned as a concrete `GitTag` in `config.toml`. Use a glob selector to scope which tags are considered, e.g. `latesttag:v*` or `latesttag:pdp/v*`. Bare `latesttag` matches all tags. @@ -1330,7 +1332,7 @@ Reports are written to `~/.foc-devnet/state/latest/scenario_report.md`. Scenarios run automatically in CI after the devnet starts. On nightly runs (or manual dispatch with `reporting` enabled), failures automatically create a GitHub issue with a full report. -CI resolves compatibility-sensitive dependencies from `ci/dependency-profiles.json`. +CI resolves compatibility-sensitive dependencies from `dependencies.toml`. Pull requests use the pinned `default` profile, while nightly `stability` runs use the latest final releases and nightly `frontier` runs pin current development branch heads to immutable commits. Nightly CI also runs manifest-declared mixed diff --git a/build.rs b/build.rs index d07d515b..99842443 100644 --- a/build.rs +++ b/build.rs @@ -59,6 +59,7 @@ fn main() { // Re-run if git info changes println!("cargo:rerun-if-changed=.git/HEAD"); println!("cargo:rerun-if-changed=.git/refs/heads/"); + println!("cargo:rerun-if-changed=dependencies.toml"); // Re-run if MockUSDFC contract files change println!("cargo:rerun-if-changed=contracts/MockUSDFC/src/MockUSDFC.sol"); diff --git a/ci/README.md b/ci/README.md index cc18d3b3..296f7b51 100644 --- a/ci/README.md +++ b/ci/README.md @@ -1,7 +1,7 @@ # CI Dependency Profiles -`dependency-profiles.json` is the central manifest for CI dependency selection. -Its resolver is located in `scripts/resolve-ci-dependencies.py`. +`dependencies.toml` is the central manifest for runtime defaults and CI +dependency selection. Its resolver is located in `scripts/resolve-dependencies.py`. ## Profiles @@ -19,22 +19,18 @@ The manifest declares valid profiles in its top-level `profiles` object: - `stability-frontier-pdp`: used by nightly CI to test stable releases except PDP, which is resolved from `frontier`. -Each component must define a selection for every component profile referenced by -the top-level profile definitions. Today those component selections are -`default`, `stability`, and `frontier`. +Components can define `default`, `stability`, and `frontier` selections. When a +profile selects a component profile that the component does not define, the +resolver uses that component's `default` selection unless the profile explicitly +overrides that component. Top-level profile definitions have a `base` component profile and can override specific components: -```json -{ - "stability-frontier-curio": { - "base": "stability", - "components": { - "curio": "frontier" - } - } -} +```toml +[profiles.stability-frontier-curio] +base = "stability" +curio = "frontier" ``` In that example, Curio resolves from its `frontier` selection while every other @@ -46,45 +42,38 @@ exist unless added there. Top-level component fields: -- `repository`: Git repository URL. +- `git`: Git repository URL. - `npm_package`: npm package name, for components that are resolved through npm metadata. - `default`, `stability`, `frontier`: component profile selections. -Profile selections always have a `strategy`. Some strategies require additional -fields. +Profile selections are inline TOML tables. The resolver infers the strategy from +the keys present in each selection. -### `config_default` +### `bundled` -Use the compiled `Config::default()` value and pass no runtime override to -`foc-devnet init`. +Use the component bundled by another dependency and pass no runtime override to +`foc-devnet init`. PDP uses this in the `default` profile so the runtime default +continues to use filecoin-services' bundled submodule. -```json -{ - "strategy": "config_default" -} +```toml +default = { bundled = true } ``` ### `git_commit` Use an exact Git commit SHA. -```json -{ - "strategy": "git_commit", - "commit": "fadc836e65804311aca3bd2276861acabe42313f" -} +```toml +default = { commit = "fadc836e65804311aca3bd2276861acabe42313f" } ``` ### `git_branch` Resolve a branch head to an immutable commit SHA before the run starts. -```json -{ - "strategy": "git_branch", - "branch": "master" -} +```toml +frontier = { branch = "master" } ``` The resolved metadata records both the branch name and the exact commit. @@ -93,70 +82,49 @@ The resolved metadata records both the branch name and the exact commit. Resolve a Git tag to an immutable commit SHA. `tag` can be an exact tag: -```json -{ - "strategy": "git_tag", - "tag": "v1.2.3" -} +```toml +default = { tag = "v1.2.3" } ``` `tag` can also be a pattern. Pattern selections choose the latest matching tag: -```json -{ - "strategy": "git_tag", - "tag": "v*" -} +```toml +stability = { tag_pattern = "v*" } ``` By default, pattern selections exclude prerelease tags such as `-rc`, `-alpha`, `-beta`, and development tags. Set `include_prereleases` to include them: -```json -{ - "strategy": "git_tag", - "tag": "v*", - "include_prereleases": true -} +```toml +stability = { tag_pattern = "v*", include_prereleases = true } ``` ### `git_submodule` Resolve a git submodule gitlink from a tag or tag pattern in another repository. -```json -{ - "strategy": "git_submodule", - "repository": "https://github.com/FilOzone/filecoin-services.git", - "tag": "v*", - "path": "service_contracts/lib/pdp" -} +```toml +stability = { submodule_git = "https://github.com/FilOzone/filecoin-services.git", tag_pattern = "v*", path = "service_contracts/lib/pdp" } ``` -The resolver first resolves `repository` and `tag` with the same rules as -`git_tag`, then reads `path` from that tree and records the submodule gitlink SHA -as the selected component commit. PDP uses this to pin the same bundled PDP -gitlink as the selected filecoin-services stability tag, even in mixed profiles -that override filecoin-services itself. +The resolver first resolves `submodule_git` and `tag_pattern` with the same +rules as `git_tag`, then reads `path` from that tree and records the submodule +gitlink SHA as the selected component commit. PDP uses this to pin the same +bundled PDP gitlink as the selected filecoin-services stability tag, even in +mixed profiles that override filecoin-services itself. -### `npm_version` +### `npm` Resolve an npm version, range, or dist-tag to a concrete package version. -```json -{ - "strategy": "npm_version", - "version": "1.0.1" -} +```toml +default = { npm = "1.0.1" } ``` -The `version` field can also be an npm dist-tag: +The `npm` field can also be an npm dist-tag: -```json -{ - "strategy": "npm_version", - "version": "latest" -} +```toml +stability = { npm = "latest" } ``` The resolver records the concrete package version selected at resolution time @@ -168,17 +136,8 @@ Some profile selections can include an optional `overrides` object. Each entry maps a package name to a `version` and a `reason` explaining why the override exists: -```json -{ - "strategy": "git_tag", - "tag": "synapse-sdk-v1.0.1", - "overrides": { - "nanoid": { - "version": "3.3.13", - "reason": "nanoid 5.x is ESM-only and breaks the CJS build" - } - } -} +```toml +default = { tag = "synapse-sdk-v1.0.1", overrides = { nanoid = { version = "3.3.13", reason = "nanoid 5.x is ESM-only and breaks the CJS build" } } } ``` Overrides are explicit profile policy. Both `version` and `reason` are required @@ -189,7 +148,7 @@ override is applied. The resolver does not infer overrides from package metadata Overrides are currently allowed only for: - `synapse-sdk`, because scenario setup controls its pnpm install. -- `filecoin-pin` selections using `npm_version`, because those install into a +- `filecoin-pin` selections using `npm`, because those install into a temporary npm project controlled by the scenario. Current consumers: @@ -201,7 +160,7 @@ Current consumers: ## Current Boundary -`resolve-ci-dependencies.py` resolves metadata. It does **not** install +`resolve-dependencies.py` resolves metadata. It does **not** install components. Installation currently lives in three places (which consume the resolved diff --git a/ci/dependency-profiles.json b/ci/dependency-profiles.json deleted file mode 100644 index babc37fa..00000000 --- a/ci/dependency-profiles.json +++ /dev/null @@ -1,130 +0,0 @@ -{ - "schema_version": 2, - "profiles": { - "default": { - "base": "default" - }, - "stability": { - "base": "stability" - }, - "frontier": { - "base": "frontier" - }, - "stability-frontier-lotus": { - "base": "stability", - "components": { - "lotus": "frontier" - } - }, - "stability-frontier-curio": { - "base": "stability", - "components": { - "curio": "frontier" - } - }, - "stability-frontier-filecoin-services": { - "base": "stability", - "components": { - "filecoin-services": "frontier" - } - }, - "stability-frontier-pdp": { - "base": "stability", - "components": { - "pdp": "frontier" - } - } - }, - "components": { - "lotus": { - "repository": "https://github.com/filecoin-project/lotus.git", - "default": { - "strategy": "config_default" - }, - "stability": { - "strategy": "git_tag", - "tag": "v*" - }, - "frontier": { - "strategy": "git_branch", - "branch": "master" - } - }, - "curio": { - "repository": "https://github.com/filecoin-project/curio.git", - "default": { - "strategy": "config_default" - }, - "stability": { - "strategy": "git_tag", - "tag": "v*" - }, - "frontier": { - "strategy": "git_branch", - "branch": "main" - } - }, - "filecoin-services": { - "repository": "https://github.com/FilOzone/filecoin-services.git", - "default": { - "strategy": "config_default" - }, - "stability": { - "strategy": "git_tag", - "tag": "v*" - }, - "frontier": { - "strategy": "git_branch", - "branch": "main" - } - }, - "pdp": { - "repository": "https://github.com/FilOzone/pdp.git", - "default": { - "strategy": "config_default" - }, - "stability": { - "strategy": "git_submodule", - "repository": "https://github.com/FilOzone/filecoin-services.git", - "tag": "v*", - "path": "service_contracts/lib/pdp" - }, - "frontier": { - "strategy": "git_branch", - "branch": "main" - } - }, - "synapse-sdk": { - "repository": "https://github.com/FilOzone/synapse-sdk.git", - "npm_package": "@filoz/synapse-sdk", - "default": { - "strategy": "git_tag", - "tag": "synapse-sdk-v1.1.1" - }, - "stability": { - "strategy": "git_tag", - "tag": "synapse-sdk-v*" - }, - "frontier": { - "strategy": "git_branch", - "branch": "master" - } - }, - "filecoin-pin": { - "repository": "https://github.com/filecoin-project/filecoin-pin.git", - "npm_package": "filecoin-pin", - "default": { - "strategy": "npm_version", - "version": "1.3.0" - }, - "stability": { - "strategy": "npm_version", - "version": "latest" - }, - "frontier": { - "strategy": "git_branch", - "branch": "master" - } - } - } -} diff --git a/dependencies.toml b/dependencies.toml new file mode 100644 index 00000000..c9890f3a --- /dev/null +++ b/dependencies.toml @@ -0,0 +1,71 @@ +# Central dependency manifest for runtime defaults and CI/scenario profiles. +# Runtime defaults are embedded into the foc-devnet binary at compile time. + +schema_version = 1 + +[dependencies.lotus] +git = "https://github.com/filecoin-project/lotus.git" +default = { tag = "v1.36.2" } +stability = { tag_pattern = "v*" } +frontier = { branch = "master" } + +[dependencies.curio] +git = "https://github.com/filecoin-project/curio.git" +default = { tag = "v1.28.3" } +stability = { tag_pattern = "v*" } +frontier = { branch = "main" } + +[dependencies.filecoin-services] +git = "https://github.com/FilOzone/filecoin-services.git" +default = { tag = "v1.3.0" } +stability = { tag_pattern = "v*" } +frontier = { branch = "main" } + +[dependencies.pdp] +git = "https://github.com/FilOzone/pdp.git" +default = { bundled = true } +stability = { submodule_git = "https://github.com/FilOzone/filecoin-services.git", tag_pattern = "v*", path = "service_contracts/lib/pdp" } +frontier = { branch = "main" } + +[dependencies.multicall3] +git = "https://github.com/mds1/multicall3.git" +default = { tag = "v3.1.0" } + +[dev-dependencies.synapse-sdk] +git = "https://github.com/FilOzone/synapse-sdk.git" +npm_package = "@filoz/synapse-sdk" +default = { tag = "synapse-sdk-v1.1.1" } +stability = { tag_pattern = "synapse-sdk-v*" } +frontier = { branch = "master" } + +[dev-dependencies.filecoin-pin] +git = "https://github.com/filecoin-project/filecoin-pin.git" +npm_package = "filecoin-pin" +default = { npm = "1.3.0" } +stability = { npm = "latest" } +frontier = { branch = "master" } + +[profiles.default] +base = "default" + +[profiles.stability] +base = "stability" + +[profiles.frontier] +base = "frontier" + +[profiles.stability-frontier-lotus] +base = "stability" +lotus = "frontier" + +[profiles.stability-frontier-curio] +base = "stability" +curio = "frontier" + +[profiles.stability-frontier-filecoin-services] +base = "stability" +filecoin-services = "frontier" + +[profiles.stability-frontier-pdp] +base = "stability" +pdp = "frontier" diff --git a/renovate.json b/renovate.json index 34ed6481..f0d58438 100644 --- a/renovate.json +++ b/renovate.json @@ -7,10 +7,10 @@ { "customType": "regex", "managerFilePatterns": [ - "/^src/config\\.rs$/" + "/^dependencies\\.toml$/" ], "matchStrings": [ - "url:\\s*\"https://github\\.com/(?[^\"]+?)(?:\\.git)?\"\\.to_string\\(\\),\\s*tag:\\s*\"(?v[0-9][^\"]*)\"" + "git\\s*=\\s*\"https://github\\.com/(?[^\"]+?)(?:\\.git)?\"\\s*\\n(?:[^\\n]*\\n){0,2}default\\s*=\\s*\\{\\s*tag\\s*=\\s*\"(?v[0-9][^\"]*)\"" ], "datasourceTemplate": "github-tags", "versioningTemplate": "semver" @@ -18,10 +18,10 @@ { "customType": "regex", "managerFilePatterns": [ - "/^ci/dependency-profiles\\.json$/" + "/^dependencies\\.toml$/" ], "matchStrings": [ - "\"tag\":\\s*\"synapse-sdk-v(?[0-9][^\"]*)\"" + "default\\s*=\\s*\\{\\s*tag\\s*=\\s*\"synapse-sdk-v(?[0-9][^\"]*)\"" ], "depNameTemplate": "FilOzone/synapse-sdk", "datasourceTemplate": "github-tags", @@ -31,10 +31,10 @@ { "customType": "regex", "managerFilePatterns": [ - "/^ci/dependency-profiles\\.json$/" + "/^dependencies\\.toml$/" ], "matchStrings": [ - "\"strategy\":\\s*\"npm_version\",\\s*\"version\":\\s*\"(?[0-9][^\"]*)\"" + "default\\s*=\\s*\\{\\s*npm\\s*=\\s*\"(?[0-9][^\"]*)\"" ], "depNameTemplate": "filecoin-pin", "datasourceTemplate": "npm", diff --git a/scenarios/dependencies.py b/scenarios/dependencies.py index e9e5d2b0..ff141384 100644 --- a/scenarios/dependencies.py +++ b/scenarios/dependencies.py @@ -5,6 +5,7 @@ import json import os +import tomllib from pathlib import Path @@ -27,37 +28,42 @@ def component(name: str) -> dict: if isinstance(value, dict): return value - manifest_path = Path(__file__).parents[1] / "ci" / "dependency-profiles.json" - manifest = json.loads(manifest_path.read_text()) - definition = manifest["components"].get(name) + manifest_path = Path(__file__).parents[1] / "dependencies.toml" + manifest = tomllib.loads(manifest_path.read_text()) + all_dependencies = { + **manifest.get("dependencies", {}), + **manifest.get("dev-dependencies", {}), + } + definition = all_dependencies.get(name) if not definition: raise RuntimeError(f"Dependency manifest has no {name!r} component") selection = definition["default"] fallback = { "name": name, - "repository": definition["repository"], - "strategy": selection["strategy"], + "repository": definition["git"], } - if "overrides" in selection: - fallback["overrides"] = selection["overrides"] - if selection["strategy"] == "git_commit": + if selection.get("bundled") is True: + fallback.update(source="bundled") + elif "commit" in selection: fallback.update( source="git", ref=selection["commit"], commit=selection["commit"], ) - elif selection["strategy"] == "git_tag": + elif "tag" in selection: fallback.update(source="git", ref=selection["tag"]) - elif selection["strategy"] == "npm_version": + elif "npm" in selection: fallback.update( source="npm", - package=definition["npm_package"], - version=selection["version"], + package=definition.get("npm_package", name), + version=selection["npm"], ) else: raise RuntimeError( - f"Local scenario fallback does not support {selection['strategy']!r}" + f"Local scenario fallback does not support {name!r} default selection" ) + if "overrides" in selection: + fallback["overrides"] = selection["overrides"] return fallback diff --git a/scripts/resolve-ci-dependencies.py b/scripts/resolve-dependencies.py similarity index 76% rename from scripts/resolve-ci-dependencies.py rename to scripts/resolve-dependencies.py index 0c87082d..b620348c 100644 --- a/scripts/resolve-ci-dependencies.py +++ b/scripts/resolve-dependencies.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Resolve CI dependency profiles to immutable versions and verify checkouts.""" +"""Resolve dependency profiles to immutable versions and verify checkouts.""" from __future__ import annotations @@ -12,9 +12,10 @@ import shlex import subprocess import tempfile +import tomllib from pathlib import Path -MANIFEST_SCHEMA_VERSION = 2 +MANIFEST_SCHEMA_VERSION = 1 INIT_COMPONENT_FLAGS = { "lotus": "--lotus", "curio": "--curio", @@ -40,29 +41,119 @@ def run_command(command: list[str]) -> str: def load_manifest(path: Path) -> dict: try: - manifest = json.loads(path.read_text()) - except (OSError, json.JSONDecodeError) as error: + raw_manifest = tomllib.loads(path.read_text()) + except (OSError, tomllib.TOMLDecodeError) as error: raise ResolutionError( f"Cannot load dependency manifest {path}: {error}" ) from error - if manifest.get("schema_version") != MANIFEST_SCHEMA_VERSION: + if raw_manifest.get("schema_version") != MANIFEST_SCHEMA_VERSION: raise ResolutionError( f"Dependency manifest schema_version must be {MANIFEST_SCHEMA_VERSION}" ) - components = manifest.get("components") - if not isinstance(components, dict): - raise ResolutionError("Dependency manifest must contain a components object") - profiles = manifest.get("profiles") - if not isinstance(profiles, dict): - raise ResolutionError("Dependency manifest must contain a profiles object") - - required = set(INIT_COMPONENT_FLAGS) | {"synapse-sdk", "filecoin-pin"} + dependencies = raw_manifest.get("dependencies") + if not isinstance(dependencies, dict): + raise ResolutionError("Dependency manifest must contain a dependencies table") + dev_dependencies = raw_manifest.get("dev-dependencies", {}) + if not isinstance(dev_dependencies, dict): + raise ResolutionError("Dependency manifest dev-dependencies must be a table") + + components = normalize_components({**dependencies, **dev_dependencies}) + profiles = normalize_profiles(raw_manifest.get("profiles")) + + required = set(INIT_COMPONENT_FLAGS) | { + "multicall3", + "synapse-sdk", + "filecoin-pin", + } missing = sorted(required - set(components)) if missing: raise ResolutionError(f"Dependency manifest is missing: {', '.join(missing)}") validate_profiles(profiles, components) - return manifest + return { + "schema_version": MANIFEST_SCHEMA_VERSION, + "profiles": profiles, + "components": components, + } + + +def normalize_components(raw_components: dict) -> dict: + components = {} + for name, raw_component in raw_components.items(): + if not isinstance(raw_component, dict): + raise ResolutionError(f"Dependency {name!r} must be a table") + repository = raw_component.get("git") + if not isinstance(repository, str) or not repository: + raise ResolutionError(f"Dependency {name!r} git must be a string") + component = {"repository": repository} + npm_package = raw_component.get("npm_package") + if npm_package is not None: + if not isinstance(npm_package, str) or not npm_package: + raise ResolutionError( + f"Dependency {name!r} npm_package must be a string" + ) + component["npm_package"] = npm_package + for selection_name, selection in raw_component.items(): + if selection_name in {"git", "npm_package"}: + continue + component[selection_name] = normalize_selection(name, selection) + components[name] = component + return components + + +def normalize_selection(component_name: str, selection) -> dict: + if not isinstance(selection, dict): + raise ResolutionError( + f"{component_name} selection must be an inline table or table" + ) + normalized = {key: value for key, value in selection.items() if key == "overrides"} + if selection.get("bundled") is True: + normalized["strategy"] = "bundled" + elif "commit" in selection: + normalized.update(strategy="git_commit", commit=selection["commit"]) + elif "branch" in selection: + normalized.update(strategy="git_branch", branch=selection["branch"]) + elif "submodule_git" in selection: + tag = selection.get("tag_pattern") or selection.get("tag") + if not tag: + raise ResolutionError( + f"{component_name} submodule selection must define tag or tag_pattern" + ) + normalized.update( + strategy="git_submodule", + repository=selection["submodule_git"], + tag=tag, + path=selection.get("path"), + ) + elif "tag" in selection: + normalized.update(strategy="git_tag", tag=selection["tag"]) + elif "tag_pattern" in selection: + normalized.update(strategy="git_tag", tag=selection["tag_pattern"]) + elif "npm" in selection: + normalized.update(strategy="npm_version", version=selection["npm"]) + else: + raise ResolutionError( + f"{component_name} selection must define one of tag, tag_pattern, " + "branch, commit, npm, submodule_git, or bundled" + ) + if "include_prereleases" in selection: + normalized["include_prereleases"] = selection["include_prereleases"] + return normalized + + +def normalize_profiles(raw_profiles) -> dict: + if not isinstance(raw_profiles, dict): + raise ResolutionError("Dependency manifest must contain a profiles table") + profiles = {} + for profile_name, raw_profile in raw_profiles.items(): + if not isinstance(raw_profile, dict): + raise ResolutionError(f"Profile {profile_name!r} must be a table") + profile = {"base": raw_profile.get("base")} + overrides = {key: value for key, value in raw_profile.items() if key != "base"} + if overrides: + profile["components"] = overrides + profiles[profile_name] = profile + return profiles def validate_profiles(profiles: dict, components: dict) -> None: @@ -97,6 +188,11 @@ def validate_profiles(profiles: dict, components: dict) -> None: "must be a string" ) if selection_profile not in component: + if ( + component_overrides.get(component_name) is None + and "default" in component + ): + continue raise ResolutionError( f"Profile {profile_name!r} selects {selection_profile!r} for " f"{component_name}, but that component has no such selection" @@ -111,10 +207,16 @@ def component_profile_map(manifest: dict, profile_name: str) -> dict[str, str]: profile = profiles[profile_name] base = profile["base"] component_overrides = profile.get("components", {}) - return { - component_name: component_overrides.get(component_name, base) - for component_name in manifest["components"] - } + component_profiles = {} + for component_name, component in manifest["components"].items(): + selection_profile = component_overrides.get(component_name, base) + if ( + selection_profile not in component + and component_name not in component_overrides + ): + selection_profile = "default" + component_profiles[component_name] = selection_profile + return component_profiles def parse_ls_remote(output: str) -> list[tuple[str, str]]: @@ -310,8 +412,8 @@ def resolve_component( "strategy": strategy, } - if strategy == "config_default": - resolved["source"] = "config_default" + if strategy == "bundled": + resolved["source"] = "bundled" elif strategy == "git_commit": commit = selection["commit"] if not COMMIT_RE.fullmatch(commit): @@ -383,7 +485,7 @@ def build_init_args(components: dict) -> list[str]: args = [] for name, flag in INIT_COMPONENT_FLAGS.items(): component = components[name] - if component["source"] == "config_default": + if component["source"] == "bundled": continue args.extend( [ @@ -449,7 +551,7 @@ def verify(args) -> None: metadata = json.loads(args.metadata.read_text()) components = metadata["components"] for name in INIT_COMPONENT_FLAGS: - if name == "pdp" and components[name]["source"] == "config_default": + if name == "pdp" and components[name]["source"] == "bundled": continue repository_path = args.code_dir / name actual = run_command(["git", "-C", str(repository_path), "rev-parse", "HEAD"]) @@ -475,7 +577,7 @@ def parser() -> argparse.ArgumentParser: resolve_parser = subparsers.add_parser("resolve") resolve_parser.add_argument("--profile", required=True) resolve_parser.add_argument( - "--manifest", type=Path, default=Path("ci/dependency-profiles.json") + "--manifest", type=Path, default=Path("dependencies.toml") ) resolve_parser.add_argument("--output", type=Path, required=True) resolve_parser.add_argument("--github-output") @@ -496,7 +598,12 @@ def main() -> None: args = parser().parse_args() try: args.handler(args) - except (ResolutionError, KeyError, json.JSONDecodeError) as error: + except ( + ResolutionError, + KeyError, + json.JSONDecodeError, + tomllib.TOMLDecodeError, + ) as error: raise SystemExit(f"dependency resolution failed: {error}") from error diff --git a/scripts/tests/test_resolve_ci_dependencies.py b/scripts/tests/test_resolve_dependencies.py similarity index 83% rename from scripts/tests/test_resolve_ci_dependencies.py rename to scripts/tests/test_resolve_dependencies.py index c009924f..12bd062c 100644 --- a/scripts/tests/test_resolve_ci_dependencies.py +++ b/scripts/tests/test_resolve_dependencies.py @@ -7,7 +7,7 @@ from pathlib import Path from unittest.mock import patch -SCRIPT = Path(__file__).parents[1] / "resolve-ci-dependencies.py" +SCRIPT = Path(__file__).parents[1] / "resolve-dependencies.py" SPEC = importlib.util.spec_from_file_location("dependency_resolver", SCRIPT) resolver = importlib.util.module_from_spec(SPEC) SPEC.loader.exec_module(resolver) @@ -30,14 +30,51 @@ def __call__(self, command): raise AssertionError(f"Unexpected command: {command}") +def toml_value(value): + if isinstance(value, str): + return json.dumps(value) + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, dict): + items = ", ".join(f"{key} = {toml_value(val)}" for key, val in value.items()) + return "{ " + items + " }" + raise TypeError(f"Unsupported TOML test value: {value!r}") + + +def write_manifest(path, manifest): + lines = [f"schema_version = {manifest['schema_version']}", ""] + for group in ("dependencies", "dev-dependencies"): + if not manifest.get(group, {}): + lines.append(f"[{group}]") + lines.append("") + for name, component in manifest.get(group, {}).items(): + lines.append(f"[{group}.{name}]") + for key, value in component.items(): + lines.append(f"{key} = {toml_value(value)}") + lines.append("") + for name, profile in manifest["profiles"].items(): + lines.append(f"[profiles.{name}]") + for key, value in profile.items(): + lines.append(f"{key} = {toml_value(value)}") + lines.append("") + path.write_text("\n".join(lines)) + + class ResolverTests(unittest.TestCase): - def manifest(self, profiles=None, components=None): - if components is None: - components = { + def manifest(self, profiles=None, dependencies=None, dev_dependencies=None): + if dependencies is None: + dependencies = { "lotus": self.component("a", "b"), "curio": self.component("c", "d"), "filecoin-services": self.component("e", "f"), "pdp": self.component("5", "6"), + "multicall3": { + "git": "https://example.test/multicall3.git", + "default": {"commit": "9" * 40}, + }, + } + if dev_dependencies is None: + dev_dependencies = { "synapse-sdk": self.component("1", "2"), "filecoin-pin": self.component("3", "4"), } @@ -48,47 +85,42 @@ def manifest(self, profiles=None, components=None): "frontier": {"base": "frontier"}, "stability-frontier-lotus": { "base": "stability", - "components": {"lotus": "frontier"}, + "lotus": "frontier", }, "stability-frontier-curio": { "base": "stability", - "components": {"curio": "frontier"}, + "curio": "frontier", }, "stability-frontier-filecoin-services": { "base": "stability", - "components": {"filecoin-services": "frontier"}, + "filecoin-services": "frontier", }, "stability-frontier-pdp": { "base": "stability", - "components": {"pdp": "frontier"}, + "pdp": "frontier", }, } return { - "schema_version": 2, + "schema_version": 1, "profiles": profiles, - "components": components, + "dependencies": dependencies, + "dev-dependencies": dev_dependencies, } def component(self, stability_prefix, frontier_prefix): return { - "repository": "https://example.test/project.git", - "default": {"strategy": "config_default"}, - "stability": { - "strategy": "git_commit", - "commit": stability_prefix * 40, - }, - "frontier": { - "strategy": "git_commit", - "commit": frontier_prefix * 40, - }, + "git": "https://example.test/project.git", + "default": {"commit": "0" * 40}, + "stability": {"commit": stability_prefix * 40}, + "frontier": {"commit": frontier_prefix * 40}, } def resolve_manifest(self, manifest, profile): with tempfile.TemporaryDirectory() as directory: directory = Path(directory) - manifest_path = directory / "manifest.json" + manifest_path = directory / "manifest.toml" output_path = directory / "resolved.json" - manifest_path.write_text(json.dumps(manifest)) + write_manifest(manifest_path, manifest) args = type( "Args", (), @@ -107,10 +139,10 @@ def resolve_manifest(self, manifest, profile): def resolve_manifest_with_github_output(self, manifest, profile): with tempfile.TemporaryDirectory() as directory: directory = Path(directory) - manifest_path = directory / "manifest.json" + manifest_path = directory / "manifest.toml" output_path = directory / "resolved.json" github_output_path = directory / "github-output" - manifest_path.write_text(json.dumps(manifest)) + write_manifest(manifest_path, manifest) args = type( "Args", (), @@ -131,18 +163,14 @@ def resolve_manifest_with_github_output(self, manifest, profile): def pdp_git_submodule_component(self): return { - "repository": "https://example.test/pdp.git", - "default": {"strategy": "config_default"}, + "git": "https://example.test/pdp.git", + "default": {"bundled": True}, "stability": { - "strategy": "git_submodule", - "repository": "https://example.test/filecoin-services.git", - "tag": "v*", + "submodule_git": "https://example.test/filecoin-services.git", + "tag_pattern": "v*", "path": "service_contracts/lib/pdp", }, - "frontier": { - "strategy": "git_commit", - "commit": "6" * 40, - }, + "frontier": {"commit": "6" * 40}, } def test_latest_non_prerelease_tag_excludes_prereleases_and_annotated_refs(self): @@ -209,9 +237,9 @@ def test_unknown_profile_fails(self): manifest = self.manifest() with tempfile.TemporaryDirectory() as directory: directory = Path(directory) - manifest_path = directory / "manifest.json" + manifest_path = directory / "manifest.toml" output_path = directory / "resolved.json" - manifest_path.write_text(json.dumps(manifest)) + write_manifest(manifest_path, manifest) args = type( "Args", (), @@ -228,15 +256,15 @@ def test_unknown_profile_fails(self): def test_manifest_missing_component_fails(self): with tempfile.TemporaryDirectory() as directory: - path = Path(directory) / "manifest.json" - path.write_text( - json.dumps( - { - "schema_version": 2, - "profiles": {"default": {"base": "default"}}, - "components": {}, - } - ) + path = Path(directory) / "manifest.toml" + write_manifest( + path, + { + "schema_version": 1, + "profiles": {"default": {"base": "default"}}, + "dependencies": {}, + "dev-dependencies": {}, + }, ) with self.assertRaisesRegex(resolver.ResolutionError, "missing"): resolver.load_manifest(path) @@ -277,7 +305,7 @@ def test_filecoin_services_mixed_profile_emits_pdp_git_submodule(self, run_comma parent_commit = "8" * 40 pdp_commit = "7" * 40 manifest = self.manifest() - manifest["components"]["pdp"] = self.pdp_git_submodule_component() + manifest["dependencies"]["pdp"] = self.pdp_git_submodule_component() run_command.side_effect = FakeRunner( [ ( @@ -344,7 +372,15 @@ def test_filecoin_services_mixed_profile_emits_pdp_git_submodule(self, run_comma @patch.object(resolver, "run_command") def test_git_submodule_rejects_missing_gitlink(self, run_command): parent_commit = "8" * 40 - component = self.pdp_git_submodule_component() + component = { + "repository": "https://example.test/pdp.git", + "stability": { + "strategy": "git_submodule", + "repository": "https://example.test/filecoin-services.git", + "tag": "v*", + "path": "service_contracts/lib/pdp", + }, + } run_command.side_effect = FakeRunner( [ ( @@ -388,7 +424,7 @@ def test_absent_mixed_profile_is_rejected(self): "frontier": {"base": "frontier"}, "stability-frontier-curio": { "base": "stability", - "components": {"curio": "frontier"}, + "curio": "frontier", }, } manifest = self.manifest(profiles=profiles) @@ -401,13 +437,13 @@ def test_manifest_profile_rejects_unknown_component_override(self): "default": {"base": "default"}, "bad": { "base": "stability", - "components": {"missing": "frontier"}, + "missing": "frontier", }, } ) with tempfile.TemporaryDirectory() as directory: - path = Path(directory) / "manifest.json" - path.write_text(json.dumps(manifest)) + path = Path(directory) / "manifest.toml" + write_manifest(path, manifest) with self.assertRaisesRegex(resolver.ResolutionError, "unknown components"): resolver.load_manifest(path) @@ -417,13 +453,13 @@ def test_manifest_profile_rejects_missing_component_selection(self): "default": {"base": "default"}, "bad": { "base": "stability", - "components": {"lotus": "not-a-selection"}, + "lotus": "not-a-selection", }, } ) with tempfile.TemporaryDirectory() as directory: - path = Path(directory) / "manifest.json" - path.write_text(json.dumps(manifest)) + path = Path(directory) / "manifest.toml" + write_manifest(path, manifest) with self.assertRaisesRegex(resolver.ResolutionError, "no such selection"): resolver.load_manifest(path) @@ -616,9 +652,13 @@ def test_git_commit_strategy_rejects_non_sha(self): "synapse-sdk", component, "default", FakeRunner({}) ) - def test_init_args_skip_config_defaults_and_pin_other_sources(self): + def test_init_args_skip_bundled_pdp_and_pin_git_sources(self): components = { - "lotus": {"source": "config_default"}, + "lotus": { + "source": "git", + "repository": "https://example.test/lotus.git", + "commit": "aaa", + }, "curio": { "source": "git", "repository": "https://example.test/curio.git", @@ -630,31 +670,49 @@ def test_init_args_skip_config_defaults_and_pin_other_sources(self): "commit": "def", }, "pdp": { - "source": "git", + "source": "bundled", "repository": "https://example.test/pdp.git", - "commit": "123", }, } self.assertEqual( resolver.build_init_args(components), [ + "--lotus", + "gitcommit:https://example.test/lotus.git:aaa", "--curio", "gitcommit:https://example.test/curio.git:abc", "--filecoin-services", "gitcommit:https://example.test/services.git:def", - "--pdp", - "gitcommit:https://example.test/pdp.git:123", ], ) - def test_init_args_skip_default_pdp(self): + def test_init_args_include_independent_pdp(self): components = { - "lotus": {"source": "config_default"}, - "curio": {"source": "config_default"}, - "filecoin-services": {"source": "config_default"}, - "pdp": {"source": "config_default"}, + "lotus": { + "source": "git", + "repository": "https://example.test/lotus.git", + "commit": "aaa", + }, + "curio": { + "source": "git", + "repository": "https://example.test/curio.git", + "commit": "bbb", + }, + "filecoin-services": { + "source": "git", + "repository": "https://example.test/services.git", + "commit": "ccc", + }, + "pdp": { + "source": "git", + "repository": "https://example.test/pdp.git", + "commit": "ddd", + }, } - self.assertEqual(resolver.build_init_args(components), []) + self.assertEqual( + resolver.build_init_args(components)[-2:], + ["--pdp", "gitcommit:https://example.test/pdp.git:ddd"], + ) def test_cache_hash_depends_only_on_lotus_and_curio_commits(self): base = { @@ -680,8 +738,8 @@ def test_verify_records_checkouts_and_writes_cache_key(self, run_command): "schema_version": 1, "profile": "default", "components": { - **{name: {"source": "config_default"} for name in commits}, - "pdp": {"source": "config_default"}, + **{name: {"source": "git"} for name in commits}, + "pdp": {"source": "bundled"}, }, } with tempfile.TemporaryDirectory() as directory: diff --git a/src/commands/config.rs b/src/commands/config.rs index d8deceea..c80b0c12 100644 --- a/src/commands/config.rs +++ b/src/commands/config.rs @@ -3,7 +3,7 @@ //! This module provides commands for updating the foc-devnet configuration, //! specifically for changing the source locations of Lotus and Curio components. -use crate::config::{Config, Location}; +use crate::config::{default_dependency_repository, Config, Location}; use crate::paths::foc_devnet_config; use std::fs; use tracing::info; @@ -17,11 +17,7 @@ use tracing::info; /// - `local:/path/to/lotus` (local directory) /// - `gittag:https://github.com/user/lotus.git:v1.0.0` (custom URL) pub fn config_lotus(source: String) -> Result<(), Box> { - update_config_location( - "lotus", - source, - "https://github.com/filecoin-project/lotus.git", - ) + update_config_location("lotus", source, &default_dependency_repository("lotus")) } /// Configure the Curio source location in the config file. @@ -33,11 +29,7 @@ pub fn config_lotus(source: String) -> Result<(), Box> { /// - `local:/path/to/curio` (local directory) /// - `gittag:https://github.com/user/curio.git:v1.0.0` (custom URL) pub fn config_curio(source: String) -> Result<(), Box> { - update_config_location( - "curio", - source, - "https://github.com/filecoin-project/curio.git", - ) + update_config_location("curio", source, &default_dependency_repository("curio")) } /// Internal function to update a location field in the config. diff --git a/src/commands/init/config.rs b/src/commands/init/config.rs index 85fc1db8..ff451723 100644 --- a/src/commands/init/config.rs +++ b/src/commands/init/config.rs @@ -6,7 +6,7 @@ use std::fs; use tracing::info; -use crate::config::{Config, Location}; +use crate::config::{default_dependency_repository, Config, Location}; use crate::paths::foc_devnet_config; /// Generate default configuration file if it doesn't exist. @@ -93,27 +93,27 @@ fn apply_overrides( apply_location_override( &mut config.lotus, lotus_location, - "https://github.com/filecoin-project/lotus.git", + &default_dependency_repository("lotus"), )?; apply_location_override( &mut config.curio, curio_location, - "https://github.com/filecoin-project/curio.git", + &default_dependency_repository("curio"), )?; apply_location_override( &mut config.filecoin_services, filecoin_services_location, - "https://github.com/FilOzone/filecoin-services.git", + &default_dependency_repository("filecoin-services"), )?; if let Some(pdp_location) = pdp_location { let mut pdp = config.pdp.clone().unwrap_or_else(|| Location::GitBranch { - url: "https://github.com/FilOzone/pdp.git".to_string(), + url: default_dependency_repository("pdp"), branch: "main".to_string(), }); apply_location_override( &mut pdp, Some(pdp_location), - "https://github.com/FilOzone/pdp.git", + &default_dependency_repository("pdp"), )?; config.pdp = Some(pdp); } @@ -150,3 +150,31 @@ pub fn apply_location_override( } Ok(()) } + +#[cfg(test)] +mod tests { + use super::apply_location_override; + use crate::config::{default_dependency_repository, Location}; + + #[test] + fn override_uses_manifest_repository_for_local_source_defaults() { + let mut location = Location::LocalSource { + dir: "/tmp/lotus".to_string(), + }; + + apply_location_override( + &mut location, + Some("gitbranch:test-branch".to_string()), + &default_dependency_repository("lotus"), + ) + .unwrap(); + + assert_eq!( + location, + Location::GitBranch { + url: default_dependency_repository("lotus"), + branch: "test-branch".to_string(), + } + ); + } +} diff --git a/src/config.rs b/src/config.rs index 15203517..5e8a9328 100644 --- a/src/config.rs +++ b/src/config.rs @@ -5,12 +5,14 @@ //! port allocations, and executable locations for various components. use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::sync::OnceLock; /// Represents the location of an executable or source code for a component. /// /// This enum allows specifying how to obtain and run different Filecoin-related /// executables (lotus, lotus-miner, curio) in various deployment scenarios. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum Location { /// Use a local directory containing source code that needs to be built. /// @@ -38,6 +40,79 @@ pub enum Location { GitBranch { url: String, branch: String }, } +static DEPENDENCY_MANIFEST: OnceLock = OnceLock::new(); + +#[derive(Debug, Deserialize)] +struct DependencyManifest { + schema_version: u32, + dependencies: HashMap, +} + +#[derive(Debug, Deserialize)] +struct DependencyDefinition { + git: String, + default: DependencySelection, +} + +#[derive(Debug, Deserialize)] +struct DependencySelection { + tag: Option, + commit: Option, + branch: Option, + bundled: Option, +} + +fn dependency_manifest() -> &'static DependencyManifest { + DEPENDENCY_MANIFEST.get_or_init(|| { + let manifest: DependencyManifest = toml::from_str(include_str!("../dependencies.toml")) + .expect("embedded dependencies.toml must be valid"); + assert_eq!( + manifest.schema_version, 1, + "embedded dependencies.toml schema_version must be 1" + ); + manifest + }) +} + +fn dependency_definition(name: &str) -> &'static DependencyDefinition { + dependency_manifest() + .dependencies + .get(name) + .unwrap_or_else(|| panic!("embedded dependencies.toml is missing {name}")) +} + +fn default_dependency_location(name: &str) -> Option { + let definition = dependency_definition(name); + let selection = &definition.default; + + if selection.bundled == Some(true) { + return None; + } + if let Some(tag) = &selection.tag { + return Some(Location::GitTag { + url: definition.git.clone(), + tag: tag.clone(), + }); + } + if let Some(commit) = &selection.commit { + return Some(Location::GitCommit { + url: definition.git.clone(), + commit: commit.clone(), + }); + } + if let Some(branch) = &selection.branch { + return Some(Location::GitBranch { + url: definition.git.clone(), + branch: branch.clone(), + }); + } + panic!("embedded dependencies.toml default for {name} must be a git location or bundled") +} + +pub fn default_dependency_repository(name: &str) -> String { + dependency_definition(name).git.clone() +} + impl Location { /// Given a url and a selector, finds the latest tag given that selector. /// @@ -284,23 +359,15 @@ impl Default for Config { Self { port_range_start: 5700, port_range_count: 100, - lotus: Location::GitTag { - url: "https://github.com/filecoin-project/lotus.git".to_string(), - tag: "v1.36.2".to_string(), - }, - curio: Location::GitTag { - url: "https://github.com/filecoin-project/curio.git".to_string(), - tag: "v1.28.3".to_string(), - }, - filecoin_services: Location::GitTag { - url: "https://github.com/FilOzone/filecoin-services.git".to_string(), - tag: "v1.3.0".to_string(), - }, - pdp: None, - multicall3: Location::GitTag { - url: "https://github.com/mds1/multicall3.git".to_string(), - tag: "v3.1.0".to_string(), - }, + lotus: default_dependency_location("lotus") + .expect("lotus default dependency must not be bundled"), + curio: default_dependency_location("curio") + .expect("curio default dependency must not be bundled"), + filecoin_services: default_dependency_location("filecoin-services") + .expect("filecoin-services default dependency must not be bundled"), + pdp: default_dependency_location("pdp"), + multicall3: default_dependency_location("multicall3") + .expect("multicall3 default dependency must not be bundled"), approved_pdp_sp_count: 2, endorsed_pdp_sp_count: 1, active_pdp_sp_count: 2, @@ -343,7 +410,7 @@ impl Config { #[cfg(test)] mod tests { - use super::{Config, Location}; + use super::{default_dependency_repository, Config, Location}; const DEFAULT_URL: &str = "https://github.com/default/repo.git"; @@ -479,6 +546,40 @@ mod tests { assert!(parsed.pdp.is_none()); } + #[test] + fn default_config_uses_manifest_dependency_defaults() { + let config = Config::default(); + + assert_eq!( + config.lotus, + Location::GitTag { + url: default_dependency_repository("lotus"), + tag: "v1.36.2".to_string(), + } + ); + assert_eq!( + config.curio, + Location::GitTag { + url: default_dependency_repository("curio"), + tag: "v1.28.3".to_string(), + } + ); + assert_eq!( + config.filecoin_services, + Location::GitTag { + url: default_dependency_repository("filecoin-services"), + tag: "v1.3.0".to_string(), + } + ); + assert_eq!( + config.multicall3, + Location::GitTag { + url: default_dependency_repository("multicall3"), + tag: "v3.1.0".to_string(), + } + ); + } + #[test] fn config_serializes_configured_pdp() { let config = Config {