diff --git a/CHANGELOG.md b/CHANGELOG.md index e248e2706..f7dc8b076 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ air-gapped signing # Unreleased +* feat: a canister that uses a `recipe` can now declare its own `sync` steps, which previously was rejected outright. They run after the sync steps the recipe renders, so a recipe's post-deployment work stays intact and yours is appended to it. `recipe` and `build` remain mutually exclusive. + # v1.4.0 * feat: a canister can now declare `upgrade_args` alongside `init_args`, in its own manifest and as a per-canister environment override. It is passed when `icp deploy` upgrades the canister, where `init_args` is passed when it installs or reinstalls it. It takes exactly the forms `init_args` does (inline Candid string, or `{ value | path, format }`), and paths resolve against the canister's own directory the same way. A canister that declares no `upgrade_args` is upgraded with its `init_args`, as before, and `--args` / `--args-file` still override whichever applies. diff --git a/crates/icp-cli/tests/recipe_tests.rs b/crates/icp-cli/tests/recipe_tests.rs index 85a889b2c..e10554f00 100644 --- a/crates/icp-cli/tests/recipe_tests.rs +++ b/crates/icp-cli/tests/recipe_tests.rs @@ -332,3 +332,67 @@ fn recipe_local_file_valid_checksum() { .assert() .success(); } + +/// A canister may declare sync steps alongside a recipe; they land after the +/// steps the recipe renders. +#[test] +fn recipe_with_manifest_sync_steps() { + let ctx = TestContext::new(); + + // Setup project + let project_dir = ctx.create_project_dir("icp"); + + // Recipe rendering a sync step of its own + write_string( + &project_dir.join("recipe.hbs"), // path + indoc! {r#" + build: + steps: + - type: script + command: echo "test" > "$ICP_WASM_OUTPUT_PATH" + sync: + steps: + - type: script + command: echo from-recipe + "#}, // contents + ) + .expect("failed to write recipe template"); + + let pm = indoc! {" + canisters: + - name: my-canister + recipe: + type: file://./recipe.hbs + sync: + steps: + - type: script + command: echo from-manifest + "}; + + write_string( + &project_dir.join("icp.yaml"), // path + pm, // contents + ) + .expect("failed to write project manifest"); + + // The effective configuration holds both steps, the recipe's first + let assert = ctx + .icp() + .current_dir(project_dir) + .args(["project", "show"]) + .assert() + .success(); + let stdout = String::from_utf8(assert.get_output().stdout.clone()) + .expect("`icp project show` output is not UTF-8"); + + let recipe_at = stdout + .find("echo from-recipe") + .unwrap_or_else(|| panic!("recipe's sync step missing from:\n{stdout}")); + let manifest_at = stdout + .find("echo from-manifest") + .unwrap_or_else(|| panic!("manifest's sync step missing from:\n{stdout}")); + assert!( + recipe_at < manifest_at, + "the manifest's sync step should follow the recipe's, got:\n{stdout}" + ); +} diff --git a/crates/icp/src/manifest/canister.rs b/crates/icp/src/manifest/canister.rs index 97ba42211..edc6c80b9 100644 --- a/crates/icp/src/manifest/canister.rs +++ b/crates/icp/src/manifest/canister.rs @@ -170,29 +170,23 @@ impl<'de> Deserialize<'de> for CanisterManifest { // let has_recipe = temp_map.contains_key(&recipe_key); let has_build = temp_map.contains_key(&build_key); - let has_sync = temp_map.contains_key(&sync_key); - match (has_recipe, has_build, has_sync) { - (true, true, _) => { + match (has_recipe, has_build) { + (true, true) => { // Can't have a recipe and a build Err(Error::custom(format!( "Canister {name} cannot have both a `recipe` and a `build` section" ))) } - (true, false, true) => { - // Can't have a recipe and a sync sections - Err(Error::custom(format!( - "Canister {name} cannot have both a `recipe` and a `sync` section" - ))) - } - (false, false, _) => { + (false, false) => { // We must have recipe or build Err(Error::custom(format!( "Canister {name} must have a `recipe` or a `build` section" ))) } - (true, false, false) => { - // We have a a recipe + (true, false) => { + // We have a recipe, optionally with sync steps of its + // own to run after the recipe's let recipe: Recipe = serde_yaml::from_value( temp_map .remove(&recipe_key) @@ -203,6 +197,19 @@ impl<'de> Deserialize<'de> for CanisterManifest { Error::custom(format!("Canister {name} failed to parse recipe: {}", e)) })?; + // An explicit `sync: null` deserializes to `None`, as + // it does in the build/sync variant + let sync: Option = + if let Some(sync_value) = temp_map.remove(&sync_key) { + serde_yaml::from_value(sync_value).map_err(|e| { + Error::custom(format!( + "Canister {name} failed to parse sync instructions: {e}" + )) + })? + } else { + None + }; + if !temp_map.is_empty() { return Err(Error::custom(format!( "Unrecognized fields in canister `{name}`." @@ -214,10 +221,10 @@ impl<'de> Deserialize<'de> for CanisterManifest { settings, init_args, upgrade_args, - instructions: Instructions::Recipe { recipe }, + instructions: Instructions::Recipe { recipe, sync }, }) } - (false, true, _) => { + (false, true) => { // We have a build section // Try to deserialize as BuildSync variant @@ -262,6 +269,10 @@ impl<'de> Deserialize<'de> for CanisterManifest { pub enum Instructions { Recipe { recipe: Recipe, + + /// Additional sync steps, run after the ones the recipe renders. + #[serde(skip_serializing_if = "Option::is_none")] + sync: Option, }, BuildSync { @@ -629,7 +640,8 @@ mod tests { recipe_type: RecipeType::File("my-recipe".to_string()), configuration: HashMap::new(), sha256: None, - } + }, + sync: None, }, }, ); @@ -659,7 +671,8 @@ mod tests { ("key-2".to_string(), "value-2".into()) ]), sha256: None, - } + }, + sync: None, }, }, ); @@ -691,7 +704,8 @@ mod tests { "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" .to_string() ), - } + }, + sync: None, }, }, ); @@ -722,12 +736,95 @@ mod tests { recipe_type: RecipeType::File("my-recipe".to_string()), configuration: HashMap::new(), sha256: None, - } + }, + sync: None, + }, + }, + ); + } + + #[test] + fn recipe_with_sync() { + assert_eq!( + validate_canister_yaml(indoc! {r#" + name: my-canister + recipe: + type: file://my-recipe + sync: + steps: + - type: script + command: echo hi + "#}), + CanisterManifest { + name: "my-canister".to_string(), + settings: ManifestSettings::default(), + init_args: None, + upgrade_args: None, + instructions: Instructions::Recipe { + recipe: Recipe { + recipe_type: RecipeType::File("my-recipe".to_string()), + configuration: HashMap::new(), + sha256: None, + }, + sync: Some(SyncSteps { + steps: vec![SyncStep::Script(script::Adapter { + command: script::CommandField::Command("echo hi".to_string()), + })] + }), + }, + }, + ); + } + + #[test] + fn recipe_with_null_sync() { + assert_eq!( + validate_canister_yaml(indoc! {r#" + name: my-canister + recipe: + type: file://my-recipe + sync: + "#}), + CanisterManifest { + name: "my-canister".to_string(), + settings: ManifestSettings::default(), + init_args: None, + upgrade_args: None, + instructions: Instructions::Recipe { + recipe: Recipe { + recipe_type: RecipeType::File("my-recipe".to_string()), + configuration: HashMap::new(), + sha256: None, + }, + sync: None, }, }, ); } + #[test] + fn recipe_with_invalid_sync() { + match serde_yaml::from_str::(indoc! {r#" + name: my-canister + recipe: + type: file://my-recipe + sync: + steps: + - type: nonsense + "#}) + { + Ok(_) => panic!("an unknown sync step type should not deserialize"), + Err(err) => { + let err_msg = format!("{err}"); + if !err_msg.contains("Canister my-canister failed to parse sync instructions") { + panic!( + "expected 'Canister my-canister failed to parse sync instructions' error but got: {err}" + ); + } + } + }; + } + #[test] fn build_steps() { assert_eq!( diff --git a/crates/icp/src/project.rs b/crates/icp/src/project.rs index cf18ea9b7..20f23fed5 100644 --- a/crates/icp/src/project.rs +++ b/crates/icp/src/project.rs @@ -399,7 +399,7 @@ async fn build_manifest_canisters( let registry_recipe = match &m.instructions { Instructions::BuildSync { .. } => None, - Instructions::Recipe { recipe } => match &recipe.recipe_type { + Instructions::Recipe { recipe, .. } => match &recipe.recipe_type { RecipeType::Registry { .. } => Some(recipe.recipe_type.to_string()), _ => None, }, @@ -416,7 +416,10 @@ async fn build_manifest_canisters( ), // Recipe - Instructions::Recipe { recipe } => { + Instructions::Recipe { + recipe, + sync: extra_sync, + } => { let fetched = recipe_resolver .resolve(recipe) @@ -445,7 +448,12 @@ async fn build_manifest_canisters( })?; } - steps + // The manifest's own sync steps run after the recipe's. + let (build, mut sync) = steps; + if let Some(extra_sync) = extra_sync { + sync.steps.extend(extra_sync.steps.iter().cloned()); + } + (build, sync) } }; @@ -1537,6 +1545,112 @@ pub async fn consolidate_manifest( }) } +#[cfg(test)] +mod recipe_sync_tests { + use super::*; + use crate::canister::recipe::{Fetched, Resolve, ResolveError}; + use crate::manifest::canister::SyncStep; + use crate::manifest::recipe::Recipe; + use camino_tempfile::Utf8TempDir; + + /// Hands back one fixed template for every recipe, without touching the + /// network or the cache. + struct FixedResolver(&'static str); + + #[async_trait::async_trait] + impl Resolve for FixedResolver { + async fn resolve(&self, _recipe: &Recipe) -> Result { + Ok(Fetched { + template: self.0.to_owned(), + pending_cache: None, + }) + } + } + + const TEMPLATE: &str = indoc::indoc! {r#" + build: + steps: + - type: script + command: build.sh + sync: + steps: + - type: script + command: echo recipe + "#}; + + async fn consolidate(pdir: &Path) -> Result { + let m: ProjectManifest = load_manifest_from_path(&pdir.join(PROJECT_MANIFEST)) + .await + .expect("failed to parse project manifest"); + consolidate_manifest(pdir, &FixedResolver(TEMPLATE), &m).await + } + + /// The commands of a canister's sync steps, which are all script steps here. + fn sync_commands(p: &Project, key: &str) -> Vec { + p.canisters + .get(key) + .expect("canister not found") + .1 + .sync + .steps + .iter() + .map(|s| match s { + SyncStep::Script(adapter) => adapter.command.as_vec().join(" "), + other => panic!("expected a script sync step, got {other:?}"), + }) + .collect() + } + + /// A canister may add sync steps of its own on top of a recipe's; they run + /// after the ones the recipe renders. + #[tokio::test] + async fn manifest_sync_steps_follow_the_recipes() { + let tmp = Utf8TempDir::new().unwrap(); + std::fs::write( + tmp.path().join(PROJECT_MANIFEST), + indoc::indoc! {r#" + canisters: + - name: backend + recipe: + type: file://recipe.hbs + sync: + steps: + - type: script + command: echo manifest + "#}, + ) + .unwrap(); + + let p = consolidate(tmp.path()).await.unwrap(); + + assert_eq!( + sync_commands(&p, "backend"), + ["echo recipe", "echo manifest"] + ); + } + + /// Without a `sync` section, a recipe canister still gets exactly the + /// recipe's own sync steps. + #[tokio::test] + async fn recipe_sync_steps_alone_when_manifest_has_none() { + let tmp = Utf8TempDir::new().unwrap(); + std::fs::write( + tmp.path().join(PROJECT_MANIFEST), + indoc::indoc! {r#" + canisters: + - name: backend + recipe: + type: file://recipe.hbs + "#}, + ) + .unwrap(); + + let p = consolidate(tmp.path()).await.unwrap(); + + assert_eq!(sync_commands(&p, "backend"), ["echo recipe"]); + } +} + #[cfg(test)] mod dependency_tests { use super::*; diff --git a/docs/concepts/recipes.md b/docs/concepts/recipes.md index b8b31b89d..494db939a 100644 --- a/docs/concepts/recipes.md +++ b/docs/concepts/recipes.md @@ -39,6 +39,13 @@ canisters: - cp target/wasm32-unknown-unknown/release/my_backend.wasm "$ICP_WASM_OUTPUT_PATH" ``` +### Extra Sync Steps + +A recipe defines the canister's build, so `recipe` and `build` are mutually +exclusive. `sync` is not: a canister using a recipe may declare its own `sync` +steps, which run after the recipe's. See +[Using Recipes](../guides/using-recipes.md#adding-your-own-sync-steps). + ## Recipe Sources Recipes can come from three sources: diff --git a/docs/guides/using-recipes.md b/docs/guides/using-recipes.md index 9201dcce5..55388bec2 100644 --- a/docs/guides/using-recipes.md +++ b/docs/guides/using-recipes.md @@ -163,6 +163,27 @@ canisters: API_KEY: "secret" ``` +## Adding Your Own Sync Steps + +A recipe canister can also declare a `sync` section of its own, for +post-deployment work the recipe does not cover. Those steps run after the +recipe's own sync steps: + +```yaml +canisters: + - name: backend + recipe: + type: "@dfinity/rust@v3.0.0" + configuration: + package: backend + sync: + steps: + - type: script + command: ./scripts/seed-data.sh +``` + +`build` remains exclusive with `recipe`: the recipe is what defines the build. + ## Next Steps - [Recipes](../concepts/recipes.md) — Understand how recipes work diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 090d34cc3..2c52e1d2e 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -61,7 +61,7 @@ canisters: | `settings` | object | No | Canister settings | | `init_args` | string or object | No | Initialization arguments (see [Install Args](#install-args)) | | `upgrade_args` | string or object | No | Upgrade arguments; defaults to `init_args` (see [Install Args](#install-args)) | -| `recipe` | object | No | Recipe reference (alternative to build) | +| `recipe` | object | No | Recipe reference (alternative to build; may be combined with `sync`) | ## Build Steps @@ -231,6 +231,26 @@ canisters: | `sha256` | string | Conditional | Required for remote URLs | | `configuration` | object | No | Parameters passed to recipe template | +### Adding Sync Steps to a Recipe + +A canister that uses a recipe may declare a `sync` section of its own. Its steps +run after the ones the recipe renders, in the order written: + +```yaml +canisters: + - name: frontend + recipe: + type: "@dfinity/asset-canister@v2.2.1" + configuration: + dir: dist + sync: + steps: + - type: script + command: ./scripts/warm-cache.sh +``` + +A `recipe` still cannot be combined with `build` — the recipe defines the build. + ### Recipe Type Formats ```yaml diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index db41b4197..feeeab593 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -34,29 +34,6 @@ "type": "object" }, "Adapter2": { - "anyOf": [ - { - "$ref": "#/$defs/LocalSource", - "description": "Local path on-disk to read a WASM file from" - }, - { - "$ref": "#/$defs/RemoteSource", - "description": "Remote url to fetch a WASM file from" - } - ], - "description": "Configuration for a wasm source — used by adapters that load a `.wasm` file\neither from a local path or from a remote URL.", - "properties": { - "sha256": { - "description": "Optional sha256 checksum of the WASM", - "type": [ - "string", - "null" - ] - } - }, - "type": "object" - }, - "Adapter3": { "anyOf": [ { "$ref": "#/$defs/LocalSource", @@ -125,6 +102,29 @@ }, "type": "object" }, + "Adapter3": { + "anyOf": [ + { + "$ref": "#/$defs/LocalSource", + "description": "Local path on-disk to read a WASM file from" + }, + { + "$ref": "#/$defs/RemoteSource", + "description": "Remote url to fetch a WASM file from" + } + ], + "description": "Configuration for a wasm source — used by adapters that load a `.wasm` file\neither from a local path or from a remote URL.", + "properties": { + "sha256": { + "description": "Optional sha256 checksum of the WASM", + "type": [ + "string", + "null" + ] + } + }, + "type": "object" + }, "ArgsFormat": { "description": "Format specifier for canister call/install args content.", "oneOf": [ @@ -163,7 +163,7 @@ "type": "object" }, { - "$ref": "#/$defs/Adapter2", + "$ref": "#/$defs/Adapter3", "description": "Represents a pre-built canister.\nThis variant allows for retrieving a canister WASM from various sources.", "properties": { "type": { @@ -626,7 +626,7 @@ "type": "object" }, { - "$ref": "#/$defs/Adapter3", + "$ref": "#/$defs/Adapter2", "description": "Represents a sync step executed by a WebAssembly plugin running inside\na wasmtime WASI sandbox. The plugin can call canister methods on exactly\nthe canister being synced and read the paths declared in `files`.", "properties": { "type": { @@ -664,6 +664,17 @@ "properties": { "recipe": { "$ref": "#/$defs/Recipe" + }, + "sync": { + "anyOf": [ + { + "$ref": "#/$defs/SyncSteps" + }, + { + "type": "null" + } + ], + "description": "Additional sync steps, run after the ones the recipe renders." } }, "required": [ diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index e0af63c9b..1b34eda33 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -34,29 +34,6 @@ "type": "object" }, "Adapter2": { - "anyOf": [ - { - "$ref": "#/$defs/LocalSource", - "description": "Local path on-disk to read a WASM file from" - }, - { - "$ref": "#/$defs/RemoteSource", - "description": "Remote url to fetch a WASM file from" - } - ], - "description": "Configuration for a wasm source — used by adapters that load a `.wasm` file\neither from a local path or from a remote URL.", - "properties": { - "sha256": { - "description": "Optional sha256 checksum of the WASM", - "type": [ - "string", - "null" - ] - } - }, - "type": "object" - }, - "Adapter3": { "anyOf": [ { "$ref": "#/$defs/LocalSource", @@ -125,6 +102,29 @@ }, "type": "object" }, + "Adapter3": { + "anyOf": [ + { + "$ref": "#/$defs/LocalSource", + "description": "Local path on-disk to read a WASM file from" + }, + { + "$ref": "#/$defs/RemoteSource", + "description": "Remote url to fetch a WASM file from" + } + ], + "description": "Configuration for a wasm source — used by adapters that load a `.wasm` file\neither from a local path or from a remote URL.", + "properties": { + "sha256": { + "description": "Optional sha256 checksum of the WASM", + "type": [ + "string", + "null" + ] + } + }, + "type": "object" + }, "ArgsFormat": { "description": "Format specifier for canister call/install args content.", "oneOf": [ @@ -163,7 +163,7 @@ "type": "object" }, { - "$ref": "#/$defs/Adapter2", + "$ref": "#/$defs/Adapter3", "description": "Represents a pre-built canister.\nThis variant allows for retrieving a canister WASM from various sources.", "properties": { "type": { @@ -199,6 +199,17 @@ "properties": { "recipe": { "$ref": "#/$defs/Recipe" + }, + "sync": { + "anyOf": [ + { + "$ref": "#/$defs/SyncSteps" + }, + { + "type": "null" + } + ], + "description": "Additional sync steps, run after the ones the recipe renders." } }, "required": [ @@ -1170,7 +1181,7 @@ "type": "object" }, { - "$ref": "#/$defs/Adapter3", + "$ref": "#/$defs/Adapter2", "description": "Represents a sync step executed by a WebAssembly plugin running inside\na wasmtime WASI sandbox. The plugin can call canister methods on exactly\nthe canister being synced and read the paths declared in `files`.", "properties": { "type": {