diff --git a/.github/workflows/dep_build_wasm_examples.yml b/.github/workflows/dep_build_wasm_examples.yml index ee2fe4cf..3c75b027 100644 --- a/.github/workflows/dep_build_wasm_examples.yml +++ b/.github/workflows/dep_build_wasm_examples.yml @@ -54,7 +54,8 @@ jobs: build/*/pulley/*.aot src/tests/rust_guests/component_sample/wit/component-world.wasm src/tests/c_guests/wasmsamples/components/runcomponent-world.wasm - key: wasm-modules-${{ matrix.arch.name }}-${{ matrix.wasmtime_version.name }}-${{ hashFiles('src/tests/**', 'src/hyperlight_wasm_aot/**', 'Cargo.lock', '.cargo/config.toml', 'rust-toolchain.toml', 'Justfile', 'c.just', '.github/workflows/dep_build_wasm_examples.yml') }} + build/*/component-sample-runtime + key: wasm-modules-${{ matrix.arch.name }}-${{ matrix.wasmtime_version.name }}-${{ hashFiles('src/tests/**', 'src/hyperlight_wasm_aot/**', 'src/hyperlight_wasm_runtime/**', 'src/hyperlight_wasm_macro/**', 'Cargo.lock', '.cargo/config.toml', 'rust-toolchain.toml', 'Justfile', 'c.just', '.github/workflows/dep_build_wasm_examples.yml') }} - name: Remove default clang if: steps.modules-cache.outputs.cache-hit != 'true' run: sudo rm /usr/bin/clang @@ -106,6 +107,7 @@ jobs: build/*/pulley/*.aot src/tests/rust_guests/component_sample/wit/component-world.wasm src/tests/c_guests/wasmsamples/components/runcomponent-world.wasm + build/*/component-sample-runtime key: ${{ steps.modules-cache.outputs.cache-primary-key }} # Build jobs compile examples against the WIT worlds, so they ship with the modules. - name: Upload Wasm Modules (wasmtime ${{ matrix.wasmtime_version.name }}) @@ -118,3 +120,4 @@ jobs: build/*/pulley/*.aot src/tests/rust_guests/component_sample/wit/component-world.wasm src/tests/c_guests/wasmsamples/components/runcomponent-world.wasm + build/*/component-sample-runtime diff --git a/CHANGELOG.md b/CHANGELOG.md index dd3464c1..ea3459a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). * Experimental: add the `experimental_runtime_file` feature, with `SandboxBuilder::with_runtime_file` to load the guest runtime from a file. Snapshots of such a sandbox cannot be saved. * Experimental: add the `hyperlight-wasm-capi` crate, a C API that calls component functions with arguments and results in Hyperlight's component wire format. It is not published. * Experimental: add the `Hyperlight.Wasm` and `Hyperlight.Wasm.Build` .NET packages, which run components from .NET with typed C# bindings generated from WIT at build time. See `dotnet/README.md`. They are not published. +* Experimental: add `hyperlight-wasm-aot experimental-build-runtime`, which builds a guest runtime for a WIT world. ### Changed diff --git a/Cargo.lock b/Cargo.lock index 3cea544e..d39e0c7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2090,10 +2090,13 @@ dependencies = [ name = "hyperlight-wasm-aot" version = "0.16.0" dependencies = [ + "cargo-hyperlight", "cargo-util-schemas", "cargo_metadata", "clap", + "hyperlight-wasm-runtime", "object 0.39.1", + "tar", "wasmtime 36.0.18", "wasmtime 47.0.4", ] diff --git a/Justfile b/Justfile index e5dfae2c..bf8602eb 100644 --- a/Justfile +++ b/Justfile @@ -152,8 +152,8 @@ examples-ci target=default-target features="": (examples-modules target features # Runs the examples-ci examples on the modules build-ci-examples builds. examples-ci-prebuilt target=default-target features="": (run-examples-modules target features) (run-examples-components target features) (use-pulley-examples target) (run-examples-pulley target features) -# Builds every module the CI build jobs load, for the given profiles. C modules build -# in release only. Pulley builds go to build//pulley. +# Builds every module the CI build jobs load, and the .NET tests' guest runtime, for the +# given profiles. C modules build in release only. Pulley builds go to build//pulley. build-ci-examples features="" profiles="debug release": (build-wasm-examples "release" features) #!/usr/bin/env bash set -euo pipefail @@ -168,6 +168,7 @@ build-ci-examples features="" profiles="debug release": (build-wasm-examples "re mv build/$target/rust_wasm_samples.aot build/$target/component_sample.aot build/$target/pulley/ just build-rust-wasm-examples $target {{ features }} just build-rust-component-examples $target {{ features }} + just build-component-sample-runtime $target {{ features }} done [private] @@ -183,15 +184,18 @@ run-examples-components target=default-target features="": {{ wit-world }} {{ wit-world-name-greeter }} cargo run {{ if features =="" {''} else {"--no-default-features -F kvm -F " + features } }} --profile={{ if target == "debug" {"dev"} else { target } }} --example component_greeter_example {{ wit-world }} cargo test -p hyperlight-wasm-capi {{ if features =="" {''} else {"--no-default-features -F kvm -F " + features } }} --profile={{ if target == "debug" {"dev"} else { target } }} -- --include-ignored -# Builds the native libraries and a guest runtime for the component_sample example world, for the .NET tests -dotnet-test-assets target=default-target features="": ensure-cargo-hyperlight +# Builds the native libraries for the .NET tests +dotnet-test-assets target=default-target features="": cargo build -p hyperlight-wasm-capi -p hyperlight-wasm-csharp-bindgen {{ if features =="" {''} else {"--no-default-features -F kvm -F " + features } }} --profile={{ if target == "debug" {"dev"} else { target } }} - {{ wit-world }} cargo hyperlight build {{ if features =~ "wasmtime_latest" {"--no-default-features -F wasmtime_latest"} else {""} }} --profile={{ if target == "debug" {"dev"} else { target } }} --manifest-path src/hyperlight_wasm_runtime/Cargo.toml --target-dir target/component-sample-runtime --locked + +# Builds a guest runtime for the component_sample example world into build//component-sample-runtime +build-component-sample-runtime target=default-target features="": + cargo run -p hyperlight-wasm-aot -- experimental-build-runtime src/tests/rust_guests/component_sample/wit/example.wit build/{{ target }}/component-sample-runtime --world example --target-dir target/component-sample-runtime {{ if target == "debug" {"--debug"} else {""} }} {{ if features =~ "wasmtime_latest" {"--wasmtime-version latest"} else {""} }} # Runs the .NET tests. Needs the .NET 8 SDK or later. -dotnet-test target=default-target features="": (build-rust-component-examples target features) (dotnet-test-prebuilt target features) +dotnet-test target=default-target features="": (build-rust-component-examples target features) (build-component-sample-runtime target features) (dotnet-test-prebuilt target features) -# Runs the .NET tests on the component_sample component that build-ci-examples builds. +# Runs the .NET tests on the component and runtime that build-ci-examples builds. dotnet-test-prebuilt target=default-target features="": (dotnet-test-assets target features) dotnet test dotnet/tests/Hyperlight.Wasm.Tests -c {{ if target == "debug" {"Debug"} else {"Release"} }} diff --git a/dotnet/README.md b/dotnet/README.md index 36cfb46e..6b0a52d0 100644 --- a/dotnet/README.md +++ b/dotnet/README.md @@ -44,7 +44,7 @@ uint sum = component.Adder.Add(1, 2); ## Guest runtime and components -A component needs a guest runtime built for its WIT world, because the runtime's guest bindings are generated when it is compiled. Building a runtime needs Rust and a hyperlight-wasm checkout of the same version as the packages: +A component needs a guest runtime built for its WIT world, because the runtime's guest bindings are generated when it is compiled. Building a runtime needs Rust. From a hyperlight-wasm checkout of the same version as the packages: ```sh wasm-tools component wit example.wit -w -o example-world.wasm @@ -54,6 +54,15 @@ WIT_WORLD=example-world.wasm cargo hyperlight build --release \ Precompile the component with `hyperlight-wasm-aot compile --component`. The runtime, the AOT tool and the packages must come from the same hyperlight-wasm version. +### Building the runtime without a checkout + +Experimental: the `experimental-build-runtime` subcommand of `hyperlight-wasm-aot` builds the runtime without a hyperlight-wasm checkout. It needs rustup and clang, and builds with Rust 1.94.1 unless `RUSTUP_TOOLCHAIN` is set: + +```sh +cargo install --locked hyperlight-wasm-aot +hyperlight-wasm-aot experimental-build-runtime example.wit example-runtime --world example +``` + ## Sandboxes * Register host functions, then load a component, then call it. diff --git a/dotnet/tests/Hyperlight.Wasm.Tests/Hyperlight.Wasm.Tests.csproj b/dotnet/tests/Hyperlight.Wasm.Tests/Hyperlight.Wasm.Tests.csproj index 00dc0cc4..2d281010 100644 --- a/dotnet/tests/Hyperlight.Wasm.Tests/Hyperlight.Wasm.Tests.csproj +++ b/dotnet/tests/Hyperlight.Wasm.Tests/Hyperlight.Wasm.Tests.csproj @@ -29,14 +29,10 @@ - - - aarch64 - x86_64 - + - + diff --git a/src/hyperlight_wasm_aot/Cargo.toml b/src/hyperlight_wasm_aot/Cargo.toml index 000b2749..a1a93887 100644 --- a/src/hyperlight_wasm_aot/Cargo.toml +++ b/src/hyperlight_wasm_aot/Cargo.toml @@ -21,6 +21,14 @@ clap = { version = "4.6", features = ["derive"] } cargo_metadata = "0.23" cargo-util-schemas = "=0.14.1" object = { version = "0.39.1", default-features = false, features = ["read_core", "elf"] } +cargo-hyperlight = "0.1.14" +tar = "0.4.46" +# Makes Cargo fetch the runtime source that `experimental-build-runtime` builds. +hyperlight-wasm-runtime.workspace = true + +[build-dependencies] +cargo_metadata = "0.23" +tar = "0.4.46" [features] gdb = ["wasmtime/debug-builtins", "wasmtime_lts/debug-builtins"] diff --git a/src/hyperlight_wasm_aot/build.rs b/src/hyperlight_wasm_aot/build.rs new file mode 100644 index 00000000..2409858f --- /dev/null +++ b/src/hyperlight_wasm_aot/build.rs @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 The Hyperlight Authors. + +// Records where `experimental-build-runtime` finds the hyperlight-wasm-runtime +// source, in OUT_DIR/runtime_source.rs. + +use std::path::PathBuf; +use std::{env, fs}; + +use cargo_metadata::MetadataCommand; + +fn main() { + println!("cargo::rerun-if-changed=build.rs"); + + let metadata = MetadataCommand::new() + .exec() + .expect("could not run cargo metadata"); + let runtime = metadata + .packages + .iter() + .find(|p| *p.name == "hyperlight-wasm-runtime") + .expect("hyperlight-wasm-runtime not found in cargo metadata"); + let manifest = runtime.manifest_path.as_std_path(); + let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap()); + + let registry = runtime + .source + .as_ref() + .is_some_and(|s| s.repr.starts_with("registry+") || s.repr.starts_with("sparse+")); + let source = if !registry { + // A checkout, including a git one, builds the runtime in place, against + // its workspace's other crates. A packaged copy would use their released + // versions. + format!("RuntimeSource::Path({manifest:?})") + } else { + // Cargo can garbage collect registry sources, so embed them. + let prefix = format!("hyperlight-wasm-runtime-{}", runtime.version); + let archive = out_dir.join("runtime-source.tar"); + let mut builder = tar::Builder::new(fs::File::create(&archive).unwrap()); + builder + .append_dir_all(&prefix, manifest.parent().unwrap()) + .unwrap(); + builder.finish().unwrap(); + format!("RuntimeSource::Archive({prefix:?}, include_bytes!({archive:?}))") + }; + fs::write( + out_dir.join("runtime_source.rs"), + format!("const RUNTIME_SOURCE: RuntimeSource = {source};\n"), + ) + .unwrap(); +} diff --git a/src/hyperlight_wasm_aot/src/main.rs b/src/hyperlight_wasm_aot/src/main.rs index 0b39d404..2addf2ad 100644 --- a/src/hyperlight_wasm_aot/src/main.rs +++ b/src/hyperlight_wasm_aot/src/main.rs @@ -15,7 +15,8 @@ limitations under the License. */ use std::fmt::Display; -use std::path::Path; +use std::fs; +use std::path::{Path, PathBuf}; use cargo_metadata::{MetadataCommand, Package}; use cargo_util_schemas::manifest::PackageName; @@ -100,6 +101,149 @@ enum Commands { #[arg(long, value_enum, default_value = "lts")] wasmtime_version: WasmtimeVersion, }, + + /// Experimental: build a guest runtime for a WIT world + /// + /// Needs rustup and clang. Builds with Rust 1.94.1 unless RUSTUP_TOOLCHAIN is + /// set. Use a runtime with hosts and precompiled components from this version + /// of hyperlight-wasm. + ExperimentalBuildRuntime { + /// The `.wit` file or wasm-encoded WIT package that defines the world + wit: PathBuf, + + /// The output file path + output: PathBuf, + + /// The world to build for. Defaults to the last world in the package. + #[arg(long)] + world: Option, + + /// Build with debug info and optimizations disabled + #[arg(long)] + debug: bool, + + /// Wasmtime version the runtime uses. Precompile components with the same version. + #[arg(long, value_enum, default_value = "lts")] + wasmtime_version: WasmtimeVersion, + + /// Directory for build artifacts, reused by later builds + #[arg(long, default_value = "target/hyperlight-wasm-runtime")] + target_dir: PathBuf, + }, +} + +// Only one variant is built, chosen by build.rs. +#[allow(dead_code)] +enum RuntimeSource { + /// The runtime's manifest in a checkout. + Path(&'static str), + /// The top-level directory name and a tar of the runtime crate. + Archive(&'static str, &'static [u8]), +} + +include!(concat!(env!("OUT_DIR"), "/runtime_source.rs")); + +type BoxError = Box; + +/// The toolchain that builds the runtime when `RUSTUP_TOOLCHAIN` is unset. +/// Keep it in sync with rust-toolchain.toml. +const RUNTIME_TOOLCHAIN: &str = "1.94.1"; + +/// Returns the runtime's manifest, extracting an embedded source into `target_dir`. +fn runtime_manifest(target_dir: &Path) -> Result { + match RUNTIME_SOURCE { + RuntimeSource::Path(manifest) => Ok(PathBuf::from(manifest)), + RuntimeSource::Archive(prefix, archive) => { + let parent = target_dir.join("source"); + let source = parent.join(prefix); + // The marker shows that an earlier extraction finished. + let marker = source.join(".hyperlight-wasm-aot-extracted"); + if !marker.exists() { + if source.exists() { + fs::remove_dir_all(&source)?; + } + tar::Archive::new(archive).unpack(&parent)?; + // Keep Cargo from treating an enclosing project as the workspace. + let manifest = source.join("Cargo.toml"); + let mut contents = fs::read_to_string(&manifest)?; + contents.push_str("\n[workspace]\n"); + fs::write(&manifest, contents)?; + fs::write(&marker, "")?; + } + Ok(source.join("Cargo.toml")) + } + } +} + +fn build_runtime( + wit: &Path, + output: &Path, + world: Option<&str>, + debug: bool, + wasmtime_version: WasmtimeVersion, + target_dir: &Path, +) -> Result<(), BoxError> { + // The runtime resolves a relative WIT_WORLD against its own manifest. + let wit = wit + .canonicalize() + .map_err(|e| format!("could not read {}: {e}", wit.display()))?; + let manifest = runtime_manifest(target_dir)?; + let target = format!("{}-hyperlight-none", std::env::consts::ARCH); + let (profile, profile_dir) = if debug { + ("dev", "debug") + } else { + ("release", "release") + }; + + // Stable Rust 1.95 and later reject the guest's custom target. cargo-hyperlight + // also adds rust-src only when RUSTUP_TOOLCHAIN is set in this process. + if std::env::var_os("RUSTUP_TOOLCHAIN").is_none() { + // SAFETY: the process has not started other threads. + unsafe { std::env::set_var("RUSTUP_TOOLCHAIN", RUNTIME_TOOLCHAIN) }; + } + + let mut cmd = cargo_hyperlight::cargo()?; + cmd.env_clear_cargo() + .arg("build") + .arg("--profile") + .arg(profile) + .arg("--target") + .arg(&target) + .arg("--target-dir") + .arg(target_dir) + .arg("--manifest-path") + .arg(&manifest) + .arg("--locked") + .env("WIT_WORLD", &wit); + match world { + Some(world) => cmd.env("WIT_WORLD_NAME", world), + None => cmd.env_remove("WIT_WORLD_NAME"), + }; + if wasmtime_version == WasmtimeVersion::Latest { + cmd.args(["--no-default-features", "--features", "wasmtime_latest"]); + } + cmd.status()?; + + let built = target_dir + .join(&target) + .join(profile_dir) + .join("hyperlight-wasm-runtime"); + if let Some(parent) = output.parent().filter(|p| !p.as_os_str().is_empty()) { + fs::create_dir_all(parent)?; + } + fs::copy(&built, output).map_err(|e| { + format!( + "could not copy {} to {}: {e}", + built.display(), + output.display() + ) + })?; + println!( + "Built guest runtime for {}: {}", + wit.display(), + output.display() + ); + Ok(()) } /// Precompile bytes using the LTS wasmtime version @@ -356,6 +500,26 @@ fn main() { } } } + Commands::ExperimentalBuildRuntime { + wit, + output, + world, + debug, + wasmtime_version, + target_dir, + } => { + if let Err(e) = build_runtime( + &wit, + &output, + world.as_deref(), + debug, + wasmtime_version, + &target_dir, + ) { + eprintln!("Error - {e}"); + std::process::exit(1); + } + } } } diff --git a/src/hyperlight_wasm_macro/src/lib.rs b/src/hyperlight_wasm_macro/src/lib.rs index 3b4c0a37..35d421c6 100644 --- a/src/hyperlight_wasm_macro/src/lib.rs +++ b/src/hyperlight_wasm_macro/src/lib.rs @@ -27,14 +27,20 @@ mod wasmguest; /// wasmtime linker for component imports (which are implemented by /// calling to the Hyperlight host). /// +/// `WIT_WORLD` is a `.wit` file or a wasm-encoded WIT package. +/// /// If the WIT file contains multiple worlds, set the `WIT_WORLD_NAME` /// environment variable to select a specific world by name. If not set, /// the last world in the file will be used. #[proc_macro] pub fn wasm_guest_bindgen(_: proc_macro::TokenStream) -> proc_macro::TokenStream { - let path = std::env::var_os("WIT_WORLD").unwrap(); + let path = std::path::PathBuf::from(std::env::var_os("WIT_WORLD").unwrap()); let world_name = std::env::var("WIT_WORLD_NAME").ok(); - util::read_wit_type_from_file(path, world_name, |kebab_name, ct| { + let source = match path.extension().and_then(|e| e.to_str()) { + Some("wit") => util::WitSource::Wit(path), + _ => util::WitSource::Wasm(path), + }; + util::read_wit_type(source, world_name, |kebab_name, ct| { let decls = emit::run_state(true, true, |s| { // Emit type/trait definitions for all instances in the world rtypes::emit_toplevel(s, &kebab_name, ct);