diff --git a/e2e/configs/gateway/docker.toml b/e2e/configs/gateway/docker.toml new file mode 100644 index 000000000..59baed1d7 --- /dev/null +++ b/e2e/configs/gateway/docker.toml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[openshell] +version = 1 + +[openshell.gateway] +bind_address = "127.0.0.1:8080" +log_level = "info" +compute_drivers = ["docker"] +disable_tls = true + +[openshell.gateway.auth] +allow_unauthenticated_users = true + +[openshell.gateway.gateway_jwt] +signing_key_path = ".cache/openshell-e2e/gateway-jwt/signing.pem" +public_key_path = ".cache/openshell-e2e/gateway-jwt/public.pem" +kid_path = ".cache/openshell-e2e/gateway-jwt/kid" +gateway_id = "openshell-e2e" +ttl_secs = 0 + +[openshell.drivers.docker] +default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" +image_pull_policy = "IfNotPresent" +sandbox_namespace = "openshell-e2e" +supervisor_image = "localhost/openshell/supervisor:e2e-vm" diff --git a/e2e/configs/gateway/podman.toml b/e2e/configs/gateway/podman.toml new file mode 100644 index 000000000..2ad86099f --- /dev/null +++ b/e2e/configs/gateway/podman.toml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[openshell] +version = 1 + +[openshell.gateway] +bind_address = "127.0.0.1:8080" +log_level = "info" +compute_drivers = ["podman"] +disable_tls = true + +[openshell.gateway.auth] +allow_unauthenticated_users = true + +[openshell.gateway.gateway_jwt] +signing_key_path = ".cache/openshell-e2e/gateway-jwt/signing.pem" +public_key_path = ".cache/openshell-e2e/gateway-jwt/public.pem" +kid_path = ".cache/openshell-e2e/gateway-jwt/kid" +gateway_id = "openshell-e2e" +ttl_secs = 0 + +[openshell.drivers.podman] +default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" +image_pull_policy = "missing" +network_name = "openshell-e2e" +grpc_endpoint = "http://10.89.0.1:8080" +host_gateway_ip = "10.89.0.1" +supervisor_image = "localhost/openshell/supervisor:e2e-vm" diff --git a/e2e/run.sh b/e2e/run.sh new file mode 100755 index 000000000..967f240c2 --- /dev/null +++ b/e2e/run.sh @@ -0,0 +1,616 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Build the current checkout, run its gateway on the host or in a disposable +# Nix test guest, and execute one named host-side E2E suite against that gateway. + +set -Eeuo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +# shellcheck disable=SC1091 +source "${ROOT}/e2e/support/gateway-common.sh" +# shellcheck disable=SC1091 +source "${ROOT}/tasks/scripts/build-env.sh" + +e2e_preserve_mise_dirs + +usage() { + cat <<'EOF' +Usage: + e2e/run.sh [--vm DISTRO] [--with CONFIG ...] \ + --gateway-config PATH --suite NAME + +Options: + --vm DISTRO Run the gateway in a Nix test guest + --with CONFIG Apply a Nix test-guest configuration; repeatable + --gateway-config PATH + Fully resolved gateway TOML + --suite NAME Rust suite at e2e/rust/tests/NAME.rs + -h, --help Show this help + +Omit --vm and --with to run the gateway on the host. Supplying --with without +--vm selects the Ubuntu test guest. Set OPENSHELL_E2E_KEEP=1 to retain state. +EOF +} + +die() { + echo "ERROR: $*" >&2 + exit 2 +} + +require_value() { + local option=$1 + local count=$2 + local value=${3:-} + + if [ "${count}" -lt 2 ] || [ -z "${value}" ]; then + die "${option} requires a value" + fi + case "${value}" in + --*) die "${option} requires a value" ;; + esac +} + +resolve_file() { + local path=$1 + + if [ ! -f "${path}" ]; then + return 1 + fi + python3 - "${path}" <<'PY' +import os +import sys + +print(os.path.realpath(sys.argv[1])) +PY +} + +catalog_has_entry() { + local catalog=$1 + local section=$2 + local name=$3 + + printf '%s\n' "${catalog}" | awk -v wanted_section="${section}:" -v wanted_name="${name}" ' + $0 == wanted_section { + in_section = 1 + next + } + /^[^[:space:]]/ { + in_section = 0 + } + in_section && $0 == " " wanted_name { + found = 1 + } + END { + exit(found ? 0 : 1) + } + ' +} + +vm= +gateway_config= +suite_name= +with_configurations=() + +while [ "$#" -gt 0 ]; do + case "$1" in + --vm) + require_value "$1" "$#" "${2:-}" + vm=$2 + shift 2 + ;; + --with) + require_value "$1" "$#" "${2:-}" + with_configurations+=("$2") + shift 2 + ;; + --gateway-config) + require_value "$1" "$#" "${2:-}" + gateway_config=$2 + shift 2 + ;; + --suite) + require_value "$1" "$#" "${2:-}" + suite_name=$2 + shift 2 + ;; + -h | --help) + usage + exit 0 + ;; + *) + die "unknown argument: $1" + ;; + esac +done + +if [ -z "${gateway_config}" ]; then + die "--gateway-config is required" +fi +if [ -z "${suite_name}" ]; then + die "--suite is required" +fi +if ! command -v python3 >/dev/null 2>&1; then + die "python3 is required" +fi +gateway_config_source=${gateway_config} +if ! gateway_config="$(resolve_file "${gateway_config_source}")"; then + die "gateway config does not exist: ${gateway_config_source}" +fi +gateway_driver="$(python3 -c ' +import sys, tomllib +print(tomllib.load(open(sys.argv[1], "rb"))["openshell"]["gateway"]["compute_drivers"][0]) +' "${gateway_config}")" +if [[ ! ${suite_name} =~ ^[a-z0-9][a-z0-9-]*$ ]]; then + die "suite name must contain only lowercase letters, digits, and hyphens: ${suite_name}" +fi +suite_path="${ROOT}/e2e/rust/tests/${suite_name}.rs" +if [ ! -f "${suite_path}" ]; then + die "unknown suite: ${suite_name}" +fi +mode=host +if [ -n "${vm}" ] || [ "${#with_configurations[@]}" -gt 0 ]; then + mode=vm + if [ -z "${vm}" ]; then + vm=ubuntu + fi +fi +if [ "${mode}" = vm ]; then + if [[ ! ${vm} =~ ^[a-z0-9][a-z0-9-]*$ ]]; then + die "invalid VM distro name: ${vm}" + fi + for configuration in "${with_configurations[@]}"; do + if [[ ! ${configuration} =~ ^[a-z0-9][a-z0-9-]*$ ]]; then + die "invalid VM configuration name: ${configuration}" + fi + done + if ! command -v nix >/dev/null 2>&1; then + die "Nix is required for VM mode" + fi + if ! command -v base64 >/dev/null 2>&1; then + die "base64 is required for VM mode" + fi + if ! vm_catalog="$(cd "${ROOT}" && nix run .#test-guest -- --list)"; then + die "failed to read the Nix test-guest catalog" + fi + if ! catalog_has_entry "${vm_catalog}" Distros "${vm}"; then + die "unknown VM distro in the Nix test-guest catalog: ${vm}" + fi + for configuration in "${with_configurations[@]}"; do + if ! catalog_has_entry "${vm_catalog}" Configurations "${configuration}"; then + die "unknown VM configuration in the Nix test-guest catalog: ${configuration}" + fi + done +fi + +gateway_ready_timeout=${OPENSHELL_E2E_GATEWAY_READY_TIMEOUT:-600} +if [[ ! ${gateway_ready_timeout} =~ ^[1-9][0-9]*$ ]]; then + die "OPENSHELL_E2E_GATEWAY_READY_TIMEOUT must be a positive integer" +fi +if ! command -v mise >/dev/null 2>&1; then + die "mise is required to build OpenShell" +fi +if ! command -v openssl >/dev/null 2>&1; then + die "OpenSSL is required to generate sandbox JWT keys" +fi + +case "$(uname -m)" in +x86_64 | amd64) + linux_musl_target=x86_64-unknown-linux-musl + linux_gateway_rust_target=x86_64-unknown-linux-gnu + linux_gateway_zig_target=x86_64-unknown-linux-gnu.2.28 + ;; +aarch64 | arm64) + linux_musl_target=aarch64-unknown-linux-musl + linux_gateway_rust_target=aarch64-unknown-linux-gnu + linux_gateway_zig_target=aarch64-unknown-linux-gnu.2.28 + ;; +*) + die "unsupported host architecture: $(uname -m)" + ;; +esac + +cargo_jobs=() +if [ -n "${CARGO_BUILD_JOBS:-}" ]; then + cargo_jobs=(-j "${CARGO_BUILD_JOBS}") +fi + +cd "${ROOT}" +target_dir="$(e2e_cargo_target_dir "${ROOT}" mise x -- cargo)" + +ensure_build_nofile_limit + +echo "==> Building native host openshell CLI" +mise x -- cargo build "${cargo_jobs[@]}" -p openshell-cli --bin openshell +host_cli_bin="${target_dir}/debug/openshell" + +echo "==> Preparing ${linux_musl_target} build target" +mise x -- rustup target add "${linux_musl_target}" >/dev/null + +echo "==> Building Linux openshell-sandbox (${linux_musl_target})" +mise x -- cargo zigbuild "${cargo_jobs[@]}" \ + --release \ + --target "${linux_musl_target}" \ + -p openshell-sandbox \ + --bin openshell-sandbox +linux_sandbox_bin="${target_dir}/${linux_musl_target}/release/openshell-sandbox" + +host_gateway_bin= +guest_gateway_bin= +if [ "${mode}" = host ]; then + echo "==> Building native host openshell-gateway" + mise x -- cargo build "${cargo_jobs[@]}" \ + -p openshell-server \ + --bin openshell-gateway \ + --features bundled-z3 + host_gateway_bin="${target_dir}/debug/openshell-gateway" +else + echo "==> Preparing ${linux_gateway_rust_target} build target" + mise x -- rustup target add "${linux_gateway_rust_target}" >/dev/null + echo "==> Building Linux openshell-gateway (${linux_gateway_zig_target})" + ( + eval "$( + "${ROOT}/tasks/scripts/setup-zig-cc-wrapper.sh" \ + "${linux_gateway_zig_target}" \ + "${linux_gateway_zig_target}" \ + "${target_dir}/zig-gnu-wrapper/e2e" + )" + mise x -- cargo zigbuild "${cargo_jobs[@]}" \ + --release \ + --target "${linux_gateway_zig_target}" \ + -p openshell-server \ + --bin openshell-gateway \ + --features bundled-z3 + ) + guest_gateway_bin="${target_dir}/${linux_gateway_rust_target}/release/openshell-gateway" +fi + +expected_binaries=("${host_cli_bin}" "${linux_sandbox_bin}") +if [ "${mode}" = host ]; then + expected_binaries+=("${host_gateway_bin}") +else + expected_binaries+=("${guest_gateway_bin}") +fi +for binary in "${expected_binaries[@]}"; do + if [ ! -x "${binary}" ]; then + echo "ERROR: expected built binary at ${binary}" >&2 + exit 1 + fi +done + +run_parent="${ROOT}/.cache/openshell-e2e/runs" +mkdir -p "${run_parent}" +run_dir="$(mktemp -d "${run_parent%/}/run.XXXXXX")" +if ! command -v tar >/dev/null 2>&1; then + die "tar is required to package the supervisor image" +fi +supervisor_image=localhost/openshell/supervisor:e2e-vm +supervisor_rootfs="${run_dir}/supervisor-rootfs" +supervisor_archive="${run_dir}/supervisor.tar" +mkdir -p "${supervisor_rootfs}" +install -m 0555 "${linux_sandbox_bin}" "${supervisor_rootfs}/openshell-sandbox" +tar -C "${supervisor_rootfs}" -cf "${supervisor_archive}" openshell-sandbox +child_pid= +runtime_log= +keep=0 +if [ "${OPENSHELL_E2E_KEEP:-0}" = 1 ]; then + keep=1 +fi + +start_child() { + local working_dir=$1 + local log_path=$2 + shift 2 + + ( + cd "${working_dir}" + exec python3 -c \ + 'import os, sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])' \ + "$@" + ) >"${log_path}" 2>&1 & + child_pid=$! +} + +# Invoked by the EXIT trap through cleanup. +# shellcheck disable=SC2329 +stop_child() { + local pid=$1 + local signal_target="-${pid}" + + if [ -z "${pid}" ] || ! kill -0 "${pid}" 2>/dev/null; then + return + fi + kill -TERM -- "${signal_target}" 2>/dev/null || true + for _ in $(seq 1 30); do + if ! kill -0 "${pid}" 2>/dev/null; then + break + fi + sleep 1 + done + if kill -0 "${pid}" 2>/dev/null; then + kill -KILL -- "${signal_target}" 2>/dev/null || true + fi + wait "${pid}" 2>/dev/null || true +} + +# Invoked by EXIT, INT, and TERM traps. +# shellcheck disable=SC2329 +cleanup() { + local status=$? + + trap - EXIT INT TERM + stop_child "${child_pid}" + if [ "${status}" -ne 0 ] && [ -n "${runtime_log}" ] && [ -f "${runtime_log}" ]; then + echo "=== ${mode} gateway log ===" >&2 + cat "${runtime_log}" >&2 + echo "=== end ${mode} gateway log ===" >&2 + fi + if [ "${keep}" -eq 1 ]; then + echo "Kept E2E runner state at ${run_dir}" >&2 + else + rm -rf "${run_dir}" + fi + exit "${status}" +} + +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +jwt_source_dir="${run_dir}/gateway-jwt" +host_runtime_dir= +if [ "${mode}" = host ]; then + host_runtime_dir="${run_dir}/host-runtime" + jwt_source_dir="${host_runtime_dir}/.cache/openshell-e2e/gateway-jwt" +fi +e2e_generate_gateway_jwt "${jwt_source_dir}" + +host_port="$(e2e_pick_port)" +guest_port= +if [ "${mode}" = vm ]; then + guest_port=8080 +fi + +export XDG_CONFIG_HOME="${run_dir}/host/config" +export XDG_DATA_HOME="${run_dir}/host/data" +export XDG_STATE_HOME="${run_dir}/host/state" +mkdir -p "${XDG_CONFIG_HOME}" "${XDG_DATA_HOME}" "${XDG_STATE_HOME}" + +gateway_name="openshell-e2e-${mode}-${host_port}" +gateway_endpoint="http://127.0.0.1:${host_port}" +export OPENSHELL_GATEWAY_ENDPOINT="${gateway_endpoint}" +export OPENSHELL_GATEWAY="${gateway_name}" +export OPENSHELL_BIN="${host_cli_bin}" + +if [ "${mode}" = host ]; then + case "${gateway_driver}" in + docker) + e2e_align_docker_host_with_cli_context + docker import \ + --change 'ENTRYPOINT ["/openshell-sandbox"]' \ + "${supervisor_archive}" \ + "${supervisor_image}" >/dev/null + ;; + podman) + podman import \ + --change 'ENTRYPOINT ["/openshell-sandbox"]' \ + "${supervisor_archive}" \ + "${supervisor_image}" >/dev/null + ;; + esac + + runtime_log="${run_dir}/gateway.log" + echo "==> Starting host gateway at ${gateway_endpoint}" + start_child \ + "${host_runtime_dir}" \ + "${runtime_log}" \ + "${host_gateway_bin}" \ + --config "${gateway_config}" \ + --bind-address 127.0.0.1 \ + --port "${host_port}" \ + --disable-tls +else + runtime_log="${run_dir}/vm.log" + guest_launcher="${run_dir}/launch-gateway.sh" + guest_launcher_path=/home/openshell/.cache/openshell-e2e/bin/launch-gateway + guest_supervisor_archive_path=/home/openshell/.cache/openshell-e2e/supervisor.tar + config_payload="$(base64 <"${gateway_config}" | tr -d '\r\n')" + jwt_signing_payload="$(base64 <"${jwt_source_dir}/signing.pem" | tr -d '\r\n')" + jwt_public_payload="$(base64 <"${jwt_source_dir}/public.pem" | tr -d '\r\n')" + jwt_kid_payload="$(base64 <"${jwt_source_dir}/kid" | tr -d '\r\n')" + cat >"${guest_launcher}" < Timing: \${label}: \$((SECONDS - started_at))s" +} + +phase_started_at=\${SECONDS} +umask 077 +state_root=/home/openshell/.cache/openshell-e2e +config_path=\${state_root}/gateway.toml +jwt_root=\${state_root}/gateway-jwt +sudo chown -R "\$(id -u):\$(id -g)" /home/openshell/.cache +chmod 0700 "\${state_root}" +mkdir -p "\${state_root}/xdg/cache" "\${state_root}/xdg/config" "\${state_root}/xdg/data" "\${state_root}/xdg/state" "\${jwt_root}" +printf '%s' '${config_payload}' | base64 --decode >"\${config_path}" +printf '%s' '${jwt_signing_payload}' | base64 --decode >"\${jwt_root}/signing.pem" +printf '%s' '${jwt_public_payload}' | base64 --decode >"\${jwt_root}/public.pem" +printf '%s' '${jwt_kid_payload}' | base64 --decode >"\${jwt_root}/kid" +chmod 0600 "\${config_path}" +chmod 0600 "\${jwt_root}/signing.pem" "\${jwt_root}/public.pem" "\${jwt_root}/kid" +export XDG_CONFIG_HOME=\${state_root}/xdg/config +export XDG_CACHE_HOME=\${state_root}/xdg/cache +export XDG_DATA_HOME=\${state_root}/xdg/data +export XDG_STATE_HOME=\${state_root}/xdg/state +report_timing "guest gateway setup" "\${phase_started_at}" +phase_started_at=\${SECONDS} +case '${gateway_driver}' in +docker) + docker import \ + --change 'ENTRYPOINT ["/openshell-sandbox"]' \ + "${guest_supervisor_archive_path}" \ + "${supervisor_image}" >/dev/null + ;; +podman) + podman --url "unix:///run/user/\$(id -u)/podman/podman.sock" import \ + --change 'ENTRYPOINT ["/openshell-sandbox"]' \ + "${guest_supervisor_archive_path}" \ + "${supervisor_image}" >/dev/null + ;; +esac +report_timing "${gateway_driver} supervisor import" "\${phase_started_at}" +if [ '${gateway_driver}' = podman ]; then + phase_started_at=\${SECONDS} + # Keep the primary gateway listener on loopback and bridge the rootless + # container callback path temporarily. Remove this relay when #2492 lands. + relay_socket=\${state_root}/podman-gateway.sock + rm -f "\${relay_socket}" + socat "UNIX-LISTEN:\${relay_socket},fork" TCP:127.0.0.1:8080 & + host_relay_pid=\$! + for _ in \$(seq 1 50); do + [ -S "\${relay_socket}" ] && break + sleep 0.1 + done + if [ ! -S "\${relay_socket}" ] || ! kill -0 "\${host_relay_pid}" 2>/dev/null; then + echo "ERROR: failed to start the host-side Podman gateway relay" >&2 + exit 1 + fi + env -u XDG_CONFIG_HOME -u XDG_DATA_HOME -u XDG_STATE_HOME \ + podman unshare --rootless-netns \ + socat TCP-LISTEN:8080,bind=0.0.0.0,reuseaddr,fork "UNIX-CONNECT:\${relay_socket}" & + network_relay_pid=\$! + sleep 1 + if ! kill -0 "\${network_relay_pid}" 2>/dev/null; then + echo "ERROR: failed to start the rootless-network Podman gateway relay" >&2 + exit 1 + fi + report_timing "Podman network relays" "\${phase_started_at}" +fi +cd /home/openshell +exec /usr/local/bin/openshell-gateway \ + --config "\${config_path}" \ + --bind-address 127.0.0.1 \ + --port ${guest_port} \ + --disable-tls +EOF + chmod 0700 "${guest_launcher}" + + vm_args=( + nix run .#test-guest -- + --distro "${vm}" + ) + for configuration in "${with_configurations[@]}"; do + vm_args+=(--with "${configuration}") + done + vm_args+=( + --copy "${guest_gateway_bin}:/usr/local/bin/openshell-gateway" + --copy "${guest_launcher}:${guest_launcher_path}" + --copy "${supervisor_archive}:${guest_supervisor_archive_path}" + --forward-port "${host_port}:${guest_port}" + ) + if [ "${keep}" -eq 1 ]; then + vm_args+=(--keep) + fi + vm_args+=(-- "${guest_launcher_path}") + + echo "==> Starting ${vm} test guest gateway at ${gateway_endpoint}" + start_child "${ROOT}" "${runtime_log}" "${vm_args[@]}" +fi + +probe_gateway() { + python3 - "${OPENSHELL_BIN}" "${1}" <<'PY' +import os +import subprocess +import sys + +with open(sys.argv[2], "wb") as output: + try: + result = subprocess.run( + [sys.argv[1], "status"], + env={**os.environ, "NO_COLOR": "1"}, + stdout=output, + stderr=subprocess.STDOUT, + timeout=5, + check=False, + ) + except subprocess.TimeoutExpired: + raise SystemExit(124) +raise SystemExit(result.returncode) +PY +} + +wait_for_gateway() { + local started_at=${SECONDS} + local elapsed=0 + local process_status + local probe_log="${run_dir}/gateway-probe.log" + local reported_timings=0 + local timing_count + + report_vm_progress() { + if [ "${mode}" != vm ]; then + return + fi + timing_count="$(grep -c '^==> Timing:' "${runtime_log}" || true)" + if [ "${timing_count}" -le "${reported_timings}" ]; then + return + fi + sed -n 's/^==> Timing: / /p' "${runtime_log}" | + sed -n "$((reported_timings + 1)),${timing_count}p" + reported_timings=${timing_count} + } + + echo "==> Waiting up to ${gateway_ready_timeout}s for gateway readiness" + while :; do + elapsed=$((SECONDS - started_at)) + if [ "${elapsed}" -ge "${gateway_ready_timeout}" ]; then + break + fi + if ! kill -0 "${child_pid}" 2>/dev/null; then + if wait "${child_pid}"; then + process_status=0 + else + process_status=$? + fi + child_pid= + echo "ERROR: ${mode} gateway process exited before becoming ready" >&2 + if [ "${process_status}" -eq 0 ]; then + return 1 + fi + return "${process_status}" + fi + report_vm_progress + if probe_gateway "${probe_log}" && + grep -q "Connected" "${probe_log}"; then + report_vm_progress + echo "==> Gateway ready after ${elapsed}s" + return 0 + fi + sleep 1 + done + + echo "ERROR: gateway did not become ready within ${gateway_ready_timeout}s" >&2 + if [ -s "${probe_log}" ]; then + echo "=== last gateway probe ===" >&2 + cat "${probe_log}" >&2 + echo "=== end last gateway probe ===" >&2 + fi + return 1 +} + +wait_for_gateway + +echo "==> Running E2E suite: ${suite_name}" +cd "${ROOT}" +cargo test \ + --manifest-path e2e/rust/Cargo.toml \ + --features e2e \ + --test "${suite_name}" \ + -- --nocapture diff --git a/e2e/support/gateway-common.sh b/e2e/support/gateway-common.sh index e7fa6d15f..6e25b30e0 100644 --- a/e2e/support/gateway-common.sh +++ b/e2e/support/gateway-common.sh @@ -7,6 +7,12 @@ e2e_cargo_target_dir() { local root=$1 + shift + local cargo_command=(cargo) + + if [ "$#" -gt 0 ]; then + cargo_command=("$@") + fi if [ -n "${CARGO_TARGET_DIR:-}" ]; then case "${CARGO_TARGET_DIR}" in @@ -16,7 +22,7 @@ e2e_cargo_target_dir() { return 0 fi - cargo metadata --format-version=1 --no-deps \ + "${cargo_command[@]}" metadata --format-version=1 --no-deps \ | python3 -c 'import json, sys; print(json.load(sys.stdin)["target_directory"])' } diff --git a/nix/test-guest/cache-lib.sh b/nix/test-guest/cache-lib.sh index e5d5f0a7b..cfc0c2392 100644 --- a/nix/test-guest/cache-lib.sh +++ b/nix/test-guest/cache-lib.sh @@ -129,27 +129,16 @@ test_vm_cache_local_entry_valid() { local distro_name=$3 shift 3 local entry - local expected_sha - local actual_sha entry=$(test_vm_cache_entry_dir "${root}" "${key}") + # Cache builders and OCI pulls verify the disk before atomically installing + # the complete entry. Keep cache-hit validation metadata-only so launching a + # guest does not hash and inspect the entire QCOW2 on every run. [ -f "${entry}/complete" ] && - [ -f "${entry}/disk.qcow2" ] && + [ -s "${entry}/disk.qcow2" ] && [ -f "${entry}/metadata.json" ] && test_vm_cache_metadata_matches \ - "${entry}/metadata.json" "${key}" "${distro_name}" "$@" || - return 1 - - expected_sha=$(jq -er ' - .disk_sha256 | - select(type == "string" and test("^[a-f0-9]{64}$")) - ' "${entry}/metadata.json") || - return 1 - actual_sha=$(test_vm_cache_sha256 "${entry}/disk.qcow2") || - return 1 - - [ "${actual_sha}" = "${expected_sha}" ] && - test_vm_cache_validate_disk "${entry}/disk.qcow2" + "${entry}/metadata.json" "${key}" "${distro_name}" "$@" } test_vm_cache_validate_disk() { diff --git a/nix/test-guest/configuration/artifacts.yml b/nix/test-guest/configuration/artifacts.yml deleted file mode 100644 index 42e1dc47a..000000000 --- a/nix/test-guest/configuration/artifacts.yml +++ /dev/null @@ -1,88 +0,0 @@ ---- -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# PROTOTYPE: Install per-run artifacts in a disposable test guest. - -- name: Install test artifacts - hosts: test_vm - become: true - gather_facts: false - - vars: - test_guest_packages: [] - test_guest_copies: [] - test_guest_package_staging_dir: /tmp/openshell-test-guest/packages - - tasks: - - name: Install guest packages - when: test_guest_packages | length > 0 - block: - - name: Create package staging directory - ansible.builtin.file: - path: "{{ test_guest_package_staging_dir }}" - state: directory - mode: "0755" - - - name: Upload guest packages - ansible.builtin.copy: - src: "{{ item }}" - dest: >- - {{ test_guest_package_staging_dir }}/package-{{ - ansible_loop.index0 }}.{{ test_guest_package_family }} - mode: "0644" - loop: "{{ test_guest_packages }}" - loop_control: - extended: true - label: "{{ item }}" - register: test_guest_staged_packages - - - name: Refresh Debian package metadata - ansible.builtin.apt: - update_cache: true - when: test_guest_package_family == "deb" - - - name: Install Debian packages - ansible.builtin.apt: - deb: "{{ item.dest }}" - state: present - loop: "{{ test_guest_staged_packages.results }}" - loop_control: - label: "{{ item.dest }}" - when: test_guest_package_family == "deb" - - - name: Install RPM packages - ansible.builtin.dnf: - name: "{{ item.dest }}" - # --install explicitly selects this local test artifact. - disable_gpg_check: true - state: present - loop: "{{ test_guest_staged_packages.results }}" - loop_control: - label: "{{ item.dest }}" - when: test_guest_package_family == "rpm" - - always: - - name: Remove staged guest packages - ansible.builtin.file: - path: "{{ test_guest_package_staging_dir }}" - state: absent - - - name: Create artifact destination directories - ansible.builtin.file: - path: "{{ item.destination | dirname }}" - state: directory - loop: "{{ test_guest_copies }}" - loop_control: - label: "{{ item.destination }}" - - - name: Copy executable artifacts - ansible.builtin.copy: - src: "{{ item.source }}" - dest: "{{ item.destination }}" - owner: root - group: root - mode: "0755" - loop: "{{ test_guest_copies }}" - loop_control: - label: "{{ item.destination }}" diff --git a/nix/test-guest/default.nix b/nix/test-guest/default.nix index a0870bbd2..2cfc77227 100644 --- a/nix/test-guest/default.nix +++ b/nix/test-guest/default.nix @@ -53,6 +53,7 @@ let runtimeInputs = [ qemu pkgs.python3Packages.ansible-core + pkgs.python3Packages.virt-firmware pkgs.coreutils pkgs.gnugrep pkgs.jq @@ -71,7 +72,6 @@ let export OPENSHELL_TEST_GUEST_CACHE_LIB=${./cache-lib.sh} export OPENSHELL_TEST_GUEST_CACHE_RUNNER=${./cache.sh} export OPENSHELL_TEST_GUEST_CACHE_SEAL=${./cache-seal.sh} - export OPENSHELL_TEST_GUEST_ARTIFACT_PLAYBOOK=${./configuration/artifacts.yml} export OPENSHELL_TEST_GUEST_RUNNER=${./run.sh} export TEST_GUEST_BASH=${pkgs.bash}/bin/bash export TEST_GUEST_QEMU=${qemuBinary} diff --git a/nix/test-guest/run.sh b/nix/test-guest/run.sh index a6eb6176a..68266fcc4 100644 --- a/nix/test-guest/run.sh +++ b/nix/test-guest/run.sh @@ -30,7 +30,6 @@ EOF if [ "${OPENSHELL_TEST_GUEST_RUNTIME:-}" != 1 ] || [ ! -d "${OPENSHELL_TEST_GUEST_DISTROS:-}" ] || [ ! -d "${OPENSHELL_TEST_GUEST_CONFIGURATIONS:-}" ] || - [ ! -r "${OPENSHELL_TEST_GUEST_ARTIFACT_PLAYBOOK:-}" ] || [ ! -r "${OPENSHELL_TEST_GUEST_CACHE_LIB:-}" ] || [ ! -r "${OPENSHELL_TEST_GUEST_CACHE_RUNNER:-}" ]; then echo "run this script through 'nix run .#test-guest -- ...'" >&2 @@ -265,6 +264,14 @@ fi # shellcheck disable=SC1090 . "${OPENSHELL_TEST_GUEST_CACHE_LIB}" +report_timing() { + local label=$1 + local started_at=$2 + + echo "==> Timing: ${label}: $((SECONDS - started_at))s" +} + +phase_started_at=${SECONDS} prepared_image=0 TEST_GUEST_IMAGE= if [ -n "${OPENSHELL_TEST_GUEST_IMAGE_OVERRIDE:-}" ]; then @@ -306,7 +313,9 @@ if [ "${prepared_image}" -eq 0 ]; then echo "==> Realizing the pinned ${distro} cloud image" TEST_GUEST_IMAGE=$(nix build --no-link --print-out-paths "${TEST_GUEST_IMAGE_DRV}^out") fi +report_timing "guest image resolution" "${phase_started_at}" +phase_started_at=${SECONDS} umask 077 run_parent=${TMPDIR:-/tmp}/openshell-test-guest mkdir -p "${run_parent}" @@ -315,6 +324,7 @@ ssh_control_dir=$(mktemp -d /tmp/openshell-test-guest-ssh.XXXXXX) overlay=${run_dir}/disk.qcow2 seed=${run_dir}/seed.iso vars=${run_dir}/firmware-vars.fd +vars_json=${run_dir}/firmware-vars.json private_key=${run_dir}/id_ed25519 ssh_control_path=${ssh_control_dir}/ctl serial_log=${run_dir}/serial.log @@ -322,9 +332,10 @@ qemu_log=${run_dir}/qemu.log qemu_pid= ssh_port= ssh_args=() +ssh_forward_args=() +scp_args=() ansible_config=${run_dir}/ansible.cfg ansible_inventory=${run_dir}/inventory.ini -artifacts_vars=${run_dir}/artifacts.json show_logs() { if [ -s "${qemu_log}" ]; then @@ -405,7 +416,18 @@ if [ "${prepared_image}" -eq 0 ]; then fi cp "${TEST_GUEST_FIRMWARE_VARS}" "${vars}" chmod 0600 "${vars}" +# The generic EDK2 varstore defaults to a five-second boot-manager timeout. +# Seed the standard global Timeout variable before the disposable VM starts. +cat >"${vars_json}" <<'EOF' +{"version":2,"variables":[{"name":"Timeout","guid":"8be4df61-93ca-11d2-aa0d-00e098032b8c","attr":7,"data":"0000"}]} +EOF +virt-fw-vars \ + --loglevel WARNING \ + --inplace "${vars}" \ + --set-json "${vars_json}" +report_timing "VM runtime preparation" "${phase_started_at}" +phase_started_at=${SECONDS} for attempt in $(seq 1 5); do if [ -n "${requested_ssh_port}" ]; then ssh_port=${requested_ssh_port} @@ -418,11 +440,6 @@ for attempt in $(seq 1 5); do done fi netdev_arg="user,id=net0,hostfwd=tcp:127.0.0.1:${ssh_port}-:22" - for forward_spec in "${forward_ports[@]}"; do - host_port=${forward_spec%%:*} - guest_port=${forward_spec#*:} - netdev_arg+=",hostfwd=tcp:127.0.0.1:${host_port}-:${guest_port}" - done : >"${qemu_log}" echo "==> Booting ${distro} (${TEST_GUEST_ARCHITECTURE}) with QEMU/${TEST_GUEST_ACCELERATOR}" "${TEST_GUEST_QEMU}" \ @@ -472,6 +489,7 @@ ssh_args=( -i "${private_key}" -p "${ssh_port}" -o BatchMode=yes + -o Compression=yes -o ConnectTimeout=5 -o ControlMaster=auto -o ControlPersist=60 @@ -481,6 +499,32 @@ ssh_args=( -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ) +scp_args=( + -F /dev/null + -C + -i "${private_key}" + -P "${ssh_port}" + -o BatchMode=yes + -o Compression=yes + -o ConnectTimeout=5 + -o ControlMaster=auto + -o ControlPersist=60 + -o "ControlPath=${ssh_control_path}" + -o IdentitiesOnly=yes + -o LogLevel=ERROR + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null +) +if [ "${#forward_ports[@]}" -gt 0 ]; then + ssh_forward_args+=(-o ExitOnForwardFailure=yes) + for forward_spec in "${forward_ports[@]}"; do + host_port=${forward_spec%%:*} + guest_port=${forward_spec#*:} + ssh_forward_args+=( + -L "127.0.0.1:${host_port}:127.0.0.1:${guest_port}" + ) + done +fi echo "==> Waiting up to ${ssh_wait_seconds} seconds for SSH on 127.0.0.1:${ssh_port}" ssh_ready=0 @@ -501,7 +545,9 @@ if [ "${ssh_ready}" -ne 1 ]; then echo "SSH did not become ready within ${ssh_wait_seconds} seconds" >&2 exit 1 fi +report_timing "VM boot and SSH" "${phase_started_at}" +phase_started_at=${SECONDS} echo "==> Validating ${distro}" # Profile values come from the trusted Nix-generated catalog. # shellcheck disable=SC2029 @@ -510,6 +556,7 @@ ssh "${ssh_args[@]}" openshell@127.0.0.1 \ # cloud-init returns 2 when it completes with recoverable errors. Fedora can # report that status for an initial transient-hostname warning even though the # requested user and SSH configuration were applied successfully. +report_timing "guest validation" "${phase_started_at}" cat >"${ansible_config}" <[^:]+):(?.*)$")) - ' "${copies[@]}" - ) - jq -n \ - --arg package_family "${TEST_GUEST_PACKAGE_FAMILY}" \ - --argjson packages "${package_artifacts}" \ - --argjson copies "${copy_artifacts}" \ - '{test_guest_package_family: $package_family, test_guest_packages: $packages, test_guest_copies: $copies}' \ - >"${artifacts_vars}" - - echo "==> Installing per-run artifacts" - ANSIBLE_CONFIG="${ansible_config}" ANSIBLE_NOCOLOR=1 \ - ansible-playbook \ - --extra-vars "@${artifacts_vars}" \ - "${OPENSHELL_TEST_GUEST_ARTIFACT_PLAYBOOK}" + phase_started_at=${SECONDS} + artifact_staging_dir=/tmp/openshell-test-guest-artifacts-$$ + ssh "${ssh_args[@]}" openshell@127.0.0.1 \ + "install -d -m 0700 -- '${artifact_staging_dir}'" + + remote_packages=() + artifact_index=0 + for package in "${packages[@]}"; do + remote_path=${artifact_staging_dir}/package-${artifact_index}.${TEST_GUEST_PACKAGE_FAMILY} + echo "==> Copying package: ${package##*/}" + scp -q "${scp_args[@]}" \ + "${package}" "openshell@127.0.0.1:${remote_path}" + remote_packages+=("${remote_path}") + artifact_index=$((artifact_index + 1)) + done + + if [ "${#remote_packages[@]}" -gt 0 ]; then + printf -v quoted_packages ' %q' "${remote_packages[@]}" + case "${TEST_GUEST_PACKAGE_FAMILY}" in + deb) + ssh "${ssh_args[@]}" openshell@127.0.0.1 \ + "sudo apt-get update >/dev/null && sudo apt-get install -y --${quoted_packages}" + ;; + rpm) + ssh "${ssh_args[@]}" openshell@127.0.0.1 \ + "sudo dnf install -y --nogpgcheck --${quoted_packages}" + ;; + esac + fi + + artifact_index=0 + for copy_spec in "${copies[@]}"; do + source_path=${copy_spec%%:*} + destination=${copy_spec#*:} + remote_path=${artifact_staging_dir}/copy-${artifact_index} + echo "==> Copying artifact: ${destination}" + scp -q "${scp_args[@]}" \ + "${source_path}" "openshell@127.0.0.1:${remote_path}" + printf -v install_command \ + 'sudo install -D -m 0755 -- %q %q' \ + "${remote_path}" "${destination}" + ssh "${ssh_args[@]}" openshell@127.0.0.1 "${install_command}" + artifact_index=$((artifact_index + 1)) + done + + ssh "${ssh_args[@]}" openshell@127.0.0.1 \ + "rm -rf -- '${artifact_staging_dir}'" + report_timing "artifact transfer" "${phase_started_at}" fi # Configuration may change the test user's groups. Close the SSH control @@ -565,12 +641,13 @@ ssh "${ssh_args[@]}" -O exit openshell@127.0.0.1 >/dev/null 2>&1 || true echo "==> Test guest ready: ${distro} (SSH port ${ssh_port})" if [ "${#guest_command[@]}" -eq 0 ]; then - ssh -t "${ssh_args[@]}" openshell@127.0.0.1 + ssh -t "${ssh_args[@]}" "${ssh_forward_args[@]}" openshell@127.0.0.1 else printf -v quoted_command '%q ' "${guest_command[@]}" # quoted_command is shell-escaped locally before it reaches the guest. # shellcheck disable=SC2029 - ssh "${ssh_args[@]}" openshell@127.0.0.1 "bash -lc $(printf '%q' "${quoted_command}")" + ssh "${ssh_args[@]}" "${ssh_forward_args[@]}" \ + openshell@127.0.0.1 "bash -lc $(printf '%q' "${quoted_command}")" fi echo "==> Shutting down ${distro}" diff --git a/tasks/test.toml b/tasks/test.toml index 7ecee2c4e..ceb1c3008 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -37,6 +37,10 @@ hide = true description = "Run all end-to-end tests (Rust + Python + MCP)" depends = ["e2e:rust", "e2e:python", "e2e:mcp"] +["e2e:test"] +description = "Build the current checkout and run a named host or Nix test-guest E2E suite" +run = "e2e/run.sh" + ["e2e:gpu"] description = "Run Docker GPU end-to-end tests" depends = ["e2e:docker:gpu"]