From 0244500de7e279c52765af285b2a3f07affb8d04 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 22 Sep 2026 08:19:59 -0700 Subject: [PATCH 1/2] fix(dstack-util): measure os-image-hash once across a KMS failover os-image-hash was extended inside the per-URL request, before the CA cert was parsed. If that parse failed, the next KMS URL extended it again and RTMR3 could no longer be replayed. Measure only after a request fully succeeds, and report every KMS's error instead of just the first. --- dstack/dstack-util/src/system_setup.rs | 42 +++++++++++--------------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/dstack/dstack-util/src/system_setup.rs b/dstack/dstack-util/src/system_setup.rs index b58fa9546..c45b95edf 100644 --- a/dstack/dstack-util/src/system_setup.rs +++ b/dstack/dstack-util/src/system_setup.rs @@ -2306,7 +2306,7 @@ impl<'a> Stage0<'a> { self.shared.dir.join(APP_KEYS) } - async fn request_app_keys_from_kms_url(&self, kms_url: String) -> Result { + async fn request_app_keys_from_kms_url(&self, kms_url: String) -> Result<(AppKeys, Vec)> { info!("Requesting app keys from KMS: {kms_url}"); let tmp_ca = { info!("Getting temp ca cert"); @@ -2340,9 +2340,6 @@ impl<'a> Stage0<'a> { .await .context("Failed to get app key")?; - emit_runtime_event("os-image-hash", &response.os_image_hash) - .context("failed to extend os-image-hash to the launch measurement")?; - let (_, ca_pem) = x509_parser::pem::parse_x509_pem(tmp_ca.ca_cert.as_bytes()) .context("Failed to parse ca cert")?; let x509 = ca_pem.parse_x509().context("Failed to parse ca cert")?; @@ -2362,34 +2359,31 @@ impl<'a> Stage0<'a> { tmp_ca_cert: tmp_ca.temp_ca_cert, }, }; - Ok(keys) + Ok((keys, response.os_image_hash)) } async fn request_app_keys_from_kms(&self) -> Result { if self.shared.sys_config.kms_urls.is_empty() { bail!("No KMS URLs are set"); } - let keys = 'out: { - let mut error = anyhow!("unknown error"); - for (i, kms_url) in self.shared.sys_config.kms_urls.iter().enumerate() { - let kms_url = kms_rpc_url(kms_url); - let response = self.request_app_keys_from_kms_url(kms_url.clone()).await; - match response { - Ok(response) => { - break 'out response; - } - Err(err) => { - warn!("Failed to get app keys from KMS {kms_url}: {err:?}"); - // Record the first error - if i == 0 { - error = err; - } - } + let mut errors = vec![]; + for kms_url in &self.shared.sys_config.kms_urls { + let kms_url = kms_rpc_url(kms_url); + match self.request_app_keys_from_kms_url(kms_url.clone()).await { + Ok((keys, os_image_hash)) => { + // Measured here, outside the per-URL request, so that a + // failover can never extend os-image-hash twice. + emit_runtime_event("os-image-hash", &os_image_hash) + .context("failed to extend os-image-hash to the launch measurement")?; + return Ok(keys); + } + Err(err) => { + warn!("Failed to get app keys from KMS {kms_url}: {err:?}"); + errors.push(format!("{kms_url}: {err:#}")); } } - return Err(error).context("Failed to get app keys from KMS"); - }; - Ok(keys) + } + bail!("Failed to get app keys from KMS: {}", errors.join("; ")) } fn verify_key_provider_id(&self, provider_id: &[u8]) -> Result<()> { From 1a78afb6c40b2612a8fb0edf74a894f67c57c1a6 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 22 Sep 2026 09:07:42 -0700 Subject: [PATCH 2/2] fix(dstack-util): keep KMS failover error as a context layer, lowercase messages --- dstack/dstack-util/src/system_setup.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dstack/dstack-util/src/system_setup.rs b/dstack/dstack-util/src/system_setup.rs index c45b95edf..0d1d537d7 100644 --- a/dstack/dstack-util/src/system_setup.rs +++ b/dstack/dstack-util/src/system_setup.rs @@ -2378,12 +2378,12 @@ impl<'a> Stage0<'a> { return Ok(keys); } Err(err) => { - warn!("Failed to get app keys from KMS {kms_url}: {err:?}"); + warn!("failed to get app keys from KMS {kms_url}: {err:?}"); errors.push(format!("{kms_url}: {err:#}")); } } } - bail!("Failed to get app keys from KMS: {}", errors.join("; ")) + Err(anyhow!(errors.join("; "))).context("failed to get app keys from KMS") } fn verify_key_provider_id(&self, provider_id: &[u8]) -> Result<()> {