From 5e4a0fb5996ebd99e2b27077e7db8be5252963e4 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 19 Sep 2026 23:56:55 -0700 Subject: [PATCH 1/6] test(verifier): pin the TCB status the result contract reports --- dstack/dstack-attest/src/attestation.rs | 110 ++++++++++++++++ dstack/verifier/README.md | 3 +- dstack/verifier/src/verification.rs | 164 ++++++++++++++++++++---- 3 files changed, 251 insertions(+), 26 deletions(-) diff --git a/dstack/dstack-attest/src/attestation.rs b/dstack/dstack-attest/src/attestation.rs index e55d63827..2373b401c 100644 --- a/dstack/dstack-attest/src/attestation.rs +++ b/dstack/dstack-attest/src/attestation.rs @@ -2615,6 +2615,116 @@ pub struct AppInfo { mod tests { use super::*; + /// Build a TD10 verified report carrying `status`. Everything else is the + /// shape `validate_tcb` accepts, so a row that fails failed on the status. + fn td10_verified_report(status: &str) -> TdxVerifiedReport { + TdxVerifiedReport { + status: status.to_string(), + advisory_ids: vec!["INTEL-SA-00001".to_string()], + report: Report::TD10(TDReport10 { + tee_tcb_svn: [0; 16], + mr_seam: [0; 48], + mr_signer_seam: [0; 48], + seam_attributes: [0; 8], + td_attributes: [0; 8], + xfam: [0; 8], + mr_td: [0; 48], + mr_config_id: [0; 48], + mr_owner: [0; 48], + mr_owner_config: [0; 48], + rt_mr0: [0; 48], + rt_mr1: [0; 48], + rt_mr2: [0; 48], + rt_mr3: [0; 48], + report_data: [0; 64], + }), + ppid: Vec::new(), + qe_status: dcap_qvl::tcb_info::TcbStatusWithAdvisory::new( + dcap_qvl::tcb_info::TcbStatus::UpToDate, + Vec::new(), + ), + platform_status: dcap_qvl::tcb_info::TcbStatusWithAdvisory::new( + dcap_qvl::tcb_info::TcbStatus::UpToDate, + Vec::new(), + ), + } + } + + /// `validate_tcb` does not read `status`, by design: whether a non-current + /// TCB is acceptable is a downstream policy decision, and the verifier + /// surfaces the string instead of ruling on it + /// (`docs/security/security-model.md`, "TCB status is surfaced, not + /// gated"). Enumerate every `dcap_qvl::tcb_info::TcbStatus` so the day that + /// stops being true, it stops here rather than in a relying party's logs. + /// + /// `Revoked` is in the list too, and it passes here as well: the only thing + /// that keeps a revoked TCB out is `dcap_qvl`'s own `is_valid()`, which + /// runs before this function ever sees the report. Nothing in dstack + /// re-checks it. + #[test] + fn validate_tcb_accepts_every_tcb_status_string() { + use dcap_qvl::tcb_info::TcbStatus; + + let statuses = [ + TcbStatus::UpToDate, + TcbStatus::SWHardeningNeeded, + TcbStatus::ConfigurationNeeded, + TcbStatus::ConfigurationAndSWHardeningNeeded, + TcbStatus::OutOfDate, + TcbStatus::OutOfDateConfigurationNeeded, + TcbStatus::Revoked, + ]; + for status in statuses { + let status = serde_json::to_value(status).unwrap(); + let status = status.as_str().unwrap(); + assert!( + validate_tcb(&td10_verified_report(status)).is_ok(), + "validate_tcb now rejects {status}; \ + docs/security/security-model.md says it surfaces the status instead" + ); + } + } + + /// The three invariants `validate_tcb` does enforce. They are hard + /// invariants rather than policy: a debug TD's measurements mean nothing, + /// and a non-zero SEAM signer or service TD is a TD this verifier does not + /// model. + #[test] + fn validate_tcb_rejects_debug_mode_and_unexpected_seam_measurements() { + let mut debug = td10_verified_report("UpToDate"); + if let Report::TD10(report) = &mut debug.report { + report.td_attributes[0] |= 0x01; + } + assert!(validate_tcb(&debug) + .unwrap_err() + .to_string() + .contains("Debug mode")); + + let mut signer_seam = td10_verified_report("UpToDate"); + if let Report::TD10(report) = &mut signer_seam.report { + report.mr_signer_seam[0] = 1; + } + assert!(validate_tcb(&signer_seam) + .unwrap_err() + .to_string() + .contains("mr signer seam")); + + let td10 = td10_verified_report("UpToDate"); + let Report::TD10(base) = td10.report.clone() else { + unreachable!("built as TD10") + }; + let mut service_td = td10; + service_td.report = Report::TD15(TDReport15 { + base, + tee_tcb_svn2: [0; 16], + mr_service_td: [1; 48], + }); + assert!(validate_tcb(&service_td) + .unwrap_err() + .to_string() + .contains("mr service td")); + } + #[test] fn app_info_defaults_missing_init_script_hashes() { let app_info: AppInfo = serde_json::from_value(serde_json::json!({ diff --git a/dstack/verifier/README.md b/dstack/verifier/README.md index 8529dc1f6..0a0a01210 100644 --- a/dstack/verifier/README.md +++ b/dstack/verifier/README.md @@ -42,7 +42,7 @@ against the returned evidence. "os_image_version": "0.5.10", // dstack OS version, null if unknown "tee_variant": "dstack-tdx", // dstack-tdx | dstack-gcp-tdx | dstack-nitro-enclave | dstack-amd-sev-snp | dstack-aws-nitro-tpm "report_data": "hex-encoded-64-byte-report-data", - "tcb_status": "UpToDate", + "tcb_status": "UpToDate", // surfaced, not gated: is_valid ignores it "advisory_ids": [], "key_provider": { "name": "kms", "id": "hex-string" }, // decoded; null if absent "app_info": { @@ -291,6 +291,7 @@ Beyond pass/fail, the result carries a few descriptive fields so a relying party - **`os_image_version`** — the dstack OS version (e.g. `0.5.10`), useful for enforcing a minimum version. - **`tee_variant`** — the TEE variant that produced the verified quote, serialized as `TeeVariant`: `dstack-tdx`, `dstack-gcp-tdx`, `dstack-nitro-enclave`, `dstack-amd-sev-snp`, or `dstack-aws-nitro-tpm`. - **`acpi_tables_verified`** — whether TDX ACPI table contents were verified. This is useful for relying parties that require `requirements.tdx_measure_acpi_tables = true`. +- **`tcb_status`** and **`advisory_ids`** — the platform TCB status (`UpToDate`, `OutOfDate`, `ConfigurationNeeded`, …) and the Intel advisories that go with it. **`is_valid` does not depend on this.** dstack surfaces the status and leaves the ruling to the relying party, because how much a non-current TCB matters differs per deployment; see [TCB status is surfaced, not gated](../../docs/security/security-model.md#tcb-status-is-surfaced-not-gated). A relying party that wants a current TCB must check `tcb_status == "UpToDate"` itself. The one status the verifier does reject is `Revoked`, which `dcap-qvl` fails before it reaches the result. `tcb_status` is `null` on platforms with no TCB surface (Nitro Enclave), so an equality check against `"UpToDate"` fails closed there. - **`key_provider`** — the decoded `app_info.key_provider_info` (`{name, id}`); `name` is e.g. `kms` or `local`. A `local` key provider means the CVM is not KMS-backed, which is itself a dev/insecure posture signal. The raw bytes remain in `app_info.key_provider_info`. - **`boot_info`** — the policy object a relying party should feed to its auth/governance layer. For AWS EC2 NitroTPM this includes `teeVariant = dstack-aws-nitro-tpm`, PCR4/7/12-derived `osImageHash`, PCR14-bound `mrAggregated`, app identity, instance/device identity, and a `tcbStatus` normalized to `UpToDate`. diff --git a/dstack/verifier/src/verification.rs b/dstack/verifier/src/verification.rs index 191736726..d3bb2526c 100644 --- a/dstack/verifier/src/verification.rs +++ b/dstack/verifier/src/verification.rs @@ -717,37 +717,56 @@ impl CvmVerifier { } else { bail!("Quote is required"); }; - let mut details = VerificationDetails::default(); - let debug = request.debug.unwrap_or(false); let attestation = attestation.into_v1(); - let verified = attestation.verify(&self.attestation_verifier).await; - let verified_attestation = match verified { - Ok(att) => { - details.quote_verified = true; - details.tee_variant = Some(att.quote.variant()); - // keep the top-level tcb_status consistent with the - // boot_info.tcbStatus fed to the auth policy (notably AWS - // NitroTPM, which is normalized to "UpToDate" there). - let (tcb_status, advisory_ids) = policy_tcb_fields(&att); - details.tcb_status = (!tcb_status.is_empty()).then_some(tcb_status); - details.advisory_ids = advisory_ids; - details.report_data = Some(hex::encode(att.report_data)); - att - } + let verified_attestation = match attestation.verify(&self.attestation_verifier).await { + Ok(att) => att, Err(e) => { return Ok(VerificationResponse { is_valid: false, - details, + details: VerificationDetails::default(), reason: Some(format!("Quote verification failed: {e:#}")), }); } }; + Ok(self + .verify_attested(&verified_attestation, request_vm_config, debug) + .await) + } + + /// Everything `/verify` reports once the quote itself has verified. + /// + /// Split out from `verify` so the result contract can be exercised against + /// a real verified attestation with one field varied at a time. The TCB + /// status is the field that needs it: dstack surfaces it rather than gating + /// on it (`docs/security/security-model.md`), which is only a safe default + /// as long as every relying party can see it, so what `is_valid` and + /// `tcb_status` say for a non-`UpToDate` platform has to be pinned by a + /// test rather than inferred from `validate_tcb` not mentioning it. + async fn verify_attested( + &self, + verified_attestation: &VerifiedAttestation, + request_vm_config: String, + debug: bool, + ) -> VerificationResponse { + // keep the top-level tcb_status consistent with the + // boot_info.tcbStatus fed to the auth policy (notably AWS + // NitroTPM, which is normalized to "UpToDate" there). + let (tcb_status, advisory_ids) = policy_tcb_fields(verified_attestation); + let mut details = VerificationDetails { + quote_verified: true, + tee_variant: Some(verified_attestation.quote.variant()), + tcb_status: (!tcb_status.is_empty()).then_some(tcb_status), + advisory_ids, + report_data: Some(hex::encode(verified_attestation.report_data)), + ..Default::default() + }; + // Step 3: Verify os-image-hash matches using dstack-mr let verified = self .verify_os_image_hash( request_vm_config.clone(), - &verified_attestation, + verified_attestation, debug, &mut details, ) @@ -755,11 +774,11 @@ impl CvmVerifier { let vm_config = match verified { Ok(vm_config) => vm_config, Err(e) => { - return Ok(VerificationResponse { + return VerificationResponse { is_valid: false, details, reason: Some(format!("OS image hash verification failed: {e:#}")), - }); + }; } }; details.os_image_hash_verified = true; @@ -767,7 +786,7 @@ impl CvmVerifier { Ok(mut info) => { info.os_image_hash = vm_config.os_image_hash; details.boot_info = Some(policy_boot_info_from_verified_app_info( - &verified_attestation, + verified_attestation, &info, )); details.event_log_verified = true; @@ -775,19 +794,19 @@ impl CvmVerifier { details.app_info = Some(info); } Err(e) => { - return Ok(VerificationResponse { + return VerificationResponse { is_valid: false, details, reason: Some(format!("Event log verification failed: {}", e)), - }); + }; } }; - Ok(VerificationResponse { + VerificationResponse { is_valid: true, details, reason: None, - }) + } } pub async fn verify_os_image_hash( @@ -2231,6 +2250,101 @@ mod tests { ); } + /// `/verify` reports the TCB status; it does not gate on it. + /// + /// `docs/security/security-model.md` ("TCB status is surfaced, not gated") + /// makes that a deliberate policy: whether an `OutOfDate` platform is + /// acceptable belongs to the relying party, not to the verification + /// primitive. The risk in a documented fail-open is that nothing pins it -- + /// `validate_tcb` simply never mentions `status`, and the only gate is + /// `dcap_qvl`'s own `is_valid()`, a dependency that can change under a + /// version bump without a line of dstack changing. + /// + /// So drive a real verified TDX attestation through the whole result path + /// with only the status varied, and pin the exact `is_valid`/`tcb_status` + /// pair each one produces. A future dcap-qvl that starts rejecting + /// `OutOfDate` is not wrong -- but it changes what `/verify` answers, and + /// this is where that has to be noticed. + #[tokio::test] + async fn tcb_status_is_surfaced_without_changing_is_valid() { + let request: VerificationRequest = + serde_json::from_str(include_str!("../fixtures/tdx-lite-attestation.json")) + .expect("TDX lite verifier fixture parses"); + let cache = tempfile::tempdir().expect("temp cache dir"); + let verifier = CvmVerifier::new( + cache.path().join("cache").display().to_string(), + "http://127.0.0.1:9/should-not-download/{OS_IMAGE_HASH}.tar.gz".to_string(), + Duration::from_secs(1), + test_attestation_verifier(), + ); + let mut verified = VersionedAttestation::from_bytes( + &request.attestation.expect("fixture carries an attestation"), + ) + .expect("fixture decodes") + .into_v1() + .verify(&verifier.attestation_verifier) + .await + .expect("fixture quote verifies"); + + // Every status dcap-qvl can return, plus the empty string a platform + // with no TCB surface produces. `Revoked` is listed for completeness: + // dcap-qvl rejects it before the report reaches this path, so it can + // only be reached by constructing the report, and if it ever does reach + // here it is reported like any other status. + let rows = [ + ("UpToDate", Some("UpToDate")), + ("SWHardeningNeeded", Some("SWHardeningNeeded")), + ("ConfigurationNeeded", Some("ConfigurationNeeded")), + ( + "ConfigurationAndSWHardeningNeeded", + Some("ConfigurationAndSWHardeningNeeded"), + ), + ("OutOfDate", Some("OutOfDate")), + ( + "OutOfDateConfigurationNeeded", + Some("OutOfDateConfigurationNeeded"), + ), + ("Revoked", Some("Revoked")), + // No TCB surface: reported as null so a relying party requiring + // "UpToDate" fails closed rather than matching a fabricated string. + ("", None), + ]; + for (status, expected) in rows { + let DstackVerifiedReport::DstackTdx(report) = &mut verified.report else { + panic!("fixture is expected to be a plain dstack TDX attestation"); + }; + report.status = status.to_string(); + report.advisory_ids = vec!["INTEL-SA-00001".to_string()]; + + let response = verifier + .verify_attested(&verified, String::new(), false) + .await; + assert!( + response.is_valid, + "{status}: TCB status is surfaced, not gated: {:?}", + response.reason + ); + assert_eq!( + response.details.tcb_status.as_deref(), + expected, + "{status}: tcb_status is the one field a relying party can gate on" + ); + assert_eq!( + response.details.advisory_ids, + vec!["INTEL-SA-00001".to_string()], + "{status}: advisory ids must reach the relying party too" + ); + let boot_info = response + .details + .boot_info + .expect("a valid result carries policy boot info"); + assert_eq!( + boot_info.tcb_status, status, + "{status}: the auth-policy payload must carry the same status" + ); + } + } + /// The fixture was captured from a real CVM, so its RTMR0 ACPI digests are /// whatever QEMU actually produced. Reproducing them without the image /// proves the generator agrees with hardware, not just with itself. From 38b84604dc7da9714115aa508f246b8d3f4af7f1 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 19 Sep 2026 23:57:50 -0700 Subject: [PATCH 2/6] docs(verifier): say which path reports os_image_is_dev, and what to gate on instead --- dstack/verifier/README.md | 14 ++++++--- dstack/verifier/src/types.rs | 20 ++++++++++-- dstack/verifier/src/verification.rs | 49 +++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 7 deletions(-) diff --git a/dstack/verifier/README.md b/dstack/verifier/README.md index 0a0a01210..42723b2ea 100644 --- a/dstack/verifier/README.md +++ b/dstack/verifier/README.md @@ -38,8 +38,8 @@ against the returned evidence. "event_log_verified": true, // See "Verification Process" for semantics "os_image_hash_verified": true, "acpi_tables_verified": true, // true only when TDX ACPI table contents are verified - "os_image_is_dev": false, // true=dev image, false=prod, null=unknown/N/A - "os_image_version": "0.5.10", // dstack OS version, null if unknown + "os_image_is_dev": false, // true=dev, false=prod; null on every path but TDX legacy + "os_image_version": "0.5.10", // dstack OS version; null on the same paths "tee_variant": "dstack-tdx", // dstack-tdx | dstack-gcp-tdx | dstack-nitro-enclave | dstack-amd-sev-snp | dstack-aws-nitro-tpm "report_data": "hex-encoded-64-byte-report-data", "tcb_status": "UpToDate", // surfaced, not gated: is_valid ignores it @@ -287,12 +287,16 @@ keeps verifying; one that changes it surfaces as a digest mismatch. Beyond pass/fail, the result carries a few descriptive fields so a relying party can apply its own policy: -- **`os_image_is_dev`** — `true` for a development OS image, `false` for production. Dev images are built for local testing and are not hardened for production use, so a relying party generally wants to reject them. -- **`os_image_version`** — the dstack OS version (e.g. `0.5.10`), useful for enforcing a minimum version. +- **`os_image_is_dev`** — `true` for a development OS image, `false` for production, `null` when the evidence does not carry it. Dev images are built for local testing and are not hardened for production use, so a relying party generally wants to reject them — but see the note below: on every path except TDX legacy this field is `null`, and the way to reject dev images is to allowlist `os_image_hash`. +- **`os_image_version`** — the dstack OS version (e.g. `0.5.10`), useful for enforcing a minimum version. `null` on the same paths, for the same reason. - **`tee_variant`** — the TEE variant that produced the verified quote, serialized as `TeeVariant`: `dstack-tdx`, `dstack-gcp-tdx`, `dstack-nitro-enclave`, `dstack-amd-sev-snp`, or `dstack-aws-nitro-tpm`. - **`acpi_tables_verified`** — whether TDX ACPI table contents were verified. This is useful for relying parties that require `requirements.tdx_measure_acpi_tables = true`. - **`tcb_status`** and **`advisory_ids`** — the platform TCB status (`UpToDate`, `OutOfDate`, `ConfigurationNeeded`, …) and the Intel advisories that go with it. **`is_valid` does not depend on this.** dstack surfaces the status and leaves the ruling to the relying party, because how much a non-current TCB matters differs per deployment; see [TCB status is surfaced, not gated](../../docs/security/security-model.md#tcb-status-is-surfaced-not-gated). A relying party that wants a current TCB must check `tcb_status == "UpToDate"` itself. The one status the verifier does reject is `Revoked`, which `dcap-qvl` fails before it reaches the result. `tcb_status` is `null` on platforms with no TCB surface (Nitro Enclave), so an equality check against `"UpToDate"` fails closed there. - **`key_provider`** — the decoded `app_info.key_provider_info` (`{name, id}`); `name` is e.g. `kms` or `local`. A `local` key provider means the CVM is not KMS-backed, which is itself a dev/insecure posture signal. The raw bytes remain in `app_info.key_provider_info`. - **`boot_info`** — the policy object a relying party should feed to its auth/governance layer. For AWS EC2 NitroTPM this includes `teeVariant = dstack-aws-nitro-tpm`, PCR4/7/12-derived `osImageHash`, PCR14-bound `mrAggregated`, app identity, instance/device identity, and a `tcbStatus` normalized to `UpToDate`. -`os_image_is_dev` and `os_image_version` are read from the image's `metadata.json`, which is part of `sha256sum.txt` and therefore bound to the `os_image_hash` that step 3 verifies against the quote — so they are as trustworthy as the os-image-hash check itself. They are `null` when the platform does not expose them (e.g. GCP TDX / Nitro Enclave) or when the image predates the field (images without `is_dev` are always production). +`os_image_is_dev` and `os_image_version` are read from the image's `metadata.json`, which is listed in `sha256sum.txt` and therefore bound to the `os_image_hash` that step 3 verifies against the quote — so where they are reported, they are as trustworthy as the os-image-hash check itself. + +**They are reported only on the TDX legacy path.** That is the only path that downloads the image, and so the only one that has `metadata.json`'s *contents*; the self-contained paths carry `sha256sum.txt`, which binds `metadata.json`'s digest and nothing more. On TDX lite, SEV-SNP, GCP TDX, AWS NitroTPM and Nitro Enclave both fields are `null`, and `null` also covers an image built before the field existed (those are always production). + +Since TDX lite is the primary path today, **do not gate on `os_image_is_dev`**: a policy of "reject when `os_image_is_dev == true`" accepts every dev image on every self-contained path. Reject dev images the same way you pin any other property of the image — allowlist the `os_image_hash` values you have vetted. A dev image is a different image with a different hash, so the allowlist already excludes it, and `os_image_hash` is verified against the quote on every path. diff --git a/dstack/verifier/src/types.rs b/dstack/verifier/src/types.rs index d9d71442b..2dacebc51 100644 --- a/dstack/verifier/src/types.rs +++ b/dstack/verifier/src/types.rs @@ -100,9 +100,25 @@ pub struct VerificationDetails { /// It stays false where the check does not apply: GCP TDX, which measures /// through the vTPM instead, and the SEV-SNP and Nitro Enclave paths. pub acpi_tables_verified: bool, - /// dev vs prod OS image, from metadata.json (bound to os_image_hash). None if not exposed. + /// dev vs prod OS image: `Some(true)` dev, `Some(false)` production, + /// `None` not established. + /// + /// `None` is the common case, not the exception. The flag lives in the + /// image's `metadata.json`, and `metadata.json` reaches the verifier only + /// on the TDX legacy path, which downloads the image. Every other path is + /// self-contained by design: the measurement documents carry + /// `sha256sum.txt`, which binds `metadata.json`'s *digest*, never its + /// contents. So TDX lite, SEV-SNP, GCP TDX, AWS NitroTPM and Nitro Enclave + /// all report `None`, and no amount of work on the evidence changes that -- + /// `metadata.json` would have to be added to the measurement material. + /// + /// A relying party therefore cannot reject dev images by reading this + /// field. Reject them by allowlisting `os_image_hash`, which is the check + /// the self-contained paths are built around anyway: a dev image and a + /// production image are different images with different hashes. pub os_image_is_dev: Option, - /// dstack OS version, from the same metadata.json. + /// dstack OS version, from the same `metadata.json`, with the same + /// availability: set on the TDX legacy path, `None` everywhere else. pub os_image_version: Option, /// TEE variant that produced the verified quote. pub tee_variant: Option, diff --git a/dstack/verifier/src/verification.rs b/dstack/verifier/src/verification.rs index d3bb2526c..37c12a10b 100644 --- a/dstack/verifier/src/verification.rs +++ b/dstack/verifier/src/verification.rs @@ -2250,6 +2250,55 @@ mod tests { ); } + /// `os_image_is_dev` and `os_image_version` come out of the image's + /// `metadata.json`, and no self-contained path ever sees that file. + /// + /// The measurement documents carry `sha256sum.txt`, which binds + /// `metadata.json`'s digest and not its contents, so TDX lite and SEV-SNP + /// -- and with them GCP TDX, AWS NitroTPM and Nitro Enclave -- can only + /// report `null`. That is a contract, not an omission to be filled in + /// later, and it is why `verifier/README.md` tells relying parties to + /// reject dev images by allowlisting `os_image_hash` rather than by + /// reading this field. Pin it so the README and the code cannot drift: + /// a change that starts populating these on a self-contained path is a + /// change to what the evidence carries. + #[tokio::test] + async fn self_contained_paths_report_no_os_image_metadata() { + for (name, fixture) in [ + ( + "tdx-lite", + include_str!("../fixtures/tdx-lite-attestation.json"), + ), + ( + "sev-snp", + include_str!("../fixtures/sev-snp-attestation.json"), + ), + ] { + let request: VerificationRequest = + serde_json::from_str(fixture).expect("verifier fixture parses"); + let cache = tempfile::tempdir().expect("temp cache dir"); + let verifier = CvmVerifier::new( + cache.path().join("cache").display().to_string(), + "http://127.0.0.1:9/should-not-download/{OS_IMAGE_HASH}.tar.gz".to_string(), + Duration::from_secs(1), + test_attestation_verifier(), + ); + let response = verifier.verify(request).await.expect("verifier runs"); + assert!(response.is_valid, "{name}: {:?}", response.reason); + assert_eq!( + response.details.os_image_is_dev, None, + "{name}: metadata.json is not part of the self-contained evidence" + ); + assert_eq!( + response.details.os_image_version, None, + "{name}: metadata.json is not part of the self-contained evidence" + ); + // The identity that *is* established on these paths, and the one a + // relying party has to allowlist instead. + assert!(response.details.os_image_hash_verified, "{name}"); + } + } + /// `/verify` reports the TCB status; it does not gate on it. /// /// `docs/security/security-model.md` ("TCB status is surfaced, not gated") From 1f071b035fc003fee09f5ce096b4405a331eefe2 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 20 Sep 2026 00:01:31 -0700 Subject: [PATCH 3/6] feat(verifier): report which trust anchor os_image_hash_verified came from --- dstack/verifier/README.md | 6 + dstack/verifier/src/types.rs | 46 ++++++++ dstack/verifier/src/verification.rs | 167 +++++++++++++++++++++++++++- 3 files changed, 217 insertions(+), 2 deletions(-) diff --git a/dstack/verifier/README.md b/dstack/verifier/README.md index 42723b2ea..d64722d89 100644 --- a/dstack/verifier/README.md +++ b/dstack/verifier/README.md @@ -37,6 +37,7 @@ against the returned evidence. "quote_verified": true, "event_log_verified": true, // See "Verification Process" for semantics "os_image_hash_verified": true, + "os_image_hash_anchor": "measurement_document", // published_image | measurement_document | quoted_pcrs "acpi_tables_verified": true, // true only when TDX ACPI table contents are verified "os_image_is_dev": false, // true=dev, false=prod; null on every path but TDX legacy "os_image_version": "0.5.10", // dstack OS version; null on the same paths @@ -243,6 +244,10 @@ The verifier performs the following verification steps: - For the full-image TDX path, downloads or loads the image identified by `os_image_hash`, checks the image checksum manifest, uses dstack-mr to compute expected MRTD/RTMR0-2, and compares them against the verified measurements from the quote - For TDX lite, AMD SEV-SNP, and GCP TDX, verifies that `os_image_hash = sha256(sha256sum.txt)`, where `sha256sum.txt` is the image build's checksum manifest (` ` lines for image files), that the manifest entry for `measurement.tdx.cbor`, `measurement.snp.cbor`, or `measurement.gcp.cbor` matches the supplied measurement material, and that the measurement material replays to the quote's hardware-signed measurements or GCP TPM UKI event - For AWS NitroTPM, requires `vm_config.aws_measurement` and verifies that `os_image_hash = sha256(sha256sum.txt)` matches the measurement material and that its `boot_pcr_digest = sha256(PCR4 || PCR7 || PCR12)` matches the attested boot PCRs + - Reports which of these ran as `details.os_image_hash_anchor`. All three end by comparing recomputed measurements against hardware-signed ones, so none of them lets a requester change *what ran* — but they differ in what pins the *name*: + - `published_image` (TDX legacy) — the verifier fetched the image published under this hash and measured its bytes, so the hash names an artifact. + - `measurement_document` (TDX lite, SEV-SNP, GCP TDX, AWS NitroTPM) — the hash is the digest of a `sha256sum.txt` the requester supplied in its own `vm_config`. A requester who writes its own manifest gets `os_image_hash_verified: true` for whatever digest that manifest has. The hash is a truthful label for the material that produced the attested boot; **it is an identity only if the relying party allowlists it.** + - `quoted_pcrs` (Nitro Enclave) — the hash is computed from the signed PCR0/1/2 in the attestation document, so the requester supplies no measurement material at all. 4. **Policy Input Construction**: Emits `details.boot_info`, the canonical auth-policy payload shape used by dstack KMS `/bootAuth/app` and `/bootAuth/kms`. This object is only present on successful verification. 5. **Endpoint Certificate Verification**: In `--verify-cert` mode, verifies the dstack RA-TLS certificate extension and checks that the attestation @@ -290,6 +295,7 @@ Beyond pass/fail, the result carries a few descriptive fields so a relying party - **`os_image_is_dev`** — `true` for a development OS image, `false` for production, `null` when the evidence does not carry it. Dev images are built for local testing and are not hardened for production use, so a relying party generally wants to reject them — but see the note below: on every path except TDX legacy this field is `null`, and the way to reject dev images is to allowlist `os_image_hash`. - **`os_image_version`** — the dstack OS version (e.g. `0.5.10`), useful for enforcing a minimum version. `null` on the same paths, for the same reason. - **`tee_variant`** — the TEE variant that produced the verified quote, serialized as `TeeVariant`: `dstack-tdx`, `dstack-gcp-tdx`, `dstack-nitro-enclave`, `dstack-amd-sev-snp`, or `dstack-aws-nitro-tpm`. +- **`os_image_hash_anchor`** — what `os_image_hash_verified` is anchored to on the path that ran: `published_image`, `measurement_document`, or `quoted_pcrs`; `null` when os-image-hash verification did not complete. See step 3 of the verification process. A relying party that treats `os_image_hash` as an identity must allowlist it on every anchor, and doubly so on `measurement_document`, where the requester picked it. - **`acpi_tables_verified`** — whether TDX ACPI table contents were verified. This is useful for relying parties that require `requirements.tdx_measure_acpi_tables = true`. - **`tcb_status`** and **`advisory_ids`** — the platform TCB status (`UpToDate`, `OutOfDate`, `ConfigurationNeeded`, …) and the Intel advisories that go with it. **`is_valid` does not depend on this.** dstack surfaces the status and leaves the ruling to the relying party, because how much a non-current TCB matters differs per deployment; see [TCB status is surfaced, not gated](../../docs/security/security-model.md#tcb-status-is-surfaced-not-gated). A relying party that wants a current TCB must check `tcb_status == "UpToDate"` itself. The one status the verifier does reject is `Revoked`, which `dcap-qvl` fails before it reaches the result. `tcb_status` is `null` on platforms with no TCB surface (Nitro Enclave), so an equality check against `"UpToDate"` fails closed there. - **`key_provider`** — the decoded `app_info.key_provider_info` (`{name, id}`); `name` is e.g. `kms` or `local`. A `local` key provider means the CVM is not KMS-backed, which is itself a dev/insecure posture signal. The raw bytes remain in `app_info.key_provider_info`. diff --git a/dstack/verifier/src/types.rs b/dstack/verifier/src/types.rs index 2dacebc51..6f989a831 100644 --- a/dstack/verifier/src/types.rs +++ b/dstack/verifier/src/types.rs @@ -88,6 +88,14 @@ pub struct VerificationDetails { /// event log payloads. pub event_log_verified: bool, pub os_image_hash_verified: bool, + /// What `os_image_hash_verified` is anchored to on the path that ran. + /// + /// `None` when os-image-hash verification did not complete. + /// + /// The paths differ by an entire trust anchor and the rest of the response + /// looks identical either way, so without this a relying party cannot tell + /// which guarantee it got. See [`OsImageHashAnchor`]. + pub os_image_hash_anchor: Option, /// Indicates that TDX ACPI table contents were verified. /// /// Both dstack TDX paths set this. The full-image path recomputes the @@ -137,6 +145,44 @@ pub struct VerificationDetails { pub rtmr_debug: Option>, } +/// What binds `os_image_hash` to the boot the quote attests. +/// +/// No path lets a requester change *what ran*: all three end by comparing +/// recomputed measurements against hardware-signed ones, and a mismatch is a +/// rejection. They differ in what pins the *name* -- whether `os_image_hash` +/// identifies a published artifact, or is merely a self-consistent label the +/// requester chose for the material it supplied. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum OsImageHashAnchor { + /// TDX legacy. The verifier fetched the image identified by + /// `os_image_hash` from its configured `download_url`, required + /// `sha256(sha256sum.txt)` of what it got to equal that hash, checked every + /// file in the manifest against its digest, and recomputed MRTD and + /// RTMR0-2 from those bytes. `os_image_hash` therefore names an artifact + /// the verifier's own image source publishes under that hash. + PublishedImage, + /// TDX lite, SEV-SNP, GCP TDX and AWS NitroTPM. The requester supplied the + /// measurement material (`sha256sum.txt` plus a `measurement.*.cbor`) in + /// its own `vm_config`; the verifier required `sha256(sha256sum.txt)` to + /// equal `os_image_hash` and the CBOR to match its manifest entry, then + /// recomputed the measurements from that CBOR and required them to equal + /// the quoted ones. + /// + /// The binding is self-referential: a requester who builds its own + /// `sha256sum.txt` gets `os_image_hash_verified: true` for whatever hash + /// that file happens to have. What it cannot do is make the recomputed + /// measurements match a boot that did not happen. So the hash is a + /// truthful label for the material that produced the attested boot, and + /// nothing more -- **it is only an identity if the relying party + /// allowlists it.** + MeasurementDocument, + /// Nitro Enclave. `os_image_hash` is computed from the signed PCR0/1/2 in + /// the attestation document and compared against the declared one, so the + /// requester supplies no measurement material at all. + QuotedPcrs, +} + #[derive(Debug, Clone, Serialize)] pub struct AcpiTables { pub tables: String, diff --git a/dstack/verifier/src/verification.rs b/dstack/verifier/src/verification.rs index 37c12a10b..b6dfcfb47 100644 --- a/dstack/verifier/src/verification.rs +++ b/dstack/verifier/src/verification.rs @@ -32,8 +32,8 @@ use tokio::{io::AsyncWriteExt, process::Command}; use tracing::{debug, info, warn}; use crate::types::{ - AcpiTables, PolicyBootInfo, RtmrEventEntry, RtmrEventStatus, RtmrMismatch, VerificationDetails, - VerificationRequest, VerificationResponse, + AcpiTables, OsImageHashAnchor, PolicyBootInfo, RtmrEventEntry, RtmrEventStatus, RtmrMismatch, + VerificationDetails, VerificationRequest, VerificationResponse, }; /// Return the canonical TCB status and advisory list used by auth policy. @@ -67,6 +67,31 @@ pub fn policy_tcb_fields(attestation: &VerifiedAttestation) -> (String, Vec OsImageHashAnchor { + match quote { + AttestationQuote::DstackTdx(_) => match tdx_attestation_variant { + TdxAttestationVariant::Legacy => OsImageHashAnchor::PublishedImage, + TdxAttestationVariant::Lite => OsImageHashAnchor::MeasurementDocument, + }, + AttestationQuote::DstackGcpTdx(_) + | AttestationQuote::DstackAwsNitroTpm(_) + | AttestationQuote::DstackAmdSevSnp(_) => OsImageHashAnchor::MeasurementDocument, + AttestationQuote::DstackNitroEnclave(_) => OsImageHashAnchor::QuotedPcrs, + } +} + fn policy_boot_info_from_verified_app_info( attestation: &VerifiedAttestation, app_info: &AppInfo, @@ -782,6 +807,10 @@ impl CvmVerifier { } }; details.os_image_hash_verified = true; + details.os_image_hash_anchor = Some(os_image_hash_anchor( + &verified_attestation.quote, + vm_config.tdx_attestation_variant, + )); match verified_attestation.decode_app_info_ex(false, &request_vm_config) { Ok(mut info) => { info.os_image_hash = vm_config.os_image_hash; @@ -2250,6 +2279,140 @@ mod tests { ); } + /// TDX lite lets the requester choose `os_image_hash`, and reports + /// `os_image_hash_verified: true` for whatever it chose. + /// + /// That is the design, not a bug: on the self-contained paths the hash is + /// the digest of a `sha256sum.txt` the requester supplied, so it is a + /// truthful label for the material that produced the attested boot and + /// nothing more. What the requester cannot do is make the recomputed + /// measurements match a boot that did not happen -- rewrite the manifest + /// and MRTD/RTMR0-2 still have to equal the hardware-signed ones. + /// + /// The problem was that the response did not say which of the two + /// anchorings ran, and they differ by an entire trust anchor: on TDX + /// legacy the same `true` means the verifier fetched the image published + /// under that hash and measured its bytes. Take a real lite attestation, + /// rename its image by appending a line to the manifest it carries, and + /// require the result to carry `os_image_hash_anchor` so a relying party + /// can tell that the hash is only an identity if it allowlists it. + #[tokio::test] + async fn tdx_lite_os_image_hash_is_anchored_to_the_requesters_own_document() { + let request: VerificationRequest = + serde_json::from_str(include_str!("../fixtures/tdx-lite-attestation.json")) + .expect("TDX lite verifier fixture parses"); + let mut attestation = VersionedAttestation::from_bytes( + &request.attestation.expect("fixture carries an attestation"), + ) + .expect("fixture decodes") + .into_v1(); + + let ra_tls::attestation::StackEvidence::Dstack { config, .. } = &mut attestation.stack + else { + panic!("fixture is expected to carry a plain dstack stack evidence"); + }; + let mut vm_config: VmConfig = + serde_json::from_str(config).expect("fixture vm_config parses"); + let original_os_image_hash = vm_config.os_image_hash.clone(); + let document = vm_config + .tdx_measurement + .as_mut() + .expect("lite fixture carries a measurement document"); + // A line the manifest grammar accepts, naming a file nothing reads. + // The measurement CBOR keeps its own entry, so the document still + // verifies against the new hash -- only the name changes. + document + .checksum_file + .extend_from_slice(format!("{} decoy\n", "00".repeat(32)).as_bytes()); + let minted_os_image_hash = + dstack_types::image_hash_from_sha256sum(&document.checksum_file).to_vec(); + assert_ne!(minted_os_image_hash, original_os_image_hash); + vm_config.os_image_hash = minted_os_image_hash.clone(); + *config = serde_json::to_string(&vm_config).expect("vm_config re-serializes"); + + let cache = tempfile::tempdir().expect("temp cache dir"); + let verifier = CvmVerifier::new( + cache.path().join("cache").display().to_string(), + "http://127.0.0.1:9/should-not-download/{OS_IMAGE_HASH}.tar.gz".to_string(), + Duration::from_secs(1), + test_attestation_verifier(), + ); + let response = verifier + .verify(VerificationRequest { + quote: None, + event_log: None, + vm_config: None, + attestation: Some( + VersionedAttestation::V1 { + attestation: attestation.clone(), + } + .to_bytes() + .expect("attestation re-encodes"), + ), + debug: None, + }) + .await + .expect("verifier runs"); + + assert!(response.is_valid, "{:?}", response.reason); + assert!(response.details.os_image_hash_verified); + assert_eq!( + response + .details + .app_info + .as_ref() + .expect("a valid result carries app info") + .os_image_hash, + minted_os_image_hash, + "the reported hash is the one the requester put in its own manifest" + ); + assert_eq!( + response.details.os_image_hash_anchor, + Some(OsImageHashAnchor::MeasurementDocument), + "the result must say the hash is anchored to the requester's document" + ); + } + + /// The anchor reported for each dispatch `verify_os_image_hash` can take. + /// Legacy TDX is the one that cannot be reached from a fixture -- it + /// downloads an image -- so cover the mapping directly. + #[test] + fn every_os_image_hash_path_reports_its_own_anchor() { + assert_eq!( + os_image_hash_anchor(&empty_tdx_quote(), TdxAttestationVariant::Legacy), + OsImageHashAnchor::PublishedImage + ); + assert_eq!( + os_image_hash_anchor(&empty_tdx_quote(), TdxAttestationVariant::Lite), + OsImageHashAnchor::MeasurementDocument + ); + for quote in [ + AttestationQuote::DstackAmdSevSnp(ra_tls::attestation::SnpQuote { + report: Vec::new(), + cert_chain: Vec::new(), + mr_config: String::new(), + }), + AttestationQuote::DstackAwsNitroTpm(ra_tls::attestation::DstackAwsNitroTpmQuote { + attestation_doc: Vec::new(), + }), + ] { + assert_eq!( + os_image_hash_anchor("e, TdxAttestationVariant::Legacy), + OsImageHashAnchor::MeasurementDocument, + "the TDX scheme selector must not reach a non-TDX path" + ); + } + assert_eq!( + os_image_hash_anchor( + &AttestationQuote::DstackNitroEnclave(ra_tls::attestation::DstackNitroQuote { + nsm_quote: Vec::new(), + }), + TdxAttestationVariant::Legacy + ), + OsImageHashAnchor::QuotedPcrs + ); + } + /// `os_image_is_dev` and `os_image_version` come out of the image's /// `metadata.json`, and no self-contained path ever sees that file. /// From ee4288541494e72deca2a299b03bb6c9fd9b79f3 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 20 Sep 2026 00:04:14 -0700 Subject: [PATCH 4/6] fix(verifier): name the malformed vm_config instead of the measurement it broke --- dstack/verifier/README.md | 5 +- dstack/verifier/src/verification.rs | 127 ++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+), 1 deletion(-) diff --git a/dstack/verifier/README.md b/dstack/verifier/README.md index d64722d89..8385abb2c 100644 --- a/dstack/verifier/README.md +++ b/dstack/verifier/README.md @@ -9,7 +9,10 @@ A HTTP server that provides dstack quote verification services using the same ve Verifies a dstack attestation or quote with the provided data and VM configuration. The body can be grabbed via [getQuote](https://github.com/Dstack-TEE/dstack/blob/next/sdk/curl/api.md#3-get-quote) (Intel TDX only, and not the full evidence on GCP) or [attest](https://github.com/Dstack-TEE/dstack/blob/next/sdk/curl/api.md#7-attest) (any platform). **Request Body:** -Provide either `attestation` or (`quote` + `event_log` + `vm_config`). +Provide either `attestation` or (`quote` + `event_log` + `vm_config`). Fields the +verifier does not read are ignored, so a guest-agent `GetQuote` or `Attest` +response can be posted verbatim — `GetQuote` also answers with `report_data`, +which the verifier re-derives from the quote itself. ```json { diff --git a/dstack/verifier/src/verification.rs b/dstack/verifier/src/verification.rs index b6dfcfb47..4ac6f670b 100644 --- a/dstack/verifier/src/verification.rs +++ b/dstack/verifier/src/verification.rs @@ -856,6 +856,7 @@ impl CvmVerifier { let mut vm_config = attestation .decode_vm_config(&vm_config) .context("Failed to decode VM config")?; + Self::require_declared_os_image_hash(&vm_config)?; match &attestation.quote { AttestationQuote::DstackGcpTdx(quote) => { self.verify_os_image_hash_for_gcp_tdx(&vm_config, "e.tpm_quote)?; @@ -919,6 +920,35 @@ impl CvmVerifier { Ok(vm_config) } + /// Reject a `vm_config` that names no OS image. + /// + /// Every field of `VmConfig` carries a serde default, so any JSON object + /// deserializes into a config: `{}`, or a request that misspelled + /// `vm_config` and therefore supplied nothing at all. What comes out is an + /// all-defaults config whose `os_image_hash` is empty, and every platform + /// path below then compares a measurement or a digest against that empty + /// value and fails. Failing closed is the important part and it already + /// held -- but it failed as `os_image_hash mismatch: expected=, + /// computed=<...>`, which reads as a quote that does not match its image + /// rather than as a request that never named one. + /// + /// `os_image_hash` is a sha256 on every path: the image digest on TDX + /// legacy, `sha256(sha256sum.txt)` on the document-anchored paths, and + /// `sha256(PCR0||PCR1||PCR2)` on Nitro Enclave. Anything else is a + /// malformed request, not a failed comparison. + fn require_declared_os_image_hash(vm_config: &VmConfig) -> Result<()> { + if vm_config.os_image_hash.is_empty() { + bail!("vm_config declares no os_image_hash"); + } + if vm_config.os_image_hash.len() != 32 { + bail!( + "vm_config.os_image_hash is {} bytes, expected a 32-byte sha256", + vm_config.os_image_hash.len() + ); + } + Ok(()) + } + /// Verify the AMD SEV-SNP OS image binding. /// /// Unlike TDX (which replays RTMRs against a downloaded image), the SNP boot @@ -2279,6 +2309,103 @@ mod tests { ); } + /// A `vm_config` that names no image must say so. + /// + /// `VmConfig` defaults every field, so `{}` -- and equally a request whose + /// `vm_config` key was misspelled, which is the case that raised this -- + /// deserializes into an all-defaults config. Verification still fails + /// closed, because the path ends in a measurement comparison either way. + /// What it did not do was name the cause: the caller saw an os-image-hash + /// mismatch and went looking at the quote. + #[tokio::test] + async fn a_vm_config_that_names_no_image_is_reported_as_such() { + let request: VerificationRequest = + serde_json::from_str(include_str!("../fixtures/tdx-lite-attestation.json")) + .expect("TDX lite verifier fixture parses"); + let mut attestation = VersionedAttestation::from_bytes( + &request.attestation.expect("fixture carries an attestation"), + ) + .expect("fixture decodes") + .into_v1(); + let ra_tls::attestation::StackEvidence::Dstack { config, .. } = &mut attestation.stack + else { + panic!("fixture is expected to carry a plain dstack stack evidence"); + }; + *config = "{}".to_string(); + + let cache = tempfile::tempdir().expect("temp cache dir"); + let verifier = CvmVerifier::new( + cache.path().join("cache").display().to_string(), + "http://127.0.0.1:9/should-not-download/{OS_IMAGE_HASH}.tar.gz".to_string(), + Duration::from_secs(1), + test_attestation_verifier(), + ); + let response = verifier + .verify(VerificationRequest { + quote: None, + event_log: None, + vm_config: None, + attestation: Some( + VersionedAttestation::V1 { attestation } + .to_bytes() + .expect("attestation re-encodes"), + ), + debug: None, + }) + .await + .expect("verifier runs"); + + assert!(!response.is_valid); + let reason = response.reason.expect("an invalid result carries a reason"); + assert!( + reason.contains("vm_config declares no os_image_hash"), + "the error must name the malformed config, not the measurement: {reason}" + ); + } + + /// Both request-shaped types stay permissive about unknown fields, and + /// both have to. + /// + /// The tempting hardening is `deny_unknown_fields`, so a typo in a + /// caller's request is rejected rather than silently ignored. It is wrong + /// on both of these, for different reasons, and the reasons are worth a + /// test because neither is visible from the type: + /// + /// - `VerificationRequest` is the body `verifier/README.md` tells callers + /// to obtain from the guest agent and post verbatim + /// (`curl .../GetQuote -o quote.json` then `curl -d @quote.json`). + /// `GetQuote` answers with `report_data` alongside the three fields the + /// verifier reads, so denying unknown fields rejects the documented + /// workflow outright. + /// - `VmConfig` is a forward-compatible wire type that travels + /// VMM -> guest -> KMS and verifier. Several of its fields are + /// documented as "absent on images built before this landed", which is + /// the same compatibility running the other way: a newer VMM's field + /// must not make an older verifier reject an honest deployment. + #[test] + fn request_types_accept_fields_they_do_not_read() { + // Verbatim `GetQuote` response shape, per sdk/curl/api.md. + let get_quote = serde_json::json!({ + "quote": "00", + "event_log": "[]", + "report_data": "1234deadbeaf", + "vm_config": "{}", + }); + let request: VerificationRequest = + serde_json::from_value(get_quote).expect("the documented GetQuote body must post"); + assert_eq!(request.quote, Some(vec![0u8])); + assert_eq!(request.vm_config.as_deref(), Some("{}")); + + let newer_vmm = serde_json::json!({ + "os_image_hash": "11".repeat(32), + "cpu_count": 2, + "a_field_a_future_vmm_adds": true, + }); + let vm_config: VmConfig = + serde_json::from_value(newer_vmm).expect("a newer VMM's config must still decode"); + assert_eq!(vm_config.cpu_count, 2); + } + /// TDX lite lets the requester choose `os_image_hash`, and reports /// `os_image_hash_verified: true` for whatever it chose. /// From 31717e9805d81485ef698305d2941d0cbac96e95 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 20 Sep 2026 00:06:12 -0700 Subject: [PATCH 5/6] fix(verifier): bound the measurement cache directory --- dstack/verifier/src/verification.rs | 133 ++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) diff --git a/dstack/verifier/src/verification.rs b/dstack/verifier/src/verification.rs index 4ac6f670b..88c8bb67b 100644 --- a/dstack/verifier/src/verification.rs +++ b/dstack/verifier/src/verification.rs @@ -215,6 +215,22 @@ fn collect_rtmr_mismatch( // cached. const MEASUREMENT_CACHE_VERSION: u32 = 3; +/// Entries kept under `/measurements/`. +/// +/// One entry is a `TdxMeasurements` -- four hex digests, about 400 bytes -- +/// keyed by the VM shape it was computed from, so 1024 entries is roughly +/// 400 KB and far above what any real deployment reaches: a verifier serves a +/// handful of images across a handful of shapes. +/// +/// A cap rather than a TTL, because the entries never go stale. A VM shape +/// deterministically produces one set of measurements forever, so the only +/// reason to drop one is space. What made the directory grow was a caller +/// varying the shape: every novel shape that names a downloadable image leaves +/// a file behind, including one whose measurements then fail to match the +/// quote. Each costs the caller a full image measurement, so this is slow +/// growth rather than a flood -- but nothing stopped it. +const MEASUREMENT_CACHE_MAX_ENTRIES: usize = 1024; + #[derive(Clone, Serialize, Deserialize)] struct CachedMeasurement { version: u32, @@ -334,6 +350,45 @@ impl CvmVerifier { ) })?; debug!("Stored measurement cache entry {}", cache_key); + if let Err(e) = Self::prune_measurement_cache(&cache_dir, MEASUREMENT_CACHE_MAX_ENTRIES) { + warn!("failed to prune the measurement cache: {e:?}"); + } + Ok(()) + } + + /// Drop the oldest entries until at most `max_entries` remain. + /// + /// Only `.json` entries are considered, which is also what keeps this away + /// from a concurrent writer's `NamedTempFile`: those have no extension + /// until `persist` renames them into place. + /// + /// Best effort throughout. A cache that cannot be pruned is not a reason to + /// fail a verification, and a racing verifier may have already removed the + /// file this one picked. + fn prune_measurement_cache(cache_dir: &Path, max_entries: usize) -> Result<()> { + let mut entries = Vec::new(); + for entry in fs_err::read_dir(cache_dir).context("failed to read measurement cache")? { + let entry = entry.context("failed to read measurement cache entry")?; + let path = entry.path(); + if path.extension() != Some(OsStr::new("json")) { + continue; + } + let modified = entry + .metadata() + .and_then(|metadata| metadata.modified()) + .unwrap_or(std::time::UNIX_EPOCH); + entries.push((modified, path)); + } + let Some(excess) = entries.len().checked_sub(max_entries).filter(|n| *n > 0) else { + return Ok(()); + }; + entries.sort(); + for (_, path) in entries.iter().take(excess) { + if let Err(e) = fs_err::remove_file(path) { + debug!("failed to evict measurement cache entry: {e:?}"); + } + } + debug!("evicted {excess} measurement cache entries"); Ok(()) } @@ -2044,6 +2099,84 @@ mod tests { .is_none()); } + /// The measurement cache is bounded. + /// + /// Nothing evicted from `/measurements/`, so a caller that + /// varies the VM shape left one file per shape forever. Each entry is + /// small and each costs a full image measurement to create, so this was + /// slow growth rather than a flood -- but it had no ceiling, and the + /// eviction has to keep the newest entries (the ones a live deployment is + /// actually hitting) and leave a concurrent writer's temporary file alone. + #[test] + fn measurement_cache_evicts_the_oldest_entries_and_spares_temporaries() { + let dir = tempfile::tempdir().expect("temp cache dir"); + let cache_dir = dir.path(); + let base = std::time::UNIX_EPOCH + Duration::from_secs(1_700_000_000); + for index in 0..8u64 { + let path = cache_dir.join(format!("{index:02}.json")); + fs_err::write(&path, b"{}").unwrap(); + let file = fs_err::File::open(&path).unwrap(); + file.set_times( + std::fs::FileTimes::new().set_modified(base + Duration::from_secs(index)), + ) + .unwrap(); + } + // What `NamedTempFile::new_in` leaves in the directory until `persist` + // renames it: no extension, and newer than everything being evicted. + let in_flight = cache_dir.join(".tmpABCDEF"); + fs_err::write(&in_flight, b"partial").unwrap(); + + CvmVerifier::prune_measurement_cache(cache_dir, 3).unwrap(); + + let mut remaining: Vec = fs_err::read_dir(cache_dir) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + remaining.sort(); + assert_eq!( + remaining, + vec![ + ".tmpABCDEF".to_string(), + "05.json".to_string(), + "06.json".to_string(), + "07.json".to_string(), + ] + ); + + // Under the cap, nothing is touched. + CvmVerifier::prune_measurement_cache(cache_dir, 3).unwrap(); + assert_eq!(fs_err::read_dir(cache_dir).unwrap().count(), 4); + } + + /// The bound is applied on the write path, not just by the helper. + #[test] + fn storing_measurements_keeps_the_cache_within_its_bound() { + let dir = tempfile::tempdir().expect("temp cache dir"); + let verifier = CvmVerifier::new( + dir.path().display().to_string(), + String::new(), + Duration::from_secs(1), + test_attestation_verifier(), + ); + let measurements = TdxMeasurements { + mrtd: vec![0x11; 48], + rtmr0: vec![0x22; 48], + rtmr1: vec![0x33; 48], + rtmr2: vec![0x44; 48], + }; + for index in 0..(MEASUREMENT_CACHE_MAX_ENTRIES + 4) { + verifier + .store_measurements_in_cache(&format!("{index:064x}"), &measurements) + .unwrap(); + } + assert_eq!( + fs_err::read_dir(verifier.measurement_cache_dir()) + .unwrap() + .count(), + MEASUREMENT_CACHE_MAX_ENTRIES + ); + } + #[test] fn concurrent_measurement_cache_writes_are_atomic() { let directory = tempfile::tempdir().unwrap(); From 456a5d3ab01da4c2ed9f8e9445fbd2edf43bf16b Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 20 Sep 2026 00:08:11 -0700 Subject: [PATCH 6/6] docs(security): say what a report_data domain tag does and does not prove --- docs/security/security-model.md | 14 ++++++++ dstack/dstack-attest/src/attestation.rs | 47 +++++++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/docs/security/security-model.md b/docs/security/security-model.md index ca5647a71..05b447d30 100644 --- a/docs/security/security-model.md +++ b/docs/security/security-model.md @@ -256,6 +256,7 @@ Use this checklist to verify a workload running in a dstack CVM. - [ ] Launch event log replays correctly (RTMR3 on TDX-family platforms, PCR14 on AWS NitroTPM) - [ ] Config commitment matches the expected app/config target (on AWS: PCR14 replay; PCR8 is an optional shortcut — see the [AWS verifier runbook](../aws-ec2-production-verifier-runbook.md)) - [ ] reportData contains your challenge (replay protection) +- [ ] A `report_data` binding to a public key is treated as evidence only together with a live handshake or signature over that key — see [`report_data` domain tags are a parsing convention, not a capability](#report_data-domain-tags-are-a-parsing-convention-not-a-capability) - [ ] No security-relevant check depends on `pre_launch_script` running before the application; such checks belong in `init_script` or in the application itself **GPU verification (when required):** @@ -377,6 +378,19 @@ The one case dstack does not leave to downstream is a genuinely invalid TCB: `dc > **Future work:** this will be refactored toward a grace-period model, where an out-of-date TCB is accepted for a bounded window after a new TCB level is published rather than being a binary downstream decision. +### `report_data` domain tags are a parsing convention, not a capability + +Every quote surface a container can reach lets it choose `report_data` outright. `DstackGuest.GetQuote`, `DstackGuest.Attest` and v1 `Attest` take up to 64 bytes and use them verbatim; the legacy `Tappd.TdxQuote` additionally takes a `prefix`, which maps to `QuoteContentType::Custom(prefix)` and only saves the caller from computing the hash itself. So an application can obtain a hardware-signed quote whose `report_data` is exactly `sha512("ratls-cert:" || )`, or `sha512("kms-root-ca:" || ...)`, for a key it does not hold. + +The domain tags exist so that an external verifier can parse `report_data` unambiguously — so that an app-data quote is not mistaken for an RA-TLS binding. They are not an authorization boundary, and nothing is built on the assumption that they are: + +1. **The identity in the quote is not the app's to choose.** `app_id`, `compose_hash`, `instance_id`, `mr_system` and `mr_aggregated` are replayed out of the RTMR3 runtime event log against the hardware-signed register, and RTMR3 events are system-owned: `EmitEvent` was removed in 0.6.0 precisely so that no application can extend it. A minted quote therefore always names the CVM that minted it. It cannot impersonate another app to a remote verifier. +2. **Key possession is proven by the channel, never by the tag.** Every consumer that reads `ratls-cert:` as a possession proof also makes the peer use the key: `ra-rpc` client and server verify the binding against the public key of a *completed* TLS handshake, and the KMS verifies the CSR signature before `verify_with_ra_pubkey`. A quote minted over a key the app does not hold is unusable at all of them. +3. **For a key it does hold, it proves nothing new.** `GetTlsKey` and `IssueCert` already hand the app an RA-TLS certificate over its own key with the same identity. Minting the same binding by hand is the same statement by another route. +4. **`kms-root-ca:` is not a privilege either.** A KMS root CA is trusted because of the on-chain identity of the KMS app, and a minted quote carries the minting app's identity. Minting one with the KMS's identity requires running inside the KMS CVM, where the KMS's keys are already reachable — a CVM is a single trust domain. + +What this does mean is that a relying party must not treat "a quote exists whose `report_data` is `sha512("ratls-cert:" || K)`" as evidence that anyone holds *K*. It is evidence that the named CVM asked for that binding. Bind it to a live handshake or a signature, the way every consumer in this repository does. + ### Development modes are auditable, not production-safe dstack keeps several development switches as runtime or on-chain configuration rather than Cargo feature flags. Examples include KMS `attest_rpc_cert = false`, KMS `auth_api.type = "dev"`, and KMS contract `gateway_app_id = "any"`. These settings exist for local development and integration tests, not for production deployments. diff --git a/dstack/dstack-attest/src/attestation.rs b/dstack/dstack-attest/src/attestation.rs index 2373b401c..eaf661859 100644 --- a/dstack/dstack-attest/src/attestation.rs +++ b/dstack/dstack-attest/src/attestation.rs @@ -2615,6 +2615,53 @@ pub struct AppInfo { mod tests { use super::*; + /// The `report_data` domain tag is a parsing convention, not a capability. + /// + /// It looked like one: the legacy `Tappd.TdxQuote` surface maps a non-empty + /// `prefix` straight to `QuoteContentType::Custom`, so any container can ask + /// for a hardware-signed quote whose `report_data` is exactly the RA-TLS + /// key-possession binding for a key it does not hold. Pin that this is the + /// same primitive the modern surfaces expose rather than a gap in one of + /// them -- `DstackGuest.GetQuote` and `Attest` take the 64 bytes verbatim + /// (`pad64`), so a caller that wants a particular binding can always just + /// hash it itself. + /// + /// What actually keeps this from being a forgery is documented in + /// `docs/security/security-model.md` ("`report_data` domain tags are a + /// parsing convention, not a capability"): the quote's identity comes from + /// the system-owned RTMR3 event log, and every consumer that reads + /// `ratls-cert:` as possession also makes the peer use the key -- a + /// completed TLS handshake, or a CSR signature. + #[test] + fn report_data_domain_tags_are_reproducible_by_any_caller() { + let spki = b"a subject public key info"; + for named in [QuoteContentType::RaTlsCert, QuoteContentType::KmsRootCa] { + assert_eq!( + QuoteContentType::Custom(named.tag()).to_report_data(spki), + named.to_report_data(spki), + "{}: a custom tag reproduces the named one byte for byte", + named.tag() + ); + } + + // And `raw` skips the tag entirely, which is also what `GetQuote` and + // `Attest` do with the bytes they are handed. + let binding = QuoteContentType::RaTlsCert.to_report_data(spki); + assert_eq!( + QuoteContentType::AppData + .to_report_data_with_hash(&binding, "raw") + .unwrap(), + binding + ); + + // The binding is to one specific key, so a quote minted for one SPKI + // says nothing about any other. + assert_ne!( + QuoteContentType::RaTlsCert.to_report_data(b"another public key"), + binding + ); + } + /// Build a TD10 verified report carrying `status`. Everything else is the /// shape `validate_tcb` accepts, so a row that fails failed on the status. fn td10_verified_report(status: &str) -> TdxVerifiedReport {