diff --git a/Cargo.lock b/Cargo.lock index a13e6d9b3a2..25166e9e27f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -360,6 +360,7 @@ checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" dependencies = [ "async-trait", "axum-core", + "axum-macros", "bytes", "futures-util", "http", @@ -431,6 +432,17 @@ dependencies = [ "tower-service", ] +[[package]] +name = "axum-macros" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d123550fa8d071b7255cb0cc04dc302baa6c8c4a79f55701552684d8399bce" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.107", +] + [[package]] name = "backtrace" version = "0.3.76" @@ -7665,6 +7677,17 @@ dependencies = [ "syn 2.0.107", ] +[[package]] +name = "sfv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "890ef0e5532daf21fdaa3dbaf991417187f3a87ab69231fe69be942495746480" +dependencies = [ + "base64 0.22.1", + "indexmap 2.12.0", + "ref-cast", +] + [[package]] name = "sha1" version = "0.10.6" @@ -8024,6 +8047,7 @@ dependencies = [ "futures", "git2", "glob", + "headers", "http", "ignore", "indicatif", @@ -8131,12 +8155,15 @@ dependencies = [ "bytestring", "derive_more 0.99.20", "enum-as-inner", + "headers", "hex", + "http", "itertools 0.12.1", "proptest", "serde", "serde_json", "serde_with", + "sfv", "smallvec", "spacetimedb-lib", "spacetimedb-primitives", diff --git a/Cargo.toml b/Cargo.toml index e126213341d..16cd5475240 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -194,7 +194,7 @@ arrayvec = "0.7.2" async-channel = "2.5" async-stream = "0.3.6" async-trait = "0.1.68" -axum = { version = "0.7", features = ["tracing", "http2"] } +axum = { version = "0.7", features = ["tracing", "http2", "macros"] } axum-extra = { version = "0.9", features = ["typed-header"] } base64 = "0.21.2" bigdecimal = "0.4.7" @@ -307,6 +307,7 @@ serde_json = { version = "1.0.128", features = ["raw_value"] } serde_path_to_error = "0.1.9" serde_with = { version = "3.3.0", features = ["base64", "hex"] } serial_test = "2.0.0" +sfv = "0.15" sha3 = "0.10.0" slab = "0.4.7" sled = "0.34.7" diff --git a/crates/bindings-macro/src/environment.rs b/crates/bindings-macro/src/environment.rs index 424b4c85485..d8e02e7ec1d 100644 --- a/crates/bindings-macro/src/environment.rs +++ b/crates/bindings-macro/src/environment.rs @@ -63,11 +63,13 @@ pub(crate) fn expand(args: TokenStream, mut item: ItemStruct) -> syn::Result quote!(<#ty as ::spacetimedb::rt::EnvironmentValue>::constraint()), Some([value]) => { - quote!(::spacetimedb::spacetimedb_lib::environment::EnvVarType::StringLiteral(#value.into())) + quote!(::spacetimedb::spacetimedb_lib::db::raw_def::v10::RawEnvVarTypeV10::StringLiteral(#value.into())) } - Some(values) => quote!(::spacetimedb::spacetimedb_lib::environment::EnvVarType::Union( - ::std::vec![#(#values.into()),*] - )), + Some(values) => quote!( + ::spacetimedb::spacetimedb_lib::db::raw_def::v10::RawEnvVarTypeV10::Union( + ::std::vec![#(#values.into()),*] + ) + ), }; let constraint = if values.is_some() { quote!(<#ty as ::spacetimedb::rt::StringEnvironmentValue>::with_constraint(#constraint)) @@ -75,7 +77,7 @@ pub(crate) fn expand(args: TokenStream, mut item: ItemStruct) -> syn::Result::OPTIONAL, @@ -96,7 +98,7 @@ pub(crate) fn expand(args: TokenStream, mut item: ItemStruct) -> syn::Result syn::Result { values.push(value); } let constraint = match values.as_slice() { - [value] => quote!(::spacetimedb::spacetimedb_lib::environment::EnvVarType::StringLiteral(#value.into())), - values => quote!(::spacetimedb::spacetimedb_lib::environment::EnvVarType::Union( - ::std::vec![#(#values.into()),*] - )), + [value] => { + quote!(::spacetimedb::spacetimedb_lib::db::raw_def::v10::RawEnvVarTypeV10::StringLiteral(#value.into())) + } + values => quote!( + ::spacetimedb::spacetimedb_lib::db::raw_def::v10::RawEnvVarTypeV10::Union(::std::vec![#(#values.into()),*]) + ), }; let ident = &item.ident; Ok(quote! { impl ::spacetimedb::rt::EnvironmentValue for #ident { const OPTIONAL: bool = false; - fn constraint() -> ::spacetimedb::spacetimedb_lib::environment::EnvVarType { + fn constraint() -> ::spacetimedb::spacetimedb_lib::db::raw_def::v10::RawEnvVarTypeV10 { #constraint } diff --git a/crates/bindings/Cargo.toml b/crates/bindings/Cargo.toml index 53d7c5cdd51..70d444998e3 100644 --- a/crates/bindings/Cargo.toml +++ b/crates/bindings/Cargo.toml @@ -42,6 +42,7 @@ serde_json.workspace = true [dev-dependencies] insta.workspace = true trybuild.workspace = true +spacetimedb-lib = { path = "../lib", features = ["test"] } [lints] workspace = true diff --git a/crates/bindings/src/rt.rs b/crates/bindings/src/rt.rs index 6caf1c4ee04..a6e8821429b 100644 --- a/crates/bindings/src/rt.rs +++ b/crates/bindings/src/rt.rs @@ -5,7 +5,8 @@ use crate::table::IndexAlgo; use crate::{sys, AnonymousViewContext, IterBuf, ReducerContext, ReducerResult, SpacetimeType, Table, ViewContext}; use spacetimedb_lib::bsatn::EncodeError; use spacetimedb_lib::db::raw_def::v10::{ - CaseConversionPolicy, ExplicitNames as RawExplicitNames, RawModuleDefV10Builder, + CaseConversionPolicy, ExplicitNames as RawExplicitNames, RawEnvVarTypeV10, RawEnvironmentDeclarationV10, + RawModuleDefV10Builder, }; pub use spacetimedb_lib::db::raw_def::v9::Lifecycle as LifecycleReducer; use spacetimedb_lib::db::raw_def::v9::{RawIndexAlgorithm, TableType, ViewResultHeader}; @@ -929,7 +930,7 @@ pub fn register_case_conversion_policy(policy: CaseConversionPolicy) { pub trait EnvironmentValue: Sized { const OPTIONAL: bool; - fn constraint() -> spacetimedb_lib::environment::EnvVarType; + fn constraint() -> RawEnvVarTypeV10; /// Decode a checked host result. Errors must identify only the key, never its value. fn from_environment(value: Option, key: &str) -> Self; @@ -947,8 +948,8 @@ pub trait RequiredEnvironmentValue: EnvironmentValue {} impl EnvironmentValue for String { const OPTIONAL: bool = false; - fn constraint() -> spacetimedb_lib::environment::EnvVarType { - spacetimedb_lib::environment::EnvVarType::String + fn constraint() -> RawEnvVarTypeV10 { + RawEnvVarTypeV10::String } fn from_environment(value: Option, key: &str) -> Self { @@ -961,7 +962,7 @@ impl RequiredEnvironmentValue for String {} impl EnvironmentValue for Option { const OPTIONAL: bool = true; - fn constraint() -> spacetimedb_lib::environment::EnvVarType { + fn constraint() -> RawEnvVarTypeV10 { T::constraint() } @@ -983,9 +984,7 @@ mod string_environment_value_sealed { message = "`#[env(values(...))]` requires `String` or `Option`; map enum variants with `#[env(value = \"...\")]` instead" )] pub trait StringEnvironmentValue: EnvironmentValue + string_environment_value_sealed::Sealed { - fn with_constraint( - constraint: spacetimedb_lib::environment::EnvVarType, - ) -> spacetimedb_lib::environment::EnvVarType { + fn with_constraint(constraint: RawEnvVarTypeV10) -> RawEnvVarTypeV10 { constraint } } @@ -995,7 +994,7 @@ impl StringEnvironmentValue for Option {} /// Register declarative ENV metadata without reading any environment values. #[doc(hidden)] -pub fn register_environment(declarations: fn() -> Vec) { +pub fn register_environment(declarations: fn() -> Vec) { register_describer(move |module| { module.inner.add_environment(declarations()); }); @@ -1067,7 +1066,6 @@ extern "C" fn __describe_module__(description: BytesSink) { } // Serialize the module to bsatn. - module.inner.ensure_environment(); let module_def = module.inner.finish(); let module_def = RawModuleDef::V10(module_def); let bytes = bsatn::to_vec(&module_def).expect("unable to serialize typespace"); diff --git a/crates/bindings/tests/environment_enum_values.rs b/crates/bindings/tests/environment_enum_values.rs index 85ca9515c75..291ed04f1a5 100644 --- a/crates/bindings/tests/environment_enum_values.rs +++ b/crates/bindings/tests/environment_enum_values.rs @@ -1,5 +1,6 @@ use spacetimedb::rt::EnvironmentValue as _; -use spacetimedb::spacetimedb_lib::environment::{EnvVarType, EnvironmentDeclaration, EnvironmentSchema}; +use spacetimedb::spacetimedb_lib::environment::EnvironmentSchema; +use spacetimedb_lib::db::raw_def::v10::{RawEnvVarTypeV10, RawEnvironmentDeclarationV10}; use std::collections::BTreeMap; #[derive(Debug, PartialEq, Eq, spacetimedb::SpacetimeType, spacetimedb::EnvironmentValue)] @@ -29,14 +30,15 @@ fn typed_mappings_match_exact_schema_strings_and_optional_absence() { ]; assert_eq!( Mode::constraint(), - EnvVarType::Union(cases.iter().map(|(s, _)| s.to_string()).collect()) + RawEnvVarTypeV10::Union(cases.iter().map(|(s, _)| s.to_string()).collect()) ); assert_eq!(Option::::constraint(), Mode::constraint()); - let schema = EnvironmentSchema::new(vec![EnvironmentDeclaration { + let schema = EnvironmentSchema::new(vec![RawEnvironmentDeclarationV10 { name: "MODE".into(), ty: Mode::constraint(), optional: false, - }]) + } + .into()]) .unwrap(); for (value, variant) in cases { schema @@ -49,7 +51,7 @@ fn typed_mappings_match_exact_schema_strings_and_optional_absence() { ); } assert_eq!(Option::::from_environment(None, "MODE"), None); - assert_eq!(Literal::constraint(), EnvVarType::StringLiteral("only".into())); + assert_eq!(Literal::constraint(), RawEnvVarTypeV10::StringLiteral("only".into())); assert_eq!(Literal::from_environment(Some("only".into()), "VALUE"), Literal::Only); for rejected in ["InProgress", "ready", "in progress ", "private-unmapped-value"] { assert!(schema diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 5ea9f718833..4ca6467996d 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -42,6 +42,7 @@ dirs.workspace = true duct.workspace = true futures.workspace = true fs-err.workspace = true +headers.workspace = true http.workspace = true is-terminal.workspace = true itertools.workspace = true diff --git a/crates/cli/src/schema_extract.rs b/crates/cli/src/schema_extract.rs index 76b2ec64159..6e6d91bcc42 100644 --- a/crates/cli/src/schema_extract.rs +++ b/crates/cli/src/schema_extract.rs @@ -43,16 +43,7 @@ fn inspect_blocking(extractor: PathBuf, bytes: Vec, host_type: String) -> an } pub(crate) fn read_program(path: &std::path::Path) -> anyhow::Result> { - use std::io::Read; - let mut bytes = Vec::new(); - std::fs::File::open(path)? - .take(spacetimedb_client_api_messages::publish::MAX_MODULE_BYTES as u64 + 1) - .read_to_end(&mut bytes)?; - ensure!( - bytes.len() <= spacetimedb_client_api_messages::publish::MAX_MODULE_BYTES, - "Module exceeds publish size limit" - ); - Ok(bytes) + Ok(std::fs::read(path)?) } const MAX_SCHEMA_BYTES: u64 = 16 * 1024 * 1024; diff --git a/crates/cli/src/schema_extract/tests.rs b/crates/cli/src/schema_extract/tests.rs index 6b77ad7cb8b..ab86403e0dd 100644 --- a/crates/cli/src/schema_extract/tests.rs +++ b/crates/cli/src/schema_extract/tests.rs @@ -89,7 +89,9 @@ fn inspector(script: &str) -> (tempfile::TempDir, PathBuf) { async fn local_inspection_passes_exact_bytes_host_and_requires_success() { use spacetimedb_lib::db::raw_def::v10::{RawModuleDefV10, RawModuleDefV10Section}; let raw = RawModuleDef::V10(RawModuleDefV10 { - sections: vec![RawModuleDefV10Section::Environment(schema().into_declarations())], + sections: vec![RawModuleDefV10Section::Environment( + schema().into_declarations().into_iter().map(Into::into).collect(), + )], }); let json = serde_json::to_string(&SerdeWrapper(raw)).unwrap(); let (dir, extractor) = inspector(&format!( @@ -118,7 +120,9 @@ async fn local_inspection_passes_exact_bytes_host_and_requires_success() { async fn synchronous_generate_adapter_uses_same_exact_byte_protocol_inside_a_runtime() { use spacetimedb_lib::db::raw_def::v10::{RawModuleDefV10, RawModuleDefV10Section}; let raw = RawModuleDef::V10(RawModuleDefV10 { - sections: vec![RawModuleDefV10Section::Environment(schema().into_declarations())], + sections: vec![RawModuleDefV10Section::Environment( + schema().into_declarations().into_iter().map(Into::into).collect(), + )], }); let json = serde_json::to_string(&SerdeWrapper(raw)).unwrap(); let (_dir, extractor) = inspector(&format!( diff --git a/crates/cli/src/subcommands/env.rs b/crates/cli/src/subcommands/env.rs index c0092bad377..593b68b919a 100644 --- a/crates/cli/src/subcommands/env.rs +++ b/crates/cli/src/subcommands/env.rs @@ -97,11 +97,8 @@ async fn fetch(request: reqwest::RequestBuilder, query: Query) -> anyhow::Result .body(query.sql()?) .send() .await?; - ensure!( - response.status().is_success(), - "Environment read failed with HTTP {}", - response.status() - ); + let response = response.error_for_status().context("failed to fetch environment")?; + // TODO(noa): what are we doing here. what. why are we manually buffering. help me let mut body = Vec::new(); let limit = MAX_ENV_VARS * (MAX_ENV_KEY_BYTES + MAX_ENV_VALUE_BYTES) * 6 + 64 * 1024; let mut stream = response.bytes_stream(); diff --git a/crates/cli/src/subcommands/publish.rs b/crates/cli/src/subcommands/publish.rs index df51f6f62de..3902c8a5fbc 100644 --- a/crates/cli/src/subcommands/publish.rs +++ b/crates/cli/src/subcommands/publish.rs @@ -6,9 +6,12 @@ use anyhow::{ensure, Context}; use clap::Arg; use clap::ArgAction::{self, Set, SetTrue}; use clap::{value_parser, ArgMatches, ValueEnum}; +use headers::HeaderMapExt; use reqwest::{StatusCode, Url}; -use spacetimedb_client_api_messages::name::{is_identity, parse_database_name, PublishResult}; +use spacetimedb_client_api_messages::name::{is_identity, parse_database_name, EnvironmentPublishError, PublishResult}; use spacetimedb_client_api_messages::name::{DatabaseNameError, PrePublishResult, PrettyPrintStyle, PublishOp}; +use spacetimedb_client_api_messages::publish::{SpacetimeEnvironment, SpacetimeEnvironmentRemove}; +use spacetimedb_lib::environment::EnvironmentRemove; use std::collections::HashMap; use std::env; use std::path::PathBuf; @@ -355,68 +358,45 @@ i.e. only lowercase ASCII letters and numbers, separated by dashes."), #[derive(Default)] struct EnvironmentOptions { only: bool, - remove: Vec, - replace: bool, + remove: EnvironmentRemove, } impl EnvironmentOptions { fn from_args(args: &ArgMatches) -> anyhow::Result { - let options = Self { - only: args.get_flag("env_only"), - remove: args - .get_many::("unset_env") - .map(|keys| keys.cloned().collect()) - .unwrap_or_default(), - replace: args.get_flag("replace_env"), + let remove = if args.get_flag("replace_env") { + EnvironmentRemove::All + } else if let Some(remove) = args.get_many::("unset_env") { + let remove = remove.cloned().collect::>(); + ensure!( + remove.len() <= spacetimedb_lib::environment::MAX_ENV_VARS, + "Too many environment removals" + ); + for key in &remove { + spacetimedb_lib::environment::validate_key(key)?; + } + EnvironmentRemove::Keys(remove) + } else { + EnvironmentRemove::No }; - ensure!( - options.remove.len() <= spacetimedb_lib::environment::MAX_ENV_VARS, - "Too many environment removals" - ); - for key in &options.remove { - spacetimedb_lib::environment::validate_key(key)?; - } - Ok(options) + Ok(Self { + only: args.get_flag("env_only"), + remove, + }) } fn validate_values(&self, values: &std::collections::BTreeMap) -> anyhow::Result<()> { - ensure!( - !self.replace || self.remove.is_empty(), - "--replace-env cannot be combined with --unset-env" - ); - for key in &self.remove { - ensure!( - !values.contains_key(key), - "Environment key {key:?} is both supplied and removed" - ); + if let EnvironmentRemove::Keys(remove) = &self.remove { + for key in remove { + ensure!( + !values.contains_key(key), + "Environment key {key:?} is both supplied and removed" + ); + } } Ok(()) } } -fn publication_body( - module: &spacetimedb_schema::def::ModuleDef, - bytes: Vec, - environment: std::collections::BTreeMap, - options: &EnvironmentOptions, -) -> anyhow::Result<(&'static str, Vec)> { - options.validate_values(&environment)?; - if module.environment_declared() || !environment.is_empty() || !options.remove.is_empty() || options.replace { - let body = spacetimedb_client_api_messages::publish::PublishRequest { - module: Some(bytes), - environment, - environment_remove: options.remove.clone(), - environment_replace: options.replace, - expected_module_version: None, - } - .encode()?; - Ok((spacetimedb_client_api_messages::publish::CONTENT_TYPE, body)) - } else { - // Preserve older servers for ordinary modules without environment changes. - Ok(("application/octet-stream", bytes)) - } -} - fn confirm_and_clear( name_or_identity: &str, skip_prompt: bool, @@ -692,7 +672,7 @@ async fn execute_publish_configs<'a>( command_config.get_config_value("env"), |key| std::env::var_os(key), )?; - print!("{}", environment.display()); + print!("{}", environment); let server_address = { let url = Url::parse(&database_host)?; @@ -765,18 +745,16 @@ async fn execute_publish_configs<'a>( // Set the host type. builder = builder.query(&[("host_type", host_type)]); - let (content_type, payload) = - publication_body(&module_schema, program_bytes, environment.values, environment_options)?; - let res = builder - .header(reqwest::header::CONTENT_TYPE, content_type) - .body(payload) - .send() - .await?; - anyhow::ensure!(res.status().is_success(), "Publish failed with HTTP {}", res.status()); - let response: PublishResult = res - .json() - .await - .map_err(|_| anyhow::anyhow!("Invalid publish response"))?; + let mut request = builder.body(program_bytes).build()?; + request + .headers_mut() + .typed_insert(SpacetimeEnvironment(environment.values)); + request + .headers_mut() + .typed_insert(SpacetimeEnvironmentRemove(environment_options.remove.clone())); + + let res = client.execute(request).await?; + let response: PublishResult = res.json_or_error().await?; match response { PublishResult::Success { domain, @@ -815,6 +793,10 @@ async fn execute_publish_configs<'a>( \tspacetime publish {suggested_tld}\n", )); } + PublishResult::EnvironmentError(EnvironmentPublishError::MissingRequiredEnvironment { keys }) => { + anyhow::bail!("Missing required environment variable(s) {keys:?}") + } + PublishResult::EnvironmentError(EnvironmentPublishError::VersionConflict(e)) => anyhow::bail!(e), } } diff --git a/crates/cli/src/subcommands/publish/environment.rs b/crates/cli/src/subcommands/publish/environment.rs index 626c909ddc7..e022eb24494 100644 --- a/crates/cli/src/subcommands/publish/environment.rs +++ b/crates/cli/src/subcommands/publish/environment.rs @@ -1,11 +1,16 @@ //! Resolve explicit overrides without fetching stored secrets. use std::collections::BTreeMap; use std::ffi::OsString; +use std::fmt; pub(super) use crate::schema_extract::{inspect, read_program}; +use crate::util::ResponseExt; use anyhow::Context; +use headers::HeaderMapExt; use serde_json::Value; -use spacetimedb_lib::environment::EnvironmentSchema; +use spacetimedb_client_api_messages::name::EnvironmentPublishError; +use spacetimedb_client_api_messages::publish::SpacetimeEnvironmentRemove; +use spacetimedb_lib::environment::{EnvironmentRemove, EnvironmentSchema}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum Source { @@ -26,14 +31,13 @@ pub(super) struct Resolved { pub values: BTreeMap, pub sources: BTreeMap, } -impl Resolved { - pub fn display(&self) -> String { - use std::fmt::Write; - let mut output = String::new(); + +impl fmt::Display for Resolved { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { for (name, source) in &self.sources { - let _ = writeln!(output, "Environment {name} ({source})"); + writeln!(f, "Environment {name} ({source})")?; } - output + Ok(()) } } @@ -88,7 +92,7 @@ pub(super) async fn publish_only( options: &super::EnvironmentOptions, ) -> anyhow::Result<()> { use crate::util::{add_auth_header_opt, get_auth_header, y_or_n}; - use spacetimedb_client_api_messages::publish::{EnvironmentMetadata, PublishRequest, CONTENT_TYPE}; + use spacetimedb_client_api_messages::publish::EnvironmentMetadata; let host = config.get_host_url(server)?; let server_url = reqwest::Url::parse(&host)?; @@ -106,52 +110,44 @@ pub(super) async fn publish_only( const { &percent_encoding::NON_ALPHANUMERIC.remove(b'_').remove(b'-') }, ) .to_string(); - let url = format!("{host}/v1/database/{encoded}"); + let url = format!("{host}/v1/database/{encoded}/environment"); // Neither credentials nor publish bodies may be forwarded to redirect destinations. let client = reqwest::Client::builder() .redirect(reqwest::redirect::Policy::none()) .build()?; - let response = add_auth_header_opt(client.get(format!("{url}/environment")), &auth) - .send() - .await?; - anyhow::ensure!( - response.status().is_success(), - "Cannot read environment schema: HTTP {}", - response.status() - ); - let metadata: EnvironmentMetadata = response.json().await.context("Invalid environment metadata")?; + let response = add_auth_header_opt(client.get(&url), &auth).send().await?; + let metadata: EnvironmentMetadata = response + .json_or_error() + .await + .context("failed to fetch environment schema")?; let schema = EnvironmentSchema::new(metadata.declarations)?; let resolved = resolve(&schema, input, |key| std::env::var_os(key))?; options.validate_values(&resolved.values)?; - print!("{}", resolved.display()); - let request = PublishRequest { - module: None, - environment: resolved.values, - environment_remove: options.remove.clone(), - environment_replace: options.replace, - expected_module_version: Some(metadata.module_version), + print!("{}", resolved); + let request = if let EnvironmentRemove::All = options.remove { + client.put(url) + } else { + client.patch(url) }; - let response = add_auth_header_opt(client.put(url), &auth) - .header(reqwest::header::CONTENT_TYPE, CONTENT_TYPE) - .body(request.encode()?) - .send() - .await?; - anyhow::ensure!( - response.status().is_success(), - "Environment publish failed with HTTP {}", - response.status() - ); - match response - .json::() - .await - .map_err(|_| anyhow::anyhow!("Invalid publish response"))? - { - spacetimedb_client_api_messages::name::PublishResult::Success { database_identity, .. } => { - println!("Updated environment for database {database_identity}"); + let mut request = add_auth_header_opt(request, &auth) + .query(&[("expected_module_hash", &metadata.module_hash)]) + .json(&resolved.values) + .build()?; + if let EnvironmentRemove::Keys(_) = options.remove { + request + .headers_mut() + .typed_insert(SpacetimeEnvironmentRemove(options.remove.clone())); + } + let res = client.execute(request).await?; + let response: Result<(), EnvironmentPublishError> = res.json_or_error().await?; + match response { + Ok(()) => { + println!("Successfully updated environment"); Ok(()) } - spacetimedb_client_api_messages::name::PublishResult::PermissionDenied { .. } => { - anyhow::bail!("Permission denied publishing environment values") + Err(EnvironmentPublishError::MissingRequiredEnvironment { keys }) => { + anyhow::bail!("Missing required environment variable(s) {keys:?}") } + Err(EnvironmentPublishError::VersionConflict(e)) => anyhow::bail!(e), } } diff --git a/crates/cli/src/subcommands/publish/environment/tests.rs b/crates/cli/src/subcommands/publish/environment/tests.rs index aec691356ee..2e9642e0096 100644 --- a/crates/cli/src/subcommands/publish/environment/tests.rs +++ b/crates/cli/src/subcommands/publish/environment/tests.rs @@ -54,10 +54,10 @@ fn declared_shell_overrides_are_redacted() { ); assert_eq!(checked, vec!["A", "B", "C", "OPTIONAL"]); assert_eq!( - resolved.display(), + resolved.to_string(), "Environment A (shell)\nEnvironment B (config)\nEnvironment C (shell)\n" ); - assert!(!resolved.display().contains("sentinel")); + assert!(!resolved.to_string().contains("sentinel")); // No declaration means no ambient lookup, including PATH or credentials. let empty = resolve(&EnvironmentSchema::default(), None, |_| panic!("ambient access")).unwrap(); assert!(empty.values.is_empty()); @@ -104,7 +104,7 @@ fn invalid_inputs_fail_without_values_or_lower_priority_fallback() { .unwrap(); assert_eq!(resolved.values["UNDECLARED"], "secret"); assert!(!looked_up.iter().any(|key| key == "UNDECLARED")); - assert!(!resolved.display().contains("secret")); + assert!(!resolved.to_string().contains("secret")); } #[test] @@ -168,7 +168,6 @@ async fn actual_precompiled_declarations_are_inspected_without_server_or_values( .await .unwrap(); let schema = inspected.environment(); - assert!(inspected.environment_declared()); assert!(!schema.get("REQUIRED").unwrap().optional); assert_eq!( schema.get("MODE").unwrap().ty, @@ -177,7 +176,7 @@ async fn actual_precompiled_declarations_are_inspected_without_server_or_values( let config = serde_json::json!({"REQUIRED":"generated-local-inspection-sentinel","MODE":"ready"}); let resolved = resolve(schema, Some(&config), |_| None).unwrap(); assert_eq!(resolved.values.len(), 2); - assert!(!resolved.display().contains("generated-local-inspection-sentinel")); + assert!(!resolved.to_string().contains("generated-local-inspection-sentinel")); assert!(resolve(schema, None, |_| None).unwrap().values.is_empty()); } diff --git a/crates/cli/src/subcommands/publish/wire_tests.rs b/crates/cli/src/subcommands/publish/wire_tests.rs index daf3a265913..219f4d97277 100644 --- a/crates/cli/src/subcommands/publish/wire_tests.rs +++ b/crates/cli/src/subcommands/publish/wire_tests.rs @@ -1,54 +1,6 @@ use super::*; -use spacetimedb_lib::{ - db::raw_def::v10::{RawModuleDefV10, RawModuleDefV10Section}, - RawModuleDef, -}; -use spacetimedb_schema::def::ModuleDef; use std::collections::BTreeMap; -fn schema(declared: bool) -> ModuleDef { - let mut sections = vec![RawModuleDefV10Section::Typespace(Default::default())]; - if declared { - sections.push(RawModuleDefV10Section::Environment(vec![])); - } - ModuleDef::try_from(RawModuleDef::V10(RawModuleDefV10 { sections })).unwrap() -} - -#[test] -fn ordinary_and_explicit_empty_declarations_choose_distinct_wire_formats() { - let bytes = b"exact selected module bytes\0\xff".to_vec(); - let (kind, body) = publication_body( - &schema(false), - bytes.clone(), - BTreeMap::new(), - &EnvironmentOptions::default(), - ) - .unwrap(); - assert_eq!(kind, "application/octet-stream"); - assert_eq!(body, bytes); - let (kind, body) = publication_body( - &schema(true), - bytes.clone(), - BTreeMap::new(), - &EnvironmentOptions::default(), - ) - .unwrap(); - assert_eq!(kind, spacetimedb_client_api_messages::publish::CONTENT_TYPE); - let envelope = spacetimedb_client_api_messages::publish::PublishRequest::decode(&body).unwrap(); - assert_eq!(envelope.module, Some(bytes.clone())); - assert!(envelope.environment.is_empty()); - let (kind, body) = publication_body( - &schema(false), - bytes, - BTreeMap::from([("KEY".into(), "secret-sentinel".into())]), - &EnvironmentOptions::default(), - ) - .unwrap(); - assert_eq!(kind, spacetimedb_client_api_messages::publish::CONTENT_TYPE); - let envelope = spacetimedb_client_api_messages::publish::PublishRequest::decode(&body).unwrap(); - assert_eq!(envelope.environment["KEY"], "secret-sentinel"); -} - #[test] fn short_help_is_concise_and_long_help_explains_environment_modes() { let short = cli().render_help().to_string(); @@ -84,7 +36,7 @@ fn environment_flags_are_explicit_and_incompatible_modes_are_rejected() { .unwrap(); let options = EnvironmentOptions::from_args(&args).unwrap(); assert!(options.only); - assert_eq!(options.remove, ["A", "B"]); + assert_eq!(options.remove, EnvironmentRemove::Keys(vec!["A".into(), "B".into()])); assert!(options .validate_values(&BTreeMap::from([("A".into(), "secret-sentinel".into())])) .is_err()); @@ -92,25 +44,5 @@ fn environment_flags_are_explicit_and_incompatible_modes_are_rejected() { .try_get_matches_from(["publish", "db", "--env-only", "--replace-env"]) .unwrap(); let options = EnvironmentOptions::from_args(&args).unwrap(); - assert!(options.only && options.replace); -} - -#[test] -fn removal_and_replacement_use_json_even_for_legacy_modules() { - for options in [ - EnvironmentOptions { - remove: vec!["OLD".into()], - ..Default::default() - }, - EnvironmentOptions { - replace: true, - ..Default::default() - }, - ] { - let (kind, body) = publication_body(&schema(false), vec![1], BTreeMap::new(), &options).unwrap(); - assert_eq!(kind, spacetimedb_client_api_messages::publish::CONTENT_TYPE); - let request = spacetimedb_client_api_messages::publish::PublishRequest::decode(&body).unwrap(); - assert_eq!(request.environment_remove, options.remove); - assert_eq!(request.environment_replace, options.replace); - } + assert!(options.only && options.remove == EnvironmentRemove::All); } diff --git a/crates/cli/src/util.rs b/crates/cli/src/util.rs index 2b199f07467..9d9957473a7 100644 --- a/crates/cli/src/util.rs +++ b/crates/cli/src/util.rs @@ -52,6 +52,10 @@ pub(crate) trait ResponseExt: Sized { /// Like [`reqwest::Response::json()`], but handles non-JSON error messages gracefully. async fn json_or_error(self) -> anyhow::Result; + /// Like [`reqwest::Response::error_for_status()`], but the returned error contains + /// the error message returned in the response body if present. + async fn error_msg_for_status(self) -> anyhow::Result; + /// Transforms a status of `NOT_FOUND` into `None`. fn found(self) -> Option; } @@ -70,7 +74,6 @@ fn err_status_desc(status: http::StatusCode) -> Option<&'static str> { impl ResponseExt for reqwest::Response { async fn ensure_content_type(self, content_type: &str) -> anyhow::Result { - let status = self.status(); if self .headers() .get(http::header::CONTENT_TYPE) @@ -78,9 +81,22 @@ impl ResponseExt for reqwest::Response { { return Ok(self); } + Err(match self.error_msg_for_status().await { + Ok(res) => { + anyhow::anyhow!( + "HTTP response from url ({}) was success but did not have content-type: {content_type}", + res.url() + ) + } + Err(e) => e, + }) + } + + async fn error_msg_for_status(self) -> anyhow::Result { + let status = self.status(); let url = self.url(); let Some(status_desc) = err_status_desc(status) else { - anyhow::bail!("HTTP response from url ({url}) was success but did not have content-type: {content_type}"); + return Ok(self); }; let url = url.to_string(); let status_err = match self.error_for_status_ref() { diff --git a/crates/client-api-messages/Cargo.toml b/crates/client-api-messages/Cargo.toml index f278ac59e02..7cfaad7d3e8 100644 --- a/crates/client-api-messages/Cargo.toml +++ b/crates/client-api-messages/Cargo.toml @@ -20,6 +20,9 @@ serde_with.workspace = true smallvec.workspace = true thiserror.workspace = true derive_more.workspace = true +http.workspace = true +headers.workspace = true +sfv.workspace = true [dev-dependencies] hex.workspace = true diff --git a/crates/client-api-messages/src/lib.rs b/crates/client-api-messages/src/lib.rs index 67b58de659f..c174c310170 100644 --- a/crates/client-api-messages/src/lib.rs +++ b/crates/client-api-messages/src/lib.rs @@ -3,6 +3,5 @@ pub mod energy; pub mod http; pub mod name; -pub mod websocket; - pub mod publish; +pub mod websocket; diff --git a/crates/client-api-messages/src/name.rs b/crates/client-api-messages/src/name.rs index b835078ac4d..4642f882fa4 100644 --- a/crates/client-api-messages/src/name.rs +++ b/crates/client-api-messages/src/name.rs @@ -104,8 +104,33 @@ pub enum PublishResult { /// owned by an identity other than the identity that you provided, then you will receive /// this error. PermissionDenied { name: DatabaseName }, + + /// An invalid environment was supplied. + EnvironmentError(EnvironmentPublishError), +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub enum EnvironmentPublishError { + /// A required environment variable was missing. + MissingRequiredEnvironment { keys: Vec }, + + /// Expected module hash was different + VersionConflict(#[serde(skip)] EnvironmentVersionConflict), } +impl EnvironmentPublishError { + pub fn status_code(&self) -> http::StatusCode { + match self { + Self::MissingRequiredEnvironment { .. } => http::StatusCode::BAD_REQUEST, + Self::VersionConflict(..) => http::StatusCode::CONFLICT, + } + } +} + +#[derive(Debug, Clone, Default, thiserror::Error, serde::Serialize, serde::Deserialize)] +#[error("database program changed before publication; reload environment metadata and retry")] +pub struct EnvironmentVersionConflict; + #[derive(serde::Serialize, serde::Deserialize, Debug, Default)] pub enum MigrationPolicy { #[default] diff --git a/crates/client-api-messages/src/publish.rs b/crates/client-api-messages/src/publish.rs index c77a21d56a8..6e0aa457139 100644 --- a/crates/client-api-messages/src/publish.rs +++ b/crates/client-api-messages/src/publish.rs @@ -1,179 +1,154 @@ //! Atomic publish input. Environment values travel only in the request body. -use serde::{Deserialize, Deserializer, Serialize}; -use serde_with::{base64::Base64, serde_as}; -use spacetimedb_lib::environment::{validate_key, validate_value, MAX_ENV_VARS}; use std::collections::BTreeMap; -pub const CONTENT_TYPE: &str = "application/vnd.spacetimedb.publish+json"; -pub const MAX_MODULE_BYTES: usize = 128 * 1024 * 1024; -/// Includes base64 module expansion and worst-case JSON escaping of configuration. -pub const MAX_REQUEST_BYTES: usize = 192 * 1024 * 1024; +use http::HeaderValue; +use serde::{Deserialize, Serialize}; +use spacetimedb_lib::environment::{validate_key, validate_value, EnvironmentMap, EnvironmentRemove}; +use spacetimedb_lib::Hash; -/// Values deliberately have no Debug representation. Omission is an empty map, -/// including for a publish of an unchanged module. -#[serde_as] -#[derive(Clone, Default, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PublishRequest { - #[serde(default, skip_serializing_if = "Option::is_none")] - #[serde_as(as = "Option")] - pub module: Option>, - #[serde(default, deserialize_with = "deserialize_environment")] - pub environment: BTreeMap, - #[serde(default)] - pub environment_remove: Vec, - #[serde(default)] - pub environment_replace: bool, - #[serde(default)] - pub expected_module_version: Option, -} +#[derive(Default)] +pub struct SpacetimeEnvironment(pub EnvironmentMap); -/// Authorized environment metadata. Values never leave the database in this response. -#[derive(Clone, Serialize, Deserialize)] -pub struct EnvironmentMetadata { - pub module_version: String, - pub declarations: Vec, - pub stored_keys: Vec, -} +impl headers::Header for SpacetimeEnvironment { + fn name() -> &'static http::HeaderName { + static NAME: http::HeaderName = http::HeaderName::from_static("spacetime-environment"); + &NAME + } -#[derive(Debug, Clone, Copy, thiserror::Error)] -pub enum PublishRequestError { - #[error("invalid publish request body")] - Invalid, - #[error("publish request exceeds size limit")] - TooLarge, -} + fn decode<'i, I>(values: &mut I) -> Result + where + Self: Sized, + I: Iterator, + { + let err = headers::Error::invalid; + let mut entries = BTreeMap::new(); + for value in values { + let list = sfv::Parser::new(value) + .with_version(sfv::Version::Rfc9651) + .parse::() + .map_err(|_| err())?; + for entry in list { + let items = match entry { + sfv::ListEntry::InnerList(sfv::InnerList { items, params }) if params.is_empty() => items, + _ => return Err(err()), + }; + let [k, v] = <[sfv::Item; 2]>::try_from(items) + .ok() + .filter(|x| x.iter().all(|item| item.params.is_empty())) + .ok_or_else(err)? + .map(|x| x.bare_item); + let k: String = match k { + sfv::BareItem::Token(tok) => tok.into(), + sfv::BareItem::String(s) => s.into(), + _ => return Err(err()), + }; + let v = match v { + sfv::BareItem::String(s) => s.into(), + sfv::BareItem::DisplayString(s) => s, + _ => return Err(err()), + }; + validate_key(&k).map_err(|_| err())?; + validate_value(&v).map_err(|_| err())?; + entries.insert(k, v); + } + } + Ok(Self(entries)) + } -impl PublishRequest { - pub fn decode(body: &[u8]) -> Result { - if body.len() > MAX_REQUEST_BYTES { - return Err(PublishRequestError::TooLarge); + fn encode>(&self, values: &mut E) { + let mut ser = sfv::ListSerializer::new(); + for (k, v) in &self.0 { + let mut tuple = ser.inner_list(); + // a valid environment key is always a valid sfv::String + let _ = tuple.bare_item(as_tok_or_string(k).unwrap()); + let _ = tuple.bare_item(sfv::RefBareItem::DisplayString(v)); + let _ = tuple.finish(); + } + if let Some(header) = ser.finish() { + let mut header: HeaderValue = header.try_into().unwrap(); + header.set_sensitive(true); + values.extend([header]); } - // Never expose serde's error text: it can quote a supplied secret. - let request: Self = serde_json::from_slice(body).map_err(|_| PublishRequestError::Invalid)?; - request.validate()?; - Ok(request) } +} - pub fn encode(&self) -> Result, PublishRequestError> { - self.validate()?; - serde_json::to_vec(self).map_err(|_| PublishRequestError::Invalid) +fn as_tok_or_string(s: &str) -> Option> { + let item = match sfv::TokenRef::from_str(s) { + Ok(tok) => tok.into(), + Err(_) => sfv::StringRef::from_str(s).ok()?.into(), + }; + Some(item) +} + +#[derive(Default)] +pub struct SpacetimeEnvironmentRemove(pub EnvironmentRemove); + +impl headers::Header for SpacetimeEnvironmentRemove { + fn name() -> &'static http::HeaderName { + static NAME: http::HeaderName = http::HeaderName::from_static("spacetime-environment-remove"); + &NAME } - fn validate(&self) -> Result<(), PublishRequestError> { - if self - .module - .as_ref() - .is_some_and(|module| module.len() > MAX_MODULE_BYTES) - || self.environment.len() > MAX_ENV_VARS - { - return Err(PublishRequestError::TooLarge); - } - spacetimedb_lib::environment::EnvironmentUpdate { - values: self.environment.clone(), - remove: self.environment_remove.clone(), - replace: self.environment_replace, - } - .validate() - .map_err(|_| PublishRequestError::Invalid)?; - if self - .expected_module_version - .as_ref() - .is_some_and(|hash| hash.len() != 64 || !hash.bytes().all(|byte| byte.is_ascii_hexdigit())) - { - return Err(PublishRequestError::Invalid); + fn decode<'i, I>(values: &mut I) -> Result + where + Self: Sized, + I: Iterator, + { + let mut entries = Vec::new(); + for value in values { + let list = sfv::Parser::new(value) + .with_version(sfv::Version::Rfc9651) + .parse::() + .map_err(|_| headers::Error::invalid())?; + entries.reserve(list.len()); + for v in list { + let sfv::ListEntry::Item(sfv::Item { bare_item, params }) = v else { + return Err(headers::Error::invalid()); + }; + if !params.is_empty() { + return Err(headers::Error::invalid()); + } + let key = match bare_item { + sfv::BareItem::Token(tok) if tok.as_str() == "*" => return Ok(Self(EnvironmentRemove::All)), + sfv::BareItem::Token(tok) => tok.into(), + sfv::BareItem::String(s) => s.into(), + _ => return Err(headers::Error::invalid()), + }; + entries.push(key) + } } - for (key, value) in &self.environment { - validate_key(key).map_err(|_| PublishRequestError::Invalid)?; - validate_value(value).map_err(|_| PublishRequestError::TooLarge)?; + if entries.is_empty() { + Ok(Self(EnvironmentRemove::No)) + } else { + Ok(Self(EnvironmentRemove::Keys(entries))) } - Ok(()) } -} -fn deserialize_environment<'de, D: Deserializer<'de>>(de: D) -> Result, D::Error> { - struct Visitor; - impl<'de> serde::de::Visitor<'de> for Visitor { - type Value = BTreeMap; - fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str("a map of supplied environment strings") - } - fn visit_map>(self, mut map: A) -> Result { - use serde::de::Error; - let mut values = BTreeMap::new(); - while let Some(key) = map.next_key::()? { - if values.len() >= MAX_ENV_VARS || validate_key(&key).is_err() || values.contains_key(&key) { - return Err(A::Error::custom("invalid environment keys")); + fn encode>(&self, values: &mut E) { + match &self.0 { + EnvironmentRemove::No => {} + EnvironmentRemove::All => { + values.extend([const { HeaderValue::from_static("*") }]); + } + EnvironmentRemove::Keys(remove) => { + let mut ser = sfv::ListSerializer::new(); + for key in remove { + // a valid environment key is always a valid sfv::String + let item = as_tok_or_string(key).unwrap(); + let _ = ser.bare_item(item); } - let value = map.next_value::()?; - if validate_value(&value).is_err() { - return Err(A::Error::custom("environment value exceeds size limit")); + if let Some(header) = ser.finish() { + values.extend([header.try_into().unwrap()]); } - values.insert(key, value); } - Ok(values) } } - de.deserialize_map(Visitor) } -#[cfg(test)] -mod tests { - use super::*; - #[test] - fn roundtrip_and_omission_preserve_complete_string_input() { - let request = PublishRequest { - module: Some(vec![0, 1, 255]), - environment: BTreeMap::from([("EMPTY".into(), "".into()), ("TOKEN".into(), "雪\0false".into())]), - ..Default::default() - }; - let decoded = PublishRequest::decode(&request.encode().unwrap()).unwrap(); - assert_eq!(decoded.module, request.module); - assert_eq!(decoded.environment, request.environment); - assert!(PublishRequest::decode(br#"{"module":""}"#) - .unwrap() - .environment - .is_empty()); - } - #[test] - fn environment_only_mutation_roundtrips_and_rejects_conflicting_operations() { - let request = PublishRequest { - environment: BTreeMap::from([("FUTURE".into(), "secret-marker".into())]), - environment_remove: vec!["OPTIONAL".into()], - expected_module_version: Some("ab".repeat(32)), - ..Default::default() - }; - let bytes = request.encode().unwrap(); - assert!(serde_json::from_slice::(&bytes) - .unwrap() - .get("module") - .is_none()); - let decoded = PublishRequest::decode(&bytes).unwrap(); - assert_eq!(decoded.environment_remove, request.environment_remove); - assert_eq!(decoded.expected_module_version, request.expected_module_version); - for body in [ - r#"{"environment_replace":true,"environment_remove":["KEY"]}"#, - r#"{"environment":{"KEY":"secret-marker"},"environment_remove":["KEY"]}"#, - r#"{"environment_remove":["KEY","KEY"]}"#, - r#"{"expected_module_version":"secret-marker"}"#, - ] { - let error = PublishRequest::decode(body.as_bytes()).err().expect("must reject"); - assert!(!error.to_string().contains("secret-marker")); - } - } - - #[test] - fn malformed_inputs_and_duplicate_keys_are_rejected_without_values() { - for body in [ - r#"{"module":"","environment":{"KEY":true}}"#, - r#"{"module":"","environment":{"KEY":null}}"#, - r#"{"module":"","environment":{"KEY":"first","KEY":"secret-marker"}}"#, - r#"{"module":"","environment":{"KEY":["secret-marker"]}}"#, - r#"{"module":"secret-marker"}"#, - r#"{"module":"","unknown":"secret-marker"}"#, - ] { - let error = PublishRequest::decode(body.as_bytes()).err().expect("must reject"); - assert!(!format!("{error:?}: {error}").contains("secret-marker")); - } - } +/// Authorized environment metadata. Values never leave the database in this response. +#[derive(Clone, Serialize, Deserialize)] +pub struct EnvironmentMetadata { + pub module_hash: Hash, + pub declarations: Vec, + pub stored_keys: Vec, } diff --git a/crates/client-api/src/lib.rs b/crates/client-api/src/lib.rs index eb383a5f8c9..0480f667a02 100644 --- a/crates/client-api/src/lib.rs +++ b/crates/client-api/src/lib.rs @@ -11,13 +11,15 @@ use http::StatusCode; use spacetimedb::client::ClientActorIndex; use spacetimedb::energy::{EnergyBalance, EnergyQuanta}; +use spacetimedb::host::module_host::UpdateEnvironmentResult; use spacetimedb::host::{HostController, MigratePlanResult, ModuleHost, NoSuchModule, UpdateDatabaseResult}; use spacetimedb::identity::{AuthCtx, Identity}; use spacetimedb::messages::control_db::{Database, HostType, Node, Replica}; use spacetimedb::sql; use spacetimedb_client_api_messages::http::{SqlStmtResult, SqlStmtStats}; use spacetimedb_client_api_messages::name::{DomainName, InsertDomainResult, RegisterTldResult, SetDomainsResult, Tld}; -use spacetimedb_lib::{ProductTypeElement, ProductValue}; +use spacetimedb_lib::environment::{EnvironmentMap, EnvironmentUpdate}; +use spacetimedb_lib::{Hash, ProductTypeElement, ProductValue}; use spacetimedb_paths::server::ModuleLogsDir; use spacetimedb_schema::auto_migrate::{MigrationPolicy, PrettyPrintStyle}; use thiserror::Error; @@ -208,18 +210,20 @@ impl Host { program_bytes: Box<[u8]>, policy: MigrationPolicy, environment: spacetimedb_lib::environment::EnvironmentUpdate, - expected_module_version: Option, ) -> anyhow::Result { self.host_controller - .update_module_host( - database, - host_type, - self.replica_id, - program_bytes, - policy, - environment, - expected_module_version, - ) + .update_module_host(database, host_type, self.replica_id, program_bytes, policy, environment) + .await + } + + pub async fn update_environment( + &self, + database: Database, + environment: spacetimedb_lib::environment::EnvironmentUpdate, + expected_module_hash: Hash, + ) -> anyhow::Result { + self.host_controller + .update_module_environment(database, self.replica_id, environment, expected_module_hash) .await } } @@ -231,11 +235,6 @@ pub struct DatabaseDef { pub database_identity: Identity, /// The compiled program of the database module. pub program_bytes: Bytes, - /// Supplied overrides, never persisted in the public Database record. - pub environment: std::collections::BTreeMap, - pub environment_remove: Vec, - pub environment_replace: bool, - pub expected_module_version: Option, /// The desired number of replicas the database shall have. /// /// If `None`, the edition default is used. @@ -253,9 +252,6 @@ pub struct DatabaseDef { pub struct DatabaseResetDef { pub database_identity: Identity, pub program_bytes: Option, - pub environment: std::collections::BTreeMap, - pub environment_remove: Vec, - pub environment_replace: bool, pub num_replicas: Option, pub host_type: Option, } @@ -328,6 +324,7 @@ pub trait ControlStateWriteAccess: Send + Sync { publisher: &Identity, spec: DatabaseDef, policy: MigrationPolicy, + environment: EnvironmentUpdate, ) -> anyhow::Result>; async fn migrate_plan(&self, spec: DatabaseDef, style: PrettyPrintStyle) -> anyhow::Result; @@ -336,7 +333,12 @@ pub trait ControlStateWriteAccess: Send + Sync { /// Remove all data from a database, and reset it according to the /// given [DatabaseResetDef]. - async fn reset_database(&self, caller_identity: &Identity, spec: DatabaseResetDef) -> anyhow::Result<()>; + async fn reset_database( + &self, + caller_identity: &Identity, + spec: DatabaseResetDef, + environment: EnvironmentMap, + ) -> anyhow::Result<()>; // Energy async fn add_energy(&self, identity: &Identity, amount: EnergyQuanta) -> anyhow::Result<()>; @@ -374,6 +376,14 @@ pub trait ControlStateWriteAccess: Send + Sync { database_identity: &Identity, locked: bool, ) -> anyhow::Result<()>; + + async fn update_environment( + &self, + publisher: &Identity, + database_identity: &Identity, + environment: EnvironmentUpdate, + expected_module_hash: Hash, + ) -> anyhow::Result; } #[async_trait] @@ -442,8 +452,9 @@ impl ControlStateWriteAccess for Arc { identity: &Identity, spec: DatabaseDef, policy: MigrationPolicy, + environment: EnvironmentUpdate, ) -> anyhow::Result> { - (**self).publish_database(identity, spec, policy).await + (**self).publish_database(identity, spec, policy, environment).await } async fn migrate_plan(&self, spec: DatabaseDef, style: PrettyPrintStyle) -> anyhow::Result { @@ -454,8 +465,13 @@ impl ControlStateWriteAccess for Arc { (**self).delete_database(caller_identity, database_identity).await } - async fn reset_database(&self, caller_identity: &Identity, spec: DatabaseResetDef) -> anyhow::Result<()> { - (**self).reset_database(caller_identity, spec).await + async fn reset_database( + &self, + caller_identity: &Identity, + spec: DatabaseResetDef, + environment: EnvironmentMap, + ) -> anyhow::Result<()> { + (**self).reset_database(caller_identity, spec, environment).await } async fn add_energy(&self, identity: &Identity, amount: EnergyQuanta) -> anyhow::Result<()> { @@ -499,6 +515,18 @@ impl ControlStateWriteAccess for Arc { .set_database_lock(caller_identity, database_identity, locked) .await } + + async fn update_environment( + &self, + publisher: &Identity, + database_identity: &Identity, + environment: EnvironmentUpdate, + expected_module_hash: Hash, + ) -> anyhow::Result { + (**self) + .update_environment(publisher, database_identity, environment, expected_module_hash) + .await + } } #[async_trait] diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index 4e6bb1af152..cffe673f4de 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -1,5 +1,5 @@ -mod publish_environment; -use publish_environment::{ModuleBody, PublishBody}; +use spacetimedb_client_api_messages::publish::{SpacetimeEnvironment, SpacetimeEnvironmentRemove}; +use spacetimedb_lib::environment::{EnvironmentMap, EnvironmentRemove, EnvironmentUpdate}; use std::borrow::Cow; use std::future::Future; @@ -33,7 +33,7 @@ use log::{debug, info, warn}; use serde::Deserialize; use spacetimedb::auth::identity::ConnectionAuthCtx; use spacetimedb::database_logger::DatabaseLogger; -use spacetimedb::host::module_host::{ClientConnectedError, DurabilityExited}; +use spacetimedb::host::module_host::{ClientConnectedError, DurabilityExited, UpdateEnvironmentResult}; use spacetimedb::host::{CallResult, UpdateDatabaseResult}; use spacetimedb::host::{FunctionArgs, MigratePlanResult}; use spacetimedb::host::{ModuleHost, ReducerOutcome}; @@ -42,8 +42,9 @@ use spacetimedb::identity::Identity; use spacetimedb::messages::control_db::{Database, HostType}; use spacetimedb_client_api_messages::http::SqlStmtResult; use spacetimedb_client_api_messages::name::{ - self, DatabaseName, DomainName, MigrationPolicy, PrePublishAutoMigrateResult, PrePublishManualMigrateResult, - PrePublishResult, PrettyPrintStyle, PublishOp, PublishResult, + self, DatabaseName, DomainName, EnvironmentPublishError, EnvironmentVersionConflict, MigrationPolicy, + PrePublishAutoMigrateResult, PrePublishManualMigrateResult, PrePublishResult, PrettyPrintStyle, PublishOp, + PublishResult, }; use spacetimedb_datastore::db_metrics::DB_METRICS; use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10; @@ -579,52 +580,44 @@ impl From for DatabaseResponse { } } -fn environment_validation_error(error: &anyhow::Error) -> Option { - use spacetimedb::db::environment::EnvironmentError; +fn extract_environment_error(mut error: &anyhow::Error) -> Option> { + // TODO(noa): clean up these errors my g-d this is bad use spacetimedb::host::module_host::InitDatabaseError; - use spacetimedb_lib::environment::{validate_key, EnvironmentSchemaError, EnvironmentSchemaErrorKind}; - if let Some(InitDatabaseError::Other(error)) = error.downcast_ref::() { - return environment_validation_error(error); - } - let error = - error - .downcast_ref::() - .or_else(|| match error.downcast_ref::() { - Some(EnvironmentError::Schema(error)) => Some(error), - _ => None, - })?; - // Only typed host validation can ask the caller for a secret. Never infer - // missing keys from module failures or arbitrary diagnostic text. - if error.kind == EnvironmentSchemaErrorKind::MissingRequired - && let Some(key) = error.key.as_deref().filter(|key| validate_key(key).is_ok()) - { - return Some( - ( - StatusCode::BAD_REQUEST, - axum::Json(serde_json::json!({ - "error": "missing_required_environment", - "key": key, - })), - ) - .into(), - ); - } - Some((StatusCode::BAD_REQUEST, error.to_string()).into()) + use spacetimedb_lib::environment::EnvironmentSchemaError; + while let Some(InitDatabaseError::Other(e)) = error.downcast_ref::() { + error = e; + } + let res = if let Some(error) = error.chain().find_map(|x| x.downcast_ref::()) { + // Only typed host validation can ask the caller for a secret. Never infer + // missing keys from module failures or arbitrary diagnostic text. + if let EnvironmentSchemaError::MissingRequired { key } = error { + // TODO(noa): in the future, return multiple missing keys at once + Ok(EnvironmentPublishError::MissingRequiredEnvironment { keys: vec![key.into()] }) + } else { + Err((StatusCode::BAD_REQUEST, error.to_string()).into()) + } + } else if let Some(error) = error.downcast_ref::() { + Ok(EnvironmentPublishError::VersionConflict(error.clone())) + } else { + return None; + }; + Some(res) } fn publish_error(error: anyhow::Error) -> axum::response::ErrorResponse { - if let Some(response) = environment_validation_error(&error) { - return response; + match extract_environment_error(&error) { + Some(Ok(err)) => (err.status_code(), axum::Json(err)).into(), + Some(Err(e)) => e, + None => log_and_500(error), } - if let Some(error) = error.downcast_ref::() { - return (StatusCode::CONFLICT, error.to_string()).into(); - } - log_and_500(error) } fn publish_migration_error(error: anyhow::Error) -> axum::response::ErrorResponse { - environment_validation_error(&error) - .unwrap_or_else(|| bad_request(format!("Failed to create or update the database: {error}").into())) + match extract_environment_error(&error) { + Some(Ok(err)) => (err.status_code(), axum::Json(err)).into(), + Some(Err(e)) => e, + None => bad_request(format!("Failed to create or update the database: {error}").into()), + } } pub async fn environment_metadata( @@ -642,6 +635,96 @@ where Ok(([(http::header::CACHE_CONTROL, "no-store")], axum::Json(metadata))) } +#[derive(Deserialize)] +pub struct EnvironmentUpdateQueryParams { + expected_module_hash: Hash, +} + +pub async fn environment_set( + State(ctx): State, + Extension(ResolvedDatabase(database)): Extension, + Extension(auth): Extension, + Query(EnvironmentUpdateQueryParams { expected_module_hash }): Query, + axum::Json(values): axum::Json, +) -> EnvironmentPublishResult +where + S: ControlStateDelegate + NodeDelegate + Authorization, +{ + ctx.authorize_action(auth.claims.identity, database.database_identity, Action::UpdateDatabase) + .await?; + let remove = EnvironmentRemove::All; + let update = EnvironmentUpdate { values, remove }; + update + .validate() + .map_err(|e| (StatusCode::BAD_REQUEST, e.to_string()))?; + let result = ctx + .update_environment( + &auth.claims.identity, + &database.database_identity, + update, + expected_module_hash, + ) + .await; + + environment_publish_result(result) +} + +pub async fn environment_patch( + State(ctx): State, + Extension(ResolvedDatabase(database)): Extension, + Extension(auth): Extension, + Query(EnvironmentUpdateQueryParams { expected_module_hash }): Query, + TypedHeader(SpacetimeEnvironmentRemove(remove)): TypedHeader, + axum::Json(values): axum::Json, +) -> EnvironmentPublishResult +where + S: ControlStateDelegate + NodeDelegate + Authorization, +{ + if remove == EnvironmentRemove::All { + // if you want to fully replace, just PUT /environment + return Err(bad_request( + format!( + "{} ({})", + headers::Error::invalid(), + ::name() + ) + .into(), + )); + } + ctx.authorize_action(auth.claims.identity, database.database_identity, Action::UpdateDatabase) + .await?; + let update = EnvironmentUpdate { values, remove }; + update + .validate() + .map_err(|e| (StatusCode::BAD_REQUEST, e.to_string()))?; + let result = ctx + .update_environment( + &auth.claims.identity, + &database.database_identity, + update, + expected_module_hash, + ) + .await; + + environment_publish_result(result) +} + +type EnvironmentPublishResult = axum::response::Result<(StatusCode, axum::Json>)>; + +fn environment_publish_result(result: anyhow::Result) -> EnvironmentPublishResult { + let result = match result { + Ok(UpdateEnvironmentResult::ErrorExecutingMigration(e)) => return Err(publish_migration_error(e)), + Ok(UpdateEnvironmentResult::NoUpdateNeeded | UpdateEnvironmentResult::UpdatePerformed { .. }) => Ok(()), + Err(e) => match extract_environment_error(&e) { + Some(Ok(e)) => Err(e), + Some(Err(e)) => return Err(e), + None => return Err(log_and_500(e)), + }, + }; + let status = result.as_ref().err().map_or(StatusCode::OK, |e| e.status_code()); + Ok((status, axum::Json(result))) +} + pub async fn db_info( Extension(ResolvedDatabase(database)): Extension, ) -> axum::response::Result { @@ -901,21 +984,9 @@ pub async fn reset( host_type, }): Query, Extension(auth): Extension, - PublishBody { - program_bytes, - environment, - environment_remove, - environment_replace, - expected_module_version, - environment_only, - }: PublishBody, + TypedHeader(SpacetimeEnvironment(environment)): TypedHeader, + program_bytes: Bytes, ) -> axum::response::Result> { - if expected_module_version.is_some() { - return Err(bad_request( - "expected_module_version is not supported for database reset".into(), - )); - } - let _ = environment_only; let database_identity = database.database_identity; ctx.authorize_action(auth.claims.identity, database.database_identity, Action::ResetDatabase) @@ -934,13 +1005,11 @@ pub async fn reset( &auth.claims.identity, DatabaseResetDef { database_identity, - program_bytes, - environment, - environment_remove, - environment_replace, + program_bytes: Some(program_bytes), num_replicas, host_type: Some(host_type), }, + environment, ) .await .map_err(publish_error)?; @@ -1014,31 +1083,11 @@ pub async fn publish( organization, update_confirmation_timeout: confirmation_timeout, }): Query, + TypedHeader(SpacetimeEnvironment(environment)): TypedHeader, + TypedHeader(SpacetimeEnvironmentRemove(environment_remove)): TypedHeader, Extension(auth): Extension, - PublishBody { - program_bytes, - environment, - environment_remove, - environment_replace, - expected_module_version, - environment_only, - }: PublishBody, + program_bytes: Bytes, ) -> axum::response::Result> { - if environment_only && (clear || parent.is_some() || organization.is_some() || num_replicas.is_some()) { - return Err(bad_request( - "environment-only publication cannot change database configuration or reset data".into(), - )); - } - if environment_only && expected_module_version.is_none() { - return Err(bad_request( - "environment-only publication requires expected_module_version".into(), - )); - } - if environment_only && name_or_identity.is_none() { - return Err(bad_request( - "environment-only publication requires an existing database".into(), - )); - } // If `clear`, check that the database exists and delegate to `reset`. // If it doesn't exist, ignore the `clear` parameter. // TODO: Replace with actual redirect at the next possible version bump. @@ -1067,27 +1116,21 @@ pub async fn publish( host_type, }), Extension(auth), - PublishBody { - program_bytes, - environment, - environment_remove, - environment_replace, - expected_module_version, - environment_only, - }, + TypedHeader(SpacetimeEnvironment(environment)), + program_bytes, ) .await; } } } - let program_bytes = program_bytes.unwrap_or_default(); - let (database_identity, db_name) = if environment_only { - let name = name_or_identity.as_ref().expect("validated existing database name"); - (name.resolve(&ctx).await?, name.name()) - } else { - get_or_create_identity_and_name(&ctx, &auth, name_or_identity.as_ref()).await? - }; + if name_or_identity.is_none() && environment_remove != EnvironmentRemove::No { + return Err(bad_request( + "cannot specify spacetime-environment-remove without an existing database".into(), + )); + } + + let (database_identity, db_name) = get_or_create_identity_and_name(&ctx, &auth, name_or_identity.as_ref()).await?; let maybe_parent_database_identity = match parent.as_ref() { None => None, Some(parent) => parent.resolve(&ctx).await.map(Some)?, @@ -1107,13 +1150,6 @@ pub async fn publish( .get_database_by_identity(&database_identity) .await .map_err(log_and_500)?; - if environment_only && existing.is_none() { - return Err(( - StatusCode::NOT_FOUND, - "environment-only publication requires an existing database", - ) - .into()); - } match existing.as_ref() { None => { allow_creation(&auth)?; @@ -1157,16 +1193,16 @@ pub async fn publish( DatabaseDef { database_identity, program_bytes, - environment, - environment_remove, - environment_replace, - expected_module_version, num_replicas, host_type, parent, organization: maybe_org_identity, }, schema_migration_policy, + EnvironmentUpdate { + values: environment, + remove: environment_remove, + }, ) .await .map_err(publish_error)?; @@ -1343,7 +1379,7 @@ pub async fn pre_publish Extension(ResolvedDatabase(database)): Extension, Query(PrePublishQueryParams { style, host_type }): Query, Extension(auth): Extension, - ModuleBody(program_bytes): ModuleBody, + program_bytes: Bytes, ) -> axum::response::Result> { let database_identity = database.database_identity; @@ -1362,10 +1398,6 @@ pub async fn pre_publish DatabaseDef { database_identity, program_bytes, - environment: Default::default(), - environment_remove: Default::default(), - environment_replace: false, - expected_module_version: None, num_replicas: None, host_type, parent: None, @@ -1605,7 +1637,12 @@ pub struct DatabaseRoutes { pub call_reducer_procedure_post: MethodRouter, /// GET: /database/:name_or_identity/schema pub schema_get: MethodRouter, + /// GET: /database/:name_or_identity/environment pub environment_get: MethodRouter, + /// PUT: /database/:name_or_identity/environment + pub environment_put: MethodRouter, + /// PATCH: /database/:name_or_identity/environment + pub environment_patch: MethodRouter, /// GET: /database/:name_or_identity/logs pub logs_get: MethodRouter, /// POST: /database/:name_or_identity/sql @@ -1635,7 +1672,7 @@ where S: NodeDelegate + ControlStateDelegate + HasWebSocketOptions + Authorization + Clone + 'static, { fn default() -> Self { - use axum::routing::{any, delete, get, post, put}; + use axum::routing::{any, delete, get, patch, post, put}; Self { root_post: post(publish::), db_put: put(publish::), @@ -1649,6 +1686,8 @@ where call_reducer_procedure_post: post(call::), schema_get: get(schema::), environment_get: get(environment_metadata::), + environment_put: put(environment_set::), + environment_patch: patch(environment_patch::), logs_get: get(logs::), sql_post: post(sql::), mcp_post: post(crate::routes::mcp::mcp::), @@ -1682,6 +1721,8 @@ where .route("/call/:reducer", self.call_reducer_procedure_post) .route("/schema", self.schema_get) .route("/environment", self.environment_get) + .route("/environment", self.environment_put) + .route("/environment", self.environment_patch) .route("/logs", self.logs_get) .route("/sql", self.sql_post) .route("/mcp", self.mcp_post) @@ -1823,11 +1864,13 @@ mod tests { use spacetimedb::auth::token_validation::{TokenSigner, TokenValidationError, TokenValidator}; use spacetimedb::client::ClientActorIndex; use spacetimedb::energy::{EnergyBalance, EnergyQuanta}; + use spacetimedb::host::module_host::UpdateEnvironmentResult; use spacetimedb::identity::AuthCtx; use spacetimedb::messages::control_db::{Database, Node, Replica}; use spacetimedb_client_api_messages::name::{ DomainName, InsertDomainResult, RegisterTldResult, SetDomainsResult, Tld, }; + use spacetimedb_lib::environment::EnvironmentMap; use spacetimedb_lib::Hash; use spacetimedb_paths::server::ModuleLogsDir; use spacetimedb_paths::FromPathUnchecked; @@ -1839,11 +1882,8 @@ mod tests { #[tokio::test] async fn publish_environment_error_identifies_only_typed_missing_required_keys() { - use spacetimedb_lib::environment::{EnvironmentSchemaError, EnvironmentSchemaErrorKind}; - let missing = || EnvironmentSchemaError { - key: Some("API_KEY".into()), - kind: EnvironmentSchemaErrorKind::MissingRequired, - }; + use spacetimedb_lib::environment::EnvironmentSchemaError; + let missing = || EnvironmentSchemaError::MissingRequired { key: "API_KEY".into() }; for response in [ publish_error(anyhow::Error::new(missing()).context("publication failed")), publish_migration_error(spacetimedb::db::environment::EnvironmentError::Schema(missing()).into()), @@ -1867,22 +1907,13 @@ mod tests { assert_eq!( serde_json::from_slice::(&body).unwrap(), serde_json::json!({ - "error": "missing_required_environment", "key": "API_KEY", + "MissingRequiredEnvironment": { "keys": ["API_KEY"] } }) ); } for error in [ anyhow::anyhow!("environment key API_KEY: required value is missing"), - EnvironmentSchemaError { - key: Some("API_KEY".into()), - kind: EnvironmentSchemaErrorKind::ConstraintMismatch, - } - .into(), - EnvironmentSchemaError { - key: Some("INVALID-KEY".into()), - kind: EnvironmentSchemaErrorKind::MissingRequired, - } - .into(), + EnvironmentSchemaError::ConstraintMismatch { key: "API_KEY".into() }.into(), ] { let response = Err::<(), _>(publish_migration_error(error)).into_response(); assert_eq!(response.status(), StatusCode::BAD_REQUEST); @@ -2137,6 +2168,7 @@ mod tests { _publisher: &Identity, _spec: DatabaseDef, _policy: MigrationPolicy, + _environment: EnvironmentUpdate, ) -> anyhow::Result> { Err(anyhow::anyhow!("unused")) } @@ -2166,7 +2198,12 @@ mod tests { Err(anyhow::anyhow!("unused")) } - async fn reset_database(&self, _caller_identity: &Identity, _spec: DatabaseResetDef) -> anyhow::Result<()> { + async fn reset_database( + &self, + _caller_identity: &Identity, + _spec: DatabaseResetDef, + _environment: EnvironmentMap, + ) -> anyhow::Result<()> { Err(anyhow::anyhow!("unused")) } @@ -2199,6 +2236,16 @@ mod tests { ) -> anyhow::Result { Err(anyhow::anyhow!("unused")) } + + async fn update_environment( + &self, + _publisher: &Identity, + _database_identity: &Identity, + _environment: EnvironmentUpdate, + _expected_module_hash: Hash, + ) -> anyhow::Result { + Err(anyhow::anyhow!("unused")) + } } impl Authorization for DummyState { diff --git a/crates/client-api/src/routes/database/publish_environment.rs b/crates/client-api/src/routes/database/publish_environment.rs deleted file mode 100644 index 86d8d30f9cb..00000000000 --- a/crates/client-api/src/routes/database/publish_environment.rs +++ /dev/null @@ -1,181 +0,0 @@ -//! Bounded publish extraction. Neither errors nor Debug output retain configuration values. -use axum::body::{to_bytes, Bytes}; -use axum::extract::{FromRequest, Request}; -use axum::response::{IntoResponse, Response}; -use http::{header, StatusCode}; -use spacetimedb_client_api_messages::publish::{PublishRequest, CONTENT_TYPE, MAX_MODULE_BYTES, MAX_REQUEST_BYTES}; -use std::collections::BTreeMap; - -pub struct PublishBody { - pub program_bytes: Option, - pub environment: BTreeMap, - pub environment_remove: Vec, - pub environment_replace: bool, - pub expected_module_version: Option, - pub environment_only: bool, -} - -async fn bounded_body(request: Request, limit: usize) -> Result { - if request - .headers() - .get(header::CONTENT_LENGTH) - .and_then(|value| value.to_str().ok()) - .and_then(|value| value.parse::().ok()) - .is_some_and(|len| len > limit as u64) - { - return Err((StatusCode::PAYLOAD_TOO_LARGE, "publish request exceeds size limit").into_response()); - } - to_bytes(request.into_body(), limit) - .await - .map_err(|_| (StatusCode::PAYLOAD_TOO_LARGE, "publish request exceeds size limit").into_response()) -} - -#[async_trait::async_trait] -impl FromRequest for PublishBody { - type Rejection = Response; - - async fn from_request(request: Request, _state: &S) -> Result { - let envelope = request - .headers() - .get(header::CONTENT_TYPE) - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| { - value - .split(';') - .next() - .unwrap_or_default() - .trim() - .eq_ignore_ascii_case(CONTENT_TYPE) - }); - let bytes = bounded_body(request, if envelope { MAX_REQUEST_BYTES } else { MAX_MODULE_BYTES }).await?; - if envelope { - let request = PublishRequest::decode(&bytes) - .map_err(|_| (StatusCode::BAD_REQUEST, "invalid publish request body").into_response())?; - if request.module.as_ref().is_some_and(|module| module.is_empty()) { - return Err((StatusCode::BAD_REQUEST, "module artifact must not be empty").into_response()); - } - let environment_only = request.module.is_none(); - Ok(Self { - program_bytes: request.module.map(Into::into), - environment: request.environment, - environment_remove: request.environment_remove, - environment_replace: request.environment_replace, - expected_module_version: request - .expected_module_version - .map(spacetimedb_lib::Hash::from_hex) - .transpose() - .map_err(|_| (StatusCode::BAD_REQUEST, "invalid expected module version").into_response())?, - environment_only, - }) - } else { - // An empty legacy reset body retains the program; reset clears all data. - Ok(Self { - program_bytes: (!bytes.is_empty()).then_some(bytes), - environment: BTreeMap::new(), - environment_remove: Vec::new(), - environment_replace: false, - expected_module_version: None, - environment_only: false, - }) - } - } -} - -pub struct ModuleBody(pub Bytes); - -#[async_trait::async_trait] -impl FromRequest for ModuleBody { - type Rejection = Response; - - async fn from_request(request: Request, _state: &S) -> Result { - bounded_body(request, MAX_MODULE_BYTES).await.map(Self) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use axum::body::Body; - - #[tokio::test] - async fn legacy_envelope_and_empty_reset_have_complete_input_semantics() { - let legacy = PublishBody::from_request(Request::new(Body::from("module")), &()) - .await - .unwrap(); - assert_eq!(legacy.program_bytes.unwrap(), "module"); - assert!(legacy.environment.is_empty()); - let empty = PublishBody::from_request(Request::new(Body::empty()), &()) - .await - .unwrap(); - assert!(empty.program_bytes.is_none()); - assert!(empty.environment.is_empty()); - let input = PublishRequest { - module: Some(vec![1, 2, 3]), - environment: BTreeMap::from([("TOKEN".into(), "雪\0".into())]), - ..Default::default() - }; - let request = Request::builder() - .header(header::CONTENT_TYPE, CONTENT_TYPE) - .body(Body::from(input.encode().unwrap())) - .unwrap(); - let extracted = PublishBody::from_request(request, &()).await.unwrap(); - assert_eq!(extracted.environment, input.environment); - assert_eq!(extracted.program_bytes.unwrap(), input.module.unwrap()); - let reset = PublishRequest { - module: None, - environment: input.environment, - ..Default::default() - }; - let request = Request::builder() - .header(header::CONTENT_TYPE, CONTENT_TYPE) - .body(Body::from(reset.encode().unwrap())) - .unwrap(); - let extracted = PublishBody::from_request(request, &()).await.unwrap(); - assert!(extracted.program_bytes.is_none()); - assert_eq!(extracted.environment, reset.environment); - } - - #[tokio::test] - async fn empty_artifact_is_rejected_but_omission_is_environment_only() { - for (body, accepted) in [(r#"{"module":""}"#, false), (r#"{}"#, true)] { - let request = Request::builder() - .header(header::CONTENT_TYPE, CONTENT_TYPE) - .body(Body::from(body)) - .unwrap(); - match PublishBody::from_request(request, &()).await { - Ok(body) => { - assert!(accepted); - assert!(body.environment_only); - } - Err(error) => { - assert!(!accepted); - assert_eq!(error.status(), StatusCode::BAD_REQUEST); - } - } - } - } - - #[tokio::test] - async fn streamed_limits_apply_without_global_body_limit_and_errors_are_redacted() { - let stream = futures::stream::iter([ - Ok::<_, std::io::Error>(Bytes::from_static(b"12345")), - Ok(Bytes::from_static(b"67890")), - ]); - let error = bounded_body(Request::new(Body::from_stream(stream)), 8) - .await - .unwrap_err(); - assert_eq!(error.into_response().status(), StatusCode::PAYLOAD_TOO_LARGE); - let request = Request::builder() - .header(header::CONTENT_TYPE, CONTENT_TYPE) - .body(Body::from(r#"{"module":"","environment":{"KEY":["secret-marker"]}}"#)) - .unwrap(); - let error = PublishBody::from_request(request, &()) - .await - .err() - .unwrap() - .into_response(); - assert_eq!(error.status(), StatusCode::BAD_REQUEST); - let body = to_bytes(error.into_body(), 1024).await.unwrap(); - assert!(!String::from_utf8_lossy(&body).contains("secret-marker")); - } -} diff --git a/crates/core/src/db/environment.rs b/crates/core/src/db/environment.rs index 5c156ab4ec3..51f194e3f37 100644 --- a/crates/core/src/db/environment.rs +++ b/crates/core/src/db/environment.rs @@ -18,7 +18,7 @@ use std::collections::BTreeMap; pub enum EnvironmentError { #[error(transparent)] Validation(#[from] EnvironmentValidationError), - #[error(transparent)] + #[error("couldn't validate schema")] Schema(#[from] EnvironmentSchemaError), #[error(transparent)] Datastore(#[from] DatastoreError), diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index 26fedc29250..55e52350a41 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -1,4 +1,6 @@ -use super::module_host::{DurableOffset, EventStatus, InitDatabaseResult, ModuleHost, ModuleInfo, NoSuchModule}; +use super::module_host::{ + DurableOffset, EventStatus, InitDatabaseResult, ModuleHost, ModuleInfo, NoSuchModule, UpdateEnvironmentResult, +}; use super::scheduler::SchedulerStarter; use super::v8::V8HeapMetrics; use super::wasmtime::{WasmMemoryBytesMetric, WasmtimeRuntime}; @@ -31,6 +33,7 @@ use durability::{Durability, EmptyHistory}; use log::{info, trace, warn}; use parking_lot::Mutex; use scopeguard::{defer, guard}; +use spacetimedb_client_api_messages::name::EnvironmentVersionConflict; use spacetimedb_commitlog::SizeOnDisk; use spacetimedb_data_structures::error_stream::ErrorStream; use spacetimedb_data_structures::map::{IntMap, IntSet}; @@ -92,10 +95,6 @@ where pub type ProgramStorage = Arc; -#[derive(Debug, thiserror::Error)] -#[error("database program changed before publication; reload environment metadata and retry")] -pub struct EnvironmentVersionConflict; - /// Private complete configuration for a not-yet-initialized database generation. /// Implementations must verify the exact persisted database identity, program and /// initialization generation. A source resolving the generation at load time must @@ -581,41 +580,16 @@ impl HostController { Ok(info) } - /// Update the [`ModuleHost`] identified by `replica_id` to the given - /// program. - /// - /// The host may not be running, in which case it is spawned (see - /// [`Self::get_or_launch_module_host`] for details on what this entails). - /// - /// If the host was running, and the update fails, the previous version of - /// the host keeps running. - #[tracing::instrument(level = "trace", skip_all, err)] - #[allow(clippy::too_many_arguments)] - pub async fn update_module_host( + async fn update_module_inner( &self, database: Database, - host_type: HostType, replica_id: u64, - program_bytes: Box<[u8]>, - policy: MigrationPolicy, - environment: spacetimedb_lib::environment::EnvironmentUpdate, - expected_module_version: Option, - ) -> anyhow::Result { - environment.validate()?; - let environment_only = program_bytes.is_empty(); - anyhow::ensure!( - !environment_only || expected_module_version.is_some(), - "environment-only publication requires expected_module_version" - ); - let program = Program::from_bytes(host_type.into(), program_bytes); - trace!( - "update module host {}/{}: genesis={} update-to={}", - database.database_identity, - replica_id, - database.initial_program, - program.hash - ); - + f: impl FnOnce(HostController, Host) -> Fut + Send + 'static, + ) -> anyhow::Result + where + Fut: Future)> + Send, + R: Send + 'static, + { let Ok(mut guard) = self.acquire_write_lock(replica_id).await else { bail!("unable to lock database {} for update", database.database_identity); }; @@ -623,7 +597,6 @@ impl HostController { // `HostController::clone` is fast, // as all of its fields are either `Copy` or wrapped in `Arc`. let this = self.clone(); - let database_identity = database.database_identity; // `try_init_host` is not cancel safe, as it will spawn other async tasks // which hold a filesystem lock past when `try_init_host` returns or is cancelled. @@ -640,7 +613,7 @@ impl HostController { // Note that `tokio::spawn` only cancels its tasks when the runtime shuts down, // at which point we won't be calling `try_init_host` again anyways. let update_result = tokio::spawn(async move { - let mut host = match guard.take() { + let host = match guard.take() { None => { trace!("host not running, try_init"); this.try_init_host(database, replica_id).await?.host @@ -650,29 +623,62 @@ impl HostController { host } }; + let (host, update_result) = f(this, host).await; + + // Rejected publication leaves the existing host usable. Restore it + // before propagating validation or migration failure to the caller. + *guard = Some(host); + update_result + }) + .await??; + + Ok(update_result) + } + + /// Update the [`ModuleHost`] identified by `replica_id` to the given + /// program. + /// + /// The host may not be running, in which case it is spawned (see + /// [`Self::get_or_launch_module_host`] for details on what this entails). + /// + /// If the host was running, and the update fails, the previous version of + /// the host keeps running. + #[tracing::instrument(level = "trace", skip_all, err)] + #[allow(clippy::too_many_arguments)] + pub async fn update_module_host( + &self, + database: Database, + host_type: HostType, + replica_id: u64, + program_bytes: Box<[u8]>, + policy: MigrationPolicy, + environment: spacetimedb_lib::environment::EnvironmentUpdate, + ) -> anyhow::Result { + environment.validate()?; + let program = Program::from_bytes(host_type.into(), program_bytes); + trace!( + "update module host {}/{}: genesis={} update-to={}", + database.database_identity, + replica_id, + database.initial_program, + program.hash + ); + + let database_identity = database.database_identity; + + self.update_module_inner(database, replica_id, async move |this, mut host| { let update_result = async { let module = host.module.borrow().clone(); - if let Some(expected) = expected_module_version - && module.info.module_hash != expected - { - return Err(EnvironmentVersionConflict.into()); - } let previous = host .replica_ctx .relational_db() .with_read_only(Workload::Internal, |tx| crate::db::environment::snapshot(tx))?; let environment = environment.resulting_values(&previous)?; - if environment_only || program.hash == module.info.module_hash { + if program.hash == module.info.module_hash { if environment == previous { return Ok(UpdateDatabaseResult::NoUpdateNeeded); } - let program = module - .relational_db() - .program()? - .context("database program is not initialized")?; - return module - .update_database_with_environment(program, module.info.clone(), policy, environment) - .await; + return module.update_environment(environment).await.map(Into::into); } host.update_module( this.runtimes.clone(), @@ -687,14 +693,43 @@ impl HostController { } .await; - // Rejected publication leaves the existing host usable. Restore it - // before propagating validation or migration failure to the caller. - *guard = Some(host); - update_result + (host, update_result) }) - .await??; + .await + } - Ok(update_result) + #[tracing::instrument(level = "trace", skip_all, err)] + #[allow(clippy::too_many_arguments)] + pub async fn update_module_environment( + &self, + database: Database, + replica_id: u64, + environment: spacetimedb_lib::environment::EnvironmentUpdate, + expected_module_hash: Hash, + ) -> anyhow::Result { + environment.validate()?; + + self.update_module_inner(database, replica_id, async move |_, host| { + let update_result = async { + let module = host.module.borrow().clone(); + if module.info.module_hash != expected_module_hash { + return Err(EnvironmentVersionConflict.into()); + } + let previous = host + .replica_ctx + .relational_db() + .with_read_only(Workload::Internal, |tx| crate::db::environment::snapshot(tx))?; + let environment = environment.resulting_values(&previous)?; + if environment == previous { + return Ok(UpdateEnvironmentResult::NoUpdateNeeded); + } + module.update_environment(environment).await + } + .await; + + (host, update_result) + }) + .await } pub async fn migrate_plan( @@ -821,7 +856,7 @@ impl HostController { crate::db::environment::snapshot(tx).map(|values| values.into_keys().collect()) })?; Ok(spacetimedb_client_api_messages::publish::EnvironmentMetadata { - module_version: module.info.module_hash.to_string(), + module_hash: module.info.module_hash, declarations: module.info.module_def.environment().declarations().cloned().collect(), stored_keys, }) diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 76f605d2244..99e6806c236 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -1552,15 +1552,16 @@ mod test { } fn bind_test_environment(env: &mut InstanceEnv) -> Result { - use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10Builder; - use spacetimedb_lib::environment::{EnvVarType, EnvironmentDeclaration}; + use spacetimedb_lib::db::raw_def::v10::{ + RawEnvVarTypeV10, RawEnvironmentDeclarationV10, RawModuleDefV10Builder, + }; let mut builder = RawModuleDefV10Builder::new(); builder.add_environment( [("A", false), ("MISSING", true)] .into_iter() - .map(|(name, optional)| EnvironmentDeclaration { + .map(|(name, optional)| RawEnvironmentDeclarationV10 { name: name.into(), - ty: EnvVarType::String, + ty: RawEnvVarTypeV10::String, optional, }) .collect(), diff --git a/crates/core/src/host/mod.rs b/crates/core/src/host/mod.rs index 01e4dee6309..d0a1f23c560 100644 --- a/crates/core/src/host/mod.rs +++ b/crates/core/src/host/mod.rs @@ -25,10 +25,9 @@ mod wasm_common; pub use disk_storage::DiskStorage; pub use host_controller::{ - extract_schema, BootstrapCompletion, CallProcedureReturn, CallResult, EnvironmentVersionConflict, - ExternalDurability, ExternalStorage, HostController, HostRuntimeConfig, InitialEnvironmentSource, - MigratePlanResult, ModuleHostWithBootstrap, ProcedureCallResult, ProgramStorage, ReducerCallResult, - ReducerCallResultWithTxOffset, ReducerOutcome, + extract_schema, BootstrapCompletion, CallProcedureReturn, CallResult, ExternalDurability, ExternalStorage, + HostController, HostRuntimeConfig, InitialEnvironmentSource, MigratePlanResult, ModuleHostWithBootstrap, + ProcedureCallResult, ProgramStorage, ReducerCallResult, ReducerCallResultWithTxOffset, ReducerOutcome, }; pub use module_host::{ InitDatabaseResult, ModuleHost, NoSuchModule, ProcedureCallError, ReducerCallError, UpdateDatabaseResult, diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 1e9b27af2e8..7439b80e375 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -1598,6 +1598,41 @@ impl UpdateDatabaseResult { } } +#[derive(Debug)] +pub enum UpdateEnvironmentResult { + NoUpdateNeeded, + UpdatePerformed { + /// The transaction offset of the successful database update. + tx_offset: TransactionOffset, + /// The durable transaction offset of the database. + /// `None` if the database is in-memory only. + durable_offset: Option, + }, + ErrorExecutingMigration(anyhow::Error), +} +impl UpdateEnvironmentResult { + /// Check if an environment update was successful. + pub fn was_successful(&self) -> bool { + matches!(self, Self::UpdatePerformed { .. }) + } +} + +impl From for UpdateDatabaseResult { + fn from(value: UpdateEnvironmentResult) -> Self { + match value { + UpdateEnvironmentResult::NoUpdateNeeded => Self::NoUpdateNeeded, + UpdateEnvironmentResult::UpdatePerformed { + tx_offset, + durable_offset, + } => Self::UpdatePerformed { + tx_offset, + durable_offset, + }, + UpdateEnvironmentResult::ErrorExecutingMigration(e) => Self::ErrorExecutingMigration(e), + } + } +} + #[derive(thiserror::Error, Debug)] #[error("no such module")] pub struct NoSuchModule; @@ -3278,6 +3313,19 @@ impl ModuleHost { )? } + pub async fn update_environment( + &self, + environment: std::collections::BTreeMap, + ) -> Result { + call_instance!( + self, + "", + environment, + |environment, inst| inst.update_environment(environment), + |environment, inst| inst.update_environment(environment).await, + )? + } + pub async fn exit(&self) { // As in `Self::marked_closed`, `Relaxed` is sufficient because we're not synchronizing any external state. self.closed.store(true, std::sync::atomic::Ordering::Relaxed); diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 90a83691fd8..2b7bd84cf5f 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -66,7 +66,7 @@ use self::syscall::{ use super::module_common::{build_common_module_from_raw, run_describer, ModuleCommon}; use super::module_host::{ CallHttpHandlerParams, CallProcedureParams, CallReducerParams, InstanceManagerMetrics, ModuleInfo, - ModuleWithInstance, + ModuleWithInstance, UpdateEnvironmentResult, }; use super::UpdateDatabaseResult; use crate::client::{ClientActorId, MeteredUnboundedReceiver, MeteredUnboundedSender}; @@ -489,6 +489,13 @@ impl JsMainInstance { .await } + pub async fn update_environment( + &self, + environment: std::collections::BTreeMap, + ) -> anyhow::Result { + self.request(UpdateEnvironmentRequest { environment }).await + } + pub async fn call_reducer(&self, params: CallReducerParams) -> ReducerCallResult { self.request(CallReducerRequest { params }).await } @@ -633,6 +640,12 @@ js_main_request! { } => "update_database", anyhow::Result, UpdateDatabase } +js_main_request! { + UpdateEnvironmentRequest { + environment: std::collections::BTreeMap, + } => "update_environment", anyhow::Result, UpdateEnvironment +} + js_main_request! { CallReducerRequest { params: CallReducerParams, @@ -817,6 +830,11 @@ enum JsMainWorkerRequest { policy: MigrationPolicy, environment: std::collections::BTreeMap, }, + /// See [`JsMainInstance::update_environment`]. + UpdateEnvironment { + reply_tx: JsReplyTx>, + environment: std::collections::BTreeMap, + }, /// See [`JsMainInstance::call_reducer`]. CallReducer { reply_tx: JsReplyTx, @@ -1416,6 +1434,12 @@ fn handle_main_worker_request( let res = instance_common.update_database(program, old_module_info, policy, environment, inst); (res, false) }), + JsMainWorkerRequest::UpdateEnvironment { reply_tx, environment } => { + handle_worker_request("update_environment", reply_tx, || { + let res = instance_common.update_environment(environment, inst); + (res, false) + }) + } JsMainWorkerRequest::CallReducer { reply_tx, params } => { handle_worker_request("call_reducer", reply_tx, || { let mut call_reducer = |tx, params| instance_common.call_reducer_with_tx(tx, params, inst); diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index 5ec1fea3047..f0ca26e0da8 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -12,8 +12,8 @@ use crate::host::module_common::{build_common_module_from_raw, ModuleCommon}; use crate::host::module_host::{ call_identity_connected, init_database, CallHttpHandlerParams, CallProcedureParams, CallReducerParams, CallViewParams, ClientConnectedError, DatabaseUpdate, EventStatus, HttpHandlerCallError, InitDatabaseResult, - ModuleEvent, ModuleFunctionCall, ModuleInfo, RefInstance, SqlCommand, SqlCommandResult, ViewCallResult, - ViewCommand, ViewCommandResult, ViewOutcome, + ModuleEvent, ModuleFunctionCall, ModuleInfo, RefInstance, SqlCommand, SqlCommandResult, UpdateEnvironmentResult, + ViewCallResult, ViewCommand, ViewCommandResult, ViewOutcome, }; use crate::host::scheduler::{CallScheduledFunctionResult, ScheduledFunctionParams}; use crate::host::{ @@ -505,6 +505,13 @@ impl WasmModuleInstance { .update_database(program, old_module_info, policy, environment, &mut self.instance) } + pub fn update_environment( + &mut self, + environment: std::collections::BTreeMap, + ) -> anyhow::Result { + self.common.update_environment(environment, &mut self.instance) + } + pub fn call_reducer(&mut self, params: CallReducerParams) -> ReducerCallResult { let (res, trapped) = self.call_reducer_with_tx(None, params); self.trapped = trapped; @@ -704,7 +711,7 @@ impl InstanceCommon { inst: &mut I, ) -> Result { if program.hash == old_module_info.module_hash { - return self.update_environment(environment, inst); + return self.update_environment(environment, inst).map(Into::into); } let replica_ctx = inst.replica_ctx().clone(); let system_logger = replica_ctx.logger.system_logger(); @@ -821,11 +828,11 @@ impl InstanceCommon { /// Apply an environment publication using the installed module instance. No /// initialization, migration, program replacement, or scheduler restart occurs. - fn update_environment( + pub(crate) fn update_environment( &mut self, environment: std::collections::BTreeMap, inst: &mut I, - ) -> anyhow::Result { + ) -> anyhow::Result { let replica_ctx = inst.replica_ctx().clone(); let db = replica_ctx.relational_db(); let tx = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); @@ -846,7 +853,7 @@ impl InstanceCommon { if trapped || out.outcome != ViewOutcome::Success { let (_, metrics, reducer) = db.rollback_mut_tx(out.tx); db.report_mut_tx_metrics(reducer, metrics, None); - return Ok(UpdateDatabaseResult::ErrorExecutingMigration(anyhow::anyhow!( + return Ok(UpdateEnvironmentResult::ErrorExecutingMigration(anyhow::anyhow!( "view evaluation failed during environment publication" ))); } @@ -865,7 +872,7 @@ impl InstanceCommon { let durable_offset = db.durable_tx_offset(); let CommitAndBroadcastEventSuccess { tx_offset, .. } = commit_and_broadcast_event(&self.info.subscriptions, None, event, out.tx); - Ok(UpdateDatabaseResult::UpdatePerformed { + Ok(UpdateEnvironmentResult::UpdatePerformed { tx_offset, durable_offset, }) @@ -2147,10 +2154,9 @@ mod tests { use spacetimedb_datastore::system_tables::{ModuleKind, ST_ENV_ID}; use spacetimedb_datastore::traits::Program; use spacetimedb_lib::db::raw_def::{ - v10::{RawModuleDefV10Builder, RawModuleDefV10Section}, + v10::{RawEnvVarTypeV10, RawEnvironmentDeclarationV10, RawModuleDefV10Builder, RawModuleDefV10Section}, v9::TableAccess, }; - use spacetimedb_lib::environment::{EnvVarType, EnvironmentDeclaration}; use spacetimedb_lib::identity::AuthCtx; use spacetimedb_schema::auto_migrate::ponder_migrate; use std::collections::BTreeMap; @@ -2182,12 +2188,13 @@ mod tests { .finish(); } let mut raw = builder.finish(); - raw.sections - .push(RawModuleDefV10Section::Environment(vec![EnvironmentDeclaration { + raw.sections.push(RawModuleDefV10Section::Environment(vec![ + RawEnvironmentDeclarationV10 { name: "TOKEN".into(), - ty: EnvVarType::String, + ty: RawEnvVarTypeV10::String, optional: false, - }])); + }, + ])); raw.try_into().expect("valid ENV view module") } diff --git a/crates/lib/src/db/raw_def/v10.rs b/crates/lib/src/db/raw_def/v10.rs index 5f966da8276..580baa1b41e 100644 --- a/crates/lib/src/db/raw_def/v10.rs +++ b/crates/lib/src/db/raw_def/v10.rs @@ -103,7 +103,7 @@ pub enum RawModuleDefV10Section { Submodules(Vec), /// Declared publish-only configuration. Even an empty section requires ENV support. - Environment(Vec), + Environment(Vec), } #[derive(Debug, Clone, SpacetimeType)] @@ -144,6 +144,24 @@ pub struct RawSubmoduleV10 { pub module: RawModuleDefV10, } +#[derive(Debug, Clone, crate::SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] +pub enum RawEnvVarTypeV10 { + String, + StringLiteral(String), + Union(Vec), +} + +#[derive(Debug, Clone, crate::SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] +pub struct RawEnvironmentDeclarationV10 { + pub name: String, + pub ty: RawEnvVarTypeV10, + pub optional: bool, +} + #[derive(Debug, Clone, Copy, Default, SpacetimeType)] #[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] #[sats(crate = crate)] @@ -733,27 +751,6 @@ impl RawModuleDefV10Builder { Default::default() } - /// Declare a complete environment schema, including an explicit empty schema. - /// Repeated calls remain repeated sections so host validation rejects ambiguity. - pub fn add_environment(&mut self, declarations: Vec) -> &mut Self { - self.module - .sections - .push(RawModuleDefV10Section::Environment(declarations)); - self - } - - /// New ENV-aware bindings declare an empty schema when no declaration is registered. - pub fn ensure_environment(&mut self) { - if !self - .module - .sections - .iter() - .any(|section| matches!(section, RawModuleDefV10Section::Environment(_))) - { - self.add_environment(Vec::new()); - } - } - /// Get mutable access to the typespace section, creating it if missing. fn typespace_mut(&mut self) -> &mut Typespace { let idx = self @@ -993,6 +990,26 @@ impl RawModuleDefV10Builder { } } + /// Get mutable access to the environment section, creating it if missing. + fn environment_mut(&mut self) -> &mut Vec { + let idx = self + .module + .sections + .iter() + .position(|s| matches!(s, RawModuleDefV10Section::Environment(_))) + .unwrap_or_else(|| { + self.module + .sections + .push(RawModuleDefV10Section::Environment(Vec::new())); + self.module.sections.len() - 1 + }); + + match &mut self.module.sections[idx] { + RawModuleDefV10Section::Environment(env) => env, + _ => unreachable!("Just ensured Environment section exists"), + } + } + /// Create a table builder. /// /// Does not validate that the product_type_ref is valid; this is left to the module validation code. @@ -1309,6 +1326,12 @@ impl RawModuleDefV10Builder { .push(RawModuleDefV10Section::CaseConversionPolicy(policy)); } + /// Declare a complete environment schema. + pub fn add_environment(&mut self, declarations: Vec) -> &mut Self { + self.environment_mut().extend(declarations); + self + } + /// Finish building, consuming the builder and returning the module. /// The module should be validated before use. pub fn finish(self) -> RawModuleDefV10 { diff --git a/crates/lib/src/environment.rs b/crates/lib/src/environment.rs index ad86b19fb58..0e450b75e29 100644 --- a/crates/lib/src/environment.rs +++ b/crates/lib/src/environment.rs @@ -1,5 +1,7 @@ //! Limits shared by the database environment store and its clients. +use crate::db::raw_def::v10::{RawEnvVarTypeV10, RawEnvironmentDeclarationV10}; + pub const MAX_ENV_KEY_BYTES: usize = 256; pub const MAX_ENV_VALUE_BYTES: usize = 8 * 1024; pub const MAX_ENV_VARS: usize = 256; @@ -18,7 +20,7 @@ impl std::fmt::Display for EnvironmentValidationError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(match self { Self::InvalidKey => "invalid POSIX environment variable name (maximum 256 bytes)", - Self::ValueTooLarge => "environment value exceeds 8192 UTF-8 bytes", + Self::ValueTooLarge => "environment value too large (maximum 8192 bytes)", Self::TooManyVariables => "environment store exceeds 256 variables", }) } @@ -90,103 +92,129 @@ pub struct EnvironmentDeclaration { pub optional: bool, } +impl From for RawEnvironmentDeclarationV10 { + fn from(decl: EnvironmentDeclaration) -> Self { + Self { + name: decl.name, + ty: decl.ty.into(), + optional: decl.optional, + } + } +} + +impl From for RawEnvVarTypeV10 { + fn from(ty: EnvVarType) -> Self { + match ty { + EnvVarType::String => Self::String, + EnvVarType::StringLiteral(s) => Self::StringLiteral(s), + EnvVarType::Union(ss) => Self::Union(ss), + } + } +} + +impl From for EnvironmentDeclaration { + fn from(decl: RawEnvironmentDeclarationV10) -> Self { + Self { + name: decl.name, + ty: decl.ty.into(), + optional: decl.optional, + } + } +} + +impl From for EnvVarType { + fn from(ty: RawEnvVarTypeV10) -> Self { + match ty { + RawEnvVarTypeV10::String => Self::String, + RawEnvVarTypeV10::StringLiteral(s) => Self::StringLiteral(s), + RawEnvVarTypeV10::Union(ss) => Self::Union(ss), + } + } +} + /// An environment schema whose declarations have passed host validation. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct EnvironmentSchema { declarations: std::collections::BTreeMap, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub enum EnvironmentSchemaErrorKind { +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub enum EnvironmentSchemaError { InvalidName, TooManyDeclarations, TooManyValues, - ConflictingUpdate, - DuplicateDeclaration, - EmptyUnion, - TooManyUnionEntries, - SchemaTooLarge, - LiteralTooLarge, - Undeclared, - MissingRequired, - ValueTooLarge, - ConstraintMismatch, -} - -/// Errors identify a key and rule, and never contain a supplied or allowed value. -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub struct EnvironmentSchemaError { - pub key: Option, - pub kind: EnvironmentSchemaErrorKind, + ConflictingUpdate { key: String }, + DuplicateDeclaration { key: String }, + EmptyUnion { key: String }, + TooManyUnionEntries { key: String }, + SchemaTooLarge { key: String }, + LiteralTooLarge { key: String }, + MissingRequired { key: String }, + ValueTooLarge { key: String }, + ConstraintMismatch { key: String }, } impl std::fmt::Display for EnvironmentSchemaError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - if let Some(key) = &self.key { - write!(f, "environment key {key:?}: ")?; - } - f.write_str(match self.kind { - EnvironmentSchemaErrorKind::InvalidName => "invalid name", - EnvironmentSchemaErrorKind::TooManyDeclarations => "too many declarations", - EnvironmentSchemaErrorKind::TooManyValues => "too many stored values", - EnvironmentSchemaErrorKind::ConflictingUpdate => "conflicting environment update operations", - EnvironmentSchemaErrorKind::DuplicateDeclaration => "duplicate declaration", - EnvironmentSchemaErrorKind::EmptyUnion => "string union must not be empty", - EnvironmentSchemaErrorKind::TooManyUnionEntries => "string union has too many entries", - EnvironmentSchemaErrorKind::SchemaTooLarge => "declaration schema exceeds size limit", - EnvironmentSchemaErrorKind::LiteralTooLarge => "declared literal exceeds value size limit", - EnvironmentSchemaErrorKind::Undeclared => "key is not declared", - EnvironmentSchemaErrorKind::MissingRequired => "required value is missing", - EnvironmentSchemaErrorKind::ValueTooLarge => "value exceeds size limit", - EnvironmentSchemaErrorKind::ConstraintMismatch => "value does not satisfy its declared string constraint", - }) + let (key, msg) = match self { + Self::InvalidName => return f.write_str("invalid name"), + Self::TooManyDeclarations => return f.write_str("too many declarations"), + Self::TooManyValues => return f.write_str("too many stored values"), + Self::ConflictingUpdate { key } => (key, "conflicting environment update operations"), + Self::DuplicateDeclaration { key } => (key, "duplicate declaration"), + Self::EmptyUnion { key } => (key, "string union must not be empty"), + Self::TooManyUnionEntries { key } => (key, "string union has too many entries"), + Self::SchemaTooLarge { key } => (key, "declaration schema exceeds size limit"), + Self::LiteralTooLarge { key } => (key, "declared literal exceeds value size limit"), + Self::MissingRequired { key } => (key, "required value is missing"), + Self::ValueTooLarge { key } => (key, "value exceeds size limit"), + Self::ConstraintMismatch { key } => (key, "value does not satisfy its declared string constraint"), + }; + + write!(f, "environment key {key:?}: {msg}") } } impl std::error::Error for EnvironmentSchemaError {} impl EnvironmentSchema { + pub const fn empty() -> Self { + Self { + declarations: std::collections::BTreeMap::new(), + } + } + fn validate_metadata(declarations: &[EnvironmentDeclaration]) -> Result<(), EnvironmentSchemaError> { - use EnvironmentSchemaErrorKind as Kind; if declarations.len() > MAX_ENV_VARS { - return Err(EnvironmentSchemaError { - key: None, - kind: Kind::TooManyDeclarations, - }); + return Err(EnvironmentSchemaError::TooManyDeclarations); } let mut bytes = 0usize; for declaration in declarations { // Never retain or format unvalidated key bytes in diagnostics. - validate_key(&declaration.name).map_err(|_| EnvironmentSchemaError { - key: None, - kind: Kind::InvalidName, - })?; - let error = |kind| EnvironmentSchemaError { - key: Some(declaration.name.clone()), - kind, - }; + validate_key(&declaration.name).map_err(|_| EnvironmentSchemaError::InvalidName)?; + let key = &declaration.name; bytes += declaration.name.len(); if bytes > MAX_ENV_SCHEMA_BYTES { - return Err(error(Kind::SchemaTooLarge)); + return Err(EnvironmentSchemaError::SchemaTooLarge { key: key.clone() }); } let literals = match &declaration.ty { EnvVarType::String => &[][..], EnvVarType::StringLiteral(value) => std::slice::from_ref(value), EnvVarType::Union(values) => { if values.is_empty() { - return Err(error(Kind::EmptyUnion)); + return Err(EnvironmentSchemaError::EmptyUnion { key: key.clone() }); } if values.len() > MAX_ENV_UNION_ENTRIES { - return Err(error(Kind::TooManyUnionEntries)); + return Err(EnvironmentSchemaError::TooManyUnionEntries { key: key.clone() }); } values.as_slice() } }; for value in literals { - validate_value(value).map_err(|_| error(Kind::LiteralTooLarge))?; + validate_value(value).map_err(|_| EnvironmentSchemaError::LiteralTooLarge { key: key.clone() })?; bytes += value.len(); if bytes > MAX_ENV_SCHEMA_BYTES { - return Err(error(Kind::SchemaTooLarge)); + return Err(EnvironmentSchemaError::SchemaTooLarge { key: key.clone() }); } } } @@ -205,10 +233,7 @@ impl EnvironmentSchema { values.dedup(); } if schema.declarations.contains_key(&declaration.name) { - return Err(EnvironmentSchemaError { - key: Some(declaration.name), - kind: EnvironmentSchemaErrorKind::DuplicateDeclaration, - }); + return Err(EnvironmentSchemaError::DuplicateDeclaration { key: declaration.name }); } schema.declarations.insert(declaration.name.clone(), declaration); } @@ -216,8 +241,8 @@ impl EnvironmentSchema { } /// Check bounds before cloning raw untrusted metadata into the validated schema. - pub fn from_declarations(declarations: &[EnvironmentDeclaration]) -> Result { - Self::validate_metadata(declarations)?; + pub fn from_declarations(declarations: Vec) -> Result { + Self::validate_metadata(&declarations)?; Self::new(declarations.to_vec()) } @@ -238,37 +263,20 @@ impl EnvironmentSchema { } /// Validate the complete resulting store, including required declarations. - pub fn validate_values( - &self, - values: &std::collections::BTreeMap, - ) -> Result<(), EnvironmentSchemaError> { + pub fn validate_values(&self, values: &EnvironmentMap) -> Result<(), EnvironmentSchemaError> { self.validate_supplied_values(values)?; self.validate_required(values) } /// Validate supplied values without requiring every required key in this input. /// Undeclared values are stored strings but are not readable by module code. - pub fn validate_supplied_values( - &self, - values: &std::collections::BTreeMap, - ) -> Result<(), EnvironmentSchemaError> { - use EnvironmentSchemaErrorKind as Kind; + pub fn validate_supplied_values(&self, values: &EnvironmentMap) -> Result<(), EnvironmentSchemaError> { if values.len() > MAX_ENV_VARS { - return Err(EnvironmentSchemaError { - key: None, - kind: Kind::TooManyValues, - }); + return Err(EnvironmentSchemaError::TooManyValues); } for (name, value) in values { - validate_key(name).map_err(|_| EnvironmentSchemaError { - key: None, - kind: Kind::InvalidName, - })?; - let error = |kind| EnvironmentSchemaError { - key: Some(name.clone()), - kind, - }; - validate_value(value).map_err(|_| error(Kind::ValueTooLarge))?; + validate_key(name).map_err(|_| EnvironmentSchemaError::InvalidName)?; + validate_value(value).map_err(|_| EnvironmentSchemaError::ValueTooLarge { key: name.clone() })?; let Some(declaration) = self.get(name) else { continue }; let matches = match &declaration.ty { EnvVarType::String => true, @@ -276,21 +284,17 @@ impl EnvironmentSchema { EnvVarType::Union(allowed) => allowed.binary_search(value).is_ok(), }; if !matches { - return Err(error(Kind::ConstraintMismatch)); + return Err(EnvironmentSchemaError::ConstraintMismatch { key: name.clone() }); } } Ok(()) } - fn validate_required( - &self, - values: &std::collections::BTreeMap, - ) -> Result<(), EnvironmentSchemaError> { + fn validate_required(&self, values: &EnvironmentMap) -> Result<(), EnvironmentSchemaError> { for declaration in self.declarations() { if !declaration.optional && !values.contains_key(&declaration.name) { - return Err(EnvironmentSchemaError { - key: Some(declaration.name.clone()), - kind: EnvironmentSchemaErrorKind::MissingRequired, + return Err(EnvironmentSchemaError::MissingRequired { + key: declaration.name.clone(), }); } } @@ -301,13 +305,23 @@ impl EnvironmentSchema { /// An environment mutation. Deliberately does not implement Debug because values are secrets. #[derive(Clone, Default)] pub struct EnvironmentUpdate { - pub values: std::collections::BTreeMap, - pub remove: Vec, - pub replace: bool, + pub values: EnvironmentMap, + pub remove: EnvironmentRemove, +} + +#[derive(Clone, Default, PartialEq, Debug)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum EnvironmentRemove { + #[default] + No, + All, + Keys(Vec), } -impl From> for EnvironmentUpdate { - fn from(values: std::collections::BTreeMap) -> Self { +pub type EnvironmentMap = std::collections::BTreeMap; + +impl From for EnvironmentUpdate { + fn from(values: EnvironmentMap) -> Self { Self { values, ..Self::default() @@ -318,31 +332,17 @@ impl From> for EnvironmentUpdate { impl EnvironmentUpdate { /// Validate operations before any mutation. Diagnostics never include values. pub fn validate(&self) -> Result<(), EnvironmentSchemaError> { - use EnvironmentSchemaErrorKind as Kind; EnvironmentSchema::default().validate_supplied_values(&self.values)?; - if self.remove.len() > MAX_ENV_VARS { - return Err(EnvironmentSchemaError { - key: None, - kind: Kind::TooManyValues, - }); - } - if self.replace && !self.remove.is_empty() { - return Err(EnvironmentSchemaError { - key: None, - kind: Kind::ConflictingUpdate, - }); - } - let mut seen = std::collections::BTreeSet::new(); - for key in &self.remove { - validate_key(key).map_err(|_| EnvironmentSchemaError { - key: None, - kind: Kind::InvalidName, - })?; - if self.values.contains_key(key) || !seen.insert(key) { - return Err(EnvironmentSchemaError { - key: Some(key.clone()), - kind: Kind::ConflictingUpdate, - }); + if let EnvironmentRemove::Keys(remove) = &self.remove { + if remove.len() > MAX_ENV_VARS { + return Err(EnvironmentSchemaError::TooManyValues); + } + let mut seen = std::collections::BTreeSet::new(); + for key in remove { + validate_key(key).map_err(|_| EnvironmentSchemaError::InvalidName)?; + if self.values.contains_key(key) || !seen.insert(key) { + return Err(EnvironmentSchemaError::ConflictingUpdate { key: key.clone() }); + } } } Ok(()) @@ -350,19 +350,19 @@ impl EnvironmentUpdate { /// Resolve the new store without mutating the old store. Schema validation follows /// against the deployed schema or the schema from the proposed new module. - pub fn resulting_values( - &self, - previous: &std::collections::BTreeMap, - ) -> Result, EnvironmentSchemaError> { + pub fn resulting_values(&self, previous: &EnvironmentMap) -> Result { self.validate()?; - let mut values = if self.replace { - Default::default() - } else { - previous.clone() + let mut values = match &self.remove { + EnvironmentRemove::No => previous.clone(), + EnvironmentRemove::All => EnvironmentMap::default(), + EnvironmentRemove::Keys(remove) => { + let mut values = previous.clone(); + for key in remove { + values.remove(key); + } + values + } }; - for key in &self.remove { - values.remove(key); - } values.extend(self.values.clone()); EnvironmentSchema::default().validate_supplied_values(&values)?; Ok(values) @@ -372,6 +372,7 @@ impl EnvironmentUpdate { #[cfg(test)] mod schema_tests { use super::*; + use std::assert_matches; use std::collections::BTreeMap; fn declaration(name: &str, ty: EnvVarType, optional: bool) -> EnvironmentDeclaration { @@ -399,18 +400,18 @@ mod schema_tests { values.insert("OPTIONAL".into(), "".into()); schema.validate_values(&values).unwrap(); values.insert("MODE".into(), "False".into()); - assert_eq!( - schema.validate_values(&values).unwrap_err().kind, - EnvironmentSchemaErrorKind::ConstraintMismatch + assert_matches!( + schema.validate_values(&values).unwrap_err(), + EnvironmentSchemaError::ConstraintMismatch { .. } ); values.remove("MODE"); - assert_eq!( - schema.validate_values(&values).unwrap_err().kind, - EnvironmentSchemaErrorKind::MissingRequired + assert_matches!( + schema.validate_values(&values).unwrap_err(), + EnvironmentSchemaError::MissingRequired { .. } ); values.insert("UNDECLARED".into(), "secret-marker".into()); let error = schema.validate_values(&values).unwrap_err(); - assert_eq!(error.kind, EnvironmentSchemaErrorKind::MissingRequired); + assert_matches!(error, EnvironmentSchemaError::MissingRequired { .. }); assert!(!format!("{error:?}: {error}").contains("secret-marker")); } @@ -427,19 +428,18 @@ mod schema_tests { assert_eq!(unchanged, old); schema.validate_values(&unchanged).unwrap(); let removed = EnvironmentUpdate { - remove: vec!["REQUIRED".into()], + remove: EnvironmentRemove::Keys(vec!["REQUIRED".into()]), ..Default::default() } .resulting_values(&old) .unwrap(); - assert_eq!( - schema.validate_values(&removed).unwrap_err().kind, - EnvironmentSchemaErrorKind::MissingRequired + assert_matches!( + schema.validate_values(&removed).unwrap_err(), + EnvironmentSchemaError::MissingRequired { .. } ); let replace = EnvironmentUpdate { - replace: true, + remove: EnvironmentRemove::All, values: BTreeMap::from([("REQUIRED".into(), "ready".into())]), - ..Default::default() } .resulting_values(&old) .unwrap(); @@ -451,9 +451,9 @@ mod schema_tests { false, )]) .unwrap(); - assert_eq!( - newly_declared.validate_values(&old).unwrap_err().kind, - EnvironmentSchemaErrorKind::ConstraintMismatch + assert_matches!( + newly_declared.validate_values(&old).unwrap_err(), + EnvironmentSchemaError::ConstraintMismatch { .. } ); let corrected = EnvironmentUpdate::from(BTreeMap::from([("UNUSED".into(), "other".into())])) .resulting_values(&old) @@ -465,31 +465,24 @@ mod schema_tests { #[test] fn mutation_conflicts_and_resulting_store_limit_are_rejected() { let stored = (0..MAX_ENV_VARS).map(|i| (format!("KEY{i}"), String::new())).collect(); - assert_eq!( + assert_matches!( EnvironmentUpdate::from(BTreeMap::from([("EXTRA".into(), String::new())])) .resulting_values(&stored) - .unwrap_err() - .kind, - EnvironmentSchemaErrorKind::TooManyValues + .unwrap_err(), + EnvironmentSchemaError::TooManyValues ); for update in [ - EnvironmentUpdate { - replace: true, - remove: vec!["KEY".into()], - ..Default::default() - }, EnvironmentUpdate { values: BTreeMap::from([("KEY".into(), "secret-marker".into())]), - remove: vec!["KEY".into()], - ..Default::default() + remove: EnvironmentRemove::Keys(vec!["KEY".into()]), }, EnvironmentUpdate { - remove: vec!["KEY".into(), "KEY".into()], + remove: EnvironmentRemove::Keys(vec!["KEY".into(), "KEY".into()]), ..Default::default() }, ] { let error = update.validate().unwrap_err(); - assert_eq!(error.kind, EnvironmentSchemaErrorKind::ConflictingUpdate); + assert_matches!(error, EnvironmentSchemaError::ConflictingUpdate { .. }); assert!(!error.to_string().contains("secret-marker")); } } @@ -497,35 +490,34 @@ mod schema_tests { #[test] fn declaration_limits_count_absent_optionals_and_reject_invalid_metadata() { let optional = declaration("A", EnvVarType::String, true); - assert_eq!( - EnvironmentSchema::new(vec![optional.clone(), optional]) - .unwrap_err() - .kind, - EnvironmentSchemaErrorKind::DuplicateDeclaration + assert_matches!( + EnvironmentSchema::new(vec![optional.clone(), optional]).unwrap_err(), + EnvironmentSchemaError::DuplicateDeclaration { .. } ); - assert_eq!( + assert_matches!( EnvironmentSchema::new( (0..=MAX_ENV_VARS) .map(|i| declaration(&format!("K{i}"), EnvVarType::String, true)) .collect() ) - .unwrap_err() - .kind, - EnvironmentSchemaErrorKind::TooManyDeclarations + .unwrap_err(), + EnvironmentSchemaError::TooManyDeclarations ); for (name, constraint, expected) in [ - ("A-B", EnvVarType::String, EnvironmentSchemaErrorKind::InvalidName), - ("A", EnvVarType::Union(vec![]), EnvironmentSchemaErrorKind::EmptyUnion), + ("A-B", EnvVarType::String, EnvironmentSchemaError::InvalidName), + ( + "A", + EnvVarType::Union(vec![]), + EnvironmentSchemaError::EmptyUnion { key: "A".into() }, + ), ( "A", EnvVarType::StringLiteral("x".repeat(MAX_ENV_VALUE_BYTES + 1)), - EnvironmentSchemaErrorKind::LiteralTooLarge, + EnvironmentSchemaError::LiteralTooLarge { key: "A".into() }, ), ] { assert_eq!( - EnvironmentSchema::new(vec![declaration(name, constraint, true)]) - .unwrap_err() - .kind, + EnvironmentSchema::new(vec![declaration(name, constraint, true)]).unwrap_err(), expected ); } @@ -539,7 +531,6 @@ mod schema_tests { fn raw_metadata_is_bounded_before_copying_or_formatting_untrusted_keys() { let invalid = format!("private-marker\n{}", "x".repeat(100_000)); let error = EnvironmentSchema::new(vec![declaration(&invalid, EnvVarType::String, true)]).unwrap_err(); - assert_eq!(error.key, None); assert!(!format!("{error:?}: {error}").contains("private-marker")); let error = EnvironmentSchema::new(vec![declaration( "A", @@ -547,13 +538,13 @@ mod schema_tests { true, )]) .unwrap_err(); - assert_eq!(error.kind, EnvironmentSchemaErrorKind::TooManyUnionEntries); + assert_matches!(error, EnvironmentSchemaError::TooManyUnionEntries { .. }); let error = EnvironmentSchema::new(vec![declaration( "A", EnvVarType::Union(vec!["x".repeat(MAX_ENV_VALUE_BYTES); MAX_ENV_UNION_ENTRIES]), true, )]) .unwrap_err(); - assert_eq!(error.kind, EnvironmentSchemaErrorKind::SchemaTooLarge); + assert_matches!(error, EnvironmentSchemaError::SchemaTooLarge { .. }); } } diff --git a/crates/schema/src/def.rs b/crates/schema/src/def.rs index 674f7d7afb3..87f61de936d 100644 --- a/crates/schema/src/def.rs +++ b/crates/schema/src/def.rs @@ -18,7 +18,6 @@ use std::collections::BTreeMap; use std::fmt::{self, Debug, Write}; use std::hash::Hash; -use std::sync::LazyLock; use crate::error::{IdentifierError, ValidationErrors}; use crate::identifier::{Identifier, NamespacePath, NamespacedIdentifier}; @@ -200,18 +199,12 @@ pub enum RawModuleDefVersion { impl ModuleDef { /// The validated root environment schema. Legacy modules have an empty schema. pub fn environment(&self) -> &EnvironmentSchema { - static EMPTY: LazyLock = LazyLock::new(EnvironmentSchema::default); match &self.environment { Some(schema) => schema, - None => &EMPTY, + None => const { &EnvironmentSchema::empty() }, } } - /// Whether the raw module explicitly required environment support. - pub fn environment_declared(&self) -> bool { - self.environment.is_some() - } - /// The raw module definition version this module was authored under. pub fn raw_module_def_version(&self) -> RawModuleDefVersion { self.raw_module_def_version @@ -1116,7 +1109,9 @@ impl From for RawModuleDefV10 { let mut sections = Vec::new(); if let Some(environment) = environment { - sections.push(RawModuleDefV10Section::Environment(environment.into_declarations())); + sections.push(RawModuleDefV10Section::Environment( + environment.into_declarations().into_iter().map(|x| x.into()).collect(), + )); } let mut explicit_names = ExplicitNames::default(); diff --git a/crates/schema/src/def/validate/v10.rs b/crates/schema/src/def/validate/v10.rs index 6d6f7426668..e7d23b28cd9 100644 --- a/crates/schema/src/def/validate/v10.rs +++ b/crates/schema/src/def/validate/v10.rs @@ -350,18 +350,15 @@ pub fn validate(def: RawModuleDefV10) -> Result { } fn validate_environment(def: &RawModuleDefV10) -> Result> { - let mut sections = def.sections.iter().filter_map(|section| match section { + let Some(declarations) = def.sections.iter().find_map(|section| match section { RawModuleDefV10Section::Environment(declarations) => Some(declarations), _ => None, - }); - let Some(declarations) = sections.next() else { + }) else { return Ok(None); }; - if sections.next().is_some() { - return Err(ValidationError::RepeatedEnvironmentDeclaration.into()); - } + let declarations = declarations.iter().map(|x| x.clone().into()).collect(); let schema = spacetimedb_lib::environment::EnvironmentSchema::from_declarations(declarations) - .map_err(|error| ValidationError::Environment { error })?; + .map_err(ValidationError::from)?; Ok(Some(schema)) } @@ -3064,33 +3061,21 @@ mod tests { #[cfg(test)] mod environment_tests { use super::*; - use spacetimedb_lib::environment::{EnvVarType, EnvironmentDeclaration}; fn declared(name: &str) -> RawModuleDefV10 { RawModuleDefV10 { - sections: vec![RawModuleDefV10Section::Environment(vec![EnvironmentDeclaration { - name: name.into(), - ty: EnvVarType::String, - optional: true, - }])], + sections: vec![RawModuleDefV10Section::Environment(vec![ + RawEnvironmentDeclarationV10 { + name: name.into(), + ty: RawEnvVarTypeV10::String, + optional: true, + }, + ])], } } #[test] - fn environment_schema_round_trip_preserves_explicit_empty_and_exact_keys() { - let legacy = validate(RawModuleDefV10::default()).unwrap(); - assert!(legacy.environment().is_empty()); - assert!(!legacy.environment_declared()); - let raw: RawModuleDefV10 = legacy.into(); - assert!(!validate(raw).unwrap().environment_declared()); - let explicit = validate(RawModuleDefV10 { - sections: vec![RawModuleDefV10Section::Environment(vec![])], - }) - .unwrap(); - assert!(explicit.environment().is_empty()); - assert!(explicit.environment_declared()); - let raw: RawModuleDefV10 = explicit.into(); - assert!(validate(raw).unwrap().environment_declared()); + fn environment_schema_round_trip_preserves_exact_keys() { let module = validate(declared("Mixed_CASE")).unwrap(); assert!(module.environment().get("Mixed_CASE").is_some()); assert!(module.environment().get("mixed_case").is_none()); @@ -3104,12 +3089,6 @@ mod environment_tests { #[test] fn environment_rejects_ambiguous_sections_and_nested_declarations() { - let mut duplicate = declared("A"); - duplicate.sections.push(RawModuleDefV10Section::Environment(vec![])); - assert!(validate(duplicate) - .unwrap_err() - .into_iter() - .any(|error| matches!(error, ValidationError::RepeatedEnvironmentDeclaration))); let nested = RawModuleDefV10 { sections: vec![RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { namespace: "outer".into(), diff --git a/crates/schema/src/describe.rs b/crates/schema/src/describe.rs index ead8deb4ed2..c4adeeb005b 100644 --- a/crates/schema/src/describe.rs +++ b/crates/schema/src/describe.rs @@ -992,8 +992,8 @@ mod tests { use super::*; use crate::identifier::Identifier; use spacetimedb_lib::db::raw_def::v10::{ - CaseConversionPolicy, FunctionVisibility as RawFunctionVisibility, RawModuleDefV10Builder, - RawModuleDefV10Section, RawSubmoduleV10, + CaseConversionPolicy, FunctionVisibility as RawFunctionVisibility, RawEnvVarTypeV10, + RawEnvironmentDeclarationV10, RawModuleDefV10Builder, RawModuleDefV10Section, RawSubmoduleV10, }; use spacetimedb_lib::db::raw_def::v9::{btree, direct, hash}; use spacetimedb_lib::{ProductType, ScheduleAt}; @@ -1364,16 +1364,16 @@ mod tests { // Declared after `/webhook`, so the output shows declaration order is kept. builder.add_http_route("health", MethodOrAny::Any, "/health"); builder.add_environment(vec![ - env_declaration("API_KEY", EnvVarType::String, false), + env_declaration("API_KEY", RawEnvVarTypeV10::String, false), env_declaration( "MODE", - EnvVarType::Union(vec!["production".into(), "development".into()]), + RawEnvVarTypeV10::Union(vec!["production".into(), "development".into()]), false, ), - env_declaration("REGION", EnvVarType::StringLiteral("eu west".into()), true), + env_declaration("REGION", RawEnvVarTypeV10::StringLiteral("eu west".into()), true), env_declaration( "LOG_LEVEL", - EnvVarType::Union(vec!["info".into(), "debug".into()]), + RawEnvVarTypeV10::Union(vec!["info".into(), "debug".into()]), true, ), ]); @@ -1414,8 +1414,8 @@ mod tests { .expect("the describe fixture should be a valid module definition") } - fn env_declaration(name: &str, ty: EnvVarType, optional: bool) -> EnvironmentDeclaration { - EnvironmentDeclaration { + fn env_declaration(name: &str, ty: RawEnvVarTypeV10, optional: bool) -> RawEnvironmentDeclarationV10 { + RawEnvironmentDeclarationV10 { name: name.into(), ty, optional, diff --git a/crates/schema/src/error.rs b/crates/schema/src/error.rs index eb16c0d2a99..450d4590708 100644 --- a/crates/schema/src/error.rs +++ b/crates/schema/src/error.rs @@ -22,14 +22,6 @@ pub type ValidationErrors = ErrorStream; #[derive(thiserror::Error, Debug, PartialOrd, Ord, PartialEq, Eq)] #[non_exhaustive] pub enum ValidationError { - #[error("module has repeated environment declarations")] - RepeatedEnvironmentDeclaration, - #[error("invalid environment declaration: {error}")] - Environment { - error: spacetimedb_lib::environment::EnvironmentSchemaError, - }, - #[error("submodule {namespace:?} cannot declare environment variables")] - EnvironmentInSubmodule { namespace: String }, #[error("name `{name}` is used for multiple entities")] DuplicateName { name: RawIdentifier }, #[error("name `{name}` is used for multiple types")] @@ -191,6 +183,10 @@ pub enum ValidationError { LifecycleInSubmodule { lifecycle: Lifecycle, namespace: String }, #[error("submodule namespace `{namespace}` is {len} bytes, which exceeds the 63-byte limit")] NamespaceTooLong { namespace: RawIdentifier, len: usize }, + #[error("invalid environment declaration: {0}")] + Environment(#[from] spacetimedb_lib::environment::EnvironmentSchemaError), + #[error("submodule {namespace:?} cannot declare environment variables")] + EnvironmentInSubmodule { namespace: String }, } /// A wrapper around an `AlgebraicType` that implements `fmt::Display`. diff --git a/crates/smoketests/tests/standalone/cli/environment.rs b/crates/smoketests/tests/standalone/cli/environment.rs index 8b848ede0a5..ec9ffd9c9d3 100644 --- a/crates/smoketests/tests/standalone/cli/environment.rs +++ b/crates/smoketests/tests/standalone/cli/environment.rs @@ -1,9 +1,10 @@ //! Publish-only environment configuration through the real CLI and local server. use serde_json::{json, Value}; use spacetimedb_guard::ensure_binaries_built; +use spacetimedb_lib::Hash; use spacetimedb_smoketests::{modules, random_string, require_local_server, Smoketest}; use std::{ - fs, + assert_matches, fs, io::{Read as _, Seek as _}, path::PathBuf, process::{Child, Command, Output, Stdio}, @@ -474,17 +475,13 @@ fn cli_environment_preservation_and_environment_only_updates() { .timeout(Duration::from_secs(20)) .build() .unwrap(); - let url = format!("{}/v1/database/{}", f.test.server_url, f.database); - let denied = client.get(format!("{url}/environment")).send().unwrap(); - assert!(matches!(denied.status().as_u16(), 401 | 403)); + let url = format!("{}/v1/database/{}/environment", f.test.server_url, f.database); + let denied = client.get(&url).send().unwrap(); + assert_matches!(denied.status().as_u16(), 401 | 403); // This config and token were created by the fixture's isolated local login. let config: toml::Value = toml::from_str(&fs::read_to_string(&f.test.config_path).unwrap()).unwrap(); let token = config["spacetimedb_token"].as_str().unwrap(); - let metadata = client - .get(format!("{url}/environment")) - .bearer_auth(token) - .send() - .unwrap(); + let metadata = client.get(&url).bearer_auth(token).send().unwrap(); assert!(metadata.status().is_success()); assert_eq!(metadata.headers()["cache-control"], "no-store"); let metadata = metadata.text().unwrap(); @@ -494,19 +491,19 @@ fn cli_environment_preservation_and_environment_only_updates() { .as_array() .unwrap() .contains(&json!("SMOKE_REQUIRED"))); - for (body, status) in [ + for (body, query, status) in [ ( - json!({"environment":{"SMOKE_REQUIRED":"stale-replacement"},"expected_module_version":"00".repeat(32)}), + json!({"SMOKE_REQUIRED":"stale-replacement"}), + &[("expected_module_hash", Hash::from_hex("00".repeat(32)).unwrap())][..], 409, ), - (json!({"environment":{"SMOKE_REQUIRED":"missing-version"}}), 400), - (json!({"module":""}), 400), + (json!({"SMOKE_REQUIRED":"missing-version"}), &[], 400), ] { let response = client .put(&url) + .query(query) .bearer_auth(token) - .header("Content-Type", "application/vnd.spacetimedb.publish+json") - .body(body.to_string()) + .json(&body) .send() .unwrap(); assert_eq!(response.status().as_u16(), status); diff --git a/crates/standalone/src/control_db/environment.rs b/crates/standalone/src/control_db/environment.rs index ff15abf2a0a..bad48553a3b 100644 --- a/crates/standalone/src/control_db/environment.rs +++ b/crates/standalone/src/control_db/environment.rs @@ -17,9 +17,8 @@ //! initialization is durable, but is currently retained until reset replaces it //! or database deletion removes it atomically with both indexes and the binding. use super::*; -use spacetimedb_client_api_messages::publish::PublishRequest; +use spacetimedb_lib::environment::EnvironmentMap; use spacetimedb_lib::Hash; -use std::collections::BTreeMap; // Keep the existing tree name for on-disk compatibility. const METADATA_TREE: &str = "database_bootstrap"; @@ -85,7 +84,7 @@ impl ControlDb { } /// Recheck the exact persisted generation and program before releasing any values. - pub(crate) fn initial_environment(&self, database: &Database, replica_id: u64) -> Result> { + pub(crate) fn initial_environment(&self, database: &Database, replica_id: u64) -> Result { let databases = self.db.open_tree("database_by_identity")?; let metadata = self.db.open_tree(METADATA_TREE)?; let values = self.db.open_tree(VALUES_TREE)?; @@ -149,14 +148,8 @@ impl ControlDb { }); let bytes = result.map_err(transaction_error)?; match bytes { - None => Ok(BTreeMap::new()), - Some(bytes) => { - let request = PublishRequest::decode(&bytes).map_err(|_| invalid())?; - if request.module.as_ref().is_some_and(|module| !module.is_empty()) { - return Err(invalid()); - } - Ok(request.environment) - } + None => Ok(Default::default()), + Some(bytes) => serde_json::from_slice(&bytes).map_err(|_| invalid()), } } @@ -171,7 +164,7 @@ impl ControlDb { &self, mut database: Database, expected: Option<&Database>, - environment: BTreeMap, + environment: EnvironmentMap, previous_replicas: &[Replica], ) -> Result<(Database, Replica)> { if expected.is_none() { @@ -186,13 +179,7 @@ impl ControlDb { node_id: 0, leader: true, }; - let input = PublishRequest { - module: None, - environment, - ..Default::default() - } - .encode() - .map_err(|_| invalid())?; + let input = serde_json::to_vec(&environment).map_err(|_| invalid())?; let binding = InitializationMetadata { version: 1, database_id: database.id, @@ -300,7 +287,7 @@ fn transaction_error(error: TransactionError) -> Error { #[async_trait::async_trait] impl spacetimedb::host::InitialEnvironmentSource for ControlDb { - async fn load(&self, database: &Database, replica_id: u64) -> anyhow::Result> { + async fn load(&self, database: &Database, replica_id: u64) -> anyhow::Result { let source = self.clone(); let database = database.clone(); spacetimedb::util::asyncify(move || { @@ -314,6 +301,8 @@ impl spacetimedb::host::InitialEnvironmentSource for ControlDb { #[cfg(test)] mod tests { + use std::collections::BTreeMap; + use super::*; use spacetimedb::messages::control_db::HostType; diff --git a/crates/standalone/src/lib.rs b/crates/standalone/src/lib.rs index 4a7479cd6a8..8047eed7c87 100644 --- a/crates/standalone/src/lib.rs +++ b/crates/standalone/src/lib.rs @@ -14,6 +14,7 @@ use spacetimedb::config::{CertificateAuthority, MetadataFile, ModuleHttpConfig, use spacetimedb::db; use spacetimedb::db::persistence::{DurabilityConfig, LocalPersistenceProvider}; use spacetimedb::energy::{EnergyBalance, EnergyQuanta, NullEnergyMonitor}; +use spacetimedb::host::module_host::UpdateEnvironmentResult; use spacetimedb::host::{DiskStorage, HostController, HostRuntimeConfig, MigratePlanResult, UpdateDatabaseResult}; use spacetimedb::identity::{AuthCtx, Identity}; use spacetimedb::messages::control_db::{Database, HostType, Node, Replica}; @@ -30,6 +31,8 @@ use spacetimedb_client_api_messages::name::{ use spacetimedb_datastore::db_metrics::data_size::DATA_SIZE_METRICS; use spacetimedb_datastore::db_metrics::DB_METRICS; use spacetimedb_datastore::traits::Program; +use spacetimedb_lib::environment::{EnvironmentMap, EnvironmentUpdate}; +use spacetimedb_lib::Hash; use spacetimedb_paths::server::{ModuleLogsDir, PidFile, ServerDataDir}; use spacetimedb_paths::standalone::StandaloneDataDirExt; use spacetimedb_schema::auto_migrate::{MigrationPolicy, PrettyPrintStyle}; @@ -277,26 +280,18 @@ impl spacetimedb_client_api::ControlStateWriteAccess for StandaloneEnv { publisher: &Identity, spec: spacetimedb_client_api::DatabaseDef, policy: MigrationPolicy, + environment: EnvironmentUpdate, ) -> anyhow::Result> { let existing_db = self.control_db.get_database_by_identity(&spec.database_identity)?; - let update = spacetimedb_lib::environment::EnvironmentUpdate { - values: spec.environment, - remove: spec.environment_remove, - replace: spec.environment_replace, - }; - update.validate()?; // standalone does not support replication. let num_replicas = 1; match existing_db { // The database does not already exist, so we'll create it. None => { - anyhow::ensure!( - !spec.program_bytes.is_empty() && spec.expected_module_version.is_none(), - "initial publication requires a module and cannot require an existing version" - ); - let environment = update.resulting_values(&Default::default())?; + anyhow::ensure!(!spec.program_bytes.is_empty(), "initial publication requires a module"); + let environment = environment.values; let program = Program::from_bytes(spec.host_type.into(), &spec.program_bytes[..]); let database = Database { @@ -347,51 +342,12 @@ impl spacetimedb_client_api::ControlStateWriteAccess for StandaloneEnv { spec.host_type, spec.program_bytes.to_vec().into(), policy, - update, - spec.expected_module_version, + environment, ) .await?; if update_result.was_successful() { - let replicas = self.control_db.get_replicas_by_database(database_id)?; - let desired_replicas = num_replicas as usize; - if desired_replicas == 0 { - log::info!("Decommissioning all replicas of database {database_identity}"); - for instance in replicas { - self.delete_replica(instance.id).await?; - } - } else if desired_replicas > replicas.len() { - let n = desired_replicas - replicas.len(); - log::info!( - "Scaling up database {} from {} to {} replicas", - database_identity, - replicas.len(), - n - ); - for _ in 0..n { - self.insert_replica(Replica { - id: 0, - database_id, - node_id: 0, - leader: false, - }) - .await?; - } - } else if desired_replicas < replicas.len() { - let n = replicas.len() - desired_replicas; - log::info!( - "Scaling down database {} from {} to {} replicas", - database_identity, - replicas.len(), - n - ); - for instance in replicas.into_iter().filter(|instance| !instance.leader).take(n) { - self.delete_replica(instance.id).await?; - } - } else { - log::debug!( - "Desired replica count {desired_replicas} for database {database_identity} already satisfied" - ); - } + self.scale_replicas(database_id, &database_identity, num_replicas) + .await?; } anyhow::Ok(Some(update_result)) @@ -443,7 +399,12 @@ impl spacetimedb_client_api::ControlStateWriteAccess for StandaloneEnv { Ok(()) } - async fn reset_database(&self, caller_identity: &Identity, spec: DatabaseResetDef) -> anyhow::Result<()> { + async fn reset_database( + &self, + caller_identity: &Identity, + spec: DatabaseResetDef, + environment: EnvironmentMap, + ) -> anyhow::Result<()> { let previous = self .control_db .get_database_by_identity(&spec.database_identity)? @@ -453,12 +414,6 @@ impl spacetimedb_client_api::ControlStateWriteAccess for StandaloneEnv { "database ownership changed before reset" ); let previous = self.control_db.with_initialization_generation(previous)?; - let environment = spacetimedb_lib::environment::EnvironmentUpdate { - values: spec.environment, - remove: spec.environment_remove, - replace: spec.environment_replace, - } - .resulting_values(&Default::default())?; let mut database = previous.clone(); let program = match spec.program_bytes { Some(bytes) => { @@ -550,6 +505,86 @@ impl spacetimedb_client_api::ControlStateWriteAccess for StandaloneEnv { self.control_db.set_database_lock(database_identity, locked)?; Ok(()) } + + async fn update_environment( + &self, + publisher: &Identity, + database_identity: &Identity, + environment: EnvironmentUpdate, + expected_module_hash: Hash, + ) -> anyhow::Result { + let Some(database) = self.control_db.get_database_by_identity(database_identity)? else { + anyhow::bail!("Database not found: {}", database_identity.to_abbreviated_hex()); + }; + + // standalone does not support replication. + let num_replicas = 1; + + anyhow::ensure!( + database.owner_identity == *publisher, + "database ownership changed before publication" + ); + let database_id = database.id; + + let leader = self.leader(database_id).await?; + let update_result = leader + .update_environment(database, environment, expected_module_hash) + .await?; + if update_result.was_successful() { + self.scale_replicas(database_id, database_identity, num_replicas) + .await?; + } + + anyhow::Ok(update_result) + } +} + +impl StandaloneEnv { + async fn scale_replicas( + &self, + database_id: u64, + database_identity: &Identity, + desired_replicas: usize, + ) -> anyhow::Result<()> { + let replicas = self.control_db.get_replicas_by_database(database_id)?; + if desired_replicas == 0 { + log::info!("Decommissioning all replicas of database {database_identity}"); + for instance in replicas { + self.delete_replica(instance.id).await?; + } + } else if desired_replicas > replicas.len() { + let n = desired_replicas - replicas.len(); + log::info!( + "Scaling up database {} from {} to {} replicas", + database_identity, + replicas.len(), + n + ); + for _ in 0..n { + self.insert_replica(Replica { + id: 0, + database_id, + node_id: 0, + leader: false, + }) + .await?; + } + } else if desired_replicas < replicas.len() { + let n = replicas.len() - desired_replicas; + log::info!( + "Scaling down database {} from {} to {} replicas", + database_identity, + replicas.len(), + n + ); + for instance in replicas.into_iter().filter(|instance| !instance.leader).take(n) { + self.delete_replica(instance.id).await?; + } + } else { + log::debug!("Desired replica count {desired_replicas} for database {database_identity} already satisfied"); + } + Ok(()) + } } impl spacetimedb_client_api::Authorization for StandaloneEnv { diff --git a/crates/testing/src/modules.rs b/crates/testing/src/modules.rs index 5f7c2b23db9..6838775d722 100644 --- a/crates/testing/src/modules.rs +++ b/crates/testing/src/modules.rs @@ -16,6 +16,7 @@ use spacetimedb::util::jobs::JobCores; use spacetimedb::Identity; use spacetimedb_client_api::auth::SpacetimeAuth; use spacetimedb_client_api::routes::subscribe::{generate_random_connection_id, WebSocketOptions}; +use spacetimedb_lib::environment::{EnvironmentRemove, EnvironmentUpdate}; use spacetimedb_lib::http as st_http; use spacetimedb_lib::AlgebraicValue; use spacetimedb_paths::{RootDir, SpacetimePaths}; @@ -96,16 +97,16 @@ impl ModuleHandle { DatabaseDef { database_identity: self.db_identity, program_bytes, - environment, - environment_remove: Vec::new(), - environment_replace: true, - expected_module_version: None, num_replicas: None, host_type, parent: None, organization: None, }, MigrationPolicy::Compatible, + EnvironmentUpdate { + values: environment, + remove: EnvironmentRemove::All, + }, ) .await? .ok_or_else(|| anyhow::anyhow!("expected an update to the existing database")) @@ -423,16 +424,13 @@ impl CompiledModule { DatabaseDef { database_identity: db_identity, program_bytes: self.program_bytes(), - environment, - environment_remove: Vec::new(), - environment_replace: false, - expected_module_version: None, num_replicas: None, host_type: self.host_type, parent: None, organization: None, }, MigrationPolicy::Compatible, + environment.into(), ) .await .unwrap(); diff --git a/crates/testing/tests/environment_lifecycle.rs b/crates/testing/tests/environment_lifecycle.rs index ab945e4455c..4449946f27e 100644 --- a/crates/testing/tests/environment_lifecycle.rs +++ b/crates/testing/tests/environment_lifecycle.rs @@ -10,6 +10,7 @@ use spacetimedb_client_api::routes::subscribe::WebSocketOptions; use spacetimedb_client_api::{ ControlStateReadAccess as _, ControlStateWriteAccess as _, DatabaseDef, DatabaseResetDef, NodeDelegate as _, }; +use spacetimedb_lib::environment::{EnvironmentRemove, EnvironmentUpdate}; use spacetimedb_lib::{bsatn, sats::product, AlgebraicValue, Identity}; use spacetimedb_paths::cli::{PrivKeyPath, PubKeyPath}; use spacetimedb_paths::{server::ServerDataDir, FromPathUnchecked}; @@ -67,13 +68,9 @@ fn real_module_reopen_and_environment_only_publication_preserve_values() -> anyh ("REQUIRED".into(), "initial-required".into()), ("MODE".into(), "ready".into()), ]); - let spec = |environment| DatabaseDef { + let spec = || DatabaseDef { database_identity: Identity::ZERO, program_bytes: bytes.clone(), - environment, - environment_remove: Vec::new(), - environment_replace: false, - expected_module_version: None, num_replicas: None, host_type: HostType::Wasm, parent: None, @@ -83,7 +80,12 @@ fn real_module_reopen_and_environment_only_publication_preserve_values() -> anyh assert!(env .as_ref() .unwrap() - .publish_database(&Identity::ZERO, spec(initial.clone()), MigrationPolicy::Compatible) + .publish_database( + &Identity::ZERO, + spec(), + MigrationPolicy::Compatible, + initial.clone().into() + ) .await? .is_none()); let database = env @@ -100,8 +102,7 @@ fn real_module_reopen_and_environment_only_publication_preserve_values() -> anyh .unwrap(); log::info!("ENV standalone fixture: rejected publication preserves live host"); for changed_program in [false, true] { - let mut invalid = spec(Values::new()); - invalid.environment_replace = true; + let mut invalid = spec(); if changed_program { // An empty custom section changes the Wasm hash without changing // its declarations, exercising rejection of a candidate module. @@ -111,9 +112,15 @@ fn real_module_reopen_and_environment_only_publication_preserve_values() -> anyh } let rejected = tokio::time::timeout( std::time::Duration::from_secs(30), - env.as_ref() - .unwrap() - .publish_database(&Identity::ZERO, invalid, MigrationPolicy::Compatible), + env.as_ref().unwrap().publish_database( + &Identity::ZERO, + invalid, + MigrationPolicy::Compatible, + EnvironmentUpdate { + values: Values::new(), + remove: EnvironmentRemove::All, + }, + ), ) .await .expect("rejected publication must not hang"); @@ -128,21 +135,27 @@ fn real_module_reopen_and_environment_only_publication_preserve_values() -> anyh assert!(matches!( env.as_ref() .unwrap() - .publish_database(&Identity::ZERO, spec(Values::new()), MigrationPolicy::Compatible) + .publish_database( + &Identity::ZERO, + spec(), + MigrationPolicy::Compatible, + Values::new().into() + ) .await?, Some(UpdateDatabaseResult::NoUpdateNeeded) )); let previous_module = env.as_ref().unwrap().leader(database.id).await?.module().await?; let version = previous_module.info.module_hash; - let mut env_only = spec(Values::from([("FUTURE".into(), "undeclared".into())])); - env_only.program_bytes = Default::default(); - env_only.expected_module_version = Some(version); assert!(env .as_ref() .unwrap() - .publish_database(&Identity::ZERO, env_only, MigrationPolicy::Compatible) + .update_environment( + &Identity::ZERO, + &Identity::ZERO, + Values::from([("FUTURE".into(), "undeclared".into())]).into(), + version + ) .await? - .unwrap() .was_successful()); let current = env.as_ref().unwrap().leader(database.id).await?.module().await?; assert!(Arc::ptr_eq(&previous_module.info, ¤t.info)); @@ -159,13 +172,16 @@ fn real_module_reopen_and_environment_only_publication_preserve_values() -> anyh AlgebraicValue::from(Some("initial-required".to_owned())) ); assert!(read(env.as_ref().unwrap(), database.id, "FUTURE").await.is_err()); - let mut stale = spec(Values::new()); - stale.program_bytes = Default::default(); - stale.expected_module_version = Some(spacetimedb_lib::Hash::from_hex("00".repeat(32))?); + let expected_module_hash = spacetimedb_lib::Hash::from_hex("00".repeat(32))?; assert!(env .as_ref() .unwrap() - .publish_database(&Identity::ZERO, stale, MigrationPolicy::Compatible) + .update_environment( + &Identity::ZERO, + &Identity::ZERO, + Values::new().into(), + expected_module_hash + ) .await .is_err()); log::info!("ENV standalone fixture: same-program update"); @@ -174,7 +190,7 @@ fn real_module_reopen_and_environment_only_publication_preserve_values() -> anyh assert!(env .as_ref() .unwrap() - .publish_database(&Identity::ZERO, spec(updated), MigrationPolicy::Compatible) + .publish_database(&Identity::ZERO, spec(), MigrationPolicy::Compatible, updated.into()) .await? .unwrap() .was_successful()); @@ -193,15 +209,16 @@ fn real_module_reopen_and_environment_only_publication_preserve_values() -> anyh drop(env.take()); env = Some(StandaloneEnv::init(config, &ca, data_dir.clone(), JobCores::without_pinned_cores()).await?); // The first request after restart is env-only, before leader lookup has started a host. - let mut cold = spec(Values::from([("EMPTY".into(), "after-restart".into())])); - cold.program_bytes = Default::default(); - cold.expected_module_version = Some(version); assert!(env .as_ref() .unwrap() - .publish_database(&Identity::ZERO, cold, MigrationPolicy::Compatible) + .update_environment( + &Identity::ZERO, + &Identity::ZERO, + Values::from([("EMPTY".into(), "after-restart".into())]).into(), + version + ) .await? - .unwrap() .was_successful()); assert_eq!( read(env.as_ref().unwrap(), database.id, "EMPTY").await?, @@ -221,12 +238,10 @@ fn real_module_reopen_and_environment_only_publication_preserve_values() -> anyh DatabaseResetDef { database_identity: Identity::ZERO, program_bytes: None, - environment_remove: Default::default(), - environment_replace: false, - environment: Values::new(), num_replicas: None, host_type: None, - } + }, + Default::default() ) .await .is_err()); @@ -244,12 +259,10 @@ fn real_module_reopen_and_environment_only_publication_preserve_values() -> anyh DatabaseResetDef { database_identity: Identity::ZERO, program_bytes: None, - environment_remove: Default::default(), - environment_replace: false, - environment: reset, num_replicas: None, host_type: None, }, + reset, ) .await?; assert_ne!( diff --git a/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md b/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md index 4286906ee01..58de714009b 100644 --- a/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md +++ b/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md @@ -9,7 +9,7 @@ The HTTP endpoints in `/v1/database` allow clients to interact with Spacetime da ## At a glance | Route | Description | -|----------------------------------------------------------------------------------------------------|---------------------------------------------------| +| -------------------------------------------------------------------------------------------------- | ------------------------------------------------- | | [`POST /v1/database`](#post-v1database) | Publish a new database given its module code. | | [`PUT /v1/database/:name_or_identity`](#put-v1databasename_or_identity) | Publish to a database given its module code. | | [`GET /v1/database/:name_or_identity`](#get-v1databasename_or_identity) | Get a JSON description of a database. | @@ -114,15 +114,15 @@ Both publish endpoints accept `Content-Type: application/vnd.spacetimedb.publish `module` uses standard padded Base64. `environment` supplies string overrides for declared or undeclared names. Unspecified stored values survive by default. The server validates the resulting environment against the module's declarations and installs both in one transaction: all required values must exist and all present declared values must satisfy their constraints. An empty or omitted map preserves stored values, including when publishing unchanged module bytes. -A missing required value returns HTTP 400 with `Content-Type: application/json` and `{"error":"missing_required_environment","key":"API_KEY"}` identifying the missing key. Clients can prompt for that value and retry the publish. Other validation and module failures do not use this error code. +A missing required value returns HTTP 400 with `Content-Type: application/json` and `{"MissingRequiredEnvironment":{"key":"API_KEY"}}` identifying the missing key. Clients can prompt for that value and retry the publish. Other validation and module failures do not use this error code. Optional fields `environment_remove` (an array of keys) and `environment_replace` (a boolean, default `false`) request explicit deletion or complete replacement. A key cannot be both supplied and removed. Replacement uses only the supplied map, deleting every unspecified declared and undeclared key, and rejects any nonempty removal list. Invalid updates leave the database unchanged. -To update an existing database without a module, omit `module` and provide `expected_module_version` from `GET /v1/database/{name_or_identity}/environment`. That authorized endpoint returns `module_version`, `declarations`, and `stored_keys`, without secret values. Each declaration contains `name`, `optional`, and `ty`: `"String"`, `{"StringLiteral":"value"}`, or `{"Union":["a","b"]}`. Metadata comes from one database version. For example: +To update an existing database without a module, omit `module` and provide `expected_module_hash` from `GET /v1/database/{name_or_identity}/environment`. That authorized endpoint returns `module_version`, `declarations`, and `stored_keys`, without secret values. Each declaration contains `name`, `optional`, and `ty`: `"String"`, `{"StringLiteral":"value"}`, or `{"Union":["a","b"]}`. Metadata comes from one database version. For example: ```json { - "expected_module_version": "", + "expected_module_hash": "", "environment": { "FUTURE_KEY": "development-only-value" }, "environment_remove": ["OLD_OPTIONAL_KEY"], "environment_replace": false