From dd8cd5e67995c65b9df1a9b8091619bf52b40623 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Mon, 10 Jun 2024 16:21:24 +0200 Subject: [PATCH 01/31] Intial import with basic text replacement Originally implemented in and imported from https://github.com/stackabletech/config-utils --- crates/config-utils/.gitignore | 1 + crates/config-utils/Cargo.toml | 18 ++ crates/config-utils/src/args.rs | 26 +++ crates/config-utils/src/file_types.rs | 20 ++ crates/config-utils/src/lib.rs | 8 + crates/config-utils/src/main.rs | 28 +++ crates/config-utils/src/templating.rs | 220 ++++++++++++++++++ .../properties/security_from_env.properties | 3 + .../security_from_env.properties.expected | 3 + .../security_from_env.properties.in | 3 + .../properties/security_from_file.properties | 3 + .../security_from_file.properties.expected | 3 + .../security_from_file.properties.in | 3 + .../properties/security_untouched.properties | 3 + .../security_untouched.properties.expected | 3 + .../security_untouched.properties.in | 3 + .../tests/resources/xml/nifi_ldap.xml | 32 +++ .../resources/xml/nifi_ldap.xml.expected | 32 +++ .../tests/resources/xml/nifi_ldap.xml.in | 32 +++ crates/config-utils/tests/templating.rs | 22 ++ 20 files changed, 466 insertions(+) create mode 100644 crates/config-utils/.gitignore create mode 100644 crates/config-utils/Cargo.toml create mode 100644 crates/config-utils/src/args.rs create mode 100644 crates/config-utils/src/file_types.rs create mode 100644 crates/config-utils/src/lib.rs create mode 100644 crates/config-utils/src/main.rs create mode 100644 crates/config-utils/src/templating.rs create mode 100644 crates/config-utils/tests/resources/properties/security_from_env.properties create mode 100644 crates/config-utils/tests/resources/properties/security_from_env.properties.expected create mode 100644 crates/config-utils/tests/resources/properties/security_from_env.properties.in create mode 100644 crates/config-utils/tests/resources/properties/security_from_file.properties create mode 100644 crates/config-utils/tests/resources/properties/security_from_file.properties.expected create mode 100644 crates/config-utils/tests/resources/properties/security_from_file.properties.in create mode 100644 crates/config-utils/tests/resources/properties/security_untouched.properties create mode 100644 crates/config-utils/tests/resources/properties/security_untouched.properties.expected create mode 100644 crates/config-utils/tests/resources/properties/security_untouched.properties.in create mode 100644 crates/config-utils/tests/resources/xml/nifi_ldap.xml create mode 100644 crates/config-utils/tests/resources/xml/nifi_ldap.xml.expected create mode 100644 crates/config-utils/tests/resources/xml/nifi_ldap.xml.in create mode 100644 crates/config-utils/tests/templating.rs diff --git a/crates/config-utils/.gitignore b/crates/config-utils/.gitignore new file mode 100644 index 000000000..ea8c4bf7f --- /dev/null +++ b/crates/config-utils/.gitignore @@ -0,0 +1 @@ +/target diff --git a/crates/config-utils/Cargo.toml b/crates/config-utils/Cargo.toml new file mode 100644 index 000000000..42b2e35cc --- /dev/null +++ b/crates/config-utils/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "config-filler" +version = "0.1.0" +authors = ["Stackable GmbH "] +license = "OSL-3.0" +edition = "2021" +repository = "https://github.com/stackabletech/config-filler" + +[dependencies] +# We try hard to have a less dependencies as possible! +clap = { version = "4.5", features = ["derive"] } +lazy_static = "1.4" +memchr = "2.7" +snafu = "0.8" + +[dev-dependencies] +rstest = "0.21" +similar-asserts = "1.5" diff --git a/crates/config-utils/src/args.rs b/crates/config-utils/src/args.rs new file mode 100644 index 000000000..5df4d64d4 --- /dev/null +++ b/crates/config-utils/src/args.rs @@ -0,0 +1,26 @@ +use std::path::PathBuf; + +use clap::{Parser, Subcommand}; + +use config_filler::file_types::FileType; + +/// Utility to fill out missing variables in config files +#[derive(Debug, Parser)] +#[command(version, about, long_about = None)] +pub struct Args { + #[command(subcommand)] + pub command: Command, +} + +#[derive(Debug, Subcommand)] +pub enum Command { + Template { + /// The path to the file that should be templated + file: PathBuf, + + /// The optional file type of the file to be templated. If this is not specified this utility will try to infer + /// the type based on the file name. + #[arg(value_enum)] + file_type: Option, + }, +} diff --git a/crates/config-utils/src/file_types.rs b/crates/config-utils/src/file_types.rs new file mode 100644 index 000000000..e6d25b04b --- /dev/null +++ b/crates/config-utils/src/file_types.rs @@ -0,0 +1,20 @@ +use std::collections::HashMap; + +use clap::ValueEnum; +use lazy_static::lazy_static; + +lazy_static! { + // Yes, we could use `strum` for that, but we try to keep the dependencies minimal. + pub static ref KNOWN_FILE_TYPES: HashMap = { + let mut types = HashMap::new(); + types.insert("properties".to_owned(), FileType::Properties); + types.insert("xml".to_owned(), FileType::Xml); + types + }; +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, ValueEnum)] +pub enum FileType { + Properties, + Xml, +} diff --git a/crates/config-utils/src/lib.rs b/crates/config-utils/src/lib.rs new file mode 100644 index 000000000..ff5328049 --- /dev/null +++ b/crates/config-utils/src/lib.rs @@ -0,0 +1,8 @@ +pub mod file_types; +pub mod templating; + +pub const ENV_VAR_PATTERN_START: &str = "${env:"; +pub const ENV_VAR_PATTERN_END: &str = "}"; + +pub const FILE_PATTERN_START: &str = "${file:UTF-8:"; +pub const FILE_PATTERN_END: &str = "}"; diff --git a/crates/config-utils/src/main.rs b/crates/config-utils/src/main.rs new file mode 100644 index 000000000..4af57134e --- /dev/null +++ b/crates/config-utils/src/main.rs @@ -0,0 +1,28 @@ +use clap::Parser; +use config_filler::templating::{self, template}; +use snafu::{ResultExt, Snafu}; + +use args::{Args, Command}; + +mod args; + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("Failed to template file"))] + TemplateFile { source: templating::Error }, +} + +type Result = std::result::Result; + +#[snafu::report] +fn main() -> Result<()> { + let args = Args::parse(); + + match args.command { + Command::Template { file, file_type } => { + template(&file, file_type.as_ref()).context(TemplateFileSnafu)?; + } + } + + Ok(()) +} diff --git a/crates/config-utils/src/templating.rs b/crates/config-utils/src/templating.rs new file mode 100644 index 000000000..38bcb5e0d --- /dev/null +++ b/crates/config-utils/src/templating.rs @@ -0,0 +1,220 @@ +use std::{ + env, + fs::{self, File}, + io::{BufRead, BufReader, Write}, + path::PathBuf, +}; + +use snafu::{OptionExt, ResultExt, Snafu}; + +use crate::{ + file_types::{FileType, KNOWN_FILE_TYPES}, + ENV_VAR_PATTERN_END, ENV_VAR_PATTERN_START, FILE_PATTERN_END, FILE_PATTERN_START, +}; + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("Could not read file {file_name:?}"))] + ReadFile { + source: std::io::Error, + file_name: PathBuf, + }, + + #[snafu(display("Failed to get file extension from file {file_name:?}"))] + GetFileExtension { file_name: PathBuf }, + + #[snafu(display("Failed to convert file name {file_name:?} to string"))] + ConvertFileNameToString { file_name: PathBuf }, + + #[snafu(display("The extension {extension} is not known, can not determine file type"))] + ExtensionUnkown { extension: String }, + + #[snafu(display("Failed to create temporary file {tmp_file_name:?}"))] + CreateTemporaryFile { + source: std::io::Error, + tmp_file_name: PathBuf, + }, + + #[snafu(display("Failed to read line from file {file_name:?}"))] + ReadLine { + source: std::io::Error, + file_name: PathBuf, + }, + + #[snafu(display("Failed to write to temporary file {tmp_file_name:?}"))] + WriteToTemporaryFile { + source: std::io::Error, + tmp_file_name: PathBuf, + }, + + #[snafu(display( + "Failed to rename temporary file {tmp_file_name:?} to destination file {destination_file_name:?}" + ))] + RenameTemporaryFile { + source: std::io::Error, + tmp_file_name: PathBuf, + destination_file_name: PathBuf, + }, + + #[snafu(display( + "Could not find the end pattern {end_pattern:?} in expression {expression:?}" + ))] + FindEndPatten { + end_pattern: String, + expression: String, + }, + + #[snafu(display("Could not read file {file_name:?} for templating"))] + ReadFileForTemplating { + source: std::io::Error, + file_name: PathBuf, + }, + + #[snafu(display("Could not env var {env_var_name:?} for templating"))] + ReadEnvVarForTemplating { + source: std::env::VarError, + env_var_name: String, + }, +} + +type Result = std::result::Result; + +pub fn template(file_name: &PathBuf, file_type: Option<&FileType>) -> Result<()> { + let _file_type = match file_type { + Some(file_type) => file_type, + None => { + let extension = file_name + .extension() + .context(GetFileExtensionSnafu { file_name })? + .to_str() + .context(GetFileExtensionSnafu { file_name })?; + + KNOWN_FILE_TYPES + .get(extension) + .context(ExtensionUnkownSnafu { extension })? + } + }; + + let file = File::open(file_name).context(ReadFileSnafu { file_name })?; + let buf_reader = BufReader::new(file); + + let tmp_file_name = PathBuf::from(format!( + "{}.tmp_config_filler", + file_name + .to_str() + .context(ConvertFileNameToStringSnafu { file_name })? + )); + let mut temp_file = File::create(&tmp_file_name).context(CreateTemporaryFileSnafu { + tmp_file_name: tmp_file_name.clone(), + })?; + + for line in buf_reader.lines() { + let mut line = line.context(ReadLineSnafu { file_name })?; + + run_all_replacements_on_line(&mut line)?; + + temp_file + .write_all(line.as_bytes()) + .context(WriteToTemporaryFileSnafu { + tmp_file_name: tmp_file_name.clone(), + })?; + temp_file + .write_all(&[b'\n']) + .context(WriteToTemporaryFileSnafu { + tmp_file_name: tmp_file_name.clone(), + })?; + } + + fs::rename(&tmp_file_name, &file_name).context(RenameTemporaryFileSnafu { + tmp_file_name, + destination_file_name: file_name, + })?; + + Ok(()) +} + +fn run_all_replacements_on_line(line: &mut String) -> Result<()> { + loop { + let mut changed = false; + changed |= replace_thingy_in_line( + line, + ENV_VAR_PATTERN_START, + ENV_VAR_PATTERN_END, + replacement_action_for_env_var, + )?; + changed |= replace_thingy_in_line( + line, + FILE_PATTERN_START, + FILE_PATTERN_END, + replacement_action_for_file, + )?; + + if !changed { + break; + } + } + + Ok(()) +} + +fn replacement_action_for_file(file_name: &str) -> Result { + let file_content = + fs::read_to_string(file_name).context(ReadFileForTemplatingSnafu { file_name })?; + let file_content = file_content.trim_end_matches('\n'); + + Ok(file_content.to_owned()) +} + +fn replacement_action_for_env_var(env_var_name: &str) -> Result { + let env_var_content = + env::var(env_var_name).context(ReadEnvVarForTemplatingSnafu { env_var_name })?; + + Ok(env_var_content) +} + +/// * `line` is the current line [`String`] that should be templated. +/// * `start_pattern` must be the start pattern, e.g `${env:`. +/// * `end_pattern` must be the end pattern, `}` in the most cases. +/// * `replacement_action` must be a function that is called and get passed the [`&str`] content between the start and end +/// pattern. This can e.g. be the name of the env var or file name to read. +/// +/// Returns wether the `line` was modified. +fn replace_thingy_in_line( + line: &mut String, + start_pattern: &str, + end_pattern: &str, + replacement_action: fn(&str) -> Result, +) -> Result { + // We need to go back to forth to not destroy stuff while iterating. + // Also this is needed to correctly handle nested cases. + let matches = line + .rmatch_indices(start_pattern) + .map(|(index, _)| index) + .collect::>(); + + if matches.is_empty() { + // Nothing to do + return Ok(false); + } + + for index in matches { + debug_assert_eq!(&line[index..index + start_pattern.len()], start_pattern); + let (parameter, _) = line[index + start_pattern.len()..] + .split_once(end_pattern) + .context(FindEndPattenSnafu { + // FIXME: Truncate string to not bloat error message + expression: &line[index..], + end_pattern, + })?; + + let new_content = replacement_action(parameter)?; + + line.replace_range( + index..index + start_pattern.len() + parameter.len() + end_pattern.len(), + &new_content, + ); + } + + // We modified stuff + Ok(true) +} diff --git a/crates/config-utils/tests/resources/properties/security_from_env.properties b/crates/config-utils/tests/resources/properties/security_from_env.properties new file mode 100644 index 000000000..50867fa88 --- /dev/null +++ b/crates/config-utils/tests/resources/properties/security_from_env.properties @@ -0,0 +1,3 @@ +networkaddress.cache.negative.ttl=0 +networkaddress.cache.ttl=:1 +meine.suß.Pröpertie=42,3 diff --git a/crates/config-utils/tests/resources/properties/security_from_env.properties.expected b/crates/config-utils/tests/resources/properties/security_from_env.properties.expected new file mode 100644 index 000000000..50867fa88 --- /dev/null +++ b/crates/config-utils/tests/resources/properties/security_from_env.properties.expected @@ -0,0 +1,3 @@ +networkaddress.cache.negative.ttl=0 +networkaddress.cache.ttl=:1 +meine.suß.Pröpertie=42,3 diff --git a/crates/config-utils/tests/resources/properties/security_from_env.properties.in b/crates/config-utils/tests/resources/properties/security_from_env.properties.in new file mode 100644 index 000000000..0baad9d05 --- /dev/null +++ b/crates/config-utils/tests/resources/properties/security_from_env.properties.in @@ -0,0 +1,3 @@ +networkaddress.cache.negative.ttl=0 +networkaddress.cache.ttl=${env:DISPLAY} +meine.suß.Pröpertie=42,3 diff --git a/crates/config-utils/tests/resources/properties/security_from_file.properties b/crates/config-utils/tests/resources/properties/security_from_file.properties new file mode 100644 index 000000000..508c1b294 --- /dev/null +++ b/crates/config-utils/tests/resources/properties/security_from_file.properties @@ -0,0 +1,3 @@ +networkaddress.cache.negative.ttl=0 +networkaddress.cache.ttl=nixos +meine.suß.Pröpertie=42,3 diff --git a/crates/config-utils/tests/resources/properties/security_from_file.properties.expected b/crates/config-utils/tests/resources/properties/security_from_file.properties.expected new file mode 100644 index 000000000..508c1b294 --- /dev/null +++ b/crates/config-utils/tests/resources/properties/security_from_file.properties.expected @@ -0,0 +1,3 @@ +networkaddress.cache.negative.ttl=0 +networkaddress.cache.ttl=nixos +meine.suß.Pröpertie=42,3 diff --git a/crates/config-utils/tests/resources/properties/security_from_file.properties.in b/crates/config-utils/tests/resources/properties/security_from_file.properties.in new file mode 100644 index 000000000..629027275 --- /dev/null +++ b/crates/config-utils/tests/resources/properties/security_from_file.properties.in @@ -0,0 +1,3 @@ +networkaddress.cache.negative.ttl=0 +networkaddress.cache.ttl=${file:UTF-8:/etc/hostname} +meine.suß.Pröpertie=42,3 diff --git a/crates/config-utils/tests/resources/properties/security_untouched.properties b/crates/config-utils/tests/resources/properties/security_untouched.properties new file mode 100644 index 000000000..312d12c96 --- /dev/null +++ b/crates/config-utils/tests/resources/properties/security_untouched.properties @@ -0,0 +1,3 @@ +networkaddress.cache.negative.ttl=0 +networkaddress.cache.ttl=5 +meine.suß.Pröpertie=42,3 diff --git a/crates/config-utils/tests/resources/properties/security_untouched.properties.expected b/crates/config-utils/tests/resources/properties/security_untouched.properties.expected new file mode 100644 index 000000000..312d12c96 --- /dev/null +++ b/crates/config-utils/tests/resources/properties/security_untouched.properties.expected @@ -0,0 +1,3 @@ +networkaddress.cache.negative.ttl=0 +networkaddress.cache.ttl=5 +meine.suß.Pröpertie=42,3 diff --git a/crates/config-utils/tests/resources/properties/security_untouched.properties.in b/crates/config-utils/tests/resources/properties/security_untouched.properties.in new file mode 100644 index 000000000..312d12c96 --- /dev/null +++ b/crates/config-utils/tests/resources/properties/security_untouched.properties.in @@ -0,0 +1,3 @@ +networkaddress.cache.negative.ttl=0 +networkaddress.cache.ttl=5 +meine.suß.Pröpertie=42,3 diff --git a/crates/config-utils/tests/resources/xml/nifi_ldap.xml b/crates/config-utils/tests/resources/xml/nifi_ldap.xml new file mode 100644 index 000000000..96a811887 --- /dev/null +++ b/crates/config-utils/tests/resources/xml/nifi_ldap.xml @@ -0,0 +1,32 @@ + + + + login-identity-provider + org.apache.nifi.ldap.LdapProvider + LDAPS + + xxx_ldap_bind_username_xxx + xxx_ldap_bind_password_xxx + + THROW + 10 secs + 10 secs + + ldaps://openldap.kuttl-test-tidy-asp.svc.cluster.local:1636 + ou=my users,dc=example,dc=org + uid={0} + + NONE + /stackable/server_tls/keystore.p12 + secret + PKCS12 + /stackable/server_tls/truststore.p12 + secret + PKCS12 + TLSv1.2 + true + + USE_DN + 7 days + + diff --git a/crates/config-utils/tests/resources/xml/nifi_ldap.xml.expected b/crates/config-utils/tests/resources/xml/nifi_ldap.xml.expected new file mode 100644 index 000000000..96a811887 --- /dev/null +++ b/crates/config-utils/tests/resources/xml/nifi_ldap.xml.expected @@ -0,0 +1,32 @@ + + + + login-identity-provider + org.apache.nifi.ldap.LdapProvider + LDAPS + + xxx_ldap_bind_username_xxx + xxx_ldap_bind_password_xxx + + THROW + 10 secs + 10 secs + + ldaps://openldap.kuttl-test-tidy-asp.svc.cluster.local:1636 + ou=my users,dc=example,dc=org + uid={0} + + NONE + /stackable/server_tls/keystore.p12 + secret + PKCS12 + /stackable/server_tls/truststore.p12 + secret + PKCS12 + TLSv1.2 + true + + USE_DN + 7 days + + diff --git a/crates/config-utils/tests/resources/xml/nifi_ldap.xml.in b/crates/config-utils/tests/resources/xml/nifi_ldap.xml.in new file mode 100644 index 000000000..96a811887 --- /dev/null +++ b/crates/config-utils/tests/resources/xml/nifi_ldap.xml.in @@ -0,0 +1,32 @@ + + + + login-identity-provider + org.apache.nifi.ldap.LdapProvider + LDAPS + + xxx_ldap_bind_username_xxx + xxx_ldap_bind_password_xxx + + THROW + 10 secs + 10 secs + + ldaps://openldap.kuttl-test-tidy-asp.svc.cluster.local:1636 + ou=my users,dc=example,dc=org + uid={0} + + NONE + /stackable/server_tls/keystore.p12 + secret + PKCS12 + /stackable/server_tls/truststore.p12 + secret + PKCS12 + TLSv1.2 + true + + USE_DN + 7 days + + diff --git a/crates/config-utils/tests/templating.rs b/crates/config-utils/tests/templating.rs new file mode 100644 index 000000000..f63e9f6ab --- /dev/null +++ b/crates/config-utils/tests/templating.rs @@ -0,0 +1,22 @@ +use std::{ + fs::{self}, + path::PathBuf, +}; + +use rstest::rstest; + +use config_filler::templating::template; + +#[rstest] +fn test_file_templating(#[files("tests/resources/**/*.in")] test_file_in: PathBuf) { + let test_file = test_file_in.with_extension(""); + let test_file_expected = test_file_in.with_extension("expected"); + + fs::copy(&test_file_in, &test_file).unwrap(); + template(&test_file, None).unwrap(); + + let actual = fs::read_to_string(&test_file).unwrap(); + let expected = fs::read_to_string(&test_file_expected).unwrap(); + + similar_asserts::assert_eq!(actual, expected); +} From 35407685af41ac67952de61b96a742b15e659a80 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 11 Jun 2024 07:23:37 +0200 Subject: [PATCH 02/31] test: Improve env var tests Originally implemented in and imported from https://github.com/stackabletech/config-utils --- .../properties/security_from_env.properties | 5 +++-- .../security_from_env.properties.expected | 5 +++-- .../properties/security_from_env.properties.in | 5 +++-- crates/config-utils/tests/templating.rs | 13 +++++++++---- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/crates/config-utils/tests/resources/properties/security_from_env.properties b/crates/config-utils/tests/resources/properties/security_from_env.properties index 50867fa88..058d6f829 100644 --- a/crates/config-utils/tests/resources/properties/security_from_env.properties +++ b/crates/config-utils/tests/resources/properties/security_from_env.properties @@ -1,3 +1,4 @@ networkaddress.cache.negative.ttl=0 -networkaddress.cache.ttl=:1 -meine.suß.Pröpertie=42,3 +networkaddress.cache.ttl=foo +meine.suß.Pröpertie=42 +example-password=admin-pw= withSpace$%" &&} § diff --git a/crates/config-utils/tests/resources/properties/security_from_env.properties.expected b/crates/config-utils/tests/resources/properties/security_from_env.properties.expected index 50867fa88..058d6f829 100644 --- a/crates/config-utils/tests/resources/properties/security_from_env.properties.expected +++ b/crates/config-utils/tests/resources/properties/security_from_env.properties.expected @@ -1,3 +1,4 @@ networkaddress.cache.negative.ttl=0 -networkaddress.cache.ttl=:1 -meine.suß.Pröpertie=42,3 +networkaddress.cache.ttl=foo +meine.suß.Pröpertie=42 +example-password=admin-pw= withSpace$%" &&} § diff --git a/crates/config-utils/tests/resources/properties/security_from_env.properties.in b/crates/config-utils/tests/resources/properties/security_from_env.properties.in index 0baad9d05..2b5643c64 100644 --- a/crates/config-utils/tests/resources/properties/security_from_env.properties.in +++ b/crates/config-utils/tests/resources/properties/security_from_env.properties.in @@ -1,3 +1,4 @@ networkaddress.cache.negative.ttl=0 -networkaddress.cache.ttl=${env:DISPLAY} -meine.suß.Pröpertie=42,3 +networkaddress.cache.ttl=${env:ENV_TEST} +meine.suß.Pröpertie=42 +example-password=${env:ENV_TEST_PASSWORD} diff --git a/crates/config-utils/tests/templating.rs b/crates/config-utils/tests/templating.rs index f63e9f6ab..2e60efbe1 100644 --- a/crates/config-utils/tests/templating.rs +++ b/crates/config-utils/tests/templating.rs @@ -1,7 +1,4 @@ -use std::{ - fs::{self}, - path::PathBuf, -}; +use std::{env, fs, path::PathBuf}; use rstest::rstest; @@ -9,6 +6,8 @@ use config_filler::templating::template; #[rstest] fn test_file_templating(#[files("tests/resources/**/*.in")] test_file_in: PathBuf) { + set_example_envs(); + let test_file = test_file_in.with_extension(""); let test_file_expected = test_file_in.with_extension("expected"); @@ -20,3 +19,9 @@ fn test_file_templating(#[files("tests/resources/**/*.in")] test_file_in: PathBu similar_asserts::assert_eq!(actual, expected); } + +fn set_example_envs() { + // SAFETY: We only use a single thread to set this env vars + env::set_var("ENV_TEST", "foo"); + env::set_var("ENV_TEST_PASSWORD", "admin-pw= withSpace$%\" &&} §"); +} From 3486e05042c3206c9158f5d9a4423c44a8ab6c63 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 11 Jun 2024 07:26:02 +0200 Subject: [PATCH 03/31] chore: Rename config-filler -> config-utils Originally implemented in and imported from https://github.com/stackabletech/config-utils --- crates/config-utils/Cargo.toml | 4 ++-- crates/config-utils/src/args.rs | 2 +- crates/config-utils/src/main.rs | 2 +- crates/config-utils/src/templating.rs | 2 +- crates/config-utils/tests/templating.rs | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/config-utils/Cargo.toml b/crates/config-utils/Cargo.toml index 42b2e35cc..f226f7108 100644 --- a/crates/config-utils/Cargo.toml +++ b/crates/config-utils/Cargo.toml @@ -1,10 +1,10 @@ [package] -name = "config-filler" +name = "config-utils" version = "0.1.0" authors = ["Stackable GmbH "] license = "OSL-3.0" edition = "2021" -repository = "https://github.com/stackabletech/config-filler" +repository = "https://github.com/stackabletech/config-utils" [dependencies] # We try hard to have a less dependencies as possible! diff --git a/crates/config-utils/src/args.rs b/crates/config-utils/src/args.rs index 5df4d64d4..680d9c569 100644 --- a/crates/config-utils/src/args.rs +++ b/crates/config-utils/src/args.rs @@ -2,7 +2,7 @@ use std::path::PathBuf; use clap::{Parser, Subcommand}; -use config_filler::file_types::FileType; +use config_utils::file_types::FileType; /// Utility to fill out missing variables in config files #[derive(Debug, Parser)] diff --git a/crates/config-utils/src/main.rs b/crates/config-utils/src/main.rs index 4af57134e..9771eb304 100644 --- a/crates/config-utils/src/main.rs +++ b/crates/config-utils/src/main.rs @@ -1,5 +1,5 @@ use clap::Parser; -use config_filler::templating::{self, template}; +use config_utils::templating::{self, template}; use snafu::{ResultExt, Snafu}; use args::{Args, Command}; diff --git a/crates/config-utils/src/templating.rs b/crates/config-utils/src/templating.rs index 38bcb5e0d..661803a7a 100644 --- a/crates/config-utils/src/templating.rs +++ b/crates/config-utils/src/templating.rs @@ -99,7 +99,7 @@ pub fn template(file_name: &PathBuf, file_type: Option<&FileType>) -> Result<()> let buf_reader = BufReader::new(file); let tmp_file_name = PathBuf::from(format!( - "{}.tmp_config_filler", + "{}.tmp_config_utils", file_name .to_str() .context(ConvertFileNameToStringSnafu { file_name })? diff --git a/crates/config-utils/tests/templating.rs b/crates/config-utils/tests/templating.rs index 2e60efbe1..83cd78726 100644 --- a/crates/config-utils/tests/templating.rs +++ b/crates/config-utils/tests/templating.rs @@ -2,7 +2,7 @@ use std::{env, fs, path::PathBuf}; use rstest::rstest; -use config_filler::templating::template; +use config_utils::templating::template; #[rstest] fn test_file_templating(#[files("tests/resources/**/*.in")] test_file_in: PathBuf) { From 309c2f82b2aa6874a58fac674bbc13ce6a48067a Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 11 Jun 2024 07:59:36 +0200 Subject: [PATCH 04/31] feat: Add escaping for properties files Originally implemented in and imported from https://github.com/stackabletech/config-utils --- .../src/{file_types.rs => file_types/mod.rs} | 18 ++++++++++++ .../config-utils/src/file_types/properties.rs | 29 +++++++++++++++++++ crates/config-utils/src/file_types/xml.rs | 9 ++++++ crates/config-utils/src/templating.rs | 10 +++++-- .../properties/security_from_env.properties | 2 +- .../security_from_env.properties.expected | 2 +- 6 files changed, 65 insertions(+), 5 deletions(-) rename crates/config-utils/src/{file_types.rs => file_types/mod.rs} (60%) create mode 100644 crates/config-utils/src/file_types/properties.rs create mode 100644 crates/config-utils/src/file_types/xml.rs diff --git a/crates/config-utils/src/file_types.rs b/crates/config-utils/src/file_types/mod.rs similarity index 60% rename from crates/config-utils/src/file_types.rs rename to crates/config-utils/src/file_types/mod.rs index e6d25b04b..54db8fe00 100644 --- a/crates/config-utils/src/file_types.rs +++ b/crates/config-utils/src/file_types/mod.rs @@ -2,6 +2,11 @@ use std::collections::HashMap; use clap::ValueEnum; use lazy_static::lazy_static; +use properties::PropertiesEscaper; +use xml::XmlEscaper; + +mod properties; +mod xml; lazy_static! { // Yes, we could use `strum` for that, but we try to keep the dependencies minimal. @@ -18,3 +23,16 @@ pub enum FileType { Properties, Xml, } + +pub trait Escape { + fn escape(line: String) -> String; +} + +impl FileType { + pub fn escape(&self, line: String) -> String { + match self { + FileType::Properties => PropertiesEscaper::escape(line), + FileType::Xml => XmlEscaper::escape(line), + } + } +} diff --git a/crates/config-utils/src/file_types/properties.rs b/crates/config-utils/src/file_types/properties.rs new file mode 100644 index 000000000..ec838e541 --- /dev/null +++ b/crates/config-utils/src/file_types/properties.rs @@ -0,0 +1,29 @@ +use super::Escape; + +pub struct PropertiesEscaper; + +// https://docs.oracle.com/javase/6/docs/api/java/util/Properties.html#load(java.io.Reader) +impl Escape for PropertiesEscaper { + fn escape(line: String) -> String { + // Copied from https://github.com/adamcrume/java-properties/blob/0335bfb733444e0b9326405bc7845be449bec1f3/src/lib.rs#L809 + let mut escaped = String::new(); + for c in line.chars() { + match c { + '\\' => escaped.push_str("\\\\"), + ' ' => escaped.push_str("\\ "), + '\t' => escaped.push_str("\\t"), + '\r' => escaped.push_str("\\r"), + '\n' => escaped.push_str("\\n"), + '\x0c' => escaped.push_str("\\f"), + ':' => escaped.push_str("\\:"), + '=' => escaped.push_str("\\="), + '!' => escaped.push_str("\\!"), + '#' => escaped.push_str("\\#"), + _ if c < ' ' => escaped.push_str(&format!("\\u{:x}", c as u16)), + _ => escaped.push(c), // We don't worry about other characters, since they're taken care of below. + } + } + + escaped + } +} diff --git a/crates/config-utils/src/file_types/xml.rs b/crates/config-utils/src/file_types/xml.rs new file mode 100644 index 000000000..48d816ab7 --- /dev/null +++ b/crates/config-utils/src/file_types/xml.rs @@ -0,0 +1,9 @@ +use super::Escape; + +pub struct XmlEscaper; + +impl Escape for XmlEscaper { + fn escape(_line: String) -> String { + todo!("Implement escaping for XML files") + } +} diff --git a/crates/config-utils/src/templating.rs b/crates/config-utils/src/templating.rs index 661803a7a..9853be0ba 100644 --- a/crates/config-utils/src/templating.rs +++ b/crates/config-utils/src/templating.rs @@ -80,7 +80,7 @@ pub enum Error { type Result = std::result::Result; pub fn template(file_name: &PathBuf, file_type: Option<&FileType>) -> Result<()> { - let _file_type = match file_type { + let file_type = match file_type { Some(file_type) => file_type, None => { let extension = file_name @@ -111,7 +111,7 @@ pub fn template(file_name: &PathBuf, file_type: Option<&FileType>) -> Result<()> for line in buf_reader.lines() { let mut line = line.context(ReadLineSnafu { file_name })?; - run_all_replacements_on_line(&mut line)?; + run_all_replacements_on_line(&mut line, file_type)?; temp_file .write_all(line.as_bytes()) @@ -133,7 +133,7 @@ pub fn template(file_name: &PathBuf, file_type: Option<&FileType>) -> Result<()> Ok(()) } -fn run_all_replacements_on_line(line: &mut String) -> Result<()> { +fn run_all_replacements_on_line(line: &mut String, file_type: &FileType) -> Result<()> { loop { let mut changed = false; changed |= replace_thingy_in_line( @@ -141,12 +141,14 @@ fn run_all_replacements_on_line(line: &mut String) -> Result<()> { ENV_VAR_PATTERN_START, ENV_VAR_PATTERN_END, replacement_action_for_env_var, + file_type, )?; changed |= replace_thingy_in_line( line, FILE_PATTERN_START, FILE_PATTERN_END, replacement_action_for_file, + file_type, )?; if !changed { @@ -184,6 +186,7 @@ fn replace_thingy_in_line( start_pattern: &str, end_pattern: &str, replacement_action: fn(&str) -> Result, + file_type: &FileType, ) -> Result { // We need to go back to forth to not destroy stuff while iterating. // Also this is needed to correctly handle nested cases. @@ -208,6 +211,7 @@ fn replace_thingy_in_line( })?; let new_content = replacement_action(parameter)?; + let new_content = file_type.escape(new_content); line.replace_range( index..index + start_pattern.len() + parameter.len() + end_pattern.len(), diff --git a/crates/config-utils/tests/resources/properties/security_from_env.properties b/crates/config-utils/tests/resources/properties/security_from_env.properties index 058d6f829..d7fe3b450 100644 --- a/crates/config-utils/tests/resources/properties/security_from_env.properties +++ b/crates/config-utils/tests/resources/properties/security_from_env.properties @@ -1,4 +1,4 @@ networkaddress.cache.negative.ttl=0 networkaddress.cache.ttl=foo meine.suß.Pröpertie=42 -example-password=admin-pw= withSpace$%" &&} § +example-password=admin-pw\=\ withSpace$%"\ &&}\ § diff --git a/crates/config-utils/tests/resources/properties/security_from_env.properties.expected b/crates/config-utils/tests/resources/properties/security_from_env.properties.expected index 058d6f829..d7fe3b450 100644 --- a/crates/config-utils/tests/resources/properties/security_from_env.properties.expected +++ b/crates/config-utils/tests/resources/properties/security_from_env.properties.expected @@ -1,4 +1,4 @@ networkaddress.cache.negative.ttl=0 networkaddress.cache.ttl=foo meine.suß.Pröpertie=42 -example-password=admin-pw= withSpace$%" &&} § +example-password=admin-pw\=\ withSpace$%"\ &&}\ § From c1810838c2babc3e1c01e2c544bde2098125f3b1 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 11 Jun 2024 08:24:01 +0200 Subject: [PATCH 05/31] feat: Escape XML Originally implemented in and imported from https://github.com/stackabletech/config-utils --- .../config-utils/src/{args.rs => cli_args.rs} | 6 ++++ .../config-utils/src/file_types/properties.rs | 18 +++++++++- crates/config-utils/src/file_types/xml.rs | 34 +++++++++++++++++-- crates/config-utils/src/main.rs | 12 ++++--- crates/config-utils/src/templating.rs | 21 ++++++++---- .../properties/security_from_env.properties | 2 +- .../security_from_env.properties.expected | 2 +- .../tests/resources/xml/nifi_ldap.xml | 2 +- .../resources/xml/nifi_ldap.xml.expected | 2 +- .../tests/resources/xml/nifi_ldap.xml.in | 2 +- crates/config-utils/tests/templating.rs | 4 +-- 11 files changed, 85 insertions(+), 20 deletions(-) rename crates/config-utils/src/{args.rs => cli_args.rs} (66%) diff --git a/crates/config-utils/src/args.rs b/crates/config-utils/src/cli_args.rs similarity index 66% rename from crates/config-utils/src/args.rs rename to crates/config-utils/src/cli_args.rs index 680d9c569..156aabf6f 100644 --- a/crates/config-utils/src/args.rs +++ b/crates/config-utils/src/cli_args.rs @@ -22,5 +22,11 @@ pub enum Command { /// the type based on the file name. #[arg(value_enum)] file_type: Option, + + /// By default inserted values are automatically escaped according to the deteced file format. You can disable + /// this, e.g. when you need to insert XML tags (as they otherwise would be escaped). + /// NOTE: Please make sure to correctly escape the inserted text on your own! + #[clap(long)] + dont_escape: bool, }, } diff --git a/crates/config-utils/src/file_types/properties.rs b/crates/config-utils/src/file_types/properties.rs index ec838e541..686fd9c4c 100644 --- a/crates/config-utils/src/file_types/properties.rs +++ b/crates/config-utils/src/file_types/properties.rs @@ -20,10 +20,26 @@ impl Escape for PropertiesEscaper { '!' => escaped.push_str("\\!"), '#' => escaped.push_str("\\#"), _ if c < ' ' => escaped.push_str(&format!("\\u{:x}", c as u16)), - _ => escaped.push(c), // We don't worry about other characters, since they're taken care of below. + _ => escaped.push(c), } } escaped } } + +#[cfg(test)] +mod tests { + use super::*; + + use rstest::rstest; + + #[rstest] + #[case("foo", "foo")] + #[case("foo bar", "foo\\ bar")] + #[case(" bar", "\\ bar")] + #[case("foo<>'\"&\r\nbar", "foo<>'\"&\\r\\nbar")] + fn test_xml_escaping(#[case] input: String, #[case] expected: String) { + assert_eq!(PropertiesEscaper::escape(input), expected); + } +} diff --git a/crates/config-utils/src/file_types/xml.rs b/crates/config-utils/src/file_types/xml.rs index 48d816ab7..6209e7826 100644 --- a/crates/config-utils/src/file_types/xml.rs +++ b/crates/config-utils/src/file_types/xml.rs @@ -3,7 +3,37 @@ use super::Escape; pub struct XmlEscaper; impl Escape for XmlEscaper { - fn escape(_line: String) -> String { - todo!("Implement escaping for XML files") + fn escape(line: String) -> String { + let mut escaped = String::new(); + for c in line.chars() { + match c { + '<' => escaped.push_str("<"), + '>' => escaped.push_str(">"), + '"' => escaped.push_str("""), + '\'' => escaped.push_str("'"), + '&' => escaped.push_str("&"), + '\n' => escaped.push_str(" "), + '\r' => escaped.push_str(" "), + _ => escaped.push(c), + } + } + + escaped + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use rstest::rstest; + + #[rstest] + #[case("foo", "foo")] + #[case("foo bar", "foo bar")] + #[case(" bar", "<foo> bar")] + #[case("foo<>'\"&\r\nbar", "foo<>'"& bar")] + fn test_xml_escaping(#[case] input: String, #[case] expected: String) { + assert_eq!(XmlEscaper::escape(input), expected); } } diff --git a/crates/config-utils/src/main.rs b/crates/config-utils/src/main.rs index 9771eb304..3c2d90690 100644 --- a/crates/config-utils/src/main.rs +++ b/crates/config-utils/src/main.rs @@ -2,9 +2,9 @@ use clap::Parser; use config_utils::templating::{self, template}; use snafu::{ResultExt, Snafu}; -use args::{Args, Command}; +use cli_args::{Args, Command}; -mod args; +mod cli_args; #[derive(Debug, Snafu)] pub enum Error { @@ -19,8 +19,12 @@ fn main() -> Result<()> { let args = Args::parse(); match args.command { - Command::Template { file, file_type } => { - template(&file, file_type.as_ref()).context(TemplateFileSnafu)?; + Command::Template { + file, + file_type, + dont_escape, + } => { + template(&file, file_type.as_ref(), !dont_escape).context(TemplateFileSnafu)?; } } diff --git a/crates/config-utils/src/templating.rs b/crates/config-utils/src/templating.rs index 9853be0ba..9b3a13f44 100644 --- a/crates/config-utils/src/templating.rs +++ b/crates/config-utils/src/templating.rs @@ -70,7 +70,7 @@ pub enum Error { file_name: PathBuf, }, - #[snafu(display("Could not env var {env_var_name:?} for templating"))] + #[snafu(display("Could not read env var {env_var_name:?} for templating"))] ReadEnvVarForTemplating { source: std::env::VarError, env_var_name: String, @@ -79,7 +79,7 @@ pub enum Error { type Result = std::result::Result; -pub fn template(file_name: &PathBuf, file_type: Option<&FileType>) -> Result<()> { +pub fn template(file_name: &PathBuf, file_type: Option<&FileType>, escape: bool) -> Result<()> { let file_type = match file_type { Some(file_type) => file_type, None => { @@ -111,7 +111,7 @@ pub fn template(file_name: &PathBuf, file_type: Option<&FileType>) -> Result<()> for line in buf_reader.lines() { let mut line = line.context(ReadLineSnafu { file_name })?; - run_all_replacements_on_line(&mut line, file_type)?; + run_all_replacements_on_line(&mut line, file_type, escape)?; temp_file .write_all(line.as_bytes()) @@ -133,7 +133,11 @@ pub fn template(file_name: &PathBuf, file_type: Option<&FileType>) -> Result<()> Ok(()) } -fn run_all_replacements_on_line(line: &mut String, file_type: &FileType) -> Result<()> { +fn run_all_replacements_on_line( + line: &mut String, + file_type: &FileType, + escape: bool, +) -> Result<()> { loop { let mut changed = false; changed |= replace_thingy_in_line( @@ -142,6 +146,7 @@ fn run_all_replacements_on_line(line: &mut String, file_type: &FileType) -> Resu ENV_VAR_PATTERN_END, replacement_action_for_env_var, file_type, + escape, )?; changed |= replace_thingy_in_line( line, @@ -149,6 +154,7 @@ fn run_all_replacements_on_line(line: &mut String, file_type: &FileType) -> Resu FILE_PATTERN_END, replacement_action_for_file, file_type, + escape, )?; if !changed { @@ -187,6 +193,7 @@ fn replace_thingy_in_line( end_pattern: &str, replacement_action: fn(&str) -> Result, file_type: &FileType, + escape: bool, ) -> Result { // We need to go back to forth to not destroy stuff while iterating. // Also this is needed to correctly handle nested cases. @@ -210,8 +217,10 @@ fn replace_thingy_in_line( end_pattern, })?; - let new_content = replacement_action(parameter)?; - let new_content = file_type.escape(new_content); + let mut new_content = replacement_action(parameter)?; + if escape { + new_content = file_type.escape(new_content); + } line.replace_range( index..index + start_pattern.len() + parameter.len() + end_pattern.len(), diff --git a/crates/config-utils/tests/resources/properties/security_from_env.properties b/crates/config-utils/tests/resources/properties/security_from_env.properties index d7fe3b450..9633d5823 100644 --- a/crates/config-utils/tests/resources/properties/security_from_env.properties +++ b/crates/config-utils/tests/resources/properties/security_from_env.properties @@ -1,4 +1,4 @@ networkaddress.cache.negative.ttl=0 networkaddress.cache.ttl=foo meine.suß.Pröpertie=42 -example-password=admin-pw\=\ withSpace$%"\ &&}\ § +example-password=admin-pw\=\ withSpace$%"'\ &&}\ § diff --git a/crates/config-utils/tests/resources/properties/security_from_env.properties.expected b/crates/config-utils/tests/resources/properties/security_from_env.properties.expected index d7fe3b450..9633d5823 100644 --- a/crates/config-utils/tests/resources/properties/security_from_env.properties.expected +++ b/crates/config-utils/tests/resources/properties/security_from_env.properties.expected @@ -1,4 +1,4 @@ networkaddress.cache.negative.ttl=0 networkaddress.cache.ttl=foo meine.suß.Pröpertie=42 -example-password=admin-pw\=\ withSpace$%"\ &&}\ § +example-password=admin-pw\=\ withSpace$%"'\ &&}\ § diff --git a/crates/config-utils/tests/resources/xml/nifi_ldap.xml b/crates/config-utils/tests/resources/xml/nifi_ldap.xml index 96a811887..055f5348d 100644 --- a/crates/config-utils/tests/resources/xml/nifi_ldap.xml +++ b/crates/config-utils/tests/resources/xml/nifi_ldap.xml @@ -6,7 +6,7 @@ LDAPS xxx_ldap_bind_username_xxx - xxx_ldap_bind_password_xxx + admin-pw= withSpace$%"' &&} § THROW 10 secs diff --git a/crates/config-utils/tests/resources/xml/nifi_ldap.xml.expected b/crates/config-utils/tests/resources/xml/nifi_ldap.xml.expected index 96a811887..055f5348d 100644 --- a/crates/config-utils/tests/resources/xml/nifi_ldap.xml.expected +++ b/crates/config-utils/tests/resources/xml/nifi_ldap.xml.expected @@ -6,7 +6,7 @@ LDAPS xxx_ldap_bind_username_xxx - xxx_ldap_bind_password_xxx + admin-pw= withSpace$%"' &&} § THROW 10 secs diff --git a/crates/config-utils/tests/resources/xml/nifi_ldap.xml.in b/crates/config-utils/tests/resources/xml/nifi_ldap.xml.in index 96a811887..3a294c706 100644 --- a/crates/config-utils/tests/resources/xml/nifi_ldap.xml.in +++ b/crates/config-utils/tests/resources/xml/nifi_ldap.xml.in @@ -6,7 +6,7 @@ LDAPS xxx_ldap_bind_username_xxx - xxx_ldap_bind_password_xxx + ${env:ENV_TEST_PASSWORD} THROW 10 secs diff --git a/crates/config-utils/tests/templating.rs b/crates/config-utils/tests/templating.rs index 83cd78726..0fa0cfa8e 100644 --- a/crates/config-utils/tests/templating.rs +++ b/crates/config-utils/tests/templating.rs @@ -12,7 +12,7 @@ fn test_file_templating(#[files("tests/resources/**/*.in")] test_file_in: PathBu let test_file_expected = test_file_in.with_extension("expected"); fs::copy(&test_file_in, &test_file).unwrap(); - template(&test_file, None).unwrap(); + template(&test_file, None, true).unwrap(); let actual = fs::read_to_string(&test_file).unwrap(); let expected = fs::read_to_string(&test_file_expected).unwrap(); @@ -23,5 +23,5 @@ fn test_file_templating(#[files("tests/resources/**/*.in")] test_file_in: PathBu fn set_example_envs() { // SAFETY: We only use a single thread to set this env vars env::set_var("ENV_TEST", "foo"); - env::set_var("ENV_TEST_PASSWORD", "admin-pw= withSpace$%\" &&} §"); + env::set_var("ENV_TEST_PASSWORD", "admin-pw= withSpace$%\"' &&} §"); } From 956bd582dcf4286dc83749605ddcac65d6e77e46 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 11 Jun 2024 08:24:51 +0200 Subject: [PATCH 06/31] chore: Improve error message Originally implemented in and imported from https://github.com/stackabletech/config-utils --- crates/config-utils/src/templating.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/config-utils/src/templating.rs b/crates/config-utils/src/templating.rs index 9b3a13f44..cd0d69414 100644 --- a/crates/config-utils/src/templating.rs +++ b/crates/config-utils/src/templating.rs @@ -26,7 +26,7 @@ pub enum Error { #[snafu(display("Failed to convert file name {file_name:?} to string"))] ConvertFileNameToString { file_name: PathBuf }, - #[snafu(display("The extension {extension} is not known, can not determine file type"))] + #[snafu(display("The extension {extension} is not known, can not determine file type. Please specify the file type manually."))] ExtensionUnkown { extension: String }, #[snafu(display("Failed to create temporary file {tmp_file_name:?}"))] From c6d6311f93d7018548d0fe51288fa1719d3b8fdf Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 11 Jun 2024 08:41:35 +0200 Subject: [PATCH 07/31] test: Test reading from file Originally implemented in and imported from https://github.com/stackabletech/config-utils --- crates/config-utils/Cargo.toml | 2 +- .../security_from_nested.properties | 4 +++ .../security_from_nested.properties.expected | 4 +++ .../security_from_nested.properties.in | 4 +++ .../tests/resources/xml/nifi_ldap.xml | 2 +- .../resources/xml/nifi_ldap.xml.expected | 2 +- .../tests/resources/xml/nifi_ldap.xml.in | 2 +- crates/config-utils/tests/templating.rs | 28 +++++++++++++++++-- 8 files changed, 41 insertions(+), 7 deletions(-) create mode 100644 crates/config-utils/tests/resources/properties/security_from_nested.properties create mode 100644 crates/config-utils/tests/resources/properties/security_from_nested.properties.expected create mode 100644 crates/config-utils/tests/resources/properties/security_from_nested.properties.in diff --git a/crates/config-utils/Cargo.toml b/crates/config-utils/Cargo.toml index f226f7108..b8bd57716 100644 --- a/crates/config-utils/Cargo.toml +++ b/crates/config-utils/Cargo.toml @@ -10,9 +10,9 @@ repository = "https://github.com/stackabletech/config-utils" # We try hard to have a less dependencies as possible! clap = { version = "4.5", features = ["derive"] } lazy_static = "1.4" -memchr = "2.7" snafu = "0.8" [dev-dependencies] rstest = "0.21" similar-asserts = "1.5" +tempfile = "3.10" diff --git a/crates/config-utils/tests/resources/properties/security_from_nested.properties b/crates/config-utils/tests/resources/properties/security_from_nested.properties new file mode 100644 index 000000000..acf8cbfc2 --- /dev/null +++ b/crates/config-utils/tests/resources/properties/security_from_nested.properties @@ -0,0 +1,4 @@ +networkaddress.cache.negative.ttl=0 +networkaddress.cache.ttl=42 +meine.suß.Pröpertie=42 +example-password=admin-pw\=\ withSpace$%"'\ &&}\ § diff --git a/crates/config-utils/tests/resources/properties/security_from_nested.properties.expected b/crates/config-utils/tests/resources/properties/security_from_nested.properties.expected new file mode 100644 index 000000000..acf8cbfc2 --- /dev/null +++ b/crates/config-utils/tests/resources/properties/security_from_nested.properties.expected @@ -0,0 +1,4 @@ +networkaddress.cache.negative.ttl=0 +networkaddress.cache.ttl=42 +meine.suß.Pröpertie=42 +example-password=admin-pw\=\ withSpace$%"'\ &&}\ § diff --git a/crates/config-utils/tests/resources/properties/security_from_nested.properties.in b/crates/config-utils/tests/resources/properties/security_from_nested.properties.in new file mode 100644 index 000000000..de07da7b7 --- /dev/null +++ b/crates/config-utils/tests/resources/properties/security_from_nested.properties.in @@ -0,0 +1,4 @@ +networkaddress.cache.negative.ttl=0 +networkaddress.cache.ttl=${file:UTF-8:${env:${env:FILE_TEST_42_FILE_ENV_NAME}}} +meine.suß.Pröpertie=42 +example-password=${env:${env:ENV_TEST_PASSWORD_ENV_NAME}} diff --git a/crates/config-utils/tests/resources/xml/nifi_ldap.xml b/crates/config-utils/tests/resources/xml/nifi_ldap.xml index 055f5348d..59705090d 100644 --- a/crates/config-utils/tests/resources/xml/nifi_ldap.xml +++ b/crates/config-utils/tests/resources/xml/nifi_ldap.xml @@ -5,7 +5,7 @@ org.apache.nifi.ldap.LdapProvider LDAPS - xxx_ldap_bind_username_xxx + example user admin-pw= withSpace$%"' &&} § THROW diff --git a/crates/config-utils/tests/resources/xml/nifi_ldap.xml.expected b/crates/config-utils/tests/resources/xml/nifi_ldap.xml.expected index 055f5348d..59705090d 100644 --- a/crates/config-utils/tests/resources/xml/nifi_ldap.xml.expected +++ b/crates/config-utils/tests/resources/xml/nifi_ldap.xml.expected @@ -5,7 +5,7 @@ org.apache.nifi.ldap.LdapProvider LDAPS - xxx_ldap_bind_username_xxx + example user admin-pw= withSpace$%"' &&} § THROW diff --git a/crates/config-utils/tests/resources/xml/nifi_ldap.xml.in b/crates/config-utils/tests/resources/xml/nifi_ldap.xml.in index 3a294c706..0b204cb6b 100644 --- a/crates/config-utils/tests/resources/xml/nifi_ldap.xml.in +++ b/crates/config-utils/tests/resources/xml/nifi_ldap.xml.in @@ -5,7 +5,7 @@ org.apache.nifi.ldap.LdapProvider LDAPS - xxx_ldap_bind_username_xxx + ${env:ENV_TEST_USERNAME} ${env:ENV_TEST_PASSWORD} THROW diff --git a/crates/config-utils/tests/templating.rs b/crates/config-utils/tests/templating.rs index 0fa0cfa8e..c8fcdc247 100644 --- a/crates/config-utils/tests/templating.rs +++ b/crates/config-utils/tests/templating.rs @@ -1,12 +1,19 @@ -use std::{env, fs, path::PathBuf}; +use std::{ + env, + fs::{self, File}, + io::Write, + path::PathBuf, +}; use rstest::rstest; use config_utils::templating::template; +use tempfile::tempdir; #[rstest] fn test_file_templating(#[files("tests/resources/**/*.in")] test_file_in: PathBuf) { - set_example_envs(); + let example_dir = create_example_files(); + set_example_envs(&example_dir); let test_file = test_file_in.with_extension(""); let test_file_expected = test_file_in.with_extension("expected"); @@ -20,8 +27,23 @@ fn test_file_templating(#[files("tests/resources/**/*.in")] test_file_in: PathBu similar_asserts::assert_eq!(actual, expected); } -fn set_example_envs() { +fn set_example_envs(example_dir: &PathBuf) { // SAFETY: We only use a single thread to set this env vars env::set_var("ENV_TEST", "foo"); + env::set_var("ENV_TEST_USERNAME", "example user"); env::set_var("ENV_TEST_PASSWORD", "admin-pw= withSpace$%\"' &&} §"); + env::set_var("ENV_TEST_PASSWORD_ENV_NAME", "ENV_TEST_PASSWORD"); + + env::set_var("FILE_TEST_42_FILE", example_dir.join("42")); + env::set_var("FILE_TEST_42_FILE_ENV_NAME", "FILE_TEST_42_FILE"); +} + +/// Returns the directoy where the files reside +fn create_example_files() -> PathBuf { + let dir = tempdir().expect("Failed to create temp dir").into_path(); + + let mut file = File::create(dir.join("42")).unwrap(); + file.write_all(b"42").unwrap(); + + dir } From 849f6619d4cd23194b52e397d6916a28a8e3e559 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 11 Jun 2024 09:00:59 +0200 Subject: [PATCH 08/31] docs: Add README Originally implemented in and imported from https://github.com/stackabletech/config-utils --- crates/config-utils/README.md | 52 +++++++++++++++++++++++++++++ crates/config-utils/src/cli_args.rs | 3 +- 2 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 crates/config-utils/README.md diff --git a/crates/config-utils/README.md b/crates/config-utils/README.md new file mode 100644 index 000000000..41df045c9 --- /dev/null +++ b/crates/config-utils/README.md @@ -0,0 +1,52 @@ +# config-utils + +This utility currently only supports filling your config with contents from environmental variables or files (called templating). + +## Templating + +Imaginge the following `example.xml`: + +```xml +cat > example.xml << 'EOF' + + ${env:EXAMPLE_USERNAME} + ${file:UTF-8:example-password} + +EOF +``` + +and the following `example-password`: + +```bash +echo 'example-password <123>!' > example-password +``` + +You can run the following command to replace both placeholders: +```bash +export EXAMPLE_USERNAME=my-user + +config-utils template example.xml +``` + +Afterwards the XML looks like + +```xml + + my-user + example-password <123>! + +``` + +`config-utils` did the following steps to achieve the result: + +1. Use the file extension to determine the file type (XML in this case). You can also specify the file type manually as a CLI argument. +2. Read the env var `EXAMPLE_USERNAME`, xml-escape it and insert it +3. Read the contents of the file `example-password`, xml-escape it and insert it + +Please note that `config-utils` also supports nesting templating, so the name of the file to read can come from an env var (or even other file as well). +This looks something like `${env:${env:ENV_TEST_PASSWORD_ENV_NAME}}` + +## Currently supported file formats + +1. `.properties` files +2. XML files diff --git a/crates/config-utils/src/cli_args.rs b/crates/config-utils/src/cli_args.rs index 156aabf6f..f7ab346f3 100644 --- a/crates/config-utils/src/cli_args.rs +++ b/crates/config-utils/src/cli_args.rs @@ -4,7 +4,7 @@ use clap::{Parser, Subcommand}; use config_utils::file_types::FileType; -/// Utility to fill out missing variables in config files +/// Utility that helps you handling config files. #[derive(Debug, Parser)] #[command(version, about, long_about = None)] pub struct Args { @@ -14,6 +14,7 @@ pub struct Args { #[derive(Debug, Subcommand)] pub enum Command { + /// Fill out variables in config files from either env variables or files directly. Template { /// The path to the file that should be templated file: PathBuf, From 74098d5435c6bf4b149775df37179086d5092d15 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 11 Jun 2024 09:04:32 +0200 Subject: [PATCH 09/31] docs: typo Originally implemented in and imported from https://github.com/stackabletech/config-utils --- crates/config-utils/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/config-utils/README.md b/crates/config-utils/README.md index 41df045c9..61693a8e4 100644 --- a/crates/config-utils/README.md +++ b/crates/config-utils/README.md @@ -4,7 +4,7 @@ This utility currently only supports filling your config with contents from envi ## Templating -Imaginge the following `example.xml`: +Imagine the following `example.xml`: ```xml cat > example.xml << 'EOF' From 5eba7ef98904079597e9e820143b06f779d10d8b Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 11 Jun 2024 09:06:00 +0200 Subject: [PATCH 10/31] docs: Improve wording Originally implemented in and imported from https://github.com/stackabletech/config-utils --- crates/config-utils/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/config-utils/README.md b/crates/config-utils/README.md index 61693a8e4..1606ce180 100644 --- a/crates/config-utils/README.md +++ b/crates/config-utils/README.md @@ -43,7 +43,7 @@ Afterwards the XML looks like 2. Read the env var `EXAMPLE_USERNAME`, xml-escape it and insert it 3. Read the contents of the file `example-password`, xml-escape it and insert it -Please note that `config-utils` also supports nesting templating, so the name of the file to read can come from an env var (or even other file as well). +Please note that `config-utils` also supports nested templating, so the name of the file to read can come from an env var (or even another file as well). This looks something like `${env:${env:ENV_TEST_PASSWORD_ENV_NAME}}` ## Currently supported file formats From 9b75935e601d7e6d4bff18e093c868d822abc715 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 11 Jun 2024 09:11:18 +0200 Subject: [PATCH 11/31] chore: typo Originally implemented in and imported from https://github.com/stackabletech/config-utils --- crates/config-utils/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/config-utils/Cargo.toml b/crates/config-utils/Cargo.toml index b8bd57716..6509d3673 100644 --- a/crates/config-utils/Cargo.toml +++ b/crates/config-utils/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" repository = "https://github.com/stackabletech/config-utils" [dependencies] -# We try hard to have a less dependencies as possible! +# We try hard to have as less dependencies as possible! clap = { version = "4.5", features = ["derive"] } lazy_static = "1.4" snafu = "0.8" From e65d2e44c6ee43f9d21e9e2b1883e92f75700f7a Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 11 Jun 2024 09:13:37 +0200 Subject: [PATCH 12/31] chore: Improve .gitignore Originally implemented in and imported from https://github.com/stackabletech/config-utils --- crates/config-utils/.gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/config-utils/.gitignore b/crates/config-utils/.gitignore index ea8c4bf7f..272009d70 100644 --- a/crates/config-utils/.gitignore +++ b/crates/config-utils/.gitignore @@ -1 +1,5 @@ /target + +# The following files are mentioned as an example in the README, so let's exclude them +example.xml +example-password From c8f7d5f420fad3ffe366f527c12d476e04979c31 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 11 Jun 2024 09:44:35 +0200 Subject: [PATCH 13/31] chore: Add Apache-2.0 license Originally implemented in and imported from https://github.com/stackabletech/config-utils --- crates/config-utils/Cargo.toml | 2 +- crates/config-utils/LICENSE | 202 +++++++++++++++++++++++++++++++++ 2 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 crates/config-utils/LICENSE diff --git a/crates/config-utils/Cargo.toml b/crates/config-utils/Cargo.toml index 6509d3673..469625236 100644 --- a/crates/config-utils/Cargo.toml +++ b/crates/config-utils/Cargo.toml @@ -2,7 +2,7 @@ name = "config-utils" version = "0.1.0" authors = ["Stackable GmbH "] -license = "OSL-3.0" +license = "Apache-2.0" edition = "2021" repository = "https://github.com/stackabletech/config-utils" diff --git a/crates/config-utils/LICENSE b/crates/config-utils/LICENSE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/crates/config-utils/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From 6e2328c4a6dc5849f3daeae0c47ee35e8de473a7 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 12 Jun 2024 08:12:11 +0200 Subject: [PATCH 14/31] chore: Rename templating -> template Originally implemented in and imported from https://github.com/stackabletech/config-utils --- crates/config-utils/src/lib.rs | 2 +- crates/config-utils/src/main.rs | 4 ++-- crates/config-utils/src/{templating.rs => template.rs} | 0 crates/config-utils/tests/templating.rs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) rename crates/config-utils/src/{templating.rs => template.rs} (100%) diff --git a/crates/config-utils/src/lib.rs b/crates/config-utils/src/lib.rs index ff5328049..01a0f4c8c 100644 --- a/crates/config-utils/src/lib.rs +++ b/crates/config-utils/src/lib.rs @@ -1,5 +1,5 @@ pub mod file_types; -pub mod templating; +pub mod template; pub const ENV_VAR_PATTERN_START: &str = "${env:"; pub const ENV_VAR_PATTERN_END: &str = "}"; diff --git a/crates/config-utils/src/main.rs b/crates/config-utils/src/main.rs index 3c2d90690..7f36f8913 100644 --- a/crates/config-utils/src/main.rs +++ b/crates/config-utils/src/main.rs @@ -1,5 +1,5 @@ use clap::Parser; -use config_utils::templating::{self, template}; +use config_utils::template::{self, template}; use snafu::{ResultExt, Snafu}; use cli_args::{Args, Command}; @@ -9,7 +9,7 @@ mod cli_args; #[derive(Debug, Snafu)] pub enum Error { #[snafu(display("Failed to template file"))] - TemplateFile { source: templating::Error }, + TemplateFile { source: template::Error }, } type Result = std::result::Result; diff --git a/crates/config-utils/src/templating.rs b/crates/config-utils/src/template.rs similarity index 100% rename from crates/config-utils/src/templating.rs rename to crates/config-utils/src/template.rs diff --git a/crates/config-utils/tests/templating.rs b/crates/config-utils/tests/templating.rs index c8fcdc247..b9c9822bb 100644 --- a/crates/config-utils/tests/templating.rs +++ b/crates/config-utils/tests/templating.rs @@ -7,7 +7,7 @@ use std::{ use rstest::rstest; -use config_utils::templating::template; +use config_utils::template::template; use tempfile::tempdir; #[rstest] From 6469016794ba9d9b5776fd3bb4be461b350643bb Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 12 Jun 2024 08:34:38 +0200 Subject: [PATCH 15/31] docs: Mention Druid configuration interpolation Originally implemented in and imported from https://github.com/stackabletech/config-utils --- crates/config-utils/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/config-utils/README.md b/crates/config-utils/README.md index 1606ce180..76583f0ec 100644 --- a/crates/config-utils/README.md +++ b/crates/config-utils/README.md @@ -4,6 +4,8 @@ This utility currently only supports filling your config with contents from envi ## Templating +> **_TIP:_** The concept was heavily inspired by [Druids configuration interpolation](https://druid.apache.org/docs/latest/configuration/#configuration-interpolation). + Imagine the following `example.xml`: ```xml From cf771904e3acd5ecb12e294684f9bfd4fc3c1acd Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 12 Jun 2024 08:57:13 +0200 Subject: [PATCH 16/31] test: Improve test Originally implemented in and imported from https://github.com/stackabletech/config-utils --- .../tests/resources/properties/security_untouched.properties | 2 +- .../resources/properties/security_untouched.properties.expected | 2 +- .../tests/resources/properties/security_untouched.properties.in | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/config-utils/tests/resources/properties/security_untouched.properties b/crates/config-utils/tests/resources/properties/security_untouched.properties index 312d12c96..be2a47474 100644 --- a/crates/config-utils/tests/resources/properties/security_untouched.properties +++ b/crates/config-utils/tests/resources/properties/security_untouched.properties @@ -1,3 +1,3 @@ networkaddress.cache.negative.ttl=0 -networkaddress.cache.ttl=5 +networkaddress.cache.ttl=${FOO},${dont:replace} meine.suß.Pröpertie=42,3 diff --git a/crates/config-utils/tests/resources/properties/security_untouched.properties.expected b/crates/config-utils/tests/resources/properties/security_untouched.properties.expected index 312d12c96..be2a47474 100644 --- a/crates/config-utils/tests/resources/properties/security_untouched.properties.expected +++ b/crates/config-utils/tests/resources/properties/security_untouched.properties.expected @@ -1,3 +1,3 @@ networkaddress.cache.negative.ttl=0 -networkaddress.cache.ttl=5 +networkaddress.cache.ttl=${FOO},${dont:replace} meine.suß.Pröpertie=42,3 diff --git a/crates/config-utils/tests/resources/properties/security_untouched.properties.in b/crates/config-utils/tests/resources/properties/security_untouched.properties.in index 312d12c96..be2a47474 100644 --- a/crates/config-utils/tests/resources/properties/security_untouched.properties.in +++ b/crates/config-utils/tests/resources/properties/security_untouched.properties.in @@ -1,3 +1,3 @@ networkaddress.cache.negative.ttl=0 -networkaddress.cache.ttl=5 +networkaddress.cache.ttl=${FOO},${dont:replace} meine.suß.Pröpertie=42,3 From 3dfd62c84b486609329647b087c22a83e782e70d Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Thu, 13 Jun 2024 09:45:23 +0200 Subject: [PATCH 17/31] chore: add language attributes for github linguist Originally implemented in and imported from https://github.com/stackabletech/config-utils --- crates/config-utils/.gitattributes | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 crates/config-utils/.gitattributes diff --git a/crates/config-utils/.gitattributes b/crates/config-utils/.gitattributes new file mode 100644 index 000000000..5de03bbf4 --- /dev/null +++ b/crates/config-utils/.gitattributes @@ -0,0 +1,4 @@ +tests/resources/**/*.properties.expected linguist-language=properties +tests/resources/**/*.properties.in linguist-language=properties +tests/resources/**/*.xml.expected linguist-language=xml +tests/resources/**/*.xml.in linguist-language=xml From 1f6fc3e2ffb1731f06f36512cc89e348ba944d1e Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 12 Jun 2024 15:08:17 +0200 Subject: [PATCH 18/31] Move template.rs to src/template/mod.rs Originally implemented in and imported from https://github.com/stackabletech/config-utils --- crates/config-utils/src/{template.rs => template/mod.rs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename crates/config-utils/src/{template.rs => template/mod.rs} (100%) diff --git a/crates/config-utils/src/template.rs b/crates/config-utils/src/template/mod.rs similarity index 100% rename from crates/config-utils/src/template.rs rename to crates/config-utils/src/template/mod.rs From 33a595aabc79a598c4660d8c156a59a9cdfc3cc2 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 12 Jun 2024 15:12:00 +0200 Subject: [PATCH 19/31] Try to gitignore test files Originally implemented in and imported from https://github.com/stackabletech/config-utils --- crates/config-utils/.gitignore | 3 ++ .../properties/security_from_env.properties | 4 --- .../properties/security_from_file.properties | 3 -- .../security_from_nested.properties | 4 --- .../properties/security_untouched.properties | 3 -- .../tests/resources/xml/nifi_ldap.xml | 32 ------------------- 6 files changed, 3 insertions(+), 46 deletions(-) delete mode 100644 crates/config-utils/tests/resources/properties/security_from_env.properties delete mode 100644 crates/config-utils/tests/resources/properties/security_from_file.properties delete mode 100644 crates/config-utils/tests/resources/properties/security_from_nested.properties delete mode 100644 crates/config-utils/tests/resources/properties/security_untouched.properties delete mode 100644 crates/config-utils/tests/resources/xml/nifi_ldap.xml diff --git a/crates/config-utils/.gitignore b/crates/config-utils/.gitignore index 272009d70..f31e01c84 100644 --- a/crates/config-utils/.gitignore +++ b/crates/config-utils/.gitignore @@ -3,3 +3,6 @@ # The following files are mentioned as an example in the README, so let's exclude them example.xml example-password + +tests/resources/properties/*.properties +tests/resources/xml/*.xml diff --git a/crates/config-utils/tests/resources/properties/security_from_env.properties b/crates/config-utils/tests/resources/properties/security_from_env.properties deleted file mode 100644 index 9633d5823..000000000 --- a/crates/config-utils/tests/resources/properties/security_from_env.properties +++ /dev/null @@ -1,4 +0,0 @@ -networkaddress.cache.negative.ttl=0 -networkaddress.cache.ttl=foo -meine.suß.Pröpertie=42 -example-password=admin-pw\=\ withSpace$%"'\ &&}\ § diff --git a/crates/config-utils/tests/resources/properties/security_from_file.properties b/crates/config-utils/tests/resources/properties/security_from_file.properties deleted file mode 100644 index 508c1b294..000000000 --- a/crates/config-utils/tests/resources/properties/security_from_file.properties +++ /dev/null @@ -1,3 +0,0 @@ -networkaddress.cache.negative.ttl=0 -networkaddress.cache.ttl=nixos -meine.suß.Pröpertie=42,3 diff --git a/crates/config-utils/tests/resources/properties/security_from_nested.properties b/crates/config-utils/tests/resources/properties/security_from_nested.properties deleted file mode 100644 index acf8cbfc2..000000000 --- a/crates/config-utils/tests/resources/properties/security_from_nested.properties +++ /dev/null @@ -1,4 +0,0 @@ -networkaddress.cache.negative.ttl=0 -networkaddress.cache.ttl=42 -meine.suß.Pröpertie=42 -example-password=admin-pw\=\ withSpace$%"'\ &&}\ § diff --git a/crates/config-utils/tests/resources/properties/security_untouched.properties b/crates/config-utils/tests/resources/properties/security_untouched.properties deleted file mode 100644 index be2a47474..000000000 --- a/crates/config-utils/tests/resources/properties/security_untouched.properties +++ /dev/null @@ -1,3 +0,0 @@ -networkaddress.cache.negative.ttl=0 -networkaddress.cache.ttl=${FOO},${dont:replace} -meine.suß.Pröpertie=42,3 diff --git a/crates/config-utils/tests/resources/xml/nifi_ldap.xml b/crates/config-utils/tests/resources/xml/nifi_ldap.xml deleted file mode 100644 index 59705090d..000000000 --- a/crates/config-utils/tests/resources/xml/nifi_ldap.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - login-identity-provider - org.apache.nifi.ldap.LdapProvider - LDAPS - - example user - admin-pw= withSpace$%"' &&} § - - THROW - 10 secs - 10 secs - - ldaps://openldap.kuttl-test-tidy-asp.svc.cluster.local:1636 - ou=my users,dc=example,dc=org - uid={0} - - NONE - /stackable/server_tls/keystore.p12 - secret - PKCS12 - /stackable/server_tls/truststore.p12 - secret - PKCS12 - TLSv1.2 - true - - USE_DN - 7 days - - From fa328709c1c5436557f61c2135c9b6251ffce80c Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 12 Jun 2024 15:26:19 +0200 Subject: [PATCH 20/31] Remove test that only works on my machine :) Originally implemented in and imported from https://github.com/stackabletech/config-utils --- .../properties/security_from_file.properties.expected | 3 --- .../resources/properties/security_from_file.properties.in | 3 --- 2 files changed, 6 deletions(-) delete mode 100644 crates/config-utils/tests/resources/properties/security_from_file.properties.expected delete mode 100644 crates/config-utils/tests/resources/properties/security_from_file.properties.in diff --git a/crates/config-utils/tests/resources/properties/security_from_file.properties.expected b/crates/config-utils/tests/resources/properties/security_from_file.properties.expected deleted file mode 100644 index 508c1b294..000000000 --- a/crates/config-utils/tests/resources/properties/security_from_file.properties.expected +++ /dev/null @@ -1,3 +0,0 @@ -networkaddress.cache.negative.ttl=0 -networkaddress.cache.ttl=nixos -meine.suß.Pröpertie=42,3 diff --git a/crates/config-utils/tests/resources/properties/security_from_file.properties.in b/crates/config-utils/tests/resources/properties/security_from_file.properties.in deleted file mode 100644 index 629027275..000000000 --- a/crates/config-utils/tests/resources/properties/security_from_file.properties.in +++ /dev/null @@ -1,3 +0,0 @@ -networkaddress.cache.negative.ttl=0 -networkaddress.cache.ttl=${file:UTF-8:/etc/hostname} -meine.suß.Pröpertie=42,3 From 11dbdb98df38c8a7246cfc78e62ce67297c616ee Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Thu, 13 Jun 2024 13:56:33 +0200 Subject: [PATCH 21/31] Move TemplateCommand struct into template mod Originally implemented in and imported from https://github.com/stackabletech/config-utils --- crates/config-utils/src/cli_args.rs | 21 ++---------------- crates/config-utils/src/main.rs | 6 ++--- crates/config-utils/src/template/cli_args.rs | 23 ++++++++++++++++++++ crates/config-utils/src/template/mod.rs | 4 +++- crates/config-utils/tests/templating.rs | 8 +++---- 5 files changed, 35 insertions(+), 27 deletions(-) create mode 100644 crates/config-utils/src/template/cli_args.rs diff --git a/crates/config-utils/src/cli_args.rs b/crates/config-utils/src/cli_args.rs index f7ab346f3..e78aabd10 100644 --- a/crates/config-utils/src/cli_args.rs +++ b/crates/config-utils/src/cli_args.rs @@ -1,8 +1,6 @@ -use std::path::PathBuf; - use clap::{Parser, Subcommand}; -use config_utils::file_types::FileType; +use config_utils::template::cli_args::TemplateCommand; /// Utility that helps you handling config files. #[derive(Debug, Parser)] @@ -14,20 +12,5 @@ pub struct Args { #[derive(Debug, Subcommand)] pub enum Command { - /// Fill out variables in config files from either env variables or files directly. - Template { - /// The path to the file that should be templated - file: PathBuf, - - /// The optional file type of the file to be templated. If this is not specified this utility will try to infer - /// the type based on the file name. - #[arg(value_enum)] - file_type: Option, - - /// By default inserted values are automatically escaped according to the deteced file format. You can disable - /// this, e.g. when you need to insert XML tags (as they otherwise would be escaped). - /// NOTE: Please make sure to correctly escape the inserted text on your own! - #[clap(long)] - dont_escape: bool, - }, + Template(TemplateCommand), } diff --git a/crates/config-utils/src/main.rs b/crates/config-utils/src/main.rs index 7f36f8913..52c9471a9 100644 --- a/crates/config-utils/src/main.rs +++ b/crates/config-utils/src/main.rs @@ -1,5 +1,5 @@ use clap::Parser; -use config_utils::template::{self, template}; +use config_utils::template::{self, cli_args::TemplateCommand, template}; use snafu::{ResultExt, Snafu}; use cli_args::{Args, Command}; @@ -19,11 +19,11 @@ fn main() -> Result<()> { let args = Args::parse(); match args.command { - Command::Template { + Command::Template(TemplateCommand { file, file_type, dont_escape, - } => { + }) => { template(&file, file_type.as_ref(), !dont_escape).context(TemplateFileSnafu)?; } } diff --git a/crates/config-utils/src/template/cli_args.rs b/crates/config-utils/src/template/cli_args.rs new file mode 100644 index 000000000..7075018d9 --- /dev/null +++ b/crates/config-utils/src/template/cli_args.rs @@ -0,0 +1,23 @@ +use std::path::PathBuf; + +use clap::Parser; + +use crate::file_types::FileType; + +/// Fill out variables in config files from either env variables or files directly. +#[derive(Debug, Parser)] +pub struct TemplateCommand { + /// The path to the file that should be templated + pub file: PathBuf, + + /// The optional file type of the file to be templated. If this is not specified this utility will try to infer + /// the type based on the file name. + #[arg(value_enum)] + pub file_type: Option, + + /// By default inserted values are automatically escaped according to the deteced file format. You can disable + /// this, e.g. when you need to insert XML tags (as they otherwise would be escaped). + /// NOTE: Please make sure to correctly escape the inserted text on your own! + #[clap(long)] + pub dont_escape: bool, +} diff --git a/crates/config-utils/src/template/mod.rs b/crates/config-utils/src/template/mod.rs index cd0d69414..9f89fa97e 100644 --- a/crates/config-utils/src/template/mod.rs +++ b/crates/config-utils/src/template/mod.rs @@ -12,6 +12,8 @@ use crate::{ ENV_VAR_PATTERN_END, ENV_VAR_PATTERN_START, FILE_PATTERN_END, FILE_PATTERN_START, }; +pub mod cli_args; + #[derive(Debug, Snafu)] pub enum Error { #[snafu(display("Could not read file {file_name:?}"))] @@ -125,7 +127,7 @@ pub fn template(file_name: &PathBuf, file_type: Option<&FileType>, escape: bool) })?; } - fs::rename(&tmp_file_name, &file_name).context(RenameTemporaryFileSnafu { + fs::rename(&tmp_file_name, file_name).context(RenameTemporaryFileSnafu { tmp_file_name, destination_file_name: file_name, })?; diff --git a/crates/config-utils/tests/templating.rs b/crates/config-utils/tests/templating.rs index b9c9822bb..0d0f648ce 100644 --- a/crates/config-utils/tests/templating.rs +++ b/crates/config-utils/tests/templating.rs @@ -2,7 +2,7 @@ use std::{ env, fs::{self, File}, io::Write, - path::PathBuf, + path::{Path, PathBuf}, }; use rstest::rstest; @@ -21,13 +21,13 @@ fn test_file_templating(#[files("tests/resources/**/*.in")] test_file_in: PathBu fs::copy(&test_file_in, &test_file).unwrap(); template(&test_file, None, true).unwrap(); - let actual = fs::read_to_string(&test_file).unwrap(); - let expected = fs::read_to_string(&test_file_expected).unwrap(); + let actual = fs::read_to_string(test_file).unwrap(); + let expected = fs::read_to_string(test_file_expected).unwrap(); similar_asserts::assert_eq!(actual, expected); } -fn set_example_envs(example_dir: &PathBuf) { +fn set_example_envs(example_dir: &Path) { // SAFETY: We only use a single thread to set this env vars env::set_var("ENV_TEST", "foo"); env::set_var("ENV_TEST_USERNAME", "example user"); From 34ec60511a7767e5340f29183da0bcedc5af8545 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 25 Jun 2024 11:28:10 +0200 Subject: [PATCH 22/31] feat: Support escaped command invocations * feat: Support escaped command invocations (as needed by Druid) * refactor Originally implemented in and imported from https://github.com/stackabletech/config-utils/pull/8 --- crates/config-utils/src/lib.rs | 10 +++-- crates/config-utils/src/template/mod.rs | 42 +++++++++++-------- ...curity_escaped_command.properties.expected | 3 ++ .../security_escaped_command.properties.in | 3 ++ 4 files changed, 37 insertions(+), 21 deletions(-) create mode 100644 crates/config-utils/tests/resources/properties/security_escaped_command.properties.expected create mode 100644 crates/config-utils/tests/resources/properties/security_escaped_command.properties.in diff --git a/crates/config-utils/src/lib.rs b/crates/config-utils/src/lib.rs index 01a0f4c8c..412e320ac 100644 --- a/crates/config-utils/src/lib.rs +++ b/crates/config-utils/src/lib.rs @@ -1,8 +1,10 @@ pub mod file_types; pub mod template; -pub const ENV_VAR_PATTERN_START: &str = "${env:"; -pub const ENV_VAR_PATTERN_END: &str = "}"; +// It could be the case that the colon (:) in the start pattern has been escaped, as e.g. the product-config crate does +pub const ENV_VAR_START_PATTERNS: [&str; 2] = ["${env:", "${env\\:"]; +pub const ENV_VAR_END_PATTERN: &str = "}"; -pub const FILE_PATTERN_START: &str = "${file:UTF-8:"; -pub const FILE_PATTERN_END: &str = "}"; +// It could be the case that the colon (:) in the start pattern has been escaped, as e.g. the product-config crate does +pub const FILE_START_PATTERNS: [&str; 2] = ["${file:UTF-8:", "${file\\:UTF-8\\:"]; +pub const FILE_END_PATTERN: &str = "}"; diff --git a/crates/config-utils/src/template/mod.rs b/crates/config-utils/src/template/mod.rs index 9f89fa97e..6425244da 100644 --- a/crates/config-utils/src/template/mod.rs +++ b/crates/config-utils/src/template/mod.rs @@ -9,7 +9,7 @@ use snafu::{OptionExt, ResultExt, Snafu}; use crate::{ file_types::{FileType, KNOWN_FILE_TYPES}, - ENV_VAR_PATTERN_END, ENV_VAR_PATTERN_START, FILE_PATTERN_END, FILE_PATTERN_START, + ENV_VAR_END_PATTERN, ENV_VAR_START_PATTERNS, FILE_END_PATTERN, FILE_START_PATTERNS, }; pub mod cli_args; @@ -141,23 +141,31 @@ fn run_all_replacements_on_line( escape: bool, ) -> Result<()> { loop { + #[allow(clippy::type_complexity)] // It's only used in a single place + let mut replacements: Vec<(&str, &str, fn(&str) -> Result)> = Vec::new(); + + for start_pattern in ENV_VAR_START_PATTERNS { + replacements.push(( + start_pattern, + ENV_VAR_END_PATTERN, + replacement_action_for_env_var, + )); + } + for start_pattern in FILE_START_PATTERNS { + replacements.push((start_pattern, FILE_END_PATTERN, replacement_action_for_file)); + } + let mut changed = false; - changed |= replace_thingy_in_line( - line, - ENV_VAR_PATTERN_START, - ENV_VAR_PATTERN_END, - replacement_action_for_env_var, - file_type, - escape, - )?; - changed |= replace_thingy_in_line( - line, - FILE_PATTERN_START, - FILE_PATTERN_END, - replacement_action_for_file, - file_type, - escape, - )?; + for (start_pattern, end_pattern, replacement_action) in replacements { + changed |= replace_thingy_in_line( + line, + start_pattern, + end_pattern, + replacement_action, + file_type, + escape, + )?; + } if !changed { break; diff --git a/crates/config-utils/tests/resources/properties/security_escaped_command.properties.expected b/crates/config-utils/tests/resources/properties/security_escaped_command.properties.expected new file mode 100644 index 000000000..153bbf387 --- /dev/null +++ b/crates/config-utils/tests/resources/properties/security_escaped_command.properties.expected @@ -0,0 +1,3 @@ +# We must also recognize commands where the colon was escaped +test.from.env=foo +test.from.file=42 diff --git a/crates/config-utils/tests/resources/properties/security_escaped_command.properties.in b/crates/config-utils/tests/resources/properties/security_escaped_command.properties.in new file mode 100644 index 000000000..078eb87b0 --- /dev/null +++ b/crates/config-utils/tests/resources/properties/security_escaped_command.properties.in @@ -0,0 +1,3 @@ +# We must also recognize commands where the colon was escaped +test.from.env=${env\:ENV_TEST} +test.from.file=${file\:UTF-8\:${env\:${env\:FILE_TEST_42_FILE_ENV_NAME}}} From 957b92adb0570e6d674ee98e8777757b35359dc1 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 25 Jun 2024 13:20:36 +0200 Subject: [PATCH 23/31] Release 0.2.0 Originally implemented in and imported from https://github.com/stackabletech/config-utils/pull/9 --- crates/config-utils/CHANGELOG.md | 17 +++++++++++++++++ crates/config-utils/Cargo.toml | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 crates/config-utils/CHANGELOG.md diff --git a/crates/config-utils/CHANGELOG.md b/crates/config-utils/CHANGELOG.md new file mode 100644 index 000000000..6a74ca3fa --- /dev/null +++ b/crates/config-utils/CHANGELOG.md @@ -0,0 +1,17 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +## [Unreleased] + +## [0.2.0] - 2024-06-25 + +### Added + +- Support escaped command invocations ([#8]). + +[#8]: https://github.com/stackabletech/config-utils/pull/8 + +## [0.1.0] - 2024-06-14 + +Initial release diff --git a/crates/config-utils/Cargo.toml b/crates/config-utils/Cargo.toml index 469625236..1a4969de6 100644 --- a/crates/config-utils/Cargo.toml +++ b/crates/config-utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "config-utils" -version = "0.1.0" +version = "0.2.0" authors = ["Stackable GmbH "] license = "Apache-2.0" edition = "2021" From 50d28c197f6cbd6848f8436935f08cc3451ec034 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Thu, 5 Mar 2026 15:54:35 +0100 Subject: [PATCH 24/31] Bump Rust to 1.93.0 and dependencies * Bump Rust to 1.93.0 and dependecies Also add deny.toml, so `cargo deny check` passes * clippy * Bump Rust in CI as well * Bump snafu Originally implemented in and imported from https://github.com/stackabletech/config-utils/pull/12 --- crates/config-utils/Cargo.toml | 4 ++-- crates/config-utils/src/template/mod.rs | 2 +- crates/config-utils/tests/templating.rs | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/config-utils/Cargo.toml b/crates/config-utils/Cargo.toml index 1a4969de6..49d49b81e 100644 --- a/crates/config-utils/Cargo.toml +++ b/crates/config-utils/Cargo.toml @@ -10,9 +10,9 @@ repository = "https://github.com/stackabletech/config-utils" # We try hard to have as less dependencies as possible! clap = { version = "4.5", features = ["derive"] } lazy_static = "1.4" -snafu = "0.8" +snafu = "0.9" [dev-dependencies] -rstest = "0.21" +rstest = "0.26" similar-asserts = "1.5" tempfile = "3.10" diff --git a/crates/config-utils/src/template/mod.rs b/crates/config-utils/src/template/mod.rs index 6425244da..bca9b8b35 100644 --- a/crates/config-utils/src/template/mod.rs +++ b/crates/config-utils/src/template/mod.rs @@ -121,7 +121,7 @@ pub fn template(file_name: &PathBuf, file_type: Option<&FileType>, escape: bool) tmp_file_name: tmp_file_name.clone(), })?; temp_file - .write_all(&[b'\n']) + .write_all(b"\n") .context(WriteToTemporaryFileSnafu { tmp_file_name: tmp_file_name.clone(), })?; diff --git a/crates/config-utils/tests/templating.rs b/crates/config-utils/tests/templating.rs index 0d0f648ce..8df686b08 100644 --- a/crates/config-utils/tests/templating.rs +++ b/crates/config-utils/tests/templating.rs @@ -38,9 +38,9 @@ fn set_example_envs(example_dir: &Path) { env::set_var("FILE_TEST_42_FILE_ENV_NAME", "FILE_TEST_42_FILE"); } -/// Returns the directoy where the files reside +/// Returns the directory where the files reside fn create_example_files() -> PathBuf { - let dir = tempdir().expect("Failed to create temp dir").into_path(); + let dir = tempdir().expect("Failed to create temp dir").keep(); let mut file = File::create(dir.join("42")).unwrap(); file.write_all(b"42").unwrap(); From 65baad56bf6791d00ca56d169ef11ef0524fc861 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Thu, 21 May 2026 15:07:06 +0200 Subject: [PATCH 25/31] chore: Dependecy bumps * chore: Dependecy bumps * Bump to Rust 2024 edition * changelog * cargo fmt * Replace `lazy_static` with `std::sync::LazyLock` Originally implemented in and imported from https://github.com/stackabletech/config-utils/pull/13 --- crates/config-utils/CHANGELOG.md | 9 ++++++++- crates/config-utils/Cargo.toml | 11 +++++------ crates/config-utils/src/file_types/mod.rs | 19 ++++++++----------- crates/config-utils/src/template/mod.rs | 6 ++++-- crates/config-utils/tests/templating.rs | 16 +++++++++------- 5 files changed, 34 insertions(+), 27 deletions(-) diff --git a/crates/config-utils/CHANGELOG.md b/crates/config-utils/CHANGELOG.md index 6a74ca3fa..016cfd94e 100644 --- a/crates/config-utils/CHANGELOG.md +++ b/crates/config-utils/CHANGELOG.md @@ -2,7 +2,14 @@ All notable changes to this project will be documented in this file. -## [Unreleased] +## [0.3.0] - 2026-05-19 + +### Changed + +- Bump `clap` to 4.6 and Rust to 1.95.0 ([#13]). +- Replace `lazy_static` with `std::sync::LazyLock` ([#13]). + +[#13]: https://github.com/stackabletech/config-utils/pull/13 ## [0.2.0] - 2024-06-25 diff --git a/crates/config-utils/Cargo.toml b/crates/config-utils/Cargo.toml index 49d49b81e..e5dfce788 100644 --- a/crates/config-utils/Cargo.toml +++ b/crates/config-utils/Cargo.toml @@ -1,18 +1,17 @@ [package] name = "config-utils" -version = "0.2.0" +version = "0.3.0" authors = ["Stackable GmbH "] license = "Apache-2.0" -edition = "2021" +edition = "2024" repository = "https://github.com/stackabletech/config-utils" [dependencies] # We try hard to have as less dependencies as possible! -clap = { version = "4.5", features = ["derive"] } -lazy_static = "1.4" +clap = { version = "4.6", features = ["derive"] } snafu = "0.9" [dev-dependencies] rstest = "0.26" -similar-asserts = "1.5" -tempfile = "3.10" +similar-asserts = "2.0" +tempfile = "3.27" diff --git a/crates/config-utils/src/file_types/mod.rs b/crates/config-utils/src/file_types/mod.rs index 54db8fe00..aaac84bfe 100644 --- a/crates/config-utils/src/file_types/mod.rs +++ b/crates/config-utils/src/file_types/mod.rs @@ -1,22 +1,19 @@ -use std::collections::HashMap; +use std::{collections::HashMap, sync::LazyLock}; use clap::ValueEnum; -use lazy_static::lazy_static; use properties::PropertiesEscaper; use xml::XmlEscaper; mod properties; mod xml; -lazy_static! { - // Yes, we could use `strum` for that, but we try to keep the dependencies minimal. - pub static ref KNOWN_FILE_TYPES: HashMap = { - let mut types = HashMap::new(); - types.insert("properties".to_owned(), FileType::Properties); - types.insert("xml".to_owned(), FileType::Xml); - types - }; -} +// Yes, we could use `strum` for that, but we try to keep the dependencies minimal. +pub static KNOWN_FILE_TYPES: LazyLock> = LazyLock::new(|| { + let mut types = HashMap::new(); + types.insert("properties".to_owned(), FileType::Properties); + types.insert("xml".to_owned(), FileType::Xml); + types +}); #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, ValueEnum)] pub enum FileType { diff --git a/crates/config-utils/src/template/mod.rs b/crates/config-utils/src/template/mod.rs index bca9b8b35..ee65b2ca3 100644 --- a/crates/config-utils/src/template/mod.rs +++ b/crates/config-utils/src/template/mod.rs @@ -8,8 +8,8 @@ use std::{ use snafu::{OptionExt, ResultExt, Snafu}; use crate::{ - file_types::{FileType, KNOWN_FILE_TYPES}, ENV_VAR_END_PATTERN, ENV_VAR_START_PATTERNS, FILE_END_PATTERN, FILE_START_PATTERNS, + file_types::{FileType, KNOWN_FILE_TYPES}, }; pub mod cli_args; @@ -28,7 +28,9 @@ pub enum Error { #[snafu(display("Failed to convert file name {file_name:?} to string"))] ConvertFileNameToString { file_name: PathBuf }, - #[snafu(display("The extension {extension} is not known, can not determine file type. Please specify the file type manually."))] + #[snafu(display( + "The extension {extension} is not known, can not determine file type. Please specify the file type manually." + ))] ExtensionUnkown { extension: String }, #[snafu(display("Failed to create temporary file {tmp_file_name:?}"))] diff --git a/crates/config-utils/tests/templating.rs b/crates/config-utils/tests/templating.rs index 8df686b08..6e93b165b 100644 --- a/crates/config-utils/tests/templating.rs +++ b/crates/config-utils/tests/templating.rs @@ -29,13 +29,15 @@ fn test_file_templating(#[files("tests/resources/**/*.in")] test_file_in: PathBu fn set_example_envs(example_dir: &Path) { // SAFETY: We only use a single thread to set this env vars - env::set_var("ENV_TEST", "foo"); - env::set_var("ENV_TEST_USERNAME", "example user"); - env::set_var("ENV_TEST_PASSWORD", "admin-pw= withSpace$%\"' &&} §"); - env::set_var("ENV_TEST_PASSWORD_ENV_NAME", "ENV_TEST_PASSWORD"); - - env::set_var("FILE_TEST_42_FILE", example_dir.join("42")); - env::set_var("FILE_TEST_42_FILE_ENV_NAME", "FILE_TEST_42_FILE"); + unsafe { + env::set_var("ENV_TEST", "foo"); + env::set_var("ENV_TEST_USERNAME", "example user"); + env::set_var("ENV_TEST_PASSWORD", "admin-pw= withSpace$%\"' &&} §"); + env::set_var("ENV_TEST_PASSWORD_ENV_NAME", "ENV_TEST_PASSWORD"); + + env::set_var("FILE_TEST_42_FILE", example_dir.join("42")); + env::set_var("FILE_TEST_42_FILE_ENV_NAME", "FILE_TEST_42_FILE"); + } } /// Returns the directory where the files reside From 12fc038b7bc53d038260c5fdcbfb00266e9677bb Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Thu, 21 May 2026 23:32:11 +0200 Subject: [PATCH 26/31] chore: Release 0.4.0 0.3.0 was only release as tag (but not in the Cargo.toml), so we actually need to release 0.4.0 instead of 0.3.0 Originally implemented in and imported from https://github.com/stackabletech/config-utils/pull/14 --- crates/config-utils/CHANGELOG.md | 2 +- crates/config-utils/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/config-utils/CHANGELOG.md b/crates/config-utils/CHANGELOG.md index 016cfd94e..c9efb5305 100644 --- a/crates/config-utils/CHANGELOG.md +++ b/crates/config-utils/CHANGELOG.md @@ -2,7 +2,7 @@ All notable changes to this project will be documented in this file. -## [0.3.0] - 2026-05-19 +## [0.4.0] - 2026-05-21 ### Changed diff --git a/crates/config-utils/Cargo.toml b/crates/config-utils/Cargo.toml index e5dfce788..826b16878 100644 --- a/crates/config-utils/Cargo.toml +++ b/crates/config-utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "config-utils" -version = "0.3.0" +version = "0.4.0" authors = ["Stackable GmbH "] license = "Apache-2.0" edition = "2024" From 5a4f6009f40648213a019d11b65daaab458d0e48 Mon Sep 17 00:00:00 2001 From: Techassi Date: Tue, 30 Jun 2026 13:31:50 +0200 Subject: [PATCH 27/31] chore: Bump dependencies * chore: Bump dependencies * chore: Add changelog entry Originally implemented in and imported from https://github.com/stackabletech/config-utils/pull/15 --- crates/config-utils/CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/config-utils/CHANGELOG.md b/crates/config-utils/CHANGELOG.md index c9efb5305..3a893d27e 100644 --- a/crates/config-utils/CHANGELOG.md +++ b/crates/config-utils/CHANGELOG.md @@ -2,6 +2,14 @@ All notable changes to this project will be documented in this file. +## [Unreleased] + +### Changed + +- Bump dependencies ([#15]). + +[#15]: https://github.com/stackabletech/config-utils/pull/15 + ## [0.4.0] - 2026-05-21 ### Changed From abae5662f8486c384ee04c549d5e730ebfa05860 Mon Sep 17 00:00:00 2001 From: Techassi Date: Tue, 30 Jun 2026 15:47:42 +0200 Subject: [PATCH 28/31] chore: Release 0.5.0 Originally implemented in and imported from https://github.com/stackabletech/config-utils/pull/16 --- crates/config-utils/CHANGELOG.md | 2 ++ crates/config-utils/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/config-utils/CHANGELOG.md b/crates/config-utils/CHANGELOG.md index 3a893d27e..c258f751e 100644 --- a/crates/config-utils/CHANGELOG.md +++ b/crates/config-utils/CHANGELOG.md @@ -4,6 +4,8 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +## [0.5.0] - 2026-06-30 + ### Changed - Bump dependencies ([#15]). diff --git a/crates/config-utils/Cargo.toml b/crates/config-utils/Cargo.toml index 826b16878..5cfc8f6eb 100644 --- a/crates/config-utils/Cargo.toml +++ b/crates/config-utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "config-utils" -version = "0.4.0" +version = "0.5.0" authors = ["Stackable GmbH "] license = "Apache-2.0" edition = "2024" From 13c94dd4476042b45078ff8f87d4aa066a765ed4 Mon Sep 17 00:00:00 2001 From: Techassi Date: Wed, 26 Aug 2026 17:02:46 +0200 Subject: [PATCH 29/31] chore: Finalize import of config-utils --- Cargo.lock | 44 ++++++++++++++++++--- Cargo.toml | 2 + crates/config-utils/.pre-commit-config.yaml | 23 +++++++++++ crates/config-utils/Cargo.toml | 12 +++--- 4 files changed, 69 insertions(+), 12 deletions(-) create mode 100644 crates/config-utils/.pre-commit-config.yaml diff --git a/Cargo.lock b/Cargo.lock index 1bbbcdd63..83ea652dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -337,6 +337,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f7dc094d718f2e1c1559ad110e27eeaae14a5465d3d56dd6dbd793079fbd530" dependencies = [ "memchr", + "regex-automata", "serde_core", ] @@ -481,6 +482,17 @@ dependencies = [ "memchr", ] +[[package]] +name = "config-utils" +version = "0.5.0" +dependencies = [ + "clap", + "rstest", + "similar-asserts", + "snafu 0.9.2", + "tempfile", +] + [[package]] name = "console" version = "0.16.4" @@ -1082,7 +1094,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1800,7 +1812,7 @@ dependencies = [ "console", "globset", "once_cell", - "similar", + "similar 2.7.0", "tempfile", "walkdir", ] @@ -3259,7 +3271,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3316,7 +3328,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3663,6 +3675,26 @@ version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" +[[package]] +name = "similar" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f66ca1f7aca2474dc10c942eb22feffc897735f54cd1db90138c2fddb490987" +dependencies = [ + "bstr", + "unicode-segmentation", +] + +[[package]] +name = "similar-asserts" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "997e6ca38e97437973fc9f7f50a50d1274cacd874341a4960fea90067291038c" +dependencies = [ + "console", + "similar 3.2.0", +] + [[package]] name = "slab" version = "0.4.12" @@ -4100,7 +4132,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4876,7 +4908,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 1f7a04ed2..07311a6e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,11 +84,13 @@ serde_yaml = "0.9.34" # This is the last available version, see https://github.c # digest 0.11 and signature 3.0, which rsa 0.9 does not support. sha2 = { version = "0.10.9", features = ["oid"] } signature = "2.2.0" +similar-asserts = "2.0.0" snafu = "0.9.2" stackable-operator-derive = { path = "stackable-operator-derive" } strum = { version = "0.28.0", features = ["derive"] } syn = "3.0.3" sysinfo = "0.39.6" +tempfile = "3.27.0" time = { version = "0.3.55" } tokio = { version = "1.53.1", features = ["macros", "rt-multi-thread", "fs"] } # We use ring instead of aws-lc-rs, as this currently fails to build in "make run-dev" diff --git a/crates/config-utils/.pre-commit-config.yaml b/crates/config-utils/.pre-commit-config.yaml new file mode 100644 index 000000000..c8cb7f1bb --- /dev/null +++ b/crates/config-utils/.pre-commit-config.yaml @@ -0,0 +1,23 @@ +# We ideally want something like: https://github.com/j178/prek/issues/1869 +--- +default_language_version: + node: system + +repos: + - repo: local + hooks: + - id: cargo-test-no-default-features + name: cargo-test-no-default-features + language: system + entry: cargo test --no-default-features --package config-utils + stages: [pre-commit, pre-merge-commit] + pass_filenames: false + files: .*\.rs$|Cargo\.toml + + - id: cargo-test-all-features + name: cargo-test-no-default-features + language: system + entry: cargo test --all-features --package config-utils + stages: [pre-commit, pre-merge-commit] + pass_filenames: false + files: .*\.rs$|Cargo\.toml diff --git a/crates/config-utils/Cargo.toml b/crates/config-utils/Cargo.toml index 5cfc8f6eb..ad826e8de 100644 --- a/crates/config-utils/Cargo.toml +++ b/crates/config-utils/Cargo.toml @@ -4,14 +4,14 @@ version = "0.5.0" authors = ["Stackable GmbH "] license = "Apache-2.0" edition = "2024" -repository = "https://github.com/stackabletech/config-utils" +repository = "https://github.com/stackabletech/operator-rs" [dependencies] # We try hard to have as less dependencies as possible! -clap = { version = "4.6", features = ["derive"] } -snafu = "0.9" +clap.workspace = true +snafu.workspace = true [dev-dependencies] -rstest = "0.26" -similar-asserts = "2.0" -tempfile = "3.27" +rstest.workspace = true +similar-asserts.workspace = true +tempfile.workspace = true From 1bcf98b6116a93274520d2989faae89e50c11833 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 27 Aug 2026 08:31:06 +0200 Subject: [PATCH 30/31] re-trigger CI From cb6086c47bd21b012b714e318d488c9e0f419441 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 27 Aug 2026 09:10:34 +0200 Subject: [PATCH 31/31] chore(config-utils): Fix prek reportings --- crates/config-utils/README.md | 1 + crates/config-utils/src/cli_args.rs | 2 +- crates/config-utils/src/file_types/properties.rs | 4 ++-- crates/config-utils/src/file_types/xml.rs | 4 ++-- crates/config-utils/src/main.rs | 3 +-- crates/config-utils/tests/templating.rs | 3 +-- 6 files changed, 8 insertions(+), 9 deletions(-) diff --git a/crates/config-utils/README.md b/crates/config-utils/README.md index 76583f0ec..a35c3670e 100644 --- a/crates/config-utils/README.md +++ b/crates/config-utils/README.md @@ -24,6 +24,7 @@ echo 'example-password <123>!' > example-password ``` You can run the following command to replace both placeholders: + ```bash export EXAMPLE_USERNAME=my-user diff --git a/crates/config-utils/src/cli_args.rs b/crates/config-utils/src/cli_args.rs index e78aabd10..c8b3b14a6 100644 --- a/crates/config-utils/src/cli_args.rs +++ b/crates/config-utils/src/cli_args.rs @@ -1,6 +1,6 @@ use clap::{Parser, Subcommand}; -use config_utils::template::cli_args::TemplateCommand; +use crate::template::cli_args::TemplateCommand; /// Utility that helps you handling config files. #[derive(Debug, Parser)] diff --git a/crates/config-utils/src/file_types/properties.rs b/crates/config-utils/src/file_types/properties.rs index 686fd9c4c..0772a8045 100644 --- a/crates/config-utils/src/file_types/properties.rs +++ b/crates/config-utils/src/file_types/properties.rs @@ -30,10 +30,10 @@ impl Escape for PropertiesEscaper { #[cfg(test)] mod tests { - use super::*; - use rstest::rstest; + use super::*; + #[rstest] #[case("foo", "foo")] #[case("foo bar", "foo\\ bar")] diff --git a/crates/config-utils/src/file_types/xml.rs b/crates/config-utils/src/file_types/xml.rs index 6209e7826..5ba6f9921 100644 --- a/crates/config-utils/src/file_types/xml.rs +++ b/crates/config-utils/src/file_types/xml.rs @@ -24,10 +24,10 @@ impl Escape for XmlEscaper { #[cfg(test)] mod tests { - use super::*; - use rstest::rstest; + use super::*; + #[rstest] #[case("foo", "foo")] #[case("foo bar", "foo bar")] diff --git a/crates/config-utils/src/main.rs b/crates/config-utils/src/main.rs index 52c9471a9..5c7961540 100644 --- a/crates/config-utils/src/main.rs +++ b/crates/config-utils/src/main.rs @@ -2,8 +2,7 @@ use clap::Parser; use config_utils::template::{self, cli_args::TemplateCommand, template}; use snafu::{ResultExt, Snafu}; -use cli_args::{Args, Command}; - +use crate::cli_args::{Args, Command}; mod cli_args; #[derive(Debug, Snafu)] diff --git a/crates/config-utils/tests/templating.rs b/crates/config-utils/tests/templating.rs index 6e93b165b..4262952e0 100644 --- a/crates/config-utils/tests/templating.rs +++ b/crates/config-utils/tests/templating.rs @@ -5,9 +5,8 @@ use std::{ path::{Path, PathBuf}, }; -use rstest::rstest; - use config_utils::template::template; +use rstest::rstest; use tempfile::tempdir; #[rstest]