Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
- Environment variable overrides (`envOverrides`) are now applied after all environment
variables set by the operator. In particular, `CONTAINERDEBUG_LOG_DIRECTORY` can now be
overridden, whereas previously the operator's value always took precedence ([#838]).
- Make operations infallible where appropriate ([#852]).

### Fixed

Expand All @@ -58,6 +59,7 @@
[#844]: https://github.com/stackabletech/airflow-operator/pull/844
[#847]: https://github.com/stackabletech/airflow-operator/pull/847
[#849]: https://github.com/stackabletech/airflow-operator/pull/849
[#852]: https://github.com/stackabletech/airflow-operator/pull/852

## [26.7.0] - 2026-07-21

Expand Down
45 changes: 45 additions & 0 deletions rust/operator-binary/src/controller/build/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -693,4 +693,49 @@ mod tests {
assert_eq!(env.get("AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX"), None);
assert_eq!(env.get("AIRFLOW__WEBSERVER__PROXY_FIX_X_FOR"), None);
}

/// A cluster with the given `spec.clusterConfig.volumeMounts` (as standalone YAML).
fn cluster_with_user_volume_mounts(
executor_key: &str,
executor_config: &str,
volume_mounts: &str,
) -> ValidatedCluster {
validated_cluster_with(executor_key, executor_config, |cluster| {
cluster["spec"]["clusterConfig"]
.as_mapping_mut()
.expect("clusterConfig is a mapping")
.insert(
"volumeMounts".into(),
serde_yaml::from_str(volume_mounts).expect("valid volumeMounts YAML"),
);
})
}

/// A user-supplied volumeMount that collides with an operator-managed mount path must be
/// reported as an error (the operator's own mounts are added first and are infallible).
#[test]
fn user_volume_mount_colliding_with_config_path_is_an_error() {
let cluster = cluster_with_user_volume_mounts(
"kubernetesExecutors",
"{config: {}}",
"[{name: user-volume, mountPath: /stackable/app/config}]",
);

let Err(error) = build(&cluster) else {
panic!("the colliding mount must be rejected");
};
assert!(
matches!(
error,
super::Error::StatefulSet {
source:
crate::controller::build::resource::statefulset::Error::AddVolumeMount { .. },
..
} | super::Error::ExecutorTemplate {
source: crate::controller::build::resource::executor::Error::AddVolumeMount { .. },
}
),
"unexpected error: {error:?}"
);
}
}
11 changes: 1 addition & 10 deletions rust/operator-binary/src/controller/build/properties/env_vars.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
use std::{collections::BTreeSet, path::PathBuf, str::FromStr};
use std::{collections::BTreeSet, str::FromStr};

use snafu::Snafu;
use stackable_operator::{
constant,
crd::{authentication::oidc, git_sync},
Expand Down Expand Up @@ -81,14 +80,6 @@ constant!(PYTHONPATH: EnvVarName = "PYTHONPATH");
constant!(CONTAINERDEBUG_LOG_DIRECTORY: EnvVarName = "CONTAINERDEBUG_LOG_DIRECTORY");
constant!(STACKABLE_POST_HOOK: EnvVarName = "_STACKABLE_POST_HOOK");

#[derive(Snafu, Debug)]
pub enum Error {
#[snafu(display(
"failed to construct Git DAG folder - Is the git folder a valid path?: {dag_folder:?}"
))]
ConstructGitDagFolder { dag_folder: PathBuf },
}

/// Return environment variables to be applied to the statefulsets for the scheduler, webserver (and worker,
/// for clusters utilizing `celeryExecutor`: for clusters using `kubernetesExecutor` a different set will be
/// used which is defined in [`build_airflow_template_envs`]).
Expand Down
12 changes: 3 additions & 9 deletions rust/operator-binary/src/controller/build/resource/config_map.rs
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,6 @@ pub enum Error {
source: webserver_config::Error,
role_group: RoleGroupName,
},

#[snafu(display("failed to build ConfigMap for role group {role_group}"))]
BuildConfigMap {
source: stackable_operator::builder::configmap::Error,
role_group: RoleGroupName,
},
}

/// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator
Expand Down Expand Up @@ -98,7 +92,7 @@ pub fn build_rolegroup_config_map(
cm_builder.add_data(VECTOR_CONFIG_FILE, vector_config_file_content());
}

cm_builder.build().with_context(|_| BuildConfigMapSnafu {
role_group: role_group_name.clone(),
})
Ok(cm_builder
.build()
.expect("The ConfigMap metadata is set in this function."))
}
37 changes: 16 additions & 21 deletions rust/operator-binary/src/controller/build/resource/executor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,11 @@ use stackable_operator::{
meta::ObjectMetaBuilder,
pod::{PodBuilder, security::PodSecurityContextBuilder},
},
constants::RESTART_CONTROLLER_ENABLED_LABEL,
k8s_openapi::{
DeepMerge,
api::core::v1::{ConfigMap, PodTemplateSpec},
},
kvp::{Label, LabelError},
v2::{
builder::pod::container::{EnvVarSet, new_container_builder},
product_logging::framework::STACKABLE_LOG_DIR,
Expand Down Expand Up @@ -55,17 +55,9 @@ pub enum Error {
source: stackable_operator::builder::pod::container::Error,
},

#[snafu(display("failed to build label"))]
BuildLabel { source: LabelError },

#[snafu(display("pod template serialization"))]
PodTemplateSerde { source: serde_yaml::Error },

#[snafu(display("failed to build the pod template config map"))]
PodTemplateConfigMap {
source: stackable_operator::builder::configmap::Error,
},

#[snafu(display("failed to build shared pod resources"))]
Pod {
source: crate::controller::build::resource::pod::Error,
Expand Down Expand Up @@ -133,13 +125,16 @@ pub fn build_executor_template_config_map(
&executor_config.logging,
git_sync_resources,
))
.add_volume_mounts(cluster.volume_mounts())
.context(AddVolumeMountSnafu)?
// Operator-managed mounts first: their names and paths are constants, so they cannot
// collide with each other.
.add_volume_mount(&*CONFIG_VOLUME_NAME, CONFIG_PATH)
.context(AddVolumeMountSnafu)?
.expect("The mount paths are statically defined and there should be no duplicates.")
.add_volume_mount(&*LOG_CONFIG_VOLUME_NAME, LOG_CONFIG_DIR)
.context(AddVolumeMountSnafu)?
.expect("The mount paths are statically defined and there should be no duplicates.")
.add_volume_mount(&*LOG_VOLUME_NAME, STACKABLE_LOG_DIR)
.expect("The mount paths are statically defined and there should be no duplicates.")
// User-supplied mounts last: these can collide with the ones above, so this stays fallible.
.add_volume_mounts(cluster.volume_mounts())
.context(AddVolumeMountSnafu)?;

add_git_sync_resources(
Expand All @@ -157,16 +152,17 @@ pub fn build_executor_template_config_map(
.add_to_container(&mut airflow_container);

pb.add_container(airflow_container.build());
pb.add_volumes(cluster.volumes().clone())
.context(AddVolumeSnafu)?;
// Operator-managed volumes first (static names), user-supplied volumes last (fallible).
pb.add_volumes(volumes::create_volumes(
cluster
.role_group_resource_names(&EXECUTOR_ROLE_NAME, &EXECUTOR_ROLE_GROUP_NAME)
.role_group_config_map()
.as_ref(),
&executor_config.logging.product_container,
))
.context(AddVolumeSnafu)?;
.expect("The volume names are statically defined and there should be no duplicates.");
pb.add_volumes(cluster.volumes().clone())
.context(AddVolumeSnafu)?;

if let Some(vector_log_config) = &executor_config.logging.vector_container {
pb.add_container(build_logging_container(
Expand All @@ -182,9 +178,6 @@ pub fn build_executor_template_config_map(

let mut cm_builder = ConfigMapBuilder::new();

let restarter_label =
Label::try_from(("restarter.stackable.tech/enabled", "true")).context(BuildLabelSnafu)?;

cm_builder
.metadata(
object_meta(
Expand All @@ -196,13 +189,15 @@ pub fn build_executor_template_config_map(
&EXECUTOR_TEMPLATE_ROLE_GROUP_NAME,
),
)
.with_label(restarter_label)
.with_label(RESTART_CONTROLLER_ENABLED_LABEL.clone())
.build(),
)
.add_data(
TEMPLATE_NAME,
serde_yaml::to_string(&pod_template).context(PodTemplateSerdeSnafu)?,
);

cm_builder.build().context(PodTemplateConfigMapSnafu)
Ok(cm_builder
.build()
.expect("The ConfigMap metadata is set in this function."))
}
11 changes: 11 additions & 0 deletions rust/operator-binary/src/controller/build/resource/pod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -148,3 +148,14 @@ pub(crate) fn build_logging_container(
EnvVarSet::new(),
)
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn test_constants() {
// Test that dereferencing the constants does not panic.
let _ = *VECTOR_CONTAINER_NAME;
}
}
46 changes: 21 additions & 25 deletions rust/operator-binary/src/controller/build/resource/statefulset.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ use stackable_operator::{
security::PodSecurityContextBuilder, volume::VolumeBuilder,
},
},
constants::RESTART_CONTROLLER_ENABLED_LABEL,
k8s_openapi::{
DeepMerge,
api::{
Expand All @@ -16,7 +17,7 @@ use stackable_operator::{
apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString},
},
kube::api::ObjectMeta,
kvp::{Annotation, Label, LabelError},
kvp::Annotation,
utils::COMMON_BASH_TRAP_FUNCTIONS,
v2::{
builder::pod::{
Expand Down Expand Up @@ -63,9 +64,6 @@ pub enum Error {
source: crate::controller::build::graceful_shutdown::Error,
},

#[snafu(display("failed to build label"))]
BuildLabel { source: LabelError },

#[snafu(display("failed to add needed volume"))]
AddVolume {
source: stackable_operator::builder::pod::Error,
Expand All @@ -88,15 +86,14 @@ fn build_rolegroup_metadata(
cluster: &ValidatedCluster,
role: &AirflowRole,
role_group_name: &RoleGroupName,
prometheus_label: Label,
name: String,
) -> ObjectMeta {
object_meta(
cluster,
name,
recommended_labels_for_role_group_resources(cluster, role, role_group_name),
)
.with_label(prometheus_label)
.with_label(RESTART_CONTROLLER_ENABLED_LABEL.clone())
.build()
}

Expand Down Expand Up @@ -194,24 +191,20 @@ pub fn build_server_rolegroup_statefulset(
git_sync_resources,
));

let volume_mounts = validated_cluster.volume_mounts();
airflow_container
.add_volume_mounts(volume_mounts)
.context(AddVolumeMountSnafu)?;
// Operator-managed mounts first: their names and paths are constants, so they cannot collide
// with each other. User-supplied mounts are added last (below) and stay fallible.
airflow_container
.add_volume_mount(&*CONFIG_VOLUME_NAME, CONFIG_PATH)
.context(AddVolumeMountSnafu)?;
airflow_container
.expect("The mount paths are statically defined and there should be no duplicates.")
.add_volume_mount(&*LOG_CONFIG_VOLUME_NAME, LOG_CONFIG_DIR)
.context(AddVolumeMountSnafu)?;
airflow_container
.expect("The mount paths are statically defined and there should be no duplicates.")
.add_volume_mount(&*LOG_VOLUME_NAME, STACKABLE_LOG_DIR)
.context(AddVolumeMountSnafu)?;
.expect("The mount paths are statically defined and there should be no duplicates.");

if let AirflowExecutor::KubernetesExecutors { .. } = executor {
airflow_container
.add_volume_mount(&*TEMPLATE_VOLUME_NAME, TEMPLATE_LOCATION)
.context(AddVolumeMountSnafu)?;
.expect("The mount paths are statically defined and there should be no duplicates.");
}

// for roles with an http endpoint
Expand Down Expand Up @@ -252,9 +245,14 @@ pub fn build_server_rolegroup_statefulset(

airflow_container
.add_volume_mount(&*LISTENER_PVC_NAME, LISTENER_VOLUME_DIR)
.context(AddVolumeMountSnafu)?;
.expect("The mount paths are statically defined and there should be no duplicates.");
}

// User-supplied mounts can collide with the operator-managed ones above, so this stays fallible.
airflow_container
.add_volume_mounts(validated_cluster.volume_mounts())
.context(AddVolumeMountSnafu)?;

add_git_sync_resources(
&mut pb,
&mut airflow_container,
Expand Down Expand Up @@ -306,23 +304,25 @@ pub fn build_server_rolegroup_statefulset(
.build();
pb.add_container(metrics_container);

pb.add_volumes(validated_cluster.volumes().clone())
.context(AddVolumeSnafu)?;
// Operator-managed volumes first (static names), user-supplied volumes last (fallible).
pb.add_volumes(volumes::create_volumes(
resource_names.role_group_config_map().as_ref(),
&logging.product_container,
))
.context(AddVolumeSnafu)?;
.expect("The volume names are statically defined and there should be no duplicates.");

if let AirflowExecutor::KubernetesExecutors { .. } = executor {
pb.add_volume(
VolumeBuilder::new(&*TEMPLATE_VOLUME_NAME)
.with_config_map(validated_cluster.executor_template_configmap_name())
.build(),
)
.context(AddVolumeSnafu)?;
.expect("The volume names are statically defined and there should be no duplicates.");
}

pb.add_volumes(validated_cluster.volumes().clone())
.context(AddVolumeSnafu)?;

if let Some(vector_log_config) = &logging.vector_container {
pb.add_container(build_logging_container(
resolved_product_image,
Expand All @@ -333,14 +333,10 @@ pub fn build_server_rolegroup_statefulset(
let mut pod_template = pb.build_template();
pod_template.merge_from(validated_rg_config.pod_overrides.clone());

let restarter_label =
Label::try_from(("restarter.stackable.tech/enabled", "true")).context(BuildLabelSnafu)?;

let metadata = build_rolegroup_metadata(
validated_cluster,
airflow_role,
role_group_name,
restarter_label,
resource_names.stateful_set_name().to_string(),
);

Expand Down
Loading
Loading