diff --git a/rust/operator-binary/src/controller/build/container.rs b/rust/operator-binary/src/controller/build/container.rs index 45e93d35..50556065 100644 --- a/rust/operator-binary/src/controller/build/container.rs +++ b/rust/operator-binary/src/controller/build/container.rs @@ -17,7 +17,6 @@ use stackable_operator::{ builder::{ self, pod::{ - PodBuilder, resources::ResourceRequirementsBuilder, volume::{ ListenerOperatorVolumeSourceBuilder, ListenerOperatorVolumeSourceBuilderError, @@ -53,12 +52,7 @@ use stackable_operator::{ STACKABLE_LOG_DIR, ValidatedContainerLogConfigChoice, VectorContainerLogConfig, vector_container, }, - role_utils::{JavaCommonConfig, RoleGroupConfig}, - types::{ - common::Port, - kubernetes::{ConfigMapName, ContainerName, VolumeName}, - operator::RoleGroupName, - }, + types::kubernetes::{ConfigMapName, ContainerName, VolumeName}, }, }; use strum::{Display, EnumDiscriminants, IntoStaticStr}; @@ -67,32 +61,28 @@ use crate::{ controller::{ ValidatedCluster, build::{ - self, ResolvedRoleGroup, RoleGroupResolver, RoleSpecificValues, + self, jvm::{self, construct_global_jvm_args, construct_role_specific_jvm_args}, kerberos::KERBEROS_CONTAINER_PATH, properties::product_logging::{ - FORMAT_NAMENODES_LOG4J_CONFIG_FILE, FORMAT_ZOOKEEPER_LOG4J_CONFIG_FILE, - HDFS_LOG4J_CONFIG_FILE, MAX_FORMAT_NAMENODE_LOG_FILE_SIZE, - MAX_FORMAT_ZOOKEEPER_LOG_FILE_SIZE, MAX_HDFS_LOG_FILE_SIZE, - MAX_WAIT_NAMENODES_LOG_FILE_SIZE, MAX_ZKFC_LOG_FILE_SIZE, - WAIT_FOR_NAMENODES_LOG4J_CONFIG_FILE, ZKFC_LOG4J_CONFIG_FILE, + MAX_FORMAT_NAMENODE_LOG_FILE_SIZE, MAX_FORMAT_ZOOKEEPER_LOG_FILE_SIZE, + MAX_HDFS_LOG_FILE_SIZE, MAX_WAIT_NAMENODES_LOG_FILE_SIZE, MAX_ZKFC_LOG_FILE_SIZE, + log4j_config_file, }, + role_group::RoleGroupInputs, }, }, crd::{ DataNodeConfig, HdfsNodeRole, HdfsPodRef, JournalNodeConfig, NameNodeConfig, UpgradeState, constants::{ - DATANODE_ROOT_DATA_DIR_PREFIX, DEFAULT_DATA_NODE_METRICS_PORT, - DEFAULT_JOURNAL_NODE_METRICS_PORT, DEFAULT_NAME_NODE_METRICS_PORT, LISTENER_VOLUME_DIR, - LISTENER_VOLUME_NAME, LIVENESS_PROBE_FAILURE_THRESHOLD, - LIVENESS_PROBE_INITIAL_DELAY_SECONDS, LIVENESS_PROBE_PERIOD_SECONDS, LOG4J_PROPERTIES, - NAMENODE_ROOT_DATA_DIR, READINESS_PROBE_FAILURE_THRESHOLD, - READINESS_PROBE_INITIAL_DELAY_SECONDS, READINESS_PROBE_PERIOD_SECONDS, - SERVICE_PORT_NAME_HTTP, SERVICE_PORT_NAME_HTTPS, SERVICE_PORT_NAME_IPC, - SERVICE_PORT_NAME_RPC, STACKABLE_ROOT_DATA_DIR, + DATANODE_ROOT_DATA_DIR_PREFIX, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, + LIVENESS_PROBE_FAILURE_THRESHOLD, LIVENESS_PROBE_INITIAL_DELAY_SECONDS, + LIVENESS_PROBE_PERIOD_SECONDS, LOG4J_PROPERTIES, NAMENODE_ROOT_DATA_DIR, + READINESS_PROBE_FAILURE_THRESHOLD, READINESS_PROBE_INITIAL_DELAY_SECONDS, + READINESS_PROBE_PERIOD_SECONDS, SERVICE_PORT_NAME_HTTP, SERVICE_PORT_NAME_HTTPS, + STACKABLE_ROOT_DATA_DIR, }, storage::DataNodeStorageConfig, - v1alpha1, }, }; @@ -173,17 +163,11 @@ pub enum Error { /// the HDFS cluster. #[derive(Display)] pub enum ContainerConfig { + /// The main container of a role, named after the role it runs. Hdfs { - /// HDFS role (name-, data-, journal-node) which determines the container name. + /// The HDFS role this is the main container of. It determines the container name, the + /// port the readiness probe checks and the JMX Exporter port. role: HdfsNodeRole, - /// Port name of the IPC/RPC port, used for the readiness probe. - ipc_port_name: &'static str, - /// Port name of the web UI HTTP port, used for the liveness probe. - web_ui_http_port_name: &'static str, - /// Port name of the web UI HTTPS port, used for the liveness probe. - web_ui_https_port_name: &'static str, - /// The JMX Exporter metrics port. - metrics_port: Port, }, /// The ZooKeeper fail-over controller side container of the namenodes. Zkfc, @@ -211,42 +195,30 @@ impl ContainerConfig { const ZKFC_CONFIG_VOLUME_MOUNT_NAME: &'static str = "zkfc-config"; const ZKFC_LOG_VOLUME_MOUNT_NAME: &'static str = "zkfc-log-config"; - /// Add all main, side and init containers as well as required volumes to the pod builder. + /// The containers every role runs — the `hdfs` main container and, when enabled, the Vector + /// sidecar — and the pod volumes they and the role group need. /// - /// Every role-specific value is resolved by the caller into `resolved`; the role itself comes - /// from `C::ROLE`, the same `C` that produced it. - pub fn add_containers_and_volumes( - pb: &mut PodBuilder, - cluster: &ValidatedCluster, - cluster_info: &KubernetesClusterInfo, - role_group_name: &RoleGroupName, - rolegroup_config: &RoleGroupConfig, - resolved: &ResolvedRoleGroup, - ) -> Result<(), Error> { - let role = &C::ROLE; - let namenode_podrefs = build::pod_refs(cluster, &HdfsNodeRole::Name); + /// The containers only one role runs are named by that role's module in + /// [`role_group`](crate::controller::build::role_group). + pub(crate) fn common_containers_and_volumes( + inputs: &RoleGroupInputs, + ) -> Result<(Vec, Vec), Error> { + let cluster = inputs.cluster; + let role = &inputs.role; + let object_name = inputs.object_name(); + let resource_names = cluster.role_group_resource_names(role, &inputs.role_group_name); + + let mut containers = Vec::new(); + let mut volumes = Vec::new(); // HDFS main container let main_container_config = Self::from(*role); - let resource_names = cluster.role_group_resource_names(role, role_group_name); - let object_name = resource_names.qualified_role_group_name().to_string(); - pb.add_volumes(main_container_config.volumes( - &resolved.logging.hdfs, - resolved.role.listener_volume(), - &object_name, - )) - .context(AddVolumeSnafu)?; - pb.add_container(main_container_config.main_container( - cluster, - cluster_info, - &resolved.logging.hdfs, - rolegroup_config, - resolved, - )?); + volumes.extend(main_container_config.volumes(&inputs.hdfs_logging, &object_name)); + containers.push(main_container_config.main_container(inputs, &inputs.hdfs_logging)?); // Vector sidecar container. - if let Some(vector_logging) = &resolved.logging.vector { + if let Some(vector_logging) = &inputs.vector_logging { let vector_aggregator_config_map_name = cluster .cluster_config .logging @@ -272,14 +244,14 @@ impl ContainerConfig { ), }; - pb.add_container(vector_container( + containers.push(vector_container( &VECTOR_CONTAINER_NAME, &cluster.image, &VectorContainerLogConfig { log_config, vector_aggregator_config_map_name, }, - &cluster.role_group_resource_names(role, role_group_name), + &resource_names, &VECTOR_CONFIG_VOLUME_NAME, &VECTOR_LOG_VOLUME_NAME, EnvVarSet::new(), @@ -287,7 +259,7 @@ impl ContainerConfig { } if let Some(authentication_config) = cluster.authentication_config() { - pb.add_volume( + volumes.push( VolumeBuilder::new(&*TLS_STORE_VOLUME_NAME) .ephemeral( SecretOperatorVolumeSourceBuilder::new( @@ -303,7 +275,7 @@ impl ContainerConfig { .with_format(SecretFormat::TlsPkcs12) .with_tls_pkcs12_password(TLS_STORE_PASSWORD) .with_auto_tls_cert_lifetime( - resolved + inputs .common .requested_secret_lifetime .context(MissingSecretLifetimeSnafu)?, @@ -314,10 +286,9 @@ impl ContainerConfig { })?, ) .build(), - ) - .context(AddVolumeSnafu)?; + ); - pb.add_volume( + volumes.push( VolumeBuilder::new(&*KERBEROS_VOLUME_NAME) .ephemeral( SecretOperatorVolumeSourceBuilder::new( @@ -334,88 +305,34 @@ impl ContainerConfig { })?, ) .build(), - ) - .context(AddVolumeSnafu)?; + ); } - // The role-specific containers and their log configs come from one enum, so a container - // is never built without the log config that belongs to it. - match &resolved.role { - RoleSpecificValues::Journal => {} - RoleSpecificValues::Name { - zkfc, - format_namenodes, - format_zookeeper, - } => { - // Zookeeper fail over container - let zkfc_container_config = Self::Zkfc; - pb.add_volumes(zkfc_container_config.volumes(zkfc, None, &object_name)) - .context(AddVolumeSnafu)?; - pb.add_container(zkfc_container_config.main_container( - cluster, - cluster_info, - zkfc, - rolegroup_config, - resolved, - )?); - - // Format namenode init container - let format_namenodes_container_config = Self::FormatNameNodes; - pb.add_volumes(format_namenodes_container_config.volumes( - format_namenodes, - None, - &object_name, - )) - .context(AddVolumeSnafu)?; - pb.add_init_container(format_namenodes_container_config.init_container( - cluster, - cluster_info, - format_namenodes, - rolegroup_config, - resolved, - &namenode_podrefs, - )?); - - // Format ZooKeeper init container - let format_zookeeper_container_config = Self::FormatZooKeeper; - pb.add_volumes(format_zookeeper_container_config.volumes( - format_zookeeper, - None, - &object_name, - )) - .context(AddVolumeSnafu)?; - pb.add_init_container(format_zookeeper_container_config.init_container( - cluster, - cluster_info, - format_zookeeper, - rolegroup_config, - resolved, - &namenode_podrefs, - )?); - } - RoleSpecificValues::Data { - wait_for_namenodes, .. - } => { - // Wait for namenode init container - let wait_for_namenodes_container_config = Self::WaitForNameNodes; - pb.add_volumes(wait_for_namenodes_container_config.volumes( - wait_for_namenodes, - None, - &object_name, - )) - .context(AddVolumeSnafu)?; - pb.add_init_container(wait_for_namenodes_container_config.init_container( - cluster, - cluster_info, - wait_for_namenodes, - rolegroup_config, - resolved, - &namenode_podrefs, - )?); - } - } + Ok((containers, volumes)) + } + + /// This container, and the pod volumes it needs. + /// + /// `container_log_config` is this container's own, passed by the role module naming it. + /// `init` picks how the container is started, the only difference between the two kinds. + pub(crate) fn build_container( + &self, + inputs: &RoleGroupInputs, + container_log_config: &ContainerLogConfig, + init: bool, + ) -> Result<(Container, Vec), Error> { + let volumes = self.volumes(container_log_config, &inputs.object_name()); + + let container = if init { + // `format-namenodes` and `wait-for-namenodes` address the namenodes by pod name; + // `format-zookeeper` ignores these. + let namenode_podrefs = build::pod_refs(inputs.cluster, &HdfsNodeRole::Name); + self.init_container(inputs, container_log_config, &namenode_podrefs)? + } else { + self.main_container(inputs, container_log_config)? + }; - Ok(()) + Ok((container, volumes)) } /// The PVC templates for a namenode role group: one data PVC plus the listener PVC. @@ -491,24 +408,21 @@ impl ContainerConfig { /// - Namenode ZooKeeper fail over controller (ZKFC) /// - Datanode main process /// - Journalnode main process - fn main_container( + fn main_container( &self, - cluster: &ValidatedCluster, - cluster_info: &KubernetesClusterInfo, + inputs: &RoleGroupInputs, container_log_config: &ContainerLogConfig, - rolegroup_config: &RoleGroupConfig, - resolved: &ResolvedRoleGroup, ) -> Result { - let role = &C::ROLE; + let cluster = inputs.cluster; let mut cb = new_container_builder(self.container_name()); - let resources = self.resources(&resolved.resources); + let resources = self.resources(&inputs.resources); cb.image_from_product_image(&cluster.image) .command(Self::command()) - .args(self.args(cluster, cluster_info, role, container_log_config, &[])?) - .add_env_vars(self.env(cluster, role, rolegroup_config, resources.as_ref())?) - .add_volume_mounts(self.volume_mounts(cluster, &resolved.volume_claim_templates)) + .args(self.args(inputs, container_log_config, &[])?) + .add_env_vars(self.env(inputs, resources.as_ref())?) + .add_volume_mounts(self.volume_mounts(cluster, &inputs.volume_claim_templates)) .context(AddVolumeMountSnafu)? .add_container_ports(self.container_ports(cluster)); @@ -539,35 +453,26 @@ impl ContainerConfig { /// Creates respective init containers for: /// - Namenode (format-namenodes, format-zookeeper) /// - Datanode (wait-for-namenodes) - fn init_container( + fn init_container( &self, - cluster: &ValidatedCluster, - cluster_info: &KubernetesClusterInfo, + inputs: &RoleGroupInputs, container_log_config: &ContainerLogConfig, - rolegroup_config: &RoleGroupConfig, - resolved: &ResolvedRoleGroup, namenode_podrefs: &[HdfsPodRef], ) -> Result { - let role = &C::ROLE; + let cluster = inputs.cluster; let mut cb = new_container_builder(self.container_name()); cb.image_from_product_image(&cluster.image) .command(Self::command()) - .args(self.args( - cluster, - cluster_info, - role, - container_log_config, - namenode_podrefs, - )?) - .add_env_vars(self.env(cluster, role, rolegroup_config, None)?) - .add_volume_mounts(self.volume_mounts(cluster, &resolved.volume_claim_templates)) + .args(self.args(inputs, container_log_config, namenode_podrefs)?) + .add_env_vars(self.env(inputs, None)?) + .add_volume_mounts(self.volume_mounts(cluster, &inputs.volume_claim_templates)) .context(AddVolumeMountSnafu)?; // We use the main app container resources here in contrast to several operators (which use // hardcoded resources) due to the different code structure. // Going forward this should be replaced by calculating init container resources in the pod builder. - if let Some(resources) = self.resources(&resolved.resources) { + if let Some(resources) = self.resources(&inputs.resources) { cb.resources(resources); } @@ -577,7 +482,7 @@ impl ContainerConfig { /// Return the typed container name. fn container_name(&self) -> &'static ContainerName { match self { - ContainerConfig::Hdfs { role, .. } => match role { + ContainerConfig::Hdfs { role } => match role { HdfsNodeRole::Name => &NAMENODE_CONTAINER_NAME, HdfsNodeRole::Data => &DATANODE_CONTAINER_NAME, HdfsNodeRole::Journal => &JOURNALNODE_CONTAINER_NAME, @@ -635,12 +540,13 @@ impl ContainerConfig { /// Returns the container command arguments. fn args( &self, - cluster: &ValidatedCluster, - cluster_info: &KubernetesClusterInfo, - role: &HdfsNodeRole, + inputs: &RoleGroupInputs, container_log_config: &ContainerLogConfig, namenode_podrefs: &[HdfsPodRef], ) -> Result, Error> { + let cluster = inputs.cluster; + let cluster_info = inputs.cluster_info; + let role = &inputs.role; let mut args = String::new(); args.push_str(&self.create_config_directory_cmd()); args.push_str(&self.copy_config_xml_cmd()); @@ -659,10 +565,8 @@ impl ContainerConfig { }; match self { - ContainerConfig::Hdfs { role, .. } => { - args.push_str( - &self.copy_log4j_properties_cmd(HDFS_LOG4J_CONFIG_FILE, container_log_config), - ); + ContainerConfig::Hdfs { role } => { + args.push_str(&self.copy_log4j_properties_cmd(container_log_config)); args.push_str(&formatdoc!( r#"\ @@ -689,9 +593,7 @@ impl ContainerConfig { )); } ContainerConfig::Zkfc => { - args.push_str( - &self.copy_log4j_properties_cmd(ZKFC_LOG4J_CONFIG_FILE, container_log_config), - ); + args.push_str(&self.copy_log4j_properties_cmd(container_log_config)); args.push_str(&format!( "{hadoop_home}/bin/hdfs zkfc\n", hadoop_home = Self::HADOOP_HOME @@ -700,10 +602,7 @@ impl ContainerConfig { ContainerConfig::FormatNameNodes => { args.push_str(&bash_capture_shell_helper(self.container_name().as_ref())); - args.push_str(&self.copy_log4j_properties_cmd( - FORMAT_NAMENODES_LOG4J_CONFIG_FILE, - container_log_config, - )); + args.push_str(&self.copy_log4j_properties_cmd(container_log_config)); // First step we check for active namenodes. This step should return an active namenode // for e.g. scaling. It may fail if the active namenode is restarted and the standby // namenode takes over. @@ -771,10 +670,7 @@ impl ContainerConfig { ContainerConfig::FormatZooKeeper => { args.push_str(&bash_capture_shell_helper(self.container_name().as_ref())); - args.push_str(&self.copy_log4j_properties_cmd( - FORMAT_ZOOKEEPER_LOG4J_CONFIG_FILE, - container_log_config, - )); + args.push_str(&self.copy_log4j_properties_cmd(container_log_config)); args.push_str(&formatdoc!( r###" echo "Attempt to format ZooKeeper ZNode for $POD_NAME ..." @@ -798,10 +694,7 @@ impl ContainerConfig { ContainerConfig::WaitForNameNodes => { args.push_str(&bash_capture_shell_helper(self.container_name().as_ref())); - args.push_str(&self.copy_log4j_properties_cmd( - WAIT_FOR_NAMENODES_LOG4J_CONFIG_FILE, - container_log_config, - )); + args.push_str(&self.copy_log4j_properties_cmd(container_log_config)); if cluster.has_kerberos_enabled() { args.push_str(&Self::get_kerberos_ticket(cluster, role, cluster_info)?); } @@ -881,13 +774,13 @@ impl ContainerConfig { } /// Returns the container env variables. - fn env( + fn env( &self, - cluster: &ValidatedCluster, - role: &HdfsNodeRole, - rolegroup_config: &RoleGroupConfig, + inputs: &RoleGroupInputs, resources: Option<&ResourceRequirements>, ) -> Result, Error> { + let cluster = inputs.cluster; + let role = &inputs.role; // Maps env var name to env var object. This allows env_overrides to work // as expected (i.e. users can override the env var value). let mut env: BTreeMap = BTreeMap::new(); @@ -910,13 +803,13 @@ impl ContainerConfig { // When the users tries to start a cli tool the port is already taken by the hdfs services, // so we don't want to stuff all the config into HADOOP_OPTS, but rather into the specialized env variables // See https://github.com/stackabletech/hdfs-operator/issues/138 for details - if let ContainerConfig::Hdfs { role, .. } = self { + if let ContainerConfig::Hdfs { role } = self { let role_opts_name = role.hadoop_opts_env_var_for_role().to_string(); env.insert( role_opts_name.clone(), EnvVar { name: role_opts_name, - value: Some(self.build_hadoop_opts(cluster, resources, rolegroup_config)?), + value: Some(self.build_hadoop_opts(inputs, resources)?), ..EnvVar::default() }, ); @@ -980,7 +873,7 @@ impl ContainerConfig { ); // Overrides need to come last - let mut env_override_vars: BTreeMap = rolegroup_config + let mut env_override_vars: BTreeMap = inputs .env_overrides .clone() .into_iter() @@ -1025,24 +918,20 @@ impl ContainerConfig { initial_delay_seconds: i32, failure_threshold: i32, ) -> Option { - let ContainerConfig::Hdfs { - web_ui_http_port_name, - web_ui_https_port_name, - .. - } = self - else { + let ContainerConfig::Hdfs { .. } = self else { return None; }; + // Every role serves its web UI under the same two port names. let port = if cluster.has_https_enabled() { - web_ui_https_port_name + SERVICE_PORT_NAME_HTTPS } else { - web_ui_http_port_name + SERVICE_PORT_NAME_HTTP }; Some(Probe { // Use tcp_socket instead of http_get so that the probe is independent of the authentication settings. - tcp_socket: Some(Self::tcp_socket_action_for_port(*port)), + tcp_socket: Some(Self::tcp_socket_action_for_port(port)), period_seconds: Some(period_seconds), initial_delay_seconds: Some(initial_delay_seconds), failure_threshold: Some(failure_threshold), @@ -1058,8 +947,8 @@ impl ContainerConfig { failure_threshold: i32, ) -> Option { match self { - ContainerConfig::Hdfs { ipc_port_name, .. } => Some(Probe { - tcp_socket: Some(Self::tcp_socket_action_for_port(*ipc_port_name)), + ContainerConfig::Hdfs { role } => Some(Probe { + tcp_socket: Some(Self::tcp_socket_action_for_port(build::ipc_port_name(role))), period_seconds: Some(period_seconds), initial_delay_seconds: Some(initial_delay_seconds), failure_threshold: Some(failure_threshold), @@ -1078,20 +967,11 @@ impl ContainerConfig { /// Return the container volumes. /// - /// `container_log_config` is this container's own, chosen by the caller from - /// [`build::RoleGroupLogging`] or [`RoleSpecificValues`]. `listener_volume` is the role - /// group's ephemeral listener volume, which only the datanode main container has. - fn volumes( - &self, - container_log_config: &ContainerLogConfig, - listener_volume: Option<&Volume>, - object_name: &str, - ) -> Vec { + /// `container_log_config` is this container's own, passed by the role builder adding it. + fn volumes(&self, container_log_config: &ContainerLogConfig, object_name: &str) -> Vec { let mut volumes = vec![]; if let ContainerConfig::Hdfs { .. } = self { - volumes.extend(listener_volume.cloned()); - volumes.push( VolumeBuilder::new(ContainerConfig::STACKABLE_LOG_VOLUME_MOUNT_NAME) .empty_dir(EmptyDirVolumeSource { @@ -1166,7 +1046,7 @@ impl ContainerConfig { .build(), ); } - ContainerConfig::Hdfs { role, .. } => { + ContainerConfig::Hdfs { role } => { // JournalNode doesn't use listeners, since it's only used internally by the namenodes if let HdfsNodeRole::Name | HdfsNodeRole::Data = role { volume_mounts.push( @@ -1229,11 +1109,7 @@ impl ContainerConfig { /// This will be copied from: /// - Custom: the log dir mount of the custom config map /// - Automatic: the container config mount dir - fn copy_log4j_properties_cmd( - &self, - log4j_config_file: &str, - container_log_config: &ContainerLogConfig, - ) -> String { + fn copy_log4j_properties_cmd(&self, container_log_config: &ContainerLogConfig) -> String { let volume_mount_dirs = self.volume_mount_dirs(); let source_log4j_properties_dir = if let ContainerLogConfig { choice: Some(ContainerLogConfigChoice::Custom(_)), @@ -1247,33 +1123,29 @@ impl ContainerConfig { format!( "cp {log4j_properties_dir}/{file_name} {config_dir}/{LOG4J_PROPERTIES}\n", log4j_properties_dir = source_log4j_properties_dir, - file_name = log4j_config_file, + file_name = log4j_config_file(self), config_dir = volume_mount_dirs.final_config() ) } /// Build HADOOP_{*node}_OPTS for each namenode, datanodes and journalnodes. - fn build_hadoop_opts( + fn build_hadoop_opts( &self, - cluster: &ValidatedCluster, + inputs: &RoleGroupInputs, resources: Option<&ResourceRequirements>, - rolegroup_config: &RoleGroupConfig, ) -> Result { + let cluster = inputs.cluster; match self { - ContainerConfig::Hdfs { - role, metrics_port, .. - } => { + ContainerConfig::Hdfs { role } => { let volume_mount_dirs = self.volume_mount_dirs(); let config_dir = volume_mount_dirs.final_config(); construct_role_specific_jvm_args( role, - &rolegroup_config - .product_specific_common_config - .jvm_argument_overrides, + &inputs.jvm_argument_overrides, cluster.has_kerberos_enabled(), resources, config_dir, - metrics_port.clone(), + build::jmx_metrics_port(role), ) .with_context(|_| ConstructJvmArgumentsSnafu { role: role.to_string(), @@ -1286,7 +1158,7 @@ impl ContainerConfig { /// Container ports for the main containers namenode, datanode and journalnode. fn container_ports(&self, cluster: &ValidatedCluster) -> Vec { match self { - ContainerConfig::Hdfs { role, .. } => { + ContainerConfig::Hdfs { role } => { // data ports build::hdfs_main_container_ports(cluster, role) .into_iter() @@ -1392,29 +1264,7 @@ impl ContainerConfig { impl From for ContainerConfig { fn from(role: HdfsNodeRole) -> Self { - match role { - HdfsNodeRole::Name => Self::Hdfs { - role, - ipc_port_name: SERVICE_PORT_NAME_RPC, - web_ui_http_port_name: SERVICE_PORT_NAME_HTTP, - web_ui_https_port_name: SERVICE_PORT_NAME_HTTPS, - metrics_port: DEFAULT_NAME_NODE_METRICS_PORT, - }, - HdfsNodeRole::Data => Self::Hdfs { - role, - ipc_port_name: SERVICE_PORT_NAME_IPC, - web_ui_http_port_name: SERVICE_PORT_NAME_HTTP, - web_ui_https_port_name: SERVICE_PORT_NAME_HTTPS, - metrics_port: DEFAULT_DATA_NODE_METRICS_PORT, - }, - HdfsNodeRole::Journal => Self::Hdfs { - role, - ipc_port_name: SERVICE_PORT_NAME_RPC, - web_ui_http_port_name: SERVICE_PORT_NAME_HTTP, - web_ui_https_port_name: SERVICE_PORT_NAME_HTTPS, - metrics_port: DEFAULT_JOURNAL_NODE_METRICS_PORT, - }, - } + Self::Hdfs { role } } } diff --git a/rust/operator-binary/src/controller/build/mod.rs b/rust/operator-binary/src/controller/build/mod.rs index 2eeb36d5..485d1446 100644 --- a/rust/operator-binary/src/controller/build/mod.rs +++ b/rust/operator-binary/src/controller/build/mod.rs @@ -23,12 +23,18 @@ use stackable_operator::{ }, }, }; +use strum::IntoEnumIterator; use crate::{ controller::{ CONTROLLER_NAME, KubernetesResources, OPERATOR_NAME, PRODUCT_NAME, Prepared, ValidatedCluster, - build::resource::rbac::{build_role_binding, build_service_account}, + build::{ + resource::rbac::{build_role_binding, build_service_account}, + role_group::{ + build_datanode_role_group, build_journalnode_role_group, build_namenode_role_group, + }, + }, }, crd::{ HdfsNodeRole, HdfsPodRef, @@ -57,8 +63,8 @@ pub mod jvm; pub mod kerberos; pub mod opa; pub mod properties; -pub mod resolve; pub mod resource; +pub mod role_group; #[derive(Snafu, Debug)] pub enum Error { @@ -106,12 +112,23 @@ pub enum Error { role: HdfsNodeRole, role_group: RoleGroupName, }, -} -pub(crate) use resolve::RoleGroupResolver; -pub use resolve::{ResolvedRoleGroup, RoleGroupLogging, RoleSpecificValues}; + #[snafu(display("failed to add the listener volume for role {role} role group {role_group}", role = role.as_ref()))] + AddListenerVolume { + source: stackable_operator::builder::pod::Error, + role: HdfsNodeRole, + role_group: RoleGroupName, + }, -/// The resources of every role, accumulated one role at a time by [`build_role`]. + #[snafu(display("failed to build the containers of role {role} role group {role_group}", role = role.as_ref()))] + Container { + source: container::Error, + role: HdfsNodeRole, + role_group: RoleGroupName, + }, +} + +/// The resources of every role, accumulated one role group at a time by [`build`]. #[derive(Default)] struct RoleGroupResources { services: Vec, @@ -122,65 +139,6 @@ struct RoleGroupResources { pod_disruption_budgets: Vec, } -/// Builds every resource of every role group of one role, plus that role's PDB, appending them to -/// `rg_resources`. -fn build_role( - cluster: &ValidatedCluster, - cluster_info: &KubernetesClusterInfo, - role_group_configs: &BTreeMap< - RoleGroupName, - RoleGroupConfig, - >, - rg_resources: &mut RoleGroupResources, -) -> Result<(), Error> { - let role = &C::ROLE; - - for (role_group_name, rg_config) in role_group_configs { - build_role_group_services(cluster, role, role_group_name, &mut rg_resources.services)?; - - let selector_labels = rolegroup_selector_labels(cluster, role, role_group_name).context( - RoleGroupSelectorLabelsSnafu { - role: *role, - role_group: role_group_name.clone(), - }, - )?; - let resolved = rg_config.config.resolve(role_group_name, selector_labels)?; - - rg_resources.config_maps.push( - resource::config_map::build_rolegroup_config_map( - cluster, - cluster_info, - role_group_name, - rg_config, - &resolved, - ) - .context(ConfigMapSnafu { - role: *role, - role_group: role_group_name.clone(), - })?, - ); - rg_resources.stateful_sets.entry(C::ROLE).or_default().push( - resource::statefulset::build_rolegroup_statefulset( - cluster, - cluster_info, - role_group_name, - rg_config, - &resolved, - ) - .context(StatefulSetSnafu { - role: *role, - role_group: role_group_name.clone(), - })?, - ); - } - - if let Some(pdb) = resource::pdb::build_pdb(cluster, role) { - rg_resources.pod_disruption_budgets.push(pdb); - } - - Ok(()) -} - /// Builds every Kubernetes resource for the given validated cluster. /// /// Does not need a Kubernetes client: every external reference is already dereferenced and @@ -188,6 +146,11 @@ fn build_role( /// `cluster_info` carries static cluster information resolved at operator startup (e.g. the /// cluster domain used to build Kerberos principals), not a live client. /// +/// Each of the three loops hands its role group's typed config to that role's builder, which is +/// where everything specific to the role lives. The loops are free to be reordered: the +/// StatefulSets are keyed by role, and the apply step does not depend on the order of the other +/// three collections. +/// /// The resources are returned as flat collections. `stateful_sets` comes out in [`HdfsNodeRole`] /// order, which the apply step depends on; that is structural, from a [`BTreeMap`] flattened in /// key order, not from the order the roles are built in. @@ -200,26 +163,48 @@ pub fn build( ) -> Result, Error> { let mut built = RoleGroupResources::default(); - // These three calls are free to be reordered: the StatefulSets are keyed by role, and the - // apply step does not depend on the order of the other three collections. - build_role( - cluster, - cluster_info, - &cluster.journalnode_role_group_configs, - &mut built, - )?; - build_role( - cluster, - cluster_info, - &cluster.namenode_role_group_configs, - &mut built, - )?; - build_role( - cluster, - cluster_info, - &cluster.datanode_role_group_configs, - &mut built, - )?; + for (role_group_name, rg_config) in &cluster.journalnode_role_group_configs { + let builder = + build_journalnode_role_group(cluster, cluster_info, role_group_name, rg_config)?; + + built.services.extend(builder.build_services()?); + built.config_maps.push(builder.build_config_map()?); + built + .stateful_sets + .entry(HdfsNodeRole::Journal) + .or_default() + .push(builder.build_statefulset()?); + } + + for (role_group_name, rg_config) in &cluster.namenode_role_group_configs { + let builder = build_namenode_role_group(cluster, cluster_info, role_group_name, rg_config)?; + + built.services.extend(builder.build_services()?); + built.config_maps.push(builder.build_config_map()?); + built + .stateful_sets + .entry(HdfsNodeRole::Name) + .or_default() + .push(builder.build_statefulset()?); + } + + for (role_group_name, rg_config) in &cluster.datanode_role_group_configs { + let builder = build_datanode_role_group(cluster, cluster_info, role_group_name, rg_config)?; + + built.services.extend(builder.build_services()?); + built.config_maps.push(builder.build_config_map()?); + built + .stateful_sets + .entry(HdfsNodeRole::Data) + .or_default() + .push(builder.build_statefulset()?); + } + + for role in HdfsNodeRole::iter() { + if let Some(pdb) = resource::pdb::build_pdb(cluster, &role) { + built.pod_disruption_budgets.push(pdb); + } + } let RoleGroupResources { services, @@ -250,34 +235,6 @@ pub fn build( }) } -/// Builds the two Services for one role group. Role-agnostic: it reads nothing from the role -/// config. -fn build_role_group_services( - cluster: &ValidatedCluster, - role: &HdfsNodeRole, - role_group_name: &RoleGroupName, - services: &mut Vec, -) -> Result<(), Error> { - services.push( - resource::service::rolegroup_headless_service(cluster, role, role_group_name).context( - ServiceSnafu { - role: *role, - role_group: role_group_name.clone(), - }, - )?, - ); - services.push( - resource::service::rolegroup_metrics_service(cluster, role, role_group_name).context( - ServiceSnafu { - role: *role, - role_group: role_group_name.clone(), - }, - )?, - ); - - Ok(()) -} - /// The replica count a role group gets when it does not set one: Kubernetes runs a single pod for /// a `StatefulSet` with `replicas: null`. pub(crate) const DEFAULT_REPLICAS: u16 = 1; @@ -456,8 +413,19 @@ pub(crate) fn native_metrics_port(cluster: &ValidatedCluster, role: &HdfsNodeRol } } -/// The deprecated JMX exporter metrics port for the given `role`. -fn jmx_metrics_port(role: &HdfsNodeRole) -> Port { +/// The name of the port the given `role` serves IPC/RPC on, which its readiness probe checks. +/// +/// The datanodes call theirs `ipc`, the other two `rpc`; the same names [`role_data_ports`] +/// exposes them under. +pub(crate) fn ipc_port_name(role: &HdfsNodeRole) -> &'static str { + match role { + HdfsNodeRole::Name | HdfsNodeRole::Journal => SERVICE_PORT_NAME_RPC, + HdfsNodeRole::Data => SERVICE_PORT_NAME_IPC, + } +} + +/// The deprecated JMX Exporter metrics port for the given `role`. +pub(crate) fn jmx_metrics_port(role: &HdfsNodeRole) -> Port { match role { HdfsNodeRole::Name => DEFAULT_NAME_NODE_METRICS_PORT, HdfsNodeRole::Data => DEFAULT_DATA_NODE_METRICS_PORT, diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs b/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs index f59e9b4f..e20a7046 100644 --- a/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs @@ -10,12 +10,9 @@ use stackable_operator::{ v2::product_logging::framework::STACKABLE_LOG_DIR, }; -use crate::controller::build::{ - RoleGroupLogging, RoleSpecificValues, - container::{ - FORMAT_NAMENODES_CONTAINER_NAME, FORMAT_ZOOKEEPER_CONTAINER_NAME, - WAIT_FOR_NAMENODES_CONTAINER_NAME, ZKFC_CONTAINER_NAME, - }, +use crate::controller::build::container::{ + ContainerConfig, FORMAT_NAMENODES_CONTAINER_NAME, FORMAT_ZOOKEEPER_CONTAINER_NAME, + WAIT_FOR_NAMENODES_CONTAINER_NAME, ZKFC_CONTAINER_NAME, }; // We have a maximum of 4 continuous logging files for Namenodes. Datanodes and Journalnodes @@ -59,6 +56,13 @@ const FORMAT_NAMENODES_LOG_FILE: &str = "format-namenodes.log4j.xml"; const FORMAT_ZOOKEEPER_LOG_FILE: &str = "format-zookeeper.log4j.xml"; const WAIT_FOR_NAMENODES_LOG_FILE: &str = "wait-for-namenodes.log4j.xml"; +/// The main `hdfs` container of every role logs into this directory, whatever the container +/// itself is named (`namenode`, `datanode`, `journalnode`). +/// +/// Vector parses the `container` label out of the log path (see the `files_log4j` source in +/// `vector.yaml`), so this name reaches the aggregated logs. +const HDFS_LOG_DIR_NAME: &str = "hdfs"; + /// The vendored Vector agent configuration (`vector.yaml`). /// /// It is static: per-rolegroup values (namespace, cluster, role, role group, log/data dirs and the @@ -72,102 +76,94 @@ pub fn vector_config_file_content() -> String { VECTOR_CONFIG.to_owned() } -/// Renders the `*.log4j.properties` files for every container of this role group that uses the -/// operator's automatic logging configuration. +/// Everything about how one container logs: which file holds its `log4j.properties` in the role +/// group `ConfigMap`, which directory it logs into, which file it writes and how large that file +/// may grow. /// -/// Returns `(filename, rendered content)` pairs; containers using a custom log ConfigMap are -/// skipped, so the result is empty when none use automatic logging. -pub fn build_log4j_configs( - logging: &RoleGroupLogging, - role: &RoleSpecificValues, -) -> Vec<(&'static str, String)> { - let mut configs = Vec::new(); - - add_log4j_config_if_automatic( - &mut configs, - &logging.hdfs, - HDFS_LOG4J_CONFIG_FILE, - "hdfs", - HDFS_LOG_FILE, - MAX_HDFS_LOG_FILE_SIZE, - ); - - // Exhaustive, so a role's containers and their log4j configs cannot drift apart. - match role { - RoleSpecificValues::Journal => {} - RoleSpecificValues::Name { - zkfc, - format_namenodes, - format_zookeeper, - } => { - add_log4j_config_if_automatic( - &mut configs, - zkfc, - ZKFC_LOG4J_CONFIG_FILE, - ZKFC_CONTAINER_NAME.as_ref(), - ZKFC_LOG_FILE, - MAX_ZKFC_LOG_FILE_SIZE, - ); - add_log4j_config_if_automatic( - &mut configs, - format_namenodes, - FORMAT_NAMENODES_LOG4J_CONFIG_FILE, - FORMAT_NAMENODES_CONTAINER_NAME.as_ref(), - FORMAT_NAMENODES_LOG_FILE, - MAX_FORMAT_NAMENODE_LOG_FILE_SIZE, - ); - add_log4j_config_if_automatic( - &mut configs, - format_zookeeper, - FORMAT_ZOOKEEPER_LOG4J_CONFIG_FILE, - FORMAT_ZOOKEEPER_CONTAINER_NAME.as_ref(), - FORMAT_ZOOKEEPER_LOG_FILE, - MAX_FORMAT_ZOOKEEPER_LOG_FILE_SIZE, - ); - } - RoleSpecificValues::Data { - wait_for_namenodes, .. - } => { - add_log4j_config_if_automatic( - &mut configs, - wait_for_namenodes, - WAIT_FOR_NAMENODES_LOG4J_CONFIG_FILE, - WAIT_FOR_NAMENODES_CONTAINER_NAME.as_ref(), - WAIT_FOR_NAMENODES_LOG_FILE, - MAX_WAIT_NAMENODES_LOG_FILE_SIZE, - ); - } +/// One match, so a container's four log4j facts sit together and cannot drift apart. +struct Log4jSpec { + config_file: &'static str, + log_dir_name: &'static str, + log_file: &'static str, + max_log_file_size: MemoryQuantity, +} + +fn log4j_spec(container: &ContainerConfig) -> Log4jSpec { + match container { + ContainerConfig::Hdfs { .. } => Log4jSpec { + config_file: HDFS_LOG4J_CONFIG_FILE, + log_dir_name: HDFS_LOG_DIR_NAME, + log_file: HDFS_LOG_FILE, + max_log_file_size: MAX_HDFS_LOG_FILE_SIZE, + }, + ContainerConfig::Zkfc => Log4jSpec { + config_file: ZKFC_LOG4J_CONFIG_FILE, + log_dir_name: ZKFC_CONTAINER_NAME.as_ref(), + log_file: ZKFC_LOG_FILE, + max_log_file_size: MAX_ZKFC_LOG_FILE_SIZE, + }, + ContainerConfig::FormatNameNodes => Log4jSpec { + config_file: FORMAT_NAMENODES_LOG4J_CONFIG_FILE, + log_dir_name: FORMAT_NAMENODES_CONTAINER_NAME.as_ref(), + log_file: FORMAT_NAMENODES_LOG_FILE, + max_log_file_size: MAX_FORMAT_NAMENODE_LOG_FILE_SIZE, + }, + ContainerConfig::FormatZooKeeper => Log4jSpec { + config_file: FORMAT_ZOOKEEPER_LOG4J_CONFIG_FILE, + log_dir_name: FORMAT_ZOOKEEPER_CONTAINER_NAME.as_ref(), + log_file: FORMAT_ZOOKEEPER_LOG_FILE, + max_log_file_size: MAX_FORMAT_ZOOKEEPER_LOG_FILE_SIZE, + }, + ContainerConfig::WaitForNameNodes => Log4jSpec { + config_file: WAIT_FOR_NAMENODES_LOG4J_CONFIG_FILE, + log_dir_name: WAIT_FOR_NAMENODES_CONTAINER_NAME.as_ref(), + log_file: WAIT_FOR_NAMENODES_LOG_FILE, + max_log_file_size: MAX_WAIT_NAMENODES_LOG_FILE_SIZE, + }, } +} - configs +/// The `ConfigMap` key holding the given container's `log4j.properties`. +/// +/// The container copies the file from there into its config directory on startup, so the key the +/// `ConfigMap` is written with and the name the container copies must agree. +pub(crate) fn log4j_config_file(container: &ContainerConfig) -> &'static str { + log4j_spec(container).config_file } -fn add_log4j_config_if_automatic( - configs: &mut Vec<(&'static str, String)>, - log_config: &ContainerLogConfig, - log_config_file: &'static str, - log_dir_name: &str, - log_file: &str, - max_log_file_size: MemoryQuantity, -) { - if let ContainerLogConfig { +/// The given container's rendered `log4j.properties` and the `ConfigMap` key to store it under, +/// if that container uses the operator's automatic logging configuration. +/// +/// `None` for a container using a custom log `ConfigMap`: it mounts its own. +pub(crate) fn log4j_config( + container: &ContainerConfig, + container_log_config: &ContainerLogConfig, +) -> Option<(String, String)> { + let ContainerLogConfig { choice: Some(ContainerLogConfigChoice::Automatic(log_config)), - } = log_config - { - configs.push(( - log_config_file, - product_logging::framework::create_log4j_config( - &format!("{STACKABLE_LOG_DIR}/{log_dir_name}"), - log_file, - max_log_file_size - .scale_to(BinaryMultiple::Mebi) - .floor() - .value as u32, - CONSOLE_CONVERSION_PATTERN, - log_config, + } = container_log_config + else { + return None; + }; + + let spec = log4j_spec(container); + + Some(( + spec.config_file.to_owned(), + product_logging::framework::create_log4j_config( + &format!( + "{STACKABLE_LOG_DIR}/{log_dir_name}", + log_dir_name = spec.log_dir_name ), - )); - } + spec.log_file, + spec.max_log_file_size + .scale_to(BinaryMultiple::Mebi) + .floor() + .value as u32, + CONSOLE_CONVERSION_PATTERN, + log_config, + ), + )) } #[cfg(test)] diff --git a/rust/operator-binary/src/controller/build/resolve.rs b/rust/operator-binary/src/controller/build/resolve.rs deleted file mode 100644 index 66fcea75..00000000 --- a/rust/operator-binary/src/controller/build/resolve.rs +++ /dev/null @@ -1,268 +0,0 @@ -//! Resolving one role group into the values the shared builders cannot derive themselves. -//! -//! One [`RoleGroupResolver`] impl per role config type, so a role's resolution is written once. - -use std::{fmt::Display, marker::PhantomData}; - -use snafu::ResultExt; -use stackable_operator::{ - k8s_openapi::api::core::v1::{PersistentVolumeClaim, ResourceRequirements, Volume}, - kvp::Labels, - product_logging::spec::{ContainerLogConfig, Logging}, - v2::types::operator::RoleGroupName, -}; - -use super::{Error, ListenerVolumeSnafu, VolumeClaimTemplatesSnafu, container::ContainerConfig}; -use crate::crd::{ - CommonNodeConfig, DataNodeConfig, DataNodeContainer, HdfsNodeRole, JournalNodeConfig, - JournalNodeContainer, NameNodeConfig, NameNodeContainer, - storage::DataNodeStorageConfigInnerType, -}; - -/// The log config of the two containers every role has. Containers only one role runs carry theirs -/// in [`RoleSpecificValues`], which is the single place the role is decided. -#[derive(Debug)] -pub struct RoleGroupLogging { - /// The main `hdfs` container, which every role has. - pub hdfs: ContainerLogConfig, - /// The Vector sidecar; `None` when the Vector agent is disabled for this role group. - pub vector: Option, -} - -/// The values the shared builders cannot derive themselves, resolved by -/// [`RoleGroupResolver::resolve`], which knows the role. -/// -/// Every builder takes `RoleGroupConfig` and `ResolvedRoleGroup` together, so one role's -/// overrides and replica count cannot be paired with another role's resolved values: both are the -/// same `C` or they do not compile. -pub struct ResolvedRoleGroup { - /// The selector labels of the role group's pods, also used as the `StatefulSet` selector and - /// on its listener volume. - /// - /// We must use the selector labels and not the recommended labels for the listener volumes. - /// This is because the recommended set contains a "managed-by" label. That label triggers the - /// cluster resources to "manage" listeners, which is wrong and leads to errors. The listeners - /// are managed by the listener-operator. - pub selector_labels: Labels, - /// The role group's merged config that is common to every role. - pub common: CommonNodeConfig, - /// The resource requirements of the role group's main and init containers; the ZKFC sidecar - /// has fixed requirements of its own and ignores this. - pub resources: ResourceRequirements, - /// The `StatefulSet`'s persistent volume claim templates. - pub volume_claim_templates: Vec, - /// The values that exist for this role only. - pub role: RoleSpecificValues, - /// The log config of each of the role group's containers. - pub logging: RoleGroupLogging, - /// Ties the bundle to its config type. Needed because `C` appears in no other field, which on - /// its own does not compile (`E0392`). Private, so [`RoleGroupResolver::resolve`] is the only - /// constructor outside this module — a struct literal elsewhere is `E0451`. - _config: PhantomData, -} - -/// Everything that exists for one role only: the containers that role runs, their log configs, and -/// its storage and listener arrangements. -/// -/// An enum rather than `Option` fields, so consumers are exhaustive and three silent failures do -/// not compile: a datanode without its storage drops `dfs.datanode.data.dir` and sends its blocks -/// to container-local storage; a namenode with a pod-level listener volume collides with the -/// identically named claim template and is rejected at apply time; a container without its log -/// config falls back to Hadoop's built-in logging, uncollected by Vector. -pub enum RoleSpecificValues { - /// Journalnodes run no role-specific container, have no listener and no role-specific - /// storage configuration. - Journal, - /// Namenodes run the `zkfc` side container and the `format-namenodes` and `format-zookeeper` - /// init containers. They get their listener from a volume claim template in - /// [`ResolvedRoleGroup::volume_claim_templates`], for stable per-pod identity, so they have - /// no pod-level listener volume. - Name { - zkfc: ContainerLogConfig, - format_namenodes: ContainerLogConfig, - format_zookeeper: ContainerLogConfig, - }, - /// Datanodes run the `wait-for-namenodes` init container. They need no stable per-pod - /// identity, so their listener is an ephemeral pod volume, and they are the only role that - /// configures `dfs.datanode.data.dir`. - Data { - listener_volume: Volume, - storage: DataNodeStorageConfigInnerType, - wait_for_namenodes: ContainerLogConfig, - }, -} - -impl RoleSpecificValues { - /// The role group's ephemeral listener volume; only datanodes have one. - pub fn listener_volume(&self) -> Option<&Volume> { - match self { - Self::Data { - listener_volume, .. - } => Some(listener_volume), - Self::Journal | Self::Name { .. } => None, - } - } - - /// The datanode data volume configuration, which drives `dfs.datanode.data.dir`; `None` for - /// the other roles. - pub fn datanode_storage(&self) -> Option<&DataNodeStorageConfigInnerType> { - match self { - Self::Data { storage, .. } => Some(storage), - Self::Journal | Self::Name { .. } => None, - } - } -} - -/// The log config of the two containers every role has: the main `hdfs` container, and the Vector -/// sidecar, which is `None` when the Vector agent is disabled for the role group. -/// -/// Each role names these containers with its own enum, so this is generic over that enum rather -/// than repeated once per role. -fn common_container_logging( - logging: &Logging, - hdfs: T, - vector: T, -) -> (ContainerLogConfig, Option) -where - T: Clone + Display + Ord, -{ - ( - logging.for_container(&hdfs).into_owned(), - logging - .enable_vector_agent - .then(|| logging.for_container(&vector).into_owned()), - ) -} - -/// How to resolve one role group's role-specific values, implemented once per role config type. -/// -/// The trait supplies the role and the single role-dependent step, which is what lets -/// [`build_role`](super::build_role) be written once. The shared builders read [`Self::ROLE`] -/// instead of taking a role parameter a caller could pair with the wrong config. -pub(crate) trait RoleGroupResolver: Sized { - /// The role whose config this is. - const ROLE: HdfsNodeRole; - - /// Resolves everything the shared builders cannot derive themselves. Takes the selector - /// labels because two of the three roles need them to build their listener. - fn resolve( - &self, - role_group_name: &RoleGroupName, - selector_labels: Labels, - ) -> Result, Error>; -} - -impl RoleGroupResolver for JournalNodeConfig { - const ROLE: HdfsNodeRole = HdfsNodeRole::Journal; - - fn resolve( - &self, - _role_group_name: &RoleGroupName, - selector_labels: Labels, - ) -> Result, Error> { - let (hdfs, vector) = common_container_logging( - &self.logging, - JournalNodeContainer::Hdfs, - JournalNodeContainer::Vector, - ); - - Ok(ResolvedRoleGroup { - selector_labels, - common: self.common.clone(), - resources: self.resources.clone().into(), - volume_claim_templates: ContainerConfig::journalnode_volume_claim_templates(self), - role: RoleSpecificValues::Journal, - logging: RoleGroupLogging { hdfs, vector }, - _config: PhantomData, - }) - } -} - -impl RoleGroupResolver for NameNodeConfig { - const ROLE: HdfsNodeRole = HdfsNodeRole::Name; - - fn resolve( - &self, - role_group_name: &RoleGroupName, - selector_labels: Labels, - ) -> Result, Error> { - // Namenodes get their listener from a persistent volume claim template, for stable - // per-pod identity, rather than from an ephemeral volume. - let volume_claim_templates = - ContainerConfig::namenode_volume_claim_templates(self, &selector_labels).context( - VolumeClaimTemplatesSnafu { - role: Self::ROLE, - role_group: role_group_name.clone(), - }, - )?; - - let (hdfs, vector) = common_container_logging( - &self.logging, - NameNodeContainer::Hdfs, - NameNodeContainer::Vector, - ); - - Ok(ResolvedRoleGroup { - selector_labels, - common: self.common.clone(), - resources: self.resources.clone().into(), - volume_claim_templates, - role: RoleSpecificValues::Name { - zkfc: self - .logging - .for_container(&NameNodeContainer::Zkfc) - .into_owned(), - format_namenodes: self - .logging - .for_container(&NameNodeContainer::FormatNameNodes) - .into_owned(), - format_zookeeper: self - .logging - .for_container(&NameNodeContainer::FormatZooKeeper) - .into_owned(), - }, - logging: RoleGroupLogging { hdfs, vector }, - _config: PhantomData, - }) - } -} - -impl RoleGroupResolver for DataNodeConfig { - const ROLE: HdfsNodeRole = HdfsNodeRole::Data; - - fn resolve( - &self, - role_group_name: &RoleGroupName, - selector_labels: Labels, - ) -> Result, Error> { - // Datanodes use an ephemeral listener volume, since they need no stable per-pod identity. - let listener_volume = ContainerConfig::datanode_listener_volume(self, &selector_labels) - .context(ListenerVolumeSnafu { - role: Self::ROLE, - role_group: role_group_name.clone(), - })?; - - let (hdfs, vector) = common_container_logging( - &self.logging, - DataNodeContainer::Hdfs, - DataNodeContainer::Vector, - ); - - Ok(ResolvedRoleGroup { - selector_labels, - common: self.common.clone(), - resources: self.resources.clone().into(), - volume_claim_templates: ContainerConfig::datanode_volume_claim_templates(self), - role: RoleSpecificValues::Data { - listener_volume, - storage: self.resources.storage.clone(), - wait_for_namenodes: self - .logging - .for_container(&DataNodeContainer::WaitForNameNodes) - .into_owned(), - }, - logging: RoleGroupLogging { hdfs, vector }, - _config: PhantomData, - }) - } -} diff --git a/rust/operator-binary/src/controller/build/resource/config_map.rs b/rust/operator-binary/src/controller/build/resource/config_map.rs index f77ebf9c..dd3bf6de 100644 --- a/rust/operator-binary/src/controller/build/resource/config_map.rs +++ b/rust/operator-binary/src/controller/build/resource/config_map.rs @@ -1,30 +1,23 @@ //! Build the per-rolegroup `ConfigMap` for the HdfsCluster. +//! +//! [`common_config_map_data`] renders the files every role group gets, during the gather phase. +//! [`build_config_map`] then wraps whatever the role group ended up with, its own containers' +//! `log4j.properties` included, in the `ConfigMap` object. use snafu::{ResultExt, Snafu}; use stackable_operator::{ - builder::configmap::ConfigMapBuilder, - k8s_openapi::api::core::v1::ConfigMap, - product_logging::framework::VECTOR_CONFIG_FILE, - utils::cluster_info::KubernetesClusterInfo, - v2::{ - config_file_writer::PropertiesWriterError, - role_utils::{JavaCommonConfig, RoleGroupConfig}, - types::operator::RoleGroupName, - }, + builder::configmap::ConfigMapBuilder, k8s_openapi::api::core::v1::ConfigMap, + product_logging::framework::VECTOR_CONFIG_FILE, v2::config_file_writer::PropertiesWriterError, }; -use crate::{ - controller::{ - ValidatedCluster, - build::{ - self, ResolvedRoleGroup, RoleGroupResolver, - properties::{ - ConfigFileName, core_site, hadoop_policy, hdfs_site, product_logging, - security_properties, ssl_client, ssl_server, - }, - }, +use crate::controller::build::{ + self, + container::ContainerConfig, + properties::{ + ConfigFileName, core_site, hadoop_policy, hdfs_site, product_logging, security_properties, + ssl_client, ssl_server, }, - crd::v1alpha1, + role_group::{RoleGroupBuilder, RoleGroupInputs}, }; #[derive(Snafu, Debug)] @@ -45,85 +38,96 @@ pub enum Error { type Result = std::result::Result; -/// Builds the [`ConfigMap`] of one role group. -/// -/// Every role-specific value is resolved by the caller into `resolved`. The role comes from -/// `C::ROLE`, and `C`'s [`RoleGroupResolver`] bound ties it to `resolved`, so this cannot read one -/// role's `HdfsNodeRole` alongside another role's resolved values. The datanode storage -/// configuration comes from `resolved` rather than a separate parameter: taking it independently -/// would let a caller pass a datanode without its storage, which silently drops -/// `dfs.datanode.data.dir`. -pub fn build_rolegroup_config_map( - cluster: &ValidatedCluster, - cluster_info: &KubernetesClusterInfo, - role_group_name: &RoleGroupName, - rolegroup_config: &RoleGroupConfig, - resolved: &ResolvedRoleGroup, -) -> Result { - let role = C::ROLE; - - tracing::info!( - "Setting up ConfigMap for role {role} role group {role_group_name}", - role = role.as_ref() - ); - - let metadata = build::rolegroup_metadata(cluster, &role, role_group_name); - - let config_overrides = &rolegroup_config.config_overrides; +/// The `ConfigMap` entries every role group gets: the Hadoop XML configs, the JVM security +/// properties, the main `hdfs` container's `log4j.properties` and, when the Vector agent is +/// enabled, the static Vector config. +pub(crate) fn common_config_map_data(inputs: &RoleGroupInputs) -> Result> { + let cluster = inputs.cluster; + let cluster_info = inputs.cluster_info; + let role = &inputs.role; + let role_group_name = &inputs.role_group_name; + let config_overrides = &inputs.config_overrides; let cluster_config = &cluster.cluster_config; - let hdfs_site_xml = hdfs_site::build( - cluster, - cluster_info, - resolved.role.datanode_storage().cloned(), - config_overrides.hdfs_site_xml.clone(), - ); - let core_site_xml = core_site::build( - cluster, - role, - cluster_info, - config_overrides.core_site_xml.clone(), - ); - let hadoop_policy_xml = hadoop_policy::build(config_overrides.hadoop_policy_xml.clone()); - let ssl_server_xml = ssl_server::build( - cluster_config.authentication.is_some(), - config_overrides.ssl_server_xml.clone(), - ); - let ssl_client_xml = ssl_client::build( - cluster_config.authentication.is_some(), - config_overrides.ssl_client_xml.clone(), - ); - - let mut builder = ConfigMapBuilder::new(); - builder - .metadata(metadata.build()) - .add_data(ConfigFileName::CoreSite.to_string(), core_site_xml) - .add_data(ConfigFileName::HdfsSite.to_string(), hdfs_site_xml) - .add_data(ConfigFileName::HadoopPolicy.to_string(), hadoop_policy_xml) - .add_data(ConfigFileName::SslServer.to_string(), ssl_server_xml) - .add_data(ConfigFileName::SslClient.to_string(), ssl_client_xml) - .add_data( + let mut data = vec![ + ( + ConfigFileName::CoreSite.to_string(), + core_site::build( + cluster, + *role, + cluster_info, + config_overrides.core_site_xml.clone(), + ), + ), + ( + ConfigFileName::HdfsSite.to_string(), + hdfs_site::build( + cluster, + cluster_info, + inputs.datanode_storage.clone(), + config_overrides.hdfs_site_xml.clone(), + ), + ), + ( + ConfigFileName::HadoopPolicy.to_string(), + hadoop_policy::build(config_overrides.hadoop_policy_xml.clone()), + ), + ( + ConfigFileName::SslServer.to_string(), + ssl_server::build( + cluster_config.authentication.is_some(), + config_overrides.ssl_server_xml.clone(), + ), + ), + ( + ConfigFileName::SslClient.to_string(), + ssl_client::build( + cluster_config.authentication.is_some(), + config_overrides.ssl_client_xml.clone(), + ), + ), + ( ConfigFileName::Security.to_string(), security_properties::build(config_overrides.security_properties.clone()).with_context( |_| JvmSecurityPropertiesSnafu { rolegroup: role_group_name.to_string(), }, )?, - ); + ), + ]; - for (log_config_file, log4j_config) in - product_logging::build_log4j_configs(&resolved.logging, &resolved.role) - { - builder.add_data(log_config_file, log4j_config); - } - if resolved.logging.vector.is_some() { - builder.add_data( - VECTOR_CONFIG_FILE, + data.extend(product_logging::log4j_config( + &ContainerConfig::from(*role), + &inputs.hdfs_logging, + )); + + if inputs.vector_logging.is_some() { + data.push(( + VECTOR_CONFIG_FILE.to_owned(), product_logging::vector_config_file_content(), - ); + )); + } + + Ok(data) +} + +/// The role group's `ConfigMap`, from the entries gathered for it. +pub(crate) fn build_config_map(builder: &RoleGroupBuilder) -> Result { + let role = &builder.role; + let role_group_name = &builder.role_group_name; + + tracing::info!( + "Setting up ConfigMap for role {role} role group {role_group_name}", + role = role.as_ref() + ); + + let mut config_map = ConfigMapBuilder::new(); + config_map.metadata(build::rolegroup_metadata(builder.cluster, role, role_group_name).build()); + for (file_name, content) in &builder.config_map_data { + config_map.add_data(file_name, content); } - builder.build().with_context(|_| AssembleSnafu { + config_map.build().with_context(|_| AssembleSnafu { role: role.to_string(), role_group: role_group_name.to_string(), }) diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index a4988c69..26efd1cd 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -1,4 +1,7 @@ //! Builds the rolegroup [`StatefulSet`] for an HDFS role group. +//! +//! Everything this needs is already resolved on the [`RoleGroupBuilder`], so it reads straight +//! through: open the pod, add the containers and volumes gathered for the role group, close it. use snafu::{ResultExt, Snafu}; use stackable_operator::{ @@ -9,111 +12,90 @@ use stackable_operator::{ apimachinery::pkg::apis::meta::v1::LabelSelector, }, kube::api::ObjectMeta, - utils::cluster_info::KubernetesClusterInfo, - v2::{ - role_utils::{JavaCommonConfig, RoleGroupConfig}, - types::operator::RoleGroupName, - }, }; -use crate::{ - controller::{ - ValidatedCluster, - build::{ - self, ResolvedRoleGroup, RoleGroupResolver, - container::{self, ContainerConfig}, - graceful_shutdown::{self, add_graceful_shutdown_config}, - }, - }, - crd::v1alpha1, +use crate::controller::build::{ + self, + container::{self}, + graceful_shutdown::{self, add_graceful_shutdown_config}, + role_group::RoleGroupBuilder, }; #[derive(Snafu, Debug)] pub enum Error { - #[snafu(display("failed to create container and volume configuration"))] - FailedToCreateContainerAndVolumeConfiguration { source: container::Error }, + #[snafu(display("failed to add a volume to the Pod"))] + AddVolume { source: container::Error }, #[snafu(display("failed to configure graceful shutdown"))] GracefulShutdown { source: graceful_shutdown::Error }, + + #[snafu(display("failed to add a volume to the Pod"))] + AddPodVolume { + source: stackable_operator::builder::pod::Error, + }, } -/// Builds the [`StatefulSet`] of one role group. -/// -/// Every role-specific value is resolved by the caller into `resolved`. The role comes from -/// `C::ROLE`, and `resolved` is [`ResolvedRoleGroup`](ResolvedRoleGroup), produced by that same -/// `C`'s [`RoleGroupResolver::resolve`], so it cannot disagree with `resolved`. -pub(crate) fn build_rolegroup_statefulset( - validated: &ValidatedCluster, - cluster_info: &KubernetesClusterInfo, - role_group_name: &RoleGroupName, - rolegroup_config: &RoleGroupConfig, - resolved: &ResolvedRoleGroup, -) -> Result { - let role = &C::ROLE; +/// The role group's [`StatefulSet`]. +pub(crate) fn build_statefulset(builder: &RoleGroupBuilder) -> Result { + let cluster = builder.cluster; + let role = &builder.role; + let role_group_name = &builder.role_group_name; tracing::info!( "Setting up StatefulSet for role {role} role group {role_group_name}", role = role.as_ref() ); - let image = &validated.image; - - // PodBuilder for StatefulSet Pod template. let mut pb = PodBuilder::new(); - - let pb_metadata = ObjectMeta { - labels: Some(resolved.selector_labels.clone().into()), + pb.metadata(ObjectMeta { + labels: Some(builder.selector_labels.clone().into()), ..ObjectMeta::default() - }; - - pb.metadata(pb_metadata) - .image_pull_secrets_from_product_image(image) - .affinity(&resolved.common.affinity) - .service_account_name( - validated - .cluster_resource_names() - .service_account_name() - .to_string(), - ) - .security_context( - PodSecurityContextBuilder::with_stackable_defaults() - .fs_group(1000) - .build(), - ); - - // Adds all containers and volumes to the pod builder. - ContainerConfig::add_containers_and_volumes( - &mut pb, - validated, - cluster_info, - role_group_name, - rolegroup_config, - resolved, + }) + .image_pull_secrets_from_product_image(&cluster.image) + .affinity(&builder.common.affinity) + .service_account_name( + cluster + .cluster_resource_names() + .service_account_name() + .to_string(), ) - .context(FailedToCreateContainerAndVolumeConfigurationSnafu)?; + .security_context( + PodSecurityContextBuilder::with_stackable_defaults() + .fs_group(1000) + .build(), + ); + + pb.add_volumes(builder.pod_volumes.clone()) + .context(AddPodVolumeSnafu)?; + for container in &builder.containers { + pb.add_container(container.clone()); + } + for init_container in &builder.init_containers { + pb.add_init_container(init_container.clone()); + } - add_graceful_shutdown_config(&resolved.common, &mut pb).context(GracefulShutdownSnafu)?; + add_graceful_shutdown_config(&builder.common, &mut pb).context(GracefulShutdownSnafu)?; // The `podOverrides` were already merged (role <- role group) during validation // by the local-`framework` `with_validated_config`. let mut pod_template = pb.build_template(); - pod_template.merge_from(rolegroup_config.pod_overrides.clone()); + pod_template.merge_from(builder.pod_overrides.clone()); let statefulset_spec = StatefulSetSpec { pod_management_policy: Some("OrderedReady".to_string()), - replicas: rolegroup_config.replicas.map(i32::from), + replicas: builder.replicas.map(i32::from), selector: LabelSelector { - match_labels: Some(resolved.selector_labels.clone().into()), + match_labels: Some(builder.selector_labels.clone().into()), ..LabelSelector::default() }, service_name: Some( - validated + cluster .governing_service_name(role, role_group_name) .to_string(), ), template: pod_template, - volume_claim_templates: Some(resolved.volume_claim_templates.clone()), + volume_claim_templates: Some(builder.volume_claim_templates.clone()), ..StatefulSetSpec::default() }; @@ -121,7 +103,7 @@ pub(crate) fn build_rolegroup_statefulset( // This is due to problems that might appear when restarting pods during the initial formatting of namenodes. // See: https://github.com/stackabletech/hdfs-operator/issues/750 (disable restart-controller) // https://github.com/stackabletech/issues/issues/816 (enable restart-controller) - let metadata = build::rolegroup_metadata(validated, role, role_group_name); + let metadata = build::rolegroup_metadata(cluster, role, role_group_name); Ok(StatefulSet { metadata: metadata.build(), diff --git a/rust/operator-binary/src/controller/build/role_group/datanode.rs b/rust/operator-binary/src/controller/build/role_group/datanode.rs new file mode 100644 index 00000000..4a35065c --- /dev/null +++ b/rust/operator-binary/src/controller/build/role_group/datanode.rs @@ -0,0 +1,93 @@ +//! Gathering one datanode role group. + +use snafu::ResultExt; +use stackable_operator::{ + utils::cluster_info::KubernetesClusterInfo, v2::types::operator::RoleGroupName, +}; + +use super::{ExtraContainer, RoleGroupBuilder, RoleGroupInputs, common_container_logging}; +use crate::{ + controller::{ + DataNodeRoleGroupConfig, ValidatedCluster, + build::{ + self, Error, ListenerVolumeSnafu, RoleGroupSelectorLabelsSnafu, + container::ContainerConfig, + }, + }, + crd::{DataNodeContainer, HdfsNodeRole}, +}; + +const ROLE: HdfsNodeRole = HdfsNodeRole::Data; + +/// Gathers one datanode role group. +/// +/// Beyond the `hdfs` main container and the Vector sidecar every role runs, datanodes run one +/// init container that waits for the namenodes. +/// +/// Datanodes need no stable per-pod identity, so their listener is an ephemeral pod volume rather +/// than a claim template as the namenodes' is. They are also the only role that configures +/// `dfs.datanode.data.dir`; losing that is silent, because the datanodes then fall back to +/// Hadoop's default directory, which is container-local. +pub(crate) fn build<'a>( + cluster: &'a ValidatedCluster, + cluster_info: &'a KubernetesClusterInfo, + role_group_name: &RoleGroupName, + rg_config: &DataNodeRoleGroupConfig, +) -> Result, Error> { + let config = &rg_config.config; + + let selector_labels = build::rolegroup_selector_labels(cluster, &ROLE, role_group_name) + .context(RoleGroupSelectorLabelsSnafu { + role: ROLE, + role_group: role_group_name.clone(), + })?; + + let listener_volume = ContainerConfig::datanode_listener_volume(config, &selector_labels) + .context(ListenerVolumeSnafu { + role: ROLE, + role_group: role_group_name.clone(), + })?; + + let (hdfs_logging, vector_logging) = common_container_logging( + &config.logging, + DataNodeContainer::Hdfs, + DataNodeContainer::Vector, + ); + + let inputs = RoleGroupInputs { + cluster, + cluster_info, + role: ROLE, + role_group_name: role_group_name.clone(), + selector_labels, + common: config.common.clone(), + resources: config.resources.clone().into(), + volume_claim_templates: ContainerConfig::datanode_volume_claim_templates(config), + // First, because the pod's `volumes` are an ordered list: moving the listener volume + // changes the pod template of every datanode StatefulSet already running, which rolls + // its pods for no reason. + extra_pod_volumes: vec![listener_volume], + hdfs_logging, + vector_logging, + datanode_storage: Some(config.resources.storage.clone()), + replicas: rg_config.replicas, + config_overrides: rg_config.config_overrides.clone(), + env_overrides: rg_config.env_overrides.clone(), + pod_overrides: rg_config.pod_overrides.clone(), + jvm_argument_overrides: rg_config + .product_specific_common_config + .jvm_argument_overrides + .clone(), + }; + + RoleGroupBuilder::new( + inputs, + vec![ExtraContainer::init( + ContainerConfig::WaitForNameNodes, + config + .logging + .for_container(&DataNodeContainer::WaitForNameNodes) + .into_owned(), + )], + ) +} diff --git a/rust/operator-binary/src/controller/build/role_group/journalnode.rs b/rust/operator-binary/src/controller/build/role_group/journalnode.rs new file mode 100644 index 00000000..f765b452 --- /dev/null +++ b/rust/operator-binary/src/controller/build/role_group/journalnode.rs @@ -0,0 +1,68 @@ +//! Gathering one journalnode role group. + +use snafu::ResultExt; +use stackable_operator::{ + utils::cluster_info::KubernetesClusterInfo, v2::types::operator::RoleGroupName, +}; + +use super::{RoleGroupBuilder, RoleGroupInputs, common_container_logging}; +use crate::{ + controller::{ + JournalNodeRoleGroupConfig, ValidatedCluster, + build::{self, Error, RoleGroupSelectorLabelsSnafu, container::ContainerConfig}, + }, + crd::{HdfsNodeRole, JournalNodeContainer}, +}; + +const ROLE: HdfsNodeRole = HdfsNodeRole::Journal; + +/// Gathers one journalnode role group. +/// +/// Journalnodes run nothing beyond the `hdfs` main container and the Vector sidecar every role +/// runs, hence the empty container list. They are used only internally by the namenodes, so they +/// have no listener, and they do not configure `dfs.datanode.data.dir`. +pub(crate) fn build<'a>( + cluster: &'a ValidatedCluster, + cluster_info: &'a KubernetesClusterInfo, + role_group_name: &RoleGroupName, + rg_config: &JournalNodeRoleGroupConfig, +) -> Result, Error> { + let config = &rg_config.config; + + let selector_labels = build::rolegroup_selector_labels(cluster, &ROLE, role_group_name) + .context(RoleGroupSelectorLabelsSnafu { + role: ROLE, + role_group: role_group_name.clone(), + })?; + + let (hdfs_logging, vector_logging) = common_container_logging( + &config.logging, + JournalNodeContainer::Hdfs, + JournalNodeContainer::Vector, + ); + + let inputs = RoleGroupInputs { + cluster, + cluster_info, + role: ROLE, + role_group_name: role_group_name.clone(), + selector_labels, + common: config.common.clone(), + resources: config.resources.clone().into(), + volume_claim_templates: ContainerConfig::journalnode_volume_claim_templates(config), + extra_pod_volumes: Vec::new(), + hdfs_logging, + vector_logging, + datanode_storage: None, + replicas: rg_config.replicas, + config_overrides: rg_config.config_overrides.clone(), + env_overrides: rg_config.env_overrides.clone(), + pod_overrides: rg_config.pod_overrides.clone(), + jvm_argument_overrides: rg_config + .product_specific_common_config + .jvm_argument_overrides + .clone(), + }; + + RoleGroupBuilder::new(inputs, Vec::new()) +} diff --git a/rust/operator-binary/src/controller/build/role_group/mod.rs b/rust/operator-binary/src/controller/build/role_group/mod.rs new file mode 100644 index 00000000..ca15b43c --- /dev/null +++ b/rust/operator-binary/src/controller/build/role_group/mod.rs @@ -0,0 +1,278 @@ +//! Building the Kubernetes resources of one role group. +//! +//! Two phases, so neither has to reach into the other. First each role's module ([`namenode`], +//! [`datanode`], [`journalnode`]) gathers its [`RoleGroupInputs`] and names the containers only +//! that role runs; [`RoleGroupBuilder::new`] turns those into finished containers, pod volumes +//! and rendered `ConfigMap` entries. Then [`RoleGroupBuilder`]'s `build_*` methods emit the +//! Kubernetes objects, reading straight through fields that are already resolved. +//! +//! The emit phase is identical for every role, so it lives here once. Everything that differs +//! between roles happens in the three role modules' `build` functions. + +mod datanode; +mod journalnode; +mod namenode; + +use std::fmt::Display; + +pub(crate) use datanode::build as build_datanode_role_group; +pub(crate) use journalnode::build as build_journalnode_role_group; +pub(crate) use namenode::build as build_namenode_role_group; +use snafu::ResultExt; +use stackable_operator::{ + k8s_openapi::api::{ + apps::v1::StatefulSet, + core::v1::{ + ConfigMap, Container, PersistentVolumeClaim, PodTemplateSpec, ResourceRequirements, + Service, Volume, + }, + }, + kvp::Labels, + product_logging::spec::{ContainerLogConfig, Logging}, + utils::cluster_info::KubernetesClusterInfo, + v2::{ + builder::pod::container::EnvVarSet, jvm_argument_overrides::JvmArgumentOverrides, + types::operator::RoleGroupName, + }, +}; + +use super::{ + Error, ServiceSnafu, container::ContainerConfig, properties::product_logging, resource, +}; +use crate::{ + controller::ValidatedCluster, + crd::{CommonNodeConfig, HdfsNodeRole, storage::DataNodeStorageConfigInnerType, v1alpha1}, +}; + +/// A container only one role runs, named by that role's module together with the log config that +/// belongs to it. +pub(crate) struct ExtraContainer { + config: ContainerConfig, + logging: ContainerLogConfig, + init: bool, +} + +impl ExtraContainer { + /// A container running alongside the main one for the pod's whole lifetime. + pub(crate) fn side(config: ContainerConfig, logging: ContainerLogConfig) -> Self { + Self { + config, + logging, + init: false, + } + } + + /// A container running to completion before the main one starts. + pub(crate) fn init(config: ContainerConfig, logging: ContainerLogConfig) -> Self { + Self { + config, + logging, + init: true, + } + } +} + +/// One role group's validated configuration, as its role's module reads it off the CRD. +/// +/// [`RoleGroupBuilder::new`] consumes this: every field is either resolved into the builder's +/// finished containers and `ConfigMap` entries, or carried over to the emit phase. +pub(crate) struct RoleGroupInputs<'a> { + pub(crate) cluster: &'a ValidatedCluster, + pub(crate) cluster_info: &'a KubernetesClusterInfo, + pub(crate) role: HdfsNodeRole, + pub(crate) role_group_name: RoleGroupName, + /// The selector labels of the role group's pods, also used as the `StatefulSet` selector and + /// on the listener volumes. + /// + /// We must use the selector labels and not the recommended labels for the listener volumes. + /// This is because the recommended set contains a "managed-by" label. That label triggers the + /// cluster resources to "manage" listeners, which is wrong and leads to errors. The listeners + /// are managed by the listener-operator. + pub(crate) selector_labels: Labels, + /// The role group's merged config that is common to every role. + pub(crate) common: CommonNodeConfig, + /// The resource requirements of the role group's main and init containers; the ZKFC sidecar + /// has fixed requirements of its own and ignores this. + pub(crate) resources: ResourceRequirements, + /// The `StatefulSet`'s persistent volume claim templates. For namenodes these include the + /// listener claim template. + pub(crate) volume_claim_templates: Vec, + /// Pod-level volumes beyond those the containers bring with them. Only datanodes have one, + /// their ephemeral listener volume. + pub(crate) extra_pod_volumes: Vec, + /// The log config of the main `hdfs` container, which every role runs. + pub(crate) hdfs_logging: ContainerLogConfig, + /// The log config of the Vector sidecar; `None` when the Vector agent is disabled for this + /// role group. + pub(crate) vector_logging: Option, + /// The data volume configuration behind `dfs.datanode.data.dir`; `Some` only for datanodes, + /// the one role that configures it. + pub(crate) datanode_storage: Option, + /// The role group's replica count; `None` when unset, which counts as one replica. + pub(crate) replicas: Option, + pub(crate) config_overrides: v1alpha1::HdfsConfigOverrides, + pub(crate) env_overrides: EnvVarSet, + pub(crate) pod_overrides: PodTemplateSpec, + pub(crate) jvm_argument_overrides: JvmArgumentOverrides, +} + +impl RoleGroupInputs<'_> { + /// The name the role group's owned objects share. + pub(crate) fn object_name(&self) -> String { + self.cluster + .role_group_resource_names(&self.role, &self.role_group_name) + .qualified_role_group_name() + .to_string() + } +} + +/// Everything one role group's Kubernetes objects are built from, already resolved. +/// +/// These fields are outputs rather than configuration: the containers are built and the +/// `ConfigMap` entries rendered. The `build_*` methods therefore read straight through, and +/// nothing about which role this is reaches them. +pub(crate) struct RoleGroupBuilder<'a> { + pub(crate) cluster: &'a ValidatedCluster, + pub(crate) role: HdfsNodeRole, + pub(crate) role_group_name: RoleGroupName, + pub(crate) selector_labels: Labels, + /// Carried for the pod's affinity and its graceful shutdown timeout. + pub(crate) common: CommonNodeConfig, + pub(crate) replicas: Option, + pub(crate) pod_overrides: PodTemplateSpec, + pub(crate) volume_claim_templates: Vec, + /// The main `hdfs` container, the Vector sidecar when it is enabled, and the role's own side + /// containers, in the order the role named them. + pub(crate) containers: Vec, + /// The role's init containers, in the order the role named them. + pub(crate) init_containers: Vec, + /// Every pod-level volume the containers and the role need. + pub(crate) pod_volumes: Vec, + /// The rendered `ConfigMap` entries, as (file name, content). + pub(crate) config_map_data: Vec<(String, String)>, +} + +impl<'a> RoleGroupBuilder<'a> { + /// Resolves one role group: builds the containers every role runs plus the `extra_containers` + /// this role named, renders its `ConfigMap` entries and collects its pod volumes. + pub(crate) fn new( + inputs: RoleGroupInputs<'a>, + extra_containers: Vec, + ) -> Result { + let role = inputs.role; + let role_group_name = inputs.role_group_name.clone(); + + let container_error = |source| Error::Container { + source, + role, + role_group: role_group_name.clone(), + }; + + // The role's own volumes come first: the pod's `volumes` are an ordered list, and + // reordering them changes the pod template of every StatefulSet already running. + let mut pod_volumes = inputs.extra_pod_volumes.clone(); + let (mut containers, container_volumes) = + ContainerConfig::common_containers_and_volumes(&inputs).map_err(container_error)?; + pod_volumes.extend(container_volumes); + + let mut config_map_data = + resource::config_map::common_config_map_data(&inputs).map_err(|source| { + Error::ConfigMap { + source, + role, + role_group: role_group_name.clone(), + } + })?; + + let mut init_containers = Vec::new(); + for extra in &extra_containers { + let (container, volumes) = extra + .config + .build_container(&inputs, &extra.logging, extra.init) + .map_err(container_error)?; + + if extra.init { + init_containers.push(container); + } else { + containers.push(container); + } + pod_volumes.extend(volumes); + config_map_data.extend(product_logging::log4j_config(&extra.config, &extra.logging)); + } + + Ok(Self { + cluster: inputs.cluster, + role, + role_group_name: inputs.role_group_name, + selector_labels: inputs.selector_labels, + common: inputs.common, + replicas: inputs.replicas, + pod_overrides: inputs.pod_overrides, + volume_claim_templates: inputs.volume_claim_templates, + containers, + init_containers, + pod_volumes, + config_map_data, + }) + } + + /// The headless and metrics `Service`s. + pub(crate) fn build_services(&self) -> Result, Error> { + let context = || ServiceSnafu { + role: self.role, + role_group: self.role_group_name.clone(), + }; + + Ok(vec![ + resource::service::rolegroup_headless_service( + self.cluster, + &self.role, + &self.role_group_name, + ) + .with_context(|_| context())?, + resource::service::rolegroup_metrics_service( + self.cluster, + &self.role, + &self.role_group_name, + ) + .with_context(|_| context())?, + ]) + } + + pub(crate) fn build_config_map(&self) -> Result { + resource::config_map::build_config_map(self).map_err(|source| Error::ConfigMap { + source, + role: self.role, + role_group: self.role_group_name.clone(), + }) + } + + pub(crate) fn build_statefulset(&self) -> Result { + resource::statefulset::build_statefulset(self).map_err(|source| Error::StatefulSet { + source, + role: self.role, + role_group: self.role_group_name.clone(), + }) + } +} + +/// The log configs of the two containers every role runs: the main `hdfs` container, and the +/// Vector sidecar, which is `None` when the Vector agent is disabled for the role group. +/// +/// Each role names these containers with its own enum, so this is generic over that enum rather +/// than repeated once per role. +fn common_container_logging( + logging: &Logging, + hdfs: T, + vector: T, +) -> (ContainerLogConfig, Option) +where + T: Clone + Display + Ord, +{ + ( + logging.for_container(&hdfs).into_owned(), + logging + .enable_vector_agent + .then(|| logging.for_container(&vector).into_owned()), + ) +} diff --git a/rust/operator-binary/src/controller/build/role_group/namenode.rs b/rust/operator-binary/src/controller/build/role_group/namenode.rs new file mode 100644 index 00000000..6685ca31 --- /dev/null +++ b/rust/operator-binary/src/controller/build/role_group/namenode.rs @@ -0,0 +1,109 @@ +//! Gathering one namenode role group. + +use snafu::ResultExt; +use stackable_operator::{ + utils::cluster_info::KubernetesClusterInfo, v2::types::operator::RoleGroupName, +}; + +use super::{ExtraContainer, RoleGroupBuilder, RoleGroupInputs, common_container_logging}; +use crate::{ + controller::{ + NameNodeRoleGroupConfig, ValidatedCluster, + build::{ + self, Error, RoleGroupSelectorLabelsSnafu, VolumeClaimTemplatesSnafu, + container::ContainerConfig, + }, + }, + crd::{HdfsNodeRole, NameNodeContainer}, +}; + +const ROLE: HdfsNodeRole = HdfsNodeRole::Name; + +/// Gathers one namenode role group. +/// +/// Beyond the `hdfs` main container and the Vector sidecar every role runs, namenodes run the +/// ZooKeeper fail-over controller alongside them and two init containers that format the +/// namenodes and ZooKeeper. +/// +/// Their listener comes from a persistent volume claim template, for stable per-pod identity, so +/// it is among the volume claim templates rather than a pod-level volume. A pod volume and a +/// claim template of the same name would be rejected at apply time, which is why a role has only +/// ever one of the two. +pub(crate) fn build<'a>( + cluster: &'a ValidatedCluster, + cluster_info: &'a KubernetesClusterInfo, + role_group_name: &RoleGroupName, + rg_config: &NameNodeRoleGroupConfig, +) -> Result, Error> { + let config = &rg_config.config; + + let selector_labels = build::rolegroup_selector_labels(cluster, &ROLE, role_group_name) + .context(RoleGroupSelectorLabelsSnafu { + role: ROLE, + role_group: role_group_name.clone(), + })?; + + let volume_claim_templates = + ContainerConfig::namenode_volume_claim_templates(config, &selector_labels).context( + VolumeClaimTemplatesSnafu { + role: ROLE, + role_group: role_group_name.clone(), + }, + )?; + + let (hdfs_logging, vector_logging) = common_container_logging( + &config.logging, + NameNodeContainer::Hdfs, + NameNodeContainer::Vector, + ); + + let inputs = RoleGroupInputs { + cluster, + cluster_info, + role: ROLE, + role_group_name: role_group_name.clone(), + selector_labels, + common: config.common.clone(), + resources: config.resources.clone().into(), + volume_claim_templates, + extra_pod_volumes: Vec::new(), + hdfs_logging, + vector_logging, + datanode_storage: None, + replicas: rg_config.replicas, + config_overrides: rg_config.config_overrides.clone(), + env_overrides: rg_config.env_overrides.clone(), + pod_overrides: rg_config.pod_overrides.clone(), + jvm_argument_overrides: rg_config + .product_specific_common_config + .jvm_argument_overrides + .clone(), + }; + + RoleGroupBuilder::new( + inputs, + vec![ + ExtraContainer::side( + ContainerConfig::Zkfc, + config + .logging + .for_container(&NameNodeContainer::Zkfc) + .into_owned(), + ), + ExtraContainer::init( + ContainerConfig::FormatNameNodes, + config + .logging + .for_container(&NameNodeContainer::FormatNameNodes) + .into_owned(), + ), + ExtraContainer::init( + ContainerConfig::FormatZooKeeper, + config + .logging + .for_container(&NameNodeContainer::FormatZooKeeper) + .into_owned(), + ), + ], + ) +} diff --git a/rust/operator-binary/src/hdfs_controller.rs b/rust/operator-binary/src/hdfs_controller.rs index 7734aa21..10d406ed 100644 --- a/rust/operator-binary/src/hdfs_controller.rs +++ b/rust/operator-binary/src/hdfs_controller.rs @@ -159,18 +159,15 @@ mod test { use std::str::FromStr; use stackable_operator::{ - builder::pod::PodBuilder, client::Client, commons::networking::DomainName, kube::{ Client as KubeClient, Config, - api::ObjectMeta, runtime::{ controller::Action, events::{Recorder, Reporter}, }, }, - kvp::Labels, utils::cluster_info::KubernetesClusterInfo, v2::types::operator::RoleGroupName, }; @@ -178,10 +175,8 @@ mod test { use super::*; use crate::{ HDFS_FULL_CONTROLLER_NAME, - controller::build::{RoleGroupResolver, container::ContainerConfig}, - test_support::{ - datanode_config, datanode_role_group_config, deserialize_cluster, validate_cluster, - }, + controller::build::role_group::build_datanode_role_group, + test_support::{datanode_role_group_config, deserialize_cluster, validate_cluster}, }; #[test] @@ -224,26 +219,29 @@ spec: let validated_cluster = validate_cluster(&hdfs); let role_group_name = RoleGroupName::from_str("default").unwrap(); let role_group_config = datanode_role_group_config(&validated_cluster, &role_group_name); - // Resolved through the production path, so this test cannot drift from what the build - // step actually hands the container builder. - let resolved = datanode_config(&validated_cluster, &role_group_name) - .resolve(&role_group_name, Labels::new()) - .expect("the datanode role group should resolve"); - - let mut pb = PodBuilder::new(); - pb.metadata(ObjectMeta::default()); - ContainerConfig::add_containers_and_volumes( - &mut pb, + let cluster_info = KubernetesClusterInfo { + cluster_domain: DomainName::try_from("cluster.local").unwrap(), + }; + // Built through the production path, so this test cannot drift from what the build step + // actually produces. + let builder = build_datanode_role_group( &validated_cluster, - &KubernetesClusterInfo { - cluster_domain: DomainName::try_from("cluster.local").unwrap(), - }, + &cluster_info, &role_group_name, role_group_config, - &resolved, ) - .unwrap(); - let containers = pb.build().unwrap().spec.unwrap().containers; + .expect("the datanode role group builder should be constructed"); + + let stateful_set = builder + .build_statefulset() + .expect("the datanode StatefulSet should build"); + let containers = stateful_set + .spec + .expect("the StatefulSet has a spec") + .template + .spec + .expect("the Pod template has a spec") + .containers; let env_vars = containers .iter() .find(|c| c.name == role.to_string())