From 6d8f2052e82acd04ea98f43e48d9b874f9f90412 Mon Sep 17 00:00:00 2001 From: Tvion Date: Fri, 4 Sep 2026 17:22:16 +0500 Subject: [PATCH 1/5] fix: [CPCAP-14941] Perfrom patroni label migration via patch API --- operator/api/patroni/v1/patronicore_types.go | 2 +- operator/pkg/deployment/patroni.go | 2 +- operator/pkg/helper/patroni_core_helper.go | 86 ++++++++++++++++++-- operator/pkg/patroni/patroni.go | 42 +++++----- operator/pkg/reconciler/patroni.go | 28 ++----- operator/pkg/util/wait_util.go | 6 +- 6 files changed, 112 insertions(+), 54 deletions(-) diff --git a/operator/api/patroni/v1/patronicore_types.go b/operator/api/patroni/v1/patronicore_types.go index 12d0ec29..2201775b 100644 --- a/operator/api/patroni/v1/patronicore_types.go +++ b/operator/api/patroni/v1/patronicore_types.go @@ -113,7 +113,7 @@ type Patroni struct { PatroniResourcesLimitMemory string `json:"patroniResourcesLimitMemory,omitempty"` // +kubebuilder:validation:Enum=DEBUG;INFO;WARNING;ERROR;CRITICAL // +kubebuilder:default=WARNING - PatroniLogLevel string `json:"patroniLogLevel,omitempty"` + PatroniLogLevel string `json:"patroniLogLevel,omitempty"` } type External struct { diff --git a/operator/pkg/deployment/patroni.go b/operator/pkg/deployment/patroni.go index 4f1268fa..a89d8d50 100644 --- a/operator/pkg/deployment/patroni.go +++ b/operator/pkg/deployment/patroni.go @@ -362,7 +362,7 @@ func NewPatroniStatefulset(cr *patroniv1.PatroniCore, deploymentIdx int, cluster }, }) } - + // TLS Section if cr.Spec.Tls != nil && cr.Spec.Tls.Enabled { logger.Info("Mount TLS secret volume") diff --git a/operator/pkg/helper/patroni_core_helper.go b/operator/pkg/helper/patroni_core_helper.go index 0c06d578..bc4530c9 100644 --- a/operator/pkg/helper/patroni_core_helper.go +++ b/operator/pkg/helper/patroni_core_helper.go @@ -33,6 +33,7 @@ import ( "github.com/Netcracker/pgskipper-operator/pkg/util" "github.com/pkg/errors" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" @@ -722,17 +723,86 @@ func (ph *PatroniHelper) GetStatefulSetIds(statefulsets []*appsv1.StatefulSet) ( return ids, nil } -func (ph *PatroniHelper) ReloadPatroniIfLegacyMasterLabel(clusterName, patroniUrl string) error { - pods, err := ph.GetPodsByLabel(map[string]string{ - util.PatroniPgTypeLabelKey: util.PatroniRoleMaster, - util.PatroniClusterLabelKey: clusterName, - }) +func (ph *PatroniHelper) ReloadPatroniIfLegacyMaster(clusterName, patroniConfigMapName, configMapKey string) error { + leaderPodList, err := ph.GetPodsByLabel(MasterLabel) if err != nil { return err } - if len(pods.Items) == 0 { + if len(leaderPodList.Items) == 0 { return nil } - logger.Info("Legacy pgtype=master label detected, reloading patroni config") - return patroni.ReloadPatroniConfig(patroniUrl) + logger.Info("Legacy pgtype=master label detected, modifying and restarting patroni pods") + + patroniConfigMap, err := ph.GetConfigMap(patroniConfigMapName) + if err != nil { + logger.Error("Failed to find patroni config map", zap.Error(err)) + return err + } + + patroniConfigMap, err = patroni.UpdateKubernetesSettings(patroniConfigMap, util.PatroniRolePrimary, configMapKey) + if err != nil { + logger.Error("Failed to update patroni config map", zap.Error(err)) + return err + } + + _, err = ph.CreateOrUpdateConfigMap(patroniConfigMap) + if err != nil { + logger.Error("Failed to update patroni config map", zap.Error(err)) + return err + } + + followerPods, err := ph.GetPodsByLabel(ReplicasLabel) + if err != nil { + return err + } + + for _, pod := range followerPods.Items { + logger.Info(fmt.Sprintf("Restarting Patroni pod %s", pod.Name)) + if err := ph.RestartPatroniPod(pod.Name); err != nil { + return err + } + } + + if err := util.WaitForReplicas(ReplicasLabel, ph.cr.Spec.Patroni.Replicas-1); err != nil { + return err + } + + for _, leaderPod := range leaderPodList.Items { + if err := ph.RestartPatroniPod(leaderPod.Name); err != nil { + return err + } + } + + if err := util.WaitForPatroni(&ph.cr, MasterLabel, ReplicasLabel); err != nil { + return err + } + return nil +} + +func (ph *PatroniHelper) RestartPatroniPod(podName string) error { + return ph.RestartPatroniPodWithWait(podName, 0) +} + +func (ph *PatroniHelper) RestartPatroniPodWithWait(podName string, waitTimeout time.Duration) error { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: util.GetNameSpace(), + }, + } + stsName := strings.TrimSuffix(pod.Name, "-0") + if err := ph.ScaleStatefulSet(stsName, 0); err != nil { + return err + } + + if err := util.WaitDeletePod(pod); err != nil { + return err + } + + time.Sleep(waitTimeout) + + if err := ph.ScaleStatefulSet(stsName, 1); err != nil { + return err + } + return nil } diff --git a/operator/pkg/patroni/patroni.go b/operator/pkg/patroni/patroni.go index 67ad13c4..c4f0adb2 100644 --- a/operator/pkg/patroni/patroni.go +++ b/operator/pkg/patroni/patroni.go @@ -650,26 +650,28 @@ func Switchover(patroniURL, leader string) error { return nil } -func ReloadPatroniConfig(patroniUrl string) error { - hosts, err := getPatroniHosts(patroniUrl) - if err != nil { - return err +func UpdateKubernetesSettings(configMap *corev1.ConfigMap, labelValue string, configMapKey string) (*corev1.ConfigMap, error) { + var config map[string]interface{} + if err := yaml.Unmarshal([]byte(configMap.Data[configMapKey]), &config); err != nil { + logger.Error("Could not unmarshal patroni config map", zap.Error(err)) + return configMap, err } - client := &http.Client{} - for _, host := range hosts { - resp, err := patroniPost(client, host+"reload", nil) - if err != nil { - return fmt.Errorf("patroni reload failed for %s: %w", host, err) - } - defer func() { - if resp != nil && resp.Body != nil { - _ = resp.Body.Close() - } - }() - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted { - return fmt.Errorf("patroni reload failed for %s: status %s", host, resp.Status) - } - logger.Info(fmt.Sprintf("Patroni config reloaded on %s", host)) + + kubernetes, ok := config["kubernetes"].(map[interface{}]interface{}) + if !ok { + err := fmt.Errorf("invalid kubernetes configuration") + logger.Error(err.Error()) + return configMap, err } - return nil + + kubernetes["leader_label_value"] = labelValue + + result, err := yaml.Marshal(config) + if err != nil { + logger.Error("Could not marshal patroni config map", zap.Error(err)) + return configMap, err + } + + configMap.Data[configMapKey] = string(result) + return configMap, nil } diff --git a/operator/pkg/reconciler/patroni.go b/operator/pkg/reconciler/patroni.go index 04a4ffd9..22b2de3a 100644 --- a/operator/pkg/reconciler/patroni.go +++ b/operator/pkg/reconciler/patroni.go @@ -78,6 +78,12 @@ func (r *PatroniReconciler) Reconcile() error { cr := r.cr patroniSpec := cr.Spec.Patroni patroniConfigMap := deployment.ConfigMapForPatroni(r.cluster.ClusterName, r.cluster.PatroniCM, r.cluster.ConfigMapKey) + + if err := r.helper.ReloadPatroniIfLegacyMaster(r.cluster.ClusterName, patroniConfigMap.Name, r.cluster.ConfigMapKey); err != nil { + logger.Error("Patroni restart for primary label migration failed", zap.Error(err)) + return err + } + isStandbyClusterPresent := patroni.IsStandbyClusterConfigurationExist(cr) isPgbackrestUsed := cr.Spec.PgBackRest != nil @@ -142,11 +148,6 @@ func (r *PatroniReconciler) Reconcile() error { return err } - if err := r.helper.ReloadPatroniIfLegacyMasterLabel(r.cluster.ClusterName, r.cluster.PatroniUrl); err != nil { - logger.Error("Patroni reload for primary label migration failed", zap.Error(err)) - return err - } - pgParamsConfigMap := deployment.ConfigMapForPostgreSQL(r.cluster.ClusterName, r.cluster.PatroniPropertiesCM) if _, err := r.helper.CreateOrUpdateConfigMap(pgParamsConfigMap); err != nil { logger.Error(fmt.Sprintf("Cannot create config map %s", pgParamsConfigMap.Name), zap.Error(err)) @@ -610,22 +611,7 @@ func (r *PatroniReconciler) processPatroniPvcResize(pvcs []*corev1.PersistentVol logger.Info(fmt.Sprintf("Restarting StatefulSet %s to complete PVC resize, attempt %d, waiting %s", statefulSetName, attempt, delay)) - if err := r.helper.ScaleStatefulSet(statefulSetName, 0); err != nil { - return err - } - - if err := opUtil.WaitDeletePod(&corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: podName, - Namespace: r.cr.Namespace, - }, - }); err != nil { - return err - } - - time.Sleep(delay) - - if err := r.helper.ScaleStatefulSet(statefulSetName, 1); err != nil { + if err := r.helper.RestartPatroniPodWithWait(podName, delay); err != nil { return err } diff --git a/operator/pkg/util/wait_util.go b/operator/pkg/util/wait_util.go index 4e973a91..5d19a38e 100644 --- a/operator/pkg/util/wait_util.go +++ b/operator/pkg/util/wait_util.go @@ -244,8 +244,8 @@ func WaitForLeader(patroniMasterSelector map[string]string) error { }) } -func waitForReplicas(patroniReplicasSelector map[string]string, numberOfReplicas int, timeout time.Duration) error { - return wait.PollUntilContextTimeout(context.Background(), time.Second, timeout, true, func(ctx context.Context) (done bool, err error) { +func WaitForReplicas(patroniReplicasSelector map[string]string, numberOfReplicas int) error { + return wait.PollUntilContextTimeout(context.Background(), time.Second, getWaitTimeout(), true, func(ctx context.Context) (done bool, err error) { return checkPodsByLabel(patroniReplicasSelector, numberOfReplicas) }) } @@ -279,7 +279,7 @@ func WaitForPatroniWithReplicaTimeout(cr *v1.PatroniCore, patroniMasterSelector timeout = operatorTimeout } - if err := waitForReplicas(patroniReplicasSelector, cr.Spec.Patroni.Replicas-1, timeout); err != nil { + if err := WaitForReplicas(patroniReplicasSelector, cr.Spec.Patroni.Replicas-1); err != nil { uLog.Error("Failed to wait for replicas, exiting", zap.Error(err)) return err } From e7e6e032d9febee18932ef4fb2561d49d19e6351 Mon Sep 17 00:00:00 2001 From: Tvion Date: Fri, 4 Sep 2026 17:23:05 +0500 Subject: [PATCH 2/5] feat: set default Patroni log level to WARNING in setEnv.sh --- services/patroni/scripts/setEnv.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/services/patroni/scripts/setEnv.sh b/services/patroni/scripts/setEnv.sh index 0c094dac..264f1d71 100755 --- a/services/patroni/scripts/setEnv.sh +++ b/services/patroni/scripts/setEnv.sh @@ -15,6 +15,7 @@ LOG_LEVEL=${LOG_LEVEL:-info} +export PATRONI_LOG_LEVEL=${PATRONI_LOG_LEVEL:-WARNING} if [[ -z "${POD_IDENTITY}" ]]; then From 1cfde389299a6cd54fc7b1f752810494146e7f89 Mon Sep 17 00:00:00 2001 From: Tvion Date: Fri, 4 Sep 2026 17:45:42 +0500 Subject: [PATCH 3/5] fix: [CPCAP-14941] Perfrom patroni label migration via patch API --- operator/pkg/helper/patroni_core_helper.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/operator/pkg/helper/patroni_core_helper.go b/operator/pkg/helper/patroni_core_helper.go index bc4530c9..6077c082 100644 --- a/operator/pkg/helper/patroni_core_helper.go +++ b/operator/pkg/helper/patroni_core_helper.go @@ -724,7 +724,10 @@ func (ph *PatroniHelper) GetStatefulSetIds(statefulsets []*appsv1.StatefulSet) ( } func (ph *PatroniHelper) ReloadPatroniIfLegacyMaster(clusterName, patroniConfigMapName, configMapKey string) error { - leaderPodList, err := ph.GetPodsByLabel(MasterLabel) + formerLeaderLabels := map[string]string{ + util.PatroniPgTypeLabelKey: util.PatroniRoleMaster, + } + leaderPodList, err := ph.GetPodsByLabel(formerLeaderLabels) if err != nil { return err } From 235d8944992d9f5a9c7388a3b0e9a3c84e66ad6e Mon Sep 17 00:00:00 2001 From: Tvion Date: Fri, 4 Sep 2026 18:46:47 +0500 Subject: [PATCH 4/5] fix: [CPCAP-14941] Perfrom patroni label migration via patch API --- operator/pkg/util/wait_util.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/operator/pkg/util/wait_util.go b/operator/pkg/util/wait_util.go index 5d19a38e..06d2de81 100644 --- a/operator/pkg/util/wait_util.go +++ b/operator/pkg/util/wait_util.go @@ -245,7 +245,11 @@ func WaitForLeader(patroniMasterSelector map[string]string) error { } func WaitForReplicas(patroniReplicasSelector map[string]string, numberOfReplicas int) error { - return wait.PollUntilContextTimeout(context.Background(), time.Second, getWaitTimeout(), true, func(ctx context.Context) (done bool, err error) { + return waitForReplicasWithTimeout(patroniReplicasSelector, numberOfReplicas, getWaitTimeout()) +} + +func waitForReplicasWithTimeout(patroniReplicasSelector map[string]string, numberOfReplicas int, timeout time.Duration) error { + return wait.PollUntilContextTimeout(context.Background(), time.Second, timeout, true, func(ctx context.Context) (done bool, err error) { return checkPodsByLabel(patroniReplicasSelector, numberOfReplicas) }) } @@ -279,7 +283,7 @@ func WaitForPatroniWithReplicaTimeout(cr *v1.PatroniCore, patroniMasterSelector timeout = operatorTimeout } - if err := WaitForReplicas(patroniReplicasSelector, cr.Spec.Patroni.Replicas-1); err != nil { + if err := waitForReplicasWithTimeout(patroniReplicasSelector, cr.Spec.Patroni.Replicas-1, timeout); err != nil { uLog.Error("Failed to wait for replicas, exiting", zap.Error(err)) return err } From be8d305bdea1d6ad0912b9db649a6d33b4e156e8 Mon Sep 17 00:00:00 2001 From: Tvion Date: Fri, 4 Sep 2026 18:47:47 +0500 Subject: [PATCH 5/5] fix: [CPCAP-14941] Perfrom patroni label migration via patch API --- operator/pkg/helper/patroni_core_helper.go | 2 +- operator/pkg/reconciler/patroni.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/operator/pkg/helper/patroni_core_helper.go b/operator/pkg/helper/patroni_core_helper.go index 6077c082..cba19fdb 100644 --- a/operator/pkg/helper/patroni_core_helper.go +++ b/operator/pkg/helper/patroni_core_helper.go @@ -723,7 +723,7 @@ func (ph *PatroniHelper) GetStatefulSetIds(statefulsets []*appsv1.StatefulSet) ( return ids, nil } -func (ph *PatroniHelper) ReloadPatroniIfLegacyMaster(clusterName, patroniConfigMapName, configMapKey string) error { +func (ph *PatroniHelper) ReloadPatroniIfLegacyLeader(clusterName, patroniConfigMapName, configMapKey string) error { formerLeaderLabels := map[string]string{ util.PatroniPgTypeLabelKey: util.PatroniRoleMaster, } diff --git a/operator/pkg/reconciler/patroni.go b/operator/pkg/reconciler/patroni.go index 22b2de3a..729dd6ff 100644 --- a/operator/pkg/reconciler/patroni.go +++ b/operator/pkg/reconciler/patroni.go @@ -79,7 +79,7 @@ func (r *PatroniReconciler) Reconcile() error { patroniSpec := cr.Spec.Patroni patroniConfigMap := deployment.ConfigMapForPatroni(r.cluster.ClusterName, r.cluster.PatroniCM, r.cluster.ConfigMapKey) - if err := r.helper.ReloadPatroniIfLegacyMaster(r.cluster.ClusterName, patroniConfigMap.Name, r.cluster.ConfigMapKey); err != nil { + if err := r.helper.ReloadPatroniIfLegacyLeader(r.cluster.ClusterName, patroniConfigMap.Name, r.cluster.ConfigMapKey); err != nil { logger.Error("Patroni restart for primary label migration failed", zap.Error(err)) return err }