fix: [CPCAP-14941] Perform patroni label migration via patch API

operator/api/patroni/v1/patronicore_types.go

@@ -113,7 +113,7 @@ type Patroni struct {
 	PatroniResourcesLimitMemory  string                   `json:"patroniResourcesLimitMemory,omitempty"`
 	// +kubebuilder:validation:Enum=DEBUG;INFO;WARNING;ERROR;CRITICAL
 	// +kubebuilder:default=WARNING
-	PatroniLogLevel              string                   `json:"patroniLogLevel,omitempty"`
+	PatroniLogLevel string `json:"patroniLogLevel,omitempty"`
 }
 
 type External struct {

operator/pkg/deployment/patroni.go

@@ -362,7 +362,7 @@ func NewPatroniStatefulset(cr *patroniv1.PatroniCore, deploymentIdx int, cluster
 			},
 		})
 	}
-	
+
 	// TLS Section
 	if cr.Spec.Tls != nil && cr.Spec.Tls.Enabled {
 		logger.Info("Mount TLS secret volume")

operator/pkg/helper/patroni_core_helper.go

@@ -33,6 +33,7 @@ import (
 	"github.com/Netcracker/pgskipper-operator/pkg/util"
 	"github.com/pkg/errors"
 	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -722,17 +723,89 @@ func (ph *PatroniHelper) GetStatefulSetIds(statefulsets []*appsv1.StatefulSet) (
 	return ids, nil
 }
 
-func (ph *PatroniHelper) ReloadPatroniIfLegacyMasterLabel(clusterName, patroniUrl string) error {
-	pods, err := ph.GetPodsByLabel(map[string]string{
-		util.PatroniPgTypeLabelKey:  util.PatroniRoleMaster,
-		util.PatroniClusterLabelKey: clusterName,
-	})
+func (ph *PatroniHelper) ReloadPatroniIfLegacyLeader(clusterName, patroniConfigMapName, configMapKey string) error {
+	formerLeaderLabels := map[string]string{
+		util.PatroniPgTypeLabelKey: util.PatroniRoleMaster,
+	}
+	leaderPodList, err := ph.GetPodsByLabel(formerLeaderLabels)
 	if err != nil {
 		return err
 	}
-	if len(pods.Items) == 0 {
+	if len(leaderPodList.Items) == 0 {
 		return nil
 	}
-	logger.Info("Legacy pgtype=master label detected, reloading patroni config")
-	return patroni.ReloadPatroniConfig(patroniUrl)
+	logger.Info("Legacy pgtype=master label detected, modifying and restarting patroni pods")
+
+	patroniConfigMap, err := ph.GetConfigMap(patroniConfigMapName)
+	if err != nil {
+		logger.Error("Failed to find patroni config map", zap.Error(err))
+		return err
+	}
+
+	patroniConfigMap, err = patroni.UpdateKubernetesSettings(patroniConfigMap, util.PatroniRolePrimary, configMapKey)
+	if err != nil {
+		logger.Error("Failed to update patroni config map", zap.Error(err))
+		return err
+	}
+
+	_, err = ph.CreateOrUpdateConfigMap(patroniConfigMap)
+	if err != nil {
+		logger.Error("Failed to update patroni config map", zap.Error(err))
+		return err
+	}
+
+	followerPods, err := ph.GetPodsByLabel(ReplicasLabel)
+	if err != nil {
+		return err
+	}
+
+	for _, pod := range followerPods.Items {
+		logger.Info(fmt.Sprintf("Restarting Patroni pod %s", pod.Name))
+		if err := ph.RestartPatroniPod(pod.Name); err != nil {
+			return err
+		}
+	}
+
+	if err := util.WaitForReplicas(ReplicasLabel, ph.cr.Spec.Patroni.Replicas-1); err != nil {
+		return err
+	}
+
+	for _, leaderPod := range leaderPodList.Items {
+		if err := ph.RestartPatroniPod(leaderPod.Name); err != nil {
+			return err
+		}
+	}
+
+	if err := util.WaitForPatroni(&ph.cr, MasterLabel, ReplicasLabel); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (ph *PatroniHelper) RestartPatroniPod(podName string) error {
+	return ph.RestartPatroniPodWithWait(podName, 0)
+}
+
+func (ph *PatroniHelper) RestartPatroniPodWithWait(podName string, waitTimeout time.Duration) error {
+	pod := &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      podName,
+			Namespace: util.GetNameSpace(),
+		},
+	}
+	stsName := strings.TrimSuffix(pod.Name, "-0")
+	if err := ph.ScaleStatefulSet(stsName, 0); err != nil {
+		return err
+	}
+
+	if err := util.WaitDeletePod(pod); err != nil {
+		return err
+	}
+
+	time.Sleep(waitTimeout)
+
+	if err := ph.ScaleStatefulSet(stsName, 1); err != nil {
+		return err
+	}
+	return nil
 }

operator/pkg/patroni/patroni.go

@@ -650,26 +650,28 @@ func Switchover(patroniURL, leader string) error {
 	return nil
 }
 
-func ReloadPatroniConfig(patroniUrl string) error {
-	hosts, err := getPatroniHosts(patroniUrl)
-	if err != nil {
-		return err
+func UpdateKubernetesSettings(configMap *corev1.ConfigMap, labelValue string, configMapKey string) (*corev1.ConfigMap, error) {
+	var config map[string]interface{}
+	if err := yaml.Unmarshal([]byte(configMap.Data[configMapKey]), &config); err != nil {
+		logger.Error("Could not unmarshal patroni config map", zap.Error(err))
+		return configMap, err
 	}
-	client := &http.Client{}
-	for _, host := range hosts {
-		resp, err := patroniPost(client, host+"reload", nil)
-		if err != nil {
-			return fmt.Errorf("patroni reload failed for %s: %w", host, err)
-		}
-		defer func() {
-			if resp != nil && resp.Body != nil {
-				_ = resp.Body.Close()
-			}
-		}()
-		if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
-			return fmt.Errorf("patroni reload failed for %s: status %s", host, resp.Status)
-		}
-		logger.Info(fmt.Sprintf("Patroni config reloaded on %s", host))
+
+	kubernetes, ok := config["kubernetes"].(map[interface{}]interface{})
+	if !ok {
+		err := fmt.Errorf("invalid kubernetes configuration")
+		logger.Error(err.Error())
+		return configMap, err
 	}
-	return nil
+
+	kubernetes["leader_label_value"] = labelValue
+
+	result, err := yaml.Marshal(config)
+	if err != nil {
+		logger.Error("Could not marshal patroni config map", zap.Error(err))
+		return configMap, err
+	}
+
+	configMap.Data[configMapKey] = string(result)
+	return configMap, nil
 }

operator/pkg/reconciler/patroni.go

@@ -78,6 +78,12 @@ func (r *PatroniReconciler) Reconcile() error {
 	cr := r.cr
 	patroniSpec := cr.Spec.Patroni
 	patroniConfigMap := deployment.ConfigMapForPatroni(r.cluster.ClusterName, r.cluster.PatroniCM, r.cluster.ConfigMapKey)
+
+	if err := r.helper.ReloadPatroniIfLegacyLeader(r.cluster.ClusterName, patroniConfigMap.Name, r.cluster.ConfigMapKey); err != nil {
+		logger.Error("Patroni restart for primary label migration failed", zap.Error(err))
+		return err
+	}
+
 	isStandbyClusterPresent := patroni.IsStandbyClusterConfigurationExist(cr)
 	isPgbackrestUsed := cr.Spec.PgBackRest != nil
 
@@ -142,11 +148,6 @@ func (r *PatroniReconciler) Reconcile() error {
 		return err
 	}
 
-	if err := r.helper.ReloadPatroniIfLegacyMasterLabel(r.cluster.ClusterName, r.cluster.PatroniUrl); err != nil {
-		logger.Error("Patroni reload for primary label migration failed", zap.Error(err))
-		return err
-	}
-
 	pgParamsConfigMap := deployment.ConfigMapForPostgreSQL(r.cluster.ClusterName, r.cluster.PatroniPropertiesCM)
 	if _, err := r.helper.CreateOrUpdateConfigMap(pgParamsConfigMap); err != nil {
 		logger.Error(fmt.Sprintf("Cannot create config map %s", pgParamsConfigMap.Name), zap.Error(err))
@@ -610,22 +611,7 @@ func (r *PatroniReconciler) processPatroniPvcResize(pvcs []*corev1.PersistentVol
 
 		logger.Info(fmt.Sprintf("Restarting StatefulSet %s to complete PVC resize, attempt %d, waiting %s", statefulSetName, attempt, delay))
 
-		if err := r.helper.ScaleStatefulSet(statefulSetName, 0); err != nil {
-			return err
-		}
-
-		if err := opUtil.WaitDeletePod(&corev1.Pod{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      podName,
-				Namespace: r.cr.Namespace,
-			},
-		}); err != nil {
-			return err
-		}
-
-		time.Sleep(delay)
-
-		if err := r.helper.ScaleStatefulSet(statefulSetName, 1); err != nil {
+		if err := r.helper.RestartPatroniPodWithWait(podName, delay); err != nil {
 			return err
 		}
 

operator/pkg/util/wait_util.go

@@ -244,7 +244,11 @@ func WaitForLeader(patroniMasterSelector map[string]string) error {
 	})
 }
 
-func waitForReplicas(patroniReplicasSelector map[string]string, numberOfReplicas int, timeout time.Duration) error {
+func WaitForReplicas(patroniReplicasSelector map[string]string, numberOfReplicas int) error {
+	return waitForReplicasWithTimeout(patroniReplicasSelector, numberOfReplicas, getWaitTimeout())
+}
+
+func waitForReplicasWithTimeout(patroniReplicasSelector map[string]string, numberOfReplicas int, timeout time.Duration) error {
 	return wait.PollUntilContextTimeout(context.Background(), time.Second, timeout, true, func(ctx context.Context) (done bool, err error) {
 		return checkPodsByLabel(patroniReplicasSelector, numberOfReplicas)
 	})
@@ -279,7 +283,7 @@ func WaitForPatroniWithReplicaTimeout(cr *v1.PatroniCore, patroniMasterSelector
 			timeout = operatorTimeout
 		}
 
-		if err := waitForReplicas(patroniReplicasSelector, cr.Spec.Patroni.Replicas-1, timeout); err != nil {
+		if err := waitForReplicasWithTimeout(patroniReplicasSelector, cr.Spec.Patroni.Replicas-1, timeout); err != nil {
 			uLog.Error("Failed to wait for replicas, exiting", zap.Error(err))
 			return err
 		}

services/patroni/scripts/setEnv.sh

@@ -15,6 +15,7 @@
 
 
 LOG_LEVEL=${LOG_LEVEL:-info}
+export PATRONI_LOG_LEVEL=${PATRONI_LOG_LEVEL:-WARNING}
 
 if [[ -z "${POD_IDENTITY}" ]];
 then