Kube cluster upgrade

pkg/cluster/cluster.go

@@ -16,6 +16,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/pkg/api/v1"
 	"k8s.io/client-go/pkg/apis/apps/v1beta1"
+	policybeta1 "k8s.io/client-go/pkg/apis/policy/v1beta1"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/cache"
 
@@ -44,10 +45,11 @@ type Config struct {
 }
 
 type kubeResources struct {
-	Services    map[PostgresRole]*v1.Service
-	Endpoint    *v1.Endpoints
-	Secrets     map[types.UID]*v1.Secret
-	Statefulset *v1beta1.StatefulSet
+	Services            map[PostgresRole]*v1.Service
+	Endpoint            *v1.Endpoints
+	Secrets             map[types.UID]*v1.Secret
+	Statefulset         *v1beta1.StatefulSet
+	PodDisruptionBudget *policybeta1.PodDisruptionBudget
 	//Pods are treated separately
 	//PVCs are treated separately
 }
@@ -259,6 +261,12 @@ func (c *Cluster) Create() error {
 		}
 	}
 
+	pdb, err := c.createPodDisruptionBudget()
+	if err != nil {
+		return fmt.Errorf("could not create pod disruption budget: %v", err)
+	}
+	c.logger.Infof("pod disruption budget %q has been successfully created", util.NameFromMeta(pdb.ObjectMeta))
+
 	err = c.listResources()
 	if err != nil {
 		c.logger.Errorf("could not list resources: %v", err)
@@ -334,6 +342,12 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *v1beta1.StatefulSet) *comp
 		needsRollUpdate = true
 		reasons = append(reasons, "new statefulset's terminationGracePeriodSeconds  doesn't match the current one")
 	}
+	if !reflect.DeepEqual(c.Statefulset.Spec.Template.Spec.Affinity, statefulSet.Spec.Template.Spec.Affinity) {
+		needsReplace = true
+		needsRollUpdate = true
+		reasons = append(reasons, "new statefulset's pod affinity doesn't match the current one")
+	}
+
 	// Some generated fields like creationTimestamp make it not possible to use DeepCompare on Spec.Template.ObjectMeta
 	if !reflect.DeepEqual(c.Statefulset.Spec.Template.Labels, statefulSet.Spec.Template.Labels) {
 		needsReplace = true
@@ -522,6 +536,15 @@ func (c *Cluster) Update(newSpec *spec.Postgresql) error {
 		c.logger.Infof("volumes have been updated successfully")
 	}
 
+	newPDB := c.generatePodDisruptionBudget()
+	if match, reason := c.samePDBWith(newPDB); !match {
+		c.logPDBChanges(c.PodDisruptionBudget, newPDB, true, reason)
+		if err := c.updatePodDisruptionBudget(newPDB); err != nil {
+			c.setStatus(spec.ClusterStatusUpdateFailed)
+			return fmt.Errorf("could not update pod disruption budget: %v", err)
+		}
+	}
+
 	c.setStatus(spec.ClusterStatusRunning)
 
 	return nil
@@ -555,6 +578,10 @@ func (c *Cluster) Delete() error {
 		}
 	}
 
+	if err := c.deletePodDisruptionBudget(); err != nil {
+		return fmt.Errorf("could not delete pod disruption budget: %v", err)
+	}
+
 	return nil
 }
 
@@ -690,11 +717,12 @@ func (c *Cluster) GetStatus() *spec.ClusterStatus {
 		Status:  c.Status,
 		Spec:    c.Spec,
 
-		MasterService:  c.GetServiceMaster(),
-		ReplicaService: c.GetServiceReplica(),
-		Endpoint:       c.GetEndpoint(),
-		StatefulSet:    c.GetStatefulSet(),
-		CurrentProcess: c.GetCurrentProcess(),
+		MasterService:       c.GetServiceMaster(),
+		ReplicaService:      c.GetServiceReplica(),
+		Endpoint:            c.GetEndpoint(),
+		StatefulSet:         c.GetStatefulSet(),
+		PodDisruptionBudget: c.GetPodDisruptionBudget(),
+		CurrentProcess:      c.GetCurrentProcess(),
 
 		Error: c.Error,
 	}
@@ -733,3 +761,13 @@ func (c *Cluster) ManualFailover(curMaster *v1.Pod, candidate spec.NamespacedNam
 
 	return nil
 }
+
+// Lock locks the cluster
+func (c *Cluster) Lock() {
+	c.mu.Lock()
+}
+
+// Unlock unlocks the cluster
+func (c *Cluster) Unlock() {
+	c.mu.Unlock()
+}

pkg/cluster/k8sres.go

@@ -10,6 +10,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/pkg/api/v1"
 	"k8s.io/client-go/pkg/apis/apps/v1beta1"
+	policybeta1 "k8s.io/client-go/pkg/apis/policy/v1beta1"
 
 	"github.com/zalando-incubator/postgres-operator/pkg/spec"
 	"github.com/zalando-incubator/postgres-operator/pkg/util/constants"
@@ -67,6 +68,10 @@ func (c *Cluster) serviceName(role PostgresRole) string {
 	return name
 }
 
+func (c *Cluster) podDisruptionBudgetName() string {
+	return c.OpConfig.PDBNameFormat.Format("cluster", c.Spec.ClusterName)
+}
+
 func (c *Cluster) resourceRequirements(resources spec.Resources) (*v1.ResourceRequirements, error) {
 	var err error
 
@@ -226,6 +231,25 @@ PATRONI_INITDB_PARAMS:
 	return string(result)
 }
 
+func (c *Cluster) nodeAffinity() *v1.Affinity {
+	matchExpressions := make([]v1.NodeSelectorRequirement, 0)
+	for k, v := range c.OpConfig.EOLNodeLabel {
+		matchExpressions = append(matchExpressions, v1.NodeSelectorRequirement{
+			Key:      k,
+			Operator: v1.NodeSelectorOpNotIn,
+			Values:   []string{v},
+		})
+	}
+
+	return &v1.Affinity{
+		NodeAffinity: &v1.NodeAffinity{
+			RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
+				NodeSelectorTerms: []v1.NodeSelectorTerm{{MatchExpressions: matchExpressions}},
+			},
+		},
+	}
+}
+
 func (c *Cluster) generatePodTemplate(resourceRequirements *v1.ResourceRequirements,
 	pgParameters *spec.PostgresqlParam,
 	patroniParameters *spec.Patroni,
@@ -347,6 +371,7 @@ func (c *Cluster) generatePodTemplate(resourceRequirements *v1.ResourceRequireme
 		ServiceAccountName:            c.OpConfig.ServiceAccountName,
 		TerminationGracePeriodSeconds: &terminateGracePeriodSeconds,
 		Containers:                    []v1.Container{container},
+		Affinity:                      c.nodeAffinity(),
 	}
 
 	template := v1.PodTemplateSpec{
@@ -573,6 +598,25 @@ func (c *Cluster) generateCloneEnvironment(description *spec.CloneDescription) [
 	return result
 }
 
+func (c *Cluster) generatePodDisruptionBudget() *policybeta1.PodDisruptionBudget {
+	minAvailable := intstr.FromInt(1)
+	matchLabels := c.OpConfig.ClusterLabels
+	matchLabels[c.OpConfig.ClusterNameLabel] = c.Name
+
+	return &policybeta1.PodDisruptionBudget{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      c.podDisruptionBudgetName(),
+			Namespace: c.Namespace,
+		},
+		Spec: policybeta1.PodDisruptionBudgetSpec{
+			MinAvailable: &minAvailable,
+			Selector: &metav1.LabelSelector{
+				MatchLabels: matchLabels,
+			},
+		},
+	}
+}
+
 // getClusterServiceConnectionParameters fetches cluster host name and port
 // TODO: perhaps we need to query the service (i.e. if non-standard port is used?)
 // TODO: handle clusters in different namespaces

pkg/cluster/pod.go

@@ -2,6 +2,7 @@ package cluster
 
 import (
 	"fmt"
+	"math/rand"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/pkg/api/v1"
@@ -71,7 +72,7 @@ func (c *Cluster) deletePods() error {
 }
 
 func (c *Cluster) deletePod(podName spec.NamespacedName) error {
-	c.setProcessName("deleting %q pod", podName)
+	c.setProcessName("deleting pod %q", podName)
 	ch := c.registerPodSubscriber(podName)
 	defer c.unregisterPodSubscriber(podName)
 
@@ -87,6 +88,7 @@ func (c *Cluster) deletePod(podName spec.NamespacedName) error {
 }
 
 func (c *Cluster) unregisterPodSubscriber(podName spec.NamespacedName) {
+	c.logger.Debugf("unsubscribing from pod %q events", podName)
 	c.podSubscribersMu.Lock()
 	defer c.podSubscribersMu.Unlock()
 
@@ -99,6 +101,7 @@ func (c *Cluster) unregisterPodSubscriber(podName spec.NamespacedName) {
 }
 
 func (c *Cluster) registerPodSubscriber(podName spec.NamespacedName) chan spec.PodEvent {
+	c.logger.Debugf("subscribing to pod %q", podName)
 	c.podSubscribersMu.Lock()
 	defer c.podSubscribersMu.Unlock()
 
@@ -111,9 +114,135 @@ func (c *Cluster) registerPodSubscriber(podName spec.NamespacedName) chan spec.P
 	return ch
 }
 
-func (c *Cluster) recreatePod(podName spec.NamespacedName) error {
-	c.setProcessName("recreating %q pod", podName)
+func (c *Cluster) movePodFromEndOfLifeNode(pod *v1.Pod) (*v1.Pod, error) {
+	podName := util.NameFromMeta(pod.ObjectMeta)
 
+	if eol, err := c.podIsEndOfLife(pod); err != nil {
+		return nil, fmt.Errorf("could not get node %q: %v", pod.Spec.NodeName, err)
+	} else if !eol {
+		c.logger.Infof("pod %q is already on a live node", podName)
+		return pod, nil
+	}
+
+	c.setProcessName("moving pod %q out of end-of-life node %q", podName, pod.Spec.NodeName)
+	c.logger.Infof("moving pod %q out of the end-of-life node %q", podName, pod.Spec.NodeName)
+
+	if err := c.recreatePod(podName); err != nil {
+		return nil, fmt.Errorf("could not move pod: %v", err)
+	}
+
+	newPod, err := c.KubeClient.Pods(podName.Namespace).Get(podName.Name, metav1.GetOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("could not get pod: %v", err)
+	}
+
+	if newPod.Spec.NodeName == pod.Spec.NodeName {
+		return nil, fmt.Errorf("pod %q remained on the same node", podName)
+	}
+
+	if eol, err := c.podIsEndOfLife(newPod); err != nil {
+		return nil, fmt.Errorf("could not get node %q: %v", pod.Spec.NodeName, err)
+	} else if eol {
+		c.logger.Warningf("pod %q moved to end-of-life node %q", podName, newPod.Spec.NodeName)
+		return newPod, nil
+	}
+
+	c.logger.Infof("pod %q moved from node %q to node %q", podName, pod.Spec.NodeName, newPod.Spec.NodeName)
+
+	return newPod, nil
+}
+
+func (c *Cluster) masterCandidate(oldNodeName string) (*v1.Pod, error) {
+	replicas, err := c.getRolePods(Replica)
+	if err != nil {
+		return nil, fmt.Errorf("could not get replica pods: %v", err)
+	}
+
+	for i, pod := range replicas {
+		// look for replicas running on live nodes. Ignore errors when querying the nodes.
+		if pod.Spec.NodeName != oldNodeName {
+			eol, err := c.podIsEndOfLife(&pod)
+			if err == nil && !eol {
+				return &replicas[i], nil
+			}
+		}
+	}
+	c.logger.Debug("no available master candidates on live nodes")
+	return &replicas[rand.Intn(len(replicas))], nil
+}
+
+// MigrateMasterPod migrates master pod via failover to a replica
+func (c *Cluster) MigrateMasterPod(podName spec.NamespacedName) error {
+	oldMaster, err := c.KubeClient.Pods(podName.Namespace).Get(podName.Name, metav1.GetOptions{})
+
+	if err != nil {
+		return fmt.Errorf("could not get pod: %v", err)
+	}
+
+	c.logger.Infof("migrating master pod %q", podName)
+
+	if eol, err := c.podIsEndOfLife(oldMaster); err != nil {
+		return fmt.Errorf("could not get node %q: %v", oldMaster.Spec.NodeName, err)
+	} else if !eol {
+		c.logger.Debugf("pod is already on a live node")
+		return nil
+	}
+
+	if role := PostgresRole(oldMaster.Labels[c.OpConfig.PodRoleLabel]); role != Master {
+		c.logger.Warningf("pod %q is not a master", podName)
+		return nil
+	}
+
+	masterCandidatePod, err := c.masterCandidate(oldMaster.Spec.NodeName)
+	if err != nil {
+		return fmt.Errorf("could not get new master candidate: %v", err)
+	}
+
+	pod, err := c.movePodFromEndOfLifeNode(masterCandidatePod)
+	if err != nil {
+		return fmt.Errorf("could not move pod: %v", err)
+	}
+
+	masterCandidateName := util.NameFromMeta(pod.ObjectMeta)
+	if err := c.ManualFailover(oldMaster, masterCandidateName); err != nil {
+		return fmt.Errorf("could not failover to pod %q: %v", masterCandidateName, err)
+	}
+
+	_, err = c.movePodFromEndOfLifeNode(oldMaster)
+	if err != nil {
+		return fmt.Errorf("could not move pod: %v", err)
+	}
+
+	return nil
+}
+
+// MigrateReplicaPod recreates pod on a new node
+func (c *Cluster) MigrateReplicaPod(podName spec.NamespacedName, fromNodeName string) error {
+	replicaPod, err := c.KubeClient.Pods(podName.Namespace).Get(podName.Name, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("could not get pod: %v", err)
+	}
+
+	c.logger.Infof("migrating replica pod %q", podName)
+
+	if replicaPod.Spec.NodeName != fromNodeName {
+		c.logger.Infof("pod %q has already migrated to node %q", podName, replicaPod.Spec.NodeName)
+		return nil
+	}
+
+	if role := PostgresRole(replicaPod.Labels[c.OpConfig.PodRoleLabel]); role != Replica {
+		return fmt.Errorf("pod %q is not a replica", podName)
+	}
+
+	_, err = c.movePodFromEndOfLifeNode(replicaPod)
+	if err != nil {
+		return fmt.Errorf("could not move pod: %v", err)
+	}
+
+	return nil
+}
+
+func (c *Cluster) recreatePod(podName spec.NamespacedName) error {
 	ch := c.registerPodSubscriber(podName)
 	defer c.unregisterPodSubscriber(podName)
 
@@ -127,7 +256,7 @@ func (c *Cluster) recreatePod(podName spec.NamespacedName) error {
 	if err := c.waitForPodLabel(ch, nil); err != nil {
 		return err
 	}
-	c.logger.Infof("pod %q is ready", podName)
+	c.logger.Infof("pod %q has been recreated", podName)
 
 	return nil
 }
@@ -184,3 +313,12 @@ func (c *Cluster) recreatePods() error {
 
 	return nil
 }
+
+func (c *Cluster) podIsEndOfLife(pod *v1.Pod) (bool, error) {
+	node, err := c.KubeClient.Nodes().Get(pod.Spec.NodeName, metav1.GetOptions{})
+	if err != nil {
+		return false, err
+	}
+	return node.Spec.Unschedulable || util.MapContains(node.Labels, c.OpConfig.EOLNodeLabel), nil
+
+}

pkg/cluster/resources.go

@@ -9,6 +9,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/pkg/api/v1"
 	"k8s.io/client-go/pkg/apis/apps/v1beta1"
+	policybeta1 "k8s.io/client-go/pkg/apis/policy/v1beta1"
 
 	"github.com/zalando-incubator/postgres-operator/pkg/spec"
 	"github.com/zalando-incubator/postgres-operator/pkg/util"
@@ -61,10 +62,21 @@ func (c *Cluster) loadResources() error {
 		c.logger.Errorf("could not get statefulset: %v", err)
 	}
 
+	pdb, err := c.KubeClient.PodDisruptionBudgets(ns).Get(c.podDisruptionBudgetName(), metav1.GetOptions{})
+	if err == nil {
+		c.PodDisruptionBudget = pdb
+	} else if !k8sutil.ResourceNotFound(err) {
+		c.logger.Errorf("could not get pod disruption budget: %v", err)
+	}
+
 	return nil
 }
 
 func (c *Cluster) listResources() error {
+	if c.PodDisruptionBudget != nil {
+		c.logger.Infof("found pod disruption budget: %q (uid: %q)", util.NameFromMeta(c.PodDisruptionBudget.ObjectMeta), c.PodDisruptionBudget.UID)
+	}
+
 	if c.Statefulset != nil {
 		c.logger.Infof("found statefulset: %q (uid: %q)", util.NameFromMeta(c.Statefulset.ObjectMeta), c.Statefulset.UID)
 	}
@@ -400,6 +412,54 @@ func (c *Cluster) createEndpoint() (*v1.Endpoints, error) {
 	return endpoints, nil
 }
 
+func (c *Cluster) createPodDisruptionBudget() (*policybeta1.PodDisruptionBudget, error) {
+	if c.PodDisruptionBudget != nil {
+		return nil, fmt.Errorf("pod disruption budget already exists in the cluster")
+	}
+	podDisruptionBudgetSpec := c.generatePodDisruptionBudget()
+	podDisruptionBudget, err := c.KubeClient.
+		PodDisruptionBudgets(podDisruptionBudgetSpec.Namespace).
+		Create(podDisruptionBudgetSpec)
+
+	if err != nil {
+		return nil, err
+	}
+	c.PodDisruptionBudget = podDisruptionBudget
+
+	return podDisruptionBudget, nil
+}
+
+func (c *Cluster) updatePodDisruptionBudget(pdb *policybeta1.PodDisruptionBudget) error {
+	if c.podEventsQueue == nil {
+		return fmt.Errorf("there is no pod disruption budget in the cluster")
+	}
+
+	newPdb, err := c.KubeClient.PodDisruptionBudgets(pdb.Namespace).Update(pdb)
+	if err != nil {
+		return fmt.Errorf("could not update pod disruption budget: %v", err)
+	}
+	c.PodDisruptionBudget = newPdb
+
+	return nil
+}
+
+func (c *Cluster) deletePodDisruptionBudget() error {
+	c.logger.Debug("deleting pod disruption budget")
+	if c.PodDisruptionBudget == nil {
+		return fmt.Errorf("there is no pod disruption budget in the cluster")
+	}
+	err := c.KubeClient.
+		PodDisruptionBudgets(c.PodDisruptionBudget.Namespace).
+		Delete(c.PodDisruptionBudget.Namespace, c.deleteOptions)
+	if err != nil {
+		return fmt.Errorf("could not delete pod disruption budget: %v", err)
+	}
+	c.logger.Infof("pod disruption budget %q has been deleted", util.NameFromMeta(c.PodDisruptionBudget.ObjectMeta))
+	c.PodDisruptionBudget = nil
+
+	return nil
+}
+
 func (c *Cluster) deleteEndpoint() error {
 	c.setProcessName("deleting endpoint")
 	c.logger.Debugln("deleting endpoint")
@@ -408,7 +468,7 @@ func (c *Cluster) deleteEndpoint() error {
 	}
 	err := c.KubeClient.Endpoints(c.Endpoint.Namespace).Delete(c.Endpoint.Name, c.deleteOptions)
 	if err != nil {
-		return err
+		return fmt.Errorf("could not delete endpoint: %v", err)
 	}
 	c.logger.Infof("endpoint %q has been deleted", util.NameFromMeta(c.Endpoint.ObjectMeta))
 	c.Endpoint = nil
@@ -492,3 +552,8 @@ func (c *Cluster) GetEndpoint() *v1.Endpoints {
 func (c *Cluster) GetStatefulSet() *v1beta1.StatefulSet {
 	return c.Statefulset
 }
+
+// GetPodDisruptionBudget returns cluster's kubernetes PodDisruptionBudget
+func (c *Cluster) GetPodDisruptionBudget() *policybeta1.PodDisruptionBudget {
+	return c.PodDisruptionBudget
+}

pkg/cluster/sync.go

@@ -2,6 +2,9 @@ package cluster
 
 import (
 	"fmt"
+	"reflect"
+
+	policybeta1 "k8s.io/client-go/pkg/apis/policy/v1beta1"
 
 	"github.com/zalando-incubator/postgres-operator/pkg/spec"
 	"github.com/zalando-incubator/postgres-operator/pkg/util"
@@ -95,6 +98,12 @@ func (c *Cluster) Sync(newSpec *spec.Postgresql) (err error) {
 		return
 	}
 
+	c.logger.Debug("syncing pod disruption budgets")
+	if err = c.syncPodDisruptionBudget(); err != nil {
+		err = fmt.Errorf("could not sync pod disruption budget: %v", err)
+		return
+	}
+
 	return
 }
 
@@ -140,6 +149,20 @@ func (c *Cluster) syncEndpoint() error {
 	return nil
 }
 
+func (c *Cluster) syncPodDisruptionBudget() error {
+	if c.PodDisruptionBudget == nil {
+		c.logger.Infof("could not find the cluster's pod disruption budget")
+		pdb, err := c.createPodDisruptionBudget()
+		if err != nil {
+			return fmt.Errorf("could not create pod disruption budget: %v", err)
+		}
+		c.logger.Infof("created missing pod disruption budget %q", util.NameFromMeta(pdb.ObjectMeta))
+		return nil
+	}
+
+	return nil
+}
+
 func (c *Cluster) syncStatefulSet() error {
 	cSpec := c.Spec
 	var rollUpdate bool
@@ -248,6 +271,17 @@ func (c *Cluster) syncVolumes() error {
 	if err := c.resizeVolumes(c.Spec.Volume, []volumes.VolumeResizer{&volumes.EBSVolumeResizer{}}); err != nil {
 		return fmt.Errorf("could not sync volumes: %v", err)
 	}
+
 	c.logger.Infof("volumes have been synced successfully")
+
 	return nil
 }
+
+func (c *Cluster) samePDBWith(pdb *policybeta1.PodDisruptionBudget) (match bool, reason string) {
+	match = reflect.DeepEqual(pdb.Spec, c.PodDisruptionBudget.Spec)
+	if !match {
+		reason = "new service spec doesn't match the current one"
+	}
+
+	return
+}

pkg/cluster/util.go

@@ -11,6 +11,7 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/client-go/pkg/api/v1"
 	"k8s.io/client-go/pkg/apis/apps/v1beta1"
+	policybeta1 "k8s.io/client-go/pkg/apis/policy/v1beta1"
 
 	"github.com/zalando-incubator/postgres-operator/pkg/spec"
 	"github.com/zalando-incubator/postgres-operator/pkg/util"
@@ -97,11 +98,21 @@ func metadataAnnotationsPatch(annotations map[string]string) string {
 	return fmt.Sprintf(constants.ServiceMetadataAnnotationReplaceFormat, annotationsString)
 }
 
-func (c *Cluster) logStatefulSetChanges(old, new *v1beta1.StatefulSet, isUpdate bool, reasons []string) {
+func (c *Cluster) logPDBChanges(old, new *policybeta1.PodDisruptionBudget, isUpdate bool, reason string) {
 	if isUpdate {
-		c.logger.Infof("statefulset %q has been changed",
+		c.logger.Infof("pod disruption budget %q has been changed", util.NameFromMeta(old.ObjectMeta))
+	} else {
+		c.logger.Infof("pod disruption budget %q is not in the desired state and needs to be updated",
 			util.NameFromMeta(old.ObjectMeta),
 		)
+	}
+
+	c.logger.Debugf("diff\n%s\n", util.PrettyDiff(old.Spec, new.Spec))
+}
+
+func (c *Cluster) logStatefulSetChanges(old, new *v1beta1.StatefulSet, isUpdate bool, reasons []string) {
+	if isUpdate {
+		c.logger.Infof("statefulset %q has been changed", util.NameFromMeta(old.ObjectMeta))
 	} else {
 		c.logger.Infof("statefulset %q is not in the desired state and needs to be updated",
 			util.NameFromMeta(old.ObjectMeta),
@@ -340,15 +351,11 @@ func (c *Cluster) credentialSecretNameForCluster(username string, clusterName st
 
 	return c.OpConfig.SecretNameTemplate.Format(
 		"username", strings.Replace(username, "_", "-", -1),
-		"clustername", clusterName,
+		"cluster", clusterName,
 		"tprkind", constants.CRDKind,
 		"tprgroup", constants.CRDGroup)
 }
 
-func (c *Cluster) podSpiloRole(pod *v1.Pod) PostgresRole {
-	return PostgresRole(pod.Labels[c.OpConfig.PodRoleLabel])
-}
-
 func masterCandidate(replicas []spec.NamespacedName) spec.NamespacedName {
 	return replicas[rand.Intn(len(replicas))]
 }