allow in place pw rotation of system users (#1953)

* allow in place pw rotation of system users
* block postgres user from rotation
* mark pooler pods for replacement
* adding podsGetter where pooler is synced in unit tests
* move rotation code in extra function

pkg/cluster/connection_pooler.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/r3labs/diff"
 	"github.com/sirupsen/logrus"
@@ -20,6 +21,7 @@ import (
 	"github.com/zalando/postgres-operator/pkg/util/config"
 	"github.com/zalando/postgres-operator/pkg/util/constants"
 	"github.com/zalando/postgres-operator/pkg/util/k8sutil"
+	"github.com/zalando/postgres-operator/pkg/util/retryutil"
 )
 
 // ConnectionPoolerObjects K8s objects that are belong to connection pooler
@@ -73,27 +75,36 @@ func needReplicaConnectionPoolerWorker(spec *acidv1.PostgresSpec) bool {
 		*spec.EnableReplicaConnectionPooler
 }
 
+// when listing pooler k8s objects
+func (c *Cluster) poolerLabelsSet(addExtraLabels bool) labels.Set {
+	poolerLabels := c.labelsSet(addExtraLabels)
+
+	// TODO should be config values
+	poolerLabels["application"] = "db-connection-pooler"
+
+	return poolerLabels
+}
+
 // Return connection pooler labels selector, which should from one point of view
 // inherit most of the labels from the cluster itself, but at the same time
 // have e.g. different `application` label, so that recreatePod operation will
 // not interfere with it (it lists all the pods via labels, and if there would
 // be no difference, it will recreate also pooler pods).
 func (c *Cluster) connectionPoolerLabels(role PostgresRole, addExtraLabels bool) *metav1.LabelSelector {
-	poolerLabels := c.labelsSet(addExtraLabels)
+	poolerLabelsSet := c.poolerLabelsSet(addExtraLabels)
 
 	// TODO should be config values
-	poolerLabels["application"] = "db-connection-pooler"
-	poolerLabels["connection-pooler"] = c.connectionPoolerName(role)
+	poolerLabelsSet["connection-pooler"] = c.connectionPoolerName(role)
 
 	if addExtraLabels {
 		extraLabels := map[string]string{}
 		extraLabels[c.OpConfig.PodRoleLabel] = string(role)
 
-		poolerLabels = labels.Merge(poolerLabels, extraLabels)
+		poolerLabelsSet = labels.Merge(poolerLabelsSet, extraLabels)
 	}
 
 	return &metav1.LabelSelector{
-		MatchLabels:      poolerLabels,
+		MatchLabels:      poolerLabelsSet,
 		MatchExpressions: nil,
 	}
 }
@@ -442,6 +453,14 @@ func (c *Cluster) shouldCreateLoadBalancerForPoolerService(role PostgresRole, sp
 	}
 }
 
+func (c *Cluster) listPoolerPods(listOptions metav1.ListOptions) ([]v1.Pod, error) {
+	pods, err := c.KubeClient.Pods(c.Namespace).List(context.TODO(), listOptions)
+	if err != nil {
+		return nil, fmt.Errorf("could not get list of pooler pods: %v", err)
+	}
+	return pods.Items, nil
+}
+
 //delete connection pooler
 func (c *Cluster) deleteConnectionPooler(role PostgresRole) (err error) {
 	c.logger.Infof("deleting connection pooler spilo-role=%s", role)
@@ -820,6 +839,7 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql
 	var (
 		deployment    *appsv1.Deployment
 		newDeployment *appsv1.Deployment
+		pods          []v1.Pod
 		service       *v1.Service
 		newService    *v1.Service
 		err           error
@@ -909,6 +929,34 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql
 		c.ConnectionPooler[role].Deployment = deployment
 	}
 
+	// check if pooler pods must be replaced due to secret update
+	listOptions := metav1.ListOptions{
+		LabelSelector: labels.Set(c.connectionPoolerLabels(role, true).MatchLabels).String(),
+	}
+	pods, err = c.listPoolerPods(listOptions)
+	if err != nil {
+		return nil, err
+	}
+	for i, pod := range pods {
+		if c.getRollingUpdateFlagFromPod(&pod) {
+			podName := util.NameFromMeta(pods[i].ObjectMeta)
+			err := retryutil.Retry(1*time.Second, 5*time.Second,
+				func() (bool, error) {
+					err2 := c.KubeClient.Pods(podName.Namespace).Delete(
+						context.TODO(),
+						podName.Name,
+						c.deleteOptions)
+					if err2 != nil {
+						return false, err2
+					}
+					return true, nil
+				})
+			if err != nil {
+				return nil, fmt.Errorf("could not delete pooler pod: %v", err)
+			}
+		}
+	}
+
 	if service, err = c.KubeClient.Services(c.Namespace).Get(context.TODO(), c.connectionPoolerName(role), metav1.GetOptions{}); err == nil {
 		c.ConnectionPooler[role].Service = service
 		desiredSvc := c.generateConnectionPoolerService(c.ConnectionPooler[role])