From e98439e5b672a2510b9344ec963586ce037ab8eb Mon Sep 17 00:00:00 2001
From: Rafia Sabih <rafia.sabih@gmail.com>
Date: Thu, 18 Nov 2021 09:55:33 +0100
Subject: [PATCH 01/58] Add log messages for usernames (#1692)

* add log messages for usernames
* document behavior better in logs

Co-authored-by: Felix Kunde <felix-kunde@gmx.de>
---
 docs/user.md           | 13 +++++++++----
 pkg/cluster/cluster.go |  1 +
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/docs/user.md b/docs/user.md
index d8783c42f..e0873f274 100644
--- a/docs/user.md
+++ b/docs/user.md
@@ -141,14 +141,18 @@ other roles.
 
 To define the secrets for the users in a different namespace than that of the
 cluster, one can set `enable_cross_namespace_secret` and declare the namespace
-for the secrets in the manifest in the following manner,
+for the secrets in the manifest in the following manner (note, that it has to
+be reflected in the `database` section, too),
 
 ```yaml
 spec:
   users:
-  #users with secret in dfferent namespace
-   appspace.db_user:
+    # users with secret in different namespace
+    appspace.db_user:
     - createdb
+  databases:
+    # namespace notation is part of user name
+    app_db: appspace.db_user
 ```
 
 Here, anything before the first dot is considered the namespace and the text after
@@ -554,7 +558,8 @@ schema creation. This means they are currently not set when `defaultUsers`
 For all LOGIN roles the operator will create K8s secrets in the namespace
 specified in `secretNamespace`, if `enable_cross_namespace_secret` is set to
 `true` in the config. Otherwise, they are created in the same namespace like
-the Postgres cluster.
+the Postgres cluster. Unlike roles specified with `namespace.username` under
+`users`, the namespace will not be part of the role name here.
 
 ```yaml
 spec:
diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go
index 8e1dcb22e..967f9d530 100644
--- a/pkg/cluster/cluster.go
+++ b/pkg/cluster/cluster.go
@@ -1176,6 +1176,7 @@ func (c *Cluster) initRobotUsers() error {
 			if strings.Contains(username, ".") {
 				splits := strings.Split(username, ".")
 				namespace = splits[0]
+				c.logger.Warningf("enable_cross_namespace_secret is set. Database role name contains the respective namespace i.e. %s is the created user", username)
 			}
 		}
 

From a55639d6b7b7182aab11f39c4088ec360c15a84b Mon Sep 17 00:00:00 2001
From: bloever <bendik@incident.no>
Date: Fri, 19 Nov 2021 11:23:44 +0100
Subject: [PATCH 02/58] Adds required Azure backup env variable WALG_AZ_PREFIX
 (#1688)

---
 docs/administrator.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/administrator.md b/docs/administrator.md
index f03326f5a..e0c03e875 100644
--- a/docs/administrator.md
+++ b/docs/administrator.md
@@ -885,6 +885,7 @@ data:
   USE_WALG_BACKUP: "true"
   USE_WALG_RESTORE: "true"
   CLONE_USE_WALG_RESTORE: "true"
+  WALG_AZ_PREFIX: "azure://container-name/$(SCOPE)/$(PGVERSION)" # Enables Azure Backups (SCOPE = Cluster name) (PGVERSION = Postgres version) 
 ```
 
 3. Setup your operator configuration values. With the `psql-backup-creds`

From e1504745df5dc3e9a484883c8304a3f412ada479 Mon Sep 17 00:00:00 2001
From: mujx <mujx@tuta.io>
Date: Fri, 19 Nov 2021 12:23:55 +0200
Subject: [PATCH 03/58] Fix typo on postgres-operator-ui values (#1680)

---
 charts/postgres-operator-ui/values.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charts/postgres-operator-ui/values.yaml b/charts/postgres-operator-ui/values.yaml
index 06004b1ca..ff1555b13 100644
--- a/charts/postgres-operator-ui/values.yaml
+++ b/charts/postgres-operator-ui/values.yaml
@@ -39,7 +39,7 @@ resources:
 
 # configure UI ENVs
 envs:
-  # IMPORTANT: While operator chart and UI chart are idendependent, this is the interface between
+  # IMPORTANT: While operator chart and UI chart are independent, this is the interface between
   # UI and operator API. Insert the service name of the operator API here!
   operatorApiUrl: "http://postgres-operator:8080"
   operatorClusterNameLabel: "cluster-name"

From 0b00372c78967d47c0cdf78b4d98ef4467eb09ea Mon Sep 17 00:00:00 2001
From: Olle Larsson <olle.larsson@elastisys.com>
Date: Fri, 19 Nov 2021 11:25:49 +0100
Subject: [PATCH 04/58] Remove helm2 support from helm chart (#1679)

---
 charts/postgres-operator/crds/operatorconfigurations.yaml | 2 --
 charts/postgres-operator/crds/postgresqls.yaml            | 2 --
 charts/postgres-operator/crds/postgresteams.yaml          | 2 --
 charts/postgres-operator/templates/crds.yaml              | 6 ------
 charts/postgres-operator/values.yaml                      | 5 -----
 5 files changed, 17 deletions(-)
 delete mode 100644 charts/postgres-operator/templates/crds.yaml

diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml
index 90a632fe5..b74ebbb0a 100644
--- a/charts/postgres-operator/crds/operatorconfigurations.yaml
+++ b/charts/postgres-operator/crds/operatorconfigurations.yaml
@@ -4,8 +4,6 @@ metadata:
   name: operatorconfigurations.acid.zalan.do
   labels:
     app.kubernetes.io/name: postgres-operator
-  annotations:
-    "helm.sh/hook": crd-install
 spec:
   group: acid.zalan.do
   names:
diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml
index 9ac4cfb3a..1b5f36db8 100644
--- a/charts/postgres-operator/crds/postgresqls.yaml
+++ b/charts/postgres-operator/crds/postgresqls.yaml
@@ -4,8 +4,6 @@ metadata:
   name: postgresqls.acid.zalan.do
   labels:
     app.kubernetes.io/name: postgres-operator
-  annotations:
-    "helm.sh/hook": crd-install
 spec:
   group: acid.zalan.do
   names:
diff --git a/charts/postgres-operator/crds/postgresteams.yaml b/charts/postgres-operator/crds/postgresteams.yaml
index fbf873b84..b7a36848d 100644
--- a/charts/postgres-operator/crds/postgresteams.yaml
+++ b/charts/postgres-operator/crds/postgresteams.yaml
@@ -4,8 +4,6 @@ metadata:
   name: postgresteams.acid.zalan.do
   labels:
     app.kubernetes.io/name: postgres-operator
-  annotations:
-    "helm.sh/hook": crd-install
 spec:
   group: acid.zalan.do
   names:
diff --git a/charts/postgres-operator/templates/crds.yaml b/charts/postgres-operator/templates/crds.yaml
deleted file mode 100644
index 733830014..000000000
--- a/charts/postgres-operator/templates/crds.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-{{ if .Values.crd.create }}
-{{- range $path, $bytes := .Files.Glob "crds/*.yaml" }}
-{{ $.Files.Get $path }}
----
-{{- end }}
-{{- end }}
diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml
index 1ad096403..9e2303f3b 100644
--- a/charts/postgres-operator/values.yaml
+++ b/charts/postgres-operator/values.yaml
@@ -362,11 +362,6 @@ rbac:
   # Specifies whether RBAC resources should be created
   create: true
 
-crd:
-  # Specifies whether custom resource definitions should be created
-  # When using helm3, this is ignored; instead use "--skip-crds" to skip.
-  create: true
-
 serviceAccount:
   # Specifies whether a ServiceAccount should be created
   create: true

From a6d4f23f90fe12822a19e758ff400233c9212e00 Mon Sep 17 00:00:00 2001
From: Philip Sieder <philip.sieder@gmx.de>
Date: Fri, 19 Nov 2021 11:28:17 +0100
Subject: [PATCH 05/58] Add user facing ClusterRoles to Helm chart (#1675)

To allow the usage of the [aggregate ClusterRoles]
(https://github.com/zalando/postgres-operator/blob/master/manifests/user-facing-clusterroles.yaml),
an option in the Helm chart is added.

Fixes https://github.com/zalando/postgres-operator/issues/1668

Co-authored-by: Philip Sieder <philip.sieder@tallence.com>
---
 .../templates/user-facing-clusterroles.yaml   | 71 +++++++++++++++++++
 charts/postgres-operator/values.yaml          |  2 +
 docs/administrator.md                         |  4 +-
 3 files changed, 76 insertions(+), 1 deletion(-)
 create mode 100644 charts/postgres-operator/templates/user-facing-clusterroles.yaml

diff --git a/charts/postgres-operator/templates/user-facing-clusterroles.yaml b/charts/postgres-operator/templates/user-facing-clusterroles.yaml
new file mode 100644
index 000000000..d7db347b2
--- /dev/null
+++ b/charts/postgres-operator/templates/user-facing-clusterroles.yaml
@@ -0,0 +1,71 @@
+{{ if .Values.rbac.createAggregateClusterRoles }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    rbac.authorization.k8s.io/aggregate-to-admin: "true"
+    app.kubernetes.io/name: {{ template "postgres-operator.name" . }}
+    helm.sh/chart: {{ template "postgres-operator.chart" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+  name: {{ template "postgres-operator.fullname" . }}:users:admin
+rules:
+- apiGroups:
+  - acid.zalan.do
+  resources:
+  - postgresqls
+  - postgresqls/status
+  verbs:
+  - create
+  - delete
+  - deletecollection
+  - get
+  - list
+  - patch
+  - update
+  - watch
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    rbac.authorization.k8s.io/aggregate-to-edit: "true"
+    app.kubernetes.io/name: {{ template "postgres-operator.name" . }}
+    helm.sh/chart: {{ template "postgres-operator.chart" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+  name: {{ template "postgres-operator.fullname" . }}:users:edit
+rules:
+- apiGroups:
+  - acid.zalan.do
+  resources:
+  - postgresqls
+  verbs:
+  - create
+  - update
+  - patch
+  - delete
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    rbac.authorization.k8s.io/aggregate-to-view: "true"
+    app.kubernetes.io/name: {{ template "postgres-operator.name" . }}
+    helm.sh/chart: {{ template "postgres-operator.chart" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+  name: {{ template "postgres-operator.fullname" . }}:users:view
+rules:
+- apiGroups:
+  - acid.zalan.do
+  resources:
+  - postgresqls
+  - postgresqls/status
+  verbs:
+  - get
+  - list
+  - watch
+{{ end }}
diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml
index 9e2303f3b..d660de0ab 100644
--- a/charts/postgres-operator/values.yaml
+++ b/charts/postgres-operator/values.yaml
@@ -361,6 +361,8 @@ configConnectionPooler:
 rbac:
   # Specifies whether RBAC resources should be created
   create: true
+  # Specifies whether ClusterRoles that are aggregated into the K8s default roles should be created. (https://kubernetes.io/docs/reference/access-authn-authz/rbac/#default-roles-and-role-bindings)
+  createAggregateClusterRoles: false
 
 serviceAccount:
   # Specifies whether a ServiceAccount should be created
diff --git a/docs/administrator.md b/docs/administrator.md
index e0c03e875..551ee5523 100644
--- a/docs/administrator.md
+++ b/docs/administrator.md
@@ -291,6 +291,8 @@ kubectl create -f manifests/user-facing-clusterroles.yaml
 It creates zalando-postgres-operator:user:view, :edit and :admin clusterroles
 that are aggregated into the K8s [default roles](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#default-roles-and-role-bindings).
 
+For Helm deployments setting `rbac.createAggregateClusterRoles: true` adds these clusterroles to the deployment.
+
 ## Use taints and tolerations for dedicated PostgreSQL nodes
 
 To ensure Postgres pods are running on nodes without any other application pods,
@@ -762,7 +764,7 @@ WALE_S3_PREFIX=$WAL_S3_BUCKET/spilo/{WAL_BUCKET_SCOPE_PREFIX}{SCOPE}{WAL_BUCKET_
 ```
 
 The operator sets the prefix to an empty string so that spilo will generate it
-from the configured `WAL_S3_BUCKET`. 
+from the configured `WAL_S3_BUCKET`.
 
 :warning: When you overwrite the configuration by defining `WAL_S3_BUCKET` in
 the [pod_environment_configmap](#custom-pod-environment-variables) you have

From fbd980af78ee811986f27f0273a620ddb6a8f39d Mon Sep 17 00:00:00 2001
From: Dmitry Volodin <dmvolod@gmail.com>
Date: Wed, 24 Nov 2021 14:19:30 +0300
Subject: [PATCH 06/58] Operator fails with nil pointer dereference when CR
 validation is not passed (#1697)

---
 pkg/controller/controller.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go
index f992ff782..5d9e00bc8 100644
--- a/pkg/controller/controller.go
+++ b/pkg/controller/controller.go
@@ -95,7 +95,9 @@ func NewController(controllerConfig *spec.ControllerConfig, controllerId string)
 	// disabling the sending of events also to the logoutput
 	// the operator currently duplicates a lot of log entries with this setup
 	// eventBroadcaster.StartLogging(logger.Infof)
-	recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: myComponentName})
+	scheme := scheme.Scheme
+	acidv1.AddToScheme(scheme)
+	recorder := eventBroadcaster.NewRecorder(scheme, v1.EventSource{Component: myComponentName})
 
 	c := &Controller{
 		config:           *controllerConfig,

From 3e275d122a54fbf17c40db9564f762ea0754b61c Mon Sep 17 00:00:00 2001
From: Jan Mussler <janm81@gmail.com>
Date: Mon, 29 Nov 2021 12:47:18 +0100
Subject: [PATCH 07/58] Allow individual teams to do auto upgrade via operator.
 (#1699)

* Allow whitelisting of teams to do auto upgrade upgrade via operator.

Co-authored-by: Felix Kunde <felix-kunde@gmx.de>
---
 .../crds/operatorconfigurations.yaml          |  4 ++++
 charts/postgres-operator/values.yaml          |  4 ++++
 docs/reference/operator_parameters.md         |  4 ++++
 docs/user.md                                  |  7 +++----
 manifests/configmap.yaml                      |  1 +
 manifests/operatorconfiguration.crd.yaml      |  4 ++++
 ...gresql-operator-default-configuration.yaml |  2 ++
 pkg/apis/acid.zalan.do/v1/crds.go             |  8 ++++++++
 .../v1/operator_configuration_type.go         |  7 ++++---
 .../acid.zalan.do/v1/zz_generated.deepcopy.go |  7 ++++++-
 pkg/cluster/majorversionupgrade.go            | 19 ++++++++++++++++++-
 pkg/controller/operator_config.go             |  1 +
 pkg/util/config/config.go                     |  1 +
 13 files changed, 60 insertions(+), 9 deletions(-)

diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml
index b74ebbb0a..043129516 100644
--- a/charts/postgres-operator/crds/operatorconfigurations.yaml
+++ b/charts/postgres-operator/crds/operatorconfigurations.yaml
@@ -131,6 +131,10 @@ spec:
                   major_version_upgrade_mode:
                     type: string
                     default: "off"
+                  major_version_upgrade_team_allow_list:
+                    type: array
+                    items:
+                      type: string
                   minimal_major_version:
                     type: string
                     default: "9.6"
diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml
index d660de0ab..65619845a 100644
--- a/charts/postgres-operator/values.yaml
+++ b/charts/postgres-operator/values.yaml
@@ -64,6 +64,10 @@ configUsers:
 configMajorVersionUpgrade:
   # "off": no upgrade, "manual": manifest triggers action, "full": minimal version violation triggers too
   major_version_upgrade_mode: "off"
+  # upgrades will only be carried out for clusters of listed teams when mode is "off"
+  # major_version_upgrade_team_allow_list:
+  # - acid
+
   # minimal Postgres major version that will not automatically be upgraded
   minimal_major_version: "9.6"
   # target Postgres major version when upgrading clusters automatically
diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md
index bba917d88..00febcf89 100644
--- a/docs/reference/operator_parameters.md
+++ b/docs/reference/operator_parameters.md
@@ -184,6 +184,10 @@ CRD-configuration, they are grouped under the `major_version_upgrade` key.
   Note, that with all three modes increasing the version in the manifest will
   trigger a rolling update of the pods. The default is `"off"`.
 
+* **major_version_upgrade_team_allow_list**
+  Upgrades will only be carried out for clusters of listed teams when mode is
+  set to "off". The default is empty.
+
 * **minimal_major_version**
   The minimal Postgres major version that will not automatically be upgraded
   when `major_version_upgrade_mode` is set to `"full"`. The default is `"9.6"`.
diff --git a/docs/user.md b/docs/user.md
index e0873f274..a2a65e63f 100644
--- a/docs/user.md
+++ b/docs/user.md
@@ -603,10 +603,9 @@ spec:
 ```
 
 Some extensions require SUPERUSER rights on creation unless they are not
-whitelisted by the [pgextwlist](https://github.com/dimitri/pgextwlist)
-extension, that is shipped with the Spilo image. To see which extensions are
-on the list check the `extwlist.extension` parameter in the postgresql.conf
-file.
+allowed by the [pgextwlist](https://github.com/dimitri/pgextwlist) extension,
+that is shipped with the Spilo image. To see which extensions are on the list
+check the `extwlist.extension` parameter in the postgresql.conf file.
 
 ```bash
 SHOW extwlist.extensions;
diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml
index e91e2f19e..7d3e14ce3 100644
--- a/manifests/configmap.yaml
+++ b/manifests/configmap.yaml
@@ -77,6 +77,7 @@ data:
   logical_backup_s3_sse: "AES256"
   logical_backup_schedule: "30 00 * * *"
   major_version_upgrade_mode: "manual"
+  # major_version_upgrade_team_allow_list: ""
   master_dns_name_format: "{cluster}.{team}.{hostedzone}"
   # master_pod_move_timeout: 20m
   # max_instances: "-1"
diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml
index d202125f3..bb64995ab 100644
--- a/manifests/operatorconfiguration.crd.yaml
+++ b/manifests/operatorconfiguration.crd.yaml
@@ -129,6 +129,10 @@ spec:
                   major_version_upgrade_mode:
                     type: string
                     default: "off"
+                  major_version_upgrade_team_allow_list:
+                    type: array
+                    items:
+                      type: string
                   minimal_major_version:
                     type: string
                     default: "9.6"
diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml
index 24d496b36..02d558543 100644
--- a/manifests/postgresql-operator-default-configuration.yaml
+++ b/manifests/postgresql-operator-default-configuration.yaml
@@ -28,6 +28,8 @@ configuration:
     super_username: postgres
   major_version_upgrade:
     major_version_upgrade_mode: "off"
+    # major_version_upgrade_team_allow_list:
+    # - acid
     minimal_major_version: "9.6"
     target_major_version: "14"
   kubernetes:
diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go
index 582b1379e..76fbbfa48 100644
--- a/pkg/apis/acid.zalan.do/v1/crds.go
+++ b/pkg/apis/acid.zalan.do/v1/crds.go
@@ -1019,6 +1019,14 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{
 							"major_version_upgrade_mode": {
 								Type: "string",
 							},
+							"major_version_upgrade_team_allow_list": {
+								Type: "array",
+								Items: &apiextv1.JSONSchemaPropsOrArray{
+									Schema: &apiextv1.JSONSchemaProps{
+										Type: "string",
+									},
+								},
+							},
 							"minimal_major_version": {
 								Type: "string",
 							},
diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
index 6d0dd136a..f8eb5b5d1 100644
--- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
+++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
@@ -43,9 +43,10 @@ type PostgresUsersConfiguration struct {
 
 // MajorVersionUpgradeConfiguration defines how to execute major version upgrades of Postgres.
 type MajorVersionUpgradeConfiguration struct {
-	MajorVersionUpgradeMode string `json:"major_version_upgrade_mode" default:"off"` // off - no actions, manual - manifest triggers action, full - manifest and minimal version violation trigger upgrade
-	MinimalMajorVersion     string `json:"minimal_major_version" default:"9.6"`
-	TargetMajorVersion      string `json:"target_major_version" default:"14"`
+	MajorVersionUpgradeMode          string   `json:"major_version_upgrade_mode" default:"off"` // off - no actions, manual - manifest triggers action, full - manifest and minimal version violation trigger upgrade
+	MajorVersionUpgradeTeamAllowList []string `json:"major_version_upgrade_team_allow_list,omitempty"`
+	MinimalMajorVersion              string   `json:"minimal_major_version" default:"9.6"`
+	TargetMajorVersion               string   `json:"target_major_version" default:"14"`
 }
 
 // KubernetesMetaConfiguration defines k8s conf required for all Postgres clusters and the operator itself
diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
index c0be8fdf9..7a8984ce4 100644
--- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
+++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
@@ -318,6 +318,11 @@ func (in *MaintenanceWindow) DeepCopy() *MaintenanceWindow {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *MajorVersionUpgradeConfiguration) DeepCopyInto(out *MajorVersionUpgradeConfiguration) {
 	*out = *in
+	if in.MajorVersionUpgradeTeamAllowList != nil {
+		in, out := &in.MajorVersionUpgradeTeamAllowList, &out.MajorVersionUpgradeTeamAllowList
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	return
 }
 
@@ -386,7 +391,7 @@ func (in *OperatorConfigurationData) DeepCopyInto(out *OperatorConfigurationData
 		}
 	}
 	out.PostgresUsersConfiguration = in.PostgresUsersConfiguration
-	out.MajorVersionUpgrade = in.MajorVersionUpgrade
+	in.MajorVersionUpgrade.DeepCopyInto(&out.MajorVersionUpgrade)
 	in.Kubernetes.DeepCopyInto(&out.Kubernetes)
 	out.PostgresPodResources = in.PostgresPodResources
 	out.Timeouts = in.Timeouts
diff --git a/pkg/cluster/majorversionupgrade.go b/pkg/cluster/majorversionupgrade.go
index edb55c882..60048e20d 100644
--- a/pkg/cluster/majorversionupgrade.go
+++ b/pkg/cluster/majorversionupgrade.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	"github.com/zalando/postgres-operator/pkg/spec"
+	"github.com/zalando/postgres-operator/pkg/util"
 	v1 "k8s.io/api/core/v1"
 )
 
@@ -44,9 +45,25 @@ func (c *Cluster) GetDesiredMajorVersion() string {
 	return c.Spec.PgVersion
 }
 
+func (c *Cluster) isUpgradeAllowedForTeam(owningTeam string) bool {
+	allowedTeams := c.OpConfig.MajorVersionUpgradeTeamAllowList
+
+	if len(allowedTeams) == 0 {
+		return false
+	}
+
+	return util.SliceContains(allowedTeams, owningTeam)
+}
+
+/*
+  Execute upgrade when mode is set to manual or full or when the owning team is allowed for upgrade (and mode is "off").
+
+  Manual upgrade means, it is triggered by the user via manifest version change
+  Full upgrade means, operator also determines the minimal version used accross all clusters and upgrades violators.
+*/
 func (c *Cluster) majorVersionUpgrade() error {
 
-	if c.OpConfig.MajorVersionUpgradeMode == "off" {
+	if c.OpConfig.MajorVersionUpgradeMode == "off" && !c.isUpgradeAllowedForTeam(c.Spec.TeamID) {
 		return nil
 	}
 
diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go
index fc56dbf96..275898d8e 100644
--- a/pkg/controller/operator_config.go
+++ b/pkg/controller/operator_config.go
@@ -56,6 +56,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 
 	// major version upgrade config
 	result.MajorVersionUpgradeMode = util.Coalesce(fromCRD.MajorVersionUpgrade.MajorVersionUpgradeMode, "off")
+	result.MajorVersionUpgradeTeamAllowList = fromCRD.MajorVersionUpgrade.MajorVersionUpgradeTeamAllowList
 	result.MinimalMajorVersion = util.Coalesce(fromCRD.MajorVersionUpgrade.MinimalMajorVersion, "9.6")
 	result.TargetMajorVersion = util.Coalesce(fromCRD.MajorVersionUpgrade.TargetMajorVersion, "14")
 
diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go
index 78e0a6c49..71bf406e4 100644
--- a/pkg/util/config/config.go
+++ b/pkg/util/config/config.go
@@ -212,6 +212,7 @@ type Config struct {
 	EnablePgVersionEnvVar                  bool              `name:"enable_pgversion_env_var" default:"true"`
 	EnableSpiloWalPathCompat               bool              `name:"enable_spilo_wal_path_compat" default:"false"`
 	MajorVersionUpgradeMode                string            `name:"major_version_upgrade_mode" default:"off"`
+	MajorVersionUpgradeTeamAllowList       []string          `name:"major_version_upgrade_team_allow_list" default:""`
 	MinimalMajorVersion                    string            `name:"minimal_major_version" default:"9.6"`
 	TargetMajorVersion                     string            `name:"target_major_version" default:"14"`
 }

From f7858ffb70f524ed37ff90b25d1868a4ec9717e9 Mon Sep 17 00:00:00 2001
From: Felix Kunde <felix-kunde@gmx.de>
Date: Mon, 29 Nov 2021 12:49:12 +0100
Subject: [PATCH 08/58] Initialize arrays of errors / error messages + minor
 refactoring (#1701)

* init error arrays correctly
* avoid nilPointer when syncing connectionPooler
* getInfrastructureRoles should return error
* fix unit tests and return type for getInfrastructureRoles
---
 pkg/cluster/connection_pooler.go |  3 +-
 pkg/cluster/resources.go         |  9 +++--
 pkg/cluster/sync_test.go         |  9 ++---
 pkg/cluster/volumes.go           |  2 +-
 pkg/controller/util.go           | 27 +++++++-------
 pkg/controller/util_test.go      | 60 +++++++-------------------------
 pkg/util/users/users.go          |  4 +--
 7 files changed, 38 insertions(+), 76 deletions(-)

diff --git a/pkg/cluster/connection_pooler.go b/pkg/cluster/connection_pooler.go
index 5bde71458..c5c55350f 100644
--- a/pkg/cluster/connection_pooler.go
+++ b/pkg/cluster/connection_pooler.go
@@ -712,7 +712,8 @@ func (c *Cluster) syncConnectionPooler(oldSpec, newSpec *acidv1.Postgresql, Look
 	if (!needSync && len(masterChanges) <= 0 && len(replicaChanges) <= 0) &&
 		((!needConnectionPooler(&newSpec.Spec) && (c.ConnectionPooler == nil || !needConnectionPooler(&oldSpec.Spec))) ||
 			(c.ConnectionPooler != nil && needConnectionPooler(&newSpec.Spec) &&
-				(c.ConnectionPooler[Master].LookupFunction || c.ConnectionPooler[Replica].LookupFunction))) {
+				((c.ConnectionPooler[Master] != nil && c.ConnectionPooler[Master].LookupFunction) ||
+					(c.ConnectionPooler[Replica] != nil && c.ConnectionPooler[Replica].LookupFunction)))) {
 		c.logger.Debugln("syncing pooler is not required")
 		return nil, nil
 	}
diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go
index f078c6434..48a82ee04 100644
--- a/pkg/cluster/resources.go
+++ b/pkg/cluster/resources.go
@@ -496,18 +496,17 @@ func (c *Cluster) deleteEndpoint(role PostgresRole) error {
 
 func (c *Cluster) deleteSecrets() error {
 	c.setProcessName("deleting secrets")
-	var errors []string
-	errorCount := 0
+	errors := make([]string, 0)
+
 	for uid, secret := range c.Secrets {
 		err := c.deleteSecret(uid, *secret)
 		if err != nil {
 			errors = append(errors, fmt.Sprintf("%v", err))
-			errorCount++
 		}
 	}
 
-	if errorCount > 0 {
-		return fmt.Errorf("could not delete all secrets: %v", errors)
+	if len(errors) > 0 {
+		return fmt.Errorf("could not delete all secrets: %v", strings.Join(errors, `', '`))
 	}
 
 	return nil
diff --git a/pkg/cluster/sync_test.go b/pkg/cluster/sync_test.go
index c6e2a8357..3e3fc9318 100644
--- a/pkg/cluster/sync_test.go
+++ b/pkg/cluster/sync_test.go
@@ -196,17 +196,15 @@ func TestCheckAndSetGlobalPostgreSQLConfiguration(t *testing.T) {
 	cluster.patroni = p
 	mockPod := newMockPod("192.168.100.1")
 
-	// simulate existing config that differs with cluster.Spec
+	// simulate existing config that differs from cluster.Spec
 	tests := []struct {
 		subtest       string
-		pod           *v1.Pod
 		patroni       acidv1.Patroni
 		pgParams      map[string]string
 		restartMaster bool
 	}{
 		{
 			subtest: "Patroni and Postgresql.Parameters differ - restart replica first",
-			pod:     mockPod,
 			patroni: acidv1.Patroni{
 				TTL: 30, // desired 20
 			},
@@ -218,7 +216,6 @@ func TestCheckAndSetGlobalPostgreSQLConfiguration(t *testing.T) {
 		},
 		{
 			subtest: "multiple Postgresql.Parameters differ - restart replica first",
-			pod:     mockPod,
 			patroni: acidv1.Patroni{
 				TTL: 20,
 			},
@@ -230,7 +227,6 @@ func TestCheckAndSetGlobalPostgreSQLConfiguration(t *testing.T) {
 		},
 		{
 			subtest: "desired max_connections bigger - restart replica first",
-			pod:     mockPod,
 			patroni: acidv1.Patroni{
 				TTL: 20,
 			},
@@ -242,7 +238,6 @@ func TestCheckAndSetGlobalPostgreSQLConfiguration(t *testing.T) {
 		},
 		{
 			subtest: "desired max_connections smaller - restart master first",
-			pod:     mockPod,
 			patroni: acidv1.Patroni{
 				TTL: 20,
 			},
@@ -255,7 +250,7 @@ func TestCheckAndSetGlobalPostgreSQLConfiguration(t *testing.T) {
 	}
 
 	for _, tt := range tests {
-		requireMasterRestart, err := cluster.checkAndSetGlobalPostgreSQLConfiguration(tt.pod, tt.patroni, tt.pgParams)
+		requireMasterRestart, err := cluster.checkAndSetGlobalPostgreSQLConfiguration(mockPod, tt.patroni, tt.pgParams)
 		assert.NoError(t, err)
 		if requireMasterRestart != tt.restartMaster {
 			t.Errorf("%s - %s: unexpect master restart strategy, got %v, expected %v", testName, tt.subtest, requireMasterRestart, tt.restartMaster)
diff --git a/pkg/cluster/volumes.go b/pkg/cluster/volumes.go
index 4962319ed..78533ef54 100644
--- a/pkg/cluster/volumes.go
+++ b/pkg/cluster/volumes.go
@@ -88,7 +88,7 @@ func (c *Cluster) syncUnderlyingEBSVolume() error {
 	awsGp3 := aws.String("gp3")
 	awsIo2 := aws.String("io2")
 
-	errors := []string{}
+	errors := make([]string, 0)
 
 	for _, volume := range c.EBSVolumes {
 		var modifyIops *int64
diff --git a/pkg/controller/util.go b/pkg/controller/util.go
index 8aa891c09..563734ac9 100644
--- a/pkg/controller/util.go
+++ b/pkg/controller/util.go
@@ -195,13 +195,12 @@ func (c *Controller) getInfrastructureRoleDefinitions() []*config.Infrastructure
 
 func (c *Controller) getInfrastructureRoles(
 	rolesSecrets []*config.InfrastructureRole) (
-	map[string]spec.PgUser, []error) {
-
-	var errors []error
-	var noRolesProvided = true
+	map[string]spec.PgUser, error) {
 
+	errors := make([]string, 0)
+	noRolesProvided := true
 	roles := []spec.PgUser{}
-	uniqRoles := map[string]spec.PgUser{}
+	uniqRoles := make(map[string]spec.PgUser)
 
 	// To be compatible with the legacy implementation we need to return nil if
 	// the provided secret name is empty. The equivalent situation in the
@@ -214,37 +213,39 @@ func (c *Controller) getInfrastructureRoles(
 	}
 
 	if noRolesProvided {
-		return nil, nil
+		return uniqRoles, nil
 	}
 
 	for _, secret := range rolesSecrets {
 		infraRoles, err := c.getInfrastructureRole(secret)
 
 		if err != nil || infraRoles == nil {
-			c.logger.Debugf("Cannot get infrastructure role: %+v", *secret)
+			c.logger.Debugf("cannot get infrastructure role: %+v", *secret)
 
 			if err != nil {
-				errors = append(errors, err)
+				errors = append(errors, fmt.Sprintf("%v", err))
 			}
 
 			continue
 		}
 
-		for _, r := range infraRoles {
-			roles = append(roles, r)
-		}
+		roles = append(roles, infraRoles...)
 	}
 
 	for _, r := range roles {
 		if _, exists := uniqRoles[r.Name]; exists {
-			msg := "Conflicting infrastructure roles: roles[%s] = (%q, %q)"
+			msg := "conflicting infrastructure roles: roles[%s] = (%q, %q)"
 			c.logger.Debugf(msg, r.Name, uniqRoles[r.Name], r)
 		}
 
 		uniqRoles[r.Name] = r
 	}
 
-	return uniqRoles, errors
+	if len(errors) > 0 {
+		return uniqRoles, fmt.Errorf(strings.Join(errors, `', '`))
+	}
+
+	return uniqRoles, nil
 }
 
 // Generate list of users representing one infrastructure role based on its
diff --git a/pkg/controller/util_test.go b/pkg/controller/util_test.go
index d8e4c3782..a4ca17728 100644
--- a/pkg/controller/util_test.go
+++ b/pkg/controller/util_test.go
@@ -7,6 +7,7 @@ import (
 
 	b64 "encoding/base64"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/zalando/postgres-operator/pkg/spec"
 	"github.com/zalando/postgres-operator/pkg/util/config"
 	"github.com/zalando/postgres-operator/pkg/util/k8sutil"
@@ -90,21 +91,21 @@ func TestClusterWorkerID(t *testing.T) {
 // not exist, or empty) and the old format.
 func TestOldInfrastructureRoleFormat(t *testing.T) {
 	var testTable = []struct {
-		secretName     spec.NamespacedName
-		expectedRoles  map[string]spec.PgUser
-		expectedErrors []error
+		secretName    spec.NamespacedName
+		expectedRoles map[string]spec.PgUser
+		expectedError error
 	}{
 		{
 			// empty secret name
 			spec.NamespacedName{},
-			nil,
+			map[string]spec.PgUser{},
 			nil,
 		},
 		{
 			// secret does not exist
 			spec.NamespacedName{Namespace: v1.NamespaceDefault, Name: "null"},
 			map[string]spec.PgUser{},
-			[]error{fmt.Errorf(`could not get infrastructure roles secret default/null: NotFound`)},
+			fmt.Errorf(`could not get infrastructure roles secret default/null: NotFound`),
 		},
 		{
 			spec.NamespacedName{
@@ -129,7 +130,7 @@ func TestOldInfrastructureRoleFormat(t *testing.T) {
 		},
 	}
 	for _, test := range testTable {
-		roles, errors := utilTestController.getInfrastructureRoles(
+		roles, err := utilTestController.getInfrastructureRoles(
 			[]*config.InfrastructureRole{
 				&config.InfrastructureRole{
 					SecretName:  test.secretName,
@@ -140,22 +141,9 @@ func TestOldInfrastructureRoleFormat(t *testing.T) {
 				},
 			})
 
-		if len(errors) != len(test.expectedErrors) {
+		if err != nil && err.Error() != test.expectedError.Error() {
 			t.Errorf("expected error '%v' does not match the actual error '%v'",
-				test.expectedErrors, errors)
-		}
-
-		for idx := range errors {
-			err := errors[idx]
-			expectedErr := test.expectedErrors[idx]
-
-			if err != expectedErr {
-				if err != nil && expectedErr != nil && err.Error() == expectedErr.Error() {
-					continue
-				}
-				t.Errorf("expected error '%v' does not match the actual error '%v'",
-					expectedErr, err)
-			}
+				test.expectedError, err)
 		}
 
 		if !reflect.DeepEqual(roles, test.expectedRoles) {
@@ -169,9 +157,8 @@ func TestOldInfrastructureRoleFormat(t *testing.T) {
 // corresponding secrets. Here we test the new format.
 func TestNewInfrastructureRoleFormat(t *testing.T) {
 	var testTable = []struct {
-		secrets        []spec.NamespacedName
-		expectedRoles  map[string]spec.PgUser
-		expectedErrors []error
+		secrets       []spec.NamespacedName
+		expectedRoles map[string]spec.PgUser
 	}{
 		// one secret with one configmap
 		{
@@ -196,7 +183,6 @@ func TestNewInfrastructureRoleFormat(t *testing.T) {
 					Flags:    []string{"createdb"},
 				},
 			},
-			nil,
 		},
 		// multiple standalone secrets
 		{
@@ -224,7 +210,6 @@ func TestNewInfrastructureRoleFormat(t *testing.T) {
 					MemberOf: []string{"new-test-inrole2"},
 				},
 			},
-			nil,
 		},
 	}
 	for _, test := range testTable {
@@ -239,27 +224,8 @@ func TestNewInfrastructureRoleFormat(t *testing.T) {
 			})
 		}
 
-		roles, errors := utilTestController.getInfrastructureRoles(definitions)
-		if len(errors) != len(test.expectedErrors) {
-			t.Errorf("expected error does not match the actual error:\n%+v\n%+v",
-				test.expectedErrors, errors)
-
-			// Stop and do not do any further checks
-			return
-		}
-
-		for idx := range errors {
-			err := errors[idx]
-			expectedErr := test.expectedErrors[idx]
-
-			if err != expectedErr {
-				if err != nil && expectedErr != nil && err.Error() == expectedErr.Error() {
-					continue
-				}
-				t.Errorf("expected error '%v' does not match the actual error '%v'",
-					expectedErr, err)
-			}
-		}
+		roles, err := utilTestController.getInfrastructureRoles(definitions)
+		assert.NoError(t, err)
 
 		if !reflect.DeepEqual(roles, test.expectedRoles) {
 			t.Errorf("expected roles output/the actual:\n%#v\n%#v",
diff --git a/pkg/util/users/users.go b/pkg/util/users/users.go
index 3da933644..821350bb7 100644
--- a/pkg/util/users/users.go
+++ b/pkg/util/users/users.go
@@ -101,7 +101,7 @@ func (strategy DefaultUserSyncStrategy) ProduceSyncRequests(dbUsers spec.PgUserM
 // ExecuteSyncRequests makes actual database changes from the requests passed in its arguments.
 func (strategy DefaultUserSyncStrategy) ExecuteSyncRequests(requests []spec.PgSyncUserRequest, db *sql.DB) error {
 	var reqretries []spec.PgSyncUserRequest
-	var errors []string
+	errors := make([]string, 0)
 	for _, request := range requests {
 		switch request.Kind {
 		case spec.PGSyncUserAdd:
@@ -138,7 +138,7 @@ func (strategy DefaultUserSyncStrategy) ExecuteSyncRequests(requests []spec.PgSy
 				return err
 			}
 		} else {
-			return fmt.Errorf("could not execute sync requests for users: %v", errors)
+			return fmt.Errorf("could not execute sync requests for users: %v", strings.Join(errors, `', '`))
 		}
 	}
 

From 1ed16fadca3bd40c314223720ad735fafffd5b24 Mon Sep 17 00:00:00 2001
From: Felix Kunde <felix-kunde@gmx.de>
Date: Thu, 2 Dec 2021 14:10:58 +0100
Subject: [PATCH 09/58] make sure upgrade script runs on the master (#1715)

* make sure upgrade script runs on the master
* show a bit more logs from upgrade script
---
 pkg/cluster/majorversionupgrade.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pkg/cluster/majorversionupgrade.go b/pkg/cluster/majorversionupgrade.go
index 60048e20d..36def164c 100644
--- a/pkg/cluster/majorversionupgrade.go
+++ b/pkg/cluster/majorversionupgrade.go
@@ -83,7 +83,7 @@ func (c *Cluster) majorVersionUpgrade() error {
 
 	var masterPod *v1.Pod
 
-	for _, pod := range pods {
+	for i, pod := range pods {
 		ps, _ := c.patroni.GetMemberData(&pod)
 
 		if ps.State != "running" {
@@ -92,7 +92,7 @@ func (c *Cluster) majorVersionUpgrade() error {
 		}
 
 		if ps.Role == "master" {
-			masterPod = &pod
+			masterPod = &pods[i]
 			c.currentMajorVersion = ps.ServerVersion
 		}
 	}
@@ -112,7 +112,7 @@ func (c *Cluster) majorVersionUpgrade() error {
 				return err
 			}
 
-			c.logger.Infof("upgrade action triggered and command completed: %s", result[:50])
+			c.logger.Infof("upgrade action triggered and command completed: %s", result[:100])
 			c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Major Version Upgrade", "Upgrade from %d to %d finished", c.currentMajorVersion, desiredVersion)
 		}
 	}

From def9e1d6884ba6d60d511c1b913a5bed7a0ef1d2 Mon Sep 17 00:00:00 2001
From: James McDonald <james@jamesmcdonald.com>
Date: Fri, 3 Dec 2021 11:24:29 +0100
Subject: [PATCH 10/58] Support standby replication from GS (GCS) (#1446)

* Add support for manual gs_wal_path in standby
* Remove separate standby version configuration
* Remove setting standby path via cluster/uid/version
Picking up the version doesn't work reliably without making changes to
Spilo. It's clearer to just specify the full S3/GS bucket path.

Co-authored-by: Felix Kunde <felix-kunde@gmx.de>
---
 .../postgres-operator/crds/postgresqls.yaml   |  4 +-
 docs/reference/cluster_manifest.md            | 10 +++--
 docs/user.md                                  | 10 ++++-
 pkg/apis/acid.zalan.do/v1/crds.go             |  6 ++-
 pkg/apis/acid.zalan.do/v1/postgresql_type.go  |  1 +
 pkg/cluster/k8sres.go                         | 40 ++++++++++++++-----
 6 files changed, 52 insertions(+), 19 deletions(-)

diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml
index 1b5f36db8..d6e1dd94f 100644
--- a/charts/postgres-operator/crds/postgresqls.yaml
+++ b/charts/postgres-operator/crds/postgresqls.yaml
@@ -464,11 +464,11 @@ spec:
                 type: integer
               standby:
                 type: object
-                required:
-                  - s3_wal_path
                 properties:
                   s3_wal_path:
                     type: string
+                  gs_wal_path:
+                    type: string
               teamId:
                 type: string
               tls:
diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md
index bf01b2a5c..3b6393550 100644
--- a/docs/reference/cluster_manifest.md
+++ b/docs/reference/cluster_manifest.md
@@ -366,12 +366,16 @@ under the `clone` top-level key and do not affect the already running cluster.
 ## Standby cluster
 
 On startup, an existing `standby` top-level key creates a standby Postgres
-cluster streaming from a remote location. So far only streaming from a S3 WAL
-archive is supported.
+cluster streaming from a remote location. So far streaming from S3 and GCS WAL
+archives is supported.
 
 * **s3_wal_path**
   the url to S3 bucket containing the WAL archive of the remote primary.
-  Required when the `standby` section is present.
+  Optional, but `s3_wal_path`  or `gs_wal_path` is required.
+
+* **gs_wal_path**
+  the url to GS bucket containing the WAL archive of the remote primary.
+  Optional, but `s3_wal_path`  or `gs_wal_path` is required.
 
 ## Volume properties
 
diff --git a/docs/user.md b/docs/user.md
index a2a65e63f..572d832ab 100644
--- a/docs/user.md
+++ b/docs/user.md
@@ -798,8 +798,8 @@ different location than its source database. Unlike cloning, the PostgreSQL
 version between source and target cluster has to be the same.
 
 To start a cluster as standby, add the following `standby` section in the YAML
-file and specify the S3 bucket path. An empty path will result in an error and
-no statefulset will be created.
+file. Specify the S3/GS bucket path. Omitting both settings will result in an error
+and no statefulset will be created.
 
 ```yaml
 spec:
@@ -807,6 +807,12 @@ spec:
     s3_wal_path: "s3://<bucketname>/spilo/<source_db_cluster>/<UID>/wal/<PGVERSION>"
 ```
 
+```yaml
+spec:
+  standby:
+    gs_wal_path: "gs://<bucketname>/spilo/<source_db_cluster>/<UID>/wal/<PGVERSION>"
+```
+
 At the moment, the operator only allows to stream from the WAL archive of the
 master. Thus, it is recommended to deploy standby clusters with only [one pod](https://github.com/zalando/postgres-operator/blob/master/manifests/standby-manifest.yaml#L10).
 You can raise the instance count when detaching. Note, that the same pod role
diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go
index 76fbbfa48..fae5a09f2 100644
--- a/pkg/apis/acid.zalan.do/v1/crds.go
+++ b/pkg/apis/acid.zalan.do/v1/crds.go
@@ -649,12 +649,14 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 						Type: "integer",
 					},
 					"standby": {
-						Type:     "object",
-						Required: []string{"s3_wal_path"},
+						Type: "object",
 						Properties: map[string]apiextv1.JSONSchemaProps{
 							"s3_wal_path": {
 								Type: "string",
 							},
+							"gs_wal_path": {
+								Type: "string",
+							},
 						},
 					},
 					"teamId": {
diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go
index 079cb8b98..57f9ae04a 100644
--- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go
+++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go
@@ -166,6 +166,7 @@ type Patroni struct {
 // StandbyDescription contains s3 wal path
 type StandbyDescription struct {
 	S3WalPath string `json:"s3_wal_path,omitempty"`
+	GSWalPath string `json:"gs_wal_path,omitempty"`
 }
 
 // TLSDescription specs TLS properties
diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go
index 366570c3c..e7d8ea376 100644
--- a/pkg/cluster/k8sres.go
+++ b/pkg/cluster/k8sres.go
@@ -1058,8 +1058,9 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
 	sort.Slice(customPodEnvVarsList,
 		func(i, j int) bool { return customPodEnvVarsList[i].Name < customPodEnvVarsList[j].Name })
 
-	if spec.StandbyCluster != nil && spec.StandbyCluster.S3WalPath == "" {
-		return nil, fmt.Errorf("s3_wal_path is empty for standby cluster")
+	if spec.StandbyCluster != nil && spec.StandbyCluster.S3WalPath == "" &&
+		spec.StandbyCluster.GSWalPath == "" {
+		return nil, fmt.Errorf("one of s3_wal_path or gs_wal_path must be set for standby cluster")
 	}
 
 	// backward compatible check for InitContainers
@@ -1874,17 +1875,36 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription)
 func (c *Cluster) generateStandbyEnvironment(description *acidv1.StandbyDescription) []v1.EnvVar {
 	result := make([]v1.EnvVar, 0)
 
-	if description.S3WalPath == "" {
+	if description.S3WalPath == "" && description.GSWalPath == "" {
 		return nil
 	}
-	// standby with S3, find out the bucket to setup standby
-	msg := "Standby from S3 bucket using custom parsed S3WalPath from the manifest %s "
-	c.logger.Infof(msg, description.S3WalPath)
 
-	result = append(result, v1.EnvVar{
-		Name:  "STANDBY_WALE_S3_PREFIX",
-		Value: description.S3WalPath,
-	})
+	if description.S3WalPath != "" {
+		// standby with S3, find out the bucket to setup standby
+		msg := "Standby from S3 bucket using custom parsed S3WalPath from the manifest %s "
+		c.logger.Infof(msg, description.S3WalPath)
+
+		result = append(result, v1.EnvVar{
+			Name:  "STANDBY_WALE_S3_PREFIX",
+			Value: description.S3WalPath,
+		})
+	} else if description.GSWalPath != "" {
+		msg := "Standby from GS bucket using custom parsed GSWalPath from the manifest %s "
+		c.logger.Infof(msg, description.GSWalPath)
+
+		envs := []v1.EnvVar{
+			{
+				Name:  "STANDBY_WALE_GS_PREFIX",
+				Value: description.GSWalPath,
+			},
+			{
+				Name:  "STANDBY_GOOGLE_APPLICATION_CREDENTIALS",
+				Value: c.OpConfig.GCPCredentials,
+			},
+		}
+		result = append(result, envs...)
+
+	}
 
 	result = append(result, v1.EnvVar{Name: "STANDBY_METHOD", Value: "STANDBY_WITH_WALE"})
 	result = append(result, v1.EnvVar{Name: "STANDBY_WAL_BUCKET_SCOPE_PREFIX", Value: ""})

From 895961847159357045a9cff720ba1de75911c39a Mon Sep 17 00:00:00 2001
From: Dmitry Volodin <dmvolod@gmail.com>
Date: Mon, 6 Dec 2021 13:21:07 +0300
Subject: [PATCH 11/58] Make test fails due to keeping github.com autogenerated
 folder in root (#1713)

---
 .github/workflows/run_e2e.yaml                   |  2 +-
 Makefile                                         |  3 +++
 hack/update-codegen.sh                           | 16 ++++++++++++++--
 hack/verify-codegen.sh                           |  5 ++---
 .../acid.zalan.do/v1/zz_generated.deepcopy.go    |  1 +
 5 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/run_e2e.yaml b/.github/workflows/run_e2e.yaml
index a8e76f0ca..f7d984519 100644
--- a/.github/workflows/run_e2e.yaml
+++ b/.github/workflows/run_e2e.yaml
@@ -20,6 +20,6 @@ jobs:
     - name: Compile
       run: make linux
     - name: Run unit tests
-      run: go test ./...
+      run: make test
     - name: Run end-2-end tests
       run: make e2e
diff --git a/Makefile b/Makefile
index b83f7156b..07605c0a3 100644
--- a/Makefile
+++ b/Makefile
@@ -103,5 +103,8 @@ test:
 	hack/verify-codegen.sh
 	GO111MODULE=on go test ./...
 
+codegen:
+	hack/update-codegen.sh
+
 e2e: docker # build operator image to be tested
 	cd e2e; make e2etest
diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh
index 280da9385..d304004dc 100755
--- a/hack/update-codegen.sh
+++ b/hack/update-codegen.sh
@@ -4,10 +4,22 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+GENERATED_PACKAGE_ROOT="github.com"
+OPERATOR_PACKAGE_ROOT="${GENERATED_PACKAGE_ROOT}/zalando/postgres-operator"
 SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/..
-CODEGEN_PKG=${CODEGEN_PKG:-$(cd ${SCRIPT_ROOT}; ls -d -1 ./vendor/k8s.io/code-generator 2>/dev/null || echo ${GOPATH}/src/k8s.io/code-generator)}
+TARGET_CODE_DIR=${1-${SCRIPT_ROOT}/pkg}
+CODEGEN_PKG=${CODEGEN_PKG:-$(cd "${SCRIPT_ROOT}"; ls -d -1 ./vendor/k8s.io/code-generator 2>/dev/null || echo "${GOPATH}"/src/k8s.io/code-generator)}
+
+cleanup() {
+    rm -rf "${GENERATED_PACKAGE_ROOT}"
+}
+trap "cleanup" EXIT SIGINT
 
 bash "${CODEGEN_PKG}/generate-groups.sh" all \
-  github.com/zalando/postgres-operator/pkg/generated github.com/zalando/postgres-operator/pkg/apis \
+  "${OPERATOR_PACKAGE_ROOT}/pkg/generated" "${OPERATOR_PACKAGE_ROOT}/pkg/apis" \
   "acid.zalan.do:v1" \
   --go-header-file "${SCRIPT_ROOT}"/hack/custom-boilerplate.go.txt
+
+cp -r "${OPERATOR_PACKAGE_ROOT}"/pkg/* "${TARGET_CODE_DIR}"
+
+cleanup
diff --git a/hack/verify-codegen.sh b/hack/verify-codegen.sh
index 68710015e..451ac76b7 100755
--- a/hack/verify-codegen.sh
+++ b/hack/verify-codegen.sh
@@ -19,15 +19,14 @@ cleanup
 mkdir -p "${TMP_DIFFROOT}"
 cp -a "${DIFFROOT}"/* "${TMP_DIFFROOT}"
 
-"${SCRIPT_ROOT}/hack/update-codegen.sh"
+"${SCRIPT_ROOT}/hack/update-codegen.sh" "${TMP_DIFFROOT}"
 echo "diffing ${DIFFROOT} against freshly generated codegen"
 ret=0
 diff -Naupr "${DIFFROOT}" "${TMP_DIFFROOT}" || ret=$?
-cp -a "${TMP_DIFFROOT}"/* "${DIFFROOT}"
 if [[ $ret -eq 0 ]]
 then
     echo "${DIFFROOT} up to date."
 else
-    echo "${DIFFROOT} is out of date. Please run hack/update-codegen.sh"
+    echo "${DIFFROOT} is out of date. Please run 'make codegen'"
     exit 1
 fi
diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
index 7a8984ce4..61bbfde9c 100644
--- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
+++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
@@ -1,3 +1,4 @@
+//go:build !ignore_autogenerated
 // +build !ignore_autogenerated
 
 /*

From 07fd4ec00bad302864330958a6fa691d9e12eed4 Mon Sep 17 00:00:00 2001
From: Felix Kunde <felix-kunde@gmx.de>
Date: Tue, 14 Dec 2021 10:35:21 +0100
Subject: [PATCH 12/58] choose switchover candidate based on lag and role
 (#1700)

* choose switchover candidate based on lowest lag in MB and role (in synchronous mode)
---
 pkg/cluster/pod.go               | 54 +++++++++++++++++++-
 pkg/cluster/pod_test.go          | 86 ++++++++++++++++++++++++++++++++
 pkg/cluster/types.go             | 11 ++--
 pkg/cluster/util.go              |  5 --
 pkg/util/patroni/patroni.go      | 37 ++++++++++++++
 pkg/util/patroni/patroni_test.go | 49 ++++++++++++++++++
 6 files changed, 231 insertions(+), 11 deletions(-)
 create mode 100644 pkg/cluster/pod_test.go

diff --git a/pkg/cluster/pod.go b/pkg/cluster/pod.go
index 229648dd1..b5415dc4e 100644
--- a/pkg/cluster/pod.go
+++ b/pkg/cluster/pod.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"math/rand"
+	"sort"
 	"strconv"
 	"time"
 
@@ -459,8 +460,13 @@ func (c *Cluster) recreatePods(pods []v1.Pod, switchoverCandidates []spec.Namesp
 		// 1. we have not observed a new master pod when re-creating former replicas
 		// 2. we know possible switchover targets even when no replicas were recreated
 		if newMasterPod == nil && len(replicas) > 0 {
-			if err := c.Switchover(masterPod, masterCandidate(replicas)); err != nil {
-				c.logger.Warningf("could not perform switch over: %v", err)
+			masterCandidate, err := c.getSwitchoverCandidate(masterPod)
+			if err != nil {
+				// do not recreate master now so it will keep the update flag and switchover will be retried on next sync
+				return fmt.Errorf("skipping switchover: %v", err)
+			}
+			if err := c.Switchover(masterPod, masterCandidate); err != nil {
+				return fmt.Errorf("could not perform switch over: %v", err)
 			}
 		} else if newMasterPod == nil && len(replicas) == 0 {
 			c.logger.Warningf("cannot perform switch over before re-creating the pod: no replicas")
@@ -475,6 +481,50 @@ func (c *Cluster) recreatePods(pods []v1.Pod, switchoverCandidates []spec.Namesp
 	return nil
 }
 
+func (c *Cluster) getSwitchoverCandidate(master *v1.Pod) (spec.NamespacedName, error) {
+
+	var members []patroni.ClusterMember
+	candidates := make([]patroni.ClusterMember, 0)
+	syncCandidates := make([]patroni.ClusterMember, 0)
+
+	err := retryutil.Retry(1*time.Second, 5*time.Second,
+		func() (bool, error) {
+			var err error
+			members, err = c.patroni.GetClusterMembers(master)
+
+			if err != nil {
+				return false, err
+			}
+			return true, nil
+		},
+	)
+	if err != nil {
+		return spec.NamespacedName{}, fmt.Errorf("failed to get Patroni cluster members: %s", err)
+	}
+
+	for _, member := range members {
+		if PostgresRole(member.Role) != Leader && PostgresRole(member.Role) != StandbyLeader && member.State == "running" {
+			candidates = append(candidates, member)
+			if PostgresRole(member.Role) == SyncStandby {
+				syncCandidates = append(syncCandidates, member)
+			}
+		}
+	}
+
+	// pick candidate with lowest lag
+	// if sync_standby replicas were found assume synchronous_mode is enabled and ignore other candidates list
+	if len(syncCandidates) > 0 {
+		sort.Slice(syncCandidates, func(i, j int) bool { return syncCandidates[i].LagInMb < syncCandidates[j].LagInMb })
+		return spec.NamespacedName{Namespace: master.Namespace, Name: syncCandidates[0].Name}, nil
+	}
+	if len(candidates) > 0 {
+		sort.Slice(candidates, func(i, j int) bool { return candidates[i].LagInMb < candidates[j].LagInMb })
+		return spec.NamespacedName{Namespace: master.Namespace, Name: candidates[0].Name}, nil
+	}
+
+	return spec.NamespacedName{}, fmt.Errorf("no switchover candidate found")
+}
+
 func (c *Cluster) podIsEndOfLife(pod *v1.Pod) (bool, error) {
 	node, err := c.KubeClient.Nodes().Get(context.TODO(), pod.Spec.NodeName, metav1.GetOptions{})
 	if err != nil {
diff --git a/pkg/cluster/pod_test.go b/pkg/cluster/pod_test.go
new file mode 100644
index 000000000..a533ebafd
--- /dev/null
+++ b/pkg/cluster/pod_test.go
@@ -0,0 +1,86 @@
+package cluster
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"testing"
+
+	"github.com/golang/mock/gomock"
+	"github.com/zalando/postgres-operator/mocks"
+	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
+	"github.com/zalando/postgres-operator/pkg/spec"
+	"github.com/zalando/postgres-operator/pkg/util/k8sutil"
+	"github.com/zalando/postgres-operator/pkg/util/patroni"
+)
+
+func TestGetSwitchoverCandidate(t *testing.T) {
+	testName := "test getting right switchover candidate"
+	namespace := "default"
+
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	var cluster = New(Config{}, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder)
+
+	// simulate different member scenarios
+	tests := []struct {
+		subtest           string
+		clusterJson       string
+		expectedCandidate spec.NamespacedName
+		expectedError     error
+	}{
+		{
+			subtest:           "choose sync_standby over replica",
+			clusterJson:       `{"members": [{"name": "acid-test-cluster-0", "role": "leader", "state": "running", "api_url": "http://192.168.100.1:8008/patroni", "host": "192.168.100.1", "port": 5432, "timeline": 1}, {"name": "acid-test-cluster-1", "role": "sync_standby", "state": "running", "api_url": "http://192.168.100.2:8008/patroni", "host": "192.168.100.2", "port": 5432, "timeline": 1, "lag": 0}, {"name": "acid-test-cluster-2", "role": "replica", "state": "running", "api_url": "http://192.168.100.3:8008/patroni", "host": "192.168.100.3", "port": 5432, "timeline": 1, "lag": 0}]}`,
+			expectedCandidate: spec.NamespacedName{Namespace: namespace, Name: "acid-test-cluster-1"},
+			expectedError:     nil,
+		},
+		{
+			subtest:           "choose replica with lowest lag",
+			clusterJson:       `{"members": [{"name": "acid-test-cluster-0", "role": "leader", "state": "running", "api_url": "http://192.168.100.1:8008/patroni", "host": "192.168.100.1", "port": 5432, "timeline": 1}, {"name": "acid-test-cluster-1", "role": "replica", "state": "running", "api_url": "http://192.168.100.2:8008/patroni", "host": "192.168.100.2", "port": 5432, "timeline": 1, "lag": 5}, {"name": "acid-test-cluster-2", "role": "replica", "state": "running", "api_url": "http://192.168.100.3:8008/patroni", "host": "192.168.100.3", "port": 5432, "timeline": 1, "lag": 2}]}`,
+			expectedCandidate: spec.NamespacedName{Namespace: namespace, Name: "acid-test-cluster-2"},
+			expectedError:     nil,
+		},
+		{
+			subtest:           "choose first replica when lag is equal evrywhere",
+			clusterJson:       `{"members": [{"name": "acid-test-cluster-0", "role": "leader", "state": "running", "api_url": "http://192.168.100.1:8008/patroni", "host": "192.168.100.1", "port": 5432, "timeline": 1}, {"name": "acid-test-cluster-1", "role": "replica", "state": "running", "api_url": "http://192.168.100.2:8008/patroni", "host": "192.168.100.2", "port": 5432, "timeline": 1, "lag": 5}, {"name": "acid-test-cluster-2", "role": "replica", "state": "running", "api_url": "http://192.168.100.3:8008/patroni", "host": "192.168.100.3", "port": 5432, "timeline": 1, "lag": 5}]}`,
+			expectedCandidate: spec.NamespacedName{Namespace: namespace, Name: "acid-test-cluster-1"},
+			expectedError:     nil,
+		},
+		{
+			subtest:           "no running replica available",
+			clusterJson:       `{"members": [{"name": "acid-test-cluster-0", "role": "leader", "state": "running", "api_url": "http://192.168.100.1:8008/patroni", "host": "192.168.100.1", "port": 5432, "timeline": 2}, {"name": "acid-test-cluster-1", "role": "replica", "state": "starting", "api_url": "http://192.168.100.2:8008/patroni", "host": "192.168.100.2", "port": 5432, "timeline": 2}]}`,
+			expectedCandidate: spec.NamespacedName{},
+			expectedError:     fmt.Errorf("no switchover candidate found"),
+		},
+	}
+
+	for _, tt := range tests {
+		// mocking cluster members
+		r := ioutil.NopCloser(bytes.NewReader([]byte(tt.clusterJson)))
+
+		response := http.Response{
+			StatusCode: 200,
+			Body:       r,
+		}
+
+		mockClient := mocks.NewMockHTTPClient(ctrl)
+		mockClient.EXPECT().Get(gomock.Any()).Return(&response, nil).AnyTimes()
+
+		p := patroni.New(patroniLogger, mockClient)
+		cluster.patroni = p
+		mockMasterPod := newMockPod("192.168.100.1")
+		mockMasterPod.Namespace = namespace
+
+		candidate, err := cluster.getSwitchoverCandidate(mockMasterPod)
+		if err != nil && err.Error() != tt.expectedError.Error() {
+			t.Errorf("%s - %s: unexpected error, %v", testName, tt.subtest, err)
+		}
+
+		if candidate != tt.expectedCandidate {
+			t.Errorf("%s - %s: unexpect switchover candidate, got %s, expected %s", testName, tt.subtest, candidate, tt.expectedCandidate)
+		}
+	}
+}
diff --git a/pkg/cluster/types.go b/pkg/cluster/types.go
index 199914ccc..67b4ee395 100644
--- a/pkg/cluster/types.go
+++ b/pkg/cluster/types.go
@@ -14,11 +14,14 @@ import (
 type PostgresRole string
 
 const (
-	// Master role
-	Master PostgresRole = "master"
-
-	// Replica role
+	// spilo roles
+	Master  PostgresRole = "master"
 	Replica PostgresRole = "replica"
+
+	// roles returned by Patroni cluster endpoint
+	Leader        PostgresRole = "leader"
+	StandbyLeader PostgresRole = "standby_leader"
+	SyncStandby   PostgresRole = "sync_standby"
 )
 
 // PodEventType represents the type of a pod-related event
diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go
index 4350f9d56..43c3282d4 100644
--- a/pkg/cluster/util.go
+++ b/pkg/cluster/util.go
@@ -6,7 +6,6 @@ import (
 	"encoding/gob"
 	"encoding/json"
 	"fmt"
-	"math/rand"
 	"reflect"
 	"sort"
 	"strings"
@@ -525,10 +524,6 @@ func (c *Cluster) credentialSecretNameForCluster(username string, clusterName st
 		"tprgroup", acidzalando.GroupName)
 }
 
-func masterCandidate(replicas []spec.NamespacedName) spec.NamespacedName {
-	return replicas[rand.Intn(len(replicas))]
-}
-
 func cloneSpec(from *acidv1.Postgresql) (*acidv1.Postgresql, error) {
 	var (
 		buf    bytes.Buffer
diff --git a/pkg/util/patroni/patroni.go b/pkg/util/patroni/patroni.go
index 4de3b8201..f030659ca 100644
--- a/pkg/util/patroni/patroni.go
+++ b/pkg/util/patroni/patroni.go
@@ -21,6 +21,7 @@ import (
 const (
 	failoverPath = "/failover"
 	configPath   = "/config"
+	clusterPath  = "/cluster"
 	statusPath   = "/patroni"
 	restartPath  = "/restart"
 	apiPort      = 8008
@@ -29,6 +30,7 @@ const (
 
 // Interface describe patroni methods
 type Interface interface {
+	GetClusterMembers(master *v1.Pod) ([]ClusterMember, error)
 	Switchover(master *v1.Pod, candidate string) error
 	SetPostgresParameters(server *v1.Pod, options map[string]string) error
 	GetMemberData(server *v1.Pod) (MemberData, error)
@@ -175,6 +177,20 @@ func (p *Patroni) SetConfig(server *v1.Pod, config map[string]interface{}) error
 	return p.httpPostOrPatch(http.MethodPatch, apiURLString+configPath, buf)
 }
 
+// ClusterMembers array of cluster members from Patroni API
+type ClusterMembers struct {
+	Members []ClusterMember `json:"members"`
+}
+
+// ClusterMember cluster member data from Patroni API
+type ClusterMember struct {
+	Name     string `json:"name"`
+	Role     string `json:"role"`
+	State    string `json:"state"`
+	Timeline int    `json:"timeline"`
+	LagInMb  int    `json:"lag"`
+}
+
 // MemberDataPatroni child element
 type MemberDataPatroni struct {
 	Version string `json:"version"`
@@ -246,6 +262,27 @@ func (p *Patroni) Restart(server *v1.Pod) error {
 	return nil
 }
 
+// GetClusterMembers read cluster data from patroni API
+func (p *Patroni) GetClusterMembers(server *v1.Pod) ([]ClusterMember, error) {
+
+	apiURLString, err := apiURL(server)
+	if err != nil {
+		return []ClusterMember{}, err
+	}
+	body, err := p.httpGet(apiURLString + clusterPath)
+	if err != nil {
+		return []ClusterMember{}, err
+	}
+
+	data := ClusterMembers{}
+	err = json.Unmarshal([]byte(body), &data)
+	if err != nil {
+		return []ClusterMember{}, err
+	}
+
+	return data.Members, nil
+}
+
 // GetMemberData read member data from patroni API
 func (p *Patroni) GetMemberData(server *v1.Pod) (MemberData, error) {
 
diff --git a/pkg/util/patroni/patroni_test.go b/pkg/util/patroni/patroni_test.go
index 5a6b2657c..48d27c8dc 100644
--- a/pkg/util/patroni/patroni_test.go
+++ b/pkg/util/patroni/patroni_test.go
@@ -85,6 +85,55 @@ func TestApiURL(t *testing.T) {
 	}
 }
 
+func TestGetClusterMembers(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	expectedClusterMemberData := []ClusterMember{
+		{
+			Name:     "acid-test-cluster-0",
+			Role:     "leader",
+			State:    "running",
+			Timeline: 1,
+			LagInMb:  0,
+		}, {
+			Name:     "acid-test-cluster-1",
+			Role:     "sync_standby",
+			State:    "running",
+			Timeline: 1,
+			LagInMb:  0,
+		}, {
+			Name:     "acid-test-cluster-2",
+			Role:     "replica",
+			State:    "running",
+			Timeline: 1,
+			LagInMb:  0,
+		}}
+
+	json := `{"members": [{"name": "acid-test-cluster-0", "role": "leader", "state": "running", "api_url": "http://192.168.100.1:8008/patroni", "host": "192.168.100.1", "port": 5432, "timeline": 1}, {"name": "acid-test-cluster-1", "role": "sync_standby", "state": "running", "api_url": "http://192.168.100.2:8008/patroni", "host": "192.168.100.2", "port": 5432, "timeline": 1, "lag": 0}, {"name": "acid-test-cluster-2", "role": "replica", "state": "running", "api_url": "http://192.168.100.3:8008/patroni", "host": "192.168.100.3", "port": 5432, "timeline": 1, "lag": 0}]}`
+	r := ioutil.NopCloser(bytes.NewReader([]byte(json)))
+
+	response := http.Response{
+		StatusCode: 200,
+		Body:       r,
+	}
+
+	mockClient := mocks.NewMockHTTPClient(ctrl)
+	mockClient.EXPECT().Get(gomock.Any()).Return(&response, nil)
+
+	p := New(logger, mockClient)
+
+	clusterMemberData, err := p.GetClusterMembers(newMockPod("192.168.100.1"))
+
+	if !reflect.DeepEqual(expectedClusterMemberData, clusterMemberData) {
+		t.Errorf("Patroni cluster members differ: expected: %#v, got: %#v", expectedClusterMemberData, clusterMemberData)
+	}
+
+	if err != nil {
+		t.Errorf("Could not read Patroni data: %v", err)
+	}
+}
+
 func TestGetMemberData(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()

From 087c3796876f4d5cef90aa3110dcc344efdcde0d Mon Sep 17 00:00:00 2001
From: Felix Kunde <felix-kunde@gmx.de>
Date: Thu, 16 Dec 2021 15:13:30 +0100
Subject: [PATCH 13/58] move to go 1.17.4 and update dependencies (#1717)

* move to go 1.17.4 and update dependencies
* use go install for mockgen and kind
---
 .github/workflows/run_e2e.yaml   |   2 +-
 .github/workflows/run_tests.yaml |   2 +-
 Makefile                         |   6 +-
 delivery.yaml                    |   2 +-
 docker/DebugDockerfile           |   2 +-
 docs/developer.md                |   4 +-
 e2e/Makefile                     |   6 +-
 go.mod                           |  67 +++++-
 go.sum                           | 128 +++++++---
 kubectl-pg/cmd/root.go           |   3 +-
 kubectl-pg/cmd/version.go        |   5 +-
 kubectl-pg/go.mod                |  60 ++++-
 kubectl-pg/go.sum                | 386 ++++++++++++++++++++++---------
 13 files changed, 506 insertions(+), 167 deletions(-)

diff --git a/.github/workflows/run_e2e.yaml b/.github/workflows/run_e2e.yaml
index f7d984519..5dcb195d2 100644
--- a/.github/workflows/run_e2e.yaml
+++ b/.github/workflows/run_e2e.yaml
@@ -14,7 +14,7 @@ jobs:
     - uses: actions/checkout@v1
     - uses: actions/setup-go@v2
       with:
-          go-version: "^1.16.9"
+          go-version: "^1.17.4"
     - name: Make dependencies
       run: make deps mocks
     - name: Compile
diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml
index 3ef1a348d..13810d9d2 100644
--- a/.github/workflows/run_tests.yaml
+++ b/.github/workflows/run_tests.yaml
@@ -14,7 +14,7 @@ jobs:
     - uses: actions/checkout@v2
     - uses: actions/setup-go@v2
       with:
-          go-version: "^1.16.9"
+          go-version: "^1.17.4"
     - name: Make dependencies
       run: make deps mocks
     - name: Compile
diff --git a/Makefile b/Makefile
index 07605c0a3..030a8088c 100644
--- a/Makefile
+++ b/Makefile
@@ -73,7 +73,7 @@ docker: ${DOCKERDIR}/${DOCKERFILE} docker-context
 	cd "${DOCKERDIR}" && docker build --rm -t "$(IMAGE):$(TAG)$(CDP_TAG)$(DEBUG_FRESH)$(DEBUG_POSTFIX)" -f "${DOCKERFILE}" .
 
 indocker-race:
-	docker run --rm -v "${GOPATH}":"${GOPATH}" -e GOPATH="${GOPATH}" -e RACE=1 -w ${PWD} golang:1.8.1 bash -c "make linux"
+	docker run --rm -v "${GOPATH}":"${GOPATH}" -e GOPATH="${GOPATH}" -e RACE=1 -w ${PWD} golang:1.17.3 bash -c "make linux"
 
 push:
 	docker push "$(IMAGE):$(TAG)$(CDP_TAG)"
@@ -85,8 +85,8 @@ mocks:
 	GO111MODULE=on go generate ./...
 
 tools:
-	GO111MODULE=on go get k8s.io/client-go@kubernetes-1.22.3
-	GO111MODULE=on go get github.com/golang/mock/mockgen@v1.6.0
+	GO111MODULE=on go get -d k8s.io/client-go@kubernetes-1.22.4
+	GO111MODULE=on go install github.com/golang/mock/mockgen@v1.6.0
 	GO111MODULE=on go mod tidy
 
 fmt:
diff --git a/delivery.yaml b/delivery.yaml
index 99f8f2078..15a30d42f 100644
--- a/delivery.yaml
+++ b/delivery.yaml
@@ -16,7 +16,7 @@ pipeline:
         - desc: 'Install go'
           cmd: |
             cd /tmp
-            wget -q https://storage.googleapis.com/golang/go1.16.9.linux-amd64.tar.gz -O go.tar.gz
+            wget -q https://storage.googleapis.com/golang/go1.17.4.linux-amd64.tar.gz -O go.tar.gz
             tar -xf go.tar.gz
             mv go /usr/local
             ln -s /usr/local/go/bin/go /usr/bin/go
diff --git a/docker/DebugDockerfile b/docker/DebugDockerfile
index e8f51badd..1981dfc82 100644
--- a/docker/DebugDockerfile
+++ b/docker/DebugDockerfile
@@ -9,7 +9,7 @@ COPY build/* /
 RUN addgroup -g 1000 pgo
 RUN adduser -D -u 1000 -G pgo -g 'Postgres Operator' pgo
 
-RUN go get github.com/derekparker/delve/cmd/dlv
+RUN go get -d github.com/derekparker/delve/cmd/dlv
 RUN cp /root/go/bin/dlv /dlv
 RUN chown -R pgo:pgo /dlv
 
diff --git a/docs/developer.md b/docs/developer.md
index 09fa5f527..bc80d5380 100644
--- a/docs/developer.md
+++ b/docs/developer.md
@@ -7,7 +7,7 @@ features and tests.
 
 Postgres Operator is written in Go. Use the [installation instructions](https://golang.org/doc/install#install)
 if you don't have Go on your system. You won't be able to compile the operator
-with Go older than 1.16. We recommend installing [the latest one](https://golang.org/dl/).
+with Go older than 1.17. We recommend installing [the latest one](https://golang.org/dl/).
 
 Go projects expect their source code and all the dependencies to be located
 under the [GOPATH](https://github.com/golang/go/wiki/GOPATH). Normally, one
@@ -176,7 +176,7 @@ go get -u github.com/derekparker/delve/cmd/dlv
 
 ```
 RUN apk --no-cache add go git musl-dev
-RUN go get github.com/derekparker/delve/cmd/dlv
+RUN go get -d github.com/derekparker/delve/cmd/dlv
 ```
 
 * Update the `Makefile` to build the project with debugging symbols. For that
diff --git a/e2e/Makefile b/e2e/Makefile
index 6e46d7238..b2da9c86e 100644
--- a/e2e/Makefile
+++ b/e2e/Makefile
@@ -45,9 +45,9 @@ push: docker
 
 tools:
 	# install pinned version of 'kind'
-	# go get must run outside of a dir with a (module-based) Go project !
-	# otherwise go get updates project's dependencies and/or behaves differently
-	cd "/tmp" && GO111MODULE=on go get sigs.k8s.io/kind@v0.11.1
+	# go install must run outside of a dir with a (module-based) Go project !
+	# otherwise go install updates project's dependencies and/or behaves differently
+	cd "/tmp" && GO111MODULE=on go install sigs.k8s.io/kind@v0.11.1
 
 e2etest: tools copy clean
 	./run.sh main
diff --git a/go.mod b/go.mod
index ac550f810..0d46daeec 100644
--- a/go.mod
+++ b/go.mod
@@ -1,20 +1,69 @@
 module github.com/zalando/postgres-operator
 
-go 1.16
+go 1.17
 
 require (
-	github.com/aws/aws-sdk-go v1.41.16
+	github.com/aws/aws-sdk-go v1.42.18
 	github.com/golang/mock v1.6.0
-	github.com/lib/pq v1.10.3
+	github.com/lib/pq v1.10.4
 	github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d
 	github.com/r3labs/diff v1.1.0
 	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.7.0
-	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
+	golang.org/x/crypto v0.0.0-20211202192323-5770296d904e
 	gopkg.in/yaml.v2 v2.4.0
-	k8s.io/api v0.22.3
-	k8s.io/apiextensions-apiserver v0.22.3
-	k8s.io/apimachinery v0.22.3
-	k8s.io/client-go v0.22.3
-	k8s.io/code-generator v0.22.3
+	k8s.io/api v0.22.4
+	k8s.io/apiextensions-apiserver v0.22.4
+	k8s.io/apimachinery v0.22.4
+	k8s.io/client-go v0.22.4
+	k8s.io/code-generator v0.22.4
+)
+
+require (
+	github.com/PuerkitoBio/purell v1.1.1 // indirect
+	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/emicklei/go-restful v2.9.5+incompatible // indirect
+	github.com/evanphx/json-patch v4.11.0+incompatible // indirect
+	github.com/go-logr/logr v0.4.0 // indirect
+	github.com/go-openapi/jsonpointer v0.19.5 // indirect
+	github.com/go-openapi/jsonreference v0.19.5 // indirect
+	github.com/go-openapi/swag v0.19.14 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-cmp v0.5.5 // indirect
+	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/googleapis/gnostic v0.5.5 // indirect
+	github.com/imdario/mergo v0.3.5 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.11 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/mailru/easyjson v0.7.6 // indirect
+	github.com/moby/spdystream v0.2.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	golang.org/x/mod v0.5.1 // indirect
+	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
+	golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602 // indirect
+	golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 // indirect
+	golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect
+	golang.org/x/text v0.3.6 // indirect
+	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	golang.org/x/tools v0.1.7 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.26.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027 // indirect
+	k8s.io/klog/v2 v2.9.0 // indirect
+	k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c // indirect
+	k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect
+	sigs.k8s.io/yaml v1.2.0 // indirect
 )
diff --git a/go.sum b/go.sum
index 5666613bb..fda15408b 100644
--- a/go.sum
+++ b/go.sum
@@ -9,18 +9,28 @@ cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6T
 cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
 cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
 cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
+cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
+cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
 cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
+cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
 cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
 cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
@@ -50,8 +60,8 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/aws/aws-sdk-go v1.41.16 h1:rNbPreEO4K2b8LwhHOe5+iNRH3Ih35L+H9U5tLdcevo=
-github.com/aws/aws-sdk-go v1.41.16/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
+github.com/aws/aws-sdk-go v1.42.18 h1:2f/cDNwQ3e+yHxtPn1si0to3GalbNHwkRm461IjwRiM=
+github.com/aws/aws-sdk-go v1.42.18/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
 github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
@@ -153,6 +163,8 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
 github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -160,6 +172,7 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
@@ -179,7 +192,9 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@@ -187,11 +202,14 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
 github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
@@ -266,8 +284,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/lib/pq v1.10.3 h1:v9QZf2Sn6AmjXtQeFpdoq/eaNtYP6IN+7lcrygsIAtg=
-github.com/lib/pq v1.10.3/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk=
+github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
@@ -392,9 +410,12 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
 go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
@@ -408,6 +429,7 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4=
@@ -436,8 +458,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20211202192323-5770296d904e h1:MUP6MR3rJ7Gk9LEia0LP2ytiH6MuCfs7qYz+47jGdD8=
+golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -469,8 +491,9 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38=
+golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -487,6 +510,7 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -496,28 +520,37 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
 golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602 h1:0Ja1LBD+yisY6RWM/BH7TJVXWsSjs2VwBSmvSX4HdBc=
+golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -552,9 +585,15 @@ golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -566,8 +605,10 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 h1:TyHqChC80pFkXWraUUf6RuB5IqFdQieMLwwCJokV2pc=
+golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE=
@@ -621,13 +662,24 @@ golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapK
 golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA=
 golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ=
+golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -642,13 +694,21 @@ google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb
 google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
 google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
 google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
+google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -666,10 +726,20 @@ google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvx
 google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
 google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
@@ -680,7 +750,10 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ
 google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
 google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
 google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
@@ -736,18 +809,19 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.22.3 h1:wOoES2GoSkUsdped2RB4zYypPqWtvprGoKCENTOOjP4=
-k8s.io/api v0.22.3/go.mod h1:azgiXFiXqiWyLCfI62/eYBOu19rj2LKmIhFPP4+33fs=
-k8s.io/apiextensions-apiserver v0.22.3 h1:bKku7MqawIbtTZc084BZoMV4fz0WZuvCnB5E+yrQXGM=
-k8s.io/apiextensions-apiserver v0.22.3/go.mod h1:f4plF+CXeqI89jAXL0Ml4LI/kSAZ54JS94+XOX1sae8=
-k8s.io/apimachinery v0.22.3 h1:mrvBG5CZnEfwgpVqWcrRKvdsYECTrhAR6cApAgdsflk=
-k8s.io/apimachinery v0.22.3/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0=
-k8s.io/apiserver v0.22.3/go.mod h1:oam7lH/F1Kto/WTamyQYrD68fS0mGUBORAFf6x/9Mxs=
-k8s.io/client-go v0.22.3 h1:6onkOSc+YNdwq5zXE0wFXicq64rrym+mXwHu/CPVGO4=
-k8s.io/client-go v0.22.3/go.mod h1:ElDjYf8gvZsKDYexmsmnMQ0DYO8W9RwBjfQ1PI53yow=
-k8s.io/code-generator v0.22.3 h1:24xLuKySzFl1XupMarNBkpt10q0N+73R9dF7wzJO/hE=
-k8s.io/code-generator v0.22.3/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o=
-k8s.io/component-base v0.22.3/go.mod h1:kuybv1miLCMoOk3ebrqF93GbQHQx6W2287FC0YEQY6s=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+k8s.io/api v0.22.4 h1:UvyHW0ezB2oIgHAxlYoo6UJQObYXU7awuNarwoHEOjw=
+k8s.io/api v0.22.4/go.mod h1:Rgs+9gIGYC5laXQSZZ9JqT5NevNgoGiOdVWi1BAB3qk=
+k8s.io/apiextensions-apiserver v0.22.4 h1:2iGpcVyw4MnAyyXVJU2Xg6ZsbIxAOfRHo0LF5A5J0RA=
+k8s.io/apiextensions-apiserver v0.22.4/go.mod h1:kH9lxD8dbJ+k0ZizGET55lFgdGjO8t45fgZnCVdZEpw=
+k8s.io/apimachinery v0.22.4 h1:9uwcvPpukBw/Ri0EUmWz+49cnFtaoiyEhQTK+xOe7Ck=
+k8s.io/apimachinery v0.22.4/go.mod h1:yU6oA6Gnax9RrxGzVvPFFJ+mpnW6PBSqp0sx0I0HHW0=
+k8s.io/apiserver v0.22.4/go.mod h1:38WmcUZiiy41A7Aty8/VorWRa8vDGqoUzDf2XYlku0E=
+k8s.io/client-go v0.22.4 h1:aAQ1Wk+I3bjCNk35YWUqbaueqrIonkfDPJSPDDe8Kfg=
+k8s.io/client-go v0.22.4/go.mod h1:Yzw4e5e7h1LNHA4uqnMVrpEpUs1hJOiuBsJKIlRCHDA=
+k8s.io/code-generator v0.22.4 h1:h7lBa5IuEUC4OQ45q/gIip/a0iQcML2iwrRmXksau30=
+k8s.io/code-generator v0.22.4/go.mod h1:qjYl54pQ/emhkT0UxbufbREYJMWsHNNV/jSVwhYZQGw=
+k8s.io/component-base v0.22.4/go.mod h1:MrSaQy4a3tFVViff8TZL6JHYSewNCLshZCwHYM58v5A=
 k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027 h1:Uusb3oh8XcdzDF/ndlI4ToKTYVlkCSJP39SRY2mfRAw=
 k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
@@ -755,8 +829,8 @@ k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
 k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM=
 k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
-k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e h1:KLHHjkdQFomZy8+06csTWZ0m1343QqxZhR2LJ1OxCYM=
-k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=
+k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c h1:jvamsI1tn9V0S8jicyX82qaFC0H/NKxv2e5mbqsgR80=
+k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=
 k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a h1:8dYfu/Fc9Gz2rNJKB9IQRGgQOh2clmRzNIPPY1xLY5g=
 k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
diff --git a/kubectl-pg/cmd/root.go b/kubectl-pg/cmd/root.go
index 916852998..163d6f6ea 100644
--- a/kubectl-pg/cmd/root.go
+++ b/kubectl-pg/cmd/root.go
@@ -24,9 +24,10 @@ package cmd
 
 import (
 	"fmt"
+	"os"
+
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"os"
 )
 
 var rootCmd = &cobra.Command{
diff --git a/kubectl-pg/cmd/version.go b/kubectl-pg/cmd/version.go
index 91976fbc2..e9a1e8056 100644
--- a/kubectl-pg/cmd/version.go
+++ b/kubectl-pg/cmd/version.go
@@ -24,10 +24,11 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/spf13/cobra"
-	"k8s.io/client-go/kubernetes"
 	"log"
 	"strings"
+
+	"github.com/spf13/cobra"
+	"k8s.io/client-go/kubernetes"
 )
 
 var KubectlPgVersion string = "1.0"
diff --git a/kubectl-pg/go.mod b/kubectl-pg/go.mod
index 8c54afc99..954b99940 100644
--- a/kubectl-pg/go.mod
+++ b/kubectl-pg/go.mod
@@ -1,13 +1,59 @@
 module github.com/zalando/postgres-operator/kubectl-pg
 
-go 1.16
+go 1.17
 
 require (
-	github.com/spf13/cobra v1.1.3
-	github.com/spf13/viper v1.7.1
+	github.com/spf13/cobra v1.2.1
+	github.com/spf13/viper v1.9.0
 	github.com/zalando/postgres-operator v1.7.1
-	k8s.io/api v0.22.2
-	k8s.io/apiextensions-apiserver v0.22.2
-	k8s.io/apimachinery v0.22.2
-	k8s.io/client-go v0.22.2
+	k8s.io/api v0.22.4
+	k8s.io/apiextensions-apiserver v0.22.4
+	k8s.io/apimachinery v0.22.4
+	k8s.io/client-go v0.22.4
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/go-logr/logr v0.4.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-cmp v0.5.6 // indirect
+	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/googleapis/gnostic v0.5.5 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/imdario/mergo v0.3.5 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.11 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/magiconair/properties v1.8.5 // indirect
+	github.com/mitchellh/mapstructure v1.4.2 // indirect
+	github.com/moby/spdystream v0.2.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d // indirect
+	github.com/pelletier/go-toml v1.9.4 // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/spf13/afero v1.6.0 // indirect
+	github.com/spf13/cast v1.4.1 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/subosito/gotenv v1.2.0 // indirect
+	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
+	golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect
+	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
+	golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf // indirect
+	golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect
+	golang.org/x/text v0.3.6 // indirect
+	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/ini.v1 v1.63.2 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	k8s.io/klog/v2 v2.9.0 // indirect
+	k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect
+	sigs.k8s.io/yaml v1.2.0 // indirect
 )
diff --git a/kubectl-pg/go.sum b/kubectl-pg/go.sum
index f1f097b98..aaa4b2e36 100644
--- a/kubectl-pg/go.sum
+++ b/kubectl-pg/go.sum
@@ -9,35 +9,49 @@ cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6T
 cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
 cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
 cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
+cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
+cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
+cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
+cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
+cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg=
+cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8=
+cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0=
+cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY=
+cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM=
+cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY=
+cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ=
+cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
 cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
+cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
 cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
+cloud.google.com/go/firestore v1.6.0/go.mod h1:afJwI0vaXwAG54kI7A//lP/lSPDkQORQuMkv56TxEPU=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
 cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
-github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw=
 github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA=
-github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg=
-github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
 github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M=
 github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
-github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
 github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
-github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
 github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
@@ -54,14 +68,16 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/aws/aws-sdk-go v1.36.29/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
+github.com/aws/aws-sdk-go v1.41.16/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
 github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
+github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
@@ -73,23 +89,20 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
-github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
+github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
 github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
-github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
-github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -97,9 +110,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
-github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
@@ -108,20 +119,22 @@ github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
+github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
 github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
-github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -136,14 +149,10 @@ github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7
 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
 github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
-github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
 github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
 github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg=
-github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
-github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
@@ -154,7 +163,6 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -165,12 +173,16 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
+github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
+github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
@@ -183,6 +195,7 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
 github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
 github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
@@ -190,50 +203,66 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
+github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
 github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU=
 github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw=
 github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
-github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
+github.com/hashicorp/consul/api v1.10.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M=
 github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
+github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
 github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
+github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
@@ -245,10 +274,14 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
+github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY=
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
+github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
+github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
@@ -260,13 +293,11 @@ github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUB
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ=
 github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
-github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
@@ -276,41 +307,48 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
+github.com/lib/pq v1.10.3/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls=
+github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
 github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
+github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
+github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
 github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo=
+github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
-github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo=
 github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
@@ -329,27 +367,28 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWb
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
-github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
-github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
-github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
+github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM=
+github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
+github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s=
 github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
@@ -369,7 +408,6 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
-github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/r3labs/diff v1.1.0/go.mod h1:7WjXasNzi0vJetRcB/RqNl5dlIsmXcTTLmF5IoH6Xig=
@@ -378,6 +416,7 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/sagikazarmark/crypt v0.1.0/go.mod h1:B/mN0msZuINBtQ1zZLEQcegFJJf9vnYIR88KRMEuODE=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
@@ -386,36 +425,36 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
-github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
+github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY=
+github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
-github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
-github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M=
+github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA=
+github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
-github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
+github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw=
+github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
+github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
-github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk=
-github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
+github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns=
+github.com/spf13/viper v1.9.0 h1:yR6EXjTp0y0cLN8OZg1CRZmOBdI88UcGkhgyJhu6nZk=
+github.com/spf13/viper v1.9.0/go.mod h1:+i6ajR7OX2XaiBkrcZJFK21htRk7eDeLg7+O6bhUPP4=
 github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -425,21 +464,18 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
-github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
-github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
-github.com/zalando/postgres-operator v1.7.0 h1:IgTXIVLJM37bFbvbdCks+HCIq7eGSistuLoFCUVr/tY=
-github.com/zalando/postgres-operator v1.7.0/go.mod h1:zPFdBewKFBf6726fnGc4GLXYxzbtwyVaU/7XFknyTl4=
+github.com/zalando/postgres-operator v1.7.1 h1:tDh7utqbrNoCNoQjy3seZiEoO+vT6SgP2+VlnSJ5mpg=
+github.com/zalando/postgres-operator v1.7.1/go.mod h1:hZTzOQBITJvv5nDHZpGIwyKqOghomxpZQRyWOfkPzKs=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
 go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
-go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg=
 go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
 go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
 go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
@@ -451,6 +487,9 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4=
@@ -463,7 +502,6 @@ go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi
 go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE=
 go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
-go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
@@ -476,13 +514,15 @@ golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
-golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -505,6 +545,7 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
 golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
@@ -515,6 +556,7 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -532,9 +574,10 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -542,27 +585,52 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 h1:ADo5wSpq2gqaCGQWzk7S5vd//0iyyLeAratkEoG5dLE=
+golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
+golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f h1:Qmd2pbz05z7z6lm0DrgQVVPuBm92jqujBKMHMOlOQEw=
+golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -574,47 +642,70 @@ golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio=
+golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf h1:2ucpDCmfkl8Bd/FsLtiD653Wf96cW37s+iGx93zsu4k=
+golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE=
@@ -628,11 +719,9 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs=
 golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -649,11 +738,11 @@ golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBn
 golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -672,11 +761,31 @@ golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapK
 golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
+golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -691,13 +800,33 @@ google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb
 google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
 google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
 google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
+google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
+google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
+google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
+google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
+google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU=
+google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94=
+google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8=
+google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo=
+google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4=
+google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw=
+google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU=
+google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k=
+google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -715,13 +844,43 @@ google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvx
 google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
 google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
+google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
 google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
+google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
+google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
+google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24=
+google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k=
+google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k=
+google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48=
+google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48=
+google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w=
+google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
+google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -730,11 +889,24 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ
 google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
 google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
+google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
+google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
 google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
+google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
+google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
+google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
+google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -746,21 +918,23 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
 google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
-gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/ini.v1 v1.63.2 h1:tGK/CyBg7SMzb60vP1M03vNZ3VDu3wGQJwn7Sxi9r3c=
+gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
@@ -779,7 +953,6 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
@@ -788,46 +961,41 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8=
-k8s.io/api v0.22.2 h1:M8ZzAD0V6725Fjg53fKeTJxGsJvRbk4TEm/fexHMtfw=
-k8s.io/api v0.22.2/go.mod h1:y3ydYpLJAaDI+BbSe2xmGcqxiWHmWjkEeIbiwHvnPR8=
-k8s.io/apiextensions-apiserver v0.20.6/go.mod h1:qO8YMqeMmZH+lV21LUNzV41vfpoE9QVAJRA+MNqj0mo=
-k8s.io/apiextensions-apiserver v0.22.2 h1:zK7qI8Ery7j2CaN23UCFaC1hj7dMiI87n01+nKuewd4=
-k8s.io/apiextensions-apiserver v0.22.2/go.mod h1:2E0Ve/isxNl7tWLSUDgi6+cmwHi5fQRdwGVCxbC+KFA=
-k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc=
-k8s.io/apimachinery v0.22.2 h1:ejz6y/zNma8clPVfNDLnPbleBo6MpoFy/HBiBqCouVk=
-k8s.io/apimachinery v0.22.2/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0=
-k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q=
-k8s.io/apiserver v0.22.2/go.mod h1:vrpMmbyjWrgdyOvZTSpsusQq5iigKNWv9o9KlDAbBHI=
-k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0=
-k8s.io/client-go v0.22.2 h1:DaSQgs02aCC1QcwUdkKZWOeaVsQjYvWv8ZazcZ6JcHc=
-k8s.io/client-go v0.22.2/go.mod h1:sAlhrkVDf50ZHx6z4K0S40wISNTarf1r800F+RlCF6U=
-k8s.io/code-generator v0.20.6/go.mod h1:i6FmG+QxaLxvJsezvZp0q/gAEzzOz3U53KFibghWToU=
-k8s.io/code-generator v0.22.2/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o=
-k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM=
-k8s.io/component-base v0.22.2/go.mod h1:5Br2QhI9OTe79p+TzPe9JKNQYvEKbq9rTJDWllunGug=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+k8s.io/api v0.22.3/go.mod h1:azgiXFiXqiWyLCfI62/eYBOu19rj2LKmIhFPP4+33fs=
+k8s.io/api v0.22.4 h1:UvyHW0ezB2oIgHAxlYoo6UJQObYXU7awuNarwoHEOjw=
+k8s.io/api v0.22.4/go.mod h1:Rgs+9gIGYC5laXQSZZ9JqT5NevNgoGiOdVWi1BAB3qk=
+k8s.io/apiextensions-apiserver v0.22.3/go.mod h1:f4plF+CXeqI89jAXL0Ml4LI/kSAZ54JS94+XOX1sae8=
+k8s.io/apiextensions-apiserver v0.22.4 h1:2iGpcVyw4MnAyyXVJU2Xg6ZsbIxAOfRHo0LF5A5J0RA=
+k8s.io/apiextensions-apiserver v0.22.4/go.mod h1:kH9lxD8dbJ+k0ZizGET55lFgdGjO8t45fgZnCVdZEpw=
+k8s.io/apimachinery v0.22.3/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0=
+k8s.io/apimachinery v0.22.4 h1:9uwcvPpukBw/Ri0EUmWz+49cnFtaoiyEhQTK+xOe7Ck=
+k8s.io/apimachinery v0.22.4/go.mod h1:yU6oA6Gnax9RrxGzVvPFFJ+mpnW6PBSqp0sx0I0HHW0=
+k8s.io/apiserver v0.22.3/go.mod h1:oam7lH/F1Kto/WTamyQYrD68fS0mGUBORAFf6x/9Mxs=
+k8s.io/apiserver v0.22.4/go.mod h1:38WmcUZiiy41A7Aty8/VorWRa8vDGqoUzDf2XYlku0E=
+k8s.io/client-go v0.22.3/go.mod h1:ElDjYf8gvZsKDYexmsmnMQ0DYO8W9RwBjfQ1PI53yow=
+k8s.io/client-go v0.22.4 h1:aAQ1Wk+I3bjCNk35YWUqbaueqrIonkfDPJSPDDe8Kfg=
+k8s.io/client-go v0.22.4/go.mod h1:Yzw4e5e7h1LNHA4uqnMVrpEpUs1hJOiuBsJKIlRCHDA=
+k8s.io/code-generator v0.22.3/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o=
+k8s.io/code-generator v0.22.4/go.mod h1:qjYl54pQ/emhkT0UxbufbREYJMWsHNNV/jSVwhYZQGw=
+k8s.io/component-base v0.22.3/go.mod h1:kuybv1miLCMoOk3ebrqF93GbQHQx6W2287FC0YEQY6s=
+k8s.io/component-base v0.22.4/go.mod h1:MrSaQy4a3tFVViff8TZL6JHYSewNCLshZCwHYM58v5A=
 k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
-k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
 k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
 k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
-k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM=
 k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
-k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM=
 k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=
-k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=
 k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a h1:8dYfu/Fc9Gz2rNJKB9IQRGgQOh2clmRzNIPPY1xLY5g=
 k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
 sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno=
 sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
-sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

From 411abbe31eaa383299bcabc9e936e1062a7e6806 Mon Sep 17 00:00:00 2001
From: Felix Kunde <felix-kunde@gmx.de>
Date: Fri, 17 Dec 2021 12:36:23 +0100
Subject: [PATCH 14/58] handle case when Patroni returns that lag is unknown
 (#1724)

* handle case when Patroni returns that lag is unknown
* remove some prints from e2e test
---
 e2e/tests/test_e2e.py            |  9 ---------
 manifests/postgresql.crd.yaml    |  4 ++--
 pkg/cluster/k8sres_test.go       | 14 ++++----------
 pkg/cluster/pod.go               |  8 ++++++--
 pkg/util/patroni/patroni.go      | 11 ++++++-----
 pkg/util/patroni/patroni_test.go |  8 ++++----
 pkg/util/util.go                 | 16 ++++++++++++++++
 7 files changed, 38 insertions(+), 32 deletions(-)

diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py
index e28e3e087..606abc95f 100644
--- a/e2e/tests/test_e2e.py
+++ b/e2e/tests/test_e2e.py
@@ -899,7 +899,6 @@ class EndToEndTestCase(unittest.TestCase):
 
         try:
             # patch current master node with the label
-            print('patching master node: {}'.format(master_node))
             k8s.api.core_v1.patch_node(master_node, node_label_body)
 
             # add node affinity to cluster
@@ -1636,12 +1635,8 @@ class EndToEndTestCase(unittest.TestCase):
         try:
             q = exec_query.format(db_list_query, "postgres")
             q = "su postgres -c \"{}\"".format(q)
-            print('Get databases: {}'.format(q))
             result = k8s.exec_with_kubectl(pod_name, q)
             db_list = clean_list(result.stdout.split(b'\n'))
-            print('db_list: {}, stdout: {}, stderr {}'.format(
-                db_list, result.stdout, result.stderr
-            ))
         except Exception as ex:
             print('Could not get databases: {}'.format(ex))
             print('Stdout: {}'.format(result.stdout))
@@ -1665,12 +1660,8 @@ class EndToEndTestCase(unittest.TestCase):
         try:
             q = exec_query.format(query, db_name)
             q = "su postgres -c \"{}\"".format(q)
-            print('Send query: {}'.format(q))
             result = k8s.exec_with_kubectl(pod_name, q)
             result_set = clean_list(result.stdout.split(b'\n'))
-            print('result: {}, stdout: {}, stderr {}'.format(
-                result_set, result.stdout, result.stderr
-            ))
         except Exception as ex:
             print('Error on query execution: {}'.format(ex))
             print('Stdout: {}'.format(result.stdout))
diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml
index 0f7524bcc..d855b1812 100644
--- a/manifests/postgresql.crd.yaml
+++ b/manifests/postgresql.crd.yaml
@@ -462,11 +462,11 @@ spec:
                 type: integer
               standby:
                 type: object
-                required:
-                  - s3_wal_path
                 properties:
                   s3_wal_path:
                     type: string
+                  gs_wal_path:
+                    type: string
               teamId:
                 type: string
               tls:
diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go
index 0f99d5f31..c3c4a56bd 100644
--- a/pkg/cluster/k8sres_test.go
+++ b/pkg/cluster/k8sres_test.go
@@ -24,7 +24,6 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/kubernetes/fake"
 	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
 )
@@ -47,11 +46,6 @@ type ExpectedValue struct {
 	envVarValue    string
 }
 
-func toIntStr(val int) *intstr.IntOrString {
-	b := intstr.FromInt(val)
-	return &b
-}
-
 func TestGenerateSpiloJSONConfiguration(t *testing.T) {
 	var cluster = New(
 		Config{
@@ -311,7 +305,7 @@ func TestGeneratePodDisruptionBudget(t *testing.T) {
 					Labels:    map[string]string{"team": "myapp", "cluster-name": "myapp-database"},
 				},
 				Spec: policyv1beta1.PodDisruptionBudgetSpec{
-					MinAvailable: toIntStr(1),
+					MinAvailable: util.ToIntStr(1),
 					Selector: &metav1.LabelSelector{
 						MatchLabels: map[string]string{"spilo-role": "master", "cluster-name": "myapp-database"},
 					},
@@ -335,7 +329,7 @@ func TestGeneratePodDisruptionBudget(t *testing.T) {
 					Labels:    map[string]string{"team": "myapp", "cluster-name": "myapp-database"},
 				},
 				Spec: policyv1beta1.PodDisruptionBudgetSpec{
-					MinAvailable: toIntStr(0),
+					MinAvailable: util.ToIntStr(0),
 					Selector: &metav1.LabelSelector{
 						MatchLabels: map[string]string{"spilo-role": "master", "cluster-name": "myapp-database"},
 					},
@@ -359,7 +353,7 @@ func TestGeneratePodDisruptionBudget(t *testing.T) {
 					Labels:    map[string]string{"team": "myapp", "cluster-name": "myapp-database"},
 				},
 				Spec: policyv1beta1.PodDisruptionBudgetSpec{
-					MinAvailable: toIntStr(0),
+					MinAvailable: util.ToIntStr(0),
 					Selector: &metav1.LabelSelector{
 						MatchLabels: map[string]string{"spilo-role": "master", "cluster-name": "myapp-database"},
 					},
@@ -383,7 +377,7 @@ func TestGeneratePodDisruptionBudget(t *testing.T) {
 					Labels:    map[string]string{"team": "myapp", "cluster-name": "myapp-database"},
 				},
 				Spec: policyv1beta1.PodDisruptionBudgetSpec{
-					MinAvailable: toIntStr(1),
+					MinAvailable: util.ToIntStr(1),
 					Selector: &metav1.LabelSelector{
 						MatchLabels: map[string]string{"spilo-role": "master", "cluster-name": "myapp-database"},
 					},
diff --git a/pkg/cluster/pod.go b/pkg/cluster/pod.go
index b5415dc4e..805112e29 100644
--- a/pkg/cluster/pod.go
+++ b/pkg/cluster/pod.go
@@ -514,11 +514,15 @@ func (c *Cluster) getSwitchoverCandidate(master *v1.Pod) (spec.NamespacedName, e
 	// pick candidate with lowest lag
 	// if sync_standby replicas were found assume synchronous_mode is enabled and ignore other candidates list
 	if len(syncCandidates) > 0 {
-		sort.Slice(syncCandidates, func(i, j int) bool { return syncCandidates[i].LagInMb < syncCandidates[j].LagInMb })
+		sort.Slice(syncCandidates, func(i, j int) bool {
+			return util.IntFromIntStr(syncCandidates[i].Lag) < util.IntFromIntStr(syncCandidates[j].Lag)
+		})
 		return spec.NamespacedName{Namespace: master.Namespace, Name: syncCandidates[0].Name}, nil
 	}
 	if len(candidates) > 0 {
-		sort.Slice(candidates, func(i, j int) bool { return candidates[i].LagInMb < candidates[j].LagInMb })
+		sort.Slice(candidates, func(i, j int) bool {
+			return util.IntFromIntStr(candidates[i].Lag) < util.IntFromIntStr(candidates[j].Lag)
+		})
 		return spec.NamespacedName{Namespace: master.Namespace, Name: candidates[0].Name}, nil
 	}
 
diff --git a/pkg/util/patroni/patroni.go b/pkg/util/patroni/patroni.go
index f030659ca..d3b2f28f0 100644
--- a/pkg/util/patroni/patroni.go
+++ b/pkg/util/patroni/patroni.go
@@ -16,6 +16,7 @@ import (
 	"github.com/sirupsen/logrus"
 	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
 const (
@@ -184,11 +185,11 @@ type ClusterMembers struct {
 
 // ClusterMember cluster member data from Patroni API
 type ClusterMember struct {
-	Name     string `json:"name"`
-	Role     string `json:"role"`
-	State    string `json:"state"`
-	Timeline int    `json:"timeline"`
-	LagInMb  int    `json:"lag"`
+	Name     string             `json:"name"`
+	Role     string             `json:"role"`
+	State    string             `json:"state"`
+	Timeline int                `json:"timeline"`
+	Lag      intstr.IntOrString `json:"lag,omitempty"`
 }
 
 // MemberDataPatroni child element
diff --git a/pkg/util/patroni/patroni_test.go b/pkg/util/patroni/patroni_test.go
index 48d27c8dc..614f77828 100644
--- a/pkg/util/patroni/patroni_test.go
+++ b/pkg/util/patroni/patroni_test.go
@@ -15,6 +15,7 @@ import (
 
 	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
 var logger = logrus.New().WithField("test", "patroni")
@@ -95,22 +96,21 @@ func TestGetClusterMembers(t *testing.T) {
 			Role:     "leader",
 			State:    "running",
 			Timeline: 1,
-			LagInMb:  0,
 		}, {
 			Name:     "acid-test-cluster-1",
 			Role:     "sync_standby",
 			State:    "running",
 			Timeline: 1,
-			LagInMb:  0,
+			Lag:      intstr.IntOrString{IntVal: 0},
 		}, {
 			Name:     "acid-test-cluster-2",
 			Role:     "replica",
 			State:    "running",
 			Timeline: 1,
-			LagInMb:  0,
+			Lag:      intstr.IntOrString{Type: 1, StrVal: "unknown"},
 		}}
 
-	json := `{"members": [{"name": "acid-test-cluster-0", "role": "leader", "state": "running", "api_url": "http://192.168.100.1:8008/patroni", "host": "192.168.100.1", "port": 5432, "timeline": 1}, {"name": "acid-test-cluster-1", "role": "sync_standby", "state": "running", "api_url": "http://192.168.100.2:8008/patroni", "host": "192.168.100.2", "port": 5432, "timeline": 1, "lag": 0}, {"name": "acid-test-cluster-2", "role": "replica", "state": "running", "api_url": "http://192.168.100.3:8008/patroni", "host": "192.168.100.3", "port": 5432, "timeline": 1, "lag": 0}]}`
+	json := `{"members": [{"name": "acid-test-cluster-0", "role": "leader", "state": "running", "api_url": "http://192.168.100.1:8008/patroni", "host": "192.168.100.1", "port": 5432, "timeline": 1}, {"name": "acid-test-cluster-1", "role": "sync_standby", "state": "running", "api_url": "http://192.168.100.2:8008/patroni", "host": "192.168.100.2", "port": 5432, "timeline": 1, "lag": 0}, {"name": "acid-test-cluster-2", "role": "replica", "state": "running", "api_url": "http://192.168.100.3:8008/patroni", "host": "192.168.100.3", "port": 5432, "timeline": 1, "lag": "unknown"}]}`
 	r := ioutil.NopCloser(bytes.NewReader([]byte(json)))
 
 	response := http.Response{
diff --git a/pkg/util/util.go b/pkg/util/util.go
index a52925583..688153b89 100644
--- a/pkg/util/util.go
+++ b/pkg/util/util.go
@@ -8,6 +8,7 @@ import (
 	"encoding/base64"
 	"encoding/hex"
 	"fmt"
+	"math"
 	"math/big"
 	"math/rand"
 	"reflect"
@@ -19,6 +20,7 @@ import (
 	"github.com/motomux/pretty"
 	resource "k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
 
 	"github.com/zalando/postgres-operator/pkg/spec"
 	"golang.org/x/crypto/pbkdf2"
@@ -322,6 +324,20 @@ func testNil(values ...*int32) bool {
 	return false
 }
 
+// Convert int to IntOrString type
+func ToIntStr(val int) *intstr.IntOrString {
+	b := intstr.FromInt(val)
+	return &b
+}
+
+// Get int from IntOrString and return max int if string
+func IntFromIntStr(intOrStr intstr.IntOrString) int {
+	if intOrStr.Type == 1 {
+		return math.MaxInt
+	}
+	return intOrStr.IntValue()
+}
+
 // MaxInt32 : Return maximum of two integers provided via pointers. If one value
 // is not defined, return the other one. If both are not defined, result is also
 // undefined, caller needs to check for that.

From b4155bc8fb468590677fb1e808708884d744d694 Mon Sep 17 00:00:00 2001
From: Felix Kunde <felix-kunde@gmx.de>
Date: Thu, 6 Jan 2022 10:55:29 +0100
Subject: [PATCH 15/58] update year in LICENSE file to 2022 (#1734)

---
 LICENSE                                                         | 2 +-
 pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go              | 2 +-
 pkg/generated/clientset/versioned/clientset.go                  | 2 +-
 pkg/generated/clientset/versioned/doc.go                        | 2 +-
 pkg/generated/clientset/versioned/fake/clientset_generated.go   | 2 +-
 pkg/generated/clientset/versioned/fake/doc.go                   | 2 +-
 pkg/generated/clientset/versioned/fake/register.go              | 2 +-
 pkg/generated/clientset/versioned/scheme/doc.go                 | 2 +-
 pkg/generated/clientset/versioned/scheme/register.go            | 2 +-
 .../versioned/typed/acid.zalan.do/v1/acid.zalan.do_client.go    | 2 +-
 pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/doc.go | 2 +-
 .../clientset/versioned/typed/acid.zalan.do/v1/fake/doc.go      | 2 +-
 .../typed/acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go    | 2 +-
 .../typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go   | 2 +-
 .../versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go    | 2 +-
 .../versioned/typed/acid.zalan.do/v1/fake/fake_postgresteam.go  | 2 +-
 .../versioned/typed/acid.zalan.do/v1/generated_expansion.go     | 2 +-
 .../versioned/typed/acid.zalan.do/v1/operatorconfiguration.go   | 2 +-
 .../clientset/versioned/typed/acid.zalan.do/v1/postgresql.go    | 2 +-
 .../clientset/versioned/typed/acid.zalan.do/v1/postgresteam.go  | 2 +-
 .../informers/externalversions/acid.zalan.do/interface.go       | 2 +-
 .../informers/externalversions/acid.zalan.do/v1/interface.go    | 2 +-
 .../informers/externalversions/acid.zalan.do/v1/postgresql.go   | 2 +-
 .../informers/externalversions/acid.zalan.do/v1/postgresteam.go | 2 +-
 pkg/generated/informers/externalversions/factory.go             | 2 +-
 pkg/generated/informers/externalversions/generic.go             | 2 +-
 .../externalversions/internalinterfaces/factory_interfaces.go   | 2 +-
 pkg/generated/listers/acid.zalan.do/v1/expansion_generated.go   | 2 +-
 pkg/generated/listers/acid.zalan.do/v1/postgresql.go            | 2 +-
 pkg/generated/listers/acid.zalan.do/v1/postgresteam.go          | 2 +-
 30 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/LICENSE b/LICENSE
index 7c0f459a5..4a87999ea 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2021 Zalando SE
+Copyright (c) 2022 Zalando SE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
index 61bbfde9c..8c69515dd 100644
--- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
+++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
@@ -2,7 +2,7 @@
 // +build !ignore_autogenerated
 
 /*
-Copyright 2021 Compose, Zalando SE
+Copyright 2022 Compose, Zalando SE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/pkg/generated/clientset/versioned/clientset.go b/pkg/generated/clientset/versioned/clientset.go
index ab4a88735..e0e97cf09 100644
--- a/pkg/generated/clientset/versioned/clientset.go
+++ b/pkg/generated/clientset/versioned/clientset.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2021 Compose, Zalando SE
+Copyright 2022 Compose, Zalando SE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/pkg/generated/clientset/versioned/doc.go b/pkg/generated/clientset/versioned/doc.go
index ae87609f6..ebb4ab535 100644
--- a/pkg/generated/clientset/versioned/doc.go
+++ b/pkg/generated/clientset/versioned/doc.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2021 Compose, Zalando SE
+Copyright 2022 Compose, Zalando SE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/pkg/generated/clientset/versioned/fake/clientset_generated.go b/pkg/generated/clientset/versioned/fake/clientset_generated.go
index 65d918801..25daca07c 100644
--- a/pkg/generated/clientset/versioned/fake/clientset_generated.go
+++ b/pkg/generated/clientset/versioned/fake/clientset_generated.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2021 Compose, Zalando SE
+Copyright 2022 Compose, Zalando SE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/pkg/generated/clientset/versioned/fake/doc.go b/pkg/generated/clientset/versioned/fake/doc.go
index bc1c91a11..b9e99ecf2 100644
--- a/pkg/generated/clientset/versioned/fake/doc.go
+++ b/pkg/generated/clientset/versioned/fake/doc.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2021 Compose, Zalando SE
+Copyright 2022 Compose, Zalando SE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/pkg/generated/clientset/versioned/fake/register.go b/pkg/generated/clientset/versioned/fake/register.go
index c4d383aab..d63cab2ff 100644
--- a/pkg/generated/clientset/versioned/fake/register.go
+++ b/pkg/generated/clientset/versioned/fake/register.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2021 Compose, Zalando SE
+Copyright 2022 Compose, Zalando SE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/pkg/generated/clientset/versioned/scheme/doc.go b/pkg/generated/clientset/versioned/scheme/doc.go
index cd594164b..387784624 100644
--- a/pkg/generated/clientset/versioned/scheme/doc.go
+++ b/pkg/generated/clientset/versioned/scheme/doc.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2021 Compose, Zalando SE
+Copyright 2022 Compose, Zalando SE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/pkg/generated/clientset/versioned/scheme/register.go b/pkg/generated/clientset/versioned/scheme/register.go
index 8be969eb5..beb803602 100644
--- a/pkg/generated/clientset/versioned/scheme/register.go
+++ b/pkg/generated/clientset/versioned/scheme/register.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2021 Compose, Zalando SE
+Copyright 2022 Compose, Zalando SE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/acid.zalan.do_client.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/acid.zalan.do_client.go
index 5666201d4..d36078922 100644
--- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/acid.zalan.do_client.go
+++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/acid.zalan.do_client.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2021 Compose, Zalando SE
+Copyright 2022 Compose, Zalando SE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/doc.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/doc.go
index eb8fcf1f4..ba729b8ca 100644
--- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/doc.go
+++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/doc.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2021 Compose, Zalando SE
+Copyright 2022 Compose, Zalando SE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/doc.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/doc.go
index c5fd1c04b..380725443 100644
--- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/doc.go
+++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/doc.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2021 Compose, Zalando SE
+Copyright 2022 Compose, Zalando SE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go
index 03e7dda94..c235b2761 100644
--- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go
+++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2021 Compose, Zalando SE
+Copyright 2022 Compose, Zalando SE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go
index c03ea7d94..fbea4930c 100644
--- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go
+++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2021 Compose, Zalando SE
+Copyright 2022 Compose, Zalando SE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go
index 01a0ed7a4..cb39ea290 100644
--- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go
+++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2021 Compose, Zalando SE
+Copyright 2022 Compose, Zalando SE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresteam.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresteam.go
index b333ae046..59cf600aa 100644
--- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresteam.go
+++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresteam.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2021 Compose, Zalando SE
+Copyright 2022 Compose, Zalando SE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/generated_expansion.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/generated_expansion.go
index b4e99cbc8..a40f2bdb0 100644
--- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/generated_expansion.go
+++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/generated_expansion.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2021 Compose, Zalando SE
+Copyright 2022 Compose, Zalando SE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go
index be22e075d..a73fb0df6 100644
--- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go
+++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2021 Compose, Zalando SE
+Copyright 2022 Compose, Zalando SE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go
index 5241cfb54..c58ed03ed 100644
--- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go
+++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2021 Compose, Zalando SE
+Copyright 2022 Compose, Zalando SE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresteam.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresteam.go
index 96fbb882a..002254ef1 100644
--- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresteam.go
+++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresteam.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2021 Compose, Zalando SE
+Copyright 2022 Compose, Zalando SE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/pkg/generated/informers/externalversions/acid.zalan.do/interface.go b/pkg/generated/informers/externalversions/acid.zalan.do/interface.go
index 6f77564fa..b2b85d87d 100644
--- a/pkg/generated/informers/externalversions/acid.zalan.do/interface.go
+++ b/pkg/generated/informers/externalversions/acid.zalan.do/interface.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2021 Compose, Zalando SE
+Copyright 2022 Compose, Zalando SE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/pkg/generated/informers/externalversions/acid.zalan.do/v1/interface.go b/pkg/generated/informers/externalversions/acid.zalan.do/v1/interface.go
index 5c05e6d68..347c53f99 100644
--- a/pkg/generated/informers/externalversions/acid.zalan.do/v1/interface.go
+++ b/pkg/generated/informers/externalversions/acid.zalan.do/v1/interface.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2021 Compose, Zalando SE
+Copyright 2022 Compose, Zalando SE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go b/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go
index 1453af276..61d88fb22 100644
--- a/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go
+++ b/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2021 Compose, Zalando SE
+Copyright 2022 Compose, Zalando SE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresteam.go b/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresteam.go
index a19e4726f..eb15e4fad 100644
--- a/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresteam.go
+++ b/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresteam.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2021 Compose, Zalando SE
+Copyright 2022 Compose, Zalando SE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/pkg/generated/informers/externalversions/factory.go b/pkg/generated/informers/externalversions/factory.go
index e4b1efdc6..20a7f415f 100644
--- a/pkg/generated/informers/externalversions/factory.go
+++ b/pkg/generated/informers/externalversions/factory.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2021 Compose, Zalando SE
+Copyright 2022 Compose, Zalando SE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/pkg/generated/informers/externalversions/generic.go b/pkg/generated/informers/externalversions/generic.go
index 5fd693558..652ba0f77 100644
--- a/pkg/generated/informers/externalversions/generic.go
+++ b/pkg/generated/informers/externalversions/generic.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2021 Compose, Zalando SE
+Copyright 2022 Compose, Zalando SE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go b/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go
index 6d1b334bf..1046f8f48 100644
--- a/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go
+++ b/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2021 Compose, Zalando SE
+Copyright 2022 Compose, Zalando SE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/pkg/generated/listers/acid.zalan.do/v1/expansion_generated.go b/pkg/generated/listers/acid.zalan.do/v1/expansion_generated.go
index cc3e578b2..3189f5948 100644
--- a/pkg/generated/listers/acid.zalan.do/v1/expansion_generated.go
+++ b/pkg/generated/listers/acid.zalan.do/v1/expansion_generated.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2021 Compose, Zalando SE
+Copyright 2022 Compose, Zalando SE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/pkg/generated/listers/acid.zalan.do/v1/postgresql.go b/pkg/generated/listers/acid.zalan.do/v1/postgresql.go
index d2258bd01..650f8d579 100644
--- a/pkg/generated/listers/acid.zalan.do/v1/postgresql.go
+++ b/pkg/generated/listers/acid.zalan.do/v1/postgresql.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2021 Compose, Zalando SE
+Copyright 2022 Compose, Zalando SE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/pkg/generated/listers/acid.zalan.do/v1/postgresteam.go b/pkg/generated/listers/acid.zalan.do/v1/postgresteam.go
index 38073e92d..c63b8da58 100644
--- a/pkg/generated/listers/acid.zalan.do/v1/postgresteam.go
+++ b/pkg/generated/listers/acid.zalan.do/v1/postgresteam.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2021 Compose, Zalando SE
+Copyright 2022 Compose, Zalando SE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

From fe340192cabc9757062073530f8599a5c076ff2a Mon Sep 17 00:00:00 2001
From: Damian Peckett <damian@pecke.tt>
Date: Thu, 13 Jan 2022 15:20:04 +0100
Subject: [PATCH 16/58] Add configuration flag for disabling operator CRD
 creation/update (#1733)

* Make CRD registration configurable and drop RBAC permissions when CRD registration is disabled
* add generated deep copy functions

Co-authored-by: Damian Peckett <d.peckett_admin@mgmt.innovo-cloud.de>
---
 .../crds/operatorconfigurations.yaml                 |  3 +++
 charts/postgres-operator/templates/clusterrole.yaml  |  4 +++-
 charts/postgres-operator/values.yaml                 |  2 ++
 docs/reference/operator_parameters.md                |  4 ++++
 manifests/configmap.yaml                             |  1 +
 manifests/operatorconfiguration.crd.yaml             |  3 +++
 .../postgresql-operator-default-configuration.yaml   |  1 +
 pkg/apis/acid.zalan.do/v1/crds.go                    |  3 +++
 .../acid.zalan.do/v1/operator_configuration_type.go  |  1 +
 pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go   |  5 +++++
 pkg/controller/controller.go                         | 12 ++++++++----
 pkg/controller/operator_config.go                    |  1 +
 pkg/util/config/config.go                            | 11 ++++++-----
 13 files changed, 41 insertions(+), 10 deletions(-)

diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml
index 043129516..abe60a1d8 100644
--- a/charts/postgres-operator/crds/operatorconfigurations.yaml
+++ b/charts/postgres-operator/crds/operatorconfigurations.yaml
@@ -64,6 +64,9 @@ spec:
               docker_image:
                 type: string
                 default: "registry.opensource.zalan.do/acid/spilo-14:2.1-p3"
+              enable_crd_registration:
+                type: boolean
+                default: true
               enable_crd_validation:
                 type: boolean
                 default: true
diff --git a/charts/postgres-operator/templates/clusterrole.yaml b/charts/postgres-operator/templates/clusterrole.yaml
index 885bad3f7..87fd38cd2 100644
--- a/charts/postgres-operator/templates/clusterrole.yaml
+++ b/charts/postgres-operator/templates/clusterrole.yaml
@@ -40,10 +40,12 @@ rules:
   resources:
   - customresourcedefinitions
   verbs:
-  - create
   - get
+{{- if toString .Values.configGeneral.enable_crd_registration | eq "true" }}
+  - create
   - patch
   - update
+{{- end }}
 # to send events to the CRs
 - apiGroups:
   - ""
diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml
index 65619845a..bbb9e6388 100644
--- a/charts/postgres-operator/values.yaml
+++ b/charts/postgres-operator/values.yaml
@@ -20,6 +20,8 @@ enableJsonLogging: false
 
 # general configuration parameters
 configGeneral:
+  # the deployment should create/update the CRDs
+  enable_crd_registration: true
   # choose if deployment creates/updates CRDs with OpenAPIV3Validation
   enable_crd_validation: true
   # update only the statefulsets without immediately doing the rolling update
diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md
index 00febcf89..6efae773d 100644
--- a/docs/reference/operator_parameters.md
+++ b/docs/reference/operator_parameters.md
@@ -70,6 +70,10 @@ Variable names are underscore-separated words.
 
 Those are top-level keys, containing both leaf keys and groups.
 
+* **enable_crd_registration**
+  Instruct the operator to create/update the CRDs. If disabled the operator will rely on the CRDs being managed separately.
+  The default is `true`.
+
 * **enable_crd_validation**
   toggles if the operator will create or update CRDs with
   [OpenAPI v3 schema validation](https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#validation)
diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml
index 7d3e14ce3..932bd60ca 100644
--- a/manifests/configmap.yaml
+++ b/manifests/configmap.yaml
@@ -35,6 +35,7 @@ data:
   docker_image: registry.opensource.zalan.do/acid/spilo-14:2.1-p3
   # downscaler_annotations: "deployment-time,downscaler/*"
   # enable_admin_role_for_users: "true"
+  # enable_crd_registration: "true"
   # enable_crd_validation: "true"
   # enable_cross_namespace_secret: "false"
   # enable_database_access: "true"
diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml
index bb64995ab..d4b1a2996 100644
--- a/manifests/operatorconfiguration.crd.yaml
+++ b/manifests/operatorconfiguration.crd.yaml
@@ -62,6 +62,9 @@ spec:
               docker_image:
                 type: string
                 default: "registry.opensource.zalan.do/acid/spilo-14:2.1-p3"
+              enable_crd_registration:
+                type: boolean
+                default: true
               enable_crd_validation:
                 type: boolean
                 default: true
diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml
index 02d558543..2ad74b1e4 100644
--- a/manifests/postgresql-operator-default-configuration.yaml
+++ b/manifests/postgresql-operator-default-configuration.yaml
@@ -4,6 +4,7 @@ metadata:
   name: postgresql-operator-default-configuration
 configuration:
   docker_image: registry.opensource.zalan.do/acid/spilo-14:2.1-p3
+  # enable_crd_registration: true
   # enable_crd_validation: true
   # enable_lazy_spilo_upgrade: false
   enable_pgversion_env_var: true
diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go
index fae5a09f2..11187ad75 100644
--- a/pkg/apis/acid.zalan.do/v1/crds.go
+++ b/pkg/apis/acid.zalan.do/v1/crds.go
@@ -946,6 +946,9 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{
 					"docker_image": {
 						Type: "string",
 					},
+					"enable_crd_registration": {
+						Type: "boolean",
+					},
 					"enable_crd_validation": {
 						Type: "boolean",
 					},
diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
index f8eb5b5d1..a1dee6bff 100644
--- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
+++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
@@ -215,6 +215,7 @@ type OperatorLogicalBackupConfiguration struct {
 
 // OperatorConfigurationData defines the operation config
 type OperatorConfigurationData struct {
+	EnableCRDRegistration      *bool                              `json:"enable_crd_registration,omitempty"`
 	EnableCRDValidation        *bool                              `json:"enable_crd_validation,omitempty"`
 	EnableLazySpiloUpgrade     bool                               `json:"enable_lazy_spilo_upgrade,omitempty"`
 	EnablePgVersionEnvVar      bool                               `json:"enable_pgversion_env_var,omitempty"`
diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
index 8c69515dd..e9ab46382 100644
--- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
+++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
@@ -367,6 +367,11 @@ func (in *OperatorConfiguration) DeepCopyObject() runtime.Object {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *OperatorConfigurationData) DeepCopyInto(out *OperatorConfigurationData) {
 	*out = *in
+	if in.EnableCRDRegistration != nil {
+		in, out := &in.EnableCRDRegistration, &out.EnableCRDRegistration
+		*out = new(bool)
+		**out = **in
+	}
 	if in.EnableCRDValidation != nil {
 		in, out := &in.EnableCRDValidation, &out.EnableCRDValidation
 		*out = new(bool)
diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go
index 5d9e00bc8..54e50a45f 100644
--- a/pkg/controller/controller.go
+++ b/pkg/controller/controller.go
@@ -309,8 +309,10 @@ func (c *Controller) initController() {
 	c.controllerID = os.Getenv("CONTROLLER_ID")
 
 	if configObjectName := os.Getenv("POSTGRES_OPERATOR_CONFIGURATION_OBJECT"); configObjectName != "" {
-		if err := c.createConfigurationCRD(c.opConfig.EnableCRDValidation); err != nil {
-			c.logger.Fatalf("could not register Operator Configuration CustomResourceDefinition: %v", err)
+		if c.opConfig.EnableCRDRegistration != nil && *c.opConfig.EnableCRDRegistration {
+			if err := c.createConfigurationCRD(c.opConfig.EnableCRDValidation); err != nil {
+				c.logger.Fatalf("could not register Operator Configuration CustomResourceDefinition: %v", err)
+			}
 		}
 		if cfg, err := c.readOperatorConfigurationFromCRD(spec.GetOperatorNamespace(), configObjectName); err != nil {
 			c.logger.Fatalf("unable to read operator configuration: %v", err)
@@ -325,8 +327,10 @@ func (c *Controller) initController() {
 
 	c.modifyConfigFromEnvironment()
 
-	if err := c.createPostgresCRD(c.opConfig.EnableCRDValidation); err != nil {
-		c.logger.Fatalf("could not register Postgres CustomResourceDefinition: %v", err)
+	if c.opConfig.EnableCRDRegistration != nil && *c.opConfig.EnableCRDRegistration {
+		if err := c.createPostgresCRD(c.opConfig.EnableCRDValidation); err != nil {
+			c.logger.Fatalf("could not register Postgres CustomResourceDefinition: %v", err)
+		}
 	}
 
 	c.initSharedInformers()
diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go
index 275898d8e..2f5261cd2 100644
--- a/pkg/controller/operator_config.go
+++ b/pkg/controller/operator_config.go
@@ -33,6 +33,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	result := &config.Config{}
 
 	// general config
+	result.EnableCRDRegistration = util.CoalesceBool(fromCRD.EnableCRDRegistration, util.True())
 	result.EnableCRDValidation = util.CoalesceBool(fromCRD.EnableCRDValidation, util.True())
 	result.EnableLazySpiloUpgrade = fromCRD.EnableLazySpiloUpgrade
 	result.EnablePgVersionEnvVar = fromCRD.EnablePgVersionEnvVar
diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go
index 71bf406e4..bb77e6231 100644
--- a/pkg/util/config/config.go
+++ b/pkg/util/config/config.go
@@ -14,11 +14,12 @@ import (
 
 // CRD describes CustomResourceDefinition specific configuration parameters
 type CRD struct {
-	ReadyWaitInterval   time.Duration `name:"ready_wait_interval" default:"4s"`
-	ReadyWaitTimeout    time.Duration `name:"ready_wait_timeout" default:"30s"`
-	ResyncPeriod        time.Duration `name:"resync_period" default:"30m"`
-	RepairPeriod        time.Duration `name:"repair_period" default:"5m"`
-	EnableCRDValidation *bool         `name:"enable_crd_validation" default:"true"`
+	ReadyWaitInterval     time.Duration `name:"ready_wait_interval" default:"4s"`
+	ReadyWaitTimeout      time.Duration `name:"ready_wait_timeout" default:"30s"`
+	ResyncPeriod          time.Duration `name:"resync_period" default:"30m"`
+	RepairPeriod          time.Duration `name:"repair_period" default:"5m"`
+	EnableCRDRegistration *bool         `name:"enable_crd_registration" default:"true"`
+	EnableCRDValidation   *bool         `name:"enable_crd_validation" default:"true"`
 }
 
 // Resources describes kubernetes resource specific configuration parameters

From a78a619e909a1c64d4917bf6a426bcaff18dbc39 Mon Sep 17 00:00:00 2001
From: Felix Kunde <felix-kunde@gmx.de>
Date: Thu, 27 Jan 2022 15:57:24 +0100
Subject: [PATCH 17/58] toleration diff and nodeReadinessLabel merge with
 manifest matchExpressions (#1729)

* include tolerations in statefulset comparison
* provide alternative merge behavior of nodeSelectorTerms for node readiness label
* add config option to change affinity merge behavior
* reworked e2e tests around node affinity
---
 .../crds/operatorconfigurations.yaml          |   5 +
 charts/postgres-operator/values.yaml          |   3 +
 docs/administrator.md                         |  75 +++++++
 docs/reference/operator_parameters.md         |  15 +-
 docs/user.md                                  |   7 +-
 e2e/tests/k8s_api.py                          |   2 +-
 e2e/tests/test_e2e.py                         | 188 ++++++++++--------
 manifests/configmap.yaml                      |   3 +-
 manifests/operatorconfiguration.crd.yaml      |   5 +
 ...gresql-operator-default-configuration.yaml |   1 +
 pkg/apis/acid.zalan.do/v1/crds.go             |  11 +
 .../v1/operator_configuration_type.go         |   1 +
 pkg/cluster/cluster.go                        |   9 +-
 pkg/cluster/connection_pooler.go              |   2 +-
 pkg/cluster/k8sres.go                         |  20 +-
 pkg/controller/operator_config.go             |   1 +
 pkg/util/config/config.go                     |   1 +
 17 files changed, 245 insertions(+), 104 deletions(-)

diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml
index abe60a1d8..3cc08b4df 100644
--- a/charts/postgres-operator/crds/operatorconfigurations.yaml
+++ b/charts/postgres-operator/crds/operatorconfigurations.yaml
@@ -233,6 +233,11 @@ spec:
                     type: object
                     additionalProperties:
                       type: string
+                  node_readiness_label_merge:
+                    type: string
+                    enum:
+                      - "AND"
+                      - "OR"
                   oauth_token_secret_name:
                     type: string
                     default: "postgresql-operator"
diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml
index bbb9e6388..0602a0e93 100644
--- a/charts/postgres-operator/values.yaml
+++ b/charts/postgres-operator/values.yaml
@@ -132,6 +132,9 @@ configKubernetes:
   # node_readiness_label:
   #   status: ready
 
+  # defines how nodeAffinity from manifest should be merged with node_readiness_label
+  # node_readiness_label_merge: "OR"
+
   # namespaced name of the secret containing the OAuth2 token to pass to the teams API
   # oauth_token_secret_name: postgresql-operator
 
diff --git a/docs/administrator.md b/docs/administrator.md
index 551ee5523..879b677b9 100644
--- a/docs/administrator.md
+++ b/docs/administrator.md
@@ -339,6 +339,81 @@ master pods from being evicted by the K8s runtime. To prevent eviction
 completely, specify the toleration by leaving out the `tolerationSeconds` value
 (similar to how Kubernetes' own DaemonSets are configured)
 
+## Node readiness labels
+
+The operator can watch on certain node labels to detect e.g. the start of a
+Kubernetes cluster upgrade procedure and move master pods off the nodes to be
+decommissioned. Key-value pairs for these node readiness labels can be
+specified in the configuration (option name is in singular form):
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: postgres-operator
+data:
+  node_readiness_label: "status1:ready,status2:ready"
+```
+
+```yaml
+apiVersion: "acid.zalan.do/v1"
+kind: OperatorConfiguration
+metadata:
+  name: postgresql-configuration
+configuration:
+  kubernetes:
+    node_readiness_label:
+      status1: ready
+      status2: ready
+```
+
+The operator will create a `nodeAffinity` on the pods. This makes the
+`node_readiness_label` option the global configuration for defining node
+affinities for all Postgres clusters. You can have both, cluster-specific and
+global affinity, defined and they will get merged on the pods. If
+`node_readiness_label_merge` is configured to `"AND"` the node readiness
+affinity will end up under the same `matchExpressions` section(s) from the
+manifest affinity.
+
+```yaml
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: environment
+            operator: In
+            values:
+            - pci
+          - key: status1
+            operator: In
+            values:
+            - ready
+          - key: status2
+            ...
+```
+
+If `node_readiness_label_merge` is set to `"OR"` (default) the readiness label
+affinty will be appended with its own expressions block:
+
+```yaml
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: environment
+            ...
+        - matchExpressions:
+          - key: storage
+            ...
+        - matchExpressions:
+          - key: status1
+            ...
+          - key: status2
+            ...
+```
+
 ## Enable pod anti affinity
 
 To ensure Postgres pods are running on different topologies, you can use
diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md
index 6efae773d..f6b94e0a5 100644
--- a/docs/reference/operator_parameters.md
+++ b/docs/reference/operator_parameters.md
@@ -344,11 +344,16 @@ configuration they are grouped under the `kubernetes` key.
 
 * **node_readiness_label**
   a set of labels that a running and active node should possess to be
-  considered `ready`. The operator uses values of those labels to detect the
-  start of the Kubernetes cluster upgrade procedure and move master pods off
-  the nodes to be decommissioned. When the set is not empty, the operator also
-  assigns the `Affinity` clause to the Postgres pods to be scheduled only on
-  `ready` nodes. The default is empty.
+  considered `ready`. When the set is not empty, the operator assigns the
+  `nodeAffinity` clause to the Postgres pods to be scheduled only on `ready`
+  nodes. The default is empty.
+
+* **node_readiness_label_merge**
+  If a `nodeAffinity` is also specified in the postgres cluster manifest
+  it will get merged with the `node_readiness_label` affinity on the pods.
+  The merge strategy can be configured - it can either be "AND" or "OR".
+  See [user docs](../user.md#use-taints-tolerations-and-node-affinity-for-dedicated-postgresql-nodes)
+  for more details. Default is "OR".
 
 * **toleration**
   a dictionary that should contain `key`, `operator`, `value` and
diff --git a/docs/user.md b/docs/user.md
index 572d832ab..20db45979 100644
--- a/docs/user.md
+++ b/docs/user.md
@@ -671,7 +671,9 @@ configured [default requests](reference/operator_parameters.md#kubernetes-resour
 
 To ensure Postgres pods are running on nodes without any other application pods,
 you can use [taints and tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/)
-and configure the required toleration in the manifest.
+and configure the required toleration in the manifest. Tolerations can also be
+defined in the [operator config](administrator.md#use-taints-and-tolerations-for-dedicated-postgresql-nodes)
+to apply for all Postgres clusters.
 
 ```yaml
 spec:
@@ -703,6 +705,9 @@ spec:
           - pci
 ```
 
+If you need to define a `nodeAffinity` for all your Postgres clusters use the
+`node_readiness_label` [configuration](administrator.md#node-readiness-labels).
+
 ## In-place major version upgrade
 
 Starting with Spilo 13, operator supports in-place major version upgrade to a
diff --git a/e2e/tests/k8s_api.py b/e2e/tests/k8s_api.py
index c3ad1c999..f58e16b50 100644
--- a/e2e/tests/k8s_api.py
+++ b/e2e/tests/k8s_api.py
@@ -53,7 +53,7 @@ class K8s:
 
         return master_pod_node, replica_pod_nodes
 
-    def get_cluster_nodes(self, cluster_labels='cluster-name=acid-minimal-cluster', namespace='default'):
+    def get_cluster_nodes(self, cluster_labels='application=spilo,cluster-name=acid-minimal-cluster', namespace='default'):
         m = []
         r = []
         podsList = self.api.core_v1.list_namespaced_pod(namespace, label_selector=cluster_labels)
diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py
index 606abc95f..3047de259 100644
--- a/e2e/tests/test_e2e.py
+++ b/e2e/tests/test_e2e.py
@@ -286,7 +286,7 @@ class EndToEndTestCase(unittest.TestCase):
         # revert config change
         revert_resync = {
             "data": {
-                "resync_period": "30m",
+                "resync_period": "4m",
             },
         }
         k8s.update_config(revert_resync)
@@ -880,12 +880,10 @@ class EndToEndTestCase(unittest.TestCase):
 
         # verify we are in good state from potential previous tests
         self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No 2 pods running")
-        self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members("acid-minimal-cluster-0")), 2, "Postgres status did not enter running")
 
         # get nodes of master and replica(s)
-        master_node, replica_nodes = k8s.get_pg_nodes(cluster_label)
-
-        self.assertNotEqual(master_node, [])
+        master_nodes, replica_nodes = k8s.get_cluster_nodes()
+        self.assertNotEqual(master_nodes, [])
         self.assertNotEqual(replica_nodes, [])
 
         # label node with environment=postgres
@@ -898,8 +896,8 @@ class EndToEndTestCase(unittest.TestCase):
         }
 
         try:
-            # patch current master node with the label
-            k8s.api.core_v1.patch_node(master_node, node_label_body)
+            # patch master node with the label
+            k8s.api.core_v1.patch_node(master_nodes[0], node_label_body)
 
             # add node affinity to cluster
             patch_node_affinity_config = {
@@ -923,7 +921,6 @@ class EndToEndTestCase(unittest.TestCase):
                     }
                 }
             }
-
             k8s.api.custom_objects_api.patch_namespaced_custom_object(
                 group="acid.zalan.do",
                 version="v1",
@@ -934,14 +931,17 @@ class EndToEndTestCase(unittest.TestCase):
             self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
 
             # node affinity change should cause replica to relocate from replica node to master node due to node affinity requirement
-            k8s.wait_for_pod_failover(master_node, 'spilo-role=replica,' + cluster_label)
+            k8s.wait_for_pod_failover(master_nodes, 'spilo-role=replica,' + cluster_label)
+            k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label)
+            # next master will be switched over and pod needs to be replaced as well to finish the rolling update
+            k8s.wait_for_pod_failover(master_nodes, 'spilo-role=master,' + cluster_label)
             k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label)
 
             podsList = k8s.api.core_v1.list_namespaced_pod('default', label_selector=cluster_label)
             for pod in podsList.items:
                 if pod.metadata.labels.get('spilo-role') == 'replica':
-                    self.assertEqual(master_node, pod.spec.node_name,
-                         "Sanity check: expected replica to relocate to master node {}, but found on {}".format(master_node, pod.spec.node_name))
+                    self.assertEqual(master_nodes[0], pod.spec.node_name,
+                         "Sanity check: expected replica to relocate to master node {}, but found on {}".format(master_nodes[0], pod.spec.node_name))
 
                     # check that pod has correct node affinity
                     key = pod.spec.affinity.node_affinity.required_during_scheduling_ignored_during_execution.node_selector_terms[0].match_expressions[0].key
@@ -966,15 +966,17 @@ class EndToEndTestCase(unittest.TestCase):
             self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
 
             # node affinity change should cause another rolling update and relocation of replica
-            k8s.wait_for_pod_failover(replica_nodes, 'spilo-role=master,' + cluster_label)
+            k8s.wait_for_pod_failover(replica_nodes, 'spilo-role=replica,' + cluster_label)
             k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label)
 
         except timeout_decorator.TimeoutError:
             print('Operator log: {}'.format(k8s.get_operator_log()))
             raise
 
+        # toggle pod anti affinity to make sure replica and master run on separate nodes
+        self.assert_distributed_pods(replica_nodes)
+
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
-    @unittest.skip("Skipping this test until fixed")
     def test_node_readiness_label(self):
         '''
            Remove node readiness label from master node. This must cause a failover.
@@ -984,12 +986,15 @@ class EndToEndTestCase(unittest.TestCase):
         readiness_label = 'lifecycle-status'
         readiness_value = 'ready'
 
-        try:
-            # get nodes of master and replica(s) (expected target of new master)
-            current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label)
-            num_replicas = len(current_replica_nodes)
-            failover_targets = self.get_failover_targets(current_master_node, current_replica_nodes)
+        # verify we are in good state from potential previous tests
+        self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No 2 pods running")
 
+        # get nodes of master and replica(s) (expected target of new master)
+        master_nodes, replica_nodes = k8s.get_cluster_nodes()
+        self.assertNotEqual(master_nodes, [])
+        self.assertNotEqual(replica_nodes, [])
+
+        try:
             # add node_readiness_label to potential failover nodes
             patch_readiness_label = {
                 "metadata": {
@@ -998,30 +1003,43 @@ class EndToEndTestCase(unittest.TestCase):
                     }
                 }
             }
-            self.assertTrue(len(failover_targets) > 0, "No failover targets available")
-            for failover_target in failover_targets:
-                k8s.api.core_v1.patch_node(failover_target, patch_readiness_label)
+            for replica_node in replica_nodes:
+                k8s.api.core_v1.patch_node(replica_node, patch_readiness_label)
 
-            # define node_readiness_label in config map which should trigger a failover of the master
+            # define node_readiness_label in config map which should trigger a rolling update
             patch_readiness_label_config = {
                 "data": {
                     "node_readiness_label": readiness_label + ':' + readiness_value,
+                    "node_readiness_label_merge": "AND",
                 }
             }
             k8s.update_config(patch_readiness_label_config, "setting readiness label")
-            new_master_node, new_replica_nodes = self.assert_failover(
-                current_master_node, num_replicas, failover_targets, cluster_label)
+            self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
+
+            # first replica will be replaced and get the new affinity
+            # however, it might not start due to a volume node affinity conflict
+            # in this case only if the pvc and pod are deleted it can be scheduled
+            replica = k8s.get_cluster_replica_pod()
+            if replica.status.phase == 'Pending':
+                k8s.api.core_v1.delete_namespaced_persistent_volume_claim('pgdata-' + replica.metadata.name, 'default')
+                k8s.api.core_v1.delete_namespaced_pod(replica.metadata.name, 'default')
+                k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label)
+
+            # next master will be switched over and pod needs to be replaced as well to finish the rolling update
+            k8s.wait_for_pod_failover(replica_nodes, 'spilo-role=master,' + cluster_label)
+            k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label)
 
             # patch also node where master ran before
-            k8s.api.core_v1.patch_node(current_master_node, patch_readiness_label)
-
-            # toggle pod anti affinity to move replica away from master node
-            self.eventuallyTrue(lambda: self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label), "Pods are redistributed")
+            k8s.api.core_v1.patch_node(master_nodes[0], patch_readiness_label)
 
         except timeout_decorator.TimeoutError:
             print('Operator log: {}'.format(k8s.get_operator_log()))
             raise
 
+        # toggle pod anti affinity to move replica away from master node
+        self.assert_distributed_pods(master_nodes)
+
+
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
     def test_overwrite_pooler_deployment(self):
         k8s = self.k8s
@@ -1309,7 +1327,7 @@ class EndToEndTestCase(unittest.TestCase):
             patch_resync_config = {
                 "data": {
                     "pod_label_wait_timeout": "10m",
-                    "resync_period": "30m",
+                    "resync_period": "4m",
                 }
             }
             k8s.update_config(patch_resync_config, "revert resync interval and pod_label_wait_timeout")
@@ -1413,7 +1431,6 @@ class EndToEndTestCase(unittest.TestCase):
         self.eventuallyTrue(lambda: k8s.check_statefulset_annotations(cluster_label, annotations), "Annotations missing")
 
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
-    @unittest.skip("Skipping this test until fixed")
     def test_taint_based_eviction(self):
         '''
            Add taint "postgres=:NoExecute" to node with master. This must cause a failover.
@@ -1427,7 +1444,6 @@ class EndToEndTestCase(unittest.TestCase):
 
         # get nodes of master and replica(s) (expected target of new master)
         master_nodes, replica_nodes = k8s.get_cluster_nodes()
-
         self.assertNotEqual(master_nodes, [])
         self.assertNotEqual(replica_nodes, [])
 
@@ -1442,10 +1458,7 @@ class EndToEndTestCase(unittest.TestCase):
                 ]
             }
         }
-
         k8s.api.core_v1.patch_node(master_nodes[0], body)
-        self.eventuallyTrue(lambda: k8s.get_cluster_nodes()[0], replica_nodes)
-        self.assertNotEqual(lambda: k8s.get_cluster_nodes()[0], master_nodes)
 
         # add toleration to pods
         patch_toleration_config = {
@@ -1454,15 +1467,20 @@ class EndToEndTestCase(unittest.TestCase):
             }
         }
 
-        k8s.update_config(patch_toleration_config, step="allow tainted nodes")
+        try:
+            k8s.update_config(patch_toleration_config, step="allow tainted nodes")
+            self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"},
+                        "Operator does not get in sync")
 
-        self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No 2 pods running")
-        self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members("acid-minimal-cluster-0")), 2, "Postgres status did not enter running")
+            self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No 2 pods running")
+            self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members("acid-minimal-cluster-0")), 2, "Postgres status did not enter running")
+
+        except timeout_decorator.TimeoutError:
+            print('Operator log: {}'.format(k8s.get_operator_log()))
+            raise
 
         # toggle pod anti affinity to move replica away from master node
-        nm, new_replica_nodes = k8s.get_cluster_nodes()
-        new_master_node = nm[0]
-        self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label)
+        self.assert_distributed_pods(master_nodes)
 
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
     def test_zz_cluster_deletion(self):
@@ -1549,39 +1567,6 @@ class EndToEndTestCase(unittest.TestCase):
         }
         k8s.update_config(patch_delete_annotations)
 
-    def get_failover_targets(self, master_node, replica_nodes):
-        '''
-           If all pods live on the same node, failover will happen to other worker(s)
-        '''
-        k8s = self.k8s
-        k8s_master_exclusion = 'kubernetes.io/hostname!=postgres-operator-e2e-tests-control-plane'
-
-        failover_targets = [x for x in replica_nodes if x != master_node]
-        if len(failover_targets) == 0:
-            nodes = k8s.api.core_v1.list_node(label_selector=k8s_master_exclusion)
-            for n in nodes.items:
-                if n.metadata.name != master_node:
-                    failover_targets.append(n.metadata.name)
-
-        return failover_targets
-
-    def assert_failover(self, current_master_node, num_replicas, failover_targets, cluster_label):
-        '''
-           Check if master is failing over. The replica should move first to be the switchover target
-        '''
-        k8s = self.k8s
-        k8s.wait_for_pod_failover(failover_targets, 'spilo-role=master,' + cluster_label)
-        k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label)
-
-        new_master_node, new_replica_nodes = k8s.get_pg_nodes(cluster_label)
-        self.assertNotEqual(current_master_node, new_master_node,
-                            "Master on {} did not fail over to one of {}".format(current_master_node, failover_targets))
-        self.assertEqual(num_replicas, len(new_replica_nodes),
-                         "Expected {} replicas, found {}".format(num_replicas, len(new_replica_nodes)))
-        self.assert_master_is_unique()
-
-        return new_master_node, new_replica_nodes
-
     def assert_master_is_unique(self, namespace='default', clusterName="acid-minimal-cluster"):
         '''
            Check that there is a single pod in the k8s cluster with the label "spilo-role=master"
@@ -1593,14 +1578,23 @@ class EndToEndTestCase(unittest.TestCase):
         num_of_master_pods = k8s.count_pods_with_label(labels, namespace)
         self.assertEqual(num_of_master_pods, 1, "Expected 1 master pod, found {}".format(num_of_master_pods))
 
-    def assert_distributed_pods(self, master_node, replica_nodes, cluster_label):
+    @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
+    def assert_distributed_pods(self, target_nodes, cluster_labels='cluster-name=acid-minimal-cluster'):
         '''
            Other tests can lead to the situation that master and replica are on the same node.
            Toggle pod anti affinty to distribute pods accross nodes (replica in particular).
         '''
         k8s = self.k8s
-        cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster'
-        failover_targets = self.get_failover_targets(master_node, replica_nodes)
+        cluster_labels = 'application=spilo,cluster-name=acid-minimal-cluster'
+
+        # get nodes of master and replica(s)
+        master_nodes, replica_nodes = k8s.get_cluster_nodes()
+        self.assertNotEqual(master_nodes, [])
+        self.assertNotEqual(replica_nodes, [])
+
+        # if nodes are different we can quit here
+        if master_nodes[0] not in replica_nodes:
+            return True             
 
         # enable pod anti affintiy in config map which should trigger movement of replica
         patch_enable_antiaffinity = {
@@ -1608,18 +1602,40 @@ class EndToEndTestCase(unittest.TestCase):
                 "enable_pod_antiaffinity": "true"
             }
         }
-        k8s.update_config(patch_enable_antiaffinity, "enable antiaffinity")
-        self.assert_failover(master_node, len(replica_nodes), failover_targets, cluster_label)
 
-        # now disable pod anti affintiy again which will cause yet another failover
-        patch_disable_antiaffinity = {
-            "data": {
-                "enable_pod_antiaffinity": "false"
+        try:
+            k8s.update_config(patch_enable_antiaffinity, "enable antiaffinity")
+            self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
+
+            k8s.wait_for_pod_start('spilo-role=replica,' + cluster_labels)
+            k8s.wait_for_running_pods(cluster_labels, 2)
+
+            # now disable pod anti affintiy again which will cause yet another failover
+            patch_disable_antiaffinity = {
+                "data": {
+                    "enable_pod_antiaffinity": "false"
+                }
             }
-        }
-        k8s.update_config(patch_disable_antiaffinity, "disable antiaffinity")
-        k8s.wait_for_pod_start('spilo-role=master,' + cluster_label)
-        k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label)
+            k8s.update_config(patch_disable_antiaffinity, "disable antiaffinity")
+            self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
+            
+            k8s.wait_for_pod_start('spilo-role=replica,' + cluster_labels)
+            k8s.wait_for_running_pods(cluster_labels, 2)
+
+            master_nodes, replica_nodes = k8s.get_cluster_nodes()
+            self.assertNotEqual(master_nodes, [])
+            self.assertNotEqual(replica_nodes, [])
+
+            # if nodes are different we can quit here
+            for target_node in target_nodes:
+                if (target_node not in master_nodes or target_node not in replica_nodes) and master_nodes[0] in replica_nodes:
+                    print('Pods run on the same node') 
+                    return False
+
+        except timeout_decorator.TimeoutError:
+            print('Operator log: {}'.format(k8s.get_operator_log()))
+            raise
+
         return True
 
     def list_databases(self, pod_name):
diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml
index 932bd60ca..3ca1ec1b7 100644
--- a/manifests/configmap.yaml
+++ b/manifests/configmap.yaml
@@ -86,7 +86,8 @@ data:
   # min_cpu_limit: 250m
   # min_memory_limit: 250Mi
   # minimal_major_version: "9.6"
-  # node_readiness_label: ""
+  # node_readiness_label: "status:ready"
+  # node_readiness_label_merge: "OR"
   # oauth_token_secret_name: postgresql-operator
   # pam_configuration: |
   #  https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees
diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml
index d4b1a2996..0d9c73ca0 100644
--- a/manifests/operatorconfiguration.crd.yaml
+++ b/manifests/operatorconfiguration.crd.yaml
@@ -228,6 +228,11 @@ spec:
                     type: object
                     additionalProperties:
                       type: string
+                  node_readiness_label_merge:
+                    type: string
+                    enum:
+                      - "AND"
+                      - "OR"
                   oauth_token_secret_name:
                     type: string
                     default: "postgresql-operator"
diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml
index 2ad74b1e4..363f40750 100644
--- a/manifests/postgresql-operator-default-configuration.yaml
+++ b/manifests/postgresql-operator-default-configuration.yaml
@@ -70,6 +70,7 @@ configuration:
     master_pod_move_timeout: 20m
     # node_readiness_label:
     #   status: ready
+    # node_readiness_label_merge: "OR"
     oauth_token_secret_name: postgresql-operator
     pdb_name_format: "postgres-{cluster}-pdb"
     pod_antiaffinity_topology_key: "kubernetes.io/hostname"
diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go
index 11187ad75..c1e2da6fb 100644
--- a/pkg/apis/acid.zalan.do/v1/crds.go
+++ b/pkg/apis/acid.zalan.do/v1/crds.go
@@ -1167,6 +1167,17 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{
 									},
 								},
 							},
+							"node_readiness_label_merge": {
+								Type: "string",
+								Enum: []apiextv1.JSON{
+									{
+										Raw: []byte(`"AND"`),
+									},
+									{
+										Raw: []byte(`"OR"`),
+									},
+								},
+							},
 							"oauth_token_secret_name": {
 								Type: "string",
 							},
diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
index a1dee6bff..00255a125 100644
--- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
+++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
@@ -82,6 +82,7 @@ type KubernetesMetaConfiguration struct {
 	DeleteAnnotationDateKey                string                       `json:"delete_annotation_date_key,omitempty"`
 	DeleteAnnotationNameKey                string                       `json:"delete_annotation_name_key,omitempty"`
 	NodeReadinessLabel                     map[string]string            `json:"node_readiness_label,omitempty"`
+	NodeReadinessLabelMerge                string                       `json:"node_readiness_label_merge,omitempty"`
 	CustomPodAnnotations                   map[string]string            `json:"custom_pod_annotations,omitempty"`
 	// TODO: use a proper toleration structure?
 	PodToleration              map[string]string   `json:"toleration,omitempty"`
diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go
index 967f9d530..ca58c10a0 100644
--- a/pkg/cluster/cluster.go
+++ b/pkg/cluster/cluster.go
@@ -375,7 +375,6 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa
 		reasons = append(reasons, "new statefulset's number of replicas does not match the current one")
 	}
 	if !reflect.DeepEqual(c.Statefulset.Annotations, statefulSet.Annotations) {
-		match = false
 		needsReplace = true
 		reasons = append(reasons, "new statefulset's annotations do not match the current one")
 	}
@@ -406,6 +405,11 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa
 		needsRollUpdate = true
 		reasons = append(reasons, "new statefulset's pod affinity does not match the current one")
 	}
+	if len(c.Statefulset.Spec.Template.Spec.Tolerations) != len(statefulSet.Spec.Template.Spec.Tolerations) {
+		needsReplace = true
+		needsRollUpdate = true
+		reasons = append(reasons, "new statefulset's pod tolerations does not match the current one")
+	}
 
 	// Some generated fields like creationTimestamp make it not possible to use DeepCompare on Spec.Template.ObjectMeta
 	if !reflect.DeepEqual(c.Statefulset.Spec.Template.Labels, statefulSet.Spec.Template.Labels) {
@@ -427,13 +431,11 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa
 	}
 
 	if !reflect.DeepEqual(c.Statefulset.Spec.Template.Annotations, statefulSet.Spec.Template.Annotations) {
-		match = false
 		needsReplace = true
 		needsRollUpdate = true
 		reasons = append(reasons, "new statefulset's pod template metadata annotations does not match the current one")
 	}
 	if !reflect.DeepEqual(c.Statefulset.Spec.Template.Spec.SecurityContext, statefulSet.Spec.Template.Spec.SecurityContext) {
-		match = false
 		needsReplace = true
 		needsRollUpdate = true
 		reasons = append(reasons, "new statefulset's pod template security context in spec does not match the current one")
@@ -469,7 +471,6 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa
 	// we assume any change in priority happens by rolling out a new priority class
 	// changing the priority value in an existing class is not supproted
 	if c.Statefulset.Spec.Template.Spec.PriorityClassName != statefulSet.Spec.Template.Spec.PriorityClassName {
-		match = false
 		needsReplace = true
 		needsRollUpdate = true
 		reasons = append(reasons, "new statefulset's pod priority class in spec does not match the current one")
diff --git a/pkg/cluster/connection_pooler.go b/pkg/cluster/connection_pooler.go
index c5c55350f..ec9fe291d 100644
--- a/pkg/cluster/connection_pooler.go
+++ b/pkg/cluster/connection_pooler.go
@@ -309,7 +309,7 @@ func (c *Cluster) generateConnectionPoolerPodTemplate(role PostgresRole) (
 		},
 	}
 
-	nodeAffinity := nodeAffinity(c.OpConfig.NodeReadinessLabel, spec.NodeAffinity)
+	nodeAffinity := c.nodeAffinity(c.OpConfig.NodeReadinessLabel, spec.NodeAffinity)
 	if c.OpConfig.EnablePodAntiAffinity {
 		labelsSet := labels.Set(c.connectionPoolerLabels(role, false).MatchLabels)
 		podTemplate.Spec.Affinity = generatePodAffinity(labelsSet, c.OpConfig.PodAntiAffinityTopologyKey, nodeAffinity)
diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go
index e7d8ea376..fda192df8 100644
--- a/pkg/cluster/k8sres.go
+++ b/pkg/cluster/k8sres.go
@@ -327,7 +327,7 @@ func generateCapabilities(capabilities []string) *v1.Capabilities {
 	return nil
 }
 
-func nodeAffinity(nodeReadinessLabel map[string]string, nodeAffinity *v1.NodeAffinity) *v1.Affinity {
+func (c *Cluster) nodeAffinity(nodeReadinessLabel map[string]string, nodeAffinity *v1.NodeAffinity) *v1.Affinity {
 	if len(nodeReadinessLabel) == 0 && nodeAffinity == nil {
 		return nil
 	}
@@ -352,8 +352,18 @@ func nodeAffinity(nodeReadinessLabel map[string]string, nodeAffinity *v1.NodeAff
 				},
 			}
 		} else {
-			nodeAffinityCopy.RequiredDuringSchedulingIgnoredDuringExecution = &v1.NodeSelector{
-				NodeSelectorTerms: append(nodeAffinityCopy.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, nodeReadinessSelectorTerm),
+			if c.OpConfig.NodeReadinessLabelMerge == "OR" {
+				manifestTerms := nodeAffinityCopy.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms
+				manifestTerms = append(manifestTerms, nodeReadinessSelectorTerm)
+				nodeAffinityCopy.RequiredDuringSchedulingIgnoredDuringExecution = &v1.NodeSelector{
+					NodeSelectorTerms: manifestTerms,
+				}
+			} else if c.OpConfig.NodeReadinessLabelMerge == "AND" {
+				for i, nodeSelectorTerm := range nodeAffinityCopy.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms {
+					manifestExpressions := nodeSelectorTerm.MatchExpressions
+					manifestExpressions = append(manifestExpressions, matchExpressions...)
+					nodeAffinityCopy.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[i] = v1.NodeSelectorTerm{MatchExpressions: manifestExpressions}
+				}
 			}
 		}
 	}
@@ -1260,7 +1270,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
 		effectiveRunAsUser,
 		effectiveRunAsGroup,
 		effectiveFSGroup,
-		nodeAffinity(c.OpConfig.NodeReadinessLabel, spec.NodeAffinity),
+		c.nodeAffinity(c.OpConfig.NodeReadinessLabel, spec.NodeAffinity),
 		spec.SchedulerName,
 		int64(c.OpConfig.PodTerminateGracePeriod.Seconds()),
 		c.OpConfig.PodServiceAccountName,
@@ -2010,7 +2020,7 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) {
 		nil,
 		nil,
 		nil,
-		nodeAffinity(c.OpConfig.NodeReadinessLabel, nil),
+		c.nodeAffinity(c.OpConfig.NodeReadinessLabel, nil),
 		nil,
 		int64(c.OpConfig.PodTerminateGracePeriod.Seconds()),
 		c.OpConfig.PodServiceAccountName,
diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go
index 2f5261cd2..1cb36e746 100644
--- a/pkg/controller/operator_config.go
+++ b/pkg/controller/operator_config.go
@@ -110,6 +110,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	result.DeleteAnnotationDateKey = fromCRD.Kubernetes.DeleteAnnotationDateKey
 	result.DeleteAnnotationNameKey = fromCRD.Kubernetes.DeleteAnnotationNameKey
 	result.NodeReadinessLabel = fromCRD.Kubernetes.NodeReadinessLabel
+	result.NodeReadinessLabelMerge = fromCRD.Kubernetes.NodeReadinessLabelMerge
 	result.PodPriorityClassName = fromCRD.Kubernetes.PodPriorityClassName
 	result.PodManagementPolicy = util.Coalesce(fromCRD.Kubernetes.PodManagementPolicy, "ordered_ready")
 	result.MasterPodMoveTimeout = util.CoalesceDuration(time.Duration(fromCRD.Kubernetes.MasterPodMoveTimeout), "10m")
diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go
index bb77e6231..ff710640e 100644
--- a/pkg/util/config/config.go
+++ b/pkg/util/config/config.go
@@ -55,6 +55,7 @@ type Resources struct {
 	PodEnvironmentConfigMap       spec.NamespacedName `name:"pod_environment_configmap"`
 	PodEnvironmentSecret          string              `name:"pod_environment_secret"`
 	NodeReadinessLabel            map[string]string   `name:"node_readiness_label" default:""`
+	NodeReadinessLabelMerge       string              `name:"node_readiness_label_merge" default:"OR"`
 	MaxInstances                  int32               `name:"max_instances" default:"-1"`
 	MinInstances                  int32               `name:"min_instances" default:"-1"`
 	ShmVolume                     *bool               `name:"enable_shm_volume" default:"true"`

From 95301c102eed430aa183bde5add0e7ddf6ef4696 Mon Sep 17 00:00:00 2001
From: jopadi <45459238+jopadi@users.noreply.github.com>
Date: Tue, 8 Feb 2022 17:08:26 +0100
Subject: [PATCH 18/58] Update codeowners and maintainers (#1773)

* change the code owners and maintainers

Co-authored-by: Jociele Padilha <jociele.padilha@zalando.de>
---
 CODEOWNERS  | 2 +-
 MAINTAINERS | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/CODEOWNERS b/CODEOWNERS
index 398856c66..6d1a2ab6a 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -1,2 +1,2 @@
 # global owners
-*        @erthalion @sdudoladov @Jan-M @CyberDem0n @avaczi @FxKu @RafiaSabih
+*        @sdudoladov @Jan-M @CyberDem0n @FxKu @jopadi
diff --git a/MAINTAINERS b/MAINTAINERS
index 572e6d971..b2abdf858 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1,5 +1,4 @@
-Dmitrii Dolgov <dmitrii.dolgov@zalando.de>
 Sergey Dudoladov <sergey.dudoladov@zalando.de>
 Felix Kunde <felix.kunde@zalando.de>
 Jan Mussler <jan.mussler@zalando.de>
-Rafia Sabih <rafia.sabih@zalando.de>
\ No newline at end of file
+Jociele Padilha <jociele.padilha@zalando.de>
\ No newline at end of file

From 658923d10d5c37e82eb9c55b735c598ebf5dae8d Mon Sep 17 00:00:00 2001
From: Felix Kunde <felix-kunde@gmx.de>
Date: Fri, 18 Feb 2022 11:54:47 +0100
Subject: [PATCH 19/58] Password rotation in secrets (#1749)

* password rotation in K8s secrets
* add db connection to syncSecrets
* add user retention
* add e2e test
* cleanup on username mismatch if rotation was switched off
* add unit test for syncSecrets + new updateSecret func
---
 .../crds/operatorconfigurations.yaml          |   9 +
 .../postgres-operator/crds/postgresqls.yaml   |  10 +
 docs/administrator.md                         |  78 +++++++
 docs/reference/cluster_manifest.md            |  16 ++
 docs/reference/operator_parameters.md         |  22 ++
 e2e/tests/k8s_api.py                          |   3 +
 e2e/tests/test_e2e.py                         | 122 ++++++++++-
 manifests/complete-postgres-manifest.yaml     |   3 +
 manifests/configmap.yaml                      |   3 +
 manifests/operatorconfiguration.crd.yaml      |   9 +
 ...gresql-operator-default-configuration.yaml |   3 +
 manifests/postgresql.crd.yaml                 |  10 +
 pkg/apis/acid.zalan.do/v1/crds.go             |  18 ++
 .../v1/operator_configuration_type.go         |   7 +-
 pkg/apis/acid.zalan.do/v1/postgresql_type.go  |   5 +-
 .../acid.zalan.do/v1/zz_generated.deepcopy.go |  10 +
 pkg/cluster/cluster.go                        |  25 ++-
 pkg/cluster/database.go                       |  82 ++++++-
 pkg/cluster/sync.go                           | 202 ++++++++++++++----
 pkg/cluster/sync_test.go                      |  81 ++++++-
 pkg/controller/operator_config.go             |   3 +
 pkg/spec/types.go                             |   1 +
 pkg/util/config/config.go                     |   3 +
 pkg/util/users/users.go                       |  11 +
 24 files changed, 674 insertions(+), 62 deletions(-)

diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml
index 3cc08b4df..554318a97 100644
--- a/charts/postgres-operator/crds/operatorconfigurations.yaml
+++ b/charts/postgres-operator/crds/operatorconfigurations.yaml
@@ -122,6 +122,15 @@ spec:
               users:
                 type: object
                 properties:
+                  enable_password_rotation:
+                    type: boolean
+                    default: false
+                  password_rotation_interval:
+                    type: integer
+                    default: 90
+                  password_rotation_user_retention:
+                    type: integer
+                    default: 180
                   replication_username:
                      type: string
                      default: standby
diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml
index d6e1dd94f..3c03e4f3c 100644
--- a/charts/postgres-operator/crds/postgresqls.yaml
+++ b/charts/postgres-operator/crds/postgresqls.yaml
@@ -551,6 +551,16 @@ spec:
                     - SUPERUSER
                     - nosuperuser
                     - NOSUPERUSER
+              usersWithPasswordRotation:
+                type: array
+                nullable: true
+                items:
+                  type: string
+              usersWithInPlacePasswordRotation:
+                type: array
+                nullable: true
+                items:
+                  type: string
               volume:
                 type: object
                 required:
diff --git a/docs/administrator.md b/docs/administrator.md
index 879b677b9..d7ab8ef0c 100644
--- a/docs/administrator.md
+++ b/docs/administrator.md
@@ -293,6 +293,84 @@ that are aggregated into the K8s [default roles](https://kubernetes.io/docs/refe
 
 For Helm deployments setting `rbac.createAggregateClusterRoles: true` adds these clusterroles to the deployment.
 
+## Password rotation in K8s secrets
+
+The operator regularly updates credentials in the K8s secrets if the
+`enable_password_rotation` option is set to `true` in the configuration.
+It happens only for `LOGIN` roles with an associated secret (manifest roles,
+default users from `preparedDatabases`). Furthermore, there are the following
+exceptions:
+
+1. Infrastructure role secrets since rotation should happen by the infrastructure.
+2. Team API roles that connect via OAuth2 and JWT token (no secrets to these roles anyway).
+3. Database owners since ownership on database objects can not be inherited.
+4. System users such as `postgres`, `standby` and `pooler` user.
+
+The interval of days can be set with `password_rotation_interval` (default
+`90` = 90 days, minimum 1). On each rotation the user name and password values
+are replaced in the K8s secret. They belong to a newly created user named after
+the original role plus rotation date in YYMMDD format. All priviliges are
+inherited meaning that migration scripts should still grant and revoke rights
+against the original role. The timestamp of the next rotation is written to the
+secret as well. Note, if the rotation interval is decreased it is reflected in
+the secrets only if the next rotation date is more days away than the new
+length of the interval.
+
+Pods still using the previous secret values which they keep in memory continue
+to connect to the database since the password of the corresponding user is not
+replaced. However, a retention policy can be configured for users created by
+the password rotation feature with `password_rotation_user_retention`. The
+operator will ensure that this period is at least twice as long as the
+configured rotation interval, hence the default of `180` = 180 days. When
+the creation date of a rotated user is older than the retention period it
+might not get removed immediately. Only on the next user rotation it is checked
+if users can get removed. Therefore, you might want to configure the retention
+to be a multiple of the rotation interval.
+
+### Password rotation for single users
+
+From the configuration, password rotation is enabled for all secrets with the
+mentioned exceptions. If you wish to first test rotation for a single user (or
+just have it enabled only for a few secrets) you can specify it in the cluster
+manifest. The rotation and retention intervals can only be configured globally.
+
+```
+spec:
+  usersWithSecretRotation:
+  - foo_user
+  - bar_reader_user
+```
+
+### Password replacement without extra users
+
+For some use cases where the secret is only used rarely - think of a `flyway`
+user running a migration script on pod start - we do not need to create extra
+database users but can replace only the password in the K8s secret. This type
+of rotation cannot be configured globally but specified in the cluster
+manifest:
+
+```
+spec:
+  usersWithInPlaceSecretRotation:
+  - flyway
+  - bar_owner_user
+```
+
+This would be the recommended option to enable rotation in secrets of database
+owners, but only if they are not used as application users for regular read
+and write operations.
+
+### Turning off password rotation
+
+When password rotation is turned off again the operator will check if the
+`username` value in the secret matches the original username and replace it
+with the latter. A new password is assigned and the `nextRotation` field is
+cleared. A final lookup for child (rotation) users to be removed is done but
+they will only be dropped if the retention policy allows for it. This is to
+avoid sudden connection issues in pods which still use credentials of these
+users in memory. You have to remove these child users manually or re-enable
+password rotation with smaller interval so they get cleaned up.
+
 ## Use taints and tolerations for dedicated PostgreSQL nodes
 
 To ensure Postgres pods are running on nodes without any other application pods,
diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md
index 3b6393550..9d65062c0 100644
--- a/docs/reference/cluster_manifest.md
+++ b/docs/reference/cluster_manifest.md
@@ -115,6 +115,22 @@ These parameters are grouped directly under  the `spec` key in the manifest.
   create the K8s secret in that namespace. The part after the first `.` is
   considered to be the user name. Optional.
 
+* **usersWithSecretRotation**
+  list of users to enable credential rotation in K8s secrets. The rotation
+  interval can only be configured globally. On each rotation a new user will
+  be added in the database replacing the `username` value in the secret of
+  the listed user. Although, rotation users inherit all rights from the
+  original role, keep in mind that ownership is not transferred. See more
+  details in the [administrator docs](https://github.com/zalando/postgres-operator/blob/master/docs/administrator.md#password-rotation-in-k8s-secrets).
+
+* **usersWithInPlaceSecretRotation**
+  list of users to enable in-place password rotation in K8s secrets. The
+  rotation interval can only be configured globally. On each rotation the
+  password value will be replaced in the secrets which the operator reflects
+  in the database, too. List only users here that rarely connect to the
+  database, like a flyway user running a migration on Pod start. See more
+  details in the [administrator docs](https://github.com/zalando/postgres-operator/blob/master/docs/administrator.md#password-replacement-without-extra-users).
+
 * **databases**
   a map of database names to database owners for the databases that should be
   created by the operator. The owner users should already exist on the cluster
diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md
index f6b94e0a5..990909eb5 100644
--- a/docs/reference/operator_parameters.md
+++ b/docs/reference/operator_parameters.md
@@ -174,6 +174,28 @@ under the `users` key.
   Postgres username used for replication between instances. The default is
   `standby`.
 
+* **enable_password_rotation**
+  For all `LOGIN` roles that are not database owners the operator can rotate
+  credentials in the corresponding K8s secrets by replacing the username and
+  password. This means, new users will be added on each rotation inheriting
+  all priviliges from the original roles. The rotation date (in YYMMDD format)
+  is appended to the names of the new user. The timestamp of the next rotation
+  is written to the secret. The default is `false`.
+
+* **password_rotation_interval**
+  If password rotation is enabled (either from config or cluster manifest) the
+  interval can be configured with this parameter. The measure is in days which
+  means daily rotation (`1`) is the most frequent interval possible.
+  Default is `90`.
+
+* **password_rotation_user_retention**
+  To avoid an ever growing amount of new users due to password rotation the
+  operator will remove the created users again after a certain amount of days
+  has passed. The number can be configured with this parameter. However, the
+  operator will check that the retention policy is at least twice as long as
+  the rotation interval and update to this minimum in case it is not.
+  Default is `180`.
+
 ## Major version upgrades
 
 Parameters configuring automatic major version upgrades. In a
diff --git a/e2e/tests/k8s_api.py b/e2e/tests/k8s_api.py
index f58e16b50..4ae6493c5 100644
--- a/e2e/tests/k8s_api.py
+++ b/e2e/tests/k8s_api.py
@@ -321,6 +321,9 @@ class K8s:
     def get_cluster_replica_pod(self, labels='application=spilo,cluster-name=acid-minimal-cluster', namespace='default'):
         return self.get_cluster_pod('replica', labels, namespace)
 
+    def get_secret_data(self, username, clustername='acid-minimal-cluster', namespace='default'):
+        return self.api.core_v1.read_namespaced_secret(
+                "{}.{}.credentials.postgresql.acid.zalan.do".format(username.replace("_","-"), clustername), namespace).data
 
 class K8sBase:
     '''
diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py
index 3047de259..7b3711b73 100644
--- a/e2e/tests/test_e2e.py
+++ b/e2e/tests/test_e2e.py
@@ -4,8 +4,9 @@ import time
 import timeout_decorator
 import os
 import yaml
+import base64
 
-from datetime import datetime
+from datetime import datetime, date, timedelta
 from kubernetes import client
 
 from tests.k8s_api import K8s
@@ -579,6 +580,7 @@ class EndToEndTestCase(unittest.TestCase):
                             "Parameters": None,
                             "AdminRole": "",
                             "Origin": 2,
+                            "IsDbOwner": False,
                             "Deleted": False
                         })
                         return True
@@ -600,7 +602,6 @@ class EndToEndTestCase(unittest.TestCase):
         but lets pods run with the old image until they are recreated for
         reasons other than operator's activity. That works because the operator
         configures stateful sets to use "onDelete" pod update policy.
-
         The test covers:
         1) enabling lazy upgrade in existing operator deployment
         2) forcing the normal rolling upgrade by changing the operator
@@ -695,7 +696,6 @@ class EndToEndTestCase(unittest.TestCase):
         Ensure we can (a) create the cron job at user request for a specific PG cluster
                       (b) update the cluster-wide image for the logical backup pod
                       (c) delete the job at user request
-
         Limitations:
         (a) Does not run the actual batch job because there is no S3 mock to upload backups to
         (b) Assumes 'acid-minimal-cluster' exists as defined in setUp
@@ -1074,6 +1074,122 @@ class EndToEndTestCase(unittest.TestCase):
         self.eventuallyEqual(lambda: k8s.count_running_pods("connection-pooler=acid-minimal-cluster-pooler"),
                              0, "Pooler pods not scaled down")
 
+    @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
+    def test_password_rotation(self):
+        '''
+           Test password rotation and removal of users due to retention policy
+        '''
+        k8s = self.k8s
+        leader = k8s.get_cluster_leader_pod()
+        today = date.today()
+
+        # enable password rotation for owner of foo database
+        pg_patch_inplace_rotation_for_owner = {
+            "spec": {
+                "usersWithInPlaceSecretRotation": [
+                    "zalando"
+                ]
+            }
+        }
+        k8s.api.custom_objects_api.patch_namespaced_custom_object(
+            "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_inplace_rotation_for_owner)
+        self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
+
+        # check if next rotation date was set in secret
+        secret_data = k8s.get_secret_data("zalando")
+        next_rotation_timestamp = datetime.fromisoformat(str(base64.b64decode(secret_data["nextRotation"]), 'utf-8'))
+        today90days = today+timedelta(days=90)
+        self.assertEqual(today90days, next_rotation_timestamp.date(),
+                        "Unexpected rotation date in secret of zalando user: expected {}, got {}".format(today90days, next_rotation_timestamp.date()))
+
+        # create fake rotation users that should be removed by operator
+        # but have one that would still fit into the retention period
+        create_fake_rotation_user = """
+            CREATE ROLE foo_user201031 IN ROLE foo_user;
+            CREATE ROLE foo_user211031 IN ROLE foo_user;
+            CREATE ROLE foo_user"""+(today-timedelta(days=40)).strftime("%y%m%d")+""" IN ROLE foo_user;
+        """
+        self.query_database(leader.metadata.name, "postgres", create_fake_rotation_user)
+
+        # patch foo_user secret with outdated rotation date
+        fake_rotation_date = today.isoformat() + ' 00:00:00'
+        fake_rotation_date_encoded = base64.b64encode(fake_rotation_date.encode('utf-8'))
+        secret_fake_rotation = {
+            "data": {
+                "nextRotation": str(fake_rotation_date_encoded, 'utf-8'),
+            },
+        }
+        k8s.api.core_v1.patch_namespaced_secret(
+            name="foo-user.acid-minimal-cluster.credentials.postgresql.acid.zalan.do", 
+            namespace="default",
+            body=secret_fake_rotation)
+
+        # enable password rotation for all other users (foo_user)
+        # this will force a sync of secrets for further assertions
+        enable_password_rotation = {
+            "data": {
+                "enable_password_rotation": "true",
+                "password_rotation_interval": "30",
+                "password_rotation_user_retention": "30",  # should be set to 60 
+            },
+        }
+        k8s.update_config(enable_password_rotation)
+        self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"},
+                             "Operator does not get in sync")
+
+        # check if next rotation date and username have been replaced
+        secret_data = k8s.get_secret_data("foo_user")
+        secret_username = str(base64.b64decode(secret_data["username"]), 'utf-8')
+        next_rotation_timestamp = datetime.fromisoformat(str(base64.b64decode(secret_data["nextRotation"]), 'utf-8'))
+        rotation_user = "foo_user"+today.strftime("%y%m%d")
+        today30days = today+timedelta(days=30)
+
+        self.assertEqual(rotation_user, secret_username,
+                        "Unexpected username in secret of foo_user: expected {}, got {}".format(rotation_user, secret_username))
+        self.assertEqual(today30days, next_rotation_timestamp.date(),
+                        "Unexpected rotation date in secret of foo_user: expected {}, got {}".format(today30days, next_rotation_timestamp.date()))
+
+        # check if oldest fake rotation users were deleted
+        # there should only be foo_user, foo_user+today and foo_user+today-40days
+        user_query = """
+            SELECT rolname
+              FROM pg_catalog.pg_roles
+             WHERE rolname LIKE 'foo_user%';
+        """
+        self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "postgres", user_query)), 3,
+            "Found incorrect number of rotation users", 10, 5)
+
+        # disable password rotation for all other users (foo_user)
+        # and pick smaller intervals to see if the third fake rotation user is dropped 
+        enable_password_rotation = {
+            "data": {
+                "enable_password_rotation": "false",
+                "password_rotation_interval": "15",
+                "password_rotation_user_retention": "30",  # 2 * rotation interval
+            },
+        }
+        k8s.update_config(enable_password_rotation)
+        self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"},
+                             "Operator does not get in sync")
+
+        # check if username in foo_user secret is reset
+        secret_data = k8s.get_secret_data("foo_user")
+        secret_username = str(base64.b64decode(secret_data["username"]), 'utf-8')
+        next_rotation_timestamp = str(base64.b64decode(secret_data["nextRotation"]), 'utf-8')
+        self.assertEqual("foo_user", secret_username,
+                        "Unexpected username in secret of foo_user: expected {}, got {}".format("foo_user", secret_username))
+        self.assertEqual('', next_rotation_timestamp,
+                        "Unexpected rotation date in secret of foo_user: expected empty string, got {}".format(next_rotation_timestamp))
+
+        # check roles again, there should only be foo_user and foo_user+today
+        user_query = """
+            SELECT rolname
+              FROM pg_catalog.pg_roles
+             WHERE rolname LIKE 'foo_user%';
+        """
+        self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "postgres", user_query)), 2,
+            "Found incorrect number of rotation users", 10, 5)
+
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
     def test_patroni_config_update(self):
         '''
diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml
index 46e79ba21..c150b616d 100644
--- a/manifests/complete-postgres-manifest.yaml
+++ b/manifests/complete-postgres-manifest.yaml
@@ -16,6 +16,9 @@ spec:
     zalando:
     - superuser
     - createdb
+    foo_user: []
+#  usersWithSecretRotation: "foo_user"
+#  usersWithInPlaceSecretRotation: "flyway,bar_owner_user"
   enableMasterLoadBalancer: false
   enableReplicaLoadBalancer: false
   enableConnectionPooler: false # enable/disable connection pooler deployment
diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml
index 3ca1ec1b7..1a52a52da 100644
--- a/manifests/configmap.yaml
+++ b/manifests/configmap.yaml
@@ -44,6 +44,7 @@ data:
   # enable_init_containers: "true"
   # enable_lazy_spilo_upgrade: "false"
   enable_master_load_balancer: "false"
+  enable_password_rotation: "false"
   enable_pgversion_env_var: "true"
   # enable_pod_antiaffinity: "false"
   # enable_pod_disruption_budget: "true"
@@ -92,6 +93,8 @@ data:
   # pam_configuration: |
   #  https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees
   # pam_role_name: zalandos
+  # password_rotation_interval: "90"
+  # password_rotation_user_retention: "180"
   pdb_name_format: "postgres-{cluster}-pdb"
   # pod_antiaffinity_topology_key: "kubernetes.io/hostname"
   pod_deletion_wait_timeout: 10m
diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml
index 0d9c73ca0..3313377a0 100644
--- a/manifests/operatorconfiguration.crd.yaml
+++ b/manifests/operatorconfiguration.crd.yaml
@@ -120,6 +120,15 @@ spec:
               users:
                 type: object
                 properties:
+                  enable_password_rotation:
+                    type: boolean
+                    default: false
+                  password_rotation_interval:
+                    type: integer
+                    default: 90
+                  password_rotation_user_retention:
+                    type: integer
+                    default: 180
                   replication_username:
                      type: string
                      default: standby
diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml
index 363f40750..f8ccad7ce 100644
--- a/manifests/postgresql-operator-default-configuration.yaml
+++ b/manifests/postgresql-operator-default-configuration.yaml
@@ -25,6 +25,9 @@ configuration:
   #     protocol: TCP
   workers: 8
   users:
+    enable_password_rotation: false
+    password_rotation_interval: 90
+    password_rotation_user_retention: 180
     replication_username: standby
     super_username: postgres
   major_version_upgrade:
diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml
index d855b1812..dafbf9038 100644
--- a/manifests/postgresql.crd.yaml
+++ b/manifests/postgresql.crd.yaml
@@ -549,6 +549,16 @@ spec:
                     - SUPERUSER
                     - nosuperuser
                     - NOSUPERUSER
+              usersWithPasswordRotation:
+                type: array
+                nullable: true
+                items:
+                  type: string
+              usersWithInPlacePasswordRotation:
+                type: array
+                nullable: true
+                items:
+                  type: string
               volume:
                 type: object
                 required:
diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go
index c1e2da6fb..da18cd20d 100644
--- a/pkg/apis/acid.zalan.do/v1/crds.go
+++ b/pkg/apis/acid.zalan.do/v1/crds.go
@@ -833,6 +833,24 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 							},
 						},
 					},
+					"usersWithSecretRotation": {
+						Type:     "array",
+						Nullable: true,
+						Items: &apiextv1.JSONSchemaPropsOrArray{
+							Schema: &apiextv1.JSONSchemaProps{
+								Type: "string",
+							},
+						},
+					},
+					"usersWithInPlaceSecretRotation": {
+						Type:     "array",
+						Nullable: true,
+						Items: &apiextv1.JSONSchemaPropsOrArray{
+							Schema: &apiextv1.JSONSchemaProps{
+								Type: "string",
+							},
+						},
+					},
 					"volume": {
 						Type:     "object",
 						Required: []string{"size"},
diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
index 00255a125..124278e70 100644
--- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
+++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
@@ -37,8 +37,11 @@ type OperatorConfigurationList struct {
 
 // PostgresUsersConfiguration defines the system users of Postgres.
 type PostgresUsersConfiguration struct {
-	SuperUsername       string `json:"super_username,omitempty"`
-	ReplicationUsername string `json:"replication_username,omitempty"`
+	SuperUsername                 string `json:"super_username,omitempty"`
+	ReplicationUsername           string `json:"replication_username,omitempty"`
+	EnablePasswordRotation        bool   `json:"enable_password_rotation,omitempty"`
+	PasswordRotationInterval      uint32 `json:"password_rotation_interval,omitempty"`
+	PasswordRotationUserRetention uint32 `json:"password_rotation_user_retention,omitempty"`
 }
 
 // MajorVersionUpgradeConfiguration defines how to execute major version upgrades of Postgres.
diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go
index 57f9ae04a..45f139383 100644
--- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go
+++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go
@@ -53,8 +53,11 @@ type PostgresSpec struct {
 	// load balancers' source ranges are the same for master and replica services
 	AllowedSourceRanges []string `json:"allowedSourceRanges"`
 
+	Users                          map[string]UserFlags `json:"users,omitempty"`
+	UsersWithSecretRotation        []string             `json:"usersWithSecretRotation,omitempty"`
+	UsersWithInPlaceSecretRotation []string             `json:"usersWithInPlaceSecretRotation,omitempty"`
+
 	NumberOfInstances     int32                       `json:"numberOfInstances"`
-	Users                 map[string]UserFlags        `json:"users,omitempty"`
 	MaintenanceWindows    []MaintenanceWindow         `json:"maintenanceWindows,omitempty"`
 	Clone                 *CloneDescription           `json:"clone,omitempty"`
 	ClusterName           string                      `json:"-"`
diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
index e9ab46382..cfa315358 100644
--- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
+++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
@@ -641,6 +641,16 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) {
 			(*out)[key] = outVal
 		}
 	}
+	if in.UsersWithSecretRotation != nil {
+		in, out := &in.UsersWithSecretRotation, &out.UsersWithSecretRotation
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.UsersWithInPlaceSecretRotation != nil {
+		in, out := &in.UsersWithInPlaceSecretRotation, &out.UsersWithInPlaceSecretRotation
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	if in.MaintenanceWindows != nil {
 		in, out := &in.MaintenanceWindows, &out.MaintenanceWindows
 		*out = make([]MaintenanceWindow, len(*in))
diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go
index ca58c10a0..3e7f708a5 100644
--- a/pkg/cluster/cluster.go
+++ b/pkg/cluster/cluster.go
@@ -711,13 +711,18 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
 		}
 	}
 
-	// connection pooler needs one system user created, which is done in
-	// initUsers. Check if it needs to be called.
+	// check if users need to be synced
 	sameUsers := reflect.DeepEqual(oldSpec.Spec.Users, newSpec.Spec.Users) &&
 		reflect.DeepEqual(oldSpec.Spec.PreparedDatabases, newSpec.Spec.PreparedDatabases)
+	sameRotatedUsers := reflect.DeepEqual(oldSpec.Spec.UsersWithSecretRotation, newSpec.Spec.UsersWithSecretRotation) &&
+		reflect.DeepEqual(oldSpec.Spec.UsersWithInPlaceSecretRotation, newSpec.Spec.UsersWithInPlaceSecretRotation)
+
+	// connection pooler needs one system user created, which is done in
+	// initUsers. Check if it needs to be called.
 	needConnectionPooler := needMasterConnectionPoolerWorker(&newSpec.Spec) ||
 		needReplicaConnectionPoolerWorker(&newSpec.Spec)
-	if !sameUsers || needConnectionPooler {
+
+	if !sameUsers || !sameRotatedUsers || needConnectionPooler {
 		c.logger.Debugf("initialize users")
 		if err := c.initUsers(); err != nil {
 			c.logger.Errorf("could not init users: %v", err)
@@ -1001,6 +1006,7 @@ func (c *Cluster) initSystemUsers() {
 		Origin:    spec.RoleOriginSystem,
 		Name:      c.OpConfig.ReplicationUsername,
 		Namespace: c.Namespace,
+		Flags:     []string{constants.RoleFlagLogin},
 		Password:  util.RandomPassword(constants.PasswordLength),
 	}
 
@@ -1113,7 +1119,6 @@ func (c *Cluster) initPreparedDatabaseRoles() error {
 func (c *Cluster) initDefaultRoles(defaultRoles map[string]string, admin, prefix, searchPath, secretNamespace string) error {
 
 	for defaultRole, inherits := range defaultRoles {
-
 		namespace := c.Namespace
 		//if namespaced secrets are allowed
 		if secretNamespace != "" {
@@ -1136,8 +1141,10 @@ func (c *Cluster) initDefaultRoles(defaultRoles map[string]string, admin, prefix
 		}
 
 		adminRole := ""
+		isOwner := false
 		if strings.Contains(defaultRole, constants.OwnerRoleNameSuffix) {
 			adminRole = admin
+			isOwner = true
 		} else {
 			adminRole = prefix + constants.OwnerRoleNameSuffix
 		}
@@ -1151,6 +1158,7 @@ func (c *Cluster) initDefaultRoles(defaultRoles map[string]string, admin, prefix
 			MemberOf:   memberOf,
 			Parameters: map[string]string{"search_path": searchPath},
 			AdminRole:  adminRole,
+			IsDbOwner:  isOwner,
 		}
 		if currentRole, present := c.pgUsers[roleName]; present {
 			c.pgUsers[roleName] = c.resolveNameConflict(&currentRole, &newRole)
@@ -1172,6 +1180,14 @@ func (c *Cluster) initRobotUsers() error {
 		}
 		namespace := c.Namespace
 
+		// check if role is specified as database owner
+		isOwner := false
+		for _, owner := range c.Spec.Databases {
+			if username == owner {
+				isOwner = true
+			}
+		}
+
 		//if namespaced secrets are allowed
 		if c.Config.OpConfig.EnableCrossNamespaceSecret {
 			if strings.Contains(username, ".") {
@@ -1196,6 +1212,7 @@ func (c *Cluster) initRobotUsers() error {
 			Password:  util.RandomPassword(constants.PasswordLength),
 			Flags:     flags,
 			AdminRole: adminRole,
+			IsDbOwner: isOwner,
 		}
 		if currentRole, present := c.pgUsers[username]; present {
 			c.pgUsers[username] = c.resolveNameConflict(&currentRole, &newRole)
diff --git a/pkg/cluster/database.go b/pkg/cluster/database.go
index aa3a5e3be..279cf828f 100644
--- a/pkg/cluster/database.go
+++ b/pkg/cluster/database.go
@@ -14,18 +14,25 @@ import (
 	"github.com/zalando/postgres-operator/pkg/spec"
 	"github.com/zalando/postgres-operator/pkg/util/constants"
 	"github.com/zalando/postgres-operator/pkg/util/retryutil"
+	"github.com/zalando/postgres-operator/pkg/util/users"
 )
 
 const (
 	getUserSQL = `SELECT a.rolname, COALESCE(a.rolpassword, ''), a.rolsuper, a.rolinherit,
-	        a.rolcreaterole, a.rolcreatedb, a.rolcanlogin, s.setconfig,
-	        ARRAY(SELECT b.rolname
-	              FROM pg_catalog.pg_auth_members m
-	              JOIN pg_catalog.pg_authid b ON (m.roleid = b.oid)
-	             WHERE m.member = a.oid) as memberof
-	 FROM pg_catalog.pg_authid a LEFT JOIN pg_db_role_setting s ON (a.oid = s.setrole AND s.setdatabase = 0::oid)
-	 WHERE a.rolname = ANY($1)
-	 ORDER BY 1;`
+	       a.rolcreaterole, a.rolcreatedb, a.rolcanlogin, s.setconfig,
+	       ARRAY(SELECT b.rolname
+	             FROM pg_catalog.pg_auth_members m
+	             JOIN pg_catalog.pg_authid b ON (m.roleid = b.oid)
+	            WHERE m.member = a.oid) as memberof
+	FROM pg_catalog.pg_authid a LEFT JOIN pg_db_role_setting s ON (a.oid = s.setrole AND s.setdatabase = 0::oid)
+	WHERE a.rolname = ANY($1)
+	ORDER BY 1;`
+
+	getUsersForRetention = `SELECT r.rolname, right(r.rolname, 6) AS roldatesuffix
+	        FROM pg_roles r
+	        JOIN unnest($1::text[]) AS u(name) ON r.rolname LIKE u.name || '%'
+			AND right(r.rolname, 6) ~ '^[0-9\.]+$'
+			ORDER BY 1;`
 
 	getDatabasesSQL = `SELECT datname, pg_get_userbyid(datdba) AS owner FROM pg_database;`
 	getSchemasSQL   = `SELECT n.nspname AS dbschema FROM pg_catalog.pg_namespace n
@@ -227,6 +234,65 @@ func (c *Cluster) readPgUsersFromDatabase(userNames []string) (users spec.PgUser
 	return users, nil
 }
 
+func findUsersFromRotation(rotatedUsers []string, db *sql.DB) (map[string]string, error) {
+	extraUsers := make(map[string]string, 0)
+	rows, err := db.Query(getUsersForRetention, pq.Array(rotatedUsers))
+	if err != nil {
+		return nil, fmt.Errorf("query failed: %v", err)
+	}
+	defer func() {
+		if err2 := rows.Close(); err2 != nil {
+			err = fmt.Errorf("error when closing query cursor: %v", err2)
+		}
+	}()
+
+	for rows.Next() {
+		var (
+			rolname, roldatesuffix string
+		)
+		err := rows.Scan(&rolname, &roldatesuffix)
+		if err != nil {
+			return nil, fmt.Errorf("error when processing rows of deprecated users: %v", err)
+		}
+		extraUsers[rolname] = roldatesuffix
+	}
+
+	return extraUsers, nil
+}
+
+func (c *Cluster) cleanupRotatedUsers(rotatedUsers []string, db *sql.DB) error {
+	c.setProcessName("checking for rotated users to remove from the database due to configured retention")
+	extraUsers, err := findUsersFromRotation(rotatedUsers, db)
+	if err != nil {
+		return fmt.Errorf("error when querying for deprecated users from password rotation: %v", err)
+	}
+
+	// make sure user retention policy aligns with rotation interval
+	retenionDays := c.OpConfig.PasswordRotationUserRetention
+	if retenionDays < 2*c.OpConfig.PasswordRotationInterval {
+		retenionDays = 2 * c.OpConfig.PasswordRotationInterval
+		c.logger.Warnf("user retention days too few compared to rotation interval %d - setting it to %d", c.OpConfig.PasswordRotationInterval, retenionDays)
+	}
+	retentionDate := time.Now().AddDate(0, 0, int(retenionDays)*-1)
+
+	for rotatedUser, dateSuffix := range extraUsers {
+		userCreationDate, err := time.Parse("060102", dateSuffix)
+		if err != nil {
+			c.logger.Errorf("could not parse creation date suffix of user %q: %v", rotatedUser, err)
+			continue
+		}
+		if retentionDate.After(userCreationDate) {
+			c.logger.Infof("dropping user %q due to configured days in password_rotation_user_retention", rotatedUser)
+			if err = users.DropPgUser(rotatedUser, db); err != nil {
+				c.logger.Errorf("could not drop role %q: %v", rotatedUser, err)
+				continue
+			}
+		}
+	}
+
+	return nil
+}
+
 // getDatabases returns the map of current databases with owners
 // The caller is responsible for opening and closing the database connection
 func (c *Cluster) getDatabases() (dbs map[string]string, err error) {
diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go
index 2df168a6e..c00f0a189 100644
--- a/pkg/cluster/sync.go
+++ b/pkg/cluster/sync.go
@@ -611,60 +611,180 @@ func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration(pod *v1.Pod, patroniC
 	return requiresMasterRestart, nil
 }
 
+func (c *Cluster) getNextRotationDate(currentDate time.Time) (time.Time, string) {
+	nextRotationDate := currentDate.AddDate(0, 0, int(c.OpConfig.PasswordRotationInterval))
+	return nextRotationDate, nextRotationDate.Format("2006-01-02 15:04:05")
+}
+
 func (c *Cluster) syncSecrets() error {
-	var (
-		err    error
-		secret *v1.Secret
-	)
+
 	c.logger.Info("syncing secrets")
 	c.setProcessName("syncing secrets")
-	secrets := c.generateUserSecrets()
+	generatedSecrets := c.generateUserSecrets()
+	rotationUsers := make(spec.PgUserMap)
+	retentionUsers := make([]string, 0)
+	currentTime := time.Now()
 
-	for secretUsername, secretSpec := range secrets {
-		if secret, err = c.KubeClient.Secrets(secretSpec.Namespace).Create(context.TODO(), secretSpec, metav1.CreateOptions{}); err == nil {
+	for secretUsername, generatedSecret := range generatedSecrets {
+		secret, err := c.KubeClient.Secrets(generatedSecret.Namespace).Create(context.TODO(), generatedSecret, metav1.CreateOptions{})
+		if err == nil {
 			c.Secrets[secret.UID] = secret
-			c.logger.Debugf("created new secret %s, namespace: %s, uid: %s", util.NameFromMeta(secret.ObjectMeta), secretSpec.Namespace, secret.UID)
+			c.logger.Debugf("created new secret %s, namespace: %s, uid: %s", util.NameFromMeta(secret.ObjectMeta), generatedSecret.Namespace, secret.UID)
 			continue
 		}
 		if k8sutil.ResourceAlreadyExists(err) {
-			var userMap map[string]spec.PgUser
-			if secret, err = c.KubeClient.Secrets(secretSpec.Namespace).Get(context.TODO(), secretSpec.Name, metav1.GetOptions{}); err != nil {
-				return fmt.Errorf("could not get current secret: %v", err)
-			}
-			if secretUsername != string(secret.Data["username"]) {
-				c.logger.Errorf("secret %s does not contain the role %s", secretSpec.Name, secretUsername)
-				continue
-			}
-			c.Secrets[secret.UID] = secret
-			c.logger.Debugf("secret %s already exists, fetching its password", util.NameFromMeta(secret.ObjectMeta))
-			if secretUsername == c.systemUsers[constants.SuperuserKeyName].Name {
-				secretUsername = constants.SuperuserKeyName
-				userMap = c.systemUsers
-			} else if secretUsername == c.systemUsers[constants.ReplicationUserKeyName].Name {
-				secretUsername = constants.ReplicationUserKeyName
-				userMap = c.systemUsers
-			} else {
-				userMap = c.pgUsers
-			}
-			pwdUser := userMap[secretUsername]
-			// if this secret belongs to the infrastructure role and the password has changed - replace it in the secret
-			if pwdUser.Password != string(secret.Data["password"]) &&
-				pwdUser.Origin == spec.RoleOriginInfrastructure {
-
-				c.logger.Debugf("updating the secret %s from the infrastructure roles", secretSpec.Name)
-				if _, err = c.KubeClient.Secrets(secretSpec.Namespace).Update(context.TODO(), secretSpec, metav1.UpdateOptions{}); err != nil {
-					return fmt.Errorf("could not update infrastructure role secret for role %q: %v", secretUsername, err)
-				}
-			} else {
-				// for non-infrastructure role - update the role with the password from the secret
-				pwdUser.Password = string(secret.Data["password"])
-				userMap[secretUsername] = pwdUser
+			if err = c.updateSecret(secretUsername, generatedSecret, &rotationUsers, &retentionUsers, currentTime); err != nil {
+				c.logger.Warningf("syncing secret %s failed: %v", util.NameFromMeta(secret.ObjectMeta), err)
 			}
 		} else {
-			return fmt.Errorf("could not create secret for user %s: in namespace %s: %v", secretUsername, secretSpec.Namespace, err)
+			return fmt.Errorf("could not create secret for user %s: in namespace %s: %v", secretUsername, generatedSecret.Namespace, err)
 		}
 	}
 
+	// add new user with date suffix and use it in the secret of the original user
+	if len(rotationUsers) > 0 {
+		err := c.initDbConn()
+		if err != nil {
+			return fmt.Errorf("could not init db connection: %v", err)
+		}
+		pgSyncRequests := c.userSyncStrategy.ProduceSyncRequests(spec.PgUserMap{}, rotationUsers)
+		if err = c.userSyncStrategy.ExecuteSyncRequests(pgSyncRequests, c.pgDb); err != nil {
+			return fmt.Errorf("error creating database roles for password rotation: %v", err)
+		}
+		if err := c.closeDbConn(); err != nil {
+			c.logger.Errorf("could not close database connection after creating users for password rotation: %v", err)
+		}
+	}
+
+	// remove rotation users that exceed the retention interval
+	if len(retentionUsers) > 0 {
+		err := c.initDbConn()
+		if err != nil {
+			return fmt.Errorf("could not init db connection: %v", err)
+		}
+		if err = c.cleanupRotatedUsers(retentionUsers, c.pgDb); err != nil {
+			return fmt.Errorf("error removing users exceeding configured retention interval: %v", err)
+		}
+		if err := c.closeDbConn(); err != nil {
+			c.logger.Errorf("could not close database connection after removing users exceeding configured retention interval: %v", err)
+		}
+	}
+
+	return nil
+}
+
+func (c *Cluster) updateSecret(
+	secretUsername string,
+	generatedSecret *v1.Secret,
+	rotationUsers *spec.PgUserMap,
+	retentionUsers *[]string,
+	currentTime time.Time) error {
+	var (
+		secret              *v1.Secret
+		err                 error
+		updateSecret        bool
+		updateSecretMsg     string
+		nextRotationDate    time.Time
+		nextRotationDateStr string
+	)
+
+	// get the secret first
+	if secret, err = c.KubeClient.Secrets(generatedSecret.Namespace).Get(context.TODO(), generatedSecret.Name, metav1.GetOptions{}); err != nil {
+		return fmt.Errorf("could not get current secret: %v", err)
+	}
+	c.Secrets[secret.UID] = secret
+
+	// fetch user map to update later
+	var userMap map[string]spec.PgUser
+	var userKey string
+	if secretUsername == c.systemUsers[constants.SuperuserKeyName].Name {
+		userKey = constants.SuperuserKeyName
+		userMap = c.systemUsers
+	} else if secretUsername == c.systemUsers[constants.ReplicationUserKeyName].Name {
+		userKey = constants.ReplicationUserKeyName
+		userMap = c.systemUsers
+	} else {
+		userKey = secretUsername
+		userMap = c.pgUsers
+	}
+	pwdUser := userMap[userKey]
+	secretName := util.NameFromMeta(secret.ObjectMeta)
+
+	// if password rotation is enabled update password and username if rotation interval has been passed
+	if (c.OpConfig.EnablePasswordRotation && !pwdUser.IsDbOwner &&
+		pwdUser.Origin != spec.RoleOriginInfrastructure && pwdUser.Origin != spec.RoleOriginSystem) ||
+		util.SliceContains(c.Spec.UsersWithSecretRotation, secretUsername) ||
+		util.SliceContains(c.Spec.UsersWithInPlaceSecretRotation, secretUsername) {
+
+		// initialize password rotation setting first rotation date
+		nextRotationDateStr = string(secret.Data["nextRotation"])
+		if nextRotationDate, err = time.ParseInLocation("2006-01-02 15:04:05", nextRotationDateStr, time.Local); err != nil {
+			nextRotationDate, nextRotationDateStr = c.getNextRotationDate(currentTime)
+			secret.Data["nextRotation"] = []byte(nextRotationDateStr)
+			updateSecret = true
+			updateSecretMsg = fmt.Sprintf("rotation date not found in secret %q. Setting it to %s", secretName, nextRotationDateStr)
+		}
+
+		// check if next rotation can happen sooner
+		// if rotation interval has been decreased
+		currentRotationDate, _ := c.getNextRotationDate(currentTime)
+		if nextRotationDate.After(currentRotationDate) {
+			nextRotationDate = currentRotationDate
+		}
+
+		// update password and next rotation date if configured interval has passed
+		if currentTime.After(nextRotationDate) {
+			// create rotation user if role is not listed for in-place password update
+			if !util.SliceContains(c.Spec.UsersWithInPlaceSecretRotation, secretUsername) {
+				rotationUser := pwdUser
+				newRotationUsername := secretUsername + currentTime.Format("060102")
+				rotationUser.Name = newRotationUsername
+				rotationUser.MemberOf = []string{secretUsername}
+				(*rotationUsers)[newRotationUsername] = rotationUser
+				secret.Data["username"] = []byte(newRotationUsername)
+
+				// whenever there is a rotation, check if old rotation users can be deleted
+				*retentionUsers = append(*retentionUsers, secretUsername)
+			}
+			secret.Data["password"] = []byte(util.RandomPassword(constants.PasswordLength))
+
+			_, nextRotationDateStr = c.getNextRotationDate(nextRotationDate)
+			secret.Data["nextRotation"] = []byte(nextRotationDateStr)
+
+			updateSecret = true
+			updateSecretMsg = fmt.Sprintf("updating secret %q due to password rotation - next rotation date: %s", secretName, nextRotationDateStr)
+		}
+	} else {
+		// username might not match if password rotation has been disabled again
+		if secretUsername != string(secret.Data["username"]) {
+			*retentionUsers = append(*retentionUsers, secretUsername)
+			secret.Data["username"] = []byte(secretUsername)
+			secret.Data["password"] = []byte(util.RandomPassword(constants.PasswordLength))
+			secret.Data["nextRotation"] = []byte{}
+			updateSecret = true
+			updateSecretMsg = fmt.Sprintf("secret %s does not contain the role %s - updating username and resetting password", secretName, secretUsername)
+		}
+	}
+
+	// if this secret belongs to the infrastructure role and the password has changed - replace it in the secret
+	if pwdUser.Password != string(secret.Data["password"]) && pwdUser.Origin == spec.RoleOriginInfrastructure {
+		secret = generatedSecret
+		updateSecret = true
+		updateSecretMsg = fmt.Sprintf("updating the secret %s from the infrastructure roles", secretName)
+	} else {
+		// for non-infrastructure role - update the role with the password from the secret
+		pwdUser.Password = string(secret.Data["password"])
+		userMap[userKey] = pwdUser
+	}
+
+	if updateSecret {
+		c.logger.Debugln(updateSecretMsg)
+		if _, err = c.KubeClient.Secrets(secret.Namespace).Update(context.TODO(), secret, metav1.UpdateOptions{}); err != nil {
+			return fmt.Errorf("could not update secret %q: %v", secretName, err)
+		}
+		c.Secrets[secret.UID] = secret
+	}
+
 	return nil
 }
 
diff --git a/pkg/cluster/sync_test.go b/pkg/cluster/sync_test.go
index 3e3fc9318..80e2b8463 100644
--- a/pkg/cluster/sync_test.go
+++ b/pkg/cluster/sync_test.go
@@ -19,6 +19,7 @@ import (
 	"github.com/zalando/postgres-operator/mocks"
 	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
 	fakeacidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/fake"
+	"github.com/zalando/postgres-operator/pkg/spec"
 	"github.com/zalando/postgres-operator/pkg/util/config"
 	"github.com/zalando/postgres-operator/pkg/util/k8sutil"
 	"github.com/zalando/postgres-operator/pkg/util/patroni"
@@ -26,6 +27,8 @@ import (
 )
 
 var patroniLogger = logrus.New().WithField("test", "patroni")
+var acidClientSet = fakeacidv1.NewSimpleClientset()
+var clientSet = fake.NewSimpleClientset()
 
 func newMockPod(ip string) *v1.Pod {
 	return &v1.Pod{
@@ -36,9 +39,6 @@ func newMockPod(ip string) *v1.Pod {
 }
 
 func newFakeK8sSyncClient() (k8sutil.KubernetesClient, *fake.Clientset) {
-	acidClientSet := fakeacidv1.NewSimpleClientset()
-	clientSet := fake.NewSimpleClientset()
-
 	return k8sutil.KubernetesClient{
 		PodsGetter:         clientSet.CoreV1(),
 		PostgresqlsGetter:  acidClientSet.AcidV1(),
@@ -46,6 +46,12 @@ func newFakeK8sSyncClient() (k8sutil.KubernetesClient, *fake.Clientset) {
 	}, clientSet
 }
 
+func newFakeK8sSyncSecretsClient() (k8sutil.KubernetesClient, *fake.Clientset) {
+	return k8sutil.KubernetesClient{
+		SecretsGetter: clientSet.CoreV1(),
+	}, clientSet
+}
+
 func TestSyncStatefulSetsAnnotations(t *testing.T) {
 	testName := "test syncing statefulsets annotations"
 	client, _ := newFakeK8sSyncClient()
@@ -257,3 +263,72 @@ func TestCheckAndSetGlobalPostgreSQLConfiguration(t *testing.T) {
 		}
 	}
 }
+
+func TestUpdateSecret(t *testing.T) {
+	testName := "test syncing secrets"
+	client, _ := newFakeK8sSyncSecretsClient()
+
+	clusterName := "acid-test-cluster"
+	namespace := "default"
+	username := "foo"
+	secretTemplate := config.StringTemplate("{username}.{cluster}.credentials")
+	rotationUsers := make(spec.PgUserMap)
+	retentionUsers := make([]string, 0)
+	yesterday := time.Now().AddDate(0, 0, -1)
+
+	// new cluster with pvc storage resize mode and configured labels
+	var cluster = New(
+		Config{
+			OpConfig: config.Config{
+				Auth: config.Auth{
+					SecretNameTemplate:            secretTemplate,
+					EnablePasswordRotation:        true,
+					PasswordRotationInterval:      1,
+					PasswordRotationUserRetention: 3,
+				},
+				Resources: config.Resources{
+					ClusterLabels:    map[string]string{"application": "spilo"},
+					ClusterNameLabel: "cluster-name",
+				},
+			},
+		}, client, acidv1.Postgresql{}, logger, eventRecorder)
+
+	cluster.Name = clusterName
+	cluster.Namespace = namespace
+	cluster.pgUsers = map[string]spec.PgUser{}
+	cluster.Spec.Users = map[string]acidv1.UserFlags{username: {}}
+	cluster.initRobotUsers()
+
+	// create a secret for user foo
+	cluster.syncSecrets()
+
+	secret, err := cluster.KubeClient.Secrets(namespace).Get(context.TODO(), secretTemplate.Format("username", username, "cluster", clusterName), metav1.GetOptions{})
+	assert.NoError(t, err)
+	generatedSecret := cluster.Secrets[secret.UID]
+
+	// now update the secret setting next rotation date (yesterday + interval)
+	cluster.updateSecret(username, generatedSecret, &rotationUsers, &retentionUsers, yesterday)
+	updatedSecret, err := cluster.KubeClient.Secrets(namespace).Get(context.TODO(), secretTemplate.Format("username", username, "cluster", clusterName), metav1.GetOptions{})
+	assert.NoError(t, err)
+
+	nextRotation := string(updatedSecret.Data["nextRotation"])
+	_, nextRotationDate := cluster.getNextRotationDate(yesterday)
+	if nextRotation != nextRotationDate {
+		t.Errorf("%s: updated secret does not contain correct rotation date: expected %s, got %s", testName, nextRotationDate, nextRotation)
+	}
+
+	// update secret again but use current time to trigger rotation
+	cluster.updateSecret(username, generatedSecret, &rotationUsers, &retentionUsers, time.Now())
+	updatedSecret, err = cluster.KubeClient.Secrets(namespace).Get(context.TODO(), secretTemplate.Format("username", username, "cluster", clusterName), metav1.GetOptions{})
+	assert.NoError(t, err)
+
+	if len(rotationUsers) != 1 && len(retentionUsers) != 1 {
+		t.Errorf("%s: unexpected number of users to rotate - expected only foo, found %d", testName, len(rotationUsers))
+	}
+
+	secretUsername := string(updatedSecret.Data["username"])
+	rotatedUsername := username + time.Now().Format("060102")
+	if secretUsername != rotatedUsername {
+		t.Errorf("%s: updated secret does not contain correct username: expected %s, got %s", testName, rotatedUsername, secretUsername)
+	}
+}
diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go
index 1cb36e746..20958dd7b 100644
--- a/pkg/controller/operator_config.go
+++ b/pkg/controller/operator_config.go
@@ -54,6 +54,9 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	// user config
 	result.SuperUsername = util.Coalesce(fromCRD.PostgresUsersConfiguration.SuperUsername, "postgres")
 	result.ReplicationUsername = util.Coalesce(fromCRD.PostgresUsersConfiguration.ReplicationUsername, "standby")
+	result.EnablePasswordRotation = fromCRD.PostgresUsersConfiguration.EnablePasswordRotation
+	result.PasswordRotationInterval = util.CoalesceUInt32(fromCRD.PostgresUsersConfiguration.PasswordRotationInterval, 90)
+	result.PasswordRotationUserRetention = util.CoalesceUInt32(fromCRD.PostgresUsersConfiguration.DeepCopy().PasswordRotationUserRetention, 180)
 
 	// major version upgrade config
 	result.MajorVersionUpgradeMode = util.Coalesce(fromCRD.MajorVersionUpgrade.MajorVersionUpgradeMode, "off")
diff --git a/pkg/spec/types.go b/pkg/spec/types.go
index 533aae79f..202e0fa6f 100644
--- a/pkg/spec/types.go
+++ b/pkg/spec/types.go
@@ -55,6 +55,7 @@ type PgUser struct {
 	MemberOf   []string          `yaml:"inrole"`
 	Parameters map[string]string `yaml:"db_parameters"`
 	AdminRole  string            `yaml:"admin_role"`
+	IsDbOwner  bool              `yaml:"is_db_owner"`
 	Deleted    bool              `yaml:"deleted"`
 }
 
diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go
index ff710640e..fa5d3b770 100644
--- a/pkg/util/config/config.go
+++ b/pkg/util/config/config.go
@@ -100,6 +100,9 @@ type Auth struct {
 	InfrastructureRolesDefs       string                `name:"infrastructure_roles_secrets"`
 	SuperUsername                 string                `name:"super_username" default:"postgres"`
 	ReplicationUsername           string                `name:"replication_username" default:"standby"`
+	EnablePasswordRotation        bool                  `name:"enable_password_rotation" default:"false"`
+	PasswordRotationInterval      uint32                `name:"password_rotation_interval" default:"90"`
+	PasswordRotationUserRetention uint32                `name:"password_rotation_user_retention" default:"180"`
 }
 
 // Scalyr holds the configuration for the Scalyr Agent sidecar for log shipping:
diff --git a/pkg/util/users/users.go b/pkg/util/users/users.go
index 821350bb7..6bc31f6da 100644
--- a/pkg/util/users/users.go
+++ b/pkg/util/users/users.go
@@ -18,6 +18,7 @@ const (
 	alterUserRenameSQL   = `ALTER ROLE "%s" RENAME TO "%s%s"`
 	alterRoleResetAllSQL = `ALTER ROLE "%s" RESET ALL`
 	alterRoleSetSQL      = `ALTER ROLE "%s" SET %s TO %s`
+	dropUserSQL          = `SET LOCAL synchronous_commit = 'local'; DROP ROLE "%s";`
 	grantToUserSQL       = `GRANT %s TO "%s"`
 	doBlockStmt          = `SET LOCAL synchronous_commit = 'local'; DO $$ BEGIN %s; END;$$;`
 	passwordTemplate     = "ENCRYPTED PASSWORD '%s'"
@@ -288,3 +289,13 @@ func quoteParameterValue(name, val string) string {
 	}
 	return fmt.Sprintf(`'%s'`, strings.Trim(val, " "))
 }
+
+// DropPgUser to remove user created by the operator e.g. for password rotation
+func DropPgUser(user string, db *sql.DB) error {
+	query := fmt.Sprintf(dropUserSQL, user)
+	if _, err := db.Exec(query); err != nil { // TODO: Try several times
+		return err
+	}
+
+	return nil
+}

From 3ce0b1e7fa236e8d54fbf066a5fb45fa11c041fd Mon Sep 17 00:00:00 2001
From: Felix Kunde <felix-kunde@gmx.de>
Date: Fri, 18 Feb 2022 15:04:31 +0100
Subject: [PATCH 20/58] deprecate crd validation toggle and sync with manifests
 (#1781)

* deprecate crd validation toggle and sync with manifests
* fix description in pg crd manifests
* change CRD creation strategy
* affinity matchExpression has values
* lower repair period in e2e tests
---
 .../crds/operatorconfigurations.yaml          |  22 +-
 .../postgres-operator/crds/postgresqls.yaml   |  17 +-
 charts/postgres-operator/values.yaml          |   5 +-
 docs/administrator.md                         |  43 ++--
 docs/reference/operator_parameters.md         |   7 +-
 e2e/tests/test_e2e.py                         |   6 +-
 manifests/configmap.yaml                      |   2 +-
 manifests/operatorconfiguration.crd.yaml      |  15 +-
 ...gresql-operator-default-configuration.yaml |   3 +-
 manifests/postgresql.crd.yaml                 |  17 +-
 pkg/apis/acid.zalan.do/v1/crds.go             | 219 ++++++++++++------
 .../v1/operator_configuration_type.go         |   1 +
 .../acid.zalan.do/v1/zz_generated.deepcopy.go |   5 +
 pkg/controller/controller.go                  |   4 +-
 pkg/controller/operator_config.go             |   1 +
 pkg/controller/util.go                        |  47 ++--
 pkg/util/config/config.go                     |   1 +
 17 files changed, 262 insertions(+), 153 deletions(-)

diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml
index 554318a97..f510e08f5 100644
--- a/charts/postgres-operator/crds/operatorconfigurations.yaml
+++ b/charts/postgres-operator/crds/operatorconfigurations.yaml
@@ -61,6 +61,11 @@ spec:
           configuration:
             type: object
             properties:
+              crd_categories:
+                type: array
+                nullable: true
+                items:
+                  type: string
               docker_image:
                 type: string
                 default: "registry.opensource.zalan.do/acid/spilo-14:2.1-p3"
@@ -69,6 +74,7 @@ spec:
                 default: true
               enable_crd_validation:
                 type: boolean
+                description: deprecated
                 default: true
               enable_lazy_spilo_upgrade:
                 type: boolean
@@ -90,11 +96,13 @@ spec:
                 default: false
               max_instances:
                 type: integer
-                minimum: -1  # -1 = disabled
+                description: "-1 = disabled"
+                minimum: -1
                 default: -1
               min_instances:
                 type: integer
-                minimum: -1  # -1 = disabled
+                description: "-1 = disabled"
+                minimum: -1
                 default: -1
               resync_period:
                 type: string
@@ -184,12 +192,12 @@ spec:
                     type: array
                     items:
                       type: string
-                  enable_init_containers:
-                    type: boolean
-                    default: true
                   enable_cross_namespace_secret:
                     type: boolean
                     default: false
+                  enable_init_containers:
+                    type: boolean
+                    default: true
                   enable_pod_antiaffinity:
                     type: boolean
                     default: false
@@ -410,12 +418,12 @@ spec:
                     type: string
                   log_s3_bucket:
                     type: string
+                  wal_az_storage_account:
+                    type: string
                   wal_gs_bucket:
                     type: string
                   wal_s3_bucket:
                     type: string
-                  wal_az_storage_account:
-                    type: string
               logical_backup:
                 type: object
                 properties:
diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml
index 3c03e4f3c..83043084c 100644
--- a/charts/postgres-operator/crds/postgresqls.yaml
+++ b/charts/postgres-operator/crds/postgresqls.yaml
@@ -147,7 +147,7 @@ spec:
                       - "transaction"
                   numberOfInstances:
                     type: integer
-                    minimum: 2
+                    minimum: 1
                   resources:
                     type: object
                     required:
@@ -201,8 +201,9 @@ spec:
                 type: boolean
               enableShmVolume:
                 type: boolean
-              init_containers:  # deprecated
+              init_containers:
                 type: array
+                description: deprecated
                 nullable: true
                 items:
                   type: object
@@ -229,8 +230,8 @@ spec:
                     items:
                       type: object
                       required:
-                      - weight
                       - preference
+                      - weight
                       properties:
                         preference:
                           type: object
@@ -348,8 +349,9 @@ spec:
                 type: object
                 additionalProperties:
                   type: string
-              pod_priority_class_name:  # deprecated
+              pod_priority_class_name:
                 type: string
+                description: deprecated
               podPriorityClassName:
                 type: string
               postgresql:
@@ -393,8 +395,9 @@ spec:
                             type: boolean
                     secretNamespace:
                       type: string
-              replicaLoadBalancer:  # deprecated
+              replicaLoadBalancer:
                 type: boolean
+                description: deprecated
               resources:
                 type: object
                 required:
@@ -512,14 +515,14 @@ spec:
                         - PreferNoSchedule
                     tolerationSeconds:
                       type: integer
-              useLoadBalancer:  # deprecated
+              useLoadBalancer:
                 type: boolean
+                description: deprecated
               users:
                 type: object
                 additionalProperties:
                   type: array
                   nullable: true
-                  description: "Role flags specified here must not contradict each other"
                   items:
                     type: string
                     enum:
diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml
index 0602a0e93..8183894d3 100644
--- a/charts/postgres-operator/values.yaml
+++ b/charts/postgres-operator/values.yaml
@@ -22,8 +22,9 @@ enableJsonLogging: false
 configGeneral:
   # the deployment should create/update the CRDs
   enable_crd_registration: true
-  # choose if deployment creates/updates CRDs with OpenAPIV3Validation
-  enable_crd_validation: true
+  # specify categories under which crds should be listed
+  crd_categories:
+  - "all"
   # update only the statefulsets without immediately doing the rolling update
   enable_lazy_spilo_upgrade: false
   # set the PGVERSION env var instead of providing the version via postgresql.bin_dir in SPILO_CONFIGURATION
diff --git a/docs/administrator.md b/docs/administrator.md
index d7ab8ef0c..3c5d8ae46 100644
--- a/docs/administrator.md
+++ b/docs/administrator.md
@@ -3,6 +3,25 @@
 Learn how to configure and manage the Postgres Operator in your Kubernetes (K8s)
 environment.
 
+## CRD registration and validation
+
+On startup, the operator will try to register the necessary
+[CustomResourceDefinitions](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/#customresourcedefinitions)
+`Postgresql` and `OperatorConfiguration`. The latter will only get created if
+the `POSTGRES_OPERATOR_CONFIGURATION_OBJECT` [environment variable](https://github.com/zalando/postgres-operator/blob/master/manifests/postgres-operator.yaml#L36)
+is set in the deployment yaml and is not empty. If the CRDs already exists they
+will only be patched. If you do not wish the operator to create or update the
+CRDs set `enable_crd_registration` config option to `false`.
+
+CRDs are defined with a `openAPIV3Schema` structural schema against which new
+manifests of [`postgresql`](https://github.com/zalando/postgres-operator/blob/master/manifests/postgresql.crd.yaml) or [`OperatorConfiguration`](https://github.com/zalando/postgres-operator/blob/master/manifests/operatorconfiguration.crd.yaml)
+resources will be validated. On creation you can bypass the validation with 
+`kubectl create --validate=false`.
+
+By default, the operator will register the CRDs in the `all` category so
+that resources are listed on `kubectl get all` commands. The `crd_categories`
+config option allows for customization of categories.
+
 ## Upgrading the operator
 
 The Postgres Operator is upgraded by changing the docker image within the
@@ -63,30 +82,6 @@ upgrade procedure, refer to the [corresponding PR in Spilo](https://github.com/z
 When `major_version_upgrade_mode` is set to `manual` the operator will run
 the upgrade script for you after the manifest is updated and pods are rotated.
 
-## CRD Validation
-
-[CustomResourceDefinitions](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/#customresourcedefinitions)
-will be registered with schema validation by default when the operator is
-deployed. The `OperatorConfiguration` CRD will only get created if the
-`POSTGRES_OPERATOR_CONFIGURATION_OBJECT` [environment variable](https://github.com/zalando/postgres-operator/blob/master/manifests/postgres-operator.yaml#L36)
-in the deployment yaml is set and not empty.
-
-When submitting manifests of [`postgresql`](https://github.com/zalando/postgres-operator/blob/master/manifests/postgresql.crd.yaml) or
-[`OperatorConfiguration`](https://github.com/zalando/postgres-operator/blob/master/manifests/operatorconfiguration.crd.yaml) custom
-resources with kubectl, validation can be bypassed with `--validate=false`. The
-operator can also be configured to not register CRDs with validation on `ADD` or
-`UPDATE` events. Running instances are not affected when enabling the validation
-afterwards unless the manifests is not changed then. Note, that the provided CRD
-manifests contain the validation for users to understand what schema is
-enforced.
-
-Once the validation is enabled it can only be disabled manually by editing or
-patching the CRD manifest:
-
-```bash
-kubectl patch crd postgresqls.acid.zalan.do -p '{"spec":{"validation": null}}'
-```
-
 ## Non-default cluster domain
 
 If your cluster uses a DNS domain other than the default `cluster.local`, this
diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md
index 990909eb5..f3d9be88f 100644
--- a/docs/reference/operator_parameters.md
+++ b/docs/reference/operator_parameters.md
@@ -75,9 +75,12 @@ Those are top-level keys, containing both leaf keys and groups.
   The default is `true`.
 
 * **enable_crd_validation**
-  toggles if the operator will create or update CRDs with
+  *deprecated*: toggles if the operator will create or update CRDs with
   [OpenAPI v3 schema validation](https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#validation)
-  The default is `true`.
+  The default is `true`. `false` will be ignored, since `apiextensions.io/v1` requires a structural schema definition.
+
+* **crd_categories**
+  The operator will register CRDs in the `all` category by default so that they will be returned by a `kubectl get all` call. You are free to change categories or leave them empty.
 
 * **enable_lazy_spilo_upgrade**
   Instruct operator to update only the statefulsets with new images (Spilo and InitContainers) without immediately doing the rolling update. The assumption is pods will be re-started later with new images, for example due to the node rotation.
diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py
index 7b3711b73..77c14e272 100644
--- a/e2e/tests/test_e2e.py
+++ b/e2e/tests/test_e2e.py
@@ -204,7 +204,8 @@ class EndToEndTestCase(unittest.TestCase):
                 "enable_postgres_team_crd": "true",
                 "enable_team_member_deprecation": "true",
                 "role_deletion_suffix": "_delete_me",
-                "resync_period": "15s"
+                "resync_period": "15s",
+                "repair_period": "10s",
             },
         }
         k8s.update_config(enable_postgres_team_crd)
@@ -288,6 +289,7 @@ class EndToEndTestCase(unittest.TestCase):
         revert_resync = {
             "data": {
                 "resync_period": "4m",
+                "repair_period": "1m",
             },
         }
         k8s.update_config(revert_resync)
@@ -1403,6 +1405,7 @@ class EndToEndTestCase(unittest.TestCase):
             "data": {
                 "pod_label_wait_timeout": "2s",
                 "resync_period": "30s",
+                "repair_period": "10s",
             }
         }
 
@@ -1444,6 +1447,7 @@ class EndToEndTestCase(unittest.TestCase):
                 "data": {
                     "pod_label_wait_timeout": "10m",
                     "resync_period": "4m",
+                    "repair_period": "2m",
                 }
             }
             k8s.update_config(patch_resync_config, "revert resync interval and pod_label_wait_timeout")
diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml
index 1a52a52da..b3aaa3c66 100644
--- a/manifests/configmap.yaml
+++ b/manifests/configmap.yaml
@@ -22,6 +22,7 @@ data:
   # connection_pooler_number_of_instances: 2
   # connection_pooler_schema: "pooler"
   # connection_pooler_user: "pooler"
+  crd_categories: "all"
   # custom_service_annotations: "keyx:valuez,keya:valuea"
   # custom_pod_annotations: "keya:valuea,keyb:valueb"
   db_hosted_zone: db.example.com
@@ -36,7 +37,6 @@ data:
   # downscaler_annotations: "deployment-time,downscaler/*"
   # enable_admin_role_for_users: "true"
   # enable_crd_registration: "true"
-  # enable_crd_validation: "true"
   # enable_cross_namespace_secret: "false"
   # enable_database_access: "true"
   enable_ebs_gp3_migration: "false"
diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml
index 3313377a0..d086998cf 100644
--- a/manifests/operatorconfiguration.crd.yaml
+++ b/manifests/operatorconfiguration.crd.yaml
@@ -59,6 +59,11 @@ spec:
           configuration:
             type: object
             properties:
+              crd_categories:
+                type: array
+                nullable: true
+                items:
+                  type: string
               docker_image:
                 type: string
                 default: "registry.opensource.zalan.do/acid/spilo-14:2.1-p3"
@@ -67,6 +72,7 @@ spec:
                 default: true
               enable_crd_validation:
                 type: boolean
+                description: deprecated
                 default: true
               enable_lazy_spilo_upgrade:
                 type: boolean
@@ -88,11 +94,13 @@ spec:
                 default: false
               max_instances:
                 type: integer
-                minimum: -1  # -1 = disabled
+                description: "-1 = disabled"
+                minimum: -1
                 default: -1
               min_instances:
                 type: integer
-                minimum: -1  # -1 = disabled
+                description: "-1 = disabled"
+                minimum: -1
                 default: -1
               resync_period:
                 type: string
@@ -182,6 +190,9 @@ spec:
                     type: array
                     items:
                       type: string
+                  enable_cross_namespace_secret:
+                    type: boolean
+                    default: false
                   enable_init_containers:
                     type: boolean
                     default: true
diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml
index f8ccad7ce..87b5436d5 100644
--- a/manifests/postgresql-operator-default-configuration.yaml
+++ b/manifests/postgresql-operator-default-configuration.yaml
@@ -5,7 +5,8 @@ metadata:
 configuration:
   docker_image: registry.opensource.zalan.do/acid/spilo-14:2.1-p3
   # enable_crd_registration: true
-  # enable_crd_validation: true
+  # crd_categories:
+  # - all
   # enable_lazy_spilo_upgrade: false
   enable_pgversion_env_var: true
   # enable_shm_volume: true
diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml
index dafbf9038..88144dc05 100644
--- a/manifests/postgresql.crd.yaml
+++ b/manifests/postgresql.crd.yaml
@@ -145,7 +145,7 @@ spec:
                       - "transaction"
                   numberOfInstances:
                     type: integer
-                    minimum: 2
+                    minimum: 1
                   resources:
                     type: object
                     required:
@@ -199,8 +199,9 @@ spec:
                 type: boolean
               enableShmVolume:
                 type: boolean
-              init_containers:  # deprecated
+              init_containers:
                 type: array
+                description: deprecated
                 nullable: true
                 items:
                   type: object
@@ -227,8 +228,8 @@ spec:
                     items:
                       type: object
                       required:
-                      - weight
                       - preference
+                      - weight
                       properties:
                         preference:
                           type: object
@@ -346,8 +347,9 @@ spec:
                 type: object
                 additionalProperties:
                   type: string
-              pod_priority_class_name:  # deprecated
+              pod_priority_class_name:
                 type: string
+                description: deprecated
               podPriorityClassName:
                 type: string
               postgresql:
@@ -391,8 +393,9 @@ spec:
                             type: boolean
                     secretNamespace:
                       type: string
-              replicaLoadBalancer:  # deprecated
+              replicaLoadBalancer:
                 type: boolean
+                description: deprecated
               resources:
                 type: object
                 required:
@@ -510,14 +513,14 @@ spec:
                         - PreferNoSchedule
                     tolerationSeconds:
                       type: integer
-              useLoadBalancer:  # deprecated
+              useLoadBalancer:
                 type: boolean
+                description: deprecated
               users:
                 type: object
                 additionalProperties:
                   type: array
                   nullable: true
-                  description: "Role flags specified here must not contradict each other"
                   items:
                     type: string
                     enum:
diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go
index da18cd20d..844ccef04 100644
--- a/pkg/apis/acid.zalan.do/v1/crds.go
+++ b/pkg/apis/acid.zalan.do/v1/crds.go
@@ -1,6 +1,8 @@
 package v1
 
 import (
+	"fmt"
+
 	acidzalando "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do"
 	"github.com/zalando/postgres-operator/pkg/util"
 	apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
@@ -11,11 +13,13 @@ import (
 const (
 	PostgresCRDResourceKind   = "postgresql"
 	PostgresCRDResourcePlural = "postgresqls"
+	PostgresCRDResourceList   = PostgresCRDResourceKind + "List"
 	PostgresCRDResouceName    = PostgresCRDResourcePlural + "." + acidzalando.GroupName
 	PostgresCRDResourceShort  = "pg"
 
 	OperatorConfigCRDResouceKind    = "OperatorConfiguration"
 	OperatorConfigCRDResourcePlural = "operatorconfigurations"
+	OperatorConfigCRDResourceList   = OperatorConfigCRDResouceKind + "List"
 	OperatorConfigCRDResourceName   = OperatorConfigCRDResourcePlural + "." + acidzalando.GroupName
 	OperatorConfigCRDResourceShort  = "opconfig"
 )
@@ -148,7 +152,8 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 										Type: "string",
 									},
 									"targetContainers": {
-										Type: "array",
+										Type:     "array",
+										Nullable: true,
 										Items: &apiextv1.JSONSchemaPropsOrArray{
 											Schema: &apiextv1.JSONSchemaProps{
 												Type: "string",
@@ -199,9 +204,8 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 								Type: "string",
 							},
 							"timestamp": {
-								Type:        "string",
-								Description: "Date-time format that specifies a timezone as an offset relative to UTC e.g. 1996-12-19T16:39:57-08:00",
-								Pattern:     "^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\\.[0-9]+)?(([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$",
+								Type:    "string",
+								Pattern: "^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\\.[0-9]+)?(([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$",
 							},
 							"uid": {
 								Type:   "string",
@@ -242,14 +246,12 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 										Required: []string{"cpu", "memory"},
 										Properties: map[string]apiextv1.JSONSchemaProps{
 											"cpu": {
-												Type:        "string",
-												Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0",
-												Pattern:     "^(\\d+m|\\d+(\\.\\d{1,3})?)$",
+												Type:    "string",
+												Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$",
 											},
 											"memory": {
-												Type:        "string",
-												Description: "Plain integer or fixed-point integer using one of these suffixes: E, P, T, G, M, k (with or without a tailing i). Must be greater than 0",
-												Pattern:     "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$",
+												Type:    "string",
+												Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$",
 											},
 										},
 									},
@@ -258,14 +260,12 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 										Required: []string{"cpu", "memory"},
 										Properties: map[string]apiextv1.JSONSchemaProps{
 											"cpu": {
-												Type:        "string",
-												Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0",
-												Pattern:     "^(\\d+m|\\d+(\\.\\d{1,3})?)$",
+												Type:    "string",
+												Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$",
 											},
 											"memory": {
-												Type:        "string",
-												Description: "Plain integer or fixed-point integer using one of these suffixes: E, P, T, G, M, k (with or without a tailing i). Must be greater than 0",
-												Pattern:     "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$",
+												Type:    "string",
+												Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$",
 											},
 										},
 									},
@@ -283,8 +283,7 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 						Type: "object",
 						AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{
 							Schema: &apiextv1.JSONSchemaProps{
-								Type:        "string",
-								Description: "User names specified here as database owners must be declared in the users key of the spec key",
+								Type: "string",
 							},
 						},
 					},
@@ -311,7 +310,8 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 					},
 					"init_containers": {
 						Type:        "array",
-						Description: "Deprecated",
+						Description: "deprecated",
+						Nullable:    true,
 						Items: &apiextv1.JSONSchemaPropsOrArray{
 							Schema: &apiextv1.JSONSchemaProps{
 								Type:                   "object",
@@ -320,7 +320,8 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 						},
 					},
 					"initContainers": {
-						Type: "array",
+						Type:     "array",
+						Nullable: true,
 						Items: &apiextv1.JSONSchemaPropsOrArray{
 							Schema: &apiextv1.JSONSchemaProps{
 								Type:                   "object",
@@ -358,9 +359,23 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 														Type: "array",
 														Items: &apiextv1.JSONSchemaPropsOrArray{
 															Schema: &apiextv1.JSONSchemaProps{
-																Type: "object",
-																AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{
-																	Allows: true,
+																Type:     "object",
+																Required: []string{"key", "operator"},
+																Properties: map[string]apiextv1.JSONSchemaProps{
+																	"key": {
+																		Type: "string",
+																	},
+																	"operator": {
+																		Type: "string",
+																	},
+																	"values": {
+																		Type: "array",
+																		Items: &apiextv1.JSONSchemaPropsOrArray{
+																			Schema: &apiextv1.JSONSchemaProps{
+																				Type: "string",
+																			},
+																		},
+																	},
 																},
 															},
 														},
@@ -369,9 +384,23 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 														Type: "array",
 														Items: &apiextv1.JSONSchemaPropsOrArray{
 															Schema: &apiextv1.JSONSchemaProps{
-																Type: "object",
-																AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{
-																	Allows: true,
+																Type:     "object",
+																Required: []string{"key", "operator"},
+																Properties: map[string]apiextv1.JSONSchemaProps{
+																	"key": {
+																		Type: "string",
+																	},
+																	"operator": {
+																		Type: "string",
+																	},
+																	"values": {
+																		Type: "array",
+																		Items: &apiextv1.JSONSchemaPropsOrArray{
+																			Schema: &apiextv1.JSONSchemaProps{
+																				Type: "string",
+																			},
+																		},
+																	},
 																},
 															},
 														},
@@ -400,9 +429,23 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 														Type: "array",
 														Items: &apiextv1.JSONSchemaPropsOrArray{
 															Schema: &apiextv1.JSONSchemaProps{
-																Type: "object",
-																AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{
-																	Allows: true,
+																Type:     "object",
+																Required: []string{"key", "operator"},
+																Properties: map[string]apiextv1.JSONSchemaProps{
+																	"key": {
+																		Type: "string",
+																	},
+																	"operator": {
+																		Type: "string",
+																	},
+																	"values": {
+																		Type: "array",
+																		Items: &apiextv1.JSONSchemaPropsOrArray{
+																			Schema: &apiextv1.JSONSchemaProps{
+																				Type: "string",
+																			},
+																		},
+																	},
 																},
 															},
 														},
@@ -411,9 +454,23 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 														Type: "array",
 														Items: &apiextv1.JSONSchemaPropsOrArray{
 															Schema: &apiextv1.JSONSchemaProps{
-																Type: "object",
-																AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{
-																	Allows: true,
+																Type:     "object",
+																Required: []string{"key", "operator"},
+																Properties: map[string]apiextv1.JSONSchemaProps{
+																	"key": {
+																		Type: "string",
+																	},
+																	"operator": {
+																		Type: "string",
+																	},
+																	"values": {
+																		Type: "array",
+																		Items: &apiextv1.JSONSchemaPropsOrArray{
+																			Schema: &apiextv1.JSONSchemaProps{
+																				Type: "string",
+																			},
+																		},
+																	},
 																},
 															},
 														},
@@ -492,7 +549,7 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 					},
 					"pod_priority_class_name": {
 						Type:        "string",
-						Description: "Deprecated",
+						Description: "deprecated",
 					},
 					"podPriorityClassName": {
 						Type: "string",
@@ -579,7 +636,7 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 					},
 					"replicaLoadBalancer": {
 						Type:        "boolean",
-						Description: "Deprecated",
+						Description: "deprecated",
 					},
 					"resources": {
 						Type:     "object",
@@ -590,14 +647,12 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 								Required: []string{"cpu", "memory"},
 								Properties: map[string]apiextv1.JSONSchemaProps{
 									"cpu": {
-										Type:        "string",
-										Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0",
-										Pattern:     "^(\\d+m|\\d+(\\.\\d{1,3})?)$",
+										Type:    "string",
+										Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$",
 									},
 									"memory": {
-										Type:        "string",
-										Description: "Plain integer or fixed-point integer using one of these suffixes: E, P, T, G, M, k (with or without a tailing i). Must be greater than 0",
-										Pattern:     "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$",
+										Type:    "string",
+										Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$",
 									},
 								},
 							},
@@ -606,14 +661,12 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 								Required: []string{"cpu", "memory"},
 								Properties: map[string]apiextv1.JSONSchemaProps{
 									"cpu": {
-										Type:        "string",
-										Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0",
-										Pattern:     "^(\\d+m|\\d+(\\.\\d{1,3})?)$",
+										Type:    "string",
+										Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$",
 									},
 									"memory": {
-										Type:        "string",
-										Description: "Plain integer or fixed-point integer using one of these suffixes: E, P, T, G, M, k (with or without a tailing i). Must be greater than 0",
-										Pattern:     "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$",
+										Type:    "string",
+										Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$",
 									},
 								},
 							},
@@ -631,7 +684,8 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 						},
 					},
 					"sidecars": {
-						Type: "array",
+						Type:     "array",
+						Nullable: true,
 						Items: &apiextv1.JSONSchemaPropsOrArray{
 							Schema: &apiextv1.JSONSchemaProps{
 								Type:                   "object",
@@ -730,15 +784,14 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 					},
 					"useLoadBalancer": {
 						Type:        "boolean",
-						Description: "Deprecated",
+						Description: "deprecated",
 					},
 					"users": {
 						Type: "object",
 						AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{
 							Schema: &apiextv1.JSONSchemaProps{
-								Type:        "array",
-								Description: "Role flags specified here must not contradict each other",
-								Nullable:    true,
+								Type:     "array",
+								Nullable: true,
 								Items: &apiextv1.JSONSchemaPropsOrArray{
 									Schema: &apiextv1.JSONSchemaProps{
 										Type: "string",
@@ -907,9 +960,8 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 								},
 							},
 							"size": {
-								Type:        "string",
-								Description: "Value must not be zero",
-								Pattern:     "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$",
+								Type:    "string",
+								Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$",
 							},
 							"storageClass": {
 								Type: "string",
@@ -961,6 +1013,15 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{
 			"configuration": {
 				Type: "object",
 				Properties: map[string]apiextv1.JSONSchemaProps{
+					"crd_categories": {
+						Type:     "array",
+						Nullable: true,
+						Items: &apiextv1.JSONSchemaPropsOrArray{
+							Schema: &apiextv1.JSONSchemaProps{
+								Type: "string",
+							},
+						},
+					},
 					"docker_image": {
 						Type: "string",
 					},
@@ -968,7 +1029,8 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{
 						Type: "boolean",
 					},
 					"enable_crd_validation": {
-						Type: "boolean",
+						Type:        "boolean",
+						Description: "deprecated",
 					},
 					"enable_lazy_spilo_upgrade": {
 						Type: "boolean",
@@ -1013,7 +1075,8 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{
 						},
 					},
 					"sidecars": {
-						Type: "array",
+						Type:     "array",
+						Nullable: true,
 						Items: &apiextv1.JSONSchemaPropsOrArray{
 							Schema: &apiextv1.JSONSchemaProps{
 								Type:                   "object",
@@ -1124,7 +1187,8 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{
 								Type: "string",
 							},
 							"infrastructure_roles_secrets": {
-								Type: "array",
+								Type:     "array",
+								Nullable: true,
 								Items: &apiextv1.JSONSchemaPropsOrArray{
 									Schema: &apiextv1.JSONSchemaProps{
 										Type:     "object",
@@ -1626,18 +1690,27 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{
 	},
 }
 
-func buildCRD(name, kind, plural, short string, columns []apiextv1.CustomResourceColumnDefinition, validation apiextv1.CustomResourceValidation) *apiextv1.CustomResourceDefinition {
+func buildCRD(name, kind, plural, list, short string,
+	categories []string,
+	columns []apiextv1.CustomResourceColumnDefinition,
+	validation apiextv1.CustomResourceValidation) *apiextv1.CustomResourceDefinition {
 	return &apiextv1.CustomResourceDefinition{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: fmt.Sprintf("%s/%s", apiextv1.GroupName, apiextv1.SchemeGroupVersion.Version),
+			Kind:       "CustomResourceDefinition",
+		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name: name,
 		},
 		Spec: apiextv1.CustomResourceDefinitionSpec{
 			Group: SchemeGroupVersion.Group,
 			Names: apiextv1.CustomResourceDefinitionNames{
-				Plural:     plural,
-				ShortNames: []string{short},
 				Kind:       kind,
-				Categories: []string{"all"},
+				ListKind:   list,
+				Plural:     plural,
+				Singular:   kind,
+				ShortNames: []string{short},
+				Categories: categories,
 			},
 			Scope: apiextv1.NamespaceScoped,
 			Versions: []apiextv1.CustomResourceDefinitionVersion{
@@ -1657,33 +1730,25 @@ func buildCRD(name, kind, plural, short string, columns []apiextv1.CustomResourc
 }
 
 // PostgresCRD returns CustomResourceDefinition built from PostgresCRDResource
-func PostgresCRD(enableValidation *bool) *apiextv1.CustomResourceDefinition {
-	postgresCRDvalidation := apiextv1.CustomResourceValidation{}
-
-	if enableValidation != nil && *enableValidation {
-		postgresCRDvalidation = PostgresCRDResourceValidation
-	}
-
+func PostgresCRD(crdCategories []string) *apiextv1.CustomResourceDefinition {
 	return buildCRD(PostgresCRDResouceName,
 		PostgresCRDResourceKind,
 		PostgresCRDResourcePlural,
+		PostgresCRDResourceList,
 		PostgresCRDResourceShort,
+		crdCategories,
 		PostgresCRDResourceColumns,
-		postgresCRDvalidation)
+		PostgresCRDResourceValidation)
 }
 
 // ConfigurationCRD returns CustomResourceDefinition built from OperatorConfigCRDResource
-func ConfigurationCRD(enableValidation *bool) *apiextv1.CustomResourceDefinition {
-	opconfigCRDvalidation := apiextv1.CustomResourceValidation{}
-
-	if enableValidation != nil && *enableValidation {
-		opconfigCRDvalidation = OperatorConfigCRDResourceValidation
-	}
-
+func ConfigurationCRD(crdCategories []string) *apiextv1.CustomResourceDefinition {
 	return buildCRD(OperatorConfigCRDResourceName,
 		OperatorConfigCRDResouceKind,
 		OperatorConfigCRDResourcePlural,
+		OperatorConfigCRDResourceList,
 		OperatorConfigCRDResourceShort,
+		crdCategories,
 		OperatorConfigCRDResourceColumns,
-		opconfigCRDvalidation)
+		OperatorConfigCRDResourceValidation)
 }
diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
index 124278e70..1298c6834 100644
--- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
+++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
@@ -221,6 +221,7 @@ type OperatorLogicalBackupConfiguration struct {
 type OperatorConfigurationData struct {
 	EnableCRDRegistration      *bool                              `json:"enable_crd_registration,omitempty"`
 	EnableCRDValidation        *bool                              `json:"enable_crd_validation,omitempty"`
+	CRDCategories              []string                           `json:"crd_categories,omitempty"`
 	EnableLazySpiloUpgrade     bool                               `json:"enable_lazy_spilo_upgrade,omitempty"`
 	EnablePgVersionEnvVar      bool                               `json:"enable_pgversion_env_var,omitempty"`
 	EnableSpiloWalPathCompat   bool                               `json:"enable_spilo_wal_path_compat,omitempty"`
diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
index cfa315358..d960cd102 100644
--- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
+++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
@@ -377,6 +377,11 @@ func (in *OperatorConfigurationData) DeepCopyInto(out *OperatorConfigurationData
 		*out = new(bool)
 		**out = **in
 	}
+	if in.CRDCategories != nil {
+		in, out := &in.CRDCategories, &out.CRDCategories
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	if in.ShmVolume != nil {
 		in, out := &in.ShmVolume, &out.ShmVolume
 		*out = new(bool)
diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go
index 54e50a45f..de0dec69f 100644
--- a/pkg/controller/controller.go
+++ b/pkg/controller/controller.go
@@ -310,7 +310,7 @@ func (c *Controller) initController() {
 
 	if configObjectName := os.Getenv("POSTGRES_OPERATOR_CONFIGURATION_OBJECT"); configObjectName != "" {
 		if c.opConfig.EnableCRDRegistration != nil && *c.opConfig.EnableCRDRegistration {
-			if err := c.createConfigurationCRD(c.opConfig.EnableCRDValidation); err != nil {
+			if err := c.createConfigurationCRD(); err != nil {
 				c.logger.Fatalf("could not register Operator Configuration CustomResourceDefinition: %v", err)
 			}
 		}
@@ -328,7 +328,7 @@ func (c *Controller) initController() {
 	c.modifyConfigFromEnvironment()
 
 	if c.opConfig.EnableCRDRegistration != nil && *c.opConfig.EnableCRDRegistration {
-		if err := c.createPostgresCRD(c.opConfig.EnableCRDValidation); err != nil {
+		if err := c.createPostgresCRD(); err != nil {
 			c.logger.Fatalf("could not register Postgres CustomResourceDefinition: %v", err)
 		}
 	}
diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go
index 20958dd7b..fbf12bfb9 100644
--- a/pkg/controller/operator_config.go
+++ b/pkg/controller/operator_config.go
@@ -35,6 +35,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	// general config
 	result.EnableCRDRegistration = util.CoalesceBool(fromCRD.EnableCRDRegistration, util.True())
 	result.EnableCRDValidation = util.CoalesceBool(fromCRD.EnableCRDValidation, util.True())
+	result.CRDCategories = util.CoalesceStrArr(fromCRD.CRDCategories, []string{"all"})
 	result.EnableLazySpiloUpgrade = fromCRD.EnableLazySpiloUpgrade
 	result.EnablePgVersionEnvVar = fromCRD.EnablePgVersionEnvVar
 	result.EnableSpiloWalPathCompat = fromCRD.EnableSpiloWalPathCompat
diff --git a/pkg/controller/util.go b/pkg/controller/util.go
index 563734ac9..bca8082f6 100644
--- a/pkg/controller/util.go
+++ b/pkg/controller/util.go
@@ -53,28 +53,35 @@ func (c *Controller) clusterWorkerID(clusterName spec.NamespacedName) uint32 {
 	return c.clusterWorkers[clusterName]
 }
 
-func (c *Controller) createOperatorCRD(crd *apiextv1.CustomResourceDefinition) error {
-	if _, err := c.KubeClient.CustomResourceDefinitions().Create(context.TODO(), crd, metav1.CreateOptions{}); err != nil {
-		if k8sutil.ResourceAlreadyExists(err) {
-			c.logger.Infof("customResourceDefinition %q is already registered and will only be updated", crd.Name)
-
-			patch, err := json.Marshal(crd)
-			if err != nil {
-				return fmt.Errorf("could not marshal new customResourceDefintion: %v", err)
-			}
-			if _, err := c.KubeClient.CustomResourceDefinitions().Patch(
-				context.TODO(), crd.Name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil {
-				return fmt.Errorf("could not update customResourceDefinition: %v", err)
-			}
-		} else {
-			c.logger.Errorf("could not create customResourceDefinition %q: %v", crd.Name, err)
+func (c *Controller) createOperatorCRD(desiredCrd *apiextv1.CustomResourceDefinition) error {
+	crd, err := c.KubeClient.CustomResourceDefinitions().Get(context.TODO(), desiredCrd.Name, metav1.GetOptions{})
+	if k8sutil.ResourceNotFound(err) {
+		if _, err := c.KubeClient.CustomResourceDefinitions().Create(context.TODO(), desiredCrd, metav1.CreateOptions{}); err != nil {
+			return fmt.Errorf("could not create customResourceDefinition %q: %v", desiredCrd.Name, err)
+		}
+	}
+	if err != nil {
+		c.logger.Errorf("could not get customResourceDefinition %q: %v", desiredCrd.Name, err)
+	}
+	if crd != nil {
+		c.logger.Infof("customResourceDefinition %q is already registered and will only be updated", crd.Name)
+		// copy annotations and labels from existing CRD since we do not define them
+		desiredCrd.Annotations = crd.Annotations
+		desiredCrd.Labels = crd.Labels
+		patch, err := json.Marshal(desiredCrd)
+		if err != nil {
+			return fmt.Errorf("could not marshal new customResourceDefintion %q: %v", desiredCrd.Name, err)
+		}
+		if _, err := c.KubeClient.CustomResourceDefinitions().Patch(
+			context.TODO(), crd.Name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil {
+			return fmt.Errorf("could not update customResourceDefinition %q: %v", crd.Name, err)
 		}
 	} else {
 		c.logger.Infof("customResourceDefinition %q has been registered", crd.Name)
 	}
 
 	return wait.Poll(c.config.CRDReadyWaitInterval, c.config.CRDReadyWaitTimeout, func() (bool, error) {
-		c, err := c.KubeClient.CustomResourceDefinitions().Get(context.TODO(), crd.Name, metav1.GetOptions{})
+		c, err := c.KubeClient.CustomResourceDefinitions().Get(context.TODO(), desiredCrd.Name, metav1.GetOptions{})
 		if err != nil {
 			return false, err
 		}
@@ -96,12 +103,12 @@ func (c *Controller) createOperatorCRD(crd *apiextv1.CustomResourceDefinition) e
 	})
 }
 
-func (c *Controller) createPostgresCRD(enableValidation *bool) error {
-	return c.createOperatorCRD(acidv1.PostgresCRD(enableValidation))
+func (c *Controller) createPostgresCRD() error {
+	return c.createOperatorCRD(acidv1.PostgresCRD(c.opConfig.CRDCategories))
 }
 
-func (c *Controller) createConfigurationCRD(enableValidation *bool) error {
-	return c.createOperatorCRD(acidv1.ConfigurationCRD(enableValidation))
+func (c *Controller) createConfigurationCRD() error {
+	return c.createOperatorCRD(acidv1.ConfigurationCRD(c.opConfig.CRDCategories))
 }
 
 func readDecodedRole(s string) (*spec.PgUser, error) {
diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go
index fa5d3b770..0dc1004a7 100644
--- a/pkg/util/config/config.go
+++ b/pkg/util/config/config.go
@@ -20,6 +20,7 @@ type CRD struct {
 	RepairPeriod          time.Duration `name:"repair_period" default:"5m"`
 	EnableCRDRegistration *bool         `name:"enable_crd_registration" default:"true"`
 	EnableCRDValidation   *bool         `name:"enable_crd_validation" default:"true"`
+	CRDCategories         []string      `name:"crd_categories" default:"all"`
 }
 
 // Resources describes kubernetes resource specific configuration parameters

From 2d2386f51996164999028332a994798e29462ca7 Mon Sep 17 00:00:00 2001
From: Emil Bostijancic <emil.bostijancic@gmail.com>
Date: Wed, 23 Feb 2022 15:30:23 +0100
Subject: [PATCH 21/58] fix(ui): uses correct default URL value for
 WALE_S3_ENDPOINT (#1793)

Co-authored-by: Emil Bostijancic <emil.bostijancic@gmaail.com>
---
 ui/operator_ui/main.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ui/operator_ui/main.py b/ui/operator_ui/main.py
index 5fbb6d24e..dc207d35e 100644
--- a/ui/operator_ui/main.py
+++ b/ui/operator_ui/main.py
@@ -98,7 +98,7 @@ COST_MEMORY = 30.5 * 24 * float(getenv('COST_MEMORY', 0.014375))  # Memory GB m5
 
 WALE_S3_ENDPOINT = getenv(
     'WALE_S3_ENDPOINT',
-    'https+path://s3-eu-central-1.amazonaws.com:443',
+    'https+path://s3.eu-central-1.amazonaws.com:443',
 )
 
 USE_AWS_INSTANCE_PROFILE = (

From 46547c4088cc3728a7ce57eb2d777de0e3bf7343 Mon Sep 17 00:00:00 2001
From: Felix Kunde <felix-kunde@gmx.de>
Date: Fri, 25 Feb 2022 09:33:04 +0100
Subject: [PATCH 22/58] do not recreate pods if previous Patroni API calls fail
 (#1767)

* do not recreate pods if previous Patroni API calls fail
* move retry reads against Patroni API to pod.go
* remove final failover check in node affinity test
* make test_min_resource_limits more robust
---
 e2e/tests/test_e2e.py | 272 +++++++++++++++++++++---------------------
 pkg/cluster/pod.go    |  93 ++++++++-------
 pkg/cluster/sync.go   |  55 +++++----
 3 files changed, 220 insertions(+), 200 deletions(-)

diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py
index 77c14e272..d1db4991b 100644
--- a/e2e/tests/test_e2e.py
+++ b/e2e/tests/test_e2e.py
@@ -205,7 +205,7 @@ class EndToEndTestCase(unittest.TestCase):
                 "enable_team_member_deprecation": "true",
                 "role_deletion_suffix": "_delete_me",
                 "resync_period": "15s",
-                "repair_period": "10s",
+                "repair_period": "15s",
             },
         }
         k8s.update_config(enable_postgres_team_crd)
@@ -296,6 +296,133 @@ class EndToEndTestCase(unittest.TestCase):
         self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"},
                              "Operator does not get in sync")
 
+    @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
+    def test_config_update(self):
+        '''
+            Change Postgres config under Spec.Postgresql.Parameters and Spec.Patroni
+            and query Patroni config endpoint to check if manifest changes got applied
+            via restarting cluster through Patroni's rest api
+        '''
+        k8s = self.k8s
+        leader = k8s.get_cluster_leader_pod()
+        replica = k8s.get_cluster_replica_pod()
+        masterCreationTimestamp = leader.metadata.creation_timestamp
+        replicaCreationTimestamp = replica.metadata.creation_timestamp
+        new_max_connections_value = "50"
+
+        # adjust Postgres config
+        pg_patch_config = {
+            "spec": {
+                "postgresql": {
+                    "parameters": {
+                        "max_connections": new_max_connections_value
+                     }
+                 },
+                 "patroni": {
+                    "slots": {
+                        "test_slot": {
+                            "type": "physical"
+                        }
+                    },
+                    "ttl": 29,
+                    "loop_wait": 9,
+                    "retry_timeout": 9,
+                    "synchronous_mode": True
+                 }
+            }
+        }
+
+        try:
+            k8s.api.custom_objects_api.patch_namespaced_custom_object(
+                "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_config)
+            
+            self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
+
+            def compare_config():
+                effective_config = k8s.patroni_rest(leader.metadata.name, "config")
+                desired_config = pg_patch_config["spec"]["patroni"]
+                desired_parameters = pg_patch_config["spec"]["postgresql"]["parameters"]
+                effective_parameters = effective_config["postgresql"]["parameters"]
+                self.assertEqual(desired_parameters["max_connections"], effective_parameters["max_connections"],
+                            "max_connections not updated")
+                self.assertTrue(effective_config["slots"] is not None, "physical replication slot not added")
+                self.assertEqual(desired_config["ttl"], effective_config["ttl"],
+                            "ttl not updated")
+                self.assertEqual(desired_config["loop_wait"], effective_config["loop_wait"],
+                            "loop_wait not updated")
+                self.assertEqual(desired_config["retry_timeout"], effective_config["retry_timeout"],
+                            "retry_timeout not updated")
+                self.assertEqual(desired_config["synchronous_mode"], effective_config["synchronous_mode"],
+                            "synchronous_mode not updated")
+                return True
+
+            # check if Patroni config has been updated
+            self.eventuallyTrue(compare_config, "Postgres config not applied")
+
+            # make sure that pods were not recreated
+            leader = k8s.get_cluster_leader_pod()
+            replica = k8s.get_cluster_replica_pod()
+            self.assertEqual(masterCreationTimestamp, leader.metadata.creation_timestamp,
+                            "Master pod creation timestamp is updated")
+            self.assertEqual(replicaCreationTimestamp, replica.metadata.creation_timestamp,
+                            "Master pod creation timestamp is updated")
+
+            # query max_connections setting
+            setting_query = """
+               SELECT setting
+                 FROM pg_settings
+                WHERE name = 'max_connections';
+            """
+            self.eventuallyEqual(lambda: self.query_database(leader.metadata.name, "postgres", setting_query)[0], new_max_connections_value,
+                "New max_connections setting not applied on master", 10, 5)
+            self.eventuallyNotEqual(lambda: self.query_database(replica.metadata.name, "postgres", setting_query)[0], new_max_connections_value,
+                "Expected max_connections not to be updated on replica since Postgres was restarted there first", 10, 5)
+
+            # the next sync should restart the replica because it has pending_restart flag set
+            # force next sync by deleting the operator pod
+            k8s.delete_operator_pod()
+            self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
+
+            self.eventuallyEqual(lambda: self.query_database(replica.metadata.name, "postgres", setting_query)[0], new_max_connections_value,
+                "New max_connections setting not applied on replica", 10, 5)
+
+            # decrease max_connections again
+            # this time restart will be correct and new value should appear on both instances
+            lower_max_connections_value = "30"
+            pg_patch_max_connections = {
+                "spec": {
+                    "postgresql": {
+                        "parameters": {
+                            "max_connections": lower_max_connections_value
+                        }
+                    }
+                }
+            }
+
+            k8s.api.custom_objects_api.patch_namespaced_custom_object(
+                "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_max_connections)
+
+            self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
+
+            # check Patroni config again
+            pg_patch_config["spec"]["postgresql"]["parameters"]["max_connections"] = lower_max_connections_value
+            self.eventuallyTrue(compare_config, "Postgres config not applied")
+
+            # and query max_connections setting again
+            self.eventuallyEqual(lambda: self.query_database(leader.metadata.name, "postgres", setting_query)[0], lower_max_connections_value,
+                "Previous max_connections setting not applied on master", 10, 5)
+            self.eventuallyEqual(lambda: self.query_database(replica.metadata.name, "postgres", setting_query)[0], lower_max_connections_value,
+                "Previous max_connections setting not applied on replica", 10, 5)
+
+        except timeout_decorator.TimeoutError:
+            print('Operator log: {}'.format(k8s.get_operator_log()))
+            raise
+
+        # make sure cluster is in a good state for further tests
+        self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
+        self.eventuallyEqual(lambda: k8s.count_running_pods(), 2,
+                             "No 2 pods running")
+
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
     def test_cross_namespace_secrets(self):
         '''
@@ -794,7 +921,11 @@ class EndToEndTestCase(unittest.TestCase):
         Lower resource limits below configured minimum and let operator fix it
         '''
         k8s = self.k8s
-        # self.eventuallyEqual(lambda: k8s.pg_get_status(), "Running", "Cluster not healthy at start")
+        cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster'
+
+        # get nodes of master and replica(s) (expected target of new master)
+        _, replica_nodes = k8s.get_pg_nodes(cluster_label)
+        self.assertNotEqual(replica_nodes, [])
 
         # configure minimum boundaries for CPU and memory limits
         minCPULimit = '503m'
@@ -827,7 +958,9 @@ class EndToEndTestCase(unittest.TestCase):
             "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_resources)
         self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
 
-        self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No two pods running after lazy rolling upgrade")
+        # wait for switched over
+        k8s.wait_for_pod_failover(replica_nodes, 'spilo-role=master,' + cluster_label)
+        k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label)
         self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members()), 2, "Postgres status did not enter running")
 
         def verify_pod_limits():
@@ -968,15 +1101,15 @@ class EndToEndTestCase(unittest.TestCase):
             self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
 
             # node affinity change should cause another rolling update and relocation of replica
-            k8s.wait_for_pod_failover(replica_nodes, 'spilo-role=replica,' + cluster_label)
             k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label)
+            k8s.wait_for_pod_start('spilo-role=master,' + cluster_label)
 
         except timeout_decorator.TimeoutError:
             print('Operator log: {}'.format(k8s.get_operator_log()))
             raise
 
         # toggle pod anti affinity to make sure replica and master run on separate nodes
-        self.assert_distributed_pods(replica_nodes)
+        self.assert_distributed_pods(master_nodes)
 
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
     def test_node_readiness_label(self):
@@ -1192,133 +1325,6 @@ class EndToEndTestCase(unittest.TestCase):
         self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "postgres", user_query)), 2,
             "Found incorrect number of rotation users", 10, 5)
 
-    @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
-    def test_patroni_config_update(self):
-        '''
-            Change Postgres config under Spec.Postgresql.Parameters and Spec.Patroni
-            and query Patroni config endpoint to check if manifest changes got applied
-            via restarting cluster through Patroni's rest api
-        '''
-        k8s = self.k8s
-        leader = k8s.get_cluster_leader_pod()
-        replica = k8s.get_cluster_replica_pod()
-        masterCreationTimestamp = leader.metadata.creation_timestamp
-        replicaCreationTimestamp = replica.metadata.creation_timestamp
-        new_max_connections_value = "50"
-
-        # adjust Postgres config
-        pg_patch_config = {
-            "spec": {
-                "postgresql": {
-                    "parameters": {
-                        "max_connections": new_max_connections_value
-                     }
-                 },
-                 "patroni": {
-                    "slots": {
-                        "test_slot": {
-                            "type": "physical"
-                        }
-                    },
-                    "ttl": 29,
-                    "loop_wait": 9,
-                    "retry_timeout": 9,
-                    "synchronous_mode": True
-                 }
-            }
-        }
-
-        try:
-            k8s.api.custom_objects_api.patch_namespaced_custom_object(
-                "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_config)
-            
-            self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
-
-            def compare_config():
-                effective_config = k8s.patroni_rest(leader.metadata.name, "config")
-                desired_config = pg_patch_config["spec"]["patroni"]
-                desired_parameters = pg_patch_config["spec"]["postgresql"]["parameters"]
-                effective_parameters = effective_config["postgresql"]["parameters"]
-                self.assertEqual(desired_parameters["max_connections"], effective_parameters["max_connections"],
-                            "max_connections not updated")
-                self.assertTrue(effective_config["slots"] is not None, "physical replication slot not added")
-                self.assertEqual(desired_config["ttl"], effective_config["ttl"],
-                            "ttl not updated")
-                self.assertEqual(desired_config["loop_wait"], effective_config["loop_wait"],
-                            "loop_wait not updated")
-                self.assertEqual(desired_config["retry_timeout"], effective_config["retry_timeout"],
-                            "retry_timeout not updated")
-                self.assertEqual(desired_config["synchronous_mode"], effective_config["synchronous_mode"],
-                            "synchronous_mode not updated")
-                return True
-
-            # check if Patroni config has been updated
-            self.eventuallyTrue(compare_config, "Postgres config not applied")
-
-            # make sure that pods were not recreated
-            leader = k8s.get_cluster_leader_pod()
-            replica = k8s.get_cluster_replica_pod()
-            self.assertEqual(masterCreationTimestamp, leader.metadata.creation_timestamp,
-                            "Master pod creation timestamp is updated")
-            self.assertEqual(replicaCreationTimestamp, replica.metadata.creation_timestamp,
-                            "Master pod creation timestamp is updated")
-
-            # query max_connections setting
-            setting_query = """
-               SELECT setting
-                 FROM pg_settings
-                WHERE name = 'max_connections';
-            """
-            self.eventuallyEqual(lambda: self.query_database(leader.metadata.name, "postgres", setting_query)[0], new_max_connections_value,
-                "New max_connections setting not applied on master", 10, 5)
-            self.eventuallyNotEqual(lambda: self.query_database(replica.metadata.name, "postgres", setting_query)[0], new_max_connections_value,
-                "Expected max_connections not to be updated on replica since Postgres was restarted there first", 10, 5)
-
-            # the next sync should restart the replica because it has pending_restart flag set
-            # force next sync by deleting the operator pod
-            k8s.delete_operator_pod()
-            self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
-
-            self.eventuallyEqual(lambda: self.query_database(replica.metadata.name, "postgres", setting_query)[0], new_max_connections_value,
-                "New max_connections setting not applied on replica", 10, 5)
-
-            # decrease max_connections again
-            # this time restart will be correct and new value should appear on both instances
-            lower_max_connections_value = "30"
-            pg_patch_max_connections = {
-                "spec": {
-                    "postgresql": {
-                        "parameters": {
-                            "max_connections": lower_max_connections_value
-                        }
-                    }
-                }
-            }
-
-            k8s.api.custom_objects_api.patch_namespaced_custom_object(
-                "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_max_connections)
-
-            self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
-
-            # check Patroni config again
-            pg_patch_config["spec"]["postgresql"]["parameters"]["max_connections"] = lower_max_connections_value
-            self.eventuallyTrue(compare_config, "Postgres config not applied")
-
-            # and query max_connections setting again
-            self.eventuallyEqual(lambda: self.query_database(leader.metadata.name, "postgres", setting_query)[0], lower_max_connections_value,
-                "Previous max_connections setting not applied on master", 10, 5)
-            self.eventuallyEqual(lambda: self.query_database(replica.metadata.name, "postgres", setting_query)[0], lower_max_connections_value,
-                "Previous max_connections setting not applied on replica", 10, 5)
-
-        except timeout_decorator.TimeoutError:
-            print('Operator log: {}'.format(k8s.get_operator_log()))
-            raise
-
-        # make sure cluster is in a good state for further tests
-        self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
-        self.eventuallyEqual(lambda: k8s.count_running_pods(), 2,
-                             "No 2 pods running")
-
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
     def test_rolling_update_flag(self):
         '''
@@ -1405,7 +1411,7 @@ class EndToEndTestCase(unittest.TestCase):
             "data": {
                 "pod_label_wait_timeout": "2s",
                 "resync_period": "30s",
-                "repair_period": "10s",
+                "repair_period": "30s",
             }
         }
 
diff --git a/pkg/cluster/pod.go b/pkg/cluster/pod.go
index 805112e29..9e8ded844 100644
--- a/pkg/cluster/pod.go
+++ b/pkg/cluster/pod.go
@@ -13,6 +13,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 
+	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
 	"github.com/zalando/postgres-operator/pkg/spec"
 	"github.com/zalando/postgres-operator/pkg/util"
 	"github.com/zalando/postgres-operator/pkg/util/patroni"
@@ -349,6 +350,54 @@ func (c *Cluster) MigrateReplicaPod(podName spec.NamespacedName, fromNodeName st
 	return nil
 }
 
+func (c *Cluster) getPatroniConfig(pod *v1.Pod) (acidv1.Patroni, map[string]string, error) {
+	var (
+		patroniConfig acidv1.Patroni
+		pgParameters  map[string]string
+	)
+	podName := util.NameFromMeta(pod.ObjectMeta)
+	err := retryutil.Retry(1*time.Second, 5*time.Second,
+		func() (bool, error) {
+			var err error
+			patroniConfig, pgParameters, err = c.patroni.GetConfig(pod)
+
+			if err != nil {
+				return false, err
+			}
+			return true, nil
+		},
+	)
+
+	if err != nil {
+		return acidv1.Patroni{}, nil, fmt.Errorf("could not get Postgres config from pod %s: %v", podName, err)
+	}
+
+	return patroniConfig, pgParameters, nil
+}
+
+func (c *Cluster) getPatroniMemberData(pod *v1.Pod) (patroni.MemberData, error) {
+	var memberData patroni.MemberData
+	err := retryutil.Retry(1*time.Second, 5*time.Second,
+		func() (bool, error) {
+			var err error
+			memberData, err = c.patroni.GetMemberData(pod)
+
+			if err != nil {
+				return false, err
+			}
+			return true, nil
+		},
+	)
+	if err != nil {
+		return patroni.MemberData{}, fmt.Errorf("could not get member data: %v", err)
+	}
+	if memberData.State == "creating replica" {
+		return patroni.MemberData{}, fmt.Errorf("replica currently being initialized")
+	}
+
+	return memberData, nil
+}
+
 func (c *Cluster) recreatePod(podName spec.NamespacedName) (*v1.Pod, error) {
 	ch := c.registerPodSubscriber(podName)
 	defer c.unregisterPodSubscriber(podName)
@@ -380,54 +429,10 @@ func (c *Cluster) recreatePod(podName spec.NamespacedName) (*v1.Pod, error) {
 	return pod, nil
 }
 
-func (c *Cluster) isSafeToRecreatePods(pods []v1.Pod) bool {
-
-	/*
-	 Operator should not re-create pods if there is at least one replica being bootstrapped
-	 because Patroni might use other replicas to take basebackup from (see Patroni's "clonefrom" tag).
-
-	 XXX operator cannot forbid replica re-init, so we might still fail if re-init is started
-	 after this check succeeds but before a pod is re-created
-	*/
-	for _, pod := range pods {
-		c.logger.Debugf("name=%s phase=%s ip=%s", pod.Name, pod.Status.Phase, pod.Status.PodIP)
-	}
-
-	for _, pod := range pods {
-
-		var data patroni.MemberData
-
-		err := retryutil.Retry(1*time.Second, 5*time.Second,
-			func() (bool, error) {
-				var err error
-				data, err = c.patroni.GetMemberData(&pod)
-
-				if err != nil {
-					return false, err
-				}
-				return true, nil
-			},
-		)
-
-		if err != nil {
-			c.logger.Errorf("failed to get Patroni state for pod: %s", err)
-			return false
-		} else if data.State == "creating replica" {
-			c.logger.Warningf("cannot re-create replica %s: it is currently being initialized", pod.Name)
-			return false
-		}
-	}
-	return true
-}
-
 func (c *Cluster) recreatePods(pods []v1.Pod, switchoverCandidates []spec.NamespacedName) error {
 	c.setProcessName("starting to recreate pods")
 	c.logger.Infof("there are %d pods in the cluster to recreate", len(pods))
 
-	if !c.isSafeToRecreatePods(pods) {
-		return fmt.Errorf("postpone pod recreation until next Sync: recreation is unsafe because pods are being initialized")
-	}
-
 	var (
 		masterPod, newMasterPod *v1.Pod
 	)
diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go
index c00f0a189..24c68dd54 100644
--- a/pkg/cluster/sync.go
+++ b/pkg/cluster/sync.go
@@ -277,6 +277,7 @@ func (c *Cluster) syncStatefulSet() error {
 		restartMasterFirst bool
 	)
 	podsToRecreate := make([]v1.Pod, 0)
+	isSafeToRecreatePods := true
 	switchoverCandidates := make([]spec.NamespacedName, 0)
 
 	pods, err := c.listPods()
@@ -402,21 +403,20 @@ func (c *Cluster) syncStatefulSet() error {
 	// get Postgres config, compare with manifest and update via Patroni PATCH endpoint if it differs
 	// Patroni's config endpoint is just a "proxy" to DCS. It is enough to patch it only once and it doesn't matter which pod is used.
 	for i, pod := range pods {
-		emptyPatroniConfig := acidv1.Patroni{}
-		podName := util.NameFromMeta(pods[i].ObjectMeta)
-		patroniConfig, pgParameters, err := c.patroni.GetConfig(&pod)
+		patroniConfig, pgParameters, err := c.getPatroniConfig(&pod)
 		if err != nil {
-			c.logger.Warningf("could not get Postgres config from pod %s: %v", podName, err)
+			c.logger.Warningf("%v", err)
+			isSafeToRecreatePods = false
 			continue
 		}
 		restartWait = patroniConfig.LoopWait
 
 		// empty config probably means cluster is not fully initialized yet, e.g. restoring from backup
 		// do not attempt a restart
-		if !reflect.DeepEqual(patroniConfig, emptyPatroniConfig) || len(pgParameters) > 0 {
+		if !reflect.DeepEqual(patroniConfig, acidv1.Patroni{}) || len(pgParameters) > 0 {
 			restartMasterFirst, err = c.checkAndSetGlobalPostgreSQLConfiguration(&pod, patroniConfig, pgParameters)
 			if err != nil {
-				c.logger.Warningf("could not set PostgreSQL configuration options for pod %s: %v", podName, err)
+				c.logger.Warningf("could not set PostgreSQL configuration options for pod %s: %v", pods[i].Name, err)
 				continue
 			}
 			// it could take up to LoopWait to apply the config
@@ -437,50 +437,59 @@ func (c *Cluster) syncStatefulSet() error {
 			remainingPods = append(remainingPods, &pods[i])
 			continue
 		}
-		c.restartInstance(&pod, restartWait)
+		if err = c.restartInstance(&pod, restartWait); err != nil {
+			c.logger.Errorf("%v", err)
+			isSafeToRecreatePods = false
+		}
 	}
 
 	// in most cases only the master should be left to restart
 	if len(remainingPods) > 0 {
 		for _, remainingPod := range remainingPods {
-			c.restartInstance(remainingPod, restartWait)
+			if err = c.restartInstance(remainingPod, restartWait); err != nil {
+				c.logger.Errorf("%v", err)
+				isSafeToRecreatePods = false
+			}
 		}
 	}
 
 	// if we get here we also need to re-create the pods (either leftovers from the old
 	// statefulset or those that got their configuration from the outdated statefulset)
 	if len(podsToRecreate) > 0 {
-		c.logger.Debugln("performing rolling update")
-		c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", "Performing rolling update")
-		if err := c.recreatePods(podsToRecreate, switchoverCandidates); err != nil {
-			return fmt.Errorf("could not recreate pods: %v", err)
+		if isSafeToRecreatePods {
+			c.logger.Debugln("performing rolling update")
+			c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", "Performing rolling update")
+			if err := c.recreatePods(podsToRecreate, switchoverCandidates); err != nil {
+				return fmt.Errorf("could not recreate pods: %v", err)
+			}
+			c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", "Rolling update done - pods have been recreated")
+		} else {
+			c.logger.Warningf("postpone pod recreation until next sync")
 		}
-		c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", "Rolling update done - pods have been recreated")
 	}
 	return nil
 }
 
-func (c *Cluster) restartInstance(pod *v1.Pod, restartWait uint32) {
+func (c *Cluster) restartInstance(pod *v1.Pod, restartWait uint32) error {
+	// if the config update requires a restart, call Patroni restart
 	podName := util.NameFromMeta(pod.ObjectMeta)
 	role := PostgresRole(pod.Labels[c.OpConfig.PodRoleLabel])
-
-	// if the config update requires a restart, call Patroni restart
-	memberData, err := c.patroni.GetMemberData(pod)
+	memberData, err := c.getPatroniMemberData(pod)
 	if err != nil {
-		c.logger.Debugf("could not get member data of %s pod %s - skipping possible restart attempt: %v", role, podName, err)
-		return
+		return fmt.Errorf("could not restart Postgres in %s pod %s: %v", role, podName, err)
 	}
 
 	// do restart only when it is pending
 	if memberData.PendingRestart {
-		c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", fmt.Sprintf("restarting Postgres server within %s pod %s", role, pod.Name))
+		c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", fmt.Sprintf("restarting Postgres server within %s pod %s", role, podName))
 		if err := c.patroni.Restart(pod); err != nil {
-			c.logger.Warningf("could not restart Postgres server within %s pod %s: %v", role, podName, err)
-			return
+			return err
 		}
 		time.Sleep(time.Duration(restartWait) * time.Second)
-		c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", fmt.Sprintf("Postgres server restart done for %s pod %s", role, pod.Name))
+		c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", fmt.Sprintf("Postgres server restart done for %s pod %s", role, podName))
 	}
+
+	return nil
 }
 
 // AnnotationsToPropagate get the annotations to update if required

From 06c28da97dcd58250e125ab4b055ec51b47c08f0 Mon Sep 17 00:00:00 2001
From: Menzorg <menzorg@gmail.com>
Date: Fri, 25 Feb 2022 19:21:42 +0300
Subject: [PATCH 23/58] synchronous_node_count support (#1484)

* synchronous_node_count support
* notification about Patroni image version
* default synchronous_node_count to 1

Co-authored-by: Felix Kunde <felix-kunde@gmx.de>
---
 charts/postgres-operator/crds/postgresqls.yaml | 2 ++
 docs/reference/cluster_manifest.md             | 3 +++
 manifests/complete-postgres-manifest.yaml      | 1 +
 manifests/postgresql.crd.yaml                  | 2 ++
 pkg/apis/acid.zalan.do/v1/crds.go              | 3 +++
 pkg/apis/acid.zalan.do/v1/postgresql_type.go   | 1 +
 pkg/cluster/k8sres.go                          | 4 ++++
 pkg/cluster/k8sres_test.go                     | 3 ++-
 8 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml
index 83043084c..bcd212afd 100644
--- a/charts/postgres-operator/crds/postgresqls.yaml
+++ b/charts/postgres-operator/crds/postgresqls.yaml
@@ -343,6 +343,8 @@ spec:
                     type: boolean
                   synchronous_mode_strict:
                     type: boolean
+                  synchronous_node_count:
+                    type: integer
                   ttl:
                     type: integer
               podAnnotations:
diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md
index 9d65062c0..ae72d6450 100644
--- a/docs/reference/cluster_manifest.md
+++ b/docs/reference/cluster_manifest.md
@@ -303,6 +303,9 @@ explanation of `ttl` and `loop_wait` parameters.
 * **synchronous_mode_strict**
   Patroni `synchronous_mode_strict` parameter value. Can be used in addition to `synchronous_mode`. The default is set to `false`. Optional.
 
+* **synchronous_node_count**
+  Patroni `synchronous_node_count` parameter value. Note, this option is only available for Spilo images with Patroni 2.0+. The default is set to `1`. Optional.
+  
 ## Postgres container resources
 
 Those parameters define [CPU and memory requests and limits](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/)
diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml
index c150b616d..577b2711b 100644
--- a/manifests/complete-postgres-manifest.yaml
+++ b/manifests/complete-postgres-manifest.yaml
@@ -119,6 +119,7 @@ spec:
     retry_timeout: 10
     synchronous_mode: false
     synchronous_mode_strict: false
+    synchronous_node_count: 1
     maximum_lag_on_failover: 33554432
 
 # restore a Postgres DB with point-in-time-recovery
diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml
index 88144dc05..1f1b064a7 100644
--- a/manifests/postgresql.crd.yaml
+++ b/manifests/postgresql.crd.yaml
@@ -341,6 +341,8 @@ spec:
                     type: boolean
                   synchronous_mode_strict:
                     type: boolean
+                  synchronous_node_count:
+                    type: integer
                   ttl:
                     type: integer
               podAnnotations:
diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go
index 844ccef04..89a04b180 100644
--- a/pkg/apis/acid.zalan.do/v1/crds.go
+++ b/pkg/apis/acid.zalan.do/v1/crds.go
@@ -534,6 +534,9 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 							"synchronous_mode_strict": {
 								Type: "boolean",
 							},
+							"synchronous_node_count": {
+								Type: "integer",
+							},
 							"ttl": {
 								Type: "integer",
 							},
diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go
index 45f139383..a97ad35a3 100644
--- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go
+++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go
@@ -164,6 +164,7 @@ type Patroni struct {
 	Slots                 map[string]map[string]string `json:"slots,omitempty"`
 	SynchronousMode       bool                         `json:"synchronous_mode,omitempty"`
 	SynchronousModeStrict bool                         `json:"synchronous_mode_strict,omitempty"`
+	SynchronousNodeCount  uint32                       `json:"synchronous_node_count,omitempty" defaults:1`
 }
 
 // StandbyDescription contains s3 wal path
diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go
index fda192df8..a42aa2d06 100644
--- a/pkg/cluster/k8sres.go
+++ b/pkg/cluster/k8sres.go
@@ -50,6 +50,7 @@ type patroniDCS struct {
 	MaximumLagOnFailover     float32                      `json:"maximum_lag_on_failover,omitempty"`
 	SynchronousMode          bool                         `json:"synchronous_mode,omitempty"`
 	SynchronousModeStrict    bool                         `json:"synchronous_mode_strict,omitempty"`
+	SynchronousNodeCount     uint32                       `json:"synchronous_node_count,omitempty"`
 	PGBootstrapConfiguration map[string]interface{}       `json:"postgresql,omitempty"`
 	Slots                    map[string]map[string]string `json:"slots,omitempty"`
 }
@@ -262,6 +263,9 @@ PatroniInitDBParams:
 	if patroni.SynchronousModeStrict {
 		config.Bootstrap.DCS.SynchronousModeStrict = patroni.SynchronousModeStrict
 	}
+	if patroni.SynchronousNodeCount >= 1 {
+		config.Bootstrap.DCS.SynchronousNodeCount = patroni.SynchronousNodeCount
+	}
 
 	config.PgLocalConfiguration = make(map[string]interface{})
 
diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go
index c3c4a56bd..503959f28 100644
--- a/pkg/cluster/k8sres_test.go
+++ b/pkg/cluster/k8sres_test.go
@@ -91,11 +91,12 @@ func TestGenerateSpiloJSONConfiguration(t *testing.T) {
 				MaximumLagOnFailover:  33554432,
 				SynchronousMode:       true,
 				SynchronousModeStrict: true,
+				SynchronousNodeCount:  1,
 				Slots:                 map[string]map[string]string{"permanent_logical_1": {"type": "logical", "database": "foo", "plugin": "pgoutput"}},
 			},
 			role:     "zalandos",
 			opConfig: config.Config{},
-			result:   `{"postgresql":{"bin_dir":"/usr/lib/postgresql/11/bin","pg_hba":["hostssl all all 0.0.0.0/0 md5","host    all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"users":{"zalandos":{"password":"","options":["CREATEDB","NOLOGIN"]}},"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"synchronous_mode":true,"synchronous_mode_strict":true,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}}}}}`,
+			result:   `{"postgresql":{"bin_dir":"/usr/lib/postgresql/11/bin","pg_hba":["hostssl all all 0.0.0.0/0 md5","host    all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"users":{"zalandos":{"password":"","options":["CREATEDB","NOLOGIN"]}},"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"synchronous_mode":true,"synchronous_mode_strict":true,"synchronous_node_count":1,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}}}}}`,
 		},
 	}
 	for _, tt := range tests {

From 8b404fd0492de92edc7a0da924b0bebcdcd6f743 Mon Sep 17 00:00:00 2001
From: Felix Kunde <felix-kunde@gmx.de>
Date: Fri, 25 Feb 2022 17:46:26 +0100
Subject: [PATCH 24/58] minor fixes to password rotation (#1796)

* minor fixes to password rotation
* rework unit test
---
 docs/administrator.md                     |  8 +-
 e2e/tests/test_e2e.py                     |  6 +-
 manifests/complete-postgres-manifest.yaml |  8 +-
 pkg/cluster/sync.go                       | 16 ++--
 pkg/cluster/sync_test.go                  | 91 +++++++++++++++--------
 5 files changed, 82 insertions(+), 47 deletions(-)

diff --git a/docs/administrator.md b/docs/administrator.md
index 3c5d8ae46..e68427658 100644
--- a/docs/administrator.md
+++ b/docs/administrator.md
@@ -306,10 +306,10 @@ The interval of days can be set with `password_rotation_interval` (default
 are replaced in the K8s secret. They belong to a newly created user named after
 the original role plus rotation date in YYMMDD format. All priviliges are
 inherited meaning that migration scripts should still grant and revoke rights
-against the original role. The timestamp of the next rotation is written to the
-secret as well. Note, if the rotation interval is decreased it is reflected in
-the secrets only if the next rotation date is more days away than the new
-length of the interval.
+against the original role. The timestamp of the next rotation (in RFC 3339
+format, UTC timezone) is written to the secret as well. Note, if the rotation
+interval is decreased it is reflected in the secrets only if the next rotation
+date is more days away than the new length of the interval.
 
 Pods still using the previous secret values which they keep in memory continue
 to connect to the database since the password of the corresponding user is not
diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py
index d1db4991b..c4d104069 100644
--- a/e2e/tests/test_e2e.py
+++ b/e2e/tests/test_e2e.py
@@ -1232,7 +1232,7 @@ class EndToEndTestCase(unittest.TestCase):
 
         # check if next rotation date was set in secret
         secret_data = k8s.get_secret_data("zalando")
-        next_rotation_timestamp = datetime.fromisoformat(str(base64.b64decode(secret_data["nextRotation"]), 'utf-8'))
+        next_rotation_timestamp = datetime.strptime(str(base64.b64decode(secret_data["nextRotation"]), 'utf-8'), "%Y-%m-%dT%H:%M:%SZ")
         today90days = today+timedelta(days=90)
         self.assertEqual(today90days, next_rotation_timestamp.date(),
                         "Unexpected rotation date in secret of zalando user: expected {}, got {}".format(today90days, next_rotation_timestamp.date()))
@@ -1247,7 +1247,7 @@ class EndToEndTestCase(unittest.TestCase):
         self.query_database(leader.metadata.name, "postgres", create_fake_rotation_user)
 
         # patch foo_user secret with outdated rotation date
-        fake_rotation_date = today.isoformat() + ' 00:00:00'
+        fake_rotation_date = today.isoformat() + 'T00:00:00Z'
         fake_rotation_date_encoded = base64.b64encode(fake_rotation_date.encode('utf-8'))
         secret_fake_rotation = {
             "data": {
@@ -1275,7 +1275,7 @@ class EndToEndTestCase(unittest.TestCase):
         # check if next rotation date and username have been replaced
         secret_data = k8s.get_secret_data("foo_user")
         secret_username = str(base64.b64decode(secret_data["username"]), 'utf-8')
-        next_rotation_timestamp = datetime.fromisoformat(str(base64.b64decode(secret_data["nextRotation"]), 'utf-8'))
+        next_rotation_timestamp = datetime.strptime(str(base64.b64decode(secret_data["nextRotation"]), 'utf-8'), "%Y-%m-%dT%H:%M:%SZ")
         rotation_user = "foo_user"+today.strftime("%y%m%d")
         today30days = today+timedelta(days=30)
 
diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml
index 577b2711b..890f5eed3 100644
--- a/manifests/complete-postgres-manifest.yaml
+++ b/manifests/complete-postgres-manifest.yaml
@@ -17,8 +17,12 @@ spec:
     - superuser
     - createdb
     foo_user: []
-#  usersWithSecretRotation: "foo_user"
-#  usersWithInPlaceSecretRotation: "flyway,bar_owner_user"
+#    flyway: [] 
+#  usersWithSecretRotation:
+#  - foo_user
+#  usersWithInPlaceSecretRotation:
+#  - flyway
+#  - bar_owner_user
   enableMasterLoadBalancer: false
   enableReplicaLoadBalancer: false
   enableConnectionPooler: false # enable/disable connection pooler deployment
diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go
index 24c68dd54..bbf023764 100644
--- a/pkg/cluster/sync.go
+++ b/pkg/cluster/sync.go
@@ -620,11 +620,6 @@ func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration(pod *v1.Pod, patroniC
 	return requiresMasterRestart, nil
 }
 
-func (c *Cluster) getNextRotationDate(currentDate time.Time) (time.Time, string) {
-	nextRotationDate := currentDate.AddDate(0, 0, int(c.OpConfig.PasswordRotationInterval))
-	return nextRotationDate, nextRotationDate.Format("2006-01-02 15:04:05")
-}
-
 func (c *Cluster) syncSecrets() error {
 
 	c.logger.Info("syncing secrets")
@@ -682,6 +677,11 @@ func (c *Cluster) syncSecrets() error {
 	return nil
 }
 
+func (c *Cluster) getNextRotationDate(currentDate time.Time) (time.Time, string) {
+	nextRotationDate := currentDate.AddDate(0, 0, int(c.OpConfig.PasswordRotationInterval))
+	return nextRotationDate, nextRotationDate.Format(time.RFC3339)
+}
+
 func (c *Cluster) updateSecret(
 	secretUsername string,
 	generatedSecret *v1.Secret,
@@ -727,7 +727,7 @@ func (c *Cluster) updateSecret(
 
 		// initialize password rotation setting first rotation date
 		nextRotationDateStr = string(secret.Data["nextRotation"])
-		if nextRotationDate, err = time.ParseInLocation("2006-01-02 15:04:05", nextRotationDateStr, time.Local); err != nil {
+		if nextRotationDate, err = time.ParseInLocation(time.RFC3339, nextRotationDateStr, currentTime.UTC().Location()); err != nil {
 			nextRotationDate, nextRotationDateStr = c.getNextRotationDate(currentTime)
 			secret.Data["nextRotation"] = []byte(nextRotationDateStr)
 			updateSecret = true
@@ -736,7 +736,7 @@ func (c *Cluster) updateSecret(
 
 		// check if next rotation can happen sooner
 		// if rotation interval has been decreased
-		currentRotationDate, _ := c.getNextRotationDate(currentTime)
+		currentRotationDate, nextRotationDateStr := c.getNextRotationDate(currentTime)
 		if nextRotationDate.After(currentRotationDate) {
 			nextRotationDate = currentRotationDate
 		}
@@ -756,8 +756,6 @@ func (c *Cluster) updateSecret(
 				*retentionUsers = append(*retentionUsers, secretUsername)
 			}
 			secret.Data["password"] = []byte(util.RandomPassword(constants.PasswordLength))
-
-			_, nextRotationDateStr = c.getNextRotationDate(nextRotationDate)
 			secret.Data["nextRotation"] = []byte(nextRotationDateStr)
 
 			updateSecret = true
diff --git a/pkg/cluster/sync_test.go b/pkg/cluster/sync_test.go
index 80e2b8463..226555a66 100644
--- a/pkg/cluster/sync_test.go
+++ b/pkg/cluster/sync_test.go
@@ -270,13 +270,29 @@ func TestUpdateSecret(t *testing.T) {
 
 	clusterName := "acid-test-cluster"
 	namespace := "default"
-	username := "foo"
+	dbname := "app"
+	dbowner := "appowner"
 	secretTemplate := config.StringTemplate("{username}.{cluster}.credentials")
 	rotationUsers := make(spec.PgUserMap)
 	retentionUsers := make([]string, 0)
-	yesterday := time.Now().AddDate(0, 0, -1)
 
-	// new cluster with pvc storage resize mode and configured labels
+	// define manifest users and enable rotation for dbowner
+	pg := acidv1.Postgresql{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      clusterName,
+			Namespace: namespace,
+		},
+		Spec: acidv1.PostgresSpec{
+			Databases:                      map[string]string{dbname: dbowner},
+			Users:                          map[string]acidv1.UserFlags{"foo": {}, dbowner: {}},
+			UsersWithInPlaceSecretRotation: []string{dbowner},
+			Volume: acidv1.Volume{
+				Size: "1Gi",
+			},
+		},
+	}
+
+	// new cluster with enabled password rotation
 	var cluster = New(
 		Config{
 			OpConfig: config.Config{
@@ -291,44 +307,61 @@ func TestUpdateSecret(t *testing.T) {
 					ClusterNameLabel: "cluster-name",
 				},
 			},
-		}, client, acidv1.Postgresql{}, logger, eventRecorder)
+		}, client, pg, logger, eventRecorder)
 
 	cluster.Name = clusterName
 	cluster.Namespace = namespace
 	cluster.pgUsers = map[string]spec.PgUser{}
-	cluster.Spec.Users = map[string]acidv1.UserFlags{username: {}}
 	cluster.initRobotUsers()
 
-	// create a secret for user foo
+	// create secrets
+	cluster.syncSecrets()
+	// initialize rotation with current time
 	cluster.syncSecrets()
 
-	secret, err := cluster.KubeClient.Secrets(namespace).Get(context.TODO(), secretTemplate.Format("username", username, "cluster", clusterName), metav1.GetOptions{})
-	assert.NoError(t, err)
-	generatedSecret := cluster.Secrets[secret.UID]
+	dayAfterTomorrow := time.Now().AddDate(0, 0, 2)
 
-	// now update the secret setting next rotation date (yesterday + interval)
-	cluster.updateSecret(username, generatedSecret, &rotationUsers, &retentionUsers, yesterday)
-	updatedSecret, err := cluster.KubeClient.Secrets(namespace).Get(context.TODO(), secretTemplate.Format("username", username, "cluster", clusterName), metav1.GetOptions{})
-	assert.NoError(t, err)
+	for username := range cluster.Spec.Users {
+		pgUser := cluster.pgUsers[username]
 
-	nextRotation := string(updatedSecret.Data["nextRotation"])
-	_, nextRotationDate := cluster.getNextRotationDate(yesterday)
-	if nextRotation != nextRotationDate {
-		t.Errorf("%s: updated secret does not contain correct rotation date: expected %s, got %s", testName, nextRotationDate, nextRotation)
-	}
+		// first, get the secret
+		secret, err := cluster.KubeClient.Secrets(namespace).Get(context.TODO(), secretTemplate.Format("username", username, "cluster", clusterName), metav1.GetOptions{})
+		assert.NoError(t, err)
+		secretPassword := string(secret.Data["password"])
 
-	// update secret again but use current time to trigger rotation
-	cluster.updateSecret(username, generatedSecret, &rotationUsers, &retentionUsers, time.Now())
-	updatedSecret, err = cluster.KubeClient.Secrets(namespace).Get(context.TODO(), secretTemplate.Format("username", username, "cluster", clusterName), metav1.GetOptions{})
-	assert.NoError(t, err)
+		// now update the secret setting a next rotation date (tomorrow + interval)
+		cluster.updateSecret(username, secret, &rotationUsers, &retentionUsers, dayAfterTomorrow)
+		updatedSecret, err := cluster.KubeClient.Secrets(namespace).Get(context.TODO(), secretTemplate.Format("username", username, "cluster", clusterName), metav1.GetOptions{})
+		assert.NoError(t, err)
 
-	if len(rotationUsers) != 1 && len(retentionUsers) != 1 {
-		t.Errorf("%s: unexpected number of users to rotate - expected only foo, found %d", testName, len(rotationUsers))
-	}
+		// check that passwords are different
+		rotatedPassword := string(updatedSecret.Data["password"])
+		if secretPassword == rotatedPassword {
+			t.Errorf("%s: password unchanged in updated secret for %s", testName, username)
+		}
 
-	secretUsername := string(updatedSecret.Data["username"])
-	rotatedUsername := username + time.Now().Format("060102")
-	if secretUsername != rotatedUsername {
-		t.Errorf("%s: updated secret does not contain correct username: expected %s, got %s", testName, rotatedUsername, secretUsername)
+		// check that next rotation date is tomorrow + interval, not date in secret + interval
+		nextRotation := string(updatedSecret.Data["nextRotation"])
+		_, nextRotationDate := cluster.getNextRotationDate(dayAfterTomorrow)
+		if nextRotation != nextRotationDate {
+			t.Errorf("%s: updated secret of %s does not contain correct rotation date: expected %s, got %s", testName, username, nextRotationDate, nextRotation)
+		}
+
+		// compare username, when it's dbowner they should be equal because of UsersWithInPlaceSecretRotation
+		secretUsername := string(updatedSecret.Data["username"])
+		if pgUser.IsDbOwner {
+			if secretUsername != username {
+				t.Errorf("%s: username differs in updated secret: expected %s, got %s", testName, username, secretUsername)
+			}
+		} else {
+			rotatedUsername := username + dayAfterTomorrow.Format("060102")
+			if secretUsername != rotatedUsername {
+				t.Errorf("%s: updated secret does not contain correct username: expected %s, got %s", testName, rotatedUsername, secretUsername)
+			}
+
+			if len(rotationUsers) != 1 && len(retentionUsers) != 1 {
+				t.Errorf("%s: unexpected number of users to rotate - expected only %s, found %d", testName, username, len(rotationUsers))
+			}
+		}
 	}
 }

From d8a159ef1a17491813f68c7bdff600aeb811802d Mon Sep 17 00:00:00 2001
From: Felix Kunde <felix-kunde@gmx.de>
Date: Mon, 28 Feb 2022 10:09:42 +0100
Subject: [PATCH 25/58] create CDC event stream CRD (#1570)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* provide event stream API
* check manifest settings for logical decoding before creating streams
* operator updates Postgres config and creates replication user
* name FES like the Postgres cluster
* add delete case and fix updating streams + update unit test
* check if fes CRD exists before syncing
* existing slot must use the same plugin
* make id and payload columns configurable
* sync streams only when they are defined in manifest
* introduce applicationId for separate stream CRDs
* add FES to RBAC in chart
* disable streams in chart
* switch to pgoutput plugin and let operator create publications
* reflect code review and additional refactoring

Co-authored-by: Paŭlo Ebermann <paul.ebermann@zalando.de>
---
 .../postgres-operator/crds/postgresqls.yaml   |  33 ++
 .../templates/clusterrole.yaml                |  16 +
 charts/postgres-operator/values.yaml          |   3 +
 docs/reference/cluster_manifest.md            |  51 +++
 hack/update-codegen.sh                        |   2 +-
 manifests/complete-postgres-manifest.yaml     |  21 +-
 manifests/operator-service-account-rbac.yaml  |  14 +
 manifests/postgresql.crd.yaml                 |  33 ++
 pkg/apis/acid.zalan.do/v1/crds.go             |  48 +++
 pkg/apis/acid.zalan.do/v1/postgresql_type.go  |  15 +
 .../acid.zalan.do/v1/zz_generated.deepcopy.go |  53 +++
 pkg/apis/zalando.org/register.go              |   6 +
 pkg/apis/zalando.org/v1/fabriceventstream.go  |  84 ++++
 pkg/apis/zalando.org/v1/register.go           |  44 +++
 .../zalando.org/v1/zz_generated.deepcopy.go   | 227 +++++++++++
 pkg/cluster/cluster.go                        |  34 ++
 pkg/cluster/database.go                       |  69 ++++
 pkg/cluster/streams.go                        | 363 ++++++++++++++++++
 pkg/cluster/streams_test.go                   | 244 ++++++++++++
 pkg/cluster/sync.go                           |  48 ++-
 pkg/cluster/sync_test.go                      |   2 +-
 .../clientset/versioned/clientset.go          |  16 +-
 .../versioned/fake/clientset_generated.go     |   7 +
 .../clientset/versioned/fake/register.go      |   2 +
 .../clientset/versioned/scheme/register.go    |   2 +
 .../versioned/typed/zalando.org/v1/doc.go     |  26 ++
 .../typed/zalando.org/v1/fabriceventstream.go | 184 +++++++++
 .../typed/zalando.org/v1/fake/doc.go          |  26 ++
 .../v1/fake/fake_fabriceventstream.go         | 136 +++++++
 .../v1/fake/fake_zalando.org_client.go        |  46 +++
 .../zalando.org/v1/generated_expansion.go     |  27 ++
 .../zalando.org/v1/zalando.org_client.go      |  95 +++++
 .../informers/externalversions/factory.go     |   6 +
 .../informers/externalversions/generic.go     |   5 +
 .../externalversions/zalando.org/interface.go |  52 +++
 .../zalando.org/v1/fabriceventstream.go       |  96 +++++
 .../zalando.org/v1/interface.go               |  51 +++
 .../zalando.org/v1/expansion_generated.go     |  33 ++
 .../zalando.org/v1/fabriceventstream.go       | 105 +++++
 pkg/util/constants/streams.go                 |  14 +
 pkg/util/k8sutil/k8sutil.go                   |  16 +-
 pkg/util/patroni/patroni_test.go              |   6 +-
 42 files changed, 2330 insertions(+), 31 deletions(-)
 create mode 100644 pkg/apis/zalando.org/register.go
 create mode 100644 pkg/apis/zalando.org/v1/fabriceventstream.go
 create mode 100644 pkg/apis/zalando.org/v1/register.go
 create mode 100644 pkg/apis/zalando.org/v1/zz_generated.deepcopy.go
 create mode 100644 pkg/cluster/streams.go
 create mode 100644 pkg/cluster/streams_test.go
 create mode 100644 pkg/generated/clientset/versioned/typed/zalando.org/v1/doc.go
 create mode 100644 pkg/generated/clientset/versioned/typed/zalando.org/v1/fabriceventstream.go
 create mode 100644 pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/doc.go
 create mode 100644 pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/fake_fabriceventstream.go
 create mode 100644 pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/fake_zalando.org_client.go
 create mode 100644 pkg/generated/clientset/versioned/typed/zalando.org/v1/generated_expansion.go
 create mode 100644 pkg/generated/clientset/versioned/typed/zalando.org/v1/zalando.org_client.go
 create mode 100644 pkg/generated/informers/externalversions/zalando.org/interface.go
 create mode 100644 pkg/generated/informers/externalversions/zalando.org/v1/fabriceventstream.go
 create mode 100644 pkg/generated/informers/externalversions/zalando.org/v1/interface.go
 create mode 100644 pkg/generated/listers/zalando.org/v1/expansion_generated.go
 create mode 100644 pkg/generated/listers/zalando.org/v1/fabriceventstream.go
 create mode 100644 pkg/util/constants/streams.go

diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml
index bcd212afd..a7072b8cc 100644
--- a/charts/postgres-operator/crds/postgresqls.yaml
+++ b/charts/postgres-operator/crds/postgresqls.yaml
@@ -474,6 +474,39 @@ spec:
                     type: string
                   gs_wal_path:
                     type: string
+              streams:
+                type: array
+                nullable: true
+                items:
+                  type: object
+                  required:
+                    - applicationId
+                    - database
+                    - tables
+                  properties:
+                    applicationId:
+                      type: string
+                    batchSize:
+                      type: integer
+                    database:
+                      type: string
+                    filter:
+                      type: object
+                      additionalProperties:
+                        type: string
+                    tables:
+                      type: object
+                      additionalProperties:
+                        type: object
+                        required:
+                          - eventType
+                        properties:
+                          eventType:
+                            type: string
+                          idColumn:
+                            type: string
+                          payloadColumn:
+                            type: string
               teamId:
                 type: string
               tls:
diff --git a/charts/postgres-operator/templates/clusterrole.yaml b/charts/postgres-operator/templates/clusterrole.yaml
index 87fd38cd2..8b2e9136e 100644
--- a/charts/postgres-operator/templates/clusterrole.yaml
+++ b/charts/postgres-operator/templates/clusterrole.yaml
@@ -34,6 +34,22 @@ rules:
   - get
   - list
   - watch
+# all verbs allowed for event streams
+{{- if .Values.enableStreams }}
+- apiGroups:
+  - zalando.org
+  resources:
+  - fabriceventstreams
+  verbs:
+  - create
+  - delete
+  - deletecollection
+  - get
+  - list
+  - patch
+  - update
+  - watch
+{{- end }}
 # to create or get/update CRDs when starting up
 - apiGroups:
   - apiextensions.k8s.io
diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml
index 8183894d3..288efe763 100644
--- a/charts/postgres-operator/values.yaml
+++ b/charts/postgres-operator/values.yaml
@@ -368,6 +368,9 @@ configConnectionPooler:
   connection_pooler_default_cpu_limit: "1"
   connection_pooler_default_memory_limit: 100Mi
 
+# Zalando's internal CDC stream feature
+enableStreams: false
+
 rbac:
   # Specifies whether RBAC resources should be created
   create: true
diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md
index ae72d6450..cff86333e 100644
--- a/docs/reference/cluster_manifest.md
+++ b/docs/reference/cluster_manifest.md
@@ -536,3 +536,54 @@ Those parameters are grouped under the `tls` top-level key.
   relative to the "/tls/", which is mount path of the tls secret.
   If `caSecretName` is defined, the ca.crt path is relative to "/tlsca/",
   otherwise to the same "/tls/".
+
+## Change data capture streams
+
+This sections enables change data capture (CDC) streams via Postgres' 
+[logical decoding](https://www.postgresql.org/docs/14/logicaldecoding.html)
+feature and `pgoutput` plugin. While the Postgres operator takes responsibility
+for providing the setup to publish change events, it relies on external tools
+to consume them. At Zalando, we are using a workflow based on
+[Debezium Connector](https://debezium.io/documentation/reference/stable/connectors/postgresql.html)
+which can feed streams into Zalando’s distributed event broker [Nakadi](https://nakadi.io/)
+among others.
+
+The Postgres Operator creates custom resources for Zalando's internal CDC
+operator which will be used to set up the consumer part. Each stream object
+can have the following properties:
+
+* **applicationId**
+  The application name to which the database and CDC belongs to. For each
+  set of streams with a distinct `applicationId` a separate stream CR as well
+  as a separate logical replication slot will be created. This means there can
+  be different streams in the same database and streams with the same
+  `applicationId` are bundled in one stream CR. The stream CR will be called
+  like the Postgres cluster plus "-<applicationId>" suffix. Required.
+
+* **database**
+  Name of the database from where events will be published via Postgres'
+  logical decoding feature. The operator will take care of updating the
+  database configuration (setting `wal_level: logical`, creating logical
+  replication slots, using output plugin `pgoutput` and creating a dedicated
+  replication user). Required.
+
+* **tables**
+  Defines a map of table names and their properties (`eventType`, `idColumn`
+  and `payloadColumn`). The CDC operator is following the [outbox pattern](https://debezium.io/blog/2019/02/19/reliable-microservices-data-exchange-with-the-outbox-pattern/).
+  The application is responsible for putting events into a (JSON/B or VARCHAR)
+  payload column of the outbox table in the structure of the specified target
+  event type. The operator will create a [PUBLICATION](https://www.postgresql.org/docs/14/logical-replication-publication.html)
+  in Postgres for all tables specified for one `database` and `applicationId`.
+  The CDC operator will consume from it shortly after transactions are
+  committed to the outbox table. The `idColumn` will be used in telemetry for
+  the CDC operator. The names for `idColumn` and `payloadColumn` can be
+  configured. Defaults are `id` and `payload`. The target `eventType` has to
+  be defined. Required.
+
+* **filter**
+  Streamed events can be filtered by a jsonpath expression for each table.
+  Optional.
+
+* **batchSize**
+  Defines the size of batches in which events are consumed. Optional.
+  Defaults to 1.
diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh
index d304004dc..d34db9c45 100755
--- a/hack/update-codegen.sh
+++ b/hack/update-codegen.sh
@@ -17,7 +17,7 @@ trap "cleanup" EXIT SIGINT
 
 bash "${CODEGEN_PKG}/generate-groups.sh" all \
   "${OPERATOR_PACKAGE_ROOT}/pkg/generated" "${OPERATOR_PACKAGE_ROOT}/pkg/apis" \
-  "acid.zalan.do:v1" \
+  "acid.zalan.do:v1 zalando.org:v1" \
   --go-header-file "${SCRIPT_ROOT}"/hack/custom-boilerplate.go.txt
 
 cp -r "${OPERATOR_PACKAGE_ROOT}"/pkg/* "${TARGET_CODE_DIR}"
diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml
index 890f5eed3..cb2a8ee1f 100644
--- a/manifests/complete-postgres-manifest.yaml
+++ b/manifests/complete-postgres-manifest.yaml
@@ -3,6 +3,7 @@ kind: postgresql
 metadata:
   name: acid-test-cluster
 #  labels:
+#    application: test-app
 #    environment: demo
 #  annotations:
 #    "acid.zalan.do/controller": "second-operator"
@@ -17,7 +18,7 @@ spec:
     - superuser
     - createdb
     foo_user: []
-#    flyway: [] 
+#    flyway: []
 #  usersWithSecretRotation:
 #  - foo_user
 #  usersWithInPlaceSecretRotation:
@@ -203,3 +204,21 @@ spec:
 #              operator: In
 #              values:
 #                - enabled
+
+# Enables change data capture streams for defined database tables
+#  streams:
+#  - applicationId: test-app
+#    database: foo
+#    tables:
+#      data.state_pending_outbox:
+#        eventType: test-app.status-pending
+#      data.state_approved_outbox:
+#        eventType: test-app.status-approved
+#      data.orders_outbox:
+#        eventType: test-app.order-completed
+#        idColumn: o_id
+#        payloadColumn: o_payload
+#    # Optional. Filter ignores events before a certain txnId and lsn. Can be used to skip bad events
+#    filter:
+#      data.orders_outbox: "[?(@.source.txId > 500 && @.source.lsn > 123456)]"
+#    batchSize: 1000
diff --git a/manifests/operator-service-account-rbac.yaml b/manifests/operator-service-account-rbac.yaml
index f0307f6a0..c10dc5fd7 100644
--- a/manifests/operator-service-account-rbac.yaml
+++ b/manifests/operator-service-account-rbac.yaml
@@ -35,6 +35,20 @@ rules:
   - get
   - list
   - watch
+# all verbs allowed for event streams (Zalando-internal feature)
+# - apiGroups:
+#   - zalando.org
+#   resources:
+#   - fabriceventstreams
+#   verbs:
+#   - create
+#   - delete
+#   - deletecollection
+#   - get
+#   - list
+#   - patch
+#   - update
+#   - watch
 # to create or get/update CRDs when starting up
 - apiGroups:
   - apiextensions.k8s.io
diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml
index 1f1b064a7..7cf220eb7 100644
--- a/manifests/postgresql.crd.yaml
+++ b/manifests/postgresql.crd.yaml
@@ -472,6 +472,39 @@ spec:
                     type: string
                   gs_wal_path:
                     type: string
+              streams:
+                type: array
+                nullable: true
+                items:
+                  type: object
+                  required:
+                    - applicationId
+                    - database
+                    - tables
+                  properties:
+                    applicationId:
+                      type: string
+                    batchSize:
+                      type: integer
+                    database:
+                      type: string
+                    filter:
+                      type: object
+                      additionalProperties:
+                        type: string
+                    tables:
+                      type: object
+                      additionalProperties:
+                        type: object
+                        required:
+                          - eventType
+                        properties:
+                          eventType:
+                            type: string
+                          idColumn:
+                            type: string
+                          payloadColumn:
+                            type: string
               teamId:
                 type: string
               tls:
diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go
index 89a04b180..9dc3d167e 100644
--- a/pkg/apis/acid.zalan.do/v1/crds.go
+++ b/pkg/apis/acid.zalan.do/v1/crds.go
@@ -716,6 +716,54 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 							},
 						},
 					},
+					"streams": {
+						Type: "array",
+						Items: &apiextv1.JSONSchemaPropsOrArray{
+							Schema: &apiextv1.JSONSchemaProps{
+								Type:     "object",
+								Required: []string{"applicationId", "database", "tables"},
+								Properties: map[string]apiextv1.JSONSchemaProps{
+									"applicationId": {
+										Type: "string",
+									},
+									"batchSize": {
+										Type: "integer",
+									},
+									"database": {
+										Type: "string",
+									},
+									"filter": {
+										Type: "object",
+										AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{
+											Schema: &apiextv1.JSONSchemaProps{
+												Type: "string",
+											},
+										},
+									},
+									"tables": {
+										Type: "object",
+										AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{
+											Schema: &apiextv1.JSONSchemaProps{
+												Type:     "object",
+												Required: []string{"eventType"},
+												Properties: map[string]apiextv1.JSONSchemaProps{
+													"eventType": {
+														Type: "string",
+													},
+													"idColumn": {
+														Type: "string",
+													},
+													"payloadColumn": {
+														Type: "string",
+													},
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
 					"teamId": {
 						Type: "string",
 					},
diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go
index a97ad35a3..a1fc4fcf7 100644
--- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go
+++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go
@@ -77,6 +77,7 @@ type PostgresSpec struct {
 	ServiceAnnotations    map[string]string           `json:"serviceAnnotations,omitempty"`
 	TLS                   *TLSDescription             `json:"tls,omitempty"`
 	AdditionalVolumes     []AdditionalVolume          `json:"additionalVolumes,omitempty"`
+	Streams               []Stream                    `json:"streams,omitempty"`
 
 	// deprecated json tags
 	InitContainersOld       []v1.Container `json:"init_containers,omitempty"`
@@ -231,3 +232,17 @@ type ConnectionPooler struct {
 
 	Resources `json:"resources,omitempty"`
 }
+
+type Stream struct {
+	ApplicationId string                 `json:"applicationId"`
+	Database      string                 `json:"database"`
+	Tables        map[string]StreamTable `json:"tables"`
+	Filter        map[string]string      `json:"filter,omitempty"`
+	BatchSize     uint32                 `json:"batchSize,omitempty"`
+}
+
+type StreamTable struct {
+	EventType     string `json:"eventType"`
+	IdColumn      string `json:"idColumn,omitempty" defaults:"id"`
+	PayloadColumn string `json:"payloadColumn,omitempty" defaults:"payload"`
+}
diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
index d960cd102..c2298fada 100644
--- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
+++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
@@ -749,6 +749,13 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	if in.Streams != nil {
+		in, out := &in.Streams, &out.Streams
+		*out = make([]Stream, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	if in.InitContainersOld != nil {
 		in, out := &in.InitContainersOld, &out.InitContainersOld
 		*out = make([]corev1.Container, len(*in))
@@ -1152,6 +1159,52 @@ func (in *StandbyDescription) DeepCopy() *StandbyDescription {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Stream) DeepCopyInto(out *Stream) {
+	*out = *in
+	if in.Tables != nil {
+		in, out := &in.Tables, &out.Tables
+		*out = make(map[string]StreamTable, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	if in.Filter != nil {
+		in, out := &in.Filter, &out.Filter
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Stream.
+func (in *Stream) DeepCopy() *Stream {
+	if in == nil {
+		return nil
+	}
+	out := new(Stream)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *StreamTable) DeepCopyInto(out *StreamTable) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StreamTable.
+func (in *StreamTable) DeepCopy() *StreamTable {
+	if in == nil {
+		return nil
+	}
+	out := new(StreamTable)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *TLSDescription) DeepCopyInto(out *TLSDescription) {
 	*out = *in
diff --git a/pkg/apis/zalando.org/register.go b/pkg/apis/zalando.org/register.go
new file mode 100644
index 000000000..3dbd3f089
--- /dev/null
+++ b/pkg/apis/zalando.org/register.go
@@ -0,0 +1,6 @@
+package zalando
+
+const (
+	// GroupName is the group name for the operator CRDs
+	GroupName = "zalando.org"
+)
diff --git a/pkg/apis/zalando.org/v1/fabriceventstream.go b/pkg/apis/zalando.org/v1/fabriceventstream.go
new file mode 100644
index 000000000..65d082c1e
--- /dev/null
+++ b/pkg/apis/zalando.org/v1/fabriceventstream.go
@@ -0,0 +1,84 @@
+package v1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// +genclient
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// FabricEventStream defines FabricEventStream Custom Resource Definition Object.
+type FabricEventStream struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec FabricEventStreamSpec `json:"spec"`
+}
+
+// FabricEventStreamSpec defines the specification for the FabricEventStream TPR.
+type FabricEventStreamSpec struct {
+	ApplicationId string        `json:"applicationId"`
+	EventStreams  []EventStream `json:"eventStreams"`
+}
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// FabricEventStreamList defines a list of FabricEventStreams .
+type FabricEventStreamList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata"`
+
+	Items []FabricEventStream `json:"items"`
+}
+
+// EventStream defines the source, flow and sink of the event stream
+type EventStream struct {
+	EventStreamFlow   EventStreamFlow   `json:"flow"`
+	EventStreamSink   EventStreamSink   `json:"sink"`
+	EventStreamSource EventStreamSource `json:"source"`
+}
+
+// EventStreamFlow defines the flow characteristics of the event stream
+type EventStreamFlow struct {
+	Type          string `json:"type"`
+	PayloadColumn string `json:"payloadColumn,omitempty" defaults:"payload"`
+}
+
+// EventStreamSink defines the target of the event stream
+type EventStreamSink struct {
+	Type         string `json:"type"`
+	EventType    string `json:"eventType,omitempty"`
+	MaxBatchSize uint32 `json:"maxBatchSize,omitempty"`
+}
+
+// EventStreamSource defines the source of the event stream and connection for FES operator
+type EventStreamSource struct {
+	Type             string           `json:"type"`
+	Schema           string           `json:"schema,omitempty" defaults:"public"`
+	EventStreamTable EventStreamTable `json:"table"`
+	Filter           string           `json:"filter,omitempty"`
+	Connection       Connection       `json:"jdbcConnection"`
+}
+
+// EventStreamTable defines the name and ID column to be used for streaming
+type EventStreamTable struct {
+	Name     string `json:"name"`
+	IDColumn string `json:"idColumn,omitempty" defaults:"id"`
+}
+
+// Connection to be used for allowing the FES operator to connect to a database
+type Connection struct {
+	Url             string `json:"jdbcUrl"`
+	SlotName        string `json:"slotName"`
+	PluginType      string `json:"pluginType,omitempty" defaults:"wal2json"`
+	PublicationName string `json:"publicationName,omitempty"`
+	DBAuth          DBAuth `json:"databaseAuthentication"`
+}
+
+// DBAuth specifies the credentials to be used for connecting with the database
+type DBAuth struct {
+	Type        string `json:"type"`
+	Name        string `json:"name,omitempty"`
+	UserKey     string `json:"userKey,omitempty"`
+	PasswordKey string `json:"passwordKey,omitempty"`
+}
diff --git a/pkg/apis/zalando.org/v1/register.go b/pkg/apis/zalando.org/v1/register.go
new file mode 100644
index 000000000..33a2c718b
--- /dev/null
+++ b/pkg/apis/zalando.org/v1/register.go
@@ -0,0 +1,44 @@
+package v1
+
+import (
+	"github.com/zalando/postgres-operator/pkg/apis/zalando.org"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/kubernetes/scheme"
+)
+
+// APIVersion of the `fabriceventstream` CRD
+const (
+	APIVersion = "v1"
+)
+
+var (
+	schemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
+	AddToScheme   = schemeBuilder.AddToScheme
+)
+
+func init() {
+	err := AddToScheme(scheme.Scheme)
+	if err != nil {
+		panic(err)
+	}
+}
+
+// SchemeGroupVersion is the group version used to register these objects.
+var SchemeGroupVersion = schema.GroupVersion{Group: zalando.GroupName, Version: APIVersion}
+
+// Resource takes an unqualified resource and returns a Group-qualified GroupResource.
+func Resource(resource string) schema.GroupResource {
+	return SchemeGroupVersion.WithResource(resource).GroupResource()
+}
+
+// addKnownTypes adds the set of types defined in this package to the supplied scheme.
+func addKnownTypes(scheme *runtime.Scheme) error {
+	scheme.AddKnownTypes(SchemeGroupVersion,
+		&FabricEventStream{},
+		&FabricEventStreamList{},
+	)
+	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
+	return nil
+}
diff --git a/pkg/apis/zalando.org/v1/zz_generated.deepcopy.go b/pkg/apis/zalando.org/v1/zz_generated.deepcopy.go
new file mode 100644
index 000000000..a44439a94
--- /dev/null
+++ b/pkg/apis/zalando.org/v1/zz_generated.deepcopy.go
@@ -0,0 +1,227 @@
+// +build !ignore_autogenerated
+
+/*
+Copyright 2021 Compose, Zalando SE
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+// Code generated by deepcopy-gen. DO NOT EDIT.
+
+package v1
+
+import (
+	runtime "k8s.io/apimachinery/pkg/runtime"
+)
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Connection) DeepCopyInto(out *Connection) {
+	*out = *in
+	return
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Connection) DeepCopy() *Connection {
+	if in == nil {
+		return nil
+	}
+	out := new(Connection)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DBAuth) DeepCopyInto(out *DBAuth) {
+	*out = *in
+	return
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DBAuth) DeepCopy() *DBAuth {
+	if in == nil {
+		return nil
+	}
+	out := new(DBAuth)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EventStream) DeepCopyInto(out *EventStream) {
+	*out = *in
+	return
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EventStream) DeepCopy() *EventStream {
+	if in == nil {
+		return nil
+	}
+	out := new(EventStream)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EventStreamFlow) DeepCopyInto(out *EventStreamFlow) {
+	*out = *in
+	return
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EventStreamFlow) DeepCopy() *EventStreamFlow {
+	if in == nil {
+		return nil
+	}
+	out := new(EventStreamFlow)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EventStreamSink) DeepCopyInto(out *EventStreamSink) {
+	*out = *in
+	return
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EventStreamSink) DeepCopy() *EventStreamSink {
+	if in == nil {
+		return nil
+	}
+	out := new(EventStreamSink)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EventStreamSource) DeepCopyInto(out *EventStreamSource) {
+	*out = *in
+	return
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EventStreamSource) DeepCopy() *EventStreamSource {
+	if in == nil {
+		return nil
+	}
+	out := new(EventStreamSource)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EventStreamTable) DeepCopyInto(out *EventStreamTable) {
+	*out = *in
+	return
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EventStreamTable) DeepCopy() *EventStreamTable {
+	if in == nil {
+		return nil
+	}
+	out := new(EventStreamTable)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *FabricEventStreamSpec) DeepCopyInto(out *FabricEventStreamSpec) {
+	*out = *in
+	if in.EventStreams != nil {
+		in, out := &in.EventStreams, &out.EventStreams
+		*out = make([]EventStream, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FabricEventStreamSpec.
+func (in *FabricEventStreamSpec) DeepCopy() *FabricEventStreamSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(FabricEventStreamSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *FabricEventStream) DeepCopyInto(out *FabricEventStream) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Spec.DeepCopyInto(&out.Spec)
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FabricEventStream.
+func (in *FabricEventStream) DeepCopy() *FabricEventStream {
+	if in == nil {
+		return nil
+	}
+	out := new(FabricEventStream)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *FabricEventStream) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *FabricEventStreamList) DeepCopyInto(out *FabricEventStreamList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]FabricEventStream, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FabricEventStreamList.
+func (in *FabricEventStreamList) DeepCopy() *FabricEventStreamList {
+	if in == nil {
+		return nil
+	}
+	out := new(FabricEventStreamList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *FabricEventStreamList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go
index 3e7f708a5..7dbfa6b88 100644
--- a/pkg/cluster/cluster.go
+++ b/pkg/cluster/cluster.go
@@ -361,6 +361,12 @@ func (c *Cluster) Create() error {
 	// something fails, report warning
 	c.createConnectionPooler(c.installLookupFunction)
 
+	if len(c.Spec.Streams) > 0 {
+		if err = c.syncStreams(); err != nil {
+			c.logger.Errorf("could not create streams: %v", err)
+		}
+	}
+
 	return nil
 }
 
@@ -861,6 +867,13 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
 		updateFailed = true
 	}
 
+	if len(c.Spec.Streams) > 0 {
+		if err := c.syncStreams(); err != nil {
+			c.logger.Errorf("could not sync streams: %v", err)
+			updateFailed = true
+		}
+	}
+
 	if !updateFailed {
 		// Major version upgrade must only fire after success of earlier operations and should stay last
 		if err := c.majorVersionUpgrade(); err != nil {
@@ -896,6 +909,10 @@ func (c *Cluster) Delete() {
 	defer c.mu.Unlock()
 	c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Delete", "Started deletion of new cluster resources")
 
+	if err := c.deleteStreams(); err != nil {
+		c.logger.Warningf("could not delete event streams: %v", err)
+	}
+
 	// delete the backup job before the stateful set of the cluster to prevent connections to non-existing pods
 	// deleting the cron job also removes pods and batch jobs it created
 	if err := c.deleteLogicalBackupJob(); err != nil {
@@ -1050,6 +1067,23 @@ func (c *Cluster) initSystemUsers() {
 			c.systemUsers[constants.ConnectionPoolerUserKeyName] = connectionPoolerUser
 		}
 	}
+
+	// replication users for event streams are another exception
+	// the operator will create one replication user for all streams
+	if len(c.Spec.Streams) > 0 {
+		username := constants.EventStreamSourceSlotPrefix + constants.UserRoleNameSuffix
+		streamUser := spec.PgUser{
+			Origin:    spec.RoleConnectionPooler,
+			Name:      username,
+			Namespace: c.Namespace,
+			Flags:     []string{constants.RoleFlagLogin, constants.RoleFlagReplication},
+			Password:  util.RandomPassword(constants.PasswordLength),
+		}
+
+		if _, exists := c.pgUsers[username]; !exists {
+			c.pgUsers[username] = streamUser
+		}
+	}
 }
 
 func (c *Cluster) initPreparedDatabaseRoles() error {
diff --git a/pkg/cluster/database.go b/pkg/cluster/database.go
index 279cf828f..652f0d0ae 100644
--- a/pkg/cluster/database.go
+++ b/pkg/cluster/database.go
@@ -46,6 +46,13 @@ const (
 	createExtensionSQL      = `CREATE EXTENSION IF NOT EXISTS "%s" SCHEMA "%s"`
 	alterExtensionSQL       = `ALTER EXTENSION "%s" SET SCHEMA "%s"`
 
+	getPublicationsSQL = `SELECT p.pubname, string_agg(pt.schemaname || '.' || pt.tablename, ', ' ORDER BY pt.schemaname, pt.tablename)
+	        FROM pg_publication p
+			JOIN pg_publication_tables pt ON pt.pubname = p.pubname
+			GROUP BY p.pubname;`
+	createPublicationSQL = `CREATE PUBLICATION "%s" FOR TABLE %s WITH (publish = 'insert, update');`
+	alterPublicationSQL  = `ALTER PUBLICATION "%s" SET TABLE %s;`
+
 	globalDefaultPrivilegesSQL = `SET ROLE TO "%s";
 			ALTER DEFAULT PRIVILEGES GRANT USAGE ON SCHEMAS TO "%s","%s";
 			ALTER DEFAULT PRIVILEGES GRANT SELECT ON TABLES TO "%s";
@@ -572,6 +579,68 @@ func (c *Cluster) execCreateOrAlterExtension(extName, schemaName, statement, doi
 	return nil
 }
 
+// getPublications returns the list of current database publications with tables
+// The caller is responsible for opening and closing the database connection
+func (c *Cluster) getPublications() (publications map[string]string, err error) {
+	var (
+		rows *sql.Rows
+	)
+
+	if rows, err = c.pgDb.Query(getPublicationsSQL); err != nil {
+		return nil, fmt.Errorf("could not query database publications: %v", err)
+	}
+
+	defer func() {
+		if err2 := rows.Close(); err2 != nil {
+			if err != nil {
+				err = fmt.Errorf("error when closing query cursor: %v, previous error: %v", err2, err)
+			} else {
+				err = fmt.Errorf("error when closing query cursor: %v", err2)
+			}
+		}
+	}()
+
+	dbPublications := make(map[string]string)
+
+	for rows.Next() {
+		var (
+			dbPublication       string
+			dbPublicationTables string
+		)
+
+		if err = rows.Scan(&dbPublication, &dbPublicationTables); err != nil {
+			return nil, fmt.Errorf("error when processing row: %v", err)
+		}
+		dbPublications[dbPublication] = dbPublicationTables
+	}
+
+	return dbPublications, err
+}
+
+// executeCreatePublication creates new publication for given tables
+// The caller is responsible for opening and closing the database connection.
+func (c *Cluster) executeCreatePublication(pubName, tableList string) error {
+	return c.execCreateOrAlterPublication(pubName, tableList, createPublicationSQL,
+		"creating publication", "create publication")
+}
+
+// executeAlterExtension changes the table list of the given publication.
+// The caller is responsible for opening and closing the database connection.
+func (c *Cluster) executeAlterPublication(pubName, tableList string) error {
+	return c.execCreateOrAlterPublication(pubName, tableList, alterPublicationSQL,
+		"changing publication", "alter publication tables")
+}
+
+func (c *Cluster) execCreateOrAlterPublication(pubName, tableList, statement, doing, operation string) error {
+
+	c.logger.Debugf("%s %q with table list %q", doing, pubName, tableList)
+	if _, err := c.pgDb.Exec(fmt.Sprintf(statement, pubName, tableList)); err != nil {
+		return fmt.Errorf("could not execute %s: %v", operation, err)
+	}
+
+	return nil
+}
+
 // Creates a connection pool credentials lookup function in every database to
 // perform remote authentication.
 func (c *Cluster) installLookupFunction(poolerSchema, poolerUser string) error {
diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go
new file mode 100644
index 000000000..a1a1b4d4a
--- /dev/null
+++ b/pkg/cluster/streams.go
@@ -0,0 +1,363 @@
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+	"sort"
+	"strings"
+
+	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
+	zalandov1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1"
+	"github.com/zalando/postgres-operator/pkg/util"
+	"github.com/zalando/postgres-operator/pkg/util/constants"
+	"github.com/zalando/postgres-operator/pkg/util/k8sutil"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func (c *Cluster) createStreams(appId string) error {
+	c.setProcessName("creating streams")
+
+	fes := c.generateFabricEventStream(appId)
+	if _, err := c.KubeClient.FabricEventStreams(c.Namespace).Create(context.TODO(), fes, metav1.CreateOptions{}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *Cluster) updateStreams(newEventStreams *zalandov1.FabricEventStream) error {
+	c.setProcessName("updating event streams")
+
+	if _, err := c.KubeClient.FabricEventStreams(newEventStreams.Namespace).Update(context.TODO(), newEventStreams, metav1.UpdateOptions{}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *Cluster) deleteStreams() error {
+	c.setProcessName("deleting event streams")
+
+	// check if stream CRD is installed before trying a delete
+	_, err := c.KubeClient.CustomResourceDefinitions().Get(context.TODO(), constants.EventStreamCRDName, metav1.GetOptions{})
+	if k8sutil.ResourceNotFound(err) {
+		return nil
+	}
+
+	err = c.KubeClient.FabricEventStreams(c.Namespace).Delete(context.TODO(), c.Name, metav1.DeleteOptions{})
+	if err != nil {
+		return fmt.Errorf("could not delete event stream custom resource: %v", err)
+	}
+
+	return nil
+}
+
+func gatherApplicationIds(streams []acidv1.Stream) []string {
+	appIds := make([]string, 0)
+	for _, stream := range streams {
+		if !util.SliceContains(appIds, stream.ApplicationId) {
+			appIds = append(appIds, stream.ApplicationId)
+		}
+	}
+
+	return appIds
+}
+
+func (c *Cluster) syncPostgresConfig(requiredPatroniConfig acidv1.Patroni) error {
+	errorMsg := "no pods found to update config"
+
+	// if streams are defined wal_level must be switched to logical
+	requiredPgParameters := map[string]string{"wal_level": "logical"}
+
+	// apply config changes in pods
+	pods, err := c.listPods()
+	if err != nil {
+		errorMsg = fmt.Sprintf("could not list pods of the statefulset: %v", err)
+	}
+	for i, pod := range pods {
+		podName := util.NameFromMeta(pods[i].ObjectMeta)
+		effectivePatroniConfig, effectivePgParameters, err := c.patroni.GetConfig(&pod)
+		if err != nil {
+			errorMsg = fmt.Sprintf("could not get Postgres config from pod %s: %v", podName, err)
+			continue
+		}
+
+		_, err = c.checkAndSetGlobalPostgreSQLConfiguration(&pod, effectivePatroniConfig, requiredPatroniConfig, effectivePgParameters, requiredPgParameters)
+		if err != nil {
+			errorMsg = fmt.Sprintf("could not set PostgreSQL configuration options for pod %s: %v", podName, err)
+			continue
+		}
+
+		// Patroni's config endpoint is just a "proxy" to DCS. It is enough to patch it only once and it doesn't matter which pod is used
+		return nil
+	}
+
+	return fmt.Errorf(errorMsg)
+}
+
+func (c *Cluster) syncPublication(publication, dbName string, tables map[string]acidv1.StreamTable) error {
+	createPublications := make(map[string]string)
+	alterPublications := make(map[string]string)
+
+	defer func() {
+		if err := c.closeDbConn(); err != nil {
+			c.logger.Errorf("could not close database connection: %v", err)
+		}
+	}()
+
+	// check for existing publications
+	if err := c.initDbConnWithName(dbName); err != nil {
+		return fmt.Errorf("could not init database connection")
+	}
+
+	currentPublications, err := c.getPublications()
+	if err != nil {
+		return fmt.Errorf("could not get current publications: %v", err)
+	}
+
+	tableNames := make([]string, len(tables))
+	i := 0
+	for t := range tables {
+		tableName, schemaName := getTableSchema(t)
+		tableNames[i] = fmt.Sprintf("%s.%s", schemaName, tableName)
+		i++
+	}
+	sort.Strings(tableNames)
+	tableList := strings.Join(tableNames, ", ")
+
+	currentTables, exists := currentPublications[publication]
+	if !exists {
+		createPublications[publication] = tableList
+	} else if currentTables != tableList {
+		alterPublications[publication] = tableList
+	}
+
+	if len(createPublications)+len(alterPublications) == 0 {
+		return nil
+	}
+
+	for publicationName, tables := range createPublications {
+		if err = c.executeCreatePublication(publicationName, tables); err != nil {
+			return fmt.Errorf("creation of publication %q failed: %v", publicationName, err)
+		}
+	}
+	for publicationName, tables := range alterPublications {
+		if err = c.executeAlterPublication(publicationName, tables); err != nil {
+			return fmt.Errorf("update of publication %q failed: %v", publicationName, err)
+		}
+	}
+
+	return nil
+}
+
+func (c *Cluster) generateFabricEventStream(appId string) *zalandov1.FabricEventStream {
+	eventStreams := make([]zalandov1.EventStream, 0)
+
+	for _, stream := range c.Spec.Streams {
+		if stream.ApplicationId != appId {
+			continue
+		}
+		for tableName, table := range stream.Tables {
+			streamSource := c.getEventStreamSource(stream, tableName, table.IdColumn)
+			streamFlow := getEventStreamFlow(stream, table.PayloadColumn)
+			streamSink := getEventStreamSink(stream, table.EventType)
+
+			eventStreams = append(eventStreams, zalandov1.EventStream{
+				EventStreamFlow:   streamFlow,
+				EventStreamSink:   streamSink,
+				EventStreamSource: streamSource})
+		}
+	}
+
+	return &zalandov1.FabricEventStream{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: constants.EventStreamCRDApiVersion,
+			Kind:       constants.EventStreamCRDKind,
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        fmt.Sprintf("%s-%s", c.Name, appId),
+			Namespace:   c.Namespace,
+			Annotations: c.AnnotationsToPropagate(c.annotationsSet(nil)),
+			// make cluster StatefulSet the owner (like with connection pooler objects)
+			OwnerReferences: c.ownerReferences(),
+		},
+		Spec: zalandov1.FabricEventStreamSpec{
+			ApplicationId: appId,
+			EventStreams:  eventStreams,
+		},
+	}
+}
+
+func (c *Cluster) getEventStreamSource(stream acidv1.Stream, tableName, idColumn string) zalandov1.EventStreamSource {
+	table, schema := getTableSchema(tableName)
+	streamFilter := stream.Filter[tableName]
+	return zalandov1.EventStreamSource{
+		Type:             constants.EventStreamSourcePGType,
+		Schema:           schema,
+		EventStreamTable: getOutboxTable(table, idColumn),
+		Filter:           streamFilter,
+		Connection: c.getStreamConnection(
+			stream.Database,
+			constants.EventStreamSourceSlotPrefix+constants.UserRoleNameSuffix,
+			stream.ApplicationId),
+	}
+}
+
+func getEventStreamFlow(stream acidv1.Stream, payloadColumn string) zalandov1.EventStreamFlow {
+	return zalandov1.EventStreamFlow{
+		Type:          constants.EventStreamFlowPgGenericType,
+		PayloadColumn: payloadColumn,
+	}
+}
+
+func getEventStreamSink(stream acidv1.Stream, eventType string) zalandov1.EventStreamSink {
+	return zalandov1.EventStreamSink{
+		Type:         constants.EventStreamSinkNakadiType,
+		EventType:    eventType,
+		MaxBatchSize: stream.BatchSize,
+	}
+}
+
+func getTableSchema(fullTableName string) (tableName, schemaName string) {
+	schemaName = "public"
+	tableName = fullTableName
+	if strings.Contains(fullTableName, ".") {
+		schemaName = strings.Split(fullTableName, ".")[0]
+		tableName = strings.Split(fullTableName, ".")[1]
+	}
+
+	return tableName, schemaName
+}
+
+func getOutboxTable(tableName, idColumn string) zalandov1.EventStreamTable {
+	return zalandov1.EventStreamTable{
+		Name:     tableName,
+		IDColumn: idColumn,
+	}
+}
+
+func getSlotName(dbName, appId string) string {
+	return fmt.Sprintf("%s_%s_%s", constants.EventStreamSourceSlotPrefix, dbName, strings.Replace(appId, "-", "_", -1))
+}
+
+func (c *Cluster) getStreamConnection(database, user, appId string) zalandov1.Connection {
+	return zalandov1.Connection{
+		Url:        fmt.Sprintf("jdbc:postgresql://%s.%s/%s?user=%s&ssl=true&sslmode=require", c.Name, c.Namespace, database, user),
+		SlotName:   getSlotName(database, appId),
+		PluginType: constants.EventStreamSourcePluginType,
+		DBAuth: zalandov1.DBAuth{
+			Type:        constants.EventStreamSourceAuthType,
+			Name:        c.credentialSecretNameForCluster(user, c.Name),
+			UserKey:     "username",
+			PasswordKey: "password",
+		},
+	}
+}
+
+func (c *Cluster) syncStreams() error {
+
+	c.setProcessName("syncing streams")
+
+	_, err := c.KubeClient.CustomResourceDefinitions().Get(context.TODO(), constants.EventStreamCRDName, metav1.GetOptions{})
+	if k8sutil.ResourceNotFound(err) {
+		c.logger.Debugf("event stream CRD not installed, skipping")
+		return nil
+	}
+
+	slots := make(map[string]map[string]string)
+	publications := make(map[string]map[string]acidv1.StreamTable)
+
+	requiredPatroniConfig := c.Spec.Patroni
+	if len(requiredPatroniConfig.Slots) > 0 {
+		slots = requiredPatroniConfig.Slots
+	}
+
+	// gather list of required slots and publications
+	for _, stream := range c.Spec.Streams {
+		slot := map[string]string{
+			"database": stream.Database,
+			"plugin":   constants.EventStreamSourcePluginType,
+			"type":     "logical",
+		}
+		slotName := getSlotName(stream.Database, stream.ApplicationId)
+		if _, exists := slots[slotName]; !exists {
+			slots[slotName] = slot
+			publications[slotName] = stream.Tables
+		} else {
+			streamTables := publications[slotName]
+			for tableName, table := range stream.Tables {
+				if _, exists := streamTables[tableName]; !exists {
+					streamTables[tableName] = table
+				}
+			}
+			publications[slotName] = streamTables
+		}
+	}
+
+	// no slots = no streams defined
+	if len(slots) > 0 {
+		requiredPatroniConfig.Slots = slots
+	} else {
+		return nil
+	}
+
+	// add extra logical slots to Patroni config
+	c.logger.Debug("syncing Postgres config for logical decoding")
+	err = c.syncPostgresConfig(requiredPatroniConfig)
+	if err != nil {
+		return fmt.Errorf("failed to snyc Postgres config for event streaming: %v", err)
+	}
+
+	// next, create publications to each created slot
+	c.logger.Debug("syncing database publications")
+	for publication, tables := range publications {
+		// but first check for existing publications
+		dbName := slots[publication]["database"]
+		err = c.syncPublication(publication, dbName, tables)
+		if err != nil {
+			c.logger.Warningf("could not sync publication %q in database %q: %v", publication, dbName, err)
+		}
+	}
+
+	err = c.createOrUpdateStreams()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *Cluster) createOrUpdateStreams() error {
+
+	appIds := gatherApplicationIds(c.Spec.Streams)
+	for _, appId := range appIds {
+		fesName := fmt.Sprintf("%s-%s", c.Name, appId)
+		effectiveStreams, err := c.KubeClient.FabricEventStreams(c.Namespace).Get(context.TODO(), fesName, metav1.GetOptions{})
+		if err != nil {
+			if !k8sutil.ResourceNotFound(err) {
+				return fmt.Errorf("failed reading event stream %s: %v", fesName, err)
+			}
+
+			c.logger.Infof("event streams do not exist, create it")
+			err = c.createStreams(appId)
+			if err != nil {
+				return fmt.Errorf("failed creating event stream %s: %v", fesName, err)
+			}
+			c.logger.Infof("event stream %q has been successfully created", fesName)
+		} else {
+			desiredStreams := c.generateFabricEventStream(appId)
+			if !reflect.DeepEqual(effectiveStreams.Spec, desiredStreams.Spec) {
+				c.logger.Debug("updating event streams")
+				desiredStreams.ObjectMeta.ResourceVersion = effectiveStreams.ObjectMeta.ResourceVersion
+				err = c.updateStreams(desiredStreams)
+				if err != nil {
+					return fmt.Errorf("failed updating event stream %s: %v", fesName, err)
+				}
+				c.logger.Infof("event stream %q has been successfully updated", fesName)
+			}
+		}
+	}
+
+	return nil
+}
diff --git a/pkg/cluster/streams_test.go b/pkg/cluster/streams_test.go
new file mode 100644
index 000000000..89dd294ca
--- /dev/null
+++ b/pkg/cluster/streams_test.go
@@ -0,0 +1,244 @@
+package cluster
+
+import (
+	"encoding/json"
+	"fmt"
+	"reflect"
+	"strings"
+
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
+	v1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1"
+	fakezalandov1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/fake"
+	"github.com/zalando/postgres-operator/pkg/util"
+	"github.com/zalando/postgres-operator/pkg/util/config"
+	"github.com/zalando/postgres-operator/pkg/util/constants"
+	"github.com/zalando/postgres-operator/pkg/util/k8sutil"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes/fake"
+)
+
+func newFakeK8sStreamClient() (k8sutil.KubernetesClient, *fake.Clientset) {
+	zalandoClientSet := fakezalandov1.NewSimpleClientset()
+	clientSet := fake.NewSimpleClientset()
+
+	return k8sutil.KubernetesClient{
+		FabricEventStreamsGetter: zalandoClientSet.ZalandoV1(),
+		PostgresqlsGetter:        zalandoClientSet.AcidV1(),
+		PodsGetter:               clientSet.CoreV1(),
+		StatefulSetsGetter:       clientSet.AppsV1(),
+	}, clientSet
+}
+
+var (
+	clusterName string = "acid-test-cluster"
+	namespace   string = "default"
+	appId       string = "test-app"
+	dbName      string = "foo"
+	fesUser     string = constants.EventStreamSourceSlotPrefix + constants.UserRoleNameSuffix
+	fesName     string = fmt.Sprintf("%s-%s", clusterName, appId)
+	slotName    string = fmt.Sprintf("%s_%s_%s", constants.EventStreamSourceSlotPrefix, dbName, strings.Replace(appId, "-", "_", -1))
+
+	pg = acidv1.Postgresql{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Postgresql",
+			APIVersion: "acid.zalan.do/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      clusterName,
+			Namespace: namespace,
+		},
+		Spec: acidv1.PostgresSpec{
+			Databases: map[string]string{
+				dbName: dbName + constants.UserRoleNameSuffix,
+			},
+			Streams: []acidv1.Stream{
+				{
+					ApplicationId: appId,
+					Database:      "foo",
+					Tables: map[string]acidv1.StreamTable{
+						"data.bar": acidv1.StreamTable{
+							EventType:     "stream_type_a",
+							IdColumn:      "b_id",
+							PayloadColumn: "b_payload",
+						},
+					},
+					Filter: map[string]string{
+						"data.bar": "[?(@.source.txId > 500 && @.source.lsn > 123456)]",
+					},
+					BatchSize: uint32(100),
+				},
+			},
+			Volume: acidv1.Volume{
+				Size: "1Gi",
+			},
+		},
+	}
+
+	fes = &v1.FabricEventStream{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: constants.EventStreamCRDApiVersion,
+			Kind:       constants.EventStreamCRDKind,
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      fesName,
+			Namespace: namespace,
+			OwnerReferences: []metav1.OwnerReference{
+				metav1.OwnerReference{
+					APIVersion: "apps/v1",
+					Kind:       "StatefulSet",
+					Name:       "acid-test-cluster",
+					Controller: util.True(),
+				},
+			},
+		},
+		Spec: v1.FabricEventStreamSpec{
+			ApplicationId: appId,
+			EventStreams: []v1.EventStream{
+				{
+					EventStreamFlow: v1.EventStreamFlow{
+						PayloadColumn: "b_payload",
+						Type:          constants.EventStreamFlowPgGenericType,
+					},
+					EventStreamSink: v1.EventStreamSink{
+						EventType:    "stream_type_a",
+						MaxBatchSize: uint32(100),
+						Type:         constants.EventStreamSinkNakadiType,
+					},
+					EventStreamSource: v1.EventStreamSource{
+						Filter: "[?(@.source.txId > 500 && @.source.lsn > 123456)]",
+						Connection: v1.Connection{
+							DBAuth: v1.DBAuth{
+								Name:        fmt.Sprintf("fes-user.%s.credentials.postgresql.acid.zalan.do", clusterName),
+								PasswordKey: "password",
+								Type:        constants.EventStreamSourceAuthType,
+								UserKey:     "username",
+							},
+							Url:        fmt.Sprintf("jdbc:postgresql://%s.%s/foo?user=%s&ssl=true&sslmode=require", clusterName, namespace, fesUser),
+							SlotName:   slotName,
+							PluginType: constants.EventStreamSourcePluginType,
+						},
+						Schema: "data",
+						EventStreamTable: v1.EventStreamTable{
+							IDColumn: "b_id",
+							Name:     "bar",
+						},
+						Type: constants.EventStreamSourcePGType,
+					},
+				},
+			},
+		},
+	}
+)
+
+func TestGenerateFabricEventStream(t *testing.T) {
+	client, _ := newFakeK8sStreamClient()
+
+	var cluster = New(
+		Config{
+			OpConfig: config.Config{
+				Auth: config.Auth{
+					SecretNameTemplate: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}",
+				},
+				PodManagementPolicy: "ordered_ready",
+				Resources: config.Resources{
+					ClusterLabels:        map[string]string{"application": "spilo"},
+					ClusterNameLabel:     "cluster-name",
+					DefaultCPURequest:    "300m",
+					DefaultCPULimit:      "300m",
+					DefaultMemoryRequest: "300Mi",
+					DefaultMemoryLimit:   "300Mi",
+					PodRoleLabel:         "spilo-role",
+				},
+			},
+		}, client, pg, logger, eventRecorder)
+
+	cluster.Name = clusterName
+	cluster.Namespace = namespace
+
+	_, err := cluster.createStatefulSet()
+	assert.NoError(t, err)
+
+	err = cluster.createOrUpdateStreams()
+	assert.NoError(t, err)
+
+	result := cluster.generateFabricEventStream(appId)
+
+	if !reflect.DeepEqual(result, fes) {
+		t.Errorf("Malformed FabricEventStream, expected %#v, got %#v", fes, result)
+	}
+
+	streamCRD, err := cluster.KubeClient.FabricEventStreams(namespace).Get(context.TODO(), fesName, metav1.GetOptions{})
+	assert.NoError(t, err)
+
+	if !reflect.DeepEqual(streamCRD, fes) {
+		t.Errorf("Malformed FabricEventStream, expected %#v, got %#v", fes, streamCRD)
+	}
+}
+
+func TestUpdateFabricEventStream(t *testing.T) {
+	client, _ := newFakeK8sStreamClient()
+
+	var cluster = New(
+		Config{
+			OpConfig: config.Config{
+				PodManagementPolicy: "ordered_ready",
+				Resources: config.Resources{
+					ClusterLabels:        map[string]string{"application": "spilo"},
+					ClusterNameLabel:     "cluster-name",
+					DefaultCPURequest:    "300m",
+					DefaultCPULimit:      "300m",
+					DefaultMemoryRequest: "300Mi",
+					DefaultMemoryLimit:   "300Mi",
+					PodRoleLabel:         "spilo-role",
+				},
+			},
+		}, client, pg, logger, eventRecorder)
+
+	_, err := cluster.KubeClient.Postgresqls(namespace).Create(
+		context.TODO(), &pg, metav1.CreateOptions{})
+	assert.NoError(t, err)
+	err = cluster.createOrUpdateStreams()
+	assert.NoError(t, err)
+
+	var pgSpec acidv1.PostgresSpec
+	pgSpec.Streams = []acidv1.Stream{
+		{
+			ApplicationId: appId,
+			Database:      dbName,
+			Tables: map[string]acidv1.StreamTable{
+				"data.bar": acidv1.StreamTable{
+					EventType:     "stream_type_b",
+					IdColumn:      "b_id",
+					PayloadColumn: "b_payload",
+				},
+			},
+			BatchSize: uint32(250),
+		},
+	}
+	patch, err := json.Marshal(struct {
+		PostgresSpec interface{} `json:"spec"`
+	}{&pgSpec})
+	assert.NoError(t, err)
+
+	pgPatched, err := cluster.KubeClient.Postgresqls(namespace).Patch(
+		context.TODO(), cluster.Name, types.MergePatchType, patch, metav1.PatchOptions{}, "spec")
+	assert.NoError(t, err)
+
+	cluster.Postgresql.Spec = pgPatched.Spec
+	err = cluster.createOrUpdateStreams()
+	assert.NoError(t, err)
+
+	streamCRD, err := cluster.KubeClient.FabricEventStreams(namespace).Get(context.TODO(), fesName, metav1.GetOptions{})
+	assert.NoError(t, err)
+
+	result := cluster.generateFabricEventStream(appId)
+	if !reflect.DeepEqual(result, streamCRD) {
+		t.Errorf("Malformed FabricEventStream, expected %#v, got %#v", streamCRD, result)
+	}
+}
diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go
index bbf023764..a897ff318 100644
--- a/pkg/cluster/sync.go
+++ b/pkg/cluster/sync.go
@@ -81,7 +81,7 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
 		return err
 	}
 
-	c.logger.Debugf("syncing statefulsets")
+	c.logger.Debug("syncing statefulsets")
 	if err = c.syncStatefulSet(); err != nil {
 		if !k8sutil.ResourceAlreadyExists(err) {
 			err = fmt.Errorf("could not sync statefulsets: %v", err)
@@ -107,17 +107,17 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
 
 	// create database objects unless we are running without pods or disabled that feature explicitly
 	if !(c.databaseAccessDisabled() || c.getNumberOfInstances(&newSpec.Spec) <= 0 || c.Spec.StandbyCluster != nil) {
-		c.logger.Debugf("syncing roles")
+		c.logger.Debug("syncing roles")
 		if err = c.syncRoles(); err != nil {
 			err = fmt.Errorf("could not sync roles: %v", err)
 			return err
 		}
-		c.logger.Debugf("syncing databases")
+		c.logger.Debug("syncing databases")
 		if err = c.syncDatabases(); err != nil {
 			err = fmt.Errorf("could not sync databases: %v", err)
 			return err
 		}
-		c.logger.Debugf("syncing prepared databases with schemas")
+		c.logger.Debug("syncing prepared databases with schemas")
 		if err = c.syncPreparedDatabases(); err != nil {
 			err = fmt.Errorf("could not sync prepared database: %v", err)
 			return err
@@ -129,6 +129,14 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
 		return fmt.Errorf("could not sync connection pooler: %v", err)
 	}
 
+	if len(c.Spec.Streams) > 0 {
+		c.logger.Debug("syncing streams")
+		if err = c.syncStreams(); err != nil {
+			err = fmt.Errorf("could not sync streams: %v", err)
+			return err
+		}
+	}
+
 	// Major version upgrade must only run after success of all earlier operations, must remain last item in sync
 	if err := c.majorVersionUpgrade(); err != nil {
 		c.logger.Errorf("major version upgrade failed: %v", err)
@@ -401,7 +409,7 @@ func (c *Cluster) syncStatefulSet() error {
 	}
 
 	// get Postgres config, compare with manifest and update via Patroni PATCH endpoint if it differs
-	// Patroni's config endpoint is just a "proxy" to DCS. It is enough to patch it only once and it doesn't matter which pod is used.
+	// Patroni's config endpoint is just a "proxy" to DCS. It is enough to patch it only once and it doesn't matter which pod is used
 	for i, pod := range pods {
 		patroniConfig, pgParameters, err := c.getPatroniConfig(&pod)
 		if err != nil {
@@ -414,11 +422,13 @@ func (c *Cluster) syncStatefulSet() error {
 		// empty config probably means cluster is not fully initialized yet, e.g. restoring from backup
 		// do not attempt a restart
 		if !reflect.DeepEqual(patroniConfig, acidv1.Patroni{}) || len(pgParameters) > 0 {
-			restartMasterFirst, err = c.checkAndSetGlobalPostgreSQLConfiguration(&pod, patroniConfig, pgParameters)
+			// compare config returned from Patroni with what is specified in the manifest
+			restartMasterFirst, err = c.checkAndSetGlobalPostgreSQLConfiguration(&pod, patroniConfig, c.Spec.Patroni, pgParameters, c.Spec.Parameters)
 			if err != nil {
 				c.logger.Warningf("could not set PostgreSQL configuration options for pod %s: %v", pods[i].Name, err)
 				continue
 			}
+
 			// it could take up to LoopWait to apply the config
 			time.Sleep(time.Duration(restartWait)*time.Second + time.Second*2)
 			break
@@ -526,40 +536,39 @@ func (c *Cluster) AnnotationsToPropagate(annotations map[string]string) map[stri
 
 // checkAndSetGlobalPostgreSQLConfiguration checks whether cluster-wide API parameters
 // (like max_connections) have changed and if necessary sets it via the Patroni API
-func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration(pod *v1.Pod, patroniConfig acidv1.Patroni, effectivePgParameters map[string]string) (bool, error) {
+func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration(pod *v1.Pod, effectivePatroniConfig, desiredPatroniConfig acidv1.Patroni, effectivePgParameters, desiredPgParameters map[string]string) (bool, error) {
 	configToSet := make(map[string]interface{})
 	parametersToSet := make(map[string]string)
 	restartMaster := make([]bool, 0)
 	requiresMasterRestart := false
 
-	// compare options from config with c.Spec.Patroni from manifest
-	desiredPatroniConfig := c.Spec.Patroni
-	if desiredPatroniConfig.LoopWait > 0 && desiredPatroniConfig.LoopWait != patroniConfig.LoopWait {
+	// compare effective and desired Patroni config options
+	if desiredPatroniConfig.LoopWait > 0 && desiredPatroniConfig.LoopWait != effectivePatroniConfig.LoopWait {
 		configToSet["loop_wait"] = desiredPatroniConfig.LoopWait
 	}
-	if desiredPatroniConfig.MaximumLagOnFailover > 0 && desiredPatroniConfig.MaximumLagOnFailover != patroniConfig.MaximumLagOnFailover {
+	if desiredPatroniConfig.MaximumLagOnFailover > 0 && desiredPatroniConfig.MaximumLagOnFailover != effectivePatroniConfig.MaximumLagOnFailover {
 		configToSet["maximum_lag_on_failover"] = desiredPatroniConfig.MaximumLagOnFailover
 	}
-	if desiredPatroniConfig.PgHba != nil && !reflect.DeepEqual(desiredPatroniConfig.PgHba, patroniConfig.PgHba) {
+	if desiredPatroniConfig.PgHba != nil && !reflect.DeepEqual(desiredPatroniConfig.PgHba, effectivePatroniConfig.PgHba) {
 		configToSet["pg_hba"] = desiredPatroniConfig.PgHba
 	}
-	if desiredPatroniConfig.RetryTimeout > 0 && desiredPatroniConfig.RetryTimeout != patroniConfig.RetryTimeout {
+	if desiredPatroniConfig.RetryTimeout > 0 && desiredPatroniConfig.RetryTimeout != effectivePatroniConfig.RetryTimeout {
 		configToSet["retry_timeout"] = desiredPatroniConfig.RetryTimeout
 	}
-	if desiredPatroniConfig.SynchronousMode != patroniConfig.SynchronousMode {
+	if desiredPatroniConfig.SynchronousMode != effectivePatroniConfig.SynchronousMode {
 		configToSet["synchronous_mode"] = desiredPatroniConfig.SynchronousMode
 	}
-	if desiredPatroniConfig.SynchronousModeStrict != patroniConfig.SynchronousModeStrict {
+	if desiredPatroniConfig.SynchronousModeStrict != effectivePatroniConfig.SynchronousModeStrict {
 		configToSet["synchronous_mode_strict"] = desiredPatroniConfig.SynchronousModeStrict
 	}
-	if desiredPatroniConfig.TTL > 0 && desiredPatroniConfig.TTL != patroniConfig.TTL {
+	if desiredPatroniConfig.TTL > 0 && desiredPatroniConfig.TTL != effectivePatroniConfig.TTL {
 		configToSet["ttl"] = desiredPatroniConfig.TTL
 	}
 
 	// check if specified slots exist in config and if they differ
 	slotsToSet := make(map[string]map[string]string)
 	for slotName, desiredSlot := range desiredPatroniConfig.Slots {
-		if effectiveSlot, exists := patroniConfig.Slots[slotName]; exists {
+		if effectiveSlot, exists := effectivePatroniConfig.Slots[slotName]; exists {
 			if reflect.DeepEqual(desiredSlot, effectiveSlot) {
 				continue
 			}
@@ -570,8 +579,7 @@ func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration(pod *v1.Pod, patroniC
 		configToSet["slots"] = slotsToSet
 	}
 
-	// compare parameters under postgresql section with c.Spec.Postgresql.Parameters from manifest
-	desiredPgParameters := c.Spec.Parameters
+	// compare effective and desired parameters under postgresql section in Patroni config
 	for desiredOption, desiredValue := range desiredPgParameters {
 		effectiveValue := effectivePgParameters[desiredOption]
 		if isBootstrapOnlyParameter(desiredOption) && (effectiveValue != desiredValue) {
@@ -614,7 +622,7 @@ func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration(pod *v1.Pod, patroniC
 	c.logger.Debugf("patching Postgres config via Patroni API on pod %s with following options: %s",
 		podName, configToSetJson)
 	if err = c.patroni.SetConfig(pod, configToSet); err != nil {
-		return requiresMasterRestart, fmt.Errorf("could not patch postgres parameters with a pod %s: %v", podName, err)
+		return requiresMasterRestart, fmt.Errorf("could not patch postgres parameters within pod %s: %v", podName, err)
 	}
 
 	return requiresMasterRestart, nil
diff --git a/pkg/cluster/sync_test.go b/pkg/cluster/sync_test.go
index 226555a66..ea73fb97c 100644
--- a/pkg/cluster/sync_test.go
+++ b/pkg/cluster/sync_test.go
@@ -256,7 +256,7 @@ func TestCheckAndSetGlobalPostgreSQLConfiguration(t *testing.T) {
 	}
 
 	for _, tt := range tests {
-		requireMasterRestart, err := cluster.checkAndSetGlobalPostgreSQLConfiguration(mockPod, tt.patroni, tt.pgParams)
+		requireMasterRestart, err := cluster.checkAndSetGlobalPostgreSQLConfiguration(mockPod, tt.patroni, cluster.Spec.Patroni, tt.pgParams, cluster.Spec.Parameters)
 		assert.NoError(t, err)
 		if requireMasterRestart != tt.restartMaster {
 			t.Errorf("%s - %s: unexpect master restart strategy, got %v, expected %v", testName, tt.subtest, requireMasterRestart, tt.restartMaster)
diff --git a/pkg/generated/clientset/versioned/clientset.go b/pkg/generated/clientset/versioned/clientset.go
index e0e97cf09..50f0ac841 100644
--- a/pkg/generated/clientset/versioned/clientset.go
+++ b/pkg/generated/clientset/versioned/clientset.go
@@ -28,6 +28,7 @@ import (
 	"fmt"
 
 	acidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1"
+	zalandov1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1"
 	discovery "k8s.io/client-go/discovery"
 	rest "k8s.io/client-go/rest"
 	flowcontrol "k8s.io/client-go/util/flowcontrol"
@@ -36,13 +37,15 @@ import (
 type Interface interface {
 	Discovery() discovery.DiscoveryInterface
 	AcidV1() acidv1.AcidV1Interface
+	ZalandoV1() zalandov1.ZalandoV1Interface
 }
 
 // Clientset contains the clients for groups. Each group has exactly one
 // version included in a Clientset.
 type Clientset struct {
 	*discovery.DiscoveryClient
-	acidV1 *acidv1.AcidV1Client
+	acidV1    *acidv1.AcidV1Client
+	zalandoV1 *zalandov1.ZalandoV1Client
 }
 
 // AcidV1 retrieves the AcidV1Client
@@ -50,6 +53,11 @@ func (c *Clientset) AcidV1() acidv1.AcidV1Interface {
 	return c.acidV1
 }
 
+// ZalandoV1 retrieves the ZalandoV1Client
+func (c *Clientset) ZalandoV1() zalandov1.ZalandoV1Interface {
+	return c.zalandoV1
+}
+
 // Discovery retrieves the DiscoveryClient
 func (c *Clientset) Discovery() discovery.DiscoveryInterface {
 	if c == nil {
@@ -75,6 +83,10 @@ func NewForConfig(c *rest.Config) (*Clientset, error) {
 	if err != nil {
 		return nil, err
 	}
+	cs.zalandoV1, err = zalandov1.NewForConfig(&configShallowCopy)
+	if err != nil {
+		return nil, err
+	}
 
 	cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfig(&configShallowCopy)
 	if err != nil {
@@ -88,6 +100,7 @@ func NewForConfig(c *rest.Config) (*Clientset, error) {
 func NewForConfigOrDie(c *rest.Config) *Clientset {
 	var cs Clientset
 	cs.acidV1 = acidv1.NewForConfigOrDie(c)
+	cs.zalandoV1 = zalandov1.NewForConfigOrDie(c)
 
 	cs.DiscoveryClient = discovery.NewDiscoveryClientForConfigOrDie(c)
 	return &cs
@@ -97,6 +110,7 @@ func NewForConfigOrDie(c *rest.Config) *Clientset {
 func New(c rest.Interface) *Clientset {
 	var cs Clientset
 	cs.acidV1 = acidv1.New(c)
+	cs.zalandoV1 = zalandov1.New(c)
 
 	cs.DiscoveryClient = discovery.NewDiscoveryClient(c)
 	return &cs
diff --git a/pkg/generated/clientset/versioned/fake/clientset_generated.go b/pkg/generated/clientset/versioned/fake/clientset_generated.go
index 25daca07c..4c94d23a6 100644
--- a/pkg/generated/clientset/versioned/fake/clientset_generated.go
+++ b/pkg/generated/clientset/versioned/fake/clientset_generated.go
@@ -28,6 +28,8 @@ import (
 	clientset "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned"
 	acidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1"
 	fakeacidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake"
+	zalandov1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1"
+	fakezalandov1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/client-go/discovery"
@@ -89,3 +91,8 @@ var (
 func (c *Clientset) AcidV1() acidv1.AcidV1Interface {
 	return &fakeacidv1.FakeAcidV1{Fake: &c.Fake}
 }
+
+// ZalandoV1 retrieves the ZalandoV1Client
+func (c *Clientset) ZalandoV1() zalandov1.ZalandoV1Interface {
+	return &fakezalandov1.FakeZalandoV1{Fake: &c.Fake}
+}
diff --git a/pkg/generated/clientset/versioned/fake/register.go b/pkg/generated/clientset/versioned/fake/register.go
index d63cab2ff..313eeacc2 100644
--- a/pkg/generated/clientset/versioned/fake/register.go
+++ b/pkg/generated/clientset/versioned/fake/register.go
@@ -26,6 +26,7 @@ package fake
 
 import (
 	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
+	zalandov1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	runtime "k8s.io/apimachinery/pkg/runtime"
 	schema "k8s.io/apimachinery/pkg/runtime/schema"
@@ -38,6 +39,7 @@ var codecs = serializer.NewCodecFactory(scheme)
 
 var localSchemeBuilder = runtime.SchemeBuilder{
 	acidv1.AddToScheme,
+	zalandov1.AddToScheme,
 }
 
 // AddToScheme adds all types of this clientset into the given scheme. This allows composition
diff --git a/pkg/generated/clientset/versioned/scheme/register.go b/pkg/generated/clientset/versioned/scheme/register.go
index beb803602..823909bcb 100644
--- a/pkg/generated/clientset/versioned/scheme/register.go
+++ b/pkg/generated/clientset/versioned/scheme/register.go
@@ -26,6 +26,7 @@ package scheme
 
 import (
 	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
+	zalandov1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	runtime "k8s.io/apimachinery/pkg/runtime"
 	schema "k8s.io/apimachinery/pkg/runtime/schema"
@@ -38,6 +39,7 @@ var Codecs = serializer.NewCodecFactory(Scheme)
 var ParameterCodec = runtime.NewParameterCodec(Scheme)
 var localSchemeBuilder = runtime.SchemeBuilder{
 	acidv1.AddToScheme,
+	zalandov1.AddToScheme,
 }
 
 // AddToScheme adds all types of this clientset into the given scheme. This allows composition
diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1/doc.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1/doc.go
new file mode 100644
index 000000000..ba729b8ca
--- /dev/null
+++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1/doc.go
@@ -0,0 +1,26 @@
+/*
+Copyright 2022 Compose, Zalando SE
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+// This package has the automatically generated typed clients.
+package v1
diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1/fabriceventstream.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fabriceventstream.go
new file mode 100644
index 000000000..26c3431f2
--- /dev/null
+++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fabriceventstream.go
@@ -0,0 +1,184 @@
+/*
+Copyright 2022 Compose, Zalando SE
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package v1
+
+import (
+	"context"
+	"time"
+
+	v1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1"
+	scheme "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/scheme"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	types "k8s.io/apimachinery/pkg/types"
+	watch "k8s.io/apimachinery/pkg/watch"
+	rest "k8s.io/client-go/rest"
+)
+
+// FabricEventStreamsGetter has a method to return a FabricEventStreamInterface.
+// A group's client should implement this interface.
+type FabricEventStreamsGetter interface {
+	FabricEventStreams(namespace string) FabricEventStreamInterface
+}
+
+// FabricEventStreamInterface has methods to work with FabricEventStream resources.
+type FabricEventStreamInterface interface {
+	Create(ctx context.Context, fabricEventStream *v1.FabricEventStream, opts metav1.CreateOptions) (*v1.FabricEventStream, error)
+	Update(ctx context.Context, fabricEventStream *v1.FabricEventStream, opts metav1.UpdateOptions) (*v1.FabricEventStream, error)
+	Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error
+	DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error
+	Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.FabricEventStream, error)
+	List(ctx context.Context, opts metav1.ListOptions) (*v1.FabricEventStreamList, error)
+	Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error)
+	Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.FabricEventStream, err error)
+	FabricEventStreamExpansion
+}
+
+// fabricEventStreams implements FabricEventStreamInterface
+type fabricEventStreams struct {
+	client rest.Interface
+	ns     string
+}
+
+// newFabricEventStreams returns a FabricEventStreams
+func newFabricEventStreams(c *ZalandoV1Client, namespace string) *fabricEventStreams {
+	return &fabricEventStreams{
+		client: c.RESTClient(),
+		ns:     namespace,
+	}
+}
+
+// Get takes name of the fabricEventStream, and returns the corresponding fabricEventStream object, and an error if there is any.
+func (c *fabricEventStreams) Get(ctx context.Context, name string, options metav1.GetOptions) (result *v1.FabricEventStream, err error) {
+	result = &v1.FabricEventStream{}
+	err = c.client.Get().
+		Namespace(c.ns).
+		Resource("fabriceventstreams").
+		Name(name).
+		VersionedParams(&options, scheme.ParameterCodec).
+		Do(ctx).
+		Into(result)
+	return
+}
+
+// List takes label and field selectors, and returns the list of FabricEventStreams that match those selectors.
+func (c *fabricEventStreams) List(ctx context.Context, opts metav1.ListOptions) (result *v1.FabricEventStreamList, err error) {
+	var timeout time.Duration
+	if opts.TimeoutSeconds != nil {
+		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
+	}
+	result = &v1.FabricEventStreamList{}
+	err = c.client.Get().
+		Namespace(c.ns).
+		Resource("fabriceventstreams").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Timeout(timeout).
+		Do(ctx).
+		Into(result)
+	return
+}
+
+// Watch returns a watch.Interface that watches the requested fabricEventStreams.
+func (c *fabricEventStreams) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
+	var timeout time.Duration
+	if opts.TimeoutSeconds != nil {
+		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
+	}
+	opts.Watch = true
+	return c.client.Get().
+		Namespace(c.ns).
+		Resource("fabriceventstreams").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Timeout(timeout).
+		Watch(ctx)
+}
+
+// Create takes the representation of a fabricEventStream and creates it.  Returns the server's representation of the fabricEventStream, and an error, if there is any.
+func (c *fabricEventStreams) Create(ctx context.Context, fabricEventStream *v1.FabricEventStream, opts metav1.CreateOptions) (result *v1.FabricEventStream, err error) {
+	result = &v1.FabricEventStream{}
+	err = c.client.Post().
+		Namespace(c.ns).
+		Resource("fabriceventstreams").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Body(fabricEventStream).
+		Do(ctx).
+		Into(result)
+	return
+}
+
+// Update takes the representation of a fabricEventStream and updates it. Returns the server's representation of the fabricEventStream, and an error, if there is any.
+func (c *fabricEventStreams) Update(ctx context.Context, fabricEventStream *v1.FabricEventStream, opts metav1.UpdateOptions) (result *v1.FabricEventStream, err error) {
+	result = &v1.FabricEventStream{}
+	err = c.client.Put().
+		Namespace(c.ns).
+		Resource("fabriceventstreams").
+		Name(fabricEventStream.Name).
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Body(fabricEventStream).
+		Do(ctx).
+		Into(result)
+	return
+}
+
+// Delete takes name of the fabricEventStream and deletes it. Returns an error if one occurs.
+func (c *fabricEventStreams) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error {
+	return c.client.Delete().
+		Namespace(c.ns).
+		Resource("fabriceventstreams").
+		Name(name).
+		Body(&opts).
+		Do(ctx).
+		Error()
+}
+
+// DeleteCollection deletes a collection of objects.
+func (c *fabricEventStreams) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error {
+	var timeout time.Duration
+	if listOpts.TimeoutSeconds != nil {
+		timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second
+	}
+	return c.client.Delete().
+		Namespace(c.ns).
+		Resource("fabriceventstreams").
+		VersionedParams(&listOpts, scheme.ParameterCodec).
+		Timeout(timeout).
+		Body(&opts).
+		Do(ctx).
+		Error()
+}
+
+// Patch applies the patch and returns the patched fabricEventStream.
+func (c *fabricEventStreams) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.FabricEventStream, err error) {
+	result = &v1.FabricEventStream{}
+	err = c.client.Patch(pt).
+		Namespace(c.ns).
+		Resource("fabriceventstreams").
+		Name(name).
+		SubResource(subresources...).
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Body(data).
+		Do(ctx).
+		Into(result)
+	return
+}
diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/doc.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/doc.go
new file mode 100644
index 000000000..380725443
--- /dev/null
+++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/doc.go
@@ -0,0 +1,26 @@
+/*
+Copyright 2022 Compose, Zalando SE
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+// Package fake has the automatically generated clients.
+package fake
diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/fake_fabriceventstream.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/fake_fabriceventstream.go
new file mode 100644
index 000000000..8d76e06de
--- /dev/null
+++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/fake_fabriceventstream.go
@@ -0,0 +1,136 @@
+/*
+Copyright 2022 Compose, Zalando SE
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package fake
+
+import (
+	"context"
+
+	zalandoorgv1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	labels "k8s.io/apimachinery/pkg/labels"
+	schema "k8s.io/apimachinery/pkg/runtime/schema"
+	types "k8s.io/apimachinery/pkg/types"
+	watch "k8s.io/apimachinery/pkg/watch"
+	testing "k8s.io/client-go/testing"
+)
+
+// FakeFabricEventStreams implements FabricEventStreamInterface
+type FakeFabricEventStreams struct {
+	Fake *FakeZalandoV1
+	ns   string
+}
+
+var fabriceventstreamsResource = schema.GroupVersionResource{Group: "zalando.org", Version: "v1", Resource: "fabriceventstreams"}
+
+var fabriceventstreamsKind = schema.GroupVersionKind{Group: "zalando.org", Version: "v1", Kind: "FabricEventStream"}
+
+// Get takes name of the fabricEventStream, and returns the corresponding fabricEventStream object, and an error if there is any.
+func (c *FakeFabricEventStreams) Get(ctx context.Context, name string, options v1.GetOptions) (result *zalandoorgv1.FabricEventStream, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewGetAction(fabriceventstreamsResource, c.ns, name), &zalandoorgv1.FabricEventStream{})
+
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*zalandoorgv1.FabricEventStream), err
+}
+
+// List takes label and field selectors, and returns the list of FabricEventStreams that match those selectors.
+func (c *FakeFabricEventStreams) List(ctx context.Context, opts v1.ListOptions) (result *zalandoorgv1.FabricEventStreamList, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewListAction(fabriceventstreamsResource, fabriceventstreamsKind, c.ns, opts), &zalandoorgv1.FabricEventStreamList{})
+
+	if obj == nil {
+		return nil, err
+	}
+
+	label, _, _ := testing.ExtractFromListOptions(opts)
+	if label == nil {
+		label = labels.Everything()
+	}
+	list := &zalandoorgv1.FabricEventStreamList{ListMeta: obj.(*zalandoorgv1.FabricEventStreamList).ListMeta}
+	for _, item := range obj.(*zalandoorgv1.FabricEventStreamList).Items {
+		if label.Matches(labels.Set(item.Labels)) {
+			list.Items = append(list.Items, item)
+		}
+	}
+	return list, err
+}
+
+// Watch returns a watch.Interface that watches the requested fabricEventStreams.
+func (c *FakeFabricEventStreams) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) {
+	return c.Fake.
+		InvokesWatch(testing.NewWatchAction(fabriceventstreamsResource, c.ns, opts))
+
+}
+
+// Create takes the representation of a fabricEventStream and creates it.  Returns the server's representation of the fabricEventStream, and an error, if there is any.
+func (c *FakeFabricEventStreams) Create(ctx context.Context, fabricEventStream *zalandoorgv1.FabricEventStream, opts v1.CreateOptions) (result *zalandoorgv1.FabricEventStream, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewCreateAction(fabriceventstreamsResource, c.ns, fabricEventStream), &zalandoorgv1.FabricEventStream{})
+
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*zalandoorgv1.FabricEventStream), err
+}
+
+// Update takes the representation of a fabricEventStream and updates it. Returns the server's representation of the fabricEventStream, and an error, if there is any.
+func (c *FakeFabricEventStreams) Update(ctx context.Context, fabricEventStream *zalandoorgv1.FabricEventStream, opts v1.UpdateOptions) (result *zalandoorgv1.FabricEventStream, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewUpdateAction(fabriceventstreamsResource, c.ns, fabricEventStream), &zalandoorgv1.FabricEventStream{})
+
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*zalandoorgv1.FabricEventStream), err
+}
+
+// Delete takes name of the fabricEventStream and deletes it. Returns an error if one occurs.
+func (c *FakeFabricEventStreams) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error {
+	_, err := c.Fake.
+		Invokes(testing.NewDeleteAction(fabriceventstreamsResource, c.ns, name), &zalandoorgv1.FabricEventStream{})
+
+	return err
+}
+
+// DeleteCollection deletes a collection of objects.
+func (c *FakeFabricEventStreams) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error {
+	action := testing.NewDeleteCollectionAction(fabriceventstreamsResource, c.ns, listOpts)
+
+	_, err := c.Fake.Invokes(action, &zalandoorgv1.FabricEventStreamList{})
+	return err
+}
+
+// Patch applies the patch and returns the patched fabricEventStream.
+func (c *FakeFabricEventStreams) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *zalandoorgv1.FabricEventStream, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewPatchSubresourceAction(fabriceventstreamsResource, c.ns, name, pt, data, subresources...), &zalandoorgv1.FabricEventStream{})
+
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*zalandoorgv1.FabricEventStream), err
+}
diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/fake_zalando.org_client.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/fake_zalando.org_client.go
new file mode 100644
index 000000000..17786c9a7
--- /dev/null
+++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/fake_zalando.org_client.go
@@ -0,0 +1,46 @@
+/*
+Copyright 2022 Compose, Zalando SE
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package fake
+
+import (
+	v1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1"
+	rest "k8s.io/client-go/rest"
+	testing "k8s.io/client-go/testing"
+)
+
+type FakeZalandoV1 struct {
+	*testing.Fake
+}
+
+func (c *FakeZalandoV1) FabricEventStreams(namespace string) v1.FabricEventStreamInterface {
+	return &FakeFabricEventStreams{c, namespace}
+}
+
+// RESTClient returns a RESTClient that is used to communicate
+// with API server by this client implementation.
+func (c *FakeZalandoV1) RESTClient() rest.Interface {
+	var ret *rest.RESTClient
+	return ret
+}
diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1/generated_expansion.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1/generated_expansion.go
new file mode 100644
index 000000000..6a776af84
--- /dev/null
+++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1/generated_expansion.go
@@ -0,0 +1,27 @@
+/*
+Copyright 2022 Compose, Zalando SE
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package v1
+
+type FabricEventStreamExpansion interface{}
diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1/zalando.org_client.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1/zalando.org_client.go
new file mode 100644
index 000000000..937fd6b7e
--- /dev/null
+++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1/zalando.org_client.go
@@ -0,0 +1,95 @@
+/*
+Copyright 2022 Compose, Zalando SE
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package v1
+
+import (
+	v1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1"
+	"github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/scheme"
+	rest "k8s.io/client-go/rest"
+)
+
+type ZalandoV1Interface interface {
+	RESTClient() rest.Interface
+	FabricEventStreamsGetter
+}
+
+// ZalandoV1Client is used to interact with features provided by the zalando.org group.
+type ZalandoV1Client struct {
+	restClient rest.Interface
+}
+
+func (c *ZalandoV1Client) FabricEventStreams(namespace string) FabricEventStreamInterface {
+	return newFabricEventStreams(c, namespace)
+}
+
+// NewForConfig creates a new ZalandoV1Client for the given config.
+func NewForConfig(c *rest.Config) (*ZalandoV1Client, error) {
+	config := *c
+	if err := setConfigDefaults(&config); err != nil {
+		return nil, err
+	}
+	client, err := rest.RESTClientFor(&config)
+	if err != nil {
+		return nil, err
+	}
+	return &ZalandoV1Client{client}, nil
+}
+
+// NewForConfigOrDie creates a new ZalandoV1Client for the given config and
+// panics if there is an error in the config.
+func NewForConfigOrDie(c *rest.Config) *ZalandoV1Client {
+	client, err := NewForConfig(c)
+	if err != nil {
+		panic(err)
+	}
+	return client
+}
+
+// New creates a new ZalandoV1Client for the given RESTClient.
+func New(c rest.Interface) *ZalandoV1Client {
+	return &ZalandoV1Client{c}
+}
+
+func setConfigDefaults(config *rest.Config) error {
+	gv := v1.SchemeGroupVersion
+	config.GroupVersion = &gv
+	config.APIPath = "/apis"
+	config.NegotiatedSerializer = scheme.Codecs.WithoutConversion()
+
+	if config.UserAgent == "" {
+		config.UserAgent = rest.DefaultKubernetesUserAgent()
+	}
+
+	return nil
+}
+
+// RESTClient returns a RESTClient that is used to communicate
+// with API server by this client implementation.
+func (c *ZalandoV1Client) RESTClient() rest.Interface {
+	if c == nil {
+		return nil
+	}
+	return c.restClient
+}
diff --git a/pkg/generated/informers/externalversions/factory.go b/pkg/generated/informers/externalversions/factory.go
index 20a7f415f..0842aa5ae 100644
--- a/pkg/generated/informers/externalversions/factory.go
+++ b/pkg/generated/informers/externalversions/factory.go
@@ -32,6 +32,7 @@ import (
 	versioned "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned"
 	acidzalando "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/acid.zalan.do"
 	internalinterfaces "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/internalinterfaces"
+	zalandoorg "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/zalando.org"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	runtime "k8s.io/apimachinery/pkg/runtime"
 	schema "k8s.io/apimachinery/pkg/runtime/schema"
@@ -179,8 +180,13 @@ type SharedInformerFactory interface {
 	WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool
 
 	Acid() acidzalando.Interface
+	Zalando() zalandoorg.Interface
 }
 
 func (f *sharedInformerFactory) Acid() acidzalando.Interface {
 	return acidzalando.New(f, f.namespace, f.tweakListOptions)
 }
+
+func (f *sharedInformerFactory) Zalando() zalandoorg.Interface {
+	return zalandoorg.New(f, f.namespace, f.tweakListOptions)
+}
diff --git a/pkg/generated/informers/externalversions/generic.go b/pkg/generated/informers/externalversions/generic.go
index 652ba0f77..afa4c6ac2 100644
--- a/pkg/generated/informers/externalversions/generic.go
+++ b/pkg/generated/informers/externalversions/generic.go
@@ -28,6 +28,7 @@ import (
 	"fmt"
 
 	v1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
+	zalandoorgv1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1"
 	schema "k8s.io/apimachinery/pkg/runtime/schema"
 	cache "k8s.io/client-go/tools/cache"
 )
@@ -64,6 +65,10 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource
 	case v1.SchemeGroupVersion.WithResource("postgresqls"):
 		return &genericInformer{resource: resource.GroupResource(), informer: f.Acid().V1().Postgresqls().Informer()}, nil
 
+		// Group=zalando.org, Version=v1
+	case zalandoorgv1.SchemeGroupVersion.WithResource("fabriceventstreams"):
+		return &genericInformer{resource: resource.GroupResource(), informer: f.Zalando().V1().FabricEventStreams().Informer()}, nil
+
 	}
 
 	return nil, fmt.Errorf("no informer found for %v", resource)
diff --git a/pkg/generated/informers/externalversions/zalando.org/interface.go b/pkg/generated/informers/externalversions/zalando.org/interface.go
new file mode 100644
index 000000000..afece8ac3
--- /dev/null
+++ b/pkg/generated/informers/externalversions/zalando.org/interface.go
@@ -0,0 +1,52 @@
+/*
+Copyright 2022 Compose, Zalando SE
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+// Code generated by informer-gen. DO NOT EDIT.
+
+package zalando
+
+import (
+	internalinterfaces "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/internalinterfaces"
+	v1 "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/zalando.org/v1"
+)
+
+// Interface provides access to each of this group's versions.
+type Interface interface {
+	// V1 provides access to shared informers for resources in V1.
+	V1() v1.Interface
+}
+
+type group struct {
+	factory          internalinterfaces.SharedInformerFactory
+	namespace        string
+	tweakListOptions internalinterfaces.TweakListOptionsFunc
+}
+
+// New returns a new Interface.
+func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface {
+	return &group{factory: f, namespace: namespace, tweakListOptions: tweakListOptions}
+}
+
+// V1 returns a new v1.Interface.
+func (g *group) V1() v1.Interface {
+	return v1.New(g.factory, g.namespace, g.tweakListOptions)
+}
diff --git a/pkg/generated/informers/externalversions/zalando.org/v1/fabriceventstream.go b/pkg/generated/informers/externalversions/zalando.org/v1/fabriceventstream.go
new file mode 100644
index 000000000..13733e1d1
--- /dev/null
+++ b/pkg/generated/informers/externalversions/zalando.org/v1/fabriceventstream.go
@@ -0,0 +1,96 @@
+/*
+Copyright 2022 Compose, Zalando SE
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+// Code generated by informer-gen. DO NOT EDIT.
+
+package v1
+
+import (
+	"context"
+	time "time"
+
+	zalandoorgv1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1"
+	versioned "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned"
+	internalinterfaces "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/internalinterfaces"
+	v1 "github.com/zalando/postgres-operator/pkg/generated/listers/zalando.org/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	runtime "k8s.io/apimachinery/pkg/runtime"
+	watch "k8s.io/apimachinery/pkg/watch"
+	cache "k8s.io/client-go/tools/cache"
+)
+
+// FabricEventStreamInformer provides access to a shared informer and lister for
+// FabricEventStreams.
+type FabricEventStreamInformer interface {
+	Informer() cache.SharedIndexInformer
+	Lister() v1.FabricEventStreamLister
+}
+
+type fabricEventStreamInformer struct {
+	factory          internalinterfaces.SharedInformerFactory
+	tweakListOptions internalinterfaces.TweakListOptionsFunc
+	namespace        string
+}
+
+// NewFabricEventStreamInformer constructs a new informer for FabricEventStream type.
+// Always prefer using an informer factory to get a shared informer instead of getting an independent
+// one. This reduces memory footprint and number of connections to the server.
+func NewFabricEventStreamInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer {
+	return NewFilteredFabricEventStreamInformer(client, namespace, resyncPeriod, indexers, nil)
+}
+
+// NewFilteredFabricEventStreamInformer constructs a new informer for FabricEventStream type.
+// Always prefer using an informer factory to get a shared informer instead of getting an independent
+// one. This reduces memory footprint and number of connections to the server.
+func NewFilteredFabricEventStreamInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer {
+	return cache.NewSharedIndexInformer(
+		&cache.ListWatch{
+			ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
+				if tweakListOptions != nil {
+					tweakListOptions(&options)
+				}
+				return client.ZalandoV1().FabricEventStreams(namespace).List(context.TODO(), options)
+			},
+			WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+				if tweakListOptions != nil {
+					tweakListOptions(&options)
+				}
+				return client.ZalandoV1().FabricEventStreams(namespace).Watch(context.TODO(), options)
+			},
+		},
+		&zalandoorgv1.FabricEventStream{},
+		resyncPeriod,
+		indexers,
+	)
+}
+
+func (f *fabricEventStreamInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
+	return NewFilteredFabricEventStreamInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions)
+}
+
+func (f *fabricEventStreamInformer) Informer() cache.SharedIndexInformer {
+	return f.factory.InformerFor(&zalandoorgv1.FabricEventStream{}, f.defaultInformer)
+}
+
+func (f *fabricEventStreamInformer) Lister() v1.FabricEventStreamLister {
+	return v1.NewFabricEventStreamLister(f.Informer().GetIndexer())
+}
diff --git a/pkg/generated/informers/externalversions/zalando.org/v1/interface.go b/pkg/generated/informers/externalversions/zalando.org/v1/interface.go
new file mode 100644
index 000000000..329f15503
--- /dev/null
+++ b/pkg/generated/informers/externalversions/zalando.org/v1/interface.go
@@ -0,0 +1,51 @@
+/*
+Copyright 2022 Compose, Zalando SE
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+// Code generated by informer-gen. DO NOT EDIT.
+
+package v1
+
+import (
+	internalinterfaces "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/internalinterfaces"
+)
+
+// Interface provides access to all the informers in this group version.
+type Interface interface {
+	// FabricEventStreams returns a FabricEventStreamInformer.
+	FabricEventStreams() FabricEventStreamInformer
+}
+
+type version struct {
+	factory          internalinterfaces.SharedInformerFactory
+	namespace        string
+	tweakListOptions internalinterfaces.TweakListOptionsFunc
+}
+
+// New returns a new Interface.
+func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface {
+	return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions}
+}
+
+// FabricEventStreams returns a FabricEventStreamInformer.
+func (v *version) FabricEventStreams() FabricEventStreamInformer {
+	return &fabricEventStreamInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions}
+}
diff --git a/pkg/generated/listers/zalando.org/v1/expansion_generated.go b/pkg/generated/listers/zalando.org/v1/expansion_generated.go
new file mode 100644
index 000000000..5a90e7828
--- /dev/null
+++ b/pkg/generated/listers/zalando.org/v1/expansion_generated.go
@@ -0,0 +1,33 @@
+/*
+Copyright 2022 Compose, Zalando SE
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+// Code generated by lister-gen. DO NOT EDIT.
+
+package v1
+
+// FabricEventStreamListerExpansion allows custom methods to be added to
+// FabricEventStreamLister.
+type FabricEventStreamListerExpansion interface{}
+
+// FabricEventStreamNamespaceListerExpansion allows custom methods to be added to
+// FabricEventStreamNamespaceLister.
+type FabricEventStreamNamespaceListerExpansion interface{}
diff --git a/pkg/generated/listers/zalando.org/v1/fabriceventstream.go b/pkg/generated/listers/zalando.org/v1/fabriceventstream.go
new file mode 100644
index 000000000..b4c895c74
--- /dev/null
+++ b/pkg/generated/listers/zalando.org/v1/fabriceventstream.go
@@ -0,0 +1,105 @@
+/*
+Copyright 2022 Compose, Zalando SE
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+// Code generated by lister-gen. DO NOT EDIT.
+
+package v1
+
+import (
+	v1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/tools/cache"
+)
+
+// FabricEventStreamLister helps list FabricEventStreams.
+// All objects returned here must be treated as read-only.
+type FabricEventStreamLister interface {
+	// List lists all FabricEventStreams in the indexer.
+	// Objects returned here must be treated as read-only.
+	List(selector labels.Selector) (ret []*v1.FabricEventStream, err error)
+	// FabricEventStreams returns an object that can list and get FabricEventStreams.
+	FabricEventStreams(namespace string) FabricEventStreamNamespaceLister
+	FabricEventStreamListerExpansion
+}
+
+// fabricEventStreamLister implements the FabricEventStreamLister interface.
+type fabricEventStreamLister struct {
+	indexer cache.Indexer
+}
+
+// NewFabricEventStreamLister returns a new FabricEventStreamLister.
+func NewFabricEventStreamLister(indexer cache.Indexer) FabricEventStreamLister {
+	return &fabricEventStreamLister{indexer: indexer}
+}
+
+// List lists all FabricEventStreams in the indexer.
+func (s *fabricEventStreamLister) List(selector labels.Selector) (ret []*v1.FabricEventStream, err error) {
+	err = cache.ListAll(s.indexer, selector, func(m interface{}) {
+		ret = append(ret, m.(*v1.FabricEventStream))
+	})
+	return ret, err
+}
+
+// FabricEventStreams returns an object that can list and get FabricEventStreams.
+func (s *fabricEventStreamLister) FabricEventStreams(namespace string) FabricEventStreamNamespaceLister {
+	return fabricEventStreamNamespaceLister{indexer: s.indexer, namespace: namespace}
+}
+
+// FabricEventStreamNamespaceLister helps list and get FabricEventStreams.
+// All objects returned here must be treated as read-only.
+type FabricEventStreamNamespaceLister interface {
+	// List lists all FabricEventStreams in the indexer for a given namespace.
+	// Objects returned here must be treated as read-only.
+	List(selector labels.Selector) (ret []*v1.FabricEventStream, err error)
+	// Get retrieves the FabricEventStream from the indexer for a given namespace and name.
+	// Objects returned here must be treated as read-only.
+	Get(name string) (*v1.FabricEventStream, error)
+	FabricEventStreamNamespaceListerExpansion
+}
+
+// fabricEventStreamNamespaceLister implements the FabricEventStreamNamespaceLister
+// interface.
+type fabricEventStreamNamespaceLister struct {
+	indexer   cache.Indexer
+	namespace string
+}
+
+// List lists all FabricEventStreams in the indexer for a given namespace.
+func (s fabricEventStreamNamespaceLister) List(selector labels.Selector) (ret []*v1.FabricEventStream, err error) {
+	err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) {
+		ret = append(ret, m.(*v1.FabricEventStream))
+	})
+	return ret, err
+}
+
+// Get retrieves the FabricEventStream from the indexer for a given namespace and name.
+func (s fabricEventStreamNamespaceLister) Get(name string) (*v1.FabricEventStream, error) {
+	obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name)
+	if err != nil {
+		return nil, err
+	}
+	if !exists {
+		return nil, errors.NewNotFound(v1.Resource("fabriceventstream"), name)
+	}
+	return obj.(*v1.FabricEventStream), nil
+}
diff --git a/pkg/util/constants/streams.go b/pkg/util/constants/streams.go
new file mode 100644
index 000000000..bd70b719b
--- /dev/null
+++ b/pkg/util/constants/streams.go
@@ -0,0 +1,14 @@
+package constants
+
+// PostgreSQL specific constants
+const (
+	EventStreamCRDApiVersion     = "zalando.org/v1"
+	EventStreamCRDKind           = "FabricEventStream"
+	EventStreamCRDName           = "fabriceventstreams.zalando.org"
+	EventStreamSourcePGType      = "PostgresLogicalReplication"
+	EventStreamSourceSlotPrefix  = "fes"
+	EventStreamSourcePluginType  = "pgoutput"
+	EventStreamSourceAuthType    = "DatabaseAuthenticationSecret"
+	EventStreamFlowPgGenericType = "PostgresWalToGenericNakadiEvent"
+	EventStreamSinkNakadiType    = "Nakadi"
+)
diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go
index dd6ec1e8b..0897777ee 100644
--- a/pkg/util/k8sutil/k8sutil.go
+++ b/pkg/util/k8sutil/k8sutil.go
@@ -12,8 +12,9 @@ import (
 	clientbatchv1beta1 "k8s.io/client-go/kubernetes/typed/batch/v1beta1"
 
 	apiacidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
-	acidv1client "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned"
+	zalandoclient "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned"
 	acidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1"
+	zalandov1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1"
 	"github.com/zalando/postgres-operator/pkg/spec"
 	apiappsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
@@ -58,9 +59,11 @@ type KubernetesClient struct {
 	acidv1.OperatorConfigurationsGetter
 	acidv1.PostgresTeamsGetter
 	acidv1.PostgresqlsGetter
+	zalandov1.FabricEventStreamsGetter
 
-	RESTClient      rest.Interface
-	AcidV1ClientSet *acidv1client.Clientset
+	RESTClient         rest.Interface
+	AcidV1ClientSet    *zalandoclient.Clientset
+	Zalandov1ClientSet *zalandoclient.Clientset
 }
 
 type mockSecret struct {
@@ -158,14 +161,19 @@ func NewFromConfig(cfg *rest.Config) (KubernetesClient, error) {
 
 	kubeClient.CustomResourceDefinitionsGetter = apiextClient.ApiextensionsV1()
 
-	kubeClient.AcidV1ClientSet = acidv1client.NewForConfigOrDie(cfg)
+	kubeClient.AcidV1ClientSet = zalandoclient.NewForConfigOrDie(cfg)
 	if err != nil {
 		return kubeClient, fmt.Errorf("could not create acid.zalan.do clientset: %v", err)
 	}
+	kubeClient.Zalandov1ClientSet = zalandoclient.NewForConfigOrDie(cfg)
+	if err != nil {
+		return kubeClient, fmt.Errorf("could not create zalando.org clientset: %v", err)
+	}
 
 	kubeClient.OperatorConfigurationsGetter = kubeClient.AcidV1ClientSet.AcidV1()
 	kubeClient.PostgresTeamsGetter = kubeClient.AcidV1ClientSet.AcidV1()
 	kubeClient.PostgresqlsGetter = kubeClient.AcidV1ClientSet.AcidV1()
+	kubeClient.FabricEventStreamsGetter = kubeClient.Zalandov1ClientSet.ZalandoV1()
 
 	return kubeClient, nil
 }
diff --git a/pkg/util/patroni/patroni_test.go b/pkg/util/patroni/patroni_test.go
index 614f77828..60f289c6f 100644
--- a/pkg/util/patroni/patroni_test.go
+++ b/pkg/util/patroni/patroni_test.go
@@ -185,7 +185,7 @@ func TestGetConfig(t *testing.T) {
 		Slots: map[string]map[string]string{
 			"cdc": {
 				"database": "foo",
-				"plugin":   "wal2json",
+				"plugin":   "pgoutput",
 				"type":     "logical",
 			},
 		},
@@ -218,7 +218,7 @@ func TestGetConfig(t *testing.T) {
 		"wal_log_hints":                   "on",
 	}
 
-	configJson := `{"loop_wait": 10, "maximum_lag_on_failover": 33554432, "postgresql": {"parameters": {"archive_mode": "on", "archive_timeout": "1800s", "autovacuum_analyze_scale_factor": 0.02, "autovacuum_max_workers": 5, "autovacuum_vacuum_scale_factor": 0.05, "checkpoint_completion_target": 0.9, "hot_standby": "on", "log_autovacuum_min_duration": 0, "log_checkpoints": "on", "log_connections": "on", "log_disconnections": "on", "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", "log_lock_waits": "on", "log_min_duration_statement": 500, "log_statement": "ddl", "log_temp_files": 0, "max_connections": 100, "max_replication_slots": 10, "max_wal_senders": 10, "tcp_keepalives_idle": 900, "tcp_keepalives_interval": 100, "track_functions": "all", "wal_level": "hot_standby", "wal_log_hints": "on"}, "use_pg_rewind": true, "use_slots": true}, "retry_timeout": 10, "slots": {"cdc": {"database": "foo", "plugin": "wal2json", "type": "logical"}}, "ttl": 30}`
+	configJson := `{"loop_wait": 10, "maximum_lag_on_failover": 33554432, "postgresql": {"parameters": {"archive_mode": "on", "archive_timeout": "1800s", "autovacuum_analyze_scale_factor": 0.02, "autovacuum_max_workers": 5, "autovacuum_vacuum_scale_factor": 0.05, "checkpoint_completion_target": 0.9, "hot_standby": "on", "log_autovacuum_min_duration": 0, "log_checkpoints": "on", "log_connections": "on", "log_disconnections": "on", "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", "log_lock_waits": "on", "log_min_duration_statement": 500, "log_statement": "ddl", "log_temp_files": 0, "max_connections": 100, "max_replication_slots": 10, "max_wal_senders": 10, "tcp_keepalives_idle": 900, "tcp_keepalives_interval": 100, "track_functions": "all", "wal_level": "hot_standby", "wal_log_hints": "on"}, "use_pg_rewind": true, "use_slots": true}, "retry_timeout": 10, "slots": {"cdc": {"database": "foo", "plugin": "pgoutput", "type": "logical"}}, "ttl": 30}`
 	r := ioutil.NopCloser(bytes.NewReader([]byte(configJson)))
 
 	response := http.Response{
@@ -253,7 +253,7 @@ func TestSetPostgresParameters(t *testing.T) {
 		"wal_level":       "logical",
 	}
 
-	configJson := `{"loop_wait": 10, "maximum_lag_on_failover": 33554432, "postgresql": {"parameters": {"archive_mode": "on", "archive_timeout": "1800s", "autovacuum_analyze_scale_factor": 0.02, "autovacuum_max_workers": 5, "autovacuum_vacuum_scale_factor": 0.05, "checkpoint_completion_target": 0.9, "hot_standby": "on", "log_autovacuum_min_duration": 0, "log_checkpoints": "on", "log_connections": "on", "log_disconnections": "on", "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", "log_lock_waits": "on", "log_min_duration_statement": 500, "log_statement": "ddl", "log_temp_files": 0, "max_connections": 50, "max_replication_slots": 10, "max_wal_senders": 10, "tcp_keepalives_idle": 900, "tcp_keepalives_interval": 100, "track_functions": "all", "wal_level": "logical", "wal_log_hints": "on"}, "use_pg_rewind": true, "use_slots": true}, "retry_timeout": 10, "slots": {"cdc": {"database": "foo", "plugin": "wal2json", "type": "logical"}}, "ttl": 30}`
+	configJson := `{"loop_wait": 10, "maximum_lag_on_failover": 33554432, "postgresql": {"parameters": {"archive_mode": "on", "archive_timeout": "1800s", "autovacuum_analyze_scale_factor": 0.02, "autovacuum_max_workers": 5, "autovacuum_vacuum_scale_factor": 0.05, "checkpoint_completion_target": 0.9, "hot_standby": "on", "log_autovacuum_min_duration": 0, "log_checkpoints": "on", "log_connections": "on", "log_disconnections": "on", "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", "log_lock_waits": "on", "log_min_duration_statement": 500, "log_statement": "ddl", "log_temp_files": 0, "max_connections": 50, "max_replication_slots": 10, "max_wal_senders": 10, "tcp_keepalives_idle": 900, "tcp_keepalives_interval": 100, "track_functions": "all", "wal_level": "logical", "wal_log_hints": "on"}, "use_pg_rewind": true, "use_slots": true}, "retry_timeout": 10, "slots": {"cdc": {"database": "foo", "plugin": "pgoutput", "type": "logical"}}, "ttl": 30}`
 	r := ioutil.NopCloser(bytes.NewReader([]byte(configJson)))
 
 	response := http.Response{

From fb8a6c7a6823bb143bbfd6b5ec2e75d8de4b40d9 Mon Sep 17 00:00:00 2001
From: Maksim Zhylinski <maksim@zhylinski.me>
Date: Mon, 28 Feb 2022 11:35:41 +0100
Subject: [PATCH 26/58] Compare container ports in a smarter way (#1755)

* Compare ports ingoring order and considering protocol defaults

Co-authored-by: Felix Kunde <felix-kunde@gmx.de>
---
 pkg/cluster/cluster.go      |  42 ++++++++++++++-
 pkg/cluster/cluster_test.go | 101 ++++++++++++++++++++++++++++++++++++
 2 files changed, 142 insertions(+), 1 deletion(-)

diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go
index 7dbfa6b88..2afacde99 100644
--- a/pkg/cluster/cluster.go
+++ b/pkg/cluster/cluster.go
@@ -523,7 +523,7 @@ func (c *Cluster) compareContainers(description string, setA, setB []v1.Containe
 		newCheck("new statefulset %s's %s (index %d) name does not match the current one",
 			func(a, b v1.Container) bool { return a.Name != b.Name }),
 		newCheck("new statefulset %s's %s (index %d) ports do not match the current one",
-			func(a, b v1.Container) bool { return !reflect.DeepEqual(a.Ports, b.Ports) }),
+			func(a, b v1.Container) bool { return !comparePorts(a.Ports, b.Ports) }),
 		newCheck("new statefulset %s's %s (index %d) resources do not match the current ones",
 			func(a, b v1.Container) bool { return !compareResources(&a.Resources, &b.Resources) }),
 		newCheck("new statefulset %s's %s (index %d) environment does not match the current one",
@@ -634,6 +634,46 @@ func compareSpiloConfiguration(configa, configb string) bool {
 	return reflect.DeepEqual(oa, ob)
 }
 
+func areProtocolsEqual(a, b v1.Protocol) bool {
+	return a == b ||
+		(a == "" && b == v1.ProtocolTCP) ||
+		(a == v1.ProtocolTCP && b == "")
+}
+
+func comparePorts(a, b []v1.ContainerPort) bool {
+	if len(a) != len(b) {
+		return false
+	}
+
+	areContainerPortsEqual := func(a, b v1.ContainerPort) bool {
+		return a.Name == b.Name &&
+			a.HostPort == b.HostPort &&
+			areProtocolsEqual(a.Protocol, b.Protocol) &&
+			a.HostIP == b.HostIP
+	}
+
+	findByPortValue := func(portSpecs []v1.ContainerPort, port int32) (v1.ContainerPort, bool) {
+		for _, portSpec := range portSpecs {
+			if portSpec.ContainerPort == port {
+				return portSpec, true
+			}
+		}
+		return v1.ContainerPort{}, false
+	}
+
+	for _, portA := range a {
+		portB, found := findByPortValue(b, portA.ContainerPort)
+		if !found {
+			return false
+		}
+		if !areContainerPortsEqual(portA, portB) {
+			return false
+		}
+	}
+
+	return true
+}
+
 func (c *Cluster) enforceMinResourceLimits(spec *acidv1.PostgresSpec) error {
 
 	var (
diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go
index dc1f5ff03..d06cc21e1 100644
--- a/pkg/cluster/cluster_test.go
+++ b/pkg/cluster/cluster_test.go
@@ -5,6 +5,8 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/sirupsen/logrus"
 	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
 	fakeacidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/fake"
@@ -1088,3 +1090,102 @@ func TestValidUsernames(t *testing.T) {
 		}
 	}
 }
+
+func TestComparePorts(t *testing.T) {
+	testCases := []struct {
+		name     string
+		setA     []v1.ContainerPort
+		setB     []v1.ContainerPort
+		expected bool
+	}{
+		{
+			name: "different ports",
+			setA: []v1.ContainerPort{
+				{
+					Name:          "metrics",
+					ContainerPort: 9187,
+					Protocol:      v1.ProtocolTCP,
+				},
+			},
+
+			setB: []v1.ContainerPort{
+				{
+					Name:          "http",
+					ContainerPort: 80,
+					Protocol:      v1.ProtocolTCP,
+				},
+			},
+			expected: false,
+		},
+		{
+			name: "no difference",
+			setA: []v1.ContainerPort{
+				{
+					Name:          "metrics",
+					ContainerPort: 9187,
+					Protocol:      v1.ProtocolTCP,
+				},
+			},
+			setB: []v1.ContainerPort{
+				{
+					Name:          "metrics",
+					ContainerPort: 9187,
+					Protocol:      v1.ProtocolTCP,
+				},
+			},
+			expected: true,
+		},
+		{
+			name: "same ports, different order",
+			setA: []v1.ContainerPort{
+				{
+					Name:          "metrics",
+					ContainerPort: 9187,
+					Protocol:      v1.ProtocolTCP,
+				},
+				{
+					Name:          "http",
+					ContainerPort: 80,
+					Protocol:      v1.ProtocolTCP,
+				},
+			},
+			setB: []v1.ContainerPort{
+				{
+					Name:          "http",
+					ContainerPort: 80,
+					Protocol:      v1.ProtocolTCP,
+				},
+				{
+					Name:          "metrics",
+					ContainerPort: 9187,
+					Protocol:      v1.ProtocolTCP,
+				},
+			},
+			expected: true,
+		},
+		{
+			name: "same ports, but one with default protocol",
+			setA: []v1.ContainerPort{
+				{
+					Name:          "metrics",
+					ContainerPort: 9187,
+					Protocol:      v1.ProtocolTCP,
+				},
+			},
+			setB: []v1.ContainerPort{
+				{
+					Name:          "metrics",
+					ContainerPort: 9187,
+				},
+			},
+			expected: true,
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			got := comparePorts(testCase.setA, testCase.setB)
+			assert.Equal(t, testCase.expected, got)
+		})
+	}
+}

From da8398231373ea2b092762913b4aba6253f01efb Mon Sep 17 00:00:00 2001
From: Dmitry Volodin <dmvolod@gmail.com>
Date: Tue, 1 Mar 2022 19:07:37 +0300
Subject: [PATCH 27/58] inherited_labels and inherited_annotations not passed
 to PVC (#1784)

* inherited_labels and inherited_annotations not passed to PVC
* Fix developer.md related to the local operator deployment
---
 docs/developer.md            | 18 ++++++++++++++----
 pkg/cluster/k8sres.go        | 19 +++++++------------
 pkg/cluster/util_test.go     |  4 ++--
 pkg/controller/postgresql.go |  6 ------
 4 files changed, 23 insertions(+), 24 deletions(-)

diff --git a/docs/developer.md b/docs/developer.md
index bc80d5380..40636d47a 100644
--- a/docs/developer.md
+++ b/docs/developer.md
@@ -72,22 +72,32 @@ make docker
 
 # kind
 make docker
-kind load docker-image <image> --name <kind-cluster-name>
+kind load docker-image registry.opensource.zalan.do/acid/postgres-operator:${TAG} --name <kind-cluster-name>
 ```
 
-Then create a new Postgres Operator deployment. You can reuse the provided
-manifest but replace the version and tag. Don't forget to also apply
+Then create a new Postgres Operator deployment.
+
+### Deploying manually with manifests and kubectl
+
+You can reuse the provided manifest but replace the version and tag. Don't forget to also apply
 configuration and RBAC manifests first, e.g.:
 
 ```bash
 kubectl create -f manifests/configmap.yaml
 kubectl create -f manifests/operator-service-account-rbac.yaml
-sed -e "s/\(image\:.*\:\).*$/\1$TAG/" manifests/postgres-operator.yaml | kubectl create  -f -
+sed -e "s/\(image\:.*\:\).*$/\1$TAG/" -e "s/\(imagePullPolicy\:\).*$/\1 Never/" manifests/postgres-operator.yaml | kubectl create  -f -
 
 # check if the operator is coming up
 kubectl get pod -l name=postgres-operator
 ```
 
+### Deploying with Helm chart
+
+Yoy can reuse the provided Helm chart to deploy local operator build with the following command:
+```bash
+helm install postgres-operator ./charts/postgres-operator --namespace zalando-operator --set image.tag=${TAG} --set image.pullPolicy=Never
+```
+
 ## Code generation
 
 The operator employs K8s-provided code generation to obtain deep copy methods
diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go
index a42aa2d06..751606d93 100644
--- a/pkg/cluster/k8sres.go
+++ b/pkg/cluster/k8sres.go
@@ -1291,7 +1291,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
 		return nil, fmt.Errorf("could not generate pod template: %v", err)
 	}
 
-	if volumeClaimTemplate, err = generatePersistentVolumeClaimTemplate(spec.Volume.Size,
+	if volumeClaimTemplate, err = c.generatePersistentVolumeClaimTemplate(spec.Volume.Size,
 		spec.Volume.StorageClass, spec.Volume.Selector); err != nil {
 		return nil, fmt.Errorf("could not generate volume claim template: %v", err)
 	}
@@ -1540,21 +1540,12 @@ func (c *Cluster) addAdditionalVolumes(podSpec *v1.PodSpec,
 	podSpec.Volumes = volumes
 }
 
-func generatePersistentVolumeClaimTemplate(volumeSize, volumeStorageClass string,
+func (c *Cluster) generatePersistentVolumeClaimTemplate(volumeSize, volumeStorageClass string,
 	volumeSelector *metav1.LabelSelector) (*v1.PersistentVolumeClaim, error) {
 
 	var storageClassName *string
-
-	metadata := metav1.ObjectMeta{
-		Name: constants.DataVolumeName,
-	}
 	if volumeStorageClass != "" {
-		// TODO: remove the old annotation, switching completely to the StorageClassName field.
-		metadata.Annotations = map[string]string{"volume.beta.kubernetes.io/storage-class": volumeStorageClass}
 		storageClassName = &volumeStorageClass
-	} else {
-		metadata.Annotations = map[string]string{"volume.alpha.kubernetes.io/storage-class": "default"}
-		storageClassName = nil
 	}
 
 	quantity, err := resource.ParseQuantity(volumeSize)
@@ -1564,7 +1555,11 @@ func generatePersistentVolumeClaimTemplate(volumeSize, volumeStorageClass string
 
 	volumeMode := v1.PersistentVolumeFilesystem
 	volumeClaim := &v1.PersistentVolumeClaim{
-		ObjectMeta: metadata,
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        constants.DataVolumeName,
+			Annotations: c.annotationsSet(nil),
+			Labels:      c.labelsSet(true),
+		},
 		Spec: v1.PersistentVolumeClaimSpec{
 			AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
 			Resources: v1.ResourceRequirements{
diff --git a/pkg/cluster/util_test.go b/pkg/cluster/util_test.go
index 20214a085..1463a5609 100644
--- a/pkg/cluster/util_test.go
+++ b/pkg/cluster/util_test.go
@@ -98,8 +98,8 @@ func TestInheritedAnnotations(t *testing.T) {
 			t.Errorf("%s: pod template %v not inherited annotations %#v, got %#v", testName, sts.ObjectMeta.Name, inheritedAnnotations, sts.ObjectMeta.Annotations)
 		}
 		// pvc template
-		if util.MapContains(sts.Spec.VolumeClaimTemplates[0].Annotations, inheritedAnnotations) {
-			t.Errorf("%s: PVC template %v not expected to have inherited annotations %#v, got %#v", testName, sts.ObjectMeta.Name, inheritedAnnotations, sts.ObjectMeta.Annotations)
+		if !(util.MapContains(sts.Spec.VolumeClaimTemplates[0].Annotations, inheritedAnnotations)) {
+			t.Errorf("%s: PVC template %v not inherited annotations %#v, got %#v", testName, sts.ObjectMeta.Name, inheritedAnnotations, sts.ObjectMeta.Annotations)
 		}
 	}
 
diff --git a/pkg/controller/postgresql.go b/pkg/controller/postgresql.go
index b3c4d8414..590494412 100644
--- a/pkg/controller/postgresql.go
+++ b/pkg/controller/postgresql.go
@@ -505,8 +505,6 @@ func (c *Controller) postgresqlAdd(obj interface{}) {
 		// We will not get multiple Add events for the same cluster
 		c.queueClusterEvent(nil, pg, EventAdd)
 	}
-
-	return
 }
 
 func (c *Controller) postgresqlUpdate(prev, cur interface{}) {
@@ -521,8 +519,6 @@ func (c *Controller) postgresqlUpdate(prev, cur interface{}) {
 		}
 		c.queueClusterEvent(pgOld, pgNew, EventUpdate)
 	}
-
-	return
 }
 
 func (c *Controller) postgresqlDelete(obj interface{}) {
@@ -530,8 +526,6 @@ func (c *Controller) postgresqlDelete(obj interface{}) {
 	if pg != nil {
 		c.queueClusterEvent(pg, nil, EventDelete)
 	}
-
-	return
 }
 
 func (c *Controller) postgresqlCheck(obj interface{}) *acidv1.Postgresql {

From ca0c27a51bf34d41847c3ad25605a971ca8a202e Mon Sep 17 00:00:00 2001
From: david amick <snarlysodboxer@users.noreply.github.com>
Date: Tue, 1 Mar 2022 08:56:16 -0800
Subject: [PATCH 28/58] Retry when getting the pod_environment_secret (#1777)

* Retry when getting the pod_environment_secret
---
 docs/developer.md          |  7 +++++++
 pkg/cluster/k8sres.go      | 31 ++++++++++++++++++++++++-----
 pkg/cluster/k8sres_test.go | 40 +++++++++++++++++++++++++++++++-------
 3 files changed, 66 insertions(+), 12 deletions(-)

diff --git a/docs/developer.md b/docs/developer.md
index 40636d47a..31f48d92d 100644
--- a/docs/developer.md
+++ b/docs/developer.md
@@ -218,6 +218,13 @@ dlv connect 127.0.0.1:DLV_PORT
 
 ## Unit tests
 
+Prerequisites:
+
+```bash
+make deps
+make mocks
+```
+
 To run all unit tests, you can simply do:
 
 ```bash
diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go
index 751606d93..e741c6dc4 100644
--- a/pkg/cluster/k8sres.go
+++ b/pkg/cluster/k8sres.go
@@ -8,11 +8,13 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	policybeta1 "k8s.io/api/policy/v1beta1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -24,6 +26,7 @@ import (
 	"github.com/zalando/postgres-operator/pkg/util/config"
 	"github.com/zalando/postgres-operator/pkg/util/constants"
 	"github.com/zalando/postgres-operator/pkg/util/k8sutil"
+	"github.com/zalando/postgres-operator/pkg/util/retryutil"
 	batchv1 "k8s.io/api/batch/v1"
 	batchv1beta1 "k8s.io/api/batch/v1beta1"
 	"k8s.io/apimachinery/pkg/labels"
@@ -897,12 +900,30 @@ func (c *Cluster) getPodEnvironmentSecretVariables() ([]v1.EnvVar, error) {
 		return secretPodEnvVarsList, nil
 	}
 
-	secret, err := c.KubeClient.Secrets(c.Namespace).Get(
-		context.TODO(),
-		c.OpConfig.PodEnvironmentSecret,
-		metav1.GetOptions{})
+	secret := &v1.Secret{}
+	var notFoundErr error
+	err := retryutil.Retry(c.OpConfig.ResourceCheckInterval, c.OpConfig.ResourceCheckTimeout,
+		func() (bool, error) {
+			var err error
+			secret, err = c.KubeClient.Secrets(c.Namespace).Get(
+				context.TODO(),
+				c.OpConfig.PodEnvironmentSecret,
+				metav1.GetOptions{})
+			if err != nil {
+				if apierrors.IsNotFound(err) {
+					notFoundErr = err
+					return false, nil
+				}
+				return false, err
+			}
+			return true, nil
+		},
+	)
+	if notFoundErr != nil && err != nil {
+		err = errors.Wrap(notFoundErr, err.Error())
+	}
 	if err != nil {
-		return nil, fmt.Errorf("could not read Secret PodEnvironmentSecretName: %v", err)
+		return nil, errors.Wrap(err, "could not read Secret PodEnvironmentSecretName")
 	}
 
 	for k := range secret.Data {
diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go
index 503959f28..6875c9dba 100644
--- a/pkg/cluster/k8sres_test.go
+++ b/pkg/cluster/k8sres_test.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"reflect"
 	"sort"
+	"time"
 
 	"testing"
 
@@ -21,8 +22,10 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	policyv1beta1 "k8s.io/api/policy/v1beta1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes/fake"
 	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
@@ -640,8 +643,12 @@ func TestSecretVolume(t *testing.T) {
 }
 
 const (
-	testPodEnvironmentConfigMapName = "pod_env_cm"
-	testPodEnvironmentSecretName    = "pod_env_sc"
+	testPodEnvironmentConfigMapName      = "pod_env_cm"
+	testPodEnvironmentSecretName         = "pod_env_sc"
+	testPodEnvironmentObjectNotExists    = "idonotexist"
+	testPodEnvironmentSecretNameAPIError = "pod_env_sc_apierror"
+	testResourceCheckInterval            = 3
+	testResourceCheckTimeout             = 10
 )
 
 type mockSecret struct {
@@ -653,8 +660,11 @@ type mockConfigMap struct {
 }
 
 func (c *mockSecret) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.Secret, error) {
+	if name == testPodEnvironmentSecretNameAPIError {
+		return nil, fmt.Errorf("Secret PodEnvironmentSecret API error")
+	}
 	if name != testPodEnvironmentSecretName {
-		return nil, fmt.Errorf("Secret PodEnvironmentSecret not found")
+		return nil, k8serrors.NewNotFound(schema.GroupResource{Group: "core", Resource: "secret"}, name)
 	}
 	secret := &v1.Secret{}
 	secret.Name = testPodEnvironmentSecretName
@@ -723,7 +733,7 @@ func TestPodEnvironmentConfigMapVariables(t *testing.T) {
 			opConfig: config.Config{
 				Resources: config.Resources{
 					PodEnvironmentConfigMap: spec.NamespacedName{
-						Name: "idonotexist",
+						Name: testPodEnvironmentObjectNotExists,
 					},
 				},
 			},
@@ -774,6 +784,7 @@ func TestPodEnvironmentConfigMapVariables(t *testing.T) {
 
 // Test if the keys of an existing secret are properly referenced
 func TestPodEnvironmentSecretVariables(t *testing.T) {
+	maxRetries := int(testResourceCheckTimeout / testResourceCheckInterval)
 	testName := "TestPodEnvironmentSecretVariables"
 	tests := []struct {
 		subTest  string
@@ -789,16 +800,31 @@ func TestPodEnvironmentSecretVariables(t *testing.T) {
 			subTest: "Secret referenced by PodEnvironmentSecret does not exist",
 			opConfig: config.Config{
 				Resources: config.Resources{
-					PodEnvironmentSecret: "idonotexist",
+					PodEnvironmentSecret:  testPodEnvironmentObjectNotExists,
+					ResourceCheckInterval: time.Duration(testResourceCheckInterval),
+					ResourceCheckTimeout:  time.Duration(testResourceCheckTimeout),
 				},
 			},
-			err: fmt.Errorf("could not read Secret PodEnvironmentSecretName: Secret PodEnvironmentSecret not found"),
+			err: fmt.Errorf("could not read Secret PodEnvironmentSecretName: still failing after %d retries: secret.core %q not found", maxRetries, testPodEnvironmentObjectNotExists),
+		},
+		{
+			subTest: "API error during PodEnvironmentSecret retrieval",
+			opConfig: config.Config{
+				Resources: config.Resources{
+					PodEnvironmentSecret:  testPodEnvironmentSecretNameAPIError,
+					ResourceCheckInterval: time.Duration(testResourceCheckInterval),
+					ResourceCheckTimeout:  time.Duration(testResourceCheckTimeout),
+				},
+			},
+			err: fmt.Errorf("could not read Secret PodEnvironmentSecretName: Secret PodEnvironmentSecret API error"),
 		},
 		{
 			subTest: "Pod environment vars reference all keys from secret configured by PodEnvironmentSecret",
 			opConfig: config.Config{
 				Resources: config.Resources{
-					PodEnvironmentSecret: testPodEnvironmentSecretName,
+					PodEnvironmentSecret:  testPodEnvironmentSecretName,
+					ResourceCheckInterval: time.Duration(testResourceCheckInterval),
+					ResourceCheckTimeout:  time.Duration(testResourceCheckTimeout),
 				},
 			},
 			envVars: []v1.EnvVar{

From 695ad44caffa7c03e19a1b59796d93942d8cb70c Mon Sep 17 00:00:00 2001
From: "A. Stoewer" <adrian.stoewer@ibm.com>
Date: Wed, 2 Mar 2022 17:39:33 +0100
Subject: [PATCH 29/58] Logical backup retention time (#1337)

* Add optional logical backup retention time
* Set defaults for potentially unbound variables, so that the script will work with older operator versions
* Document retention time parameter for logical backups
* Add retention time parameter to resources and charts

Co-authored-by: Felix Kunde <felix-kunde@gmx.de>
---
 .../crds/operatorconfigurations.yaml          |  2 +
 charts/postgres-operator/values.yaml          |  2 +
 docker/logical-backup/dump.sh                 | 55 +++++++++++++++++++
 docs/reference/operator_parameters.md         |  5 ++
 manifests/configmap.yaml                      |  1 +
 manifests/operatorconfiguration.crd.yaml      |  2 +
 ...gresql-operator-default-configuration.yaml |  1 +
 pkg/apis/acid.zalan.do/v1/crds.go             |  3 +
 .../v1/operator_configuration_type.go         |  1 +
 pkg/cluster/k8sres.go                         |  4 ++
 pkg/controller/operator_config.go             |  1 +
 pkg/util/config/config.go                     |  1 +
 12 files changed, 78 insertions(+)

diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml
index f510e08f5..f67c04fc5 100644
--- a/charts/postgres-operator/crds/operatorconfigurations.yaml
+++ b/charts/postgres-operator/crds/operatorconfigurations.yaml
@@ -450,6 +450,8 @@ spec:
                     type: string
                   logical_backup_s3_sse:
                     type: string
+                  logical_backup_s3_retention_time:
+                    type: string
                   logical_backup_schedule:
                     type: string
                     pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$'
diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml
index 288efe763..5d38b5233 100644
--- a/charts/postgres-operator/values.yaml
+++ b/charts/postgres-operator/values.yaml
@@ -310,6 +310,8 @@ configLogicalBackup:
   logical_backup_s3_secret_access_key: ""
   # S3 server side encryption
   logical_backup_s3_sse: "AES256"
+  # S3 retention time for stored backups for example "2 week" or "7 days"
+  logical_backup_s3_retention_time: ""
   # backup schedule in the cron format
   logical_backup_schedule: "30 00 * * *"
 
diff --git a/docker/logical-backup/dump.sh b/docker/logical-backup/dump.sh
index c931dc962..85e372664 100755
--- a/docker/logical-backup/dump.sh
+++ b/docker/logical-backup/dump.sh
@@ -15,6 +15,9 @@ TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
 K8S_API_URL=https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1
 CERT=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
 
+LOGICAL_BACKUP_PROVIDER=${LOGICAL_BACKUP_PROVIDER:="s3"}
+LOGICAL_BACKUP_S3_RETENTION_TIME=${LOGICAL_BACKUP_S3_RETENTION_TIME:=""}
+
 function estimate_size {
     "$PG_BIN"/psql -tqAc "${ALL_DB_SIZE_QUERY}"
 }
@@ -28,6 +31,57 @@ function compress {
     pigz
 }
 
+function aws_delete_objects {
+    args=(
+      "--bucket=$LOGICAL_BACKUP_S3_BUCKET"
+    )
+
+    [[ ! -z "$LOGICAL_BACKUP_S3_ENDPOINT" ]] && args+=("--endpoint-url=$LOGICAL_BACKUP_S3_ENDPOINT")
+    [[ ! -z "$LOGICAL_BACKUP_S3_REGION" ]] && args+=("--region=$LOGICAL_BACKUP_S3_REGION")
+
+    aws s3api delete-objects "${args[@]}" --delete Objects=["$(printf {Key=%q}, "$@")"],Quiet=true
+}
+export -f aws_delete_objects
+
+function aws_delete_outdated {
+      if [[ -z "$LOGICAL_BACKUP_S3_RETENTION_TIME" ]] ; then
+          echo "no retention time configured: skip cleanup of outdated backups"
+          return 0
+      fi
+
+      # define cutoff date for outdated backups (day precision)
+      cutoff_date=$(date -d "$LOGICAL_BACKUP_S3_RETENTION_TIME ago" +%F)
+
+      # mimic bucket setup from Spilo
+      prefix="spilo/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"
+
+      args=(
+        "--no-paginate"
+        "--output=text"
+        "--prefix=$prefix"
+        "--bucket=$LOGICAL_BACKUP_S3_BUCKET"
+      )
+
+      [[ ! -z "$LOGICAL_BACKUP_S3_ENDPOINT" ]] && args+=("--endpoint-url=$LOGICAL_BACKUP_S3_ENDPOINT")
+      [[ ! -z "$LOGICAL_BACKUP_S3_REGION" ]] && args+=("--region=$LOGICAL_BACKUP_S3_REGION")
+
+      # list objects older than the cutoff date
+      aws s3api list-objects "${args[@]}" --query="Contents[?LastModified<='$cutoff_date'].[Key]" > /tmp/outdated-backups
+
+      # spare the last backup
+      sed -i '$d' /tmp/outdated-backups
+
+      count=$(wc -l < /tmp/outdated-backups)
+      if [[ $count == 0 ]] ; then
+        echo "no outdated backups to delete"
+        return 0
+      fi
+      echo "deleting $count outdated backups created before $cutoff_date"
+
+      # deleted outdated files in batches with 100 at a time
+      tr '\n' '\0'  < /tmp/outdated-backups | xargs -0 -P1 -n100 bash -c 'aws_delete_objects "$@"' _
+}
+
 function aws_upload {
     declare -r EXPECTED_SIZE="$1"
 
@@ -59,6 +113,7 @@ function upload {
             ;;
         *)
             aws_upload $(($(estimate_size) / DUMP_SIZE_COEFF))
+            aws_delete_outdated
             ;;
     esac
 }
diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md
index f3d9be88f..cbe31ba57 100644
--- a/docs/reference/operator_parameters.md
+++ b/docs/reference/operator_parameters.md
@@ -676,6 +676,11 @@ grouped under the `logical_backup` key.
   Specify server side encryption that S3 storage is using. If empty string
   is specified, no argument will be passed to `aws s3` command. Default: "AES256".
 
+* **logical_backup_s3_retention_time**
+  Specify a retention time for logical backups stored in S3. Backups older than the specified retention 
+  time will be deleted after a new backup was uploaded. If empty, all backups will be kept. Example values are
+  "3 days", "2 weeks", or "1 month". The default is empty.
+
 * **logical_backup_schedule**
   Backup schedule in the cron format. Please take the
   [reference schedule format](https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#schedule)
diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml
index b3aaa3c66..b9605441e 100644
--- a/manifests/configmap.yaml
+++ b/manifests/configmap.yaml
@@ -77,6 +77,7 @@ data:
   # logical_backup_s3_endpoint: ""
   # logical_backup_s3_secret_access_key: ""
   logical_backup_s3_sse: "AES256"
+  # logical_backup_s3_retention_time: ""
   logical_backup_schedule: "30 00 * * *"
   major_version_upgrade_mode: "manual"
   # major_version_upgrade_team_allow_list: ""
diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml
index d086998cf..e6c2a2d7c 100644
--- a/manifests/operatorconfiguration.crd.yaml
+++ b/manifests/operatorconfiguration.crd.yaml
@@ -448,6 +448,8 @@ spec:
                     type: string
                   logical_backup_s3_sse:
                     type: string
+                  logical_backup_s3_retention_time:
+                    type: string
                   logical_backup_schedule:
                     type: string
                     pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$'
diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml
index 87b5436d5..0641bd1b1 100644
--- a/manifests/postgresql-operator-default-configuration.yaml
+++ b/manifests/postgresql-operator-default-configuration.yaml
@@ -146,6 +146,7 @@ configuration:
     # logical_backup_s3_region: ""
     # logical_backup_s3_secret_access_key: ""
     logical_backup_s3_sse: "AES256"
+    # logical_backup_s3_retention_time: ""
     logical_backup_schedule: "30 00 * * *"
   debug:
     debug_logging: true
diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go
index 9dc3d167e..d8d46e248 100644
--- a/pkg/apis/acid.zalan.do/v1/crds.go
+++ b/pkg/apis/acid.zalan.do/v1/crds.go
@@ -1556,6 +1556,9 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{
 							"logical_backup_s3_sse": {
 								Type: "string",
 							},
+							"logical_backup_s3_retention_time": {
+								Type: "string",
+							},
 							"logical_backup_schedule": {
 								Type:    "string",
 								Pattern: "^(\\d+|\\*)(/\\d+)?(\\s+(\\d+|\\*)(/\\d+)?){4}$",
diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
index 1298c6834..00fce42b5 100644
--- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
+++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
@@ -213,6 +213,7 @@ type OperatorLogicalBackupConfiguration struct {
 	S3AccessKeyID                string `json:"logical_backup_s3_access_key_id,omitempty"`
 	S3SecretAccessKey            string `json:"logical_backup_s3_secret_access_key,omitempty"`
 	S3SSE                        string `json:"logical_backup_s3_sse,omitempty"`
+	RetentionTime                string `json:"logical_backup_s3_retention_time,omitempty"`
 	GoogleApplicationCredentials string `json:"logical_backup_google_application_credentials,omitempty"`
 	JobPrefix                    string `json:"logical_backup_job_prefix,omitempty"`
 }
diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go
index e741c6dc4..91dd20765 100644
--- a/pkg/cluster/k8sres.go
+++ b/pkg/cluster/k8sres.go
@@ -2134,6 +2134,10 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar {
 			Name:  "LOGICAL_BACKUP_S3_SSE",
 			Value: c.OpConfig.LogicalBackup.LogicalBackupS3SSE,
 		},
+		{
+			Name:  "LOGICAL_BACKUP_S3_RETENTION_TIME",
+			Value: c.OpConfig.LogicalBackup.LogicalBackupS3RetentionTime,
+		},
 		{
 			Name:  "LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX",
 			Value: getBucketScopeSuffix(string(c.Postgresql.GetUID())),
diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go
index fbf12bfb9..543254242 100644
--- a/pkg/controller/operator_config.go
+++ b/pkg/controller/operator_config.go
@@ -170,6 +170,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	result.LogicalBackupS3AccessKeyID = fromCRD.LogicalBackup.S3AccessKeyID
 	result.LogicalBackupS3SecretAccessKey = fromCRD.LogicalBackup.S3SecretAccessKey
 	result.LogicalBackupS3SSE = fromCRD.LogicalBackup.S3SSE
+	result.LogicalBackupS3RetentionTime = fromCRD.LogicalBackup.RetentionTime
 	result.LogicalBackupGoogleApplicationCredentials = fromCRD.LogicalBackup.GoogleApplicationCredentials
 	result.LogicalBackupJobPrefix = util.Coalesce(fromCRD.LogicalBackup.JobPrefix, "logical-backup-")
 
diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go
index 0dc1004a7..f32762e46 100644
--- a/pkg/util/config/config.go
+++ b/pkg/util/config/config.go
@@ -128,6 +128,7 @@ type LogicalBackup struct {
 	LogicalBackupS3AccessKeyID                string `name:"logical_backup_s3_access_key_id" default:""`
 	LogicalBackupS3SecretAccessKey            string `name:"logical_backup_s3_secret_access_key" default:""`
 	LogicalBackupS3SSE                        string `name:"logical_backup_s3_sse" default:""`
+	LogicalBackupS3RetentionTime              string `name:"logical_backup_s3_retention_time" default:""`
 	LogicalBackupGoogleApplicationCredentials string `name:"logical_backup_google_application_credentials" default:""`
 	LogicalBackupJobPrefix                    string `name:"logical_backup_job_prefix" default:"logical-backup-"`
 }

From d032e4783eb84fcc9c3056d374f4b6340f18141b Mon Sep 17 00:00:00 2001
From: Felix Kunde <felix-kunde@gmx.de>
Date: Fri, 4 Mar 2022 13:36:17 +0100
Subject: [PATCH 30/58] LoadBalancer toggles for master and replica pooler pods
 (#1799)

* Add support for pooler load balancer

Signed-off-by: Sergey Shatunov <me@prok.pw>

* Rename to enable_master_pooler_load_balancer

Signed-off-by: Sergey Shatunov <me@prok.pw>

* target port should be intval
* enhance pooler e2e test
* add new options to crds.go

Co-authored-by: Sergey Shatunov <me@prok.pw>
---
 .../crds/operatorconfigurations.yaml          |   6 +
 .../postgres-operator/crds/postgresqls.yaml   |   4 +
 charts/postgres-operator/values.yaml          |   4 +
 docs/administrator.md                         |   5 +
 docs/reference/cluster_manifest.md            |  12 ++
 docs/reference/operator_parameters.md         |  16 +-
 e2e/tests/test_e2e.py                         |  75 +++----
 manifests/configmap.yaml                      |   2 +
 manifests/operatorconfiguration.crd.yaml      |   6 +
 ...gresql-operator-default-configuration.yaml |   2 +
 manifests/postgresql.crd.yaml                 |   4 +
 pkg/apis/acid.zalan.do/v1/crds.go             |  12 ++
 .../v1/operator_configuration_type.go         |  16 +-
 pkg/apis/acid.zalan.do/v1/postgresql_type.go  |   6 +-
 .../acid.zalan.do/v1/zz_generated.deepcopy.go |  10 +
 pkg/cluster/cluster_test.go                   |   2 +-
 pkg/cluster/connection_pooler.go              |  90 ++++++---
 pkg/cluster/connection_pooler_test.go         |  12 +-
 pkg/cluster/k8sres.go                         |  41 ++--
 pkg/cluster/k8sres_test.go                    | 183 ++++++++++++++++++
 pkg/cluster/resources.go                      |  27 ++-
 pkg/cluster/sync.go                           |   4 +-
 pkg/cluster/util_test.go                      |   2 +-
 pkg/controller/operator_config.go             |  13 +-
 pkg/util/config/config.go                     |   2 +
 25 files changed, 429 insertions(+), 127 deletions(-)

diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml
index f67c04fc5..469aa5588 100644
--- a/charts/postgres-operator/crds/operatorconfigurations.yaml
+++ b/charts/postgres-operator/crds/operatorconfigurations.yaml
@@ -380,9 +380,15 @@ spec:
                   enable_master_load_balancer:
                     type: boolean
                     default: true
+                  enable_master_pooler_load_balancer:
+                    type: boolean
+                    default: false
                   enable_replica_load_balancer:
                     type: boolean
                     default: false
+                  enable_replica_pooler_load_balancer:
+                    type: boolean
+                    default: false
                   external_traffic_policy:
                     type: string
                     enum:
diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml
index a7072b8cc..5ec40bdb1 100644
--- a/charts/postgres-operator/crds/postgresqls.yaml
+++ b/charts/postgres-operator/crds/postgresqls.yaml
@@ -197,8 +197,12 @@ spec:
                 type: boolean
               enableMasterLoadBalancer:
                 type: boolean
+              enableMasterPoolerLoadBalancer:
+                type: boolean
               enableReplicaLoadBalancer:
                 type: boolean
+              enableReplicaPoolerLoadBalancer:
+                type: boolean
               enableShmVolume:
                 type: boolean
               init_containers:
diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml
index 5d38b5233..9f144ba5e 100644
--- a/charts/postgres-operator/values.yaml
+++ b/charts/postgres-operator/values.yaml
@@ -228,8 +228,12 @@ configLoadBalancer:
 
   # toggles service type load balancer pointing to the master pod of the cluster
   enable_master_load_balancer: false
+  # toggles service type load balancer pointing to the master pooler pod of the cluster
+  enable_master_pooler_load_balancer: false
   # toggles service type load balancer pointing to the replica pod of the cluster
   enable_replica_load_balancer: false
+  # toggles service type load balancer pointing to the replica pooler pod of the cluster
+  enable_replica_pooler_load_balancer: false
   # define external traffic policy for the load balancer
   external_traffic_policy: "Cluster"
   # defines the DNS name string template for the master load balancer cluster
diff --git a/docs/administrator.md b/docs/administrator.md
index e68427658..dd6bf59b0 100644
--- a/docs/administrator.md
+++ b/docs/administrator.md
@@ -750,6 +750,11 @@ lead to K8s removing this field from the manifest due to its
 Then the resultant manifest will not contain the necessary change, and the
 operator will respectively do nothing with the existing source ranges.
 
+Load balancer services can also be enabled for the [connection pooler](user.md#connection-pooler)
+pods with manifest flags `enableMasterPoolerLoadBalancer` and/or
+`enableReplicaPoolerLoadBalancer` or in the operator configuration with
+`enable_master_pooler_load_balancer` and/or `enable_replica_pooler_load_balancer`.
+
 ## Running periodic 'autorepair' scans of K8s objects
 
 The Postgres Operator periodically scans all K8s objects belonging to each
diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md
index cff86333e..bf77bd8b8 100644
--- a/docs/reference/cluster_manifest.md
+++ b/docs/reference/cluster_manifest.md
@@ -91,11 +91,23 @@ These parameters are grouped directly under  the `spec` key in the manifest.
   `enable_master_load_balancer` parameter) to define whether to enable the load
   balancer pointing to the Postgres primary. Optional.
 
+* **enableMasterPoolerLoadBalancer**
+  boolean flag to override the operator defaults (set by the
+  `enable_master_pooler_load_balancer` parameter) to define whether to enable
+  the load balancer for master pooler pods pointing to the Postgres primary.
+  Optional.
+
 * **enableReplicaLoadBalancer**
   boolean flag to override the operator defaults (set by the
   `enable_replica_load_balancer` parameter) to define whether to enable the
   load balancer pointing to the Postgres standby instances. Optional.
 
+* **enableReplicaPoolerLoadBalancer**
+  boolean flag to override the operator defaults (set by the
+  `enable_replica_pooler_load_balancer` parameter) to define whether to enable
+  the load balancer for replica pooler pods pointing to the Postgres standby
+  instances. Optional.
+
 * **allowedSourceRanges**
   when one or more load balancers are enabled for the cluster, this parameter
   defines the comma-separated range of IP networks (in CIDR-notation). The
diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md
index cbe31ba57..5988271c5 100644
--- a/docs/reference/operator_parameters.md
+++ b/docs/reference/operator_parameters.md
@@ -546,11 +546,21 @@ In the CRD-based configuration they are grouped under the `load_balancer` key.
   toggles service type load balancer pointing to the master pod of the cluster.
   Can be overridden by individual cluster settings. The default is `true`.
 
-* **enable_replica_load_balancer**
-  toggles service type load balancer pointing to the replica pod of the
-  cluster.  Can be overridden by individual cluster settings. The default is
+* **enable_master_pooler_load_balancer**
+  toggles service type load balancer pointing to the master pooler pod of the
+  cluster. Can be overridden by individual cluster settings. The default is
   `false`.
 
+* **enable_replica_load_balancer**
+  toggles service type load balancer pointing to the replica pod(s) of the
+  cluster. Can be overridden by individual cluster settings. The default is
+  `false`.
+
+* **enable_replica_pooler_load_balancer**
+  toggles service type load balancer pointing to the replica pooler pod(s) of
+  the cluster. Can be overridden by individual cluster settings. The default
+  is `false`.
+
 * **external_traffic_policy** defines external traffic policy for load
   balancers. Allowed values are `Cluster` (default) and `Local`.
 
diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py
index c4d104069..09b2c9c60 100644
--- a/e2e/tests/test_e2e.py
+++ b/e2e/tests/test_e2e.py
@@ -467,6 +467,9 @@ class EndToEndTestCase(unittest.TestCase):
         the end turn connection pooler off to not interfere with other tests.
         '''
         k8s = self.k8s
+        pooler_label = 'application=db-connection-pooler,cluster-name=acid-minimal-cluster'
+        master_pooler_label = 'connection-pooler=acid-minimal-cluster-pooler'
+        replica_pooler_label = master_pooler_label + '-repl'
         self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
 
         k8s.api.custom_objects_api.patch_namespaced_custom_object(
@@ -476,22 +479,23 @@ class EndToEndTestCase(unittest.TestCase):
                 'spec': {
                     'enableConnectionPooler': True,
                     'enableReplicaConnectionPooler': True,
+                    'enableMasterPoolerLoadBalancer': True,
+                    'enableReplicaPoolerLoadBalancer': True,
                 }
             })
+        self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
 
-        self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(), 2,
-                             "Deployment replicas is 2 default")
-        self.eventuallyEqual(lambda: k8s.count_running_pods(
-                            "connection-pooler=acid-minimal-cluster-pooler"),
-                            2, "No pooler pods found")
-        self.eventuallyEqual(lambda: k8s.count_running_pods(
-                            "connection-pooler=acid-minimal-cluster-pooler-repl"),
-                            2, "No pooler replica pods found")
-        self.eventuallyEqual(lambda: k8s.count_services_with_label(
-                            'application=db-connection-pooler,cluster-name=acid-minimal-cluster'),
-                            2, "No pooler service found")
-        self.eventuallyEqual(lambda: k8s.count_secrets_with_label('application=db-connection-pooler,cluster-name=acid-minimal-cluster'),
-                             1, "Pooler secret not created")
+        self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(), 2, "Deployment replicas is 2 default")
+        self.eventuallyEqual(lambda: k8s.count_running_pods(master_pooler_label), 2, "No pooler pods found")
+        self.eventuallyEqual(lambda: k8s.count_running_pods(replica_pooler_label), 2, "No pooler replica pods found")
+        self.eventuallyEqual(lambda: k8s.count_services_with_label(pooler_label), 2, "No pooler service found")
+        self.eventuallyEqual(lambda: k8s.count_secrets_with_label(pooler_label), 1, "Pooler secret not created")
+        self.eventuallyEqual(lambda: k8s.get_service_type(master_pooler_label+","+pooler_label),
+                             'LoadBalancer',
+                             "Expected LoadBalancer service type for master pooler pod, found {}")
+        self.eventuallyEqual(lambda: k8s.get_service_type(replica_pooler_label+","+pooler_label),
+                             'LoadBalancer',
+                             "Expected LoadBalancer service type for replica pooler pod, found {}")
 
         # Turn off only master connection pooler
         k8s.api.custom_objects_api.patch_namespaced_custom_object(
@@ -504,20 +508,17 @@ class EndToEndTestCase(unittest.TestCase):
                 }
             })
 
-        self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"},
-                             "Operator does not get in sync")
+        self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
+
         self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(name="acid-minimal-cluster-pooler-repl"), 2,
                              "Deployment replicas is 2 default")
-        self.eventuallyEqual(lambda: k8s.count_running_pods(
-                             "connection-pooler=acid-minimal-cluster-pooler"),
+        self.eventuallyEqual(lambda: k8s.count_running_pods(master_pooler_label),
                              0, "Master pooler pods not deleted")
-        self.eventuallyEqual(lambda: k8s.count_running_pods(
-                             "connection-pooler=acid-minimal-cluster-pooler-repl"),
+        self.eventuallyEqual(lambda: k8s.count_running_pods(replica_pooler_label),
                              2, "Pooler replica pods not found")
-        self.eventuallyEqual(lambda: k8s.count_services_with_label(
-                             'application=db-connection-pooler,cluster-name=acid-minimal-cluster'),
+        self.eventuallyEqual(lambda: k8s.count_services_with_label(pooler_label),
                              1, "No pooler service found")
-        self.eventuallyEqual(lambda: k8s.count_secrets_with_label('application=db-connection-pooler,cluster-name=acid-minimal-cluster'),
+        self.eventuallyEqual(lambda: k8s.count_secrets_with_label(pooler_label),
                              1, "Secret not created")
 
         # Turn off only replica connection pooler
@@ -528,20 +529,24 @@ class EndToEndTestCase(unittest.TestCase):
                 'spec': {
                     'enableConnectionPooler': True,
                     'enableReplicaConnectionPooler': False,
+                    'enableMasterPoolerLoadBalancer': False,
                 }
             })
 
-        self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"},
-                             "Operator does not get in sync")
+        self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
+
         self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(), 2,
                              "Deployment replicas is 2 default")
-        self.eventuallyEqual(lambda: k8s.count_running_pods("connection-pooler=acid-minimal-cluster-pooler"),
+        self.eventuallyEqual(lambda: k8s.count_running_pods(master_pooler_label),
                              2, "Master pooler pods not found")
-        self.eventuallyEqual(lambda: k8s.count_running_pods("connection-pooler=acid-minimal-cluster-pooler-repl"),
+        self.eventuallyEqual(lambda: k8s.count_running_pods(replica_pooler_label),
                              0, "Pooler replica pods not deleted")
-        self.eventuallyEqual(lambda: k8s.count_services_with_label('application=db-connection-pooler,cluster-name=acid-minimal-cluster'),
+        self.eventuallyEqual(lambda: k8s.count_services_with_label(pooler_label),
                              1, "No pooler service found")
-        self.eventuallyEqual(lambda: k8s.count_secrets_with_label('application=db-connection-pooler,cluster-name=acid-minimal-cluster'),
+        self.eventuallyEqual(lambda: k8s.get_service_type(master_pooler_label+","+pooler_label),
+                             'ClusterIP',
+                             "Expected LoadBalancer service type for master, found {}")
+        self.eventuallyEqual(lambda: k8s.count_secrets_with_label(pooler_label),
                              1, "Secret not created")
 
         # scale up connection pooler deployment
@@ -558,7 +563,7 @@ class EndToEndTestCase(unittest.TestCase):
 
         self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(), 3,
                              "Deployment replicas is scaled to 3")
-        self.eventuallyEqual(lambda: k8s.count_running_pods("connection-pooler=acid-minimal-cluster-pooler"),
+        self.eventuallyEqual(lambda: k8s.count_running_pods(master_pooler_label),
                              3, "Scale up of pooler pods does not work")
 
         # turn it off, keeping config should be overwritten by false
@@ -569,12 +574,13 @@ class EndToEndTestCase(unittest.TestCase):
                 'spec': {
                     'enableConnectionPooler': False,
                     'enableReplicaConnectionPooler': False,
+                    'enableReplicaPoolerLoadBalancer': False,
                 }
             })
 
-        self.eventuallyEqual(lambda: k8s.count_running_pods("connection-pooler=acid-minimal-cluster-pooler"),
+        self.eventuallyEqual(lambda: k8s.count_running_pods(master_pooler_label),
                              0, "Pooler pods not scaled down")
-        self.eventuallyEqual(lambda: k8s.count_services_with_label('application=db-connection-pooler,cluster-name=acid-minimal-cluster'),
+        self.eventuallyEqual(lambda: k8s.count_services_with_label(pooler_label),
                              0, "Pooler service not removed")
         self.eventuallyEqual(lambda: k8s.count_secrets_with_label('application=spilo,cluster-name=acid-minimal-cluster'),
                              4, "Secrets not deleted")
@@ -1177,10 +1183,11 @@ class EndToEndTestCase(unittest.TestCase):
 
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
     def test_overwrite_pooler_deployment(self):
+        pooler_name = 'acid-minimal-cluster-pooler'
         k8s = self.k8s
         k8s.create_with_kubectl("manifests/minimal-fake-pooler-deployment.yaml")
         self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
-        self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(name="acid-minimal-cluster-pooler"), 1,
+        self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(name=pooler_name), 1,
                              "Initial broken deployment not rolled out")
 
         k8s.api.custom_objects_api.patch_namespaced_custom_object(
@@ -1193,7 +1200,7 @@ class EndToEndTestCase(unittest.TestCase):
         })
 
         self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
-        self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(name="acid-minimal-cluster-pooler"), 2,
+        self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(name=pooler_name), 2,
                              "Operator did not succeed in overwriting labels")
 
         k8s.api.custom_objects_api.patch_namespaced_custom_object(
@@ -1206,7 +1213,7 @@ class EndToEndTestCase(unittest.TestCase):
         })
 
         self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
-        self.eventuallyEqual(lambda: k8s.count_running_pods("connection-pooler=acid-minimal-cluster-pooler"),
+        self.eventuallyEqual(lambda: k8s.count_running_pods("connection-pooler="+pooler_name),
                              0, "Pooler pods not scaled down")
 
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml
index b9605441e..d127299c4 100644
--- a/manifests/configmap.yaml
+++ b/manifests/configmap.yaml
@@ -44,6 +44,7 @@ data:
   # enable_init_containers: "true"
   # enable_lazy_spilo_upgrade: "false"
   enable_master_load_balancer: "false"
+  enable_master_pooler_load_balancer: "false"
   enable_password_rotation: "false"
   enable_pgversion_env_var: "true"
   # enable_pod_antiaffinity: "false"
@@ -51,6 +52,7 @@ data:
   # enable_postgres_team_crd: "false"
   # enable_postgres_team_crd_superusers: "false"
   enable_replica_load_balancer: "false"
+  enable_replica_pooler_load_balancer: "false"
   # enable_shm_volume: "true"
   # enable_sidecars: "true"
   enable_spilo_wal_path_compat: "true"
diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml
index e6c2a2d7c..d6241abea 100644
--- a/manifests/operatorconfiguration.crd.yaml
+++ b/manifests/operatorconfiguration.crd.yaml
@@ -378,9 +378,15 @@ spec:
                   enable_master_load_balancer:
                     type: boolean
                     default: true
+                  enable_master_pooler_load_balancer:
+                    type: boolean
+                    default: false
                   enable_replica_load_balancer:
                     type: boolean
                     default: false
+                  enable_replica_pooler_load_balancer:
+                    type: boolean
+                    default: false
                   external_traffic_policy:
                     type: string
                     enum:
diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml
index 0641bd1b1..7ab8565a9 100644
--- a/manifests/postgresql-operator-default-configuration.yaml
+++ b/manifests/postgresql-operator-default-configuration.yaml
@@ -119,7 +119,9 @@ configuration:
     #   keyy: valuey
     # db_hosted_zone: ""
     enable_master_load_balancer: false
+    enable_master_pooler_load_balancer: false
     enable_replica_load_balancer: false
+    enable_replica_pooler_load_balancer: false
     external_traffic_policy: "Cluster"
     master_dns_name_format: "{cluster}.{team}.{hostedzone}"
     replica_dns_name_format: "{cluster}-repl.{team}.{hostedzone}"
diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml
index 7cf220eb7..183b97fc6 100644
--- a/manifests/postgresql.crd.yaml
+++ b/manifests/postgresql.crd.yaml
@@ -195,8 +195,12 @@ spec:
                 type: boolean
               enableMasterLoadBalancer:
                 type: boolean
+              enableMasterPoolerLoadBalancer:
+                type: boolean
               enableReplicaLoadBalancer:
                 type: boolean
+              enableReplicaPoolerLoadBalancer:
+                type: boolean
               enableShmVolume:
                 type: boolean
               init_containers:
diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go
index d8d46e248..84c57b7cf 100644
--- a/pkg/apis/acid.zalan.do/v1/crds.go
+++ b/pkg/apis/acid.zalan.do/v1/crds.go
@@ -302,9 +302,15 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 					"enableMasterLoadBalancer": {
 						Type: "boolean",
 					},
+					"enableMasterPoolerLoadBalancer": {
+						Type: "boolean",
+					},
 					"enableReplicaLoadBalancer": {
 						Type: "boolean",
 					},
+					"enableReplicaPoolerLoadBalancer": {
+						Type: "boolean",
+					},
 					"enableShmVolume": {
 						Type: "boolean",
 					},
@@ -1469,9 +1475,15 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{
 							"enable_master_load_balancer": {
 								Type: "boolean",
 							},
+							"enable_master_pooler_load_balancer": {
+								Type: "boolean",
+							},
 							"enable_replica_load_balancer": {
 								Type: "boolean",
 							},
+							"enable_replica_pooler_load_balancer": {
+								Type: "boolean",
+							},
 							"external_traffic_policy": {
 								Type: "string",
 								Enum: []apiextv1.JSON{
diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
index 00fce42b5..f5c57aaa4 100644
--- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
+++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
@@ -121,13 +121,15 @@ type OperatorTimeouts struct {
 
 // LoadBalancerConfiguration defines the LB configuration
 type LoadBalancerConfiguration struct {
-	DbHostedZone              string                `json:"db_hosted_zone,omitempty"`
-	EnableMasterLoadBalancer  bool                  `json:"enable_master_load_balancer,omitempty"`
-	EnableReplicaLoadBalancer bool                  `json:"enable_replica_load_balancer,omitempty"`
-	CustomServiceAnnotations  map[string]string     `json:"custom_service_annotations,omitempty"`
-	MasterDNSNameFormat       config.StringTemplate `json:"master_dns_name_format,omitempty"`
-	ReplicaDNSNameFormat      config.StringTemplate `json:"replica_dns_name_format,omitempty"`
-	ExternalTrafficPolicy     string                `json:"external_traffic_policy" default:"Cluster"`
+	DbHostedZone                    string                `json:"db_hosted_zone,omitempty"`
+	EnableMasterLoadBalancer        bool                  `json:"enable_master_load_balancer,omitempty"`
+	EnableMasterPoolerLoadBalancer  bool                  `json:"enable_master_pooler_load_balancer,omitempty"`
+	EnableReplicaLoadBalancer       bool                  `json:"enable_replica_load_balancer,omitempty"`
+	EnableReplicaPoolerLoadBalancer bool                  `json:"enable_replica_pooler_load_balancer,omitempty"`
+	CustomServiceAnnotations        map[string]string     `json:"custom_service_annotations,omitempty"`
+	MasterDNSNameFormat             config.StringTemplate `json:"master_dns_name_format,omitempty"`
+	ReplicaDNSNameFormat            config.StringTemplate `json:"replica_dns_name_format,omitempty"`
+	ExternalTrafficPolicy           string                `json:"external_traffic_policy" default:"Cluster"`
 }
 
 // AWSGCPConfiguration defines the configuration for AWS
diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go
index a1fc4fcf7..50be9e551 100644
--- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go
+++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go
@@ -42,8 +42,10 @@ type PostgresSpec struct {
 
 	// vars that enable load balancers are pointers because it is important to know if any of them is omitted from the Postgres manifest
 	// in that case the var evaluates to nil and the value is taken from the operator config
-	EnableMasterLoadBalancer  *bool `json:"enableMasterLoadBalancer,omitempty"`
-	EnableReplicaLoadBalancer *bool `json:"enableReplicaLoadBalancer,omitempty"`
+	EnableMasterLoadBalancer        *bool `json:"enableMasterLoadBalancer,omitempty"`
+	EnableMasterPoolerLoadBalancer  *bool `json:"enableMasterPoolerLoadBalancer,omitempty"`
+	EnableReplicaLoadBalancer       *bool `json:"enableReplicaLoadBalancer,omitempty"`
+	EnableReplicaPoolerLoadBalancer *bool `json:"enableReplicaPoolerLoadBalancer,omitempty"`
 
 	// deprecated load balancer settings maintained for backward compatibility
 	// see "Load balancers" operator docs
diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
index c2298fada..91f6be563 100644
--- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
+++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
@@ -611,11 +611,21 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) {
 		*out = new(bool)
 		**out = **in
 	}
+	if in.EnableMasterPoolerLoadBalancer != nil {
+		in, out := &in.EnableMasterPoolerLoadBalancer, &out.EnableMasterPoolerLoadBalancer
+		*out = new(bool)
+		**out = **in
+	}
 	if in.EnableReplicaLoadBalancer != nil {
 		in, out := &in.EnableReplicaLoadBalancer, &out.EnableReplicaLoadBalancer
 		*out = new(bool)
 		**out = **in
 	}
+	if in.EnableReplicaPoolerLoadBalancer != nil {
+		in, out := &in.EnableReplicaPoolerLoadBalancer, &out.EnableReplicaPoolerLoadBalancer
+		*out = new(bool)
+		**out = **in
+	}
 	if in.UseLoadBalancer != nil {
 		in, out := &in.UseLoadBalancer, &out.UseLoadBalancer
 		*out = new(bool)
diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go
index d06cc21e1..fba11a6dd 100644
--- a/pkg/cluster/cluster_test.go
+++ b/pkg/cluster/cluster_test.go
@@ -1041,7 +1041,7 @@ func TestCrossNamespacedSecrets(t *testing.T) {
 					ConnectionPoolerDefaultCPULimit:      "100m",
 					ConnectionPoolerDefaultMemoryRequest: "100Mi",
 					ConnectionPoolerDefaultMemoryLimit:   "100Mi",
-					NumberOfInstances:                    int32ToPointer(1),
+					NumberOfInstances:                    k8sutil.Int32ToPointer(1),
 				},
 				PodManagementPolicy: "ordered_ready",
 				Resources: config.Resources{
diff --git a/pkg/cluster/connection_pooler.go b/pkg/cluster/connection_pooler.go
index ec9fe291d..12c0be6ae 100644
--- a/pkg/cluster/connection_pooler.go
+++ b/pkg/cluster/connection_pooler.go
@@ -247,7 +247,7 @@ func (c *Cluster) generateConnectionPoolerPodTemplate(role PostgresRole) (
 		},
 		{
 			Name:  "PGPORT",
-			Value: c.servicePort(role),
+			Value: fmt.Sprint(c.servicePort(role)),
 		},
 		{
 			Name: "PGUSER",
@@ -386,7 +386,7 @@ func (c *Cluster) generateConnectionPoolerService(connectionPooler *ConnectionPo
 			{
 				Name:       connectionPooler.Name,
 				Port:       pgPort,
-				TargetPort: intstr.IntOrString{StrVal: c.servicePort(connectionPooler.Role)},
+				TargetPort: intstr.IntOrString{IntVal: c.servicePort(connectionPooler.Role)},
 			},
 		},
 		Type: v1.ServiceTypeClusterIP,
@@ -395,6 +395,10 @@ func (c *Cluster) generateConnectionPoolerService(connectionPooler *ConnectionPo
 		},
 	}
 
+	if c.shouldCreateLoadBalancerForPoolerService(connectionPooler.Role, spec) {
+		c.configureLoadBalanceService(&serviceSpec, spec.AllowedSourceRanges)
+	}
+
 	service := &v1.Service{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        connectionPooler.Name,
@@ -415,6 +419,29 @@ func (c *Cluster) generateConnectionPoolerService(connectionPooler *ConnectionPo
 	return service
 }
 
+func (c *Cluster) shouldCreateLoadBalancerForPoolerService(role PostgresRole, spec *acidv1.PostgresSpec) bool {
+
+	switch role {
+
+	case Replica:
+		// if the value is explicitly set in a Postgresql manifest, follow this setting
+		if spec.EnableReplicaPoolerLoadBalancer != nil {
+			return *spec.EnableReplicaPoolerLoadBalancer
+		}
+		// otherwise, follow the operator configuration
+		return c.OpConfig.EnableReplicaPoolerLoadBalancer
+
+	case Master:
+		if spec.EnableMasterPoolerLoadBalancer != nil {
+			return *spec.EnableMasterPoolerLoadBalancer
+		}
+		return c.OpConfig.EnableMasterPoolerLoadBalancer
+
+	default:
+		panic(fmt.Sprintf("Unknown role %v", role))
+	}
+}
+
 //delete connection pooler
 func (c *Cluster) deleteConnectionPooler(role PostgresRole) (err error) {
 	c.logger.Infof("deleting connection pooler spilo-role=%s", role)
@@ -811,6 +838,7 @@ func (c *Cluster) syncConnectionPooler(oldSpec, newSpec *acidv1.Postgresql, Look
 func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql, role PostgresRole) (
 	SyncReason, error) {
 
+	syncReason := make([]string, 0)
 	deployment, err := c.KubeClient.
 		Deployments(c.Namespace).
 		Get(context.TODO(), c.connectionPoolerName(role), metav1.GetOptions{})
@@ -865,25 +893,26 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql
 
 		if oldSpec != nil {
 			specSync, specReason = needSyncConnectionPoolerSpecs(oldConnectionPooler, newConnectionPooler, c.logger)
+			syncReason = append(syncReason, specReason...)
 		}
 
 		defaultsSync, defaultsReason := c.needSyncConnectionPoolerDefaults(&c.Config, newConnectionPooler, deployment)
-		reason := append(specReason, defaultsReason...)
+		syncReason = append(syncReason, defaultsReason...)
 
 		if specSync || defaultsSync {
 			c.logger.Infof("Update connection pooler deployment %s, reason: %+v",
-				c.connectionPoolerName(role), reason)
+				c.connectionPoolerName(role), syncReason)
 			newDeploymentSpec, err := c.generateConnectionPoolerDeployment(c.ConnectionPooler[role])
 			if err != nil {
 				msg := "could not generate deployment for connection pooler: %v"
-				return reason, fmt.Errorf(msg, err)
+				return syncReason, fmt.Errorf(msg, err)
 			}
 
 			deployment, err := updateConnectionPoolerDeployment(c.KubeClient,
 				newDeploymentSpec)
 
 			if err != nil {
-				return reason, err
+				return syncReason, err
 			}
 			c.ConnectionPooler[role].Deployment = deployment
 		}
@@ -898,31 +927,40 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql
 		c.ConnectionPooler[role].Deployment = deployment
 	}
 
-	service, err := c.KubeClient.
-		Services(c.Namespace).
-		Get(context.TODO(), c.connectionPoolerName(role), metav1.GetOptions{})
-
-	if err != nil && k8sutil.ResourceNotFound(err) {
-		msg := "Service %s for connection pooler synchronization is not found, create it"
-		c.logger.Warningf(msg, c.connectionPoolerName(role))
-
-		serviceSpec := c.generateConnectionPoolerService(c.ConnectionPooler[role])
-		service, err := c.KubeClient.
-			Services(serviceSpec.Namespace).
-			Create(context.TODO(), serviceSpec, metav1.CreateOptions{})
-
-		if err != nil {
-			return NoSync, err
+	if svc, err := c.KubeClient.Services(c.Namespace).Get(context.TODO(), c.connectionPoolerName(role), metav1.GetOptions{}); err == nil {
+		c.ConnectionPooler[role].Service = svc
+		desiredSvc := c.generateConnectionPoolerService(c.ConnectionPooler[role])
+		if match, reason := k8sutil.SameService(svc, desiredSvc); !match {
+			syncReason = append(syncReason, reason)
+			c.logServiceChanges(role, svc, desiredSvc, false, reason)
+			updatedService, err := c.updateService(role, svc, desiredSvc)
+			if err != nil {
+				return syncReason, fmt.Errorf("could not update %s service to match desired state: %v", role, err)
+			}
+			c.ConnectionPooler[role].Service = updatedService
+			c.logger.Infof("%s service %q is in the desired state now", role, util.NameFromMeta(desiredSvc.ObjectMeta))
 		}
-		c.ConnectionPooler[role].Service = service
+		return NoSync, nil
+	}
 
-	} else if err != nil {
+	if !k8sutil.ResourceNotFound(err) {
 		msg := "could not get connection pooler service to sync: %v"
 		return NoSync, fmt.Errorf(msg, err)
-	} else {
-		// Service updates are not supported and probably not that useful anyway
-		c.ConnectionPooler[role].Service = service
 	}
 
+	c.ConnectionPooler[role].Service = nil
+	msg := "Service %s for connection pooler synchronization is not found, create it"
+	c.logger.Warningf(msg, c.connectionPoolerName(role))
+
+	serviceSpec := c.generateConnectionPoolerService(c.ConnectionPooler[role])
+	service, err := c.KubeClient.
+		Services(serviceSpec.Namespace).
+		Create(context.TODO(), serviceSpec, metav1.CreateOptions{})
+
+	if err != nil {
+		return NoSync, err
+	}
+	c.ConnectionPooler[role].Service = service
+
 	return NoSync, nil
 }
diff --git a/pkg/cluster/connection_pooler_test.go b/pkg/cluster/connection_pooler_test.go
index 9b983c7b0..da45899b4 100644
--- a/pkg/cluster/connection_pooler_test.go
+++ b/pkg/cluster/connection_pooler_test.go
@@ -27,10 +27,6 @@ func boolToPointer(value bool) *bool {
 	return &value
 }
 
-func int32ToPointer(value int32) *int32 {
-	return &value
-}
-
 func deploymentUpdated(cluster *Cluster, err error, reason SyncReason) error {
 	for _, role := range [2]PostgresRole{Master, Replica} {
 
@@ -294,7 +290,7 @@ func TestConnectionPoolerCreateDeletion(t *testing.T) {
 					ConnectionPoolerDefaultCPULimit:      "100m",
 					ConnectionPoolerDefaultMemoryRequest: "100Mi",
 					ConnectionPoolerDefaultMemoryLimit:   "100Mi",
-					NumberOfInstances:                    int32ToPointer(1),
+					NumberOfInstances:                    k8sutil.Int32ToPointer(1),
 				},
 				PodManagementPolicy: "ordered_ready",
 				Resources: config.Resources{
@@ -401,7 +397,7 @@ func TestConnectionPoolerSync(t *testing.T) {
 					ConnectionPoolerDefaultCPULimit:      "100m",
 					ConnectionPoolerDefaultMemoryRequest: "100Mi",
 					ConnectionPoolerDefaultMemoryLimit:   "100Mi",
-					NumberOfInstances:                    int32ToPointer(1),
+					NumberOfInstances:                    k8sutil.Int32ToPointer(1),
 				},
 				PodManagementPolicy: "ordered_ready",
 				Resources: config.Resources{
@@ -639,7 +635,7 @@ func TestConnectionPoolerSync(t *testing.T) {
 	for _, tt := range tests {
 		tt.cluster.OpConfig.ConnectionPooler.Image = tt.defaultImage
 		tt.cluster.OpConfig.ConnectionPooler.NumberOfInstances =
-			int32ToPointer(tt.defaultInstances)
+			k8sutil.Int32ToPointer(tt.defaultInstances)
 
 		t.Logf("running test for %s [%s]", testName, tt.subTest)
 
@@ -664,7 +660,7 @@ func TestConnectionPoolerPodSpec(t *testing.T) {
 					ReplicationUsername: replicationUserName,
 				},
 				ConnectionPooler: config.ConnectionPooler{
-					MaxDBConnections:                     int32ToPointer(60),
+					MaxDBConnections:                     k8sutil.Int32ToPointer(60),
 					ConnectionPoolerDefaultCPURequest:    "100m",
 					ConnectionPoolerDefaultCPULimit:      "100m",
 					ConnectionPoolerDefaultMemoryRequest: "100Mi",
diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go
index 91dd20765..6fa983c80 100644
--- a/pkg/cluster/k8sres.go
+++ b/pkg/cluster/k8sres.go
@@ -102,15 +102,15 @@ func (c *Cluster) serviceAddress(role PostgresRole) string {
 	return ""
 }
 
-func (c *Cluster) servicePort(role PostgresRole) string {
+func (c *Cluster) servicePort(role PostgresRole) int32 {
 	service, exist := c.Services[role]
 
 	if exist {
-		return fmt.Sprint(service.Spec.Ports[0].Port)
+		return service.Spec.Ports[0].Port
 	}
 
-	c.logger.Warningf("No service for role %s", role)
-	return ""
+	c.logger.Warningf("No service for role %s - defaulting to port 5432", role)
+	return pgPort
 }
 
 func (c *Cluster) podDisruptionBudgetName() string {
@@ -1699,23 +1699,7 @@ func (c *Cluster) generateService(role PostgresRole, spec *acidv1.PostgresSpec)
 	}
 
 	if c.shouldCreateLoadBalancerForService(role, spec) {
-
-		// spec.AllowedSourceRanges evaluates to the empty slice of zero length
-		// when omitted or set to 'null'/empty sequence in the PG manifest
-		if len(spec.AllowedSourceRanges) > 0 {
-			serviceSpec.LoadBalancerSourceRanges = spec.AllowedSourceRanges
-		} else {
-			// safe default value: lock a load balancer only to the local address unless overridden explicitly
-			serviceSpec.LoadBalancerSourceRanges = []string{localHost}
-		}
-
-		c.logger.Debugf("final load balancer source ranges as seen in a service spec (not necessarily applied): %q", serviceSpec.LoadBalancerSourceRanges)
-		serviceSpec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyType(c.OpConfig.ExternalTrafficPolicy)
-		serviceSpec.Type = v1.ServiceTypeLoadBalancer
-	} else if role == Replica {
-		// before PR #258, the replica service was only created if allocated a LB
-		// now we always create the service but warn if the LB is absent
-		c.logger.Debugf("No load balancer created for the replica service")
+		c.configureLoadBalanceService(&serviceSpec, spec.AllowedSourceRanges)
 	}
 
 	service := &v1.Service{
@@ -1731,6 +1715,21 @@ func (c *Cluster) generateService(role PostgresRole, spec *acidv1.PostgresSpec)
 	return service
 }
 
+func (c *Cluster) configureLoadBalanceService(serviceSpec *v1.ServiceSpec, sourceRanges []string) {
+	// spec.AllowedSourceRanges evaluates to the empty slice of zero length
+	// when omitted or set to 'null'/empty sequence in the PG manifest
+	if len(sourceRanges) > 0 {
+		serviceSpec.LoadBalancerSourceRanges = sourceRanges
+	} else {
+		// safe default value: lock a load balancer only to the local address unless overridden explicitly
+		serviceSpec.LoadBalancerSourceRanges = []string{localHost}
+	}
+
+	c.logger.Debugf("final load balancer source ranges as seen in a service spec (not necessarily applied): %q", serviceSpec.LoadBalancerSourceRanges)
+	serviceSpec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyType(c.OpConfig.ExternalTrafficPolicy)
+	serviceSpec.Type = v1.ServiceTypeLoadBalancer
+}
+
 func (c *Cluster) generateServiceAnnotations(role PostgresRole, spec *acidv1.PostgresSpec) map[string]string {
 	annotations := make(map[string]string)
 
diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go
index 6875c9dba..cb2fd83c0 100644
--- a/pkg/cluster/k8sres_test.go
+++ b/pkg/cluster/k8sres_test.go
@@ -27,6 +27,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/kubernetes/fake"
 	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
 )
@@ -1484,6 +1485,188 @@ func TestGenerateService(t *testing.T) {
 
 }
 
+func newLBFakeClient() (k8sutil.KubernetesClient, *fake.Clientset) {
+	clientSet := fake.NewSimpleClientset()
+
+	return k8sutil.KubernetesClient{
+		DeploymentsGetter: clientSet.AppsV1(),
+		ServicesGetter:    clientSet.CoreV1(),
+	}, clientSet
+}
+
+func getServices(serviceType v1.ServiceType, sourceRanges []string, extTrafficPolicy, clusterName string) []v1.ServiceSpec {
+	return []v1.ServiceSpec{
+		v1.ServiceSpec{
+			ExternalTrafficPolicy:    v1.ServiceExternalTrafficPolicyType(extTrafficPolicy),
+			LoadBalancerSourceRanges: sourceRanges,
+			Ports:                    []v1.ServicePort{{Name: "postgresql", Port: 5432, TargetPort: intstr.IntOrString{IntVal: 5432}}},
+			Type:                     serviceType,
+		},
+		v1.ServiceSpec{
+			ExternalTrafficPolicy:    v1.ServiceExternalTrafficPolicyType(extTrafficPolicy),
+			LoadBalancerSourceRanges: sourceRanges,
+			Ports:                    []v1.ServicePort{{Name: clusterName + "-pooler", Port: 5432, TargetPort: intstr.IntOrString{IntVal: 5432}}},
+			Selector:                 map[string]string{"connection-pooler": clusterName + "-pooler"},
+			Type:                     serviceType,
+		},
+		v1.ServiceSpec{
+			ExternalTrafficPolicy:    v1.ServiceExternalTrafficPolicyType(extTrafficPolicy),
+			LoadBalancerSourceRanges: sourceRanges,
+			Ports:                    []v1.ServicePort{{Name: "postgresql", Port: 5432, TargetPort: intstr.IntOrString{IntVal: 5432}}},
+			Selector:                 map[string]string{"spilo-role": "replica", "application": "spilo", "cluster-name": clusterName},
+			Type:                     serviceType,
+		},
+		v1.ServiceSpec{
+			ExternalTrafficPolicy:    v1.ServiceExternalTrafficPolicyType(extTrafficPolicy),
+			LoadBalancerSourceRanges: sourceRanges,
+			Ports:                    []v1.ServicePort{{Name: clusterName + "-pooler-repl", Port: 5432, TargetPort: intstr.IntOrString{IntVal: 5432}}},
+			Selector:                 map[string]string{"connection-pooler": clusterName + "-pooler-repl"},
+			Type:                     serviceType,
+		},
+	}
+}
+
+func TestEnableLoadBalancers(t *testing.T) {
+	testName := "Test enabling LoadBalancers"
+	client, _ := newLBFakeClient()
+	clusterName := "acid-test-cluster"
+	namespace := "default"
+	clusterNameLabel := "cluster-name"
+	roleLabel := "spilo-role"
+	roles := []PostgresRole{Master, Replica}
+	sourceRanges := []string{"192.186.1.2/22"}
+	extTrafficPolicy := "Cluster"
+
+	tests := []struct {
+		subTest          string
+		config           config.Config
+		pgSpec           acidv1.Postgresql
+		expectedServices []v1.ServiceSpec
+	}{
+		{
+			subTest: "LBs enabled in config, disabled in manifest",
+			config: config.Config{
+				ConnectionPooler: config.ConnectionPooler{
+					ConnectionPoolerDefaultCPURequest:    "100m",
+					ConnectionPoolerDefaultCPULimit:      "100m",
+					ConnectionPoolerDefaultMemoryRequest: "100Mi",
+					ConnectionPoolerDefaultMemoryLimit:   "100Mi",
+					NumberOfInstances:                    k8sutil.Int32ToPointer(1),
+				},
+				EnableMasterLoadBalancer:        true,
+				EnableMasterPoolerLoadBalancer:  true,
+				EnableReplicaLoadBalancer:       true,
+				EnableReplicaPoolerLoadBalancer: true,
+				ExternalTrafficPolicy:           extTrafficPolicy,
+				Resources: config.Resources{
+					ClusterLabels:    map[string]string{"application": "spilo"},
+					ClusterNameLabel: clusterNameLabel,
+					PodRoleLabel:     roleLabel,
+				},
+			},
+			pgSpec: acidv1.Postgresql{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      clusterName,
+					Namespace: namespace,
+				},
+				Spec: acidv1.PostgresSpec{
+					AllowedSourceRanges:             sourceRanges,
+					EnableConnectionPooler:          util.True(),
+					EnableReplicaConnectionPooler:   util.True(),
+					EnableMasterLoadBalancer:        util.False(),
+					EnableMasterPoolerLoadBalancer:  util.False(),
+					EnableReplicaLoadBalancer:       util.False(),
+					EnableReplicaPoolerLoadBalancer: util.False(),
+					NumberOfInstances:               1,
+					Resources: acidv1.Resources{
+						ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"},
+						ResourceLimits:   acidv1.ResourceDescription{CPU: "1", Memory: "10"},
+					},
+					TeamID: "acid",
+					Volume: acidv1.Volume{
+						Size: "1G",
+					},
+				},
+			},
+			expectedServices: getServices(v1.ServiceTypeClusterIP, nil, "", clusterName),
+		},
+		{
+			subTest: "LBs enabled in manifest, disabled in config",
+			config: config.Config{
+				ConnectionPooler: config.ConnectionPooler{
+					ConnectionPoolerDefaultCPURequest:    "100m",
+					ConnectionPoolerDefaultCPULimit:      "100m",
+					ConnectionPoolerDefaultMemoryRequest: "100Mi",
+					ConnectionPoolerDefaultMemoryLimit:   "100Mi",
+					NumberOfInstances:                    k8sutil.Int32ToPointer(1),
+				},
+				EnableMasterLoadBalancer:        false,
+				EnableMasterPoolerLoadBalancer:  false,
+				EnableReplicaLoadBalancer:       false,
+				EnableReplicaPoolerLoadBalancer: false,
+				ExternalTrafficPolicy:           extTrafficPolicy,
+				Resources: config.Resources{
+					ClusterLabels:    map[string]string{"application": "spilo"},
+					ClusterNameLabel: clusterNameLabel,
+					PodRoleLabel:     roleLabel,
+				},
+			},
+			pgSpec: acidv1.Postgresql{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      clusterName,
+					Namespace: namespace,
+				},
+				Spec: acidv1.PostgresSpec{
+					AllowedSourceRanges:             sourceRanges,
+					EnableConnectionPooler:          util.True(),
+					EnableReplicaConnectionPooler:   util.True(),
+					EnableMasterLoadBalancer:        util.True(),
+					EnableMasterPoolerLoadBalancer:  util.True(),
+					EnableReplicaLoadBalancer:       util.True(),
+					EnableReplicaPoolerLoadBalancer: util.True(),
+					NumberOfInstances:               1,
+					Resources: acidv1.Resources{
+						ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"},
+						ResourceLimits:   acidv1.ResourceDescription{CPU: "1", Memory: "10"},
+					},
+					TeamID: "acid",
+					Volume: acidv1.Volume{
+						Size: "1G",
+					},
+				},
+			},
+			expectedServices: getServices(v1.ServiceTypeLoadBalancer, sourceRanges, extTrafficPolicy, clusterName),
+		},
+	}
+
+	for _, tt := range tests {
+		var cluster = New(
+			Config{
+				OpConfig: tt.config,
+			}, client, tt.pgSpec, logger, eventRecorder)
+
+		cluster.Name = clusterName
+		cluster.Namespace = namespace
+		cluster.ConnectionPooler = map[PostgresRole]*ConnectionPoolerObjects{}
+		generatedServices := make([]v1.ServiceSpec, 0)
+		for _, role := range roles {
+			cluster.syncService(role)
+			cluster.ConnectionPooler[role] = &ConnectionPoolerObjects{
+				Name:        cluster.connectionPoolerName(role),
+				ClusterName: cluster.ClusterName,
+				Namespace:   cluster.Namespace,
+				Role:        role,
+			}
+			cluster.syncConnectionPoolerWorker(&tt.pgSpec, &tt.pgSpec, role)
+			generatedServices = append(generatedServices, cluster.Services[role].Spec)
+			generatedServices = append(generatedServices, cluster.ConnectionPooler[role].Service.Spec)
+		}
+		if !reflect.DeepEqual(tt.expectedServices, generatedServices) {
+			t.Errorf("%s %s: expected %#v but got %#v", testName, tt.subTest, tt.expectedServices, generatedServices)
+		}
+	}
+}
+
 func TestGenerateCapabilities(t *testing.T) {
 
 	testName := "TestGenerateCapabilities"
diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go
index 48a82ee04..5e5c6156e 100644
--- a/pkg/cluster/resources.go
+++ b/pkg/cluster/resources.go
@@ -275,7 +275,7 @@ func (c *Cluster) createService(role PostgresRole) (*v1.Service, error) {
 	return service, nil
 }
 
-func (c *Cluster) updateService(role PostgresRole, newService *v1.Service) error {
+func (c *Cluster) updateService(role PostgresRole, oldService *v1.Service, newService *v1.Service) (*v1.Service, error) {
 	var (
 		svc *v1.Service
 		err error
@@ -283,11 +283,7 @@ func (c *Cluster) updateService(role PostgresRole, newService *v1.Service) error
 
 	c.setProcessName("updating %v service", role)
 
-	if c.Services[role] == nil {
-		return fmt.Errorf("there is no service in the cluster")
-	}
-
-	serviceName := util.NameFromMeta(c.Services[role].ObjectMeta)
+	serviceName := util.NameFromMeta(oldService.ObjectMeta)
 
 	// update the service annotation in order to propagate ELB notation.
 	if len(newService.ObjectMeta.Annotations) > 0 {
@@ -301,39 +297,38 @@ func (c *Cluster) updateService(role PostgresRole, newService *v1.Service) error
 				"")
 
 			if err != nil {
-				return fmt.Errorf("could not replace annotations for the service %q: %v", serviceName, err)
+				return nil, fmt.Errorf("could not replace annotations for the service %q: %v", serviceName, err)
 			}
 		} else {
-			return fmt.Errorf("could not form patch for the service metadata: %v", err)
+			return nil, fmt.Errorf("could not form patch for the service metadata: %v", err)
 		}
 	}
 
 	// now, patch the service spec, but when disabling LoadBalancers do update instead
 	// patch does not work because of LoadBalancerSourceRanges field (even if set to nil)
-	oldServiceType := c.Services[role].Spec.Type
+	oldServiceType := oldService.Spec.Type
 	newServiceType := newService.Spec.Type
 	if newServiceType == "ClusterIP" && newServiceType != oldServiceType {
-		newService.ResourceVersion = c.Services[role].ResourceVersion
-		newService.Spec.ClusterIP = c.Services[role].Spec.ClusterIP
+		newService.ResourceVersion = oldService.ResourceVersion
+		newService.Spec.ClusterIP = oldService.Spec.ClusterIP
 		svc, err = c.KubeClient.Services(serviceName.Namespace).Update(context.TODO(), newService, metav1.UpdateOptions{})
 		if err != nil {
-			return fmt.Errorf("could not update service %q: %v", serviceName, err)
+			return nil, fmt.Errorf("could not update service %q: %v", serviceName, err)
 		}
 	} else {
 		patchData, err := specPatch(newService.Spec)
 		if err != nil {
-			return fmt.Errorf("could not form patch for the service %q: %v", serviceName, err)
+			return nil, fmt.Errorf("could not form patch for the service %q: %v", serviceName, err)
 		}
 
 		svc, err = c.KubeClient.Services(serviceName.Namespace).Patch(
 			context.TODO(), serviceName.Name, types.MergePatchType, patchData, metav1.PatchOptions{}, "")
 		if err != nil {
-			return fmt.Errorf("could not patch service %q: %v", serviceName, err)
+			return nil, fmt.Errorf("could not patch service %q: %v", serviceName, err)
 		}
 	}
-	c.Services[role] = svc
 
-	return nil
+	return svc, nil
 }
 
 func (c *Cluster) deleteService(role PostgresRole) error {
diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go
index a897ff318..4548a5b14 100644
--- a/pkg/cluster/sync.go
+++ b/pkg/cluster/sync.go
@@ -174,9 +174,11 @@ func (c *Cluster) syncService(role PostgresRole) error {
 		desiredSvc := c.generateService(role, &c.Spec)
 		if match, reason := k8sutil.SameService(svc, desiredSvc); !match {
 			c.logServiceChanges(role, svc, desiredSvc, false, reason)
-			if err = c.updateService(role, desiredSvc); err != nil {
+			updatedSvc, err := c.updateService(role, svc, desiredSvc)
+			if err != nil {
 				return fmt.Errorf("could not update %s service to match desired state: %v", role, err)
 			}
+			c.Services[role] = updatedSvc
 			c.logger.Infof("%s service %q is in the desired state now", role, util.NameFromMeta(desiredSvc.ObjectMeta))
 		}
 		return nil
diff --git a/pkg/cluster/util_test.go b/pkg/cluster/util_test.go
index 1463a5609..9ed644ea2 100644
--- a/pkg/cluster/util_test.go
+++ b/pkg/cluster/util_test.go
@@ -57,7 +57,7 @@ func TestInheritedAnnotations(t *testing.T) {
 					ConnectionPoolerDefaultCPULimit:      "100m",
 					ConnectionPoolerDefaultMemoryRequest: "100Mi",
 					ConnectionPoolerDefaultMemoryLimit:   "100Mi",
-					NumberOfInstances:                    int32ToPointer(1),
+					NumberOfInstances:                    k8sutil.Int32ToPointer(1),
 				},
 				PodManagementPolicy: "ordered_ready",
 				Resources: config.Resources{
diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go
index 543254242..e470fb59d 100644
--- a/pkg/controller/operator_config.go
+++ b/pkg/controller/operator_config.go
@@ -10,6 +10,7 @@ import (
 	"github.com/zalando/postgres-operator/pkg/util"
 	"github.com/zalando/postgres-operator/pkg/util/config"
 	"github.com/zalando/postgres-operator/pkg/util/constants"
+	"github.com/zalando/postgres-operator/pkg/util/k8sutil"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -24,10 +25,6 @@ func (c *Controller) readOperatorConfigurationFromCRD(configObjectNamespace, con
 	return config, nil
 }
 
-func int32ToPointer(value int32) *int32 {
-	return &value
-}
-
 // importConfigurationFromCRD is a transitional function that converts CRD configuration to the one based on the configmap
 func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigurationData) *config.Config {
 	result := &config.Config{}
@@ -141,7 +138,9 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	// load balancer config
 	result.DbHostedZone = util.Coalesce(fromCRD.LoadBalancer.DbHostedZone, "db.example.com")
 	result.EnableMasterLoadBalancer = fromCRD.LoadBalancer.EnableMasterLoadBalancer
+	result.EnableMasterPoolerLoadBalancer = fromCRD.LoadBalancer.EnableMasterPoolerLoadBalancer
 	result.EnableReplicaLoadBalancer = fromCRD.LoadBalancer.EnableReplicaLoadBalancer
+	result.EnableReplicaPoolerLoadBalancer = fromCRD.LoadBalancer.EnableReplicaPoolerLoadBalancer
 	result.CustomServiceAnnotations = fromCRD.LoadBalancer.CustomServiceAnnotations
 	result.MasterDNSNameFormat = fromCRD.LoadBalancer.MasterDNSNameFormat
 	result.ReplicaDNSNameFormat = fromCRD.LoadBalancer.ReplicaDNSNameFormat
@@ -212,11 +211,11 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	// so ensure default values here.
 	result.ConnectionPooler.NumberOfInstances = util.CoalesceInt32(
 		fromCRD.ConnectionPooler.NumberOfInstances,
-		int32ToPointer(2))
+		k8sutil.Int32ToPointer(2))
 
 	result.ConnectionPooler.NumberOfInstances = util.MaxInt32(
 		result.ConnectionPooler.NumberOfInstances,
-		int32ToPointer(2))
+		k8sutil.Int32ToPointer(2))
 
 	result.ConnectionPooler.Schema = util.Coalesce(
 		fromCRD.ConnectionPooler.Schema,
@@ -257,7 +256,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 
 	result.ConnectionPooler.MaxDBConnections = util.CoalesceInt32(
 		fromCRD.ConnectionPooler.MaxDBConnections,
-		int32ToPointer(constants.ConnectionPoolerMaxDBConnections))
+		k8sutil.Int32ToPointer(constants.ConnectionPoolerMaxDBConnections))
 
 	return result
 }
diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go
index f32762e46..fc4b73074 100644
--- a/pkg/util/config/config.go
+++ b/pkg/util/config/config.go
@@ -190,7 +190,9 @@ type Config struct {
 	EnablePostgresTeamCRD                  bool              `name:"enable_postgres_team_crd" default:"false"`
 	EnablePostgresTeamCRDSuperusers        bool              `name:"enable_postgres_team_crd_superusers" default:"false"`
 	EnableMasterLoadBalancer               bool              `name:"enable_master_load_balancer" default:"true"`
+	EnableMasterPoolerLoadBalancer         bool              `name:"enable_master_pooler_load_balancer" default:"false"`
 	EnableReplicaLoadBalancer              bool              `name:"enable_replica_load_balancer" default:"false"`
+	EnableReplicaPoolerLoadBalancer        bool              `name:"enable_replica_pooler_load_balancer" default:"false"`
 	CustomServiceAnnotations               map[string]string `name:"custom_service_annotations"`
 	CustomPodAnnotations                   map[string]string `name:"custom_pod_annotations"`
 	EnablePodAntiAffinity                  bool              `name:"enable_pod_antiaffinity" default:"false"`

From 69254abeba0b9991bb765c2673f87a09a5f8dc5d Mon Sep 17 00:00:00 2001
From: Jociele Padilha <45459238+jopadi@users.noreply.github.com>
Date: Tue, 15 Mar 2022 11:34:09 +0100
Subject: [PATCH 31/58] add new parameter for Patroni API
 (PatroniAPICheckInterval, PatroniAPICheckTimeout) (#1803)

Co-authored-by: Jociele Padilha <jociele.padilha@zalando.de>
---
 .../crds/operatorconfigurations.yaml               |  6 ++++++
 charts/postgres-operator/values.yaml               |  4 ++++
 docs/reference/operator_parameters.md              |  7 +++++++
 manifests/configmap.yaml                           |  2 ++
 manifests/operatorconfiguration.crd.yaml           |  6 ++++++
 .../postgresql-operator-default-configuration.yaml |  2 ++
 pkg/apis/acid.zalan.do/v1/crds.go                  |  6 ++++++
 .../v1/operator_configuration_type.go              | 14 ++++++++------
 pkg/cluster/pod.go                                 | 10 +++++-----
 pkg/cluster/pod_test.go                            | 10 +++++++++-
 pkg/controller/operator_config.go                  |  2 ++
 pkg/util/config/config.go                          |  2 ++
 12 files changed, 59 insertions(+), 12 deletions(-)

diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml
index 469aa5588..f10c638a4 100644
--- a/charts/postgres-operator/crds/operatorconfigurations.yaml
+++ b/charts/postgres-operator/crds/operatorconfigurations.yaml
@@ -349,6 +349,12 @@ spec:
               timeouts:
                 type: object
                 properties:
+                  patroni_api_check_interval:
+                    type: string
+                    default: "1s"
+                  patroni_api_check_timeout:
+                    type: string
+                    default: "5s"
                   pod_label_wait_timeout:
                     type: string
                     default: "10m"
diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml
index 9f144ba5e..bcaa6fca9 100644
--- a/charts/postgres-operator/values.yaml
+++ b/charts/postgres-operator/values.yaml
@@ -204,6 +204,10 @@ configPostgresPodResources:
 
 # timeouts related to some operator actions
 configTimeouts:
+  # interval between consecutive attempts of operator calling the Patroni API
+  patroni_api_check_interval: 1s
+  # timeout when waiting for successful response from Patroni API
+  patroni_api_check_timeout: 5s
   # timeout when waiting for the Postgres pods to be deleted
   pod_deletion_wait_timeout: 10m
   # timeout when waiting for pod role and cluster labels
diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md
index 5988271c5..ec4a35cbf 100644
--- a/docs/reference/operator_parameters.md
+++ b/docs/reference/operator_parameters.md
@@ -498,6 +498,13 @@ configuration `resource_check_interval` and `resource_check_timeout` have no
 effect, and the parameters are grouped under the `timeouts` key in the
 CRD-based configuration.
 
+* **PatroniAPICheckInterval**
+  the interval between consecutive attempts waiting for the return of 
+  Patroni Api. The default is `1s`.
+
+* **PatroniAPICheckTimeout**
+  the timeout for a response from Patroni Api. The default is `5s`.
+
 * **resource_check_interval**
   interval to wait between consecutive attempts to check for the presence of
   some Kubernetes resource (i.e. `StatefulSet` or `PodDisruptionBudget`). The
diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml
index d127299c4..9543ad564 100644
--- a/manifests/configmap.yaml
+++ b/manifests/configmap.yaml
@@ -96,6 +96,8 @@ data:
   # pam_configuration: |
   #  https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees
   # pam_role_name: zalandos
+  patroni_api_check_interval: "1s"
+  patroni_api_check_timeout: "5s"
   # password_rotation_interval: "90"
   # password_rotation_user_retention: "180"
   pdb_name_format: "postgres-{cluster}-pdb"
diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml
index d6241abea..7002ec2b3 100644
--- a/manifests/operatorconfiguration.crd.yaml
+++ b/manifests/operatorconfiguration.crd.yaml
@@ -347,6 +347,12 @@ spec:
               timeouts:
                 type: object
                 properties:
+                  patroni_api_check_interval:
+                    type: string
+                    default: "1s"
+                  patroni_api_check_timeout:
+                    type: string
+                    default: "5s"
                   pod_label_wait_timeout:
                     type: string
                     default: "10m"
diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml
index 7ab8565a9..c6f256c58 100644
--- a/manifests/postgresql-operator-default-configuration.yaml
+++ b/manifests/postgresql-operator-default-configuration.yaml
@@ -107,6 +107,8 @@ configuration:
     # min_cpu_limit: 250m
     # min_memory_limit: 250Mi
   timeouts:
+    patroni_api_check_interval: 1s
+    patroni_api_check_timeout: 5s
     pod_label_wait_timeout: 10m
     pod_deletion_wait_timeout: 10m
     ready_wait_interval: 4s
diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go
index 84c57b7cf..f47e2a158 100644
--- a/pkg/apis/acid.zalan.do/v1/crds.go
+++ b/pkg/apis/acid.zalan.do/v1/crds.go
@@ -1438,6 +1438,12 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{
 					"timeouts": {
 						Type: "object",
 						Properties: map[string]apiextv1.JSONSchemaProps{
+							"patroni_api_check_interval": {
+								Type: "string",
+							},
+							"patroni_api_check_timeout": {
+								Type: "string",
+							},
 							"pod_label_wait_timeout": {
 								Type: "string",
 							},
diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
index f5c57aaa4..d15194c08 100644
--- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
+++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
@@ -111,12 +111,14 @@ type PostgresPodResourcesDefaults struct {
 
 // OperatorTimeouts defines the timeout of ResourceCheck, PodWait, ReadyWait
 type OperatorTimeouts struct {
-	ResourceCheckInterval  Duration `json:"resource_check_interval,omitempty"`
-	ResourceCheckTimeout   Duration `json:"resource_check_timeout,omitempty"`
-	PodLabelWaitTimeout    Duration `json:"pod_label_wait_timeout,omitempty"`
-	PodDeletionWaitTimeout Duration `json:"pod_deletion_wait_timeout,omitempty"`
-	ReadyWaitInterval      Duration `json:"ready_wait_interval,omitempty"`
-	ReadyWaitTimeout       Duration `json:"ready_wait_timeout,omitempty"`
+	ResourceCheckInterval   Duration `json:"resource_check_interval,omitempty"`
+	ResourceCheckTimeout    Duration `json:"resource_check_timeout,omitempty"`
+	PodLabelWaitTimeout     Duration `json:"pod_label_wait_timeout,omitempty"`
+	PodDeletionWaitTimeout  Duration `json:"pod_deletion_wait_timeout,omitempty"`
+	ReadyWaitInterval       Duration `json:"ready_wait_interval,omitempty"`
+	ReadyWaitTimeout        Duration `json:"ready_wait_timeout,omitempty"`
+	PatroniAPICheckInterval Duration `json:"patroni_api_check_interval,omitempty"`
+	PatroniAPICheckTimeout  Duration `json:"patroni_api_check_timeout,omitempty"`
 }
 
 // LoadBalancerConfiguration defines the LB configuration
diff --git a/pkg/cluster/pod.go b/pkg/cluster/pod.go
index 9e8ded844..26c4c332d 100644
--- a/pkg/cluster/pod.go
+++ b/pkg/cluster/pod.go
@@ -67,7 +67,7 @@ func (c *Cluster) markRollingUpdateFlagForPod(pod *v1.Pod, msg string) error {
 		return fmt.Errorf("could not form patch for pod's rolling update flag: %v", err)
 	}
 
-	err = retryutil.Retry(1*time.Second, 5*time.Second,
+	err = retryutil.Retry(c.OpConfig.PatroniAPICheckInterval, c.OpConfig.PatroniAPICheckTimeout,
 		func() (bool, error) {
 			_, err2 := c.KubeClient.Pods(pod.Namespace).Patch(
 				context.TODO(),
@@ -356,7 +356,7 @@ func (c *Cluster) getPatroniConfig(pod *v1.Pod) (acidv1.Patroni, map[string]stri
 		pgParameters  map[string]string
 	)
 	podName := util.NameFromMeta(pod.ObjectMeta)
-	err := retryutil.Retry(1*time.Second, 5*time.Second,
+	err := retryutil.Retry(c.OpConfig.PatroniAPICheckInterval, c.OpConfig.PatroniAPICheckTimeout,
 		func() (bool, error) {
 			var err error
 			patroniConfig, pgParameters, err = c.patroni.GetConfig(pod)
@@ -377,7 +377,7 @@ func (c *Cluster) getPatroniConfig(pod *v1.Pod) (acidv1.Patroni, map[string]stri
 
 func (c *Cluster) getPatroniMemberData(pod *v1.Pod) (patroni.MemberData, error) {
 	var memberData patroni.MemberData
-	err := retryutil.Retry(1*time.Second, 5*time.Second,
+	err := retryutil.Retry(c.OpConfig.PatroniAPICheckInterval, c.OpConfig.PatroniAPICheckTimeout,
 		func() (bool, error) {
 			var err error
 			memberData, err = c.patroni.GetMemberData(pod)
@@ -403,7 +403,7 @@ func (c *Cluster) recreatePod(podName spec.NamespacedName) (*v1.Pod, error) {
 	defer c.unregisterPodSubscriber(podName)
 	stopChan := make(chan struct{})
 
-	err := retryutil.Retry(1*time.Second, 5*time.Second,
+	err := retryutil.Retry(c.OpConfig.PatroniAPICheckInterval, c.OpConfig.PatroniAPICheckTimeout,
 		func() (bool, error) {
 			err2 := c.KubeClient.Pods(podName.Namespace).Delete(
 				context.TODO(),
@@ -492,7 +492,7 @@ func (c *Cluster) getSwitchoverCandidate(master *v1.Pod) (spec.NamespacedName, e
 	candidates := make([]patroni.ClusterMember, 0)
 	syncCandidates := make([]patroni.ClusterMember, 0)
 
-	err := retryutil.Retry(1*time.Second, 5*time.Second,
+	err := retryutil.Retry(c.OpConfig.PatroniAPICheckInterval, c.OpConfig.PatroniAPICheckTimeout,
 		func() (bool, error) {
 			var err error
 			members, err = c.patroni.GetClusterMembers(master)
diff --git a/pkg/cluster/pod_test.go b/pkg/cluster/pod_test.go
index a533ebafd..068145312 100644
--- a/pkg/cluster/pod_test.go
+++ b/pkg/cluster/pod_test.go
@@ -6,11 +6,13 @@ import (
 	"io/ioutil"
 	"net/http"
 	"testing"
+	"time"
 
 	"github.com/golang/mock/gomock"
 	"github.com/zalando/postgres-operator/mocks"
 	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
 	"github.com/zalando/postgres-operator/pkg/spec"
+	"github.com/zalando/postgres-operator/pkg/util/config"
 	"github.com/zalando/postgres-operator/pkg/util/k8sutil"
 	"github.com/zalando/postgres-operator/pkg/util/patroni"
 )
@@ -22,7 +24,13 @@ func TestGetSwitchoverCandidate(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
 
-	var cluster = New(Config{}, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder)
+	var cluster = New(
+		Config{
+			OpConfig: config.Config{
+				PatroniAPICheckInterval: time.Duration(1),
+				PatroniAPICheckTimeout:  time.Duration(5),
+			},
+		}, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder)
 
 	// simulate different member scenarios
 	tests := []struct {
diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go
index e470fb59d..167cde163 100644
--- a/pkg/controller/operator_config.go
+++ b/pkg/controller/operator_config.go
@@ -134,6 +134,8 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	result.PodDeletionWaitTimeout = util.CoalesceDuration(time.Duration(fromCRD.Timeouts.PodDeletionWaitTimeout), "10m")
 	result.ReadyWaitInterval = util.CoalesceDuration(time.Duration(fromCRD.Timeouts.ReadyWaitInterval), "4s")
 	result.ReadyWaitTimeout = util.CoalesceDuration(time.Duration(fromCRD.Timeouts.ReadyWaitTimeout), "30s")
+	result.PatroniAPICheckInterval = util.CoalesceDuration(time.Duration(fromCRD.Timeouts.PatroniAPICheckInterval), "1s")
+	result.PatroniAPICheckTimeout = util.CoalesceDuration(time.Duration(fromCRD.Timeouts.PatroniAPICheckTimeout), "5s")
 
 	// load balancer config
 	result.DbHostedZone = util.Coalesce(fromCRD.LoadBalancer.DbHostedZone, "db.example.com")
diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go
index fc4b73074..21cf279f7 100644
--- a/pkg/util/config/config.go
+++ b/pkg/util/config/config.go
@@ -224,6 +224,8 @@ type Config struct {
 	MajorVersionUpgradeTeamAllowList       []string          `name:"major_version_upgrade_team_allow_list" default:""`
 	MinimalMajorVersion                    string            `name:"minimal_major_version" default:"9.6"`
 	TargetMajorVersion                     string            `name:"target_major_version" default:"14"`
+	PatroniAPICheckInterval                time.Duration     `name:"patroni_api_check_interval" default:"1s"`
+	PatroniAPICheckTimeout                 time.Duration     `name:"patroni_api_check_timeout" default:"5s"`
 }
 
 // MustMarshal marshals the config or panics

From 6ba05fee2224b7f2f6a82b301461cd5e1017b024 Mon Sep 17 00:00:00 2001
From: Felix Kunde <felix-kunde@gmx.de>
Date: Thu, 17 Mar 2022 19:22:18 +0100
Subject: [PATCH 32/58] Pooler sync fix (#1811)

* always sync pooler objects
* do not capitalize log messages in Go
* stretch pooler e2e test
---
 e2e/tests/test_e2e.py             | 13 ++++-
 pkg/cluster/connection_pooler.go  | 94 ++++++++++++-------------------
 pkg/cluster/k8sres.go             | 20 +++----
 pkg/controller/operator_config.go |  2 +-
 pkg/util/config/config.go         |  2 +-
 5 files changed, 59 insertions(+), 72 deletions(-)

diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py
index 09b2c9c60..a425354a5 100644
--- a/e2e/tests/test_e2e.py
+++ b/e2e/tests/test_e2e.py
@@ -479,8 +479,6 @@ class EndToEndTestCase(unittest.TestCase):
                 'spec': {
                     'enableConnectionPooler': True,
                     'enableReplicaConnectionPooler': True,
-                    'enableMasterPoolerLoadBalancer': True,
-                    'enableReplicaPoolerLoadBalancer': True,
                 }
             })
         self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
@@ -490,6 +488,17 @@ class EndToEndTestCase(unittest.TestCase):
         self.eventuallyEqual(lambda: k8s.count_running_pods(replica_pooler_label), 2, "No pooler replica pods found")
         self.eventuallyEqual(lambda: k8s.count_services_with_label(pooler_label), 2, "No pooler service found")
         self.eventuallyEqual(lambda: k8s.count_secrets_with_label(pooler_label), 1, "Pooler secret not created")
+
+        k8s.api.custom_objects_api.patch_namespaced_custom_object(
+            'acid.zalan.do', 'v1', 'default',
+            'postgresqls', 'acid-minimal-cluster',
+            {
+                'spec': {
+                    'enableMasterPoolerLoadBalancer': True,
+                    'enableReplicaPoolerLoadBalancer': True,
+                }
+            })
+        self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
         self.eventuallyEqual(lambda: k8s.get_service_type(master_pooler_label+","+pooler_label),
                              'LoadBalancer',
                              "Expected LoadBalancer service type for master pooler pod, found {}")
diff --git a/pkg/cluster/connection_pooler.go b/pkg/cluster/connection_pooler.go
index 12c0be6ae..5639b0283 100644
--- a/pkg/cluster/connection_pooler.go
+++ b/pkg/cluster/connection_pooler.go
@@ -3,7 +3,6 @@ package cluster
 import (
 	"context"
 	"fmt"
-	"reflect"
 	"strings"
 
 	"github.com/r3labs/diff"
@@ -344,7 +343,7 @@ func (c *Cluster) generateConnectionPoolerDeployment(connectionPooler *Connectio
 	}
 
 	if *numberOfInstances < constants.ConnectionPoolerMinInstances {
-		msg := "Adjusted number of connection pooler instances from %d to %d"
+		msg := "adjusted number of connection pooler instances from %d to %d"
 		c.logger.Warningf(msg, *numberOfInstances, constants.ConnectionPoolerMinInstances)
 
 		*numberOfInstances = constants.ConnectionPoolerMinInstances
@@ -614,7 +613,7 @@ func (c *Cluster) needSyncConnectionPoolerDefaults(Config *Config, spec *acidv1.
 		*deployment.Spec.Replicas != *config.NumberOfInstances {
 
 		sync = true
-		msg := fmt.Sprintf("NumberOfInstances is different (having %d, required %d)",
+		msg := fmt.Sprintf("numberOfInstances is different (having %d, required %d)",
 			*deployment.Spec.Replicas, *config.NumberOfInstances)
 		reasons = append(reasons, msg)
 	}
@@ -623,7 +622,7 @@ func (c *Cluster) needSyncConnectionPoolerDefaults(Config *Config, spec *acidv1.
 		poolerContainer.Image != config.Image {
 
 		sync = true
-		msg := fmt.Sprintf("DockerImage is different (having %s, required %s)",
+		msg := fmt.Sprintf("dockerImage is different (having %s, required %s)",
 			poolerContainer.Image, config.Image)
 		reasons = append(reasons, msg)
 	}
@@ -637,7 +636,7 @@ func (c *Cluster) needSyncConnectionPoolerDefaults(Config *Config, spec *acidv1.
 	// updates for new resource values).
 	if err == nil && syncResources(&poolerContainer.Resources, expectedResources) {
 		sync = true
-		msg := fmt.Sprintf("Resources are different (having %+v, required %+v)",
+		msg := fmt.Sprintf("resources are different (having %+v, required %+v)",
 			poolerContainer.Resources, expectedResources)
 		reasons = append(reasons, msg)
 	}
@@ -722,29 +721,6 @@ func (c *Cluster) syncConnectionPooler(oldSpec, newSpec *acidv1.Postgresql, Look
 	var err error
 	var connectionPoolerNeeded bool
 
-	needSync := !reflect.DeepEqual(oldSpec.Spec.ConnectionPooler, newSpec.Spec.ConnectionPooler)
-	masterChanges, err := diff.Diff(oldSpec.Spec.EnableConnectionPooler, newSpec.Spec.EnableConnectionPooler)
-	if err != nil {
-		c.logger.Error("Error in getting diff of master connection pooler changes")
-	}
-	replicaChanges, err := diff.Diff(oldSpec.Spec.EnableReplicaConnectionPooler, newSpec.Spec.EnableReplicaConnectionPooler)
-	if err != nil {
-		c.logger.Error("Error in getting diff of replica connection pooler changes")
-	}
-
-	// skip pooler sync when theres no diff or it's deactivated
-	// but, handling the case when connectionPooler is not there but it is required
-	// as per spec, hence do not skip syncing in that case, even though there
-	// is no diff in specs
-	if (!needSync && len(masterChanges) <= 0 && len(replicaChanges) <= 0) &&
-		((!needConnectionPooler(&newSpec.Spec) && (c.ConnectionPooler == nil || !needConnectionPooler(&oldSpec.Spec))) ||
-			(c.ConnectionPooler != nil && needConnectionPooler(&newSpec.Spec) &&
-				((c.ConnectionPooler[Master] != nil && c.ConnectionPooler[Master].LookupFunction) ||
-					(c.ConnectionPooler[Replica] != nil && c.ConnectionPooler[Replica].LookupFunction)))) {
-		c.logger.Debugln("syncing pooler is not required")
-		return nil, nil
-	}
-
 	logPoolerEssentials(c.logger, oldSpec, newSpec)
 
 	// Check and perform the sync requirements for each of the roles.
@@ -781,7 +757,8 @@ func (c *Cluster) syncConnectionPooler(oldSpec, newSpec *acidv1.Postgresql, Look
 			// in between
 
 			// in this case also do not forget to install lookup function
-			if !c.ConnectionPooler[role].LookupFunction {
+			// skip installation in standby clusters, since they are read-only
+			if !c.ConnectionPooler[role].LookupFunction && c.Spec.StandbyCluster == nil {
 				connectionPooler := c.Spec.ConnectionPooler
 				specSchema := ""
 				specUser := ""
@@ -838,32 +815,37 @@ func (c *Cluster) syncConnectionPooler(oldSpec, newSpec *acidv1.Postgresql, Look
 func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql, role PostgresRole) (
 	SyncReason, error) {
 
+	var (
+		deployment    *appsv1.Deployment
+		newDeployment *appsv1.Deployment
+		service       *v1.Service
+		newService    *v1.Service
+		err           error
+	)
+
 	syncReason := make([]string, 0)
-	deployment, err := c.KubeClient.
+	deployment, err = c.KubeClient.
 		Deployments(c.Namespace).
 		Get(context.TODO(), c.connectionPoolerName(role), metav1.GetOptions{})
 
 	if err != nil && k8sutil.ResourceNotFound(err) {
-		msg := "deployment %s for connection pooler synchronization is not found, create it"
-		c.logger.Warningf(msg, c.connectionPoolerName(role))
+		c.logger.Warningf("deployment %s for connection pooler synchronization is not found, create it", c.connectionPoolerName(role))
 
-		deploymentSpec, err := c.generateConnectionPoolerDeployment(c.ConnectionPooler[role])
+		newDeployment, err = c.generateConnectionPoolerDeployment(c.ConnectionPooler[role])
 		if err != nil {
-			msg = "could not generate deployment for connection pooler: %v"
-			return NoSync, fmt.Errorf(msg, err)
+			return NoSync, fmt.Errorf("could not generate deployment for connection pooler: %v", err)
 		}
 
-		deployment, err := c.KubeClient.
-			Deployments(deploymentSpec.Namespace).
-			Create(context.TODO(), deploymentSpec, metav1.CreateOptions{})
+		deployment, err = c.KubeClient.
+			Deployments(newDeployment.Namespace).
+			Create(context.TODO(), newDeployment, metav1.CreateOptions{})
 
 		if err != nil {
 			return NoSync, err
 		}
 		c.ConnectionPooler[role].Deployment = deployment
 	} else if err != nil {
-		msg := "could not get connection pooler deployment to sync: %v"
-		return NoSync, fmt.Errorf(msg, err)
+		return NoSync, fmt.Errorf("could not get connection pooler deployment to sync: %v", err)
 	} else {
 		c.ConnectionPooler[role].Deployment = deployment
 		// actual synchronization
@@ -900,16 +882,14 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql
 		syncReason = append(syncReason, defaultsReason...)
 
 		if specSync || defaultsSync {
-			c.logger.Infof("Update connection pooler deployment %s, reason: %+v",
+			c.logger.Infof("update connection pooler deployment %s, reason: %+v",
 				c.connectionPoolerName(role), syncReason)
-			newDeploymentSpec, err := c.generateConnectionPoolerDeployment(c.ConnectionPooler[role])
+			newDeployment, err = c.generateConnectionPoolerDeployment(c.ConnectionPooler[role])
 			if err != nil {
-				msg := "could not generate deployment for connection pooler: %v"
-				return syncReason, fmt.Errorf(msg, err)
+				return syncReason, fmt.Errorf("could not generate deployment for connection pooler: %v", err)
 			}
 
-			deployment, err := updateConnectionPoolerDeployment(c.KubeClient,
-				newDeploymentSpec)
+			deployment, err = updateConnectionPoolerDeployment(c.KubeClient, newDeployment)
 
 			if err != nil {
 				return syncReason, err
@@ -927,40 +907,38 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql
 		c.ConnectionPooler[role].Deployment = deployment
 	}
 
-	if svc, err := c.KubeClient.Services(c.Namespace).Get(context.TODO(), c.connectionPoolerName(role), metav1.GetOptions{}); err == nil {
-		c.ConnectionPooler[role].Service = svc
+	if service, err = c.KubeClient.Services(c.Namespace).Get(context.TODO(), c.connectionPoolerName(role), metav1.GetOptions{}); err == nil {
+		c.ConnectionPooler[role].Service = service
 		desiredSvc := c.generateConnectionPoolerService(c.ConnectionPooler[role])
-		if match, reason := k8sutil.SameService(svc, desiredSvc); !match {
+		if match, reason := k8sutil.SameService(service, desiredSvc); !match {
 			syncReason = append(syncReason, reason)
-			c.logServiceChanges(role, svc, desiredSvc, false, reason)
-			updatedService, err := c.updateService(role, svc, desiredSvc)
+			c.logServiceChanges(role, service, desiredSvc, false, reason)
+			newService, err = c.updateService(role, service, desiredSvc)
 			if err != nil {
 				return syncReason, fmt.Errorf("could not update %s service to match desired state: %v", role, err)
 			}
-			c.ConnectionPooler[role].Service = updatedService
+			c.ConnectionPooler[role].Service = newService
 			c.logger.Infof("%s service %q is in the desired state now", role, util.NameFromMeta(desiredSvc.ObjectMeta))
 		}
 		return NoSync, nil
 	}
 
 	if !k8sutil.ResourceNotFound(err) {
-		msg := "could not get connection pooler service to sync: %v"
-		return NoSync, fmt.Errorf(msg, err)
+		return NoSync, fmt.Errorf("could not get connection pooler service to sync: %v", err)
 	}
 
 	c.ConnectionPooler[role].Service = nil
-	msg := "Service %s for connection pooler synchronization is not found, create it"
-	c.logger.Warningf(msg, c.connectionPoolerName(role))
+	c.logger.Warningf("service %s for connection pooler synchronization is not found, create it", c.connectionPoolerName(role))
 
 	serviceSpec := c.generateConnectionPoolerService(c.ConnectionPooler[role])
-	service, err := c.KubeClient.
+	newService, err = c.KubeClient.
 		Services(serviceSpec.Namespace).
 		Create(context.TODO(), serviceSpec, metav1.CreateOptions{})
 
 	if err != nil {
 		return NoSync, err
 	}
-	c.ConnectionPooler[role].Service = service
+	c.ConnectionPooler[role].Service = newService
 
 	return NoSync, nil
 }
diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go
index 6fa983c80..2b2ac9c08 100644
--- a/pkg/cluster/k8sres.go
+++ b/pkg/cluster/k8sres.go
@@ -1100,7 +1100,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
 
 	// backward compatible check for InitContainers
 	if spec.InitContainersOld != nil {
-		msg := "Manifest parameter init_containers is deprecated."
+		msg := "manifest parameter init_containers is deprecated."
 		if spec.InitContainers == nil {
 			c.logger.Warningf("%s Consider using initContainers instead.", msg)
 			spec.InitContainers = spec.InitContainersOld
@@ -1111,7 +1111,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
 
 	// backward compatible check for PodPriorityClassName
 	if spec.PodPriorityClassNameOld != "" {
-		msg := "Manifest parameter pod_priority_class_name is deprecated."
+		msg := "manifest parameter pod_priority_class_name is deprecated."
 		if spec.PodPriorityClassName == "" {
 			c.logger.Warningf("%s Consider using podPriorityClassName instead.", msg)
 			spec.PodPriorityClassName = spec.PodPriorityClassNameOld
@@ -1504,13 +1504,13 @@ func (c *Cluster) addAdditionalVolumes(podSpec *v1.PodSpec,
 	mountPaths := map[string]acidv1.AdditionalVolume{}
 	for i, additionalVolume := range additionalVolumes {
 		if previousVolume, exist := mountPaths[additionalVolume.MountPath]; exist {
-			msg := "Volume %+v cannot be mounted to the same path as %+v"
+			msg := "volume %+v cannot be mounted to the same path as %+v"
 			c.logger.Warningf(msg, additionalVolume, previousVolume)
 			continue
 		}
 
 		if additionalVolume.MountPath == constants.PostgresDataMount {
-			msg := "Cannot mount volume on postgresql data directory, %+v"
+			msg := "cannot mount volume on postgresql data directory, %+v"
 			c.logger.Warningf(msg, additionalVolume)
 			continue
 		}
@@ -1523,7 +1523,7 @@ func (c *Cluster) addAdditionalVolumes(podSpec *v1.PodSpec,
 
 		for _, target := range additionalVolume.TargetContainers {
 			if target == "all" && len(additionalVolume.TargetContainers) != 1 {
-				msg := `Target containers could be either "all" or a list
+				msg := `target containers could be either "all" or a list
 						of containers, mixing those is not allowed, %+v`
 				c.logger.Warningf(msg, additionalVolume)
 				continue
@@ -1813,11 +1813,11 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription)
 			})
 	} else {
 		// cloning with S3, find out the bucket to clone
-		msg := "Clone from S3 bucket"
+		msg := "clone from S3 bucket"
 		c.logger.Info(msg, description.S3WalPath)
 
 		if description.S3WalPath == "" {
-			msg := "Figure out which S3 bucket to use from env"
+			msg := "figure out which S3 bucket to use from env"
 			c.logger.Info(msg, description.S3WalPath)
 
 			if c.OpConfig.WALES3Bucket != "" {
@@ -1861,7 +1861,7 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription)
 
 			result = append(result, envs...)
 		} else {
-			msg := "Use custom parsed S3WalPath %s from the manifest"
+			msg := "use custom parsed S3WalPath %s from the manifest"
 			c.logger.Warningf(msg, description.S3WalPath)
 
 			result = append(result, v1.EnvVar{
@@ -1910,7 +1910,7 @@ func (c *Cluster) generateStandbyEnvironment(description *acidv1.StandbyDescript
 
 	if description.S3WalPath != "" {
 		// standby with S3, find out the bucket to setup standby
-		msg := "Standby from S3 bucket using custom parsed S3WalPath from the manifest %s "
+		msg := "standby from S3 bucket using custom parsed S3WalPath from the manifest %s "
 		c.logger.Infof(msg, description.S3WalPath)
 
 		result = append(result, v1.EnvVar{
@@ -1918,7 +1918,7 @@ func (c *Cluster) generateStandbyEnvironment(description *acidv1.StandbyDescript
 			Value: description.S3WalPath,
 		})
 	} else if description.GSWalPath != "" {
-		msg := "Standby from GS bucket using custom parsed GSWalPath from the manifest %s "
+		msg := "standby from GS bucket using custom parsed GSWalPath from the manifest %s "
 		c.logger.Infof(msg, description.GSWalPath)
 
 		envs := []v1.EnvVar{
diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go
index 167cde163..2cdbb4a68 100644
--- a/pkg/controller/operator_config.go
+++ b/pkg/controller/operator_config.go
@@ -228,7 +228,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 		constants.ConnectionPoolerUserName)
 
 	if result.ConnectionPooler.User == result.SuperUsername {
-		msg := "Connection pool user is not allowed to be the same as super user, username: %s"
+		msg := "connection pool user is not allowed to be the same as super user, username: %s"
 		panic(fmt.Errorf(msg, result.ConnectionPooler.User))
 	}
 
diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go
index 21cf279f7..ec0dd6274 100644
--- a/pkg/util/config/config.go
+++ b/pkg/util/config/config.go
@@ -292,7 +292,7 @@ func validate(cfg *Config) (err error) {
 	}
 
 	if cfg.ConnectionPooler.User == cfg.SuperUsername {
-		msg := "Connection pool user is not allowed to be the same as super user, username: %s"
+		msg := "connection pool user is not allowed to be the same as super user, username: %s"
 		err = fmt.Errorf(msg, cfg.ConnectionPooler.User)
 	}
 

From 2719d411c3221e057a2d9dec36ecbd8a1f21726d Mon Sep 17 00:00:00 2001
From: Felix Kunde <felix-kunde@gmx.de>
Date: Fri, 18 Mar 2022 12:36:12 +0100
Subject: [PATCH 33/58] grant db owners to cron_admin (#1805)

* grant db owners to cron_admin
* allow specifiying more extra owner roles
* add unit test for InitAdditionalOwnerRoles
* add e2e test
---
 .../crds/operatorconfigurations.yaml          |  6 +++
 charts/postgres-operator/values.yaml          | 11 +++++
 docs/reference/operator_parameters.md         | 11 ++++-
 e2e/tests/test_e2e.py                         | 31 ++++++++++++
 manifests/configmap.yaml                      |  3 +-
 manifests/operatorconfiguration.crd.yaml      |  6 +++
 ...gresql-operator-default-configuration.yaml |  3 ++
 pkg/apis/acid.zalan.do/v1/crds.go             | 18 +++++++
 .../v1/operator_configuration_type.go         | 11 +++--
 .../acid.zalan.do/v1/zz_generated.deepcopy.go |  7 ++-
 pkg/cluster/cluster.go                        | 29 +++++++++++
 pkg/cluster/cluster_test.go                   | 48 +++++++++++++++++--
 pkg/cluster/k8sres.go                         |  2 +-
 pkg/controller/operator_config.go             |  3 +-
 pkg/spec/types.go                             |  1 +
 pkg/util/config/config.go                     |  3 +-
 pkg/util/users/users.go                       | 22 +++++----
 17 files changed, 192 insertions(+), 23 deletions(-)

diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml
index f10c638a4..5d76b9540 100644
--- a/charts/postgres-operator/crds/operatorconfigurations.yaml
+++ b/charts/postgres-operator/crds/operatorconfigurations.yaml
@@ -130,6 +130,11 @@ spec:
               users:
                 type: object
                 properties:
+                  additional_owner_roles:
+                    type: array
+                    nullable: true
+                    items:
+                      type: string
                   enable_password_rotation:
                     type: boolean
                     default: false
@@ -514,6 +519,7 @@ spec:
                       type: string
                     default:
                     - admin
+                    - cron_admin
                   role_deletion_suffix:
                     type: string
                     default: "_deleted"
diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml
index bcaa6fca9..4c373ac8d 100644
--- a/charts/postgres-operator/values.yaml
+++ b/charts/postgres-operator/values.yaml
@@ -59,6 +59,16 @@ configGeneral:
 
 # parameters describing Postgres users
 configUsers:
+  # roles to be granted to database owners
+  # additional_owner_roles:
+  # - cron_admin
+
+  # enable password rotation for app users that are not database owners
+  enable_password_rotation: false
+  # rotation interval for updating credentials in K8s secrets of app users
+  password_rotation_interval: 90
+  # retention interval to keep rotation users
+  password_rotation_user_retention: 180
   # postgres username used for replication between instances
   replication_username: standby
   # postgres superuser name to be created by initdb
@@ -348,6 +358,7 @@ configTeamsApi:
   # List of roles that cannot be overwritten by an application, team or infrastructure role
   protected_role_names:
     - admin
+    - cron_admin
   # Suffix to add if members are removed from TeamsAPI or PostgresTeam CRD
   role_deletion_suffix: "_deleted"
   # role name to grant to team members created from the Teams API
diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md
index ec4a35cbf..7692a4369 100644
--- a/docs/reference/operator_parameters.md
+++ b/docs/reference/operator_parameters.md
@@ -177,6 +177,15 @@ under the `users` key.
   Postgres username used for replication between instances. The default is
   `standby`.
 
+* **additional_owner_roles**
+  Specifies database roles that will become members of all database owners.
+  Then owners can use `SET ROLE` to obtain privileges of these roles to e.g.
+  create/update functionality from extensions as part of a migration script.
+  Note, that roles listed here should be preconfigured in the docker image
+  and already exist in the database cluster on startup. One such role can be
+  `cron_admin` which is provided by the Spilo docker image to set up cron
+  jobs inside the `postgres` database. Default is `empty`.
+
 * **enable_password_rotation**
   For all `LOGIN` roles that are not database owners the operator can rotate
   credentials in the corresponding K8s secrets by replacing the username and
@@ -770,7 +779,7 @@ key.
 
 * **protected_role_names**
   List of roles that cannot be overwritten by an application, team or
-  infrastructure role. The default is `admin`.
+  infrastructure role. The default list is `admin` and `cron_admin`.
 
 * **postgres_superuser_teams**
   List of teams which members need the superuser role in each PG database
diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py
index a425354a5..e47b4bf4c 100644
--- a/e2e/tests/test_e2e.py
+++ b/e2e/tests/test_e2e.py
@@ -158,6 +158,37 @@ class EndToEndTestCase(unittest.TestCase):
             print('Operator log: {}'.format(k8s.get_operator_log()))
             raise
 
+    @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
+    def test_additional_owner_roles(self):
+        '''
+           Test adding additional member roles to existing database owner roles
+        '''
+        k8s = self.k8s
+
+        # enable PostgresTeam CRD and lower resync
+        owner_roles = {
+            "data": {
+                "additional_owner_roles": "cron_admin",
+            },
+        }
+        k8s.update_config(owner_roles)
+        self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"},
+                             "Operator does not get in sync")
+
+        leader = k8s.get_cluster_leader_pod()
+        owner_query = """
+            SELECT a2.rolname
+              FROM pg_catalog.pg_authid a
+              JOIN pg_catalog.pg_auth_members am
+                ON a.oid = am.member
+               AND a.rolname = 'cron_admin'
+              JOIN pg_catalog.pg_authid a2
+                ON a2.oid = am.roleid
+             WHERE a2.rolname IN ('zalando', 'bar_owner', 'bar_data_owner');
+        """
+        self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "postgres", owner_query)), 3,
+            "Not all additional users found in database", 10, 5)
+
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
     def test_additional_pod_capabilities(self):
         '''
diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml
index 9543ad564..bc1fe6ffc 100644
--- a/manifests/configmap.yaml
+++ b/manifests/configmap.yaml
@@ -3,6 +3,7 @@ kind: ConfigMap
 metadata:
   name: postgres-operator
 data:
+  # additional_owner_roles: "cron_admin"
   # additional_pod_capabilities: "SYS_NICE"
   # additional_secret_mount: "some-secret-name"
   # additional_secret_mount_path: "/some/dir"
@@ -114,7 +115,7 @@ data:
   # pod_service_account_role_binding_definition: ""
   pod_terminate_grace_period: 5m
   # postgres_superuser_teams: "postgres_superusers"
-  # protected_role_names: "admin"
+  # protected_role_names: "admin,cron_admin"
   ready_wait_interval: 3s
   ready_wait_timeout: 30s
   repair_period: 5m
diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml
index 7002ec2b3..c67d4f19b 100644
--- a/manifests/operatorconfiguration.crd.yaml
+++ b/manifests/operatorconfiguration.crd.yaml
@@ -128,6 +128,11 @@ spec:
               users:
                 type: object
                 properties:
+                  additional_owner_roles:
+                    type: array
+                    nullable: true
+                    items:
+                      type: string
                   enable_password_rotation:
                     type: boolean
                     default: false
@@ -512,6 +517,7 @@ spec:
                       type: string
                     default:
                     - admin
+                    - cron_admin
                   role_deletion_suffix:
                     type: string
                     default: "_deleted"
diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml
index c6f256c58..ea5032e3c 100644
--- a/manifests/postgresql-operator-default-configuration.yaml
+++ b/manifests/postgresql-operator-default-configuration.yaml
@@ -26,6 +26,8 @@ configuration:
   #     protocol: TCP
   workers: 8
   users:
+    # additional_owner_roles: 
+    # - cron_admin
     enable_password_rotation: false
     password_rotation_interval: 90
     password_rotation_user_retention: 180
@@ -168,6 +170,7 @@ configuration:
     # - postgres_superusers
     protected_role_names:
     - admin
+    - cron_admin
     role_deletion_suffix: "_deleted"
     team_admin_role: admin
     team_api_role_configuration:
diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go
index f47e2a158..7f8178bab 100644
--- a/pkg/apis/acid.zalan.do/v1/crds.go
+++ b/pkg/apis/acid.zalan.do/v1/crds.go
@@ -1148,6 +1148,24 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{
 					"users": {
 						Type: "object",
 						Properties: map[string]apiextv1.JSONSchemaProps{
+							"additional_owner_roles": {
+								Type:     "array",
+								Nullable: true,
+								Items: &apiextv1.JSONSchemaPropsOrArray{
+									Schema: &apiextv1.JSONSchemaProps{
+										Type: "string",
+									},
+								},
+							},
+							"enable_password_rotation": {
+								Type: "boolean",
+							},
+							"password_rotation_interval": {
+								Type: "integer",
+							},
+							"password_rotation_user_retention": {
+								Type: "integer",
+							},
 							"replication_username": {
 								Type: "string",
 							},
diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
index d15194c08..3d94ae038 100644
--- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
+++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
@@ -37,11 +37,12 @@ type OperatorConfigurationList struct {
 
 // PostgresUsersConfiguration defines the system users of Postgres.
 type PostgresUsersConfiguration struct {
-	SuperUsername                 string `json:"super_username,omitempty"`
-	ReplicationUsername           string `json:"replication_username,omitempty"`
-	EnablePasswordRotation        bool   `json:"enable_password_rotation,omitempty"`
-	PasswordRotationInterval      uint32 `json:"password_rotation_interval,omitempty"`
-	PasswordRotationUserRetention uint32 `json:"password_rotation_user_retention,omitempty"`
+	SuperUsername                 string   `json:"super_username,omitempty"`
+	ReplicationUsername           string   `json:"replication_username,omitempty"`
+	AdditionalOwnerRoles          []string `json:"additional_owner_roles,omitempty"`
+	EnablePasswordRotation        bool     `json:"enable_password_rotation,omitempty"`
+	PasswordRotationInterval      uint32   `json:"password_rotation_interval,omitempty"`
+	PasswordRotationUserRetention uint32   `json:"password_rotation_user_retention,omitempty"`
 }
 
 // MajorVersionUpgradeConfiguration defines how to execute major version upgrades of Postgres.
diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
index 91f6be563..d3f09f402 100644
--- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
+++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
@@ -401,7 +401,7 @@ func (in *OperatorConfigurationData) DeepCopyInto(out *OperatorConfigurationData
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
-	out.PostgresUsersConfiguration = in.PostgresUsersConfiguration
+	in.PostgresUsersConfiguration.DeepCopyInto(&out.PostgresUsersConfiguration)
 	in.MajorVersionUpgrade.DeepCopyInto(&out.MajorVersionUpgrade)
 	in.Kubernetes.DeepCopyInto(&out.Kubernetes)
 	out.PostgresPodResources = in.PostgresPodResources
@@ -926,6 +926,11 @@ func (in *PostgresTeamSpec) DeepCopy() *PostgresTeamSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *PostgresUsersConfiguration) DeepCopyInto(out *PostgresUsersConfiguration) {
 	*out = *in
+	if in.AdditionalOwnerRoles != nil {
+		in, out := &in.AdditionalOwnerRoles, &out.AdditionalOwnerRoles
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	return
 }
 
diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go
index 2afacde99..80c59f872 100644
--- a/pkg/cluster/cluster.go
+++ b/pkg/cluster/cluster.go
@@ -228,6 +228,8 @@ func (c *Cluster) initUsers() error {
 		return fmt.Errorf("could not init human users: %v", err)
 	}
 
+	c.initAdditionalOwnerRoles()
+
 	return nil
 }
 
@@ -1297,6 +1299,33 @@ func (c *Cluster) initRobotUsers() error {
 	return nil
 }
 
+func (c *Cluster) initAdditionalOwnerRoles() {
+	for _, additionalOwner := range c.OpConfig.AdditionalOwnerRoles {
+		// fetch all database owners the additional should become a member of
+		memberOf := make([]string, 0)
+		for username, pgUser := range c.pgUsers {
+			if pgUser.IsDbOwner {
+				memberOf = append(memberOf, username)
+			}
+		}
+
+		if len(memberOf) > 1 {
+			namespace := c.Namespace
+			additionalOwnerPgUser := spec.PgUser{
+				Origin:    spec.RoleOriginSpilo,
+				MemberOf:  memberOf,
+				Name:      additionalOwner,
+				Namespace: namespace,
+			}
+			if currentRole, present := c.pgUsers[additionalOwner]; present {
+				c.pgUsers[additionalOwner] = c.resolveNameConflict(&currentRole, &additionalOwnerPgUser)
+			} else {
+				c.pgUsers[additionalOwner] = additionalOwnerPgUser
+			}
+		}
+	}
+}
+
 func (c *Cluster) initTeamMembers(teamID string, isPostgresSuperuserTeam bool) error {
 	teamMembers, err := c.getTeamMembers(teamID)
 
diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go
index fba11a6dd..98cc40f05 100644
--- a/pkg/cluster/cluster_test.go
+++ b/pkg/cluster/cluster_test.go
@@ -33,10 +33,11 @@ var cl = New(
 	Config{
 		OpConfig: config.Config{
 			PodManagementPolicy: "ordered_ready",
-			ProtectedRoles:      []string{"admin"},
+			ProtectedRoles:      []string{"admin", "cron_admin", "part_man"},
 			Auth: config.Auth{
-				SuperUsername:       superUserName,
-				ReplicationUsername: replicationUserName,
+				SuperUsername:        superUserName,
+				ReplicationUsername:  replicationUserName,
+				AdditionalOwnerRoles: []string{"cron_admin", "part_man"},
 			},
 			Resources: config.Resources{
 				DownscalerAnnotations: []string{"downscaler/*"},
@@ -44,7 +45,13 @@ var cl = New(
 		},
 	},
 	k8sutil.NewMockKubernetesClient(),
-	acidv1.Postgresql{ObjectMeta: metav1.ObjectMeta{Name: "acid-test", Namespace: "test", Annotations: map[string]string{"downscaler/downtime_replicas": "0"}}},
+	acidv1.Postgresql{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        "acid-test",
+			Namespace:   "test",
+			Annotations: map[string]string{"downscaler/downtime_replicas": "0"},
+		},
+	},
 	logger,
 	eventRecorder,
 )
@@ -132,6 +139,39 @@ func TestInitRobotUsers(t *testing.T) {
 	}
 }
 
+func TestInitAdditionalOwnerRoles(t *testing.T) {
+	testName := "TestInitAdditionalOwnerRoles"
+
+	manifestUsers := map[string]acidv1.UserFlags{"foo_owner": {}, "bar_owner": {}, "app_user": {}}
+	expectedUsers := map[string]spec.PgUser{
+		"foo_owner":  {Origin: spec.RoleOriginManifest, Name: "foo_owner", Namespace: cl.Namespace, Password: "f123", Flags: []string{"LOGIN"}, IsDbOwner: true},
+		"bar_owner":  {Origin: spec.RoleOriginManifest, Name: "bar_owner", Namespace: cl.Namespace, Password: "b123", Flags: []string{"LOGIN"}, IsDbOwner: true},
+		"app_user":   {Origin: spec.RoleOriginManifest, Name: "app_user", Namespace: cl.Namespace, Password: "a123", Flags: []string{"LOGIN"}, IsDbOwner: false},
+		"cron_admin": {Origin: spec.RoleOriginSpilo, Name: "cron_admin", Namespace: cl.Namespace, MemberOf: []string{"foo_owner", "bar_owner"}},
+		"part_man":   {Origin: spec.RoleOriginSpilo, Name: "part_man", Namespace: cl.Namespace, MemberOf: []string{"foo_owner", "bar_owner"}},
+	}
+
+	cl.Spec.Databases = map[string]string{"foo_db": "foo_owner", "bar_db": "bar_owner"}
+	cl.Spec.Users = manifestUsers
+
+	// this should set IsDbOwner field for manifest users
+	if err := cl.initRobotUsers(); err != nil {
+		t.Errorf("%s could not init manifest users", testName)
+	}
+
+	// update passwords to compare with result
+	for manifestUser := range manifestUsers {
+		pgUser := cl.pgUsers[manifestUser]
+		pgUser.Password = manifestUser[0:1] + "123"
+		cl.pgUsers[manifestUser] = pgUser
+	}
+
+	cl.initAdditionalOwnerRoles()
+	if !reflect.DeepEqual(cl.pgUsers, expectedUsers) {
+		t.Errorf("%s expected: %#v, got %#v", testName, expectedUsers, cl.pgUsers)
+	}
+}
+
 type mockOAuthTokenGetter struct {
 }
 
diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go
index 2b2ac9c08..9dcca842e 100644
--- a/pkg/cluster/k8sres.go
+++ b/pkg/cluster/k8sres.go
@@ -1622,7 +1622,7 @@ func (c *Cluster) generateUserSecrets() map[string]*v1.Secret {
 func (c *Cluster) generateSingleUserSecret(namespace string, pgUser spec.PgUser) *v1.Secret {
 	//Skip users with no password i.e. human users (they'll be authenticated using pam)
 	if pgUser.Password == "" {
-		if pgUser.Origin != spec.RoleOriginTeamsAPI {
+		if pgUser.Origin != spec.RoleOriginTeamsAPI && pgUser.Origin != spec.RoleOriginSpilo {
 			c.logger.Warningf("could not generate secret for a non-teamsAPI role %q: role has no password",
 				pgUser.Name)
 		}
diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go
index 2cdbb4a68..be7118d9e 100644
--- a/pkg/controller/operator_config.go
+++ b/pkg/controller/operator_config.go
@@ -52,6 +52,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	// user config
 	result.SuperUsername = util.Coalesce(fromCRD.PostgresUsersConfiguration.SuperUsername, "postgres")
 	result.ReplicationUsername = util.Coalesce(fromCRD.PostgresUsersConfiguration.ReplicationUsername, "standby")
+	result.AdditionalOwnerRoles = fromCRD.PostgresUsersConfiguration.AdditionalOwnerRoles
 	result.EnablePasswordRotation = fromCRD.PostgresUsersConfiguration.EnablePasswordRotation
 	result.PasswordRotationInterval = util.CoalesceUInt32(fromCRD.PostgresUsersConfiguration.PasswordRotationInterval, 90)
 	result.PasswordRotationUserRetention = util.CoalesceUInt32(fromCRD.PostgresUsersConfiguration.DeepCopy().PasswordRotationUserRetention, 180)
@@ -188,7 +189,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	result.TeamAdminRole = fromCRD.TeamsAPI.TeamAdminRole
 	result.PamRoleName = util.Coalesce(fromCRD.TeamsAPI.PamRoleName, "zalandos")
 	result.PamConfiguration = util.Coalesce(fromCRD.TeamsAPI.PamConfiguration, "https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees")
-	result.ProtectedRoles = util.CoalesceStrArr(fromCRD.TeamsAPI.ProtectedRoles, []string{"admin"})
+	result.ProtectedRoles = util.CoalesceStrArr(fromCRD.TeamsAPI.ProtectedRoles, []string{"admin", "cron_admin"})
 	result.PostgresSuperuserTeams = fromCRD.TeamsAPI.PostgresSuperuserTeams
 	result.EnablePostgresTeamCRD = fromCRD.TeamsAPI.EnablePostgresTeamCRD
 	result.EnablePostgresTeamCRDSuperusers = fromCRD.TeamsAPI.EnablePostgresTeamCRDSuperusers
diff --git a/pkg/spec/types.go b/pkg/spec/types.go
index 202e0fa6f..428bcbdc6 100644
--- a/pkg/spec/types.go
+++ b/pkg/spec/types.go
@@ -30,6 +30,7 @@ const (
 	RoleOriginManifest
 	RoleOriginInfrastructure
 	RoleOriginTeamsAPI
+	RoleOriginSpilo
 	RoleOriginSystem
 	RoleOriginBootstrap
 	RoleConnectionPooler
diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go
index ec0dd6274..e6b2c03c0 100644
--- a/pkg/util/config/config.go
+++ b/pkg/util/config/config.go
@@ -101,6 +101,7 @@ type Auth struct {
 	InfrastructureRolesDefs       string                `name:"infrastructure_roles_secrets"`
 	SuperUsername                 string                `name:"super_username" default:"postgres"`
 	ReplicationUsername           string                `name:"replication_username" default:"standby"`
+	AdditionalOwnerRoles          []string              `name:"additional_owner_roles" default:""`
 	EnablePasswordRotation        bool                  `name:"enable_password_rotation" default:"false"`
 	PasswordRotationInterval      uint32                `name:"password_rotation_interval" default:"90"`
 	PasswordRotationUserRetention uint32                `name:"password_rotation_user_retention" default:"180"`
@@ -213,7 +214,7 @@ type Config struct {
 	TeamAPIRoleConfiguration               map[string]string `name:"team_api_role_configuration" default:"log_statement:all"`
 	PodTerminateGracePeriod                time.Duration     `name:"pod_terminate_grace_period" default:"5m"`
 	PodManagementPolicy                    string            `name:"pod_management_policy" default:"ordered_ready"`
-	ProtectedRoles                         []string          `name:"protected_role_names" default:"admin"`
+	ProtectedRoles                         []string          `name:"protected_role_names" default:"admin,cron_admin"`
 	PostgresSuperuserTeams                 []string          `name:"postgres_superuser_teams" default:""`
 	SetMemoryRequestToLimit                bool              `name:"set_memory_request_to_limit" default:"false"`
 	EnableLazySpiloUpgrade                 bool              `name:"enable_lazy_spilo_upgrade" default:"false"`
diff --git a/pkg/util/users/users.go b/pkg/util/users/users.go
index 6bc31f6da..5007268d2 100644
--- a/pkg/util/users/users.go
+++ b/pkg/util/users/users.go
@@ -53,25 +53,31 @@ func (strategy DefaultUserSyncStrategy) ProduceSyncRequests(dbUsers spec.PgUserM
 			}
 		} else {
 			r := spec.PgSyncUserRequest{}
+			r.User = dbUser
 			newMD5Password := util.NewEncryptor(strategy.PasswordEncryption).PGUserPassword(newUser)
 
-			if dbUser.Password != newMD5Password {
-				r.User.Password = newMD5Password
-				r.Kind = spec.PGsyncUserAlter
+			// do not compare for roles coming from docker image
+			if newUser.Origin != spec.RoleOriginSpilo {
+				if dbUser.Password != newMD5Password {
+					r.User.Password = newMD5Password
+					r.Kind = spec.PGsyncUserAlter
+				}
+				if addNewFlags, equal := util.SubstractStringSlices(newUser.Flags, dbUser.Flags); !equal {
+					r.User.Flags = addNewFlags
+					r.Kind = spec.PGsyncUserAlter
+				}
 			}
 			if addNewRoles, equal := util.SubstractStringSlices(newUser.MemberOf, dbUser.MemberOf); !equal {
 				r.User.MemberOf = addNewRoles
 				r.Kind = spec.PGsyncUserAlter
 			}
-			if addNewFlags, equal := util.SubstractStringSlices(newUser.Flags, dbUser.Flags); !equal {
-				r.User.Flags = addNewFlags
-				r.Kind = spec.PGsyncUserAlter
-			}
 			if r.Kind == spec.PGsyncUserAlter {
 				r.User.Name = newUser.Name
 				reqs = append(reqs, r)
 			}
-			if len(newUser.Parameters) > 0 && !reflect.DeepEqual(dbUser.Parameters, newUser.Parameters) {
+			if newUser.Origin != spec.RoleOriginSpilo &&
+				len(newUser.Parameters) > 0 &&
+				!reflect.DeepEqual(dbUser.Parameters, newUser.Parameters) {
 				reqs = append(reqs, spec.PgSyncUserRequest{Kind: spec.PGSyncAlterSet, User: newUser})
 			}
 		}

From 1d88009ec4cc01bad192a1e882e00571179e91ba Mon Sep 17 00:00:00 2001
From: Felix Kunde <felix-kunde@gmx.de>
Date: Fri, 18 Mar 2022 15:06:17 +0100
Subject: [PATCH 34/58] fix comparison of event stream array (#1817)

* fix comparison of event stream array
* turn optional stream fields to pointers
---
 pkg/apis/acid.zalan.do/v1/postgresql_type.go  |  10 +-
 .../acid.zalan.do/v1/zz_generated.deepcopy.go |  29 ++-
 pkg/apis/zalando.org/v1/fabriceventstream.go  |  26 +--
 pkg/cluster/streams.go                        |  34 +++-
 pkg/cluster/streams_test.go                   | 186 +++++++++++++++---
 pkg/util/k8sutil/k8sutil.go                   |   8 +
 6 files changed, 236 insertions(+), 57 deletions(-)

diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go
index 50be9e551..5bb24ed10 100644
--- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go
+++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go
@@ -239,12 +239,12 @@ type Stream struct {
 	ApplicationId string                 `json:"applicationId"`
 	Database      string                 `json:"database"`
 	Tables        map[string]StreamTable `json:"tables"`
-	Filter        map[string]string      `json:"filter,omitempty"`
-	BatchSize     uint32                 `json:"batchSize,omitempty"`
+	Filter        map[string]*string     `json:"filter,omitempty"`
+	BatchSize     *uint32                `json:"batchSize,omitempty"`
 }
 
 type StreamTable struct {
-	EventType     string `json:"eventType"`
-	IdColumn      string `json:"idColumn,omitempty" defaults:"id"`
-	PayloadColumn string `json:"payloadColumn,omitempty" defaults:"payload"`
+	EventType     string  `json:"eventType"`
+	IdColumn      *string `json:"idColumn,omitempty"`
+	PayloadColumn *string `json:"payloadColumn,omitempty"`
 }
diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
index d3f09f402..d7c3b1a86 100644
--- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
+++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
@@ -1181,16 +1181,29 @@ func (in *Stream) DeepCopyInto(out *Stream) {
 		in, out := &in.Tables, &out.Tables
 		*out = make(map[string]StreamTable, len(*in))
 		for key, val := range *in {
-			(*out)[key] = val
+			(*out)[key] = *val.DeepCopy()
 		}
 	}
 	if in.Filter != nil {
 		in, out := &in.Filter, &out.Filter
-		*out = make(map[string]string, len(*in))
+		*out = make(map[string]*string, len(*in))
 		for key, val := range *in {
-			(*out)[key] = val
+			var outVal *string
+			if val == nil {
+				(*out)[key] = nil
+			} else {
+				in, out := &val, &outVal
+				*out = new(string)
+				**out = **in
+			}
+			(*out)[key] = outVal
 		}
 	}
+	if in.BatchSize != nil {
+		in, out := &in.BatchSize, &out.BatchSize
+		*out = new(uint32)
+		**out = **in
+	}
 	return
 }
 
@@ -1207,6 +1220,16 @@ func (in *Stream) DeepCopy() *Stream {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *StreamTable) DeepCopyInto(out *StreamTable) {
 	*out = *in
+	if in.IdColumn != nil {
+		in, out := &in.IdColumn, &out.IdColumn
+		*out = new(string)
+		**out = **in
+	}
+	if in.PayloadColumn != nil {
+		in, out := &in.PayloadColumn, &out.PayloadColumn
+		*out = new(string)
+		**out = **in
+	}
 	return
 }
 
diff --git a/pkg/apis/zalando.org/v1/fabriceventstream.go b/pkg/apis/zalando.org/v1/fabriceventstream.go
index 65d082c1e..559c52d3e 100644
--- a/pkg/apis/zalando.org/v1/fabriceventstream.go
+++ b/pkg/apis/zalando.org/v1/fabriceventstream.go
@@ -40,15 +40,15 @@ type EventStream struct {
 
 // EventStreamFlow defines the flow characteristics of the event stream
 type EventStreamFlow struct {
-	Type          string `json:"type"`
-	PayloadColumn string `json:"payloadColumn,omitempty" defaults:"payload"`
+	Type          string  `json:"type"`
+	PayloadColumn *string `json:"payloadColumn,omitempty"`
 }
 
 // EventStreamSink defines the target of the event stream
 type EventStreamSink struct {
-	Type         string `json:"type"`
-	EventType    string `json:"eventType,omitempty"`
-	MaxBatchSize uint32 `json:"maxBatchSize,omitempty"`
+	Type         string  `json:"type"`
+	EventType    string  `json:"eventType,omitempty"`
+	MaxBatchSize *uint32 `json:"maxBatchSize,omitempty"`
 }
 
 // EventStreamSource defines the source of the event stream and connection for FES operator
@@ -56,23 +56,23 @@ type EventStreamSource struct {
 	Type             string           `json:"type"`
 	Schema           string           `json:"schema,omitempty" defaults:"public"`
 	EventStreamTable EventStreamTable `json:"table"`
-	Filter           string           `json:"filter,omitempty"`
+	Filter           *string          `json:"filter,omitempty"`
 	Connection       Connection       `json:"jdbcConnection"`
 }
 
 // EventStreamTable defines the name and ID column to be used for streaming
 type EventStreamTable struct {
-	Name     string `json:"name"`
-	IDColumn string `json:"idColumn,omitempty" defaults:"id"`
+	Name     string  `json:"name"`
+	IDColumn *string `json:"idColumn,omitempty"`
 }
 
 // Connection to be used for allowing the FES operator to connect to a database
 type Connection struct {
-	Url             string `json:"jdbcUrl"`
-	SlotName        string `json:"slotName"`
-	PluginType      string `json:"pluginType,omitempty" defaults:"wal2json"`
-	PublicationName string `json:"publicationName,omitempty"`
-	DBAuth          DBAuth `json:"databaseAuthentication"`
+	Url             string  `json:"jdbcUrl"`
+	SlotName        string  `json:"slotName"`
+	PluginType      string  `json:"pluginType,omitempty"`
+	PublicationName *string `json:"publicationName,omitempty"`
+	DBAuth          DBAuth  `json:"databaseAuthentication"`
 }
 
 // DBAuth specifies the credentials to be used for connecting with the database
diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go
index a1a1b4d4a..7325fa857 100644
--- a/pkg/cluster/streams.go
+++ b/pkg/cluster/streams.go
@@ -189,7 +189,7 @@ func (c *Cluster) generateFabricEventStream(appId string) *zalandov1.FabricEvent
 	}
 }
 
-func (c *Cluster) getEventStreamSource(stream acidv1.Stream, tableName, idColumn string) zalandov1.EventStreamSource {
+func (c *Cluster) getEventStreamSource(stream acidv1.Stream, tableName string, idColumn *string) zalandov1.EventStreamSource {
 	table, schema := getTableSchema(tableName)
 	streamFilter := stream.Filter[tableName]
 	return zalandov1.EventStreamSource{
@@ -204,7 +204,7 @@ func (c *Cluster) getEventStreamSource(stream acidv1.Stream, tableName, idColumn
 	}
 }
 
-func getEventStreamFlow(stream acidv1.Stream, payloadColumn string) zalandov1.EventStreamFlow {
+func getEventStreamFlow(stream acidv1.Stream, payloadColumn *string) zalandov1.EventStreamFlow {
 	return zalandov1.EventStreamFlow{
 		Type:          constants.EventStreamFlowPgGenericType,
 		PayloadColumn: payloadColumn,
@@ -230,7 +230,7 @@ func getTableSchema(fullTableName string) (tableName, schemaName string) {
 	return tableName, schemaName
 }
 
-func getOutboxTable(tableName, idColumn string) zalandov1.EventStreamTable {
+func getOutboxTable(tableName string, idColumn *string) zalandov1.EventStreamTable {
 	return zalandov1.EventStreamTable{
 		Name:     tableName,
 		IDColumn: idColumn,
@@ -347,8 +347,8 @@ func (c *Cluster) createOrUpdateStreams() error {
 			c.logger.Infof("event stream %q has been successfully created", fesName)
 		} else {
 			desiredStreams := c.generateFabricEventStream(appId)
-			if !reflect.DeepEqual(effectiveStreams.Spec, desiredStreams.Spec) {
-				c.logger.Debug("updating event streams")
+			if match, reason := sameStreams(effectiveStreams.Spec.EventStreams, desiredStreams.Spec.EventStreams); !match {
+				c.logger.Debugf("updating event streams: %s", reason)
 				desiredStreams.ObjectMeta.ResourceVersion = effectiveStreams.ObjectMeta.ResourceVersion
 				err = c.updateStreams(desiredStreams)
 				if err != nil {
@@ -361,3 +361,27 @@ func (c *Cluster) createOrUpdateStreams() error {
 
 	return nil
 }
+
+func sameStreams(curEventStreams, newEventStreams []zalandov1.EventStream) (match bool, reason string) {
+	if len(newEventStreams) != len(curEventStreams) {
+		return false, "number of defined streams is different"
+	}
+
+	for _, newStream := range newEventStreams {
+		match = false
+		reason = "event stream specs differ"
+		for _, curStream := range curEventStreams {
+			if reflect.DeepEqual(newStream.EventStreamSource, curStream.EventStreamSource) &&
+				reflect.DeepEqual(newStream.EventStreamFlow, curStream.EventStreamFlow) &&
+				reflect.DeepEqual(newStream.EventStreamSink, curStream.EventStreamSink) {
+				match = true
+				break
+			}
+		}
+		if !match {
+			return false, reason
+		}
+	}
+
+	return true, ""
+}
diff --git a/pkg/cluster/streams_test.go b/pkg/cluster/streams_test.go
index 89dd294ca..7ebb1c89a 100644
--- a/pkg/cluster/streams_test.go
+++ b/pkg/cluster/streams_test.go
@@ -11,7 +11,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
-	v1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1"
+	zalandov1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1"
 	fakezalandov1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/fake"
 	"github.com/zalando/postgres-operator/pkg/util"
 	"github.com/zalando/postgres-operator/pkg/util/config"
@@ -63,15 +63,18 @@ var (
 					Database:      "foo",
 					Tables: map[string]acidv1.StreamTable{
 						"data.bar": acidv1.StreamTable{
-							EventType:     "stream_type_a",
-							IdColumn:      "b_id",
-							PayloadColumn: "b_payload",
+							EventType:     "stream-type-a",
+							IdColumn:      k8sutil.StringToPointer("b_id"),
+							PayloadColumn: k8sutil.StringToPointer("b_payload"),
+						},
+						"data.foobar": acidv1.StreamTable{
+							EventType: "stream-type-b",
 						},
 					},
-					Filter: map[string]string{
-						"data.bar": "[?(@.source.txId > 500 && @.source.lsn > 123456)]",
+					Filter: map[string]*string{
+						"data.bar": k8sutil.StringToPointer("[?(@.source.txId > 500 && @.source.lsn > 123456)]"),
 					},
-					BatchSize: uint32(100),
+					BatchSize: k8sutil.UInt32ToPointer(uint32(100)),
 				},
 			},
 			Volume: acidv1.Volume{
@@ -80,7 +83,7 @@ var (
 		},
 	}
 
-	fes = &v1.FabricEventStream{
+	fes = &zalandov1.FabricEventStream{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: constants.EventStreamCRDApiVersion,
 			Kind:       constants.EventStreamCRDKind,
@@ -97,23 +100,23 @@ var (
 				},
 			},
 		},
-		Spec: v1.FabricEventStreamSpec{
+		Spec: zalandov1.FabricEventStreamSpec{
 			ApplicationId: appId,
-			EventStreams: []v1.EventStream{
-				{
-					EventStreamFlow: v1.EventStreamFlow{
-						PayloadColumn: "b_payload",
+			EventStreams: []zalandov1.EventStream{
+				zalandov1.EventStream{
+					EventStreamFlow: zalandov1.EventStreamFlow{
+						PayloadColumn: k8sutil.StringToPointer("b_payload"),
 						Type:          constants.EventStreamFlowPgGenericType,
 					},
-					EventStreamSink: v1.EventStreamSink{
-						EventType:    "stream_type_a",
-						MaxBatchSize: uint32(100),
+					EventStreamSink: zalandov1.EventStreamSink{
+						EventType:    "stream-type-a",
+						MaxBatchSize: k8sutil.UInt32ToPointer(uint32(100)),
 						Type:         constants.EventStreamSinkNakadiType,
 					},
-					EventStreamSource: v1.EventStreamSource{
-						Filter: "[?(@.source.txId > 500 && @.source.lsn > 123456)]",
-						Connection: v1.Connection{
-							DBAuth: v1.DBAuth{
+					EventStreamSource: zalandov1.EventStreamSource{
+						Filter: k8sutil.StringToPointer("[?(@.source.txId > 500 && @.source.lsn > 123456)]"),
+						Connection: zalandov1.Connection{
+							DBAuth: zalandov1.DBAuth{
 								Name:        fmt.Sprintf("fes-user.%s.credentials.postgresql.acid.zalan.do", clusterName),
 								PasswordKey: "password",
 								Type:        constants.EventStreamSourceAuthType,
@@ -124,13 +127,41 @@ var (
 							PluginType: constants.EventStreamSourcePluginType,
 						},
 						Schema: "data",
-						EventStreamTable: v1.EventStreamTable{
-							IDColumn: "b_id",
+						EventStreamTable: zalandov1.EventStreamTable{
+							IDColumn: k8sutil.StringToPointer("b_id"),
 							Name:     "bar",
 						},
 						Type: constants.EventStreamSourcePGType,
 					},
 				},
+				zalandov1.EventStream{
+					EventStreamFlow: zalandov1.EventStreamFlow{
+						Type: constants.EventStreamFlowPgGenericType,
+					},
+					EventStreamSink: zalandov1.EventStreamSink{
+						EventType:    "stream-type-b",
+						MaxBatchSize: k8sutil.UInt32ToPointer(uint32(100)),
+						Type:         constants.EventStreamSinkNakadiType,
+					},
+					EventStreamSource: zalandov1.EventStreamSource{
+						Connection: zalandov1.Connection{
+							DBAuth: zalandov1.DBAuth{
+								Name:        fmt.Sprintf("fes-user.%s.credentials.postgresql.acid.zalan.do", clusterName),
+								PasswordKey: "password",
+								Type:        constants.EventStreamSourceAuthType,
+								UserKey:     "username",
+							},
+							Url:        fmt.Sprintf("jdbc:postgresql://%s.%s/foo?user=%s&ssl=true&sslmode=require", clusterName, namespace, fesUser),
+							SlotName:   slotName,
+							PluginType: constants.EventStreamSourcePluginType,
+						},
+						Schema: "data",
+						EventStreamTable: zalandov1.EventStreamTable{
+							Name: "foobar",
+						},
+						Type: constants.EventStreamSourcePGType,
+					},
+				},
 			},
 		},
 	}
@@ -161,23 +192,116 @@ func TestGenerateFabricEventStream(t *testing.T) {
 	cluster.Name = clusterName
 	cluster.Namespace = namespace
 
+	// create statefulset to have ownerReference for streams
 	_, err := cluster.createStatefulSet()
 	assert.NoError(t, err)
 
+	// create the streams
 	err = cluster.createOrUpdateStreams()
 	assert.NoError(t, err)
 
+	// compare generated stream with expected stream
 	result := cluster.generateFabricEventStream(appId)
-
-	if !reflect.DeepEqual(result, fes) {
-		t.Errorf("Malformed FabricEventStream, expected %#v, got %#v", fes, result)
+	if match, _ := sameStreams(result.Spec.EventStreams, fes.Spec.EventStreams); !match {
+		t.Errorf("malformed FabricEventStream, expected %#v, got %#v", fes, result)
 	}
 
+	// compare stream resturned from API with expected stream
 	streamCRD, err := cluster.KubeClient.FabricEventStreams(namespace).Get(context.TODO(), fesName, metav1.GetOptions{})
 	assert.NoError(t, err)
+	if match, _ := sameStreams(streamCRD.Spec.EventStreams, fes.Spec.EventStreams); !match {
+		t.Errorf("malformed FabricEventStream returned from API, expected %#v, got %#v", fes, streamCRD)
+	}
 
-	if !reflect.DeepEqual(streamCRD, fes) {
-		t.Errorf("Malformed FabricEventStream, expected %#v, got %#v", fes, streamCRD)
+	// sync streams once again
+	err = cluster.createOrUpdateStreams()
+	assert.NoError(t, err)
+
+	// compare stream resturned from API with generated stream
+	streamCRD, err = cluster.KubeClient.FabricEventStreams(namespace).Get(context.TODO(), fesName, metav1.GetOptions{})
+	assert.NoError(t, err)
+	if match, _ := sameStreams(streamCRD.Spec.EventStreams, result.Spec.EventStreams); !match {
+		t.Errorf("returned FabricEventStream differs from generated one, expected %#v, got %#v", result, streamCRD)
+	}
+}
+
+func TestSameStreams(t *testing.T) {
+	testName := "TestSameStreams"
+
+	stream1 := zalandov1.EventStream{
+		EventStreamFlow: zalandov1.EventStreamFlow{},
+		EventStreamSink: zalandov1.EventStreamSink{
+			EventType: "stream-type-a",
+		},
+		EventStreamSource: zalandov1.EventStreamSource{
+			EventStreamTable: zalandov1.EventStreamTable{
+				Name: "foo",
+			},
+		},
+	}
+
+	stream2 := zalandov1.EventStream{
+		EventStreamFlow: zalandov1.EventStreamFlow{},
+		EventStreamSink: zalandov1.EventStreamSink{
+			EventType: "stream-type-b",
+		},
+		EventStreamSource: zalandov1.EventStreamSource{
+			EventStreamTable: zalandov1.EventStreamTable{
+				Name: "bar",
+			},
+		},
+	}
+
+	tests := []struct {
+		subTest  string
+		streamsA []zalandov1.EventStream
+		streamsB []zalandov1.EventStream
+		match    bool
+		reason   string
+	}{
+		{
+			subTest:  "identical streams",
+			streamsA: []zalandov1.EventStream{stream1, stream2},
+			streamsB: []zalandov1.EventStream{stream1, stream2},
+			match:    true,
+			reason:   "",
+		},
+		{
+			subTest:  "same streams different order",
+			streamsA: []zalandov1.EventStream{stream1, stream2},
+			streamsB: []zalandov1.EventStream{stream2, stream1},
+			match:    true,
+			reason:   "",
+		},
+		{
+			subTest:  "same streams different order",
+			streamsA: []zalandov1.EventStream{stream1},
+			streamsB: []zalandov1.EventStream{stream1, stream2},
+			match:    false,
+			reason:   "number of defined streams is different",
+		},
+		{
+			subTest:  "different number of streams",
+			streamsA: []zalandov1.EventStream{stream1},
+			streamsB: []zalandov1.EventStream{stream1, stream2},
+			match:    false,
+			reason:   "number of defined streams is different",
+		},
+		{
+			subTest:  "event stream specs differ",
+			streamsA: []zalandov1.EventStream{stream1, stream2},
+			streamsB: fes.Spec.EventStreams,
+			match:    false,
+			reason:   "number of defined streams is different",
+		},
+	}
+
+	for _, tt := range tests {
+		streamsMatch, matchReason := sameStreams(tt.streamsA, tt.streamsB)
+		if streamsMatch != tt.match {
+			t.Errorf("%s %s: unexpected match result when comparing streams: got %s, epxected %s",
+				testName, tt.subTest, matchReason, tt.reason)
+		}
 	}
 }
 
@@ -213,12 +337,12 @@ func TestUpdateFabricEventStream(t *testing.T) {
 			Database:      dbName,
 			Tables: map[string]acidv1.StreamTable{
 				"data.bar": acidv1.StreamTable{
-					EventType:     "stream_type_b",
-					IdColumn:      "b_id",
-					PayloadColumn: "b_payload",
+					EventType:     "stream-type-c",
+					IdColumn:      k8sutil.StringToPointer("b_id"),
+					PayloadColumn: k8sutil.StringToPointer("b_payload"),
 				},
 			},
-			BatchSize: uint32(250),
+			BatchSize: k8sutil.UInt32ToPointer(uint32(250)),
 		},
 	}
 	patch, err := json.Marshal(struct {
diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go
index 0897777ee..fd5de8195 100644
--- a/pkg/util/k8sutil/k8sutil.go
+++ b/pkg/util/k8sutil/k8sutil.go
@@ -37,6 +37,14 @@ func Int32ToPointer(value int32) *int32 {
 	return &value
 }
 
+func UInt32ToPointer(value uint32) *uint32 {
+	return &value
+}
+
+func StringToPointer(str string) *string {
+	return &str
+}
+
 // KubernetesClient describes getters for Kubernetes objects
 type KubernetesClient struct {
 	corev1.SecretsGetter

From f3b83c0b055679a6925404d8ed3565b70dd1c0f6 Mon Sep 17 00:00:00 2001
From: Jakob Gillich <jakob@gillich.me>
Date: Fri, 18 Mar 2022 16:16:32 +0100
Subject: [PATCH 35/58] Fix empty resources spec field failing schema
 validation (#1589)

In Go, when a struct field is not set, it becomes a struct with
default values for all fields. These default values are included
during serialization. This causes issues with schema validation
where optional fields cannot be omitted because default values
are considered invalid.

This patch addresses this issue for `Resources` fields on several
types by using a pointer value.
---
 pkg/apis/acid.zalan.do/v1/postgresql_type.go |  6 ++--
 pkg/apis/acid.zalan.do/v1/util_test.go       | 12 ++++----
 pkg/cluster/cluster_test.go                  |  2 +-
 pkg/cluster/k8sres.go                        | 31 ++++++++++++--------
 pkg/cluster/k8sres_test.go                   | 14 ++++-----
 5 files changed, 35 insertions(+), 30 deletions(-)

diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go
index 5bb24ed10..52aa66726 100644
--- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go
+++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go
@@ -27,7 +27,7 @@ type PostgresSpec struct {
 	PostgresqlParam `json:"postgresql"`
 	Volume          `json:"volume,omitempty"`
 	Patroni         `json:"patroni,omitempty"`
-	Resources       `json:"resources,omitempty"`
+	*Resources      `json:"resources,omitempty"`
 
 	EnableConnectionPooler        *bool             `json:"enableConnectionPooler,omitempty"`
 	EnableReplicaConnectionPooler *bool             `json:"enableReplicaConnectionPooler,omitempty"`
@@ -199,7 +199,7 @@ type CloneDescription struct {
 
 // Sidecar defines a container to be run in the same pod as the Postgres container.
 type Sidecar struct {
-	Resources   `json:"resources,omitempty"`
+	*Resources  `json:"resources,omitempty"`
 	Name        string             `json:"name,omitempty"`
 	DockerImage string             `json:"image,omitempty"`
 	Ports       []v1.ContainerPort `json:"ports,omitempty"`
@@ -232,7 +232,7 @@ type ConnectionPooler struct {
 	DockerImage       string `json:"dockerImage,omitempty"`
 	MaxDBConnections  *int32 `json:"maxDBConnections,omitempty"`
 
-	Resources `json:"resources,omitempty"`
+	*Resources `json:"resources,omitempty"`
 }
 
 type Stream struct {
diff --git a/pkg/apis/acid.zalan.do/v1/util_test.go b/pkg/apis/acid.zalan.do/v1/util_test.go
index bf6875a82..2ff40d347 100644
--- a/pkg/apis/acid.zalan.do/v1/util_test.go
+++ b/pkg/apis/acid.zalan.do/v1/util_test.go
@@ -163,7 +163,7 @@ var unmarshalCluster = []struct {
 				"kind": "Postgresql","apiVersion": "acid.zalan.do/v1",
 				"metadata": {"name": "acid-testcluster1"}, "spec": {"teamId": 100}}`), &tmp).Error(),
 		},
-		marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":null},"status":"Invalid"}`),
+		marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"teamId":"","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":null},"status":"Invalid"}`),
 		err:     nil},
 	{
 		about: "example with /status subresource",
@@ -184,7 +184,7 @@ var unmarshalCluster = []struct {
 				"kind": "Postgresql","apiVersion": "acid.zalan.do/v1",
 				"metadata": {"name": "acid-testcluster1"}, "spec": {"teamId": 100}}`), &tmp).Error(),
 		},
-		marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":null},"status":{"PostgresClusterStatus":"Invalid"}}`),
+		marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"teamId":"","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":null},"status":{"PostgresClusterStatus":"Invalid"}}`),
 		err:     nil},
 	{
 		about: "example with detailed input manifest and deprecated pod_priority_class_name -> podPriorityClassName",
@@ -300,7 +300,7 @@ var unmarshalCluster = []struct {
 					MaximumLagOnFailover: 33554432,
 					Slots:                map[string]map[string]string{"permanent_logical_1": {"type": "logical", "database": "foo", "plugin": "pgoutput"}},
 				},
-				Resources: Resources{
+				Resources: &Resources{
 					ResourceRequests: ResourceDescription{CPU: "10m", Memory: "50Mi"},
 					ResourceLimits:   ResourceDescription{CPU: "300m", Memory: "3000Mi"},
 				},
@@ -351,7 +351,7 @@ var unmarshalCluster = []struct {
 			Status: PostgresStatus{PostgresClusterStatus: ClusterStatusInvalid},
 			Error:  errors.New("name must match {TEAM}-{NAME} format").Error(),
 		},
-		marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"teapot-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null} ,"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"acid","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":null},"status":{"PostgresClusterStatus":"Invalid"}}`),
+		marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"teapot-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"teamId":"acid","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":null},"status":{"PostgresClusterStatus":"Invalid"}}`),
 		err:     nil},
 	{
 		about: "example with clone",
@@ -373,7 +373,7 @@ var unmarshalCluster = []struct {
 			},
 			Error: "",
 		},
-		marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"acid","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":{"cluster":"team-batman"}},"status":{"PostgresClusterStatus":""}}`),
+		marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"teamId":"acid","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":{"cluster":"team-batman"}},"status":{"PostgresClusterStatus":""}}`),
 		err:     nil},
 	{
 		about: "standby example",
@@ -395,7 +395,7 @@ var unmarshalCluster = []struct {
 			},
 			Error: "",
 		},
-		marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"acid","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"standby":{"s3_wal_path":"s3://custom/path/to/bucket/"}},"status":{"PostgresClusterStatus":""}}`),
+		marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"teamId":"acid","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"standby":{"s3_wal_path":"s3://custom/path/to/bucket/"}},"status":{"PostgresClusterStatus":""}}`),
 		err:     nil},
 	{
 		about:   "expect error on malformatted JSON",
diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go
index 98cc40f05..20c1c5c93 100644
--- a/pkg/cluster/cluster_test.go
+++ b/pkg/cluster/cluster_test.go
@@ -60,7 +60,7 @@ func TestStatefulSetAnnotations(t *testing.T) {
 	testName := "CheckStatefulsetAnnotations"
 	spec := acidv1.PostgresSpec{
 		TeamID: "myapp", NumberOfInstances: 1,
-		Resources: acidv1.Resources{
+		Resources: &acidv1.Resources{
 			ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"},
 			ResourceLimits:   acidv1.ResourceDescription{CPU: "1", Memory: "10"},
 		},
diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go
index 9dcca842e..7e96adacc 100644
--- a/pkg/cluster/k8sres.go
+++ b/pkg/cluster/k8sres.go
@@ -136,11 +136,18 @@ func (c *Cluster) makeDefaultResources() acidv1.Resources {
 	}
 }
 
-func generateResourceRequirements(resources acidv1.Resources, defaultResources acidv1.Resources) (*v1.ResourceRequirements, error) {
+func generateResourceRequirements(resources *acidv1.Resources, defaultResources acidv1.Resources) (*v1.ResourceRequirements, error) {
 	var err error
 
-	specRequests := resources.ResourceRequests
-	specLimits := resources.ResourceLimits
+	var specRequests acidv1.ResourceDescription
+	var specLimits acidv1.ResourceDescription
+	if resources == nil {
+		specRequests = acidv1.ResourceDescription{}
+		specLimits = acidv1.ResourceDescription{}
+	} else {
+		specRequests = resources.ResourceRequests
+		specLimits = resources.ResourceLimits
+	}
 
 	result := v1.ResourceRequirements{}
 
@@ -513,16 +520,14 @@ func generateSidecarContainers(sidecars []acidv1.Sidecar,
 	if len(sidecars) > 0 {
 		result := make([]v1.Container, 0)
 		for index, sidecar := range sidecars {
+			var resourcesSpec acidv1.Resources
+			if sidecar.Resources == nil {
+				resourcesSpec = acidv1.Resources{}
+			} else {
+				sidecar.Resources.DeepCopyInto(&resourcesSpec)
+			}
 
-			resources, err := generateResourceRequirements(
-				makeResources(
-					sidecar.Resources.ResourceRequests.CPU,
-					sidecar.Resources.ResourceRequests.Memory,
-					sidecar.Resources.ResourceLimits.CPU,
-					sidecar.Resources.ResourceLimits.Memory,
-				),
-				defaultResources,
-			)
+			resources, err := generateResourceRequirements(&resourcesSpec, defaultResources)
 			if err != nil {
 				return nil, err
 			}
@@ -1387,7 +1392,7 @@ func generateScalyrSidecarSpec(clusterName, APIKey, serverURL, dockerImage strin
 		scalyrCPULimit,
 		scalyrMemoryLimit,
 	)
-	resourceRequirementsScalyrSidecar, err := generateResourceRequirements(resourcesScalyrSidecar, defaultResources)
+	resourceRequirementsScalyrSidecar, err := generateResourceRequirements(&resourcesScalyrSidecar, defaultResources)
 	if err != nil {
 		return nil, fmt.Errorf("invalid resources for Scalyr sidecar: %v", err)
 	}
diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go
index cb2fd83c0..daa8aa4e4 100644
--- a/pkg/cluster/k8sres_test.go
+++ b/pkg/cluster/k8sres_test.go
@@ -915,7 +915,7 @@ func TestNodeAffinity(t *testing.T) {
 	makeSpec := func(nodeAffinity *v1.NodeAffinity) acidv1.PostgresSpec {
 		return acidv1.PostgresSpec{
 			TeamID: "myapp", NumberOfInstances: 1,
-			Resources: acidv1.Resources{
+			Resources: &acidv1.Resources{
 				ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"},
 				ResourceLimits:   acidv1.ResourceDescription{CPU: "1", Memory: "10"},
 			},
@@ -1011,7 +1011,7 @@ func TestTLS(t *testing.T) {
 		},
 		Spec: acidv1.PostgresSpec{
 			TeamID: "myapp", NumberOfInstances: 1,
-			Resources: acidv1.Resources{
+			Resources: &acidv1.Resources{
 				ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"},
 				ResourceLimits:   acidv1.ResourceDescription{CPU: "1", Memory: "10"},
 			},
@@ -1130,7 +1130,7 @@ func TestAdditionalVolume(t *testing.T) {
 		},
 		Spec: acidv1.PostgresSpec{
 			TeamID: "myapp", NumberOfInstances: 1,
-			Resources: acidv1.Resources{
+			Resources: &acidv1.Resources{
 				ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"},
 				ResourceLimits:   acidv1.ResourceDescription{CPU: "1", Memory: "10"},
 			},
@@ -1236,7 +1236,7 @@ func TestSidecars(t *testing.T) {
 			},
 		},
 		TeamID: "myapp", NumberOfInstances: 1,
-		Resources: acidv1.Resources{
+		Resources: &acidv1.Resources{
 			ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"},
 			ResourceLimits:   acidv1.ResourceDescription{CPU: "1", Memory: "10"},
 		},
@@ -1249,7 +1249,7 @@ func TestSidecars(t *testing.T) {
 			},
 			acidv1.Sidecar{
 				Name: "cluster-specific-sidecar-with-resources",
-				Resources: acidv1.Resources{
+				Resources: &acidv1.Resources{
 					ResourceRequests: acidv1.ResourceDescription{CPU: "210m", Memory: "0.8Gi"},
 					ResourceLimits:   acidv1.ResourceDescription{CPU: "510m", Memory: "1.4Gi"},
 				},
@@ -1411,7 +1411,7 @@ func TestGenerateService(t *testing.T) {
 	var enableLB bool = true
 	spec = acidv1.PostgresSpec{
 		TeamID: "myapp", NumberOfInstances: 1,
-		Resources: acidv1.Resources{
+		Resources: &acidv1.Resources{
 			ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"},
 			ResourceLimits:   acidv1.ResourceDescription{CPU: "1", Memory: "10"},
 		},
@@ -1424,7 +1424,7 @@ func TestGenerateService(t *testing.T) {
 			},
 			acidv1.Sidecar{
 				Name: "cluster-specific-sidecar-with-resources",
-				Resources: acidv1.Resources{
+				Resources: &acidv1.Resources{
 					ResourceRequests: acidv1.ResourceDescription{CPU: "210m", Memory: "0.8Gi"},
 					ResourceLimits:   acidv1.ResourceDescription{CPU: "510m", Memory: "1.4Gi"},
 				},

From a020708ef14901733bc440ac8c76039b5d539adc Mon Sep 17 00:00:00 2001
From: Felix Kunde <felix-kunde@gmx.de>
Date: Mon, 21 Mar 2022 10:05:20 +0100
Subject: [PATCH 36/58] fix unit test and improve stability in e2e test (#1819)

* fix unit test and improve stability in e2e test
* fix resource handling
---
 e2e/tests/test_e2e.py       | 16 +++++++---------
 pkg/cluster/cluster.go      |  4 ++++
 pkg/cluster/cluster_test.go | 14 ++++++++++++--
 pkg/cluster/k8sres.go       | 32 +++++++++++++++-----------------
 pkg/cluster/k8sres_test.go  |  6 +++---
 5 files changed, 41 insertions(+), 31 deletions(-)

diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py
index e47b4bf4c..44c72cb44 100644
--- a/e2e/tests/test_e2e.py
+++ b/e2e/tests/test_e2e.py
@@ -208,13 +208,12 @@ class EndToEndTestCase(unittest.TestCase):
 
         try:
             k8s.update_config(patch_capabilities)
-            self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"},
-                                "Operator does not get in sync")
 
             # changed security context of postgres container should trigger a rolling update
             k8s.wait_for_pod_failover(replica_nodes, 'spilo-role=master,' + cluster_label)
             k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label)
 
+            self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
             self.eventuallyEqual(lambda: k8s.count_pods_with_container_capabilities(capabilities, cluster_label),
                                 2, "Container capabilities not updated")
 
@@ -240,8 +239,6 @@ class EndToEndTestCase(unittest.TestCase):
             },
         }
         k8s.update_config(enable_postgres_team_crd)
-        self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"},
-                             "Operator does not get in sync")
 
         k8s.api.custom_objects_api.patch_namespaced_custom_object(
         'acid.zalan.do', 'v1', 'default',
@@ -366,7 +363,7 @@ class EndToEndTestCase(unittest.TestCase):
         try:
             k8s.api.custom_objects_api.patch_namespaced_custom_object(
                 "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_config)
-            
+
             self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
 
             def compare_config():
@@ -483,7 +480,7 @@ class EndToEndTestCase(unittest.TestCase):
                     }
                 }
             })
-        
+
         self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"},
                              "Operator does not get in sync")
         self.eventuallyEqual(lambda: k8s.count_secrets_with_label("cluster-name=acid-minimal-cluster,application=spilo", self.test_namespace),
@@ -1002,12 +999,12 @@ class EndToEndTestCase(unittest.TestCase):
         }
         k8s.api.custom_objects_api.patch_namespaced_custom_object(
             "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_resources)
-        self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
+        self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"},
+                             "Operator does not get in sync")
 
         # wait for switched over
         k8s.wait_for_pod_failover(replica_nodes, 'spilo-role=master,' + cluster_label)
         k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label)
-        self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members()), 2, "Postgres status did not enter running")
 
         def verify_pod_limits():
             pods = k8s.api.core_v1.list_namespaced_pod('default', label_selector="cluster-name=acid-minimal-cluster,application=spilo").items
@@ -1109,7 +1106,8 @@ class EndToEndTestCase(unittest.TestCase):
                 plural="postgresqls",
                 name="acid-minimal-cluster",
                 body=patch_node_affinity_config)
-            self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
+            self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"},
+                                 "Operator does not get in sync")
 
             # node affinity change should cause replica to relocate from replica node to master node due to node affinity requirement
             k8s.wait_for_pod_failover(master_nodes, 'spilo-role=replica,' + cluster_label)
diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go
index 80c59f872..9c1aada79 100644
--- a/pkg/cluster/cluster.go
+++ b/pkg/cluster/cluster.go
@@ -683,6 +683,10 @@ func (c *Cluster) enforceMinResourceLimits(spec *acidv1.PostgresSpec) error {
 		err       error
 	)
 
+	if spec.Resources == nil {
+		return nil
+	}
+
 	// setting limits too low can cause unnecessary evictions / OOM kills
 	minCPULimit := c.OpConfig.MinCPULimit
 	minMemoryLimit := c.OpConfig.MinMemoryLimit
diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go
index 20c1c5c93..401b1bc94 100644
--- a/pkg/cluster/cluster_test.go
+++ b/pkg/cluster/cluster_test.go
@@ -11,6 +11,7 @@ import (
 	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
 	fakeacidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/fake"
 	"github.com/zalando/postgres-operator/pkg/spec"
+	"github.com/zalando/postgres-operator/pkg/util"
 	"github.com/zalando/postgres-operator/pkg/util/config"
 	"github.com/zalando/postgres-operator/pkg/util/constants"
 	"github.com/zalando/postgres-operator/pkg/util/k8sutil"
@@ -167,8 +168,17 @@ func TestInitAdditionalOwnerRoles(t *testing.T) {
 	}
 
 	cl.initAdditionalOwnerRoles()
-	if !reflect.DeepEqual(cl.pgUsers, expectedUsers) {
-		t.Errorf("%s expected: %#v, got %#v", testName, expectedUsers, cl.pgUsers)
+
+	for _, additionalOwnerRole := range cl.Config.OpConfig.AdditionalOwnerRoles {
+		expectedPgUser := expectedUsers[additionalOwnerRole]
+		existingPgUser, exists := cl.pgUsers[additionalOwnerRole]
+		if !exists {
+			t.Errorf("%s additional owner role %q not initilaized", testName, additionalOwnerRole)
+		}
+		if !util.IsEqualIgnoreOrder(expectedPgUser.MemberOf, existingPgUser.MemberOf) {
+			t.Errorf("%s unexpected membership of additional owner role %q: expected member of %#v, got member of %#v",
+				testName, additionalOwnerRole, expectedPgUser.MemberOf, existingPgUser.MemberOf)
+		}
 	}
 }
 
diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go
index 7e96adacc..23f3b9cd6 100644
--- a/pkg/cluster/k8sres.go
+++ b/pkg/cluster/k8sres.go
@@ -139,8 +139,8 @@ func (c *Cluster) makeDefaultResources() acidv1.Resources {
 func generateResourceRequirements(resources *acidv1.Resources, defaultResources acidv1.Resources) (*v1.ResourceRequirements, error) {
 	var err error
 
-	var specRequests acidv1.ResourceDescription
-	var specLimits acidv1.ResourceDescription
+	var specRequests, specLimits acidv1.ResourceDescription
+
 	if resources == nil {
 		specRequests = acidv1.ResourceDescription{}
 		specLimits = acidv1.ResourceDescription{}
@@ -1007,14 +1007,14 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
 
 		// controller adjusts the default memory request at operator startup
 
-		request := spec.Resources.ResourceRequests.Memory
-		if request == "" {
-			request = c.OpConfig.Resources.DefaultMemoryRequest
-		}
+		var request, limit string
 
-		limit := spec.Resources.ResourceLimits.Memory
-		if limit == "" {
+		if spec.Resources == nil {
+			request = c.OpConfig.Resources.DefaultMemoryRequest
 			limit = c.OpConfig.Resources.DefaultMemoryLimit
+		} else {
+			request = spec.Resources.ResourceRequests.Memory
+			limit = spec.Resources.ResourceRequests.Memory
 		}
 
 		isSmaller, err := util.IsSmallerQuantity(request, limit)
@@ -1024,7 +1024,6 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
 		if isSmaller {
 			c.logger.Warningf("The memory request of %v for the Postgres container is increased to match the memory limit of %v.", request, limit)
 			spec.Resources.ResourceRequests.Memory = limit
-
 		}
 
 		// controller adjusts the Scalyr sidecar request at operator startup
@@ -1034,14 +1033,14 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
 		for _, sidecar := range spec.Sidecars {
 
 			// TODO #413
-			sidecarRequest := sidecar.Resources.ResourceRequests.Memory
-			if request == "" {
-				request = c.OpConfig.Resources.DefaultMemoryRequest
-			}
+			var sidecarRequest, sidecarLimit string
 
-			sidecarLimit := sidecar.Resources.ResourceLimits.Memory
-			if limit == "" {
-				limit = c.OpConfig.Resources.DefaultMemoryLimit
+			if sidecar.Resources == nil {
+				sidecarRequest = c.OpConfig.Resources.DefaultMemoryRequest
+				sidecarLimit = c.OpConfig.Resources.DefaultMemoryLimit
+			} else {
+				sidecarRequest = sidecar.Resources.ResourceRequests.Memory
+				sidecarLimit = sidecar.Resources.ResourceRequests.Memory
 			}
 
 			isSmaller, err := util.IsSmallerQuantity(sidecarRequest, sidecarLimit)
@@ -1057,7 +1056,6 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
 	}
 
 	defaultResources := c.makeDefaultResources()
-
 	resourceRequirements, err := generateResourceRequirements(spec.Resources, defaultResources)
 	if err != nil {
 		return nil, fmt.Errorf("could not generate resource requirements: %v", err)
diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go
index daa8aa4e4..85305504f 100644
--- a/pkg/cluster/k8sres_test.go
+++ b/pkg/cluster/k8sres_test.go
@@ -1578,7 +1578,7 @@ func TestEnableLoadBalancers(t *testing.T) {
 					EnableReplicaLoadBalancer:       util.False(),
 					EnableReplicaPoolerLoadBalancer: util.False(),
 					NumberOfInstances:               1,
-					Resources: acidv1.Resources{
+					Resources: &acidv1.Resources{
 						ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"},
 						ResourceLimits:   acidv1.ResourceDescription{CPU: "1", Memory: "10"},
 					},
@@ -1625,7 +1625,7 @@ func TestEnableLoadBalancers(t *testing.T) {
 					EnableReplicaLoadBalancer:       util.True(),
 					EnableReplicaPoolerLoadBalancer: util.True(),
 					NumberOfInstances:               1,
-					Resources: acidv1.Resources{
+					Resources: &acidv1.Resources{
 						ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"},
 						ResourceLimits:   acidv1.ResourceDescription{CPU: "1", Memory: "10"},
 					},
@@ -1720,7 +1720,7 @@ func TestVolumeSelector(t *testing.T) {
 		return acidv1.PostgresSpec{
 			TeamID:            "myapp",
 			NumberOfInstances: 0,
-			Resources: acidv1.Resources{
+			Resources: &acidv1.Resources{
 				ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"},
 				ResourceLimits:   acidv1.ResourceDescription{CPU: "1", Memory: "10"},
 			},

From 38db48c7f085be6834a626d496d5bdece62f8a9d Mon Sep 17 00:00:00 2001
From: preved911 <preved.911@gmail.com>
Date: Mon, 21 Mar 2022 18:05:45 +0300
Subject: [PATCH 37/58] fixing toleration fields requirements (#1797)

Signed-off-by: Ildar Valiullin <preved.911@gmail.com>
---
 charts/postgres-operator/crds/postgresqls.yaml | 4 ----
 manifests/postgresql.crd.yaml                  | 4 ----
 pkg/apis/acid.zalan.do/v1/crds.go              | 3 +--
 3 files changed, 1 insertion(+), 10 deletions(-)

diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml
index 5ec40bdb1..4fbdf3908 100644
--- a/charts/postgres-operator/crds/postgresqls.yaml
+++ b/charts/postgres-operator/crds/postgresqls.yaml
@@ -532,10 +532,6 @@ spec:
                 type: array
                 items:
                   type: object
-                  required:
-                    - key
-                    - operator
-                    - effect
                   properties:
                     key:
                       type: string
diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml
index 183b97fc6..3fddc32d9 100644
--- a/manifests/postgresql.crd.yaml
+++ b/manifests/postgresql.crd.yaml
@@ -530,10 +530,6 @@ spec:
                 type: array
                 items:
                   type: object
-                  required:
-                    - key
-                    - operator
-                    - effect
                   properties:
                     key:
                       type: string
diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go
index 7f8178bab..0b2c5a9fb 100644
--- a/pkg/apis/acid.zalan.do/v1/crds.go
+++ b/pkg/apis/acid.zalan.do/v1/crds.go
@@ -798,8 +798,7 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 						Type: "array",
 						Items: &apiextv1.JSONSchemaPropsOrArray{
 							Schema: &apiextv1.JSONSchemaProps{
-								Type:     "object",
-								Required: []string{"key", "operator", "effect"},
+								Type: "object",
 								Properties: map[string]apiextv1.JSONSchemaProps{
 									"key": {
 										Type: "string",

From 36df1bc87c47c9cf6042e0c1e52903fd574fa930 Mon Sep 17 00:00:00 2001
From: Felix Kunde <felix-kunde@gmx.de>
Date: Thu, 24 Mar 2022 17:35:00 +0100
Subject: [PATCH 38/58] refactor GenerateResourceRequirements and provide unit
 tests (#1822)

* refactor GenerateResourceRequirements and provide unit tests
---
 .../postgres-operator/crds/postgresqls.yaml   |  18 -
 docs/reference/cluster_manifest.md            |   4 +-
 docs/reference/operator_parameters.md         |   6 +-
 manifests/postgresql.crd.yaml                 |  18 -
 pkg/apis/acid.zalan.do/v1/crds.go             |  18 +-
 pkg/apis/acid.zalan.do/v1/postgresql_type.go  |   2 +-
 .../acid.zalan.do/v1/zz_generated.deepcopy.go |  18 +-
 pkg/cluster/cluster.go                        |  57 ----
 pkg/cluster/connection_pooler.go              |  10 +-
 pkg/cluster/k8sres.go                         | 203 +++++------
 pkg/cluster/k8sres_test.go                    | 314 ++++++++++++++++++
 pkg/cluster/sync.go                           |   5 -
 pkg/cluster/util.go                           |  12 +
 13 files changed, 467 insertions(+), 218 deletions(-)

diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml
index 4fbdf3908..143f95591 100644
--- a/charts/postgres-operator/crds/postgresqls.yaml
+++ b/charts/postgres-operator/crds/postgresqls.yaml
@@ -150,15 +150,9 @@ spec:
                     minimum: 1
                   resources:
                     type: object
-                    required:
-                      - requests
-                      - limits
                     properties:
                       limits:
                         type: object
-                        required:
-                          - cpu
-                          - memory
                         properties:
                           cpu:
                             type: string
@@ -168,9 +162,6 @@ spec:
                             pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$'
                       requests:
                         type: object
-                        required:
-                          - cpu
-                          - memory
                         properties:
                           cpu:
                             type: string
@@ -406,15 +397,9 @@ spec:
                 description: deprecated
               resources:
                 type: object
-                required:
-                  - requests
-                  - limits
                 properties:
                   limits:
                     type: object
-                    required:
-                      - cpu
-                      - memory
                     properties:
                       cpu:
                         type: string
@@ -443,9 +428,6 @@ spec:
                         # than the corresponding limit.
                   requests:
                     type: object
-                    required:
-                      - cpu
-                      - memory
                     properties:
                       cpu:
                         type: string
diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md
index bf77bd8b8..c69de8d40 100644
--- a/docs/reference/cluster_manifest.md
+++ b/docs/reference/cluster_manifest.md
@@ -322,9 +322,7 @@ explanation of `ttl` and `loop_wait` parameters.
 
 Those parameters define [CPU and memory requests and limits](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/)
 for the Postgres container. They are grouped under the `resources` top-level
-key with subgroups `requests` and `limits`. The whole section is optional,
-however if you specify a request or limit you have to define everything
-(unless you are not modifying the default CRD schema validation). 
+key with subgroups `requests` and `limits`. 
 
 ### Requests
 
diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md
index 7692a4369..4e0d0d2e0 100644
--- a/docs/reference/operator_parameters.md
+++ b/docs/reference/operator_parameters.md
@@ -159,9 +159,9 @@ Those are top-level keys, containing both leaf keys and groups.
   at the cost of overprovisioning memory and potential scheduling problems for
   containers with high memory limits due to the lack of memory on Kubernetes
   cluster nodes. This affects all containers created by the operator (Postgres,
-  Scalyr sidecar, and other sidecars except **sidecars** defined in the operator
-  configuration); to set resources for the operator's own container, change the
-  [operator deployment manually](https://github.com/zalando/postgres-operator/blob/master/manifests/postgres-operator.yaml#L20).
+  connection pooler, logical backup, scalyr sidecar, and other sidecars except 
+  **sidecars** defined in the operator configuration); to set resources for the
+  operator's own container, change the [operator deployment manually](https://github.com/zalando/postgres-operator/blob/master/manifests/postgres-operator.yaml#L20).
   The default is `false`.
 
 ## Postgres users
diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml
index 3fddc32d9..93f5c1325 100644
--- a/manifests/postgresql.crd.yaml
+++ b/manifests/postgresql.crd.yaml
@@ -148,15 +148,9 @@ spec:
                     minimum: 1
                   resources:
                     type: object
-                    required:
-                      - requests
-                      - limits
                     properties:
                       limits:
                         type: object
-                        required:
-                          - cpu
-                          - memory
                         properties:
                           cpu:
                             type: string
@@ -166,9 +160,6 @@ spec:
                             pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$'
                       requests:
                         type: object
-                        required:
-                          - cpu
-                          - memory
                         properties:
                           cpu:
                             type: string
@@ -404,15 +395,9 @@ spec:
                 description: deprecated
               resources:
                 type: object
-                required:
-                  - requests
-                  - limits
                 properties:
                   limits:
                     type: object
-                    required:
-                      - cpu
-                      - memory
                     properties:
                       cpu:
                         type: string
@@ -441,9 +426,6 @@ spec:
                         # than the corresponding limit.
                   requests:
                     type: object
-                    required:
-                      - cpu
-                      - memory
                     properties:
                       cpu:
                         type: string
diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go
index 0b2c5a9fb..fe436534f 100644
--- a/pkg/apis/acid.zalan.do/v1/crds.go
+++ b/pkg/apis/acid.zalan.do/v1/crds.go
@@ -238,12 +238,10 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 								Minimum: &min1,
 							},
 							"resources": {
-								Type:     "object",
-								Required: []string{"requests", "limits"},
+								Type: "object",
 								Properties: map[string]apiextv1.JSONSchemaProps{
 									"limits": {
-										Type:     "object",
-										Required: []string{"cpu", "memory"},
+										Type: "object",
 										Properties: map[string]apiextv1.JSONSchemaProps{
 											"cpu": {
 												Type:    "string",
@@ -256,8 +254,7 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 										},
 									},
 									"requests": {
-										Type:     "object",
-										Required: []string{"cpu", "memory"},
+										Type: "object",
 										Properties: map[string]apiextv1.JSONSchemaProps{
 											"cpu": {
 												Type:    "string",
@@ -648,12 +645,10 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 						Description: "deprecated",
 					},
 					"resources": {
-						Type:     "object",
-						Required: []string{"requests", "limits"},
+						Type: "object",
 						Properties: map[string]apiextv1.JSONSchemaProps{
 							"limits": {
-								Type:     "object",
-								Required: []string{"cpu", "memory"},
+								Type: "object",
 								Properties: map[string]apiextv1.JSONSchemaProps{
 									"cpu": {
 										Type:    "string",
@@ -666,8 +661,7 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 								},
 							},
 							"requests": {
-								Type:     "object",
-								Required: []string{"cpu", "memory"},
+								Type: "object",
 								Properties: map[string]apiextv1.JSONSchemaProps{
 									"cpu": {
 										Type:    "string",
diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go
index 52aa66726..14b16541c 100644
--- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go
+++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go
@@ -167,7 +167,7 @@ type Patroni struct {
 	Slots                 map[string]map[string]string `json:"slots,omitempty"`
 	SynchronousMode       bool                         `json:"synchronous_mode,omitempty"`
 	SynchronousModeStrict bool                         `json:"synchronous_mode_strict,omitempty"`
-	SynchronousNodeCount  uint32                       `json:"synchronous_node_count,omitempty" defaults:1`
+	SynchronousNodeCount  uint32                       `json:"synchronous_node_count,omitempty" defaults:"1"`
 }
 
 // StandbyDescription contains s3 wal path
diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
index d7c3b1a86..c4f5f0774 100644
--- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
+++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
@@ -106,7 +106,11 @@ func (in *ConnectionPooler) DeepCopyInto(out *ConnectionPooler) {
 		*out = new(int32)
 		**out = **in
 	}
-	out.Resources = in.Resources
+	if in.Resources != nil {
+		in, out := &in.Resources, &out.Resources
+		*out = new(Resources)
+		**out = **in
+	}
 	return
 }
 
@@ -575,7 +579,11 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) {
 	in.PostgresqlParam.DeepCopyInto(&out.PostgresqlParam)
 	in.Volume.DeepCopyInto(&out.Volume)
 	in.Patroni.DeepCopyInto(&out.Patroni)
-	out.Resources = in.Resources
+	if in.Resources != nil {
+		in, out := &in.Resources, &out.Resources
+		*out = new(Resources)
+		**out = **in
+	}
 	if in.EnableConnectionPooler != nil {
 		in, out := &in.EnableConnectionPooler, &out.EnableConnectionPooler
 		*out = new(bool)
@@ -1132,7 +1140,11 @@ func (in *ScalyrConfiguration) DeepCopy() *ScalyrConfiguration {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Sidecar) DeepCopyInto(out *Sidecar) {
 	*out = *in
-	out.Resources = in.Resources
+	if in.Resources != nil {
+		in, out := &in.Resources, &out.Resources
+		*out = new(Resources)
+		**out = **in
+	}
 	if in.Ports != nil {
 		in, out := &in.Ports, &out.Ports
 		*out = make([]corev1.ContainerPort, len(*in))
diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go
index 9c1aada79..eec7d77df 100644
--- a/pkg/cluster/cluster.go
+++ b/pkg/cluster/cluster.go
@@ -256,10 +256,6 @@ func (c *Cluster) Create() error {
 	c.KubeClient.SetPostgresCRDStatus(c.clusterName(), acidv1.ClusterStatusCreating)
 	c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Create", "Started creation of new cluster resources")
 
-	if err = c.enforceMinResourceLimits(&c.Spec); err != nil {
-		return fmt.Errorf("could not enforce minimum resource limits: %v", err)
-	}
-
 	for _, role := range []PostgresRole{Master, Replica} {
 
 		if c.Endpoints[role] != nil {
@@ -676,50 +672,6 @@ func comparePorts(a, b []v1.ContainerPort) bool {
 	return true
 }
 
-func (c *Cluster) enforceMinResourceLimits(spec *acidv1.PostgresSpec) error {
-
-	var (
-		isSmaller bool
-		err       error
-	)
-
-	if spec.Resources == nil {
-		return nil
-	}
-
-	// setting limits too low can cause unnecessary evictions / OOM kills
-	minCPULimit := c.OpConfig.MinCPULimit
-	minMemoryLimit := c.OpConfig.MinMemoryLimit
-
-	cpuLimit := spec.Resources.ResourceLimits.CPU
-	if cpuLimit != "" {
-		isSmaller, err = util.IsSmallerQuantity(cpuLimit, minCPULimit)
-		if err != nil {
-			return fmt.Errorf("could not compare defined CPU limit %s with configured minimum value %s: %v", cpuLimit, minCPULimit, err)
-		}
-		if isSmaller {
-			c.logger.Warningf("defined CPU limit %s is below required minimum %s and will be increased", cpuLimit, minCPULimit)
-			c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "ResourceLimits", "defined CPU limit %s is below required minimum %s and will be set to it", cpuLimit, minCPULimit)
-			spec.Resources.ResourceLimits.CPU = minCPULimit
-		}
-	}
-
-	memoryLimit := spec.Resources.ResourceLimits.Memory
-	if memoryLimit != "" {
-		isSmaller, err = util.IsSmallerQuantity(memoryLimit, minMemoryLimit)
-		if err != nil {
-			return fmt.Errorf("could not compare defined memory limit %s with configured minimum value %s: %v", memoryLimit, minMemoryLimit, err)
-		}
-		if isSmaller {
-			c.logger.Warningf("defined memory limit %s is below required minimum %s and will be increased", memoryLimit, minMemoryLimit)
-			c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "ResourceLimits", "defined memory limit %s is below required minimum %s and will be set to it", memoryLimit, minMemoryLimit)
-			spec.Resources.ResourceLimits.Memory = minMemoryLimit
-		}
-	}
-
-	return nil
-}
-
 // Update changes Kubernetes objects according to the new specification. Unlike the sync case, the missing object
 // (i.e. service) is treated as an error
 // logical backup cron jobs are an exception: a user-initiated Update can enable a logical backup job
@@ -799,12 +751,6 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
 
 	// Statefulset
 	func() {
-		if err := c.enforceMinResourceLimits(&c.Spec); err != nil {
-			c.logger.Errorf("could not sync resources: %v", err)
-			updateFailed = true
-			return
-		}
-
 		oldSs, err := c.generateStatefulSet(&oldSpec.Spec)
 		if err != nil {
 			c.logger.Errorf("could not generate old statefulset spec: %v", err)
@@ -812,9 +758,6 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
 			return
 		}
 
-		// update newSpec to for latter comparison with oldSpec
-		c.enforceMinResourceLimits(&newSpec.Spec)
-
 		newSs, err := c.generateStatefulSet(&newSpec.Spec)
 		if err != nil {
 			c.logger.Errorf("could not generate new statefulset spec: %v", err)
diff --git a/pkg/cluster/connection_pooler.go b/pkg/cluster/connection_pooler.go
index 5639b0283..238e9d430 100644
--- a/pkg/cluster/connection_pooler.go
+++ b/pkg/cluster/connection_pooler.go
@@ -210,9 +210,10 @@ func (c *Cluster) generateConnectionPoolerPodTemplate(role PostgresRole) (
 		connectionPoolerSpec = &acidv1.ConnectionPooler{}
 	}
 	gracePeriod := int64(c.OpConfig.PodTerminateGracePeriod.Seconds())
-	resources, err := generateResourceRequirements(
+	resources, err := c.generateResourceRequirements(
 		connectionPoolerSpec.Resources,
-		makeDefaultConnectionPoolerResources(&c.OpConfig))
+		makeDefaultConnectionPoolerResources(&c.OpConfig),
+		connectionPoolerContainer)
 
 	effectiveDockerImage := util.Coalesce(
 		connectionPoolerSpec.DockerImage,
@@ -627,8 +628,9 @@ func (c *Cluster) needSyncConnectionPoolerDefaults(Config *Config, spec *acidv1.
 		reasons = append(reasons, msg)
 	}
 
-	expectedResources, err := generateResourceRequirements(spec.Resources,
-		makeDefaultConnectionPoolerResources(&Config.OpConfig))
+	expectedResources, err := c.generateResourceRequirements(spec.Resources,
+		makeDefaultConnectionPoolerResources(&Config.OpConfig),
+		connectionPoolerContainer)
 
 	// An error to generate expected resources means something is not quite
 	// right, but for the purpose of robustness do not panic here, just report
diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go
index 23f3b9cd6..e545be7ef 100644
--- a/pkg/cluster/k8sres.go
+++ b/pkg/cluster/k8sres.go
@@ -37,6 +37,8 @@ const (
 	patroniPGBinariesParameterName = "bin_dir"
 	patroniPGHBAConfParameterName  = "pg_hba"
 	localHost                      = "127.0.0.1/32"
+	scalyrSidecarName              = "scalyr-sidecar"
+	logicalBackupContainerName     = "logical-backup"
 	connectionPoolerContainer      = "connection-pooler"
 	pgPort                         = 5432
 )
@@ -117,9 +119,7 @@ func (c *Cluster) podDisruptionBudgetName() string {
 	return c.OpConfig.PDBNameFormat.Format("cluster", c.Name)
 }
 
-func (c *Cluster) makeDefaultResources() acidv1.Resources {
-
-	config := c.OpConfig
+func makeDefaultResources(config *config.Config) acidv1.Resources {
 
 	defaultRequests := acidv1.ResourceDescription{
 		CPU:    config.Resources.DefaultCPURequest,
@@ -136,32 +136,61 @@ func (c *Cluster) makeDefaultResources() acidv1.Resources {
 	}
 }
 
-func generateResourceRequirements(resources *acidv1.Resources, defaultResources acidv1.Resources) (*v1.ResourceRequirements, error) {
-	var err error
+func (c *Cluster) enforceMinResourceLimits(resources *v1.ResourceRequirements) error {
+	var (
+		isSmaller bool
+		err       error
+		msg       string
+	)
 
-	var specRequests, specLimits acidv1.ResourceDescription
-
-	if resources == nil {
-		specRequests = acidv1.ResourceDescription{}
-		specLimits = acidv1.ResourceDescription{}
-	} else {
-		specRequests = resources.ResourceRequests
-		specLimits = resources.ResourceLimits
+	// setting limits too low can cause unnecessary evictions / OOM kills
+	cpuLimit := resources.Limits[v1.ResourceCPU]
+	minCPULimit := c.OpConfig.MinCPULimit
+	if minCPULimit != "" {
+		isSmaller, err = util.IsSmallerQuantity(cpuLimit.String(), minCPULimit)
+		if err != nil {
+			return fmt.Errorf("could not compare defined CPU limit %s for %q container with configured minimum value %s: %v",
+				cpuLimit.String(), constants.PostgresContainerName, minCPULimit, err)
+		}
+		if isSmaller {
+			msg = fmt.Sprintf("defined CPU limit %s for %q container is below required minimum %s and will be increased",
+				cpuLimit.String(), constants.PostgresContainerName, minCPULimit)
+			c.logger.Warningf(msg)
+			c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "ResourceLimits", msg)
+			resources.Limits[v1.ResourceCPU], _ = resource.ParseQuantity(minCPULimit)
+		}
 	}
 
-	result := v1.ResourceRequirements{}
-
-	result.Requests, err = fillResourceList(specRequests, defaultResources.ResourceRequests)
-	if err != nil {
-		return nil, fmt.Errorf("could not fill resource requests: %v", err)
+	memoryLimit := resources.Limits[v1.ResourceMemory]
+	minMemoryLimit := c.OpConfig.MinMemoryLimit
+	if minMemoryLimit != "" {
+		isSmaller, err = util.IsSmallerQuantity(memoryLimit.String(), minMemoryLimit)
+		if err != nil {
+			return fmt.Errorf("could not compare defined memory limit %s for %q container with configured minimum value %s: %v",
+				memoryLimit.String(), constants.PostgresContainerName, minMemoryLimit, err)
+		}
+		if isSmaller {
+			msg = fmt.Sprintf("defined memory limit %s for %q container is below required minimum %s and will be increased",
+				memoryLimit.String(), constants.PostgresContainerName, minMemoryLimit)
+			c.logger.Warningf(msg)
+			c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "ResourceLimits", msg)
+			resources.Limits[v1.ResourceMemory], _ = resource.ParseQuantity(minMemoryLimit)
+		}
 	}
 
-	result.Limits, err = fillResourceList(specLimits, defaultResources.ResourceLimits)
-	if err != nil {
-		return nil, fmt.Errorf("could not fill resource limits: %v", err)
-	}
+	return nil
+}
 
-	return &result, nil
+func setMemoryRequestToLimit(resources *v1.ResourceRequirements, containerName string, logger *logrus.Entry) {
+
+	requests := resources.Requests[v1.ResourceMemory]
+	limits := resources.Limits[v1.ResourceMemory]
+	isSmaller := requests.Cmp(limits) == -1
+	if isSmaller {
+		logger.Warningf("memory request of %s for %q container is increased to match memory limit of %s",
+			requests.String(), containerName, limits.String())
+		resources.Requests[v1.ResourceMemory] = limits
+	}
 }
 
 func fillResourceList(spec acidv1.ResourceDescription, defaults acidv1.ResourceDescription) (v1.ResourceList, error) {
@@ -194,6 +223,44 @@ func fillResourceList(spec acidv1.ResourceDescription, defaults acidv1.ResourceD
 	return requests, nil
 }
 
+func (c *Cluster) generateResourceRequirements(
+	resources *acidv1.Resources,
+	defaultResources acidv1.Resources,
+	containerName string) (*v1.ResourceRequirements, error) {
+	var err error
+	specRequests := acidv1.ResourceDescription{}
+	specLimits := acidv1.ResourceDescription{}
+	result := v1.ResourceRequirements{}
+
+	if resources != nil {
+		specRequests = resources.ResourceRequests
+		specLimits = resources.ResourceLimits
+	}
+
+	result.Requests, err = fillResourceList(specRequests, defaultResources.ResourceRequests)
+	if err != nil {
+		return nil, fmt.Errorf("could not fill resource requests: %v", err)
+	}
+
+	result.Limits, err = fillResourceList(specLimits, defaultResources.ResourceLimits)
+	if err != nil {
+		return nil, fmt.Errorf("could not fill resource limits: %v", err)
+	}
+
+	// enforce minimum cpu and memory limits for Postgres containers only
+	if containerName == constants.PostgresContainerName {
+		if err = c.enforceMinResourceLimits(&result); err != nil {
+			return nil, fmt.Errorf("could not enforce minimum resource limits: %v", err)
+		}
+	}
+
+	if c.OpConfig.SetMemoryRequestToLimit {
+		setMemoryRequestToLimit(&result, containerName, c.logger)
+	}
+
+	return &result, nil
+}
+
 func generateSpiloJSONConfiguration(pg *acidv1.PostgresqlParam, patroni *acidv1.Patroni, pamRoleName string, EnablePgVersionEnvVar bool, logger *logrus.Entry) (string, error) {
 	config := spiloConfiguration{}
 
@@ -514,8 +581,8 @@ func generateContainer(
 	}
 }
 
-func generateSidecarContainers(sidecars []acidv1.Sidecar,
-	defaultResources acidv1.Resources, startIndex int, logger *logrus.Entry) ([]v1.Container, error) {
+func (c *Cluster) generateSidecarContainers(sidecars []acidv1.Sidecar,
+	defaultResources acidv1.Resources, startIndex int) ([]v1.Container, error) {
 
 	if len(sidecars) > 0 {
 		result := make([]v1.Container, 0)
@@ -527,7 +594,7 @@ func generateSidecarContainers(sidecars []acidv1.Sidecar,
 				sidecar.Resources.DeepCopyInto(&resourcesSpec)
 			}
 
-			resources, err := generateResourceRequirements(&resourcesSpec, defaultResources)
+			resources, err := c.generateResourceRequirements(&resourcesSpec, defaultResources, sidecar.Name)
 			if err != nil {
 				return nil, err
 			}
@@ -1002,61 +1069,9 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
 		additionalVolumes   = spec.AdditionalVolumes
 	)
 
-	// Improve me. Please.
-	if c.OpConfig.SetMemoryRequestToLimit {
-
-		// controller adjusts the default memory request at operator startup
-
-		var request, limit string
-
-		if spec.Resources == nil {
-			request = c.OpConfig.Resources.DefaultMemoryRequest
-			limit = c.OpConfig.Resources.DefaultMemoryLimit
-		} else {
-			request = spec.Resources.ResourceRequests.Memory
-			limit = spec.Resources.ResourceRequests.Memory
-		}
-
-		isSmaller, err := util.IsSmallerQuantity(request, limit)
-		if err != nil {
-			return nil, err
-		}
-		if isSmaller {
-			c.logger.Warningf("The memory request of %v for the Postgres container is increased to match the memory limit of %v.", request, limit)
-			spec.Resources.ResourceRequests.Memory = limit
-		}
-
-		// controller adjusts the Scalyr sidecar request at operator startup
-		// as this sidecar is managed separately
-
-		// adjust sidecar containers defined for that particular cluster
-		for _, sidecar := range spec.Sidecars {
-
-			// TODO #413
-			var sidecarRequest, sidecarLimit string
-
-			if sidecar.Resources == nil {
-				sidecarRequest = c.OpConfig.Resources.DefaultMemoryRequest
-				sidecarLimit = c.OpConfig.Resources.DefaultMemoryLimit
-			} else {
-				sidecarRequest = sidecar.Resources.ResourceRequests.Memory
-				sidecarLimit = sidecar.Resources.ResourceRequests.Memory
-			}
-
-			isSmaller, err := util.IsSmallerQuantity(sidecarRequest, sidecarLimit)
-			if err != nil {
-				return nil, err
-			}
-			if isSmaller {
-				c.logger.Warningf("The memory request of %v for the %v sidecar container is increased to match the memory limit of %v.", sidecar.Resources.ResourceRequests.Memory, sidecar.Name, sidecar.Resources.ResourceLimits.Memory)
-				sidecar.Resources.ResourceRequests.Memory = sidecar.Resources.ResourceLimits.Memory
-			}
-		}
-
-	}
-
-	defaultResources := c.makeDefaultResources()
-	resourceRequirements, err := generateResourceRequirements(spec.Resources, defaultResources)
+	defaultResources := makeDefaultResources(&c.OpConfig)
+	resourceRequirements, err := c.generateResourceRequirements(
+		spec.Resources, defaultResources, constants.PostgresContainerName)
 	if err != nil {
 		return nil, fmt.Errorf("could not generate resource requirements: %v", err)
 	}
@@ -1232,7 +1247,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
 			c.logger.Warningf("sidecars specified but disabled in configuration - next statefulset creation would fail")
 		}
 
-		if clusterSpecificSidecars, err = generateSidecarContainers(spec.Sidecars, defaultResources, 0, c.logger); err != nil {
+		if clusterSpecificSidecars, err = c.generateSidecarContainers(spec.Sidecars, defaultResources, 0); err != nil {
 			return nil, fmt.Errorf("could not generate sidecar containers: %v", err)
 		}
 	}
@@ -1243,7 +1258,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
 	for name, dockerImage := range c.OpConfig.SidecarImages {
 		globalSidecarsByDockerImage = append(globalSidecarsByDockerImage, acidv1.Sidecar{Name: name, DockerImage: dockerImage})
 	}
-	if globalSidecarContainersByDockerImage, err = generateSidecarContainers(globalSidecarsByDockerImage, defaultResources, len(clusterSpecificSidecars), c.logger); err != nil {
+	if globalSidecarContainersByDockerImage, err = c.generateSidecarContainers(globalSidecarsByDockerImage, defaultResources, len(clusterSpecificSidecars)); err != nil {
 		return nil, fmt.Errorf("could not generate sidecar containers: %v", err)
 	}
 	// make the resulting list reproducible
@@ -1256,7 +1271,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
 	// generate scalyr sidecar container
 	var scalyrSidecars []v1.Container
 	if scalyrSidecar, err :=
-		generateScalyrSidecarSpec(c.Name,
+		c.generateScalyrSidecarSpec(c.Name,
 			c.OpConfig.ScalyrAPIKey,
 			c.OpConfig.ScalyrServerURL,
 			c.OpConfig.ScalyrImage,
@@ -1264,8 +1279,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
 			c.OpConfig.ScalyrMemoryRequest,
 			c.OpConfig.ScalyrCPULimit,
 			c.OpConfig.ScalyrMemoryLimit,
-			defaultResources,
-			c.logger); err != nil {
+			defaultResources); err != nil {
 		return nil, fmt.Errorf("could not generate Scalyr sidecar: %v", err)
 	} else {
 		if scalyrSidecar != nil {
@@ -1375,12 +1389,12 @@ func (c *Cluster) generatePodAnnotations(spec *acidv1.PostgresSpec) map[string]s
 	return annotations
 }
 
-func generateScalyrSidecarSpec(clusterName, APIKey, serverURL, dockerImage string,
+func (c *Cluster) generateScalyrSidecarSpec(clusterName, APIKey, serverURL, dockerImage string,
 	scalyrCPURequest string, scalyrMemoryRequest string, scalyrCPULimit string, scalyrMemoryLimit string,
-	defaultResources acidv1.Resources, logger *logrus.Entry) (*v1.Container, error) {
+	defaultResources acidv1.Resources) (*v1.Container, error) {
 	if APIKey == "" || dockerImage == "" {
 		if APIKey == "" && dockerImage != "" {
-			logger.Warning("Not running Scalyr sidecar: SCALYR_API_KEY must be defined")
+			c.logger.Warning("Not running Scalyr sidecar: SCALYR_API_KEY must be defined")
 		}
 		return nil, nil
 	}
@@ -1390,7 +1404,8 @@ func generateScalyrSidecarSpec(clusterName, APIKey, serverURL, dockerImage strin
 		scalyrCPULimit,
 		scalyrMemoryLimit,
 	)
-	resourceRequirementsScalyrSidecar, err := generateResourceRequirements(&resourcesScalyrSidecar, defaultResources)
+	resourceRequirementsScalyrSidecar, err := c.generateResourceRequirements(
+		&resourcesScalyrSidecar, defaultResources, scalyrSidecarName)
 	if err != nil {
 		return nil, fmt.Errorf("invalid resources for Scalyr sidecar: %v", err)
 	}
@@ -1408,7 +1423,7 @@ func generateScalyrSidecarSpec(clusterName, APIKey, serverURL, dockerImage strin
 		env = append(env, v1.EnvVar{Name: "SCALYR_SERVER_URL", Value: serverURL})
 	}
 	return &v1.Container{
-		Name:            "scalyr-sidecar",
+		Name:            scalyrSidecarName,
 		Image:           dockerImage,
 		Env:             env,
 		ImagePullPolicy: v1.PullIfNotPresent,
@@ -1991,15 +2006,15 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) {
 	c.logger.Debug("Generating logical backup pod template")
 
 	// allocate for the backup pod the same amount of resources as for normal DB pods
-	defaultResources := c.makeDefaultResources()
-	resourceRequirements, err = generateResourceRequirements(c.Spec.Resources, defaultResources)
+	resourceRequirements, err = c.generateResourceRequirements(
+		c.Spec.Resources, makeDefaultResources(&c.OpConfig), logicalBackupContainerName)
 	if err != nil {
 		return nil, fmt.Errorf("could not generate resource requirements for logical backup pods: %v", err)
 	}
 
 	envVars := c.generateLogicalBackupPodEnvVars()
 	logicalBackupContainer := generateContainer(
-		"logical-backup",
+		logicalBackupContainerName,
 		&c.OpConfig.LogicalBackup.LogicalBackupDockerImage,
 		resourceRequirements,
 		envVars,
diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go
index 85305504f..f27f9b13b 100644
--- a/pkg/cluster/k8sres_test.go
+++ b/pkg/cluster/k8sres_test.go
@@ -30,6 +30,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/kubernetes/fake"
 	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/tools/record"
 )
 
 func newFakeK8sTestClient() (k8sutil.KubernetesClient, *fake.Clientset) {
@@ -1667,6 +1668,319 @@ func TestEnableLoadBalancers(t *testing.T) {
 	}
 }
 
+func TestGenerateResourceRequirements(t *testing.T) {
+	testName := "TestGenerateResourceRequirements"
+	client, _ := newFakeK8sTestClient()
+	clusterName := "acid-test-cluster"
+	namespace := "default"
+	clusterNameLabel := "cluster-name"
+	roleLabel := "spilo-role"
+	sidecarName := "postgres-exporter"
+
+	// two test cases will call enforceMinResourceLimits which emits 2 events per call
+	// hence bufferSize of 4 is required
+	newEventRecorder := record.NewFakeRecorder(4)
+
+	configResources := config.Resources{
+		ClusterLabels:        map[string]string{"application": "spilo"},
+		ClusterNameLabel:     clusterNameLabel,
+		DefaultCPURequest:    "100m",
+		DefaultCPULimit:      "1",
+		DefaultMemoryRequest: "100Mi",
+		DefaultMemoryLimit:   "500Mi",
+		MinCPULimit:          "250m",
+		MinMemoryLimit:       "250Mi",
+		PodRoleLabel:         roleLabel,
+	}
+
+	tests := []struct {
+		subTest           string
+		config            config.Config
+		pgSpec            acidv1.Postgresql
+		expectedResources acidv1.Resources
+	}{
+		{
+			subTest: "test generation of default resources when empty in manifest",
+			config: config.Config{
+				Resources:               configResources,
+				PodManagementPolicy:     "ordered_ready",
+				SetMemoryRequestToLimit: false,
+			},
+			pgSpec: acidv1.Postgresql{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      clusterName,
+					Namespace: namespace,
+				},
+				Spec: acidv1.PostgresSpec{
+					TeamID: "acid",
+					Volume: acidv1.Volume{
+						Size: "1G",
+					},
+				},
+			},
+			expectedResources: acidv1.Resources{
+				ResourceRequests: acidv1.ResourceDescription{CPU: "100m", Memory: "100Mi"},
+				ResourceLimits:   acidv1.ResourceDescription{CPU: "1", Memory: "500Mi"},
+			},
+		},
+		{
+			subTest: "test generation of default resources for sidecar",
+			config: config.Config{
+				Resources:               configResources,
+				PodManagementPolicy:     "ordered_ready",
+				SetMemoryRequestToLimit: false,
+			},
+			pgSpec: acidv1.Postgresql{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      clusterName,
+					Namespace: namespace,
+				},
+				Spec: acidv1.PostgresSpec{
+					Sidecars: []acidv1.Sidecar{
+						acidv1.Sidecar{
+							Name: sidecarName,
+						},
+					},
+					TeamID: "acid",
+					Volume: acidv1.Volume{
+						Size: "1G",
+					},
+				},
+			},
+			expectedResources: acidv1.Resources{
+				ResourceRequests: acidv1.ResourceDescription{CPU: "100m", Memory: "100Mi"},
+				ResourceLimits:   acidv1.ResourceDescription{CPU: "1", Memory: "500Mi"},
+			},
+		},
+		{
+			subTest: "test generation of resources when only requests are defined in manifest",
+			config: config.Config{
+				Resources:               configResources,
+				PodManagementPolicy:     "ordered_ready",
+				SetMemoryRequestToLimit: false,
+			},
+			pgSpec: acidv1.Postgresql{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      clusterName,
+					Namespace: namespace,
+				},
+				Spec: acidv1.PostgresSpec{
+					Resources: &acidv1.Resources{
+						ResourceRequests: acidv1.ResourceDescription{CPU: "50m", Memory: "50Mi"},
+					},
+					TeamID: "acid",
+					Volume: acidv1.Volume{
+						Size: "1G",
+					},
+				},
+			},
+			expectedResources: acidv1.Resources{
+				ResourceRequests: acidv1.ResourceDescription{CPU: "50m", Memory: "50Mi"},
+				ResourceLimits:   acidv1.ResourceDescription{CPU: "1", Memory: "500Mi"},
+			},
+		},
+		{
+			subTest: "test generation of resources when only memory is defined in manifest",
+			config: config.Config{
+				Resources:               configResources,
+				PodManagementPolicy:     "ordered_ready",
+				SetMemoryRequestToLimit: false,
+			},
+			pgSpec: acidv1.Postgresql{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      clusterName,
+					Namespace: namespace,
+				},
+				Spec: acidv1.PostgresSpec{
+					Resources: &acidv1.Resources{
+						ResourceRequests: acidv1.ResourceDescription{Memory: "100Mi"},
+						ResourceLimits:   acidv1.ResourceDescription{Memory: "1Gi"},
+					},
+					TeamID: "acid",
+					Volume: acidv1.Volume{
+						Size: "1G",
+					},
+				},
+			},
+			expectedResources: acidv1.Resources{
+				ResourceRequests: acidv1.ResourceDescription{CPU: "100m", Memory: "100Mi"},
+				ResourceLimits:   acidv1.ResourceDescription{CPU: "1", Memory: "1Gi"},
+			},
+		},
+		{
+			subTest: "test SetMemoryRequestToLimit flag",
+			config: config.Config{
+				Resources:               configResources,
+				PodManagementPolicy:     "ordered_ready",
+				SetMemoryRequestToLimit: true,
+			},
+			pgSpec: acidv1.Postgresql{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      clusterName,
+					Namespace: namespace,
+				},
+				Spec: acidv1.PostgresSpec{
+					TeamID: "acid",
+					Volume: acidv1.Volume{
+						Size: "1G",
+					},
+				},
+			},
+			expectedResources: acidv1.Resources{
+				ResourceRequests: acidv1.ResourceDescription{CPU: "100m", Memory: "500Mi"},
+				ResourceLimits:   acidv1.ResourceDescription{CPU: "1", Memory: "500Mi"},
+			},
+		},
+		{
+			subTest: "test SetMemoryRequestToLimit flag for sidecar container, too",
+			config: config.Config{
+				Resources:               configResources,
+				PodManagementPolicy:     "ordered_ready",
+				SetMemoryRequestToLimit: true,
+			},
+			pgSpec: acidv1.Postgresql{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      clusterName,
+					Namespace: namespace,
+				},
+				Spec: acidv1.PostgresSpec{
+					Sidecars: []acidv1.Sidecar{
+						acidv1.Sidecar{
+							Name: sidecarName,
+							Resources: &acidv1.Resources{
+								ResourceRequests: acidv1.ResourceDescription{CPU: "10m", Memory: "10Mi"},
+								ResourceLimits:   acidv1.ResourceDescription{CPU: "100m", Memory: "100Mi"},
+							},
+						},
+					},
+					TeamID: "acid",
+					Volume: acidv1.Volume{
+						Size: "1G",
+					},
+				},
+			},
+			expectedResources: acidv1.Resources{
+				ResourceRequests: acidv1.ResourceDescription{CPU: "10m", Memory: "100Mi"},
+				ResourceLimits:   acidv1.ResourceDescription{CPU: "100m", Memory: "100Mi"},
+			},
+		},
+		{
+			subTest: "test generating resources from manifest",
+			config: config.Config{
+				Resources:               configResources,
+				PodManagementPolicy:     "ordered_ready",
+				SetMemoryRequestToLimit: false,
+			},
+			pgSpec: acidv1.Postgresql{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      clusterName,
+					Namespace: namespace,
+				},
+				Spec: acidv1.PostgresSpec{
+					Resources: &acidv1.Resources{
+						ResourceRequests: acidv1.ResourceDescription{CPU: "10m", Memory: "250Mi"},
+						ResourceLimits:   acidv1.ResourceDescription{CPU: "400m", Memory: "800Mi"},
+					},
+					TeamID: "acid",
+					Volume: acidv1.Volume{
+						Size: "1G",
+					},
+				},
+			},
+			expectedResources: acidv1.Resources{
+				ResourceRequests: acidv1.ResourceDescription{CPU: "10m", Memory: "250Mi"},
+				ResourceLimits:   acidv1.ResourceDescription{CPU: "400m", Memory: "800Mi"},
+			},
+		},
+		{
+			subTest: "test enforcing min cpu and memory limit",
+			config: config.Config{
+				Resources:               configResources,
+				PodManagementPolicy:     "ordered_ready",
+				SetMemoryRequestToLimit: false,
+			},
+			pgSpec: acidv1.Postgresql{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      clusterName,
+					Namespace: namespace,
+				},
+				Spec: acidv1.PostgresSpec{
+					Resources: &acidv1.Resources{
+						ResourceRequests: acidv1.ResourceDescription{CPU: "100m", Memory: "100Mi"},
+						ResourceLimits:   acidv1.ResourceDescription{CPU: "200m", Memory: "200Mi"},
+					},
+					TeamID: "acid",
+					Volume: acidv1.Volume{
+						Size: "1G",
+					},
+				},
+			},
+			expectedResources: acidv1.Resources{
+				ResourceRequests: acidv1.ResourceDescription{CPU: "100m", Memory: "100Mi"},
+				ResourceLimits:   acidv1.ResourceDescription{CPU: "250m", Memory: "250Mi"},
+			},
+		},
+		{
+			subTest: "test min cpu and memory limit are not enforced on sidecar",
+			config: config.Config{
+				Resources:               configResources,
+				PodManagementPolicy:     "ordered_ready",
+				SetMemoryRequestToLimit: false,
+			},
+			pgSpec: acidv1.Postgresql{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      clusterName,
+					Namespace: namespace,
+				},
+				Spec: acidv1.PostgresSpec{
+					Sidecars: []acidv1.Sidecar{
+						acidv1.Sidecar{
+							Name: sidecarName,
+							Resources: &acidv1.Resources{
+								ResourceRequests: acidv1.ResourceDescription{CPU: "10m", Memory: "10Mi"},
+								ResourceLimits:   acidv1.ResourceDescription{CPU: "100m", Memory: "100Mi"},
+							},
+						},
+					},
+					TeamID: "acid",
+					Volume: acidv1.Volume{
+						Size: "1G",
+					},
+				},
+			},
+			expectedResources: acidv1.Resources{
+				ResourceRequests: acidv1.ResourceDescription{CPU: "10m", Memory: "10Mi"},
+				ResourceLimits:   acidv1.ResourceDescription{CPU: "100m", Memory: "100Mi"},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		var cluster = New(
+			Config{
+				OpConfig: tt.config,
+			}, client, tt.pgSpec, logger, newEventRecorder)
+
+		cluster.Name = clusterName
+		cluster.Namespace = namespace
+		_, err := cluster.createStatefulSet()
+		if k8sutil.ResourceAlreadyExists(err) {
+			err = cluster.syncStatefulSet()
+		}
+		assert.NoError(t, err)
+
+		containers := cluster.Statefulset.Spec.Template.Spec.Containers
+		clusterResources, err := parseResourceRequirements(containers[0].Resources)
+		if len(containers) > 1 {
+			clusterResources, err = parseResourceRequirements(containers[1].Resources)
+		}
+		assert.NoError(t, err)
+		if !reflect.DeepEqual(tt.expectedResources, clusterResources) {
+			t.Errorf("%s - %s: expected %#v but got %#v", testName, tt.subTest, tt.expectedResources, clusterResources)
+		}
+	}
+}
+
 func TestGenerateCapabilities(t *testing.T) {
 
 	testName := "TestGenerateCapabilities"
diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go
index 4548a5b14..1c011426c 100644
--- a/pkg/cluster/sync.go
+++ b/pkg/cluster/sync.go
@@ -76,11 +76,6 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
 		}
 	}
 
-	if err = c.enforceMinResourceLimits(&c.Spec); err != nil {
-		err = fmt.Errorf("could not enforce minimum resource limits: %v", err)
-		return err
-	}
-
 	c.logger.Debug("syncing statefulsets")
 	if err = c.syncStatefulSet(); err != nil {
 		if !k8sutil.ResourceAlreadyExists(err) {
diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go
index 43c3282d4..5f9a0b379 100644
--- a/pkg/cluster/util.go
+++ b/pkg/cluster/util.go
@@ -594,3 +594,15 @@ func trimCronjobName(name string) string {
 	}
 	return name
 }
+
+func parseResourceRequirements(resourcesRequirement v1.ResourceRequirements) (acidv1.Resources, error) {
+	var resources acidv1.Resources
+	resourcesJSON, err := json.Marshal(resourcesRequirement)
+	if err != nil {
+		return acidv1.Resources{}, fmt.Errorf("could not marshal K8s resources requirements")
+	}
+	if err = json.Unmarshal(resourcesJSON, &resources); err != nil {
+		return acidv1.Resources{}, fmt.Errorf("could not unmarshal K8s resources requirements into acidv1.Resources struct")
+	}
+	return resources, nil
+}

From 654d22d04a1863d7ade4a61829379b68ba7b276c Mon Sep 17 00:00:00 2001
From: Felix Kunde <felix-kunde@gmx.de>
Date: Thu, 24 Mar 2022 18:38:37 +0100
Subject: [PATCH 39/58] Configure annotations to be ignored in comparisons
 during sync (#1823)

* feat: add ignored annotations when comparing during sync

Co-authored-by: Felix Kunde <felix-kunde@gmx.de>
Co-authored-by: Moshe Immerman <moshe@flanksource.com>
---
 .../crds/operatorconfigurations.yaml          |   4 +
 charts/postgres-operator/values.yaml          |   5 +
 cmd/main.go                                   |   3 +-
 docs/reference/operator_parameters.md         |   6 +
 e2e/tests/test_e2e.py                         |  43 +++
 manifests/configmap.yaml                      |   1 +
 manifests/operatorconfiguration.crd.yaml      |   4 +
 ...gresql-operator-default-configuration.yaml |   2 +
 pkg/apis/acid.zalan.do/v1/crds.go             |   8 +
 .../v1/operator_configuration_type.go         |   1 +
 .../acid.zalan.do/v1/zz_generated.deepcopy.go |   5 +
 pkg/cluster/cluster.go                        |  67 +++-
 pkg/cluster/cluster_test.go                   | 331 ++++++++++++++++++
 pkg/cluster/connection_pooler.go              |   2 +-
 pkg/cluster/sync.go                           |   2 +-
 pkg/controller/operator_config.go             |   1 +
 pkg/spec/types.go                             |   1 +
 pkg/util/config/config.go                     |   1 +
 pkg/util/k8sutil/k8sutil.go                   |  51 ---
 pkg/util/k8sutil/k8sutil_test.go              | 311 ----------------
 20 files changed, 479 insertions(+), 370 deletions(-)
 delete mode 100644 pkg/util/k8sutil/k8sutil_test.go

diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml
index 5d76b9540..283acee2e 100644
--- a/charts/postgres-operator/crds/operatorconfigurations.yaml
+++ b/charts/postgres-operator/crds/operatorconfigurations.yaml
@@ -212,6 +212,10 @@ spec:
                   enable_sidecars:
                     type: boolean
                     default: true
+                  ignored_annotations:
+                    type: array
+                    items:
+                      type: string
                   infrastructure_roles_secret_name:
                     type: string
                   infrastructure_roles_secrets:
diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml
index 4c373ac8d..2ce26d18a 100644
--- a/charts/postgres-operator/values.yaml
+++ b/charts/postgres-operator/values.yaml
@@ -124,6 +124,11 @@ configKubernetes:
   enable_pod_disruption_budget: true
   # enables sidecar containers to run alongside Spilo in the same pod
   enable_sidecars: true
+
+  # annotations to be ignored when comparing statefulsets, services etc.
+  # ignored_annotations:
+  # - k8s.v1.cni.cncf.io/network-status
+
   # namespaced name of the secret containing infrastructure roles names and passwords
   # infrastructure_roles_secret_name: postgresql-infrastructure-roles
 
diff --git a/cmd/main.go b/cmd/main.go
index 376df0bad..0b48ac863 100644
--- a/cmd/main.go
+++ b/cmd/main.go
@@ -2,13 +2,14 @@ package main
 
 import (
 	"flag"
-	log "github.com/sirupsen/logrus"
 	"os"
 	"os/signal"
 	"sync"
 	"syscall"
 	"time"
 
+	log "github.com/sirupsen/logrus"
+
 	"github.com/zalando/postgres-operator/pkg/controller"
 	"github.com/zalando/postgres-operator/pkg/spec"
 	"github.com/zalando/postgres-operator/pkg/util/k8sutil"
diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md
index 4e0d0d2e0..d245be58d 100644
--- a/docs/reference/operator_parameters.md
+++ b/docs/reference/operator_parameters.md
@@ -287,6 +287,12 @@ configuration they are grouped under the `kubernetes` key.
   Regular expressions like `downscaler/*` etc. are also accepted. Can be used
   with [kube-downscaler](https://github.com/hjacobs/kube-downscaler).
 
+* **ignored_annotations**
+  Some K8s tools inject and update annotations out of the Postgres Operator
+  control. This can cause rolling updates on each cluster sync cycle. With
+  this option you can specify an array of annotation keys that should be
+  ignored when comparing K8s resources on sync. The default is empty.
+
 * **watched_namespace**
   The operator watches for Postgres objects in the given namespace. If not
   specified, the value is taken from the operator namespace. A special `*`
diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py
index 44c72cb44..b2977b687 100644
--- a/e2e/tests/test_e2e.py
+++ b/e2e/tests/test_e2e.py
@@ -704,6 +704,49 @@ class EndToEndTestCase(unittest.TestCase):
             print('Operator log: {}'.format(k8s.get_operator_log()))
             raise
 
+    @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
+    def test_ignored_annotations(self):
+        '''
+           Test if injected annotation does not cause replacement of resources when listed under ignored_annotations
+        '''
+        k8s = self.k8s
+
+        annotation_patch = {
+            "metadata": {
+                "annotations": {
+                    "k8s-status": "healthy"
+                },
+            }
+        }
+
+        try:
+            sts = k8s.api.apps_v1.read_namespaced_stateful_set('acid-minimal-cluster', 'default')
+            old_sts_creation_timestamp = sts.metadata.creation_timestamp
+            k8s.api.apps_v1.patch_namespaced_stateful_set(sts.metadata.name, sts.metadata.namespace, annotation_patch)
+            svc = k8s.api.core_v1.read_namespaced_service('acid-minimal-cluster', 'default')
+            old_svc_creation_timestamp = svc.metadata.creation_timestamp
+            k8s.api.core_v1.patch_namespaced_service(svc.metadata.name, svc.metadata.namespace, annotation_patch)
+
+            patch_config_ignored_annotations = {
+                "data": {
+                    "ignored_annotations": "k8s-status",
+                }
+            }
+            k8s.update_config(patch_config_ignored_annotations)
+            self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
+
+            sts = k8s.api.apps_v1.read_namespaced_stateful_set('acid-minimal-cluster', 'default')
+            new_sts_creation_timestamp = sts.metadata.creation_timestamp
+            svc = k8s.api.core_v1.read_namespaced_service('acid-minimal-cluster', 'default')
+            new_svc_creation_timestamp = svc.metadata.creation_timestamp
+
+            self.assertEqual(old_sts_creation_timestamp, new_sts_creation_timestamp, "unexpected replacement of statefulset on sync")
+            self.assertEqual(old_svc_creation_timestamp, new_svc_creation_timestamp, "unexpected replacement of master service on sync")
+
+        except timeout_decorator.TimeoutError:
+            print('Operator log: {}'.format(k8s.get_operator_log()))
+            raise
+
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
     def test_infrastructure_roles(self):
         '''
diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml
index bc1fe6ffc..130a35176 100644
--- a/manifests/configmap.yaml
+++ b/manifests/configmap.yaml
@@ -64,6 +64,7 @@ data:
   external_traffic_policy: "Cluster"
   # gcp_credentials: ""
   # kubernetes_use_configmaps: "false"
+  # ignored_annotations: ""
   # infrastructure_roles_secret_name: "postgresql-infrastructure-roles"
   # infrastructure_roles_secrets: "secretname:monitoring-roles,userkey:user,passwordkey:password,rolekey:inrole"
   # inherited_annotations: owned-by
diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml
index c67d4f19b..173e83c51 100644
--- a/manifests/operatorconfiguration.crd.yaml
+++ b/manifests/operatorconfiguration.crd.yaml
@@ -210,6 +210,10 @@ spec:
                   enable_sidecars:
                     type: boolean
                     default: true
+                  ignored_annotations:
+                    type: array
+                    items:
+                      type: string
                   infrastructure_roles_secret_name:
                     type: string
                   infrastructure_roles_secrets:
diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml
index ea5032e3c..a5ceb8d2a 100644
--- a/manifests/postgresql-operator-default-configuration.yaml
+++ b/manifests/postgresql-operator-default-configuration.yaml
@@ -59,6 +59,8 @@ configuration:
     enable_pod_antiaffinity: false
     enable_pod_disruption_budget: true
     enable_sidecars: true
+    # ignored_annotations:
+    # - k8s.v1.cni.cncf.io/network-status
     # infrastructure_roles_secret_name: "postgresql-infrastructure-roles"
     # infrastructure_roles_secrets:
     # - secretname: "monitoring-roles"
diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go
index fe436534f..77828255d 100644
--- a/pkg/apis/acid.zalan.do/v1/crds.go
+++ b/pkg/apis/acid.zalan.do/v1/crds.go
@@ -1251,6 +1251,14 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{
 							"enable_sidecars": {
 								Type: "boolean",
 							},
+							"ignored_annotations": {
+								Type: "array",
+								Items: &apiextv1.JSONSchemaPropsOrArray{
+									Schema: &apiextv1.JSONSchemaProps{
+										Type: "string",
+									},
+								},
+							},
 							"infrastructure_roles_secret_name": {
 								Type: "string",
 							},
diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
index 3d94ae038..0be275553 100644
--- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
+++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
@@ -82,6 +82,7 @@ type KubernetesMetaConfiguration struct {
 	InheritedLabels                        []string                     `json:"inherited_labels,omitempty"`
 	InheritedAnnotations                   []string                     `json:"inherited_annotations,omitempty"`
 	DownscalerAnnotations                  []string                     `json:"downscaler_annotations,omitempty"`
+	IgnoredAnnotations                     []string                     `json:"ignored_annotations,omitempty"`
 	ClusterNameLabel                       string                       `json:"cluster_name_label,omitempty"`
 	DeleteAnnotationDateKey                string                       `json:"delete_annotation_date_key,omitempty"`
 	DeleteAnnotationNameKey                string                       `json:"delete_annotation_name_key,omitempty"`
diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
index c4f5f0774..26d930f2b 100644
--- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
+++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
@@ -228,6 +228,11 @@ func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfigura
 		*out = make([]string, len(*in))
 		copy(*out, *in)
 	}
+	if in.IgnoredAnnotations != nil {
+		in, out := &in.IgnoredAnnotations, &out.IgnoredAnnotations
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	if in.NodeReadinessLabel != nil {
 		in, out := &in.NodeReadinessLabel, &out.NodeReadinessLabel
 		*out = make(map[string]string, len(*in))
diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go
index eec7d77df..e411c66e8 100644
--- a/pkg/cluster/cluster.go
+++ b/pkg/cluster/cluster.go
@@ -378,9 +378,10 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa
 		match = false
 		reasons = append(reasons, "new statefulset's number of replicas does not match the current one")
 	}
-	if !reflect.DeepEqual(c.Statefulset.Annotations, statefulSet.Annotations) {
+	if changed, reason := c.compareAnnotations(c.Statefulset.Annotations, statefulSet.Annotations); changed {
+		match = false
 		needsReplace = true
-		reasons = append(reasons, "new statefulset's annotations do not match the current one")
+		reasons = append(reasons, "new statefulset's annotations do not match: "+reason)
 	}
 
 	needsRollUpdate, reasons = c.compareContainers("initContainers", c.Statefulset.Spec.Template.Spec.InitContainers, statefulSet.Spec.Template.Spec.InitContainers, needsRollUpdate, reasons)
@@ -434,10 +435,11 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa
 		}
 	}
 
-	if !reflect.DeepEqual(c.Statefulset.Spec.Template.Annotations, statefulSet.Spec.Template.Annotations) {
+	if changed, reason := c.compareAnnotations(c.Statefulset.Spec.Template.Annotations, statefulSet.Spec.Template.Annotations); changed {
+		match = false
 		needsReplace = true
 		needsRollUpdate = true
-		reasons = append(reasons, "new statefulset's pod template metadata annotations does not match the current one")
+		reasons = append(reasons, "new statefulset's pod template metadata annotations does not match "+reason)
 	}
 	if !reflect.DeepEqual(c.Statefulset.Spec.Template.Spec.SecurityContext, statefulSet.Spec.Template.Spec.SecurityContext) {
 		needsReplace = true
@@ -672,6 +674,61 @@ func comparePorts(a, b []v1.ContainerPort) bool {
 	return true
 }
 
+func (c *Cluster) compareAnnotations(old, new map[string]string) (bool, string) {
+	reason := ""
+	ignoredAnnotations := make(map[string]bool)
+	for _, ignore := range c.OpConfig.IgnoredAnnotations {
+		ignoredAnnotations[ignore] = true
+	}
+
+	for key := range old {
+		if _, ok := ignoredAnnotations[key]; ok {
+			continue
+		}
+		if _, ok := new[key]; !ok {
+			reason += fmt.Sprintf(" Removed %q.", key)
+		}
+	}
+
+	for key := range new {
+		if _, ok := ignoredAnnotations[key]; ok {
+			continue
+		}
+		v, ok := old[key]
+		if !ok {
+			reason += fmt.Sprintf(" Added %q with value %q.", key, new[key])
+		} else if v != new[key] {
+			reason += fmt.Sprintf(" %q changed from %q to %q.", key, v, new[key])
+		}
+	}
+
+	return reason != "", reason
+
+}
+
+func (c *Cluster) compareServices(old, new *v1.Service) (bool, string) {
+	if old.Spec.Type != new.Spec.Type {
+		return false, fmt.Sprintf("new service's type %q does not match the current one %q",
+			new.Spec.Type, old.Spec.Type)
+	}
+
+	oldSourceRanges := old.Spec.LoadBalancerSourceRanges
+	newSourceRanges := new.Spec.LoadBalancerSourceRanges
+
+	/* work around Kubernetes 1.6 serializing [] as nil. See https://github.com/kubernetes/kubernetes/issues/43203 */
+	if (len(oldSourceRanges) != 0) || (len(newSourceRanges) != 0) {
+		if !util.IsEqualIgnoreOrder(oldSourceRanges, newSourceRanges) {
+			return false, "new service's LoadBalancerSourceRange does not match the current one"
+		}
+	}
+
+	if changed, reason := c.compareAnnotations(old.Annotations, new.Annotations); changed {
+		return !changed, "new service's annotations does not match the current one:" + reason
+	}
+
+	return true, ""
+}
+
 // Update changes Kubernetes objects according to the new specification. Unlike the sync case, the missing object
 // (i.e. service) is treated as an error
 // logical backup cron jobs are an exception: a user-initiated Update can enable a logical backup job
@@ -764,7 +821,7 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
 			updateFailed = true
 			return
 		}
-		if syncStatefulSet || !reflect.DeepEqual(oldSs, newSs) || !reflect.DeepEqual(oldSpec.Annotations, newSpec.Annotations) {
+		if syncStatefulSet || !reflect.DeepEqual(oldSs, newSs) {
 			c.logger.Debugf("syncing statefulsets")
 			syncStatefulSet = false
 			// TODO: avoid generating the StatefulSet object twice by passing it to syncStatefulSet
diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go
index 401b1bc94..acfecc166 100644
--- a/pkg/cluster/cluster_test.go
+++ b/pkg/cluster/cluster_test.go
@@ -3,6 +3,7 @@ package cluster
 import (
 	"fmt"
 	"reflect"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -1054,6 +1055,336 @@ func TestCompareEnv(t *testing.T) {
 	}
 }
 
+func newService(ann map[string]string, svcT v1.ServiceType, lbSr []string) *v1.Service {
+	svc := &v1.Service{
+		Spec: v1.ServiceSpec{
+			Type:                     svcT,
+			LoadBalancerSourceRanges: lbSr,
+		},
+	}
+	svc.Annotations = ann
+	return svc
+}
+
+func TestCompareServices(t *testing.T) {
+	testName := "TestCompareServices"
+	cluster := Cluster{
+		Config: Config{
+			OpConfig: config.Config{
+				Resources: config.Resources{
+					IgnoredAnnotations: []string{
+						"k8s.v1.cni.cncf.io/network-status",
+					},
+				},
+			},
+		},
+	}
+
+	tests := []struct {
+		about   string
+		current *v1.Service
+		new     *v1.Service
+		reason  string
+		match   bool
+	}{
+		{
+			about: "two equal services",
+			current: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+				},
+				v1.ServiceTypeClusterIP,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			new: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+				},
+				v1.ServiceTypeClusterIP,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			match: true,
+		},
+		{
+			about: "services differ on service type",
+			current: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+				},
+				v1.ServiceTypeClusterIP,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			new: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			match:  false,
+			reason: `new service's type "LoadBalancer" does not match the current one "ClusterIP"`,
+		},
+		{
+			about: "services differ on lb source ranges",
+			current: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			new: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"185.249.56.0/22"}),
+			match:  false,
+			reason: `new service's LoadBalancerSourceRange does not match the current one`,
+		},
+		{
+			about: "new service doesn't have lb source ranges",
+			current: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			new: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{}),
+			match:  false,
+			reason: `new service's LoadBalancerSourceRange does not match the current one`,
+		},
+		{
+			about: "services differ on DNS annotation",
+			current: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			new: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "new_clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			match:  false,
+			reason: `new service's annotations does not match the current one: "external-dns.alpha.kubernetes.io/hostname" changed from "clstr.acid.zalan.do" to "new_clstr.acid.zalan.do".`,
+		},
+		{
+			about: "services differ on AWS ELB annotation",
+			current: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			new: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: "1800",
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			match:  false,
+			reason: `new service's annotations does not match the current one: "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout" changed from "3600" to "1800".`,
+		},
+		{
+			about: "service changes existing annotation",
+			current: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+					"foo":                              "bar",
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			new: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+					"foo":                              "baz",
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			match:  false,
+			reason: `new service's annotations does not match the current one: "foo" changed from "bar" to "baz".`,
+		},
+		{
+			about: "service changes multiple existing annotations",
+			current: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+					"foo":                              "bar",
+					"bar":                              "foo",
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			new: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+					"foo":                              "baz",
+					"bar":                              "fooz",
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			match: false,
+			// Test just the prefix to avoid flakiness and map sorting
+			reason: `new service's annotations does not match the current one:`,
+		},
+		{
+			about: "service adds a new custom annotation",
+			current: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			new: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+					"foo":                              "bar",
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			match:  false,
+			reason: `new service's annotations does not match the current one: Added "foo" with value "bar".`,
+		},
+		{
+			about: "service removes a custom annotation",
+			current: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+					"foo":                              "bar",
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			new: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			match:  false,
+			reason: `new service's annotations does not match the current one: Removed "foo".`,
+		},
+		{
+			about: "service removes a custom annotation and adds a new one",
+			current: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+					"foo":                              "bar",
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			new: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+					"bar":                              "foo",
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			match:  false,
+			reason: `new service's annotations does not match the current one: Removed "foo". Added "bar" with value "foo".`,
+		},
+		{
+			about: "service removes a custom annotation, adds a new one and change another",
+			current: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+					"foo":                              "bar",
+					"zalan":                            "do",
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			new: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+					"bar":                              "foo",
+					"zalan":                            "do.com",
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			match: false,
+			// Test just the prefix to avoid flakiness and map sorting
+			reason: `new service's annotations does not match the current one: Removed "foo".`,
+		},
+		{
+			about: "service add annotations",
+			current: newService(
+				map[string]string{},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			new: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			match: false,
+			// Test just the prefix to avoid flakiness and map sorting
+			reason: `new service's annotations does not match the current one: Added `,
+		},
+		{
+			about: "ignored annotations",
+			current: newService(
+				map[string]string{},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			new: newService(
+				map[string]string{
+					"k8s.v1.cni.cncf.io/network-status": "up",
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			match: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.about, func(t *testing.T) {
+			match, reason := cluster.compareServices(tt.current, tt.new)
+			if match && !tt.match {
+				t.Logf("match=%v current=%v, old=%v reason=%s", match, tt.current.Annotations, tt.new.Annotations, reason)
+				t.Errorf("%s - expected services to do not match: %q and %q", testName, tt.current, tt.new)
+				return
+			}
+			if !match && tt.match {
+				t.Errorf("%s - expected services to be the same: %q and %q", testName, tt.current, tt.new)
+				return
+			}
+			if !match && !tt.match {
+				if !strings.HasPrefix(reason, tt.reason) {
+					t.Errorf("%s - expected reason prefix %s, found %s", testName, tt.reason, reason)
+					return
+				}
+			}
+		})
+	}
+}
+
 func TestCrossNamespacedSecrets(t *testing.T) {
 	testName := "test secrets in different namespace"
 	clientSet := fake.NewSimpleClientset()
diff --git a/pkg/cluster/connection_pooler.go b/pkg/cluster/connection_pooler.go
index 238e9d430..1fba2eed7 100644
--- a/pkg/cluster/connection_pooler.go
+++ b/pkg/cluster/connection_pooler.go
@@ -912,7 +912,7 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql
 	if service, err = c.KubeClient.Services(c.Namespace).Get(context.TODO(), c.connectionPoolerName(role), metav1.GetOptions{}); err == nil {
 		c.ConnectionPooler[role].Service = service
 		desiredSvc := c.generateConnectionPoolerService(c.ConnectionPooler[role])
-		if match, reason := k8sutil.SameService(service, desiredSvc); !match {
+		if match, reason := c.compareServices(service, desiredSvc); !match {
 			syncReason = append(syncReason, reason)
 			c.logServiceChanges(role, service, desiredSvc, false, reason)
 			newService, err = c.updateService(role, service, desiredSvc)
diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go
index 1c011426c..7ff021ceb 100644
--- a/pkg/cluster/sync.go
+++ b/pkg/cluster/sync.go
@@ -167,7 +167,7 @@ func (c *Cluster) syncService(role PostgresRole) error {
 	if svc, err = c.KubeClient.Services(c.Namespace).Get(context.TODO(), c.serviceName(role), metav1.GetOptions{}); err == nil {
 		c.Services[role] = svc
 		desiredSvc := c.generateService(role, &c.Spec)
-		if match, reason := k8sutil.SameService(svc, desiredSvc); !match {
+		if match, reason := c.compareServices(svc, desiredSvc); !match {
 			c.logServiceChanges(role, svc, desiredSvc, false, reason)
 			updatedSvc, err := c.updateService(role, svc, desiredSvc)
 			if err != nil {
diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go
index be7118d9e..8e4ad1f69 100644
--- a/pkg/controller/operator_config.go
+++ b/pkg/controller/operator_config.go
@@ -108,6 +108,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	result.InheritedLabels = fromCRD.Kubernetes.InheritedLabels
 	result.InheritedAnnotations = fromCRD.Kubernetes.InheritedAnnotations
 	result.DownscalerAnnotations = fromCRD.Kubernetes.DownscalerAnnotations
+	result.IgnoredAnnotations = fromCRD.Kubernetes.IgnoredAnnotations
 	result.ClusterNameLabel = util.Coalesce(fromCRD.Kubernetes.ClusterNameLabel, "cluster-name")
 	result.DeleteAnnotationDateKey = fromCRD.Kubernetes.DeleteAnnotationDateKey
 	result.DeleteAnnotationNameKey = fromCRD.Kubernetes.DeleteAnnotationNameKey
diff --git a/pkg/spec/types.go b/pkg/spec/types.go
index 428bcbdc6..e594a7978 100644
--- a/pkg/spec/types.go
+++ b/pkg/spec/types.go
@@ -119,6 +119,7 @@ type ControllerConfig struct {
 	CRDReadyWaitTimeout  time.Duration
 	ConfigMapName        NamespacedName
 	Namespace            string
+	IgnoredAnnotations   []string
 
 	EnableJsonLogging bool
 }
diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go
index e6b2c03c0..eb2eb07cf 100644
--- a/pkg/util/config/config.go
+++ b/pkg/util/config/config.go
@@ -42,6 +42,7 @@ type Resources struct {
 	InheritedLabels               []string            `name:"inherited_labels" default:""`
 	InheritedAnnotations          []string            `name:"inherited_annotations" default:""`
 	DownscalerAnnotations         []string            `name:"downscaler_annotations"`
+	IgnoredAnnotations            []string            `name:"ignored_annotations"`
 	ClusterNameLabel              string              `name:"cluster_name_label" default:"cluster-name"`
 	DeleteAnnotationDateKey       string              `name:"delete_annotation_date_key"`
 	DeleteAnnotationNameKey       string              `name:"delete_annotation_name_key"`
diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go
index fd5de8195..f4ea014f2 100644
--- a/pkg/util/k8sutil/k8sutil.go
+++ b/pkg/util/k8sutil/k8sutil.go
@@ -213,57 +213,6 @@ func (client *KubernetesClient) SetPostgresCRDStatus(clusterName spec.Namespaced
 	return pg, nil
 }
 
-// SameService compares the Services
-func SameService(cur, new *v1.Service) (match bool, reason string) {
-	//TODO: improve comparison
-	if cur.Spec.Type != new.Spec.Type {
-		return false, fmt.Sprintf("new service's type %q does not match the current one %q",
-			new.Spec.Type, cur.Spec.Type)
-	}
-
-	oldSourceRanges := cur.Spec.LoadBalancerSourceRanges
-	newSourceRanges := new.Spec.LoadBalancerSourceRanges
-
-	/* work around Kubernetes 1.6 serializing [] as nil. See https://github.com/kubernetes/kubernetes/issues/43203 */
-	if (len(oldSourceRanges) != 0) || (len(newSourceRanges) != 0) {
-		if !reflect.DeepEqual(oldSourceRanges, newSourceRanges) {
-			return false, "new service's LoadBalancerSourceRange does not match the current one"
-		}
-	}
-
-	match = true
-
-	reasonPrefix := "new service's annotations does not match the current one:"
-	for ann := range cur.Annotations {
-		if _, ok := new.Annotations[ann]; !ok {
-			match = false
-			if len(reason) == 0 {
-				reason = reasonPrefix
-			}
-			reason += fmt.Sprintf(" Removed '%s'.", ann)
-		}
-	}
-
-	for ann := range new.Annotations {
-		v, ok := cur.Annotations[ann]
-		if !ok {
-			if len(reason) == 0 {
-				reason = reasonPrefix
-			}
-			reason += fmt.Sprintf(" Added '%s' with value '%s'.", ann, new.Annotations[ann])
-			match = false
-		} else if v != new.Annotations[ann] {
-			if len(reason) == 0 {
-				reason = reasonPrefix
-			}
-			reason += fmt.Sprintf(" '%s' changed from '%s' to '%s'.", ann, v, new.Annotations[ann])
-			match = false
-		}
-	}
-
-	return match, reason
-}
-
 // SamePDB compares the PodDisruptionBudgets
 func SamePDB(cur, new *policybeta1.PodDisruptionBudget) (match bool, reason string) {
 	//TODO: improve comparison
diff --git a/pkg/util/k8sutil/k8sutil_test.go b/pkg/util/k8sutil/k8sutil_test.go
deleted file mode 100644
index b3e768501..000000000
--- a/pkg/util/k8sutil/k8sutil_test.go
+++ /dev/null
@@ -1,311 +0,0 @@
-package k8sutil
-
-import (
-	"strings"
-	"testing"
-
-	"github.com/zalando/postgres-operator/pkg/util/constants"
-
-	v1 "k8s.io/api/core/v1"
-)
-
-func newsService(ann map[string]string, svcT v1.ServiceType, lbSr []string) *v1.Service {
-	svc := &v1.Service{
-		Spec: v1.ServiceSpec{
-			Type:                     svcT,
-			LoadBalancerSourceRanges: lbSr,
-		},
-	}
-	svc.Annotations = ann
-	return svc
-}
-
-func TestSameService(t *testing.T) {
-	tests := []struct {
-		about   string
-		current *v1.Service
-		new     *v1.Service
-		reason  string
-		match   bool
-	}{
-		{
-			about: "two equal services",
-			current: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-				},
-				v1.ServiceTypeClusterIP,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			new: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-				},
-				v1.ServiceTypeClusterIP,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			match: true,
-		},
-		{
-			about: "services differ on service type",
-			current: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-				},
-				v1.ServiceTypeClusterIP,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			new: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			match:  false,
-			reason: `new service's type "LoadBalancer" does not match the current one "ClusterIP"`,
-		},
-		{
-			about: "services differ on lb source ranges",
-			current: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			new: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"185.249.56.0/22"}),
-			match:  false,
-			reason: `new service's LoadBalancerSourceRange does not match the current one`,
-		},
-		{
-			about: "new service doesn't have lb source ranges",
-			current: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			new: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{}),
-			match:  false,
-			reason: `new service's LoadBalancerSourceRange does not match the current one`,
-		},
-		{
-			about: "services differ on DNS annotation",
-			current: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			new: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "new_clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			match:  false,
-			reason: `new service's annotations does not match the current one: 'external-dns.alpha.kubernetes.io/hostname' changed from 'clstr.acid.zalan.do' to 'new_clstr.acid.zalan.do'.`,
-		},
-		{
-			about: "services differ on AWS ELB annotation",
-			current: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			new: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: "1800",
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			match:  false,
-			reason: `new service's annotations does not match the current one: 'service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout' changed from '3600' to '1800'.`,
-		},
-		{
-			about: "service changes existing annotation",
-			current: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-					"foo":                              "bar",
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			new: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-					"foo":                              "baz",
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			match:  false,
-			reason: `new service's annotations does not match the current one: 'foo' changed from 'bar' to 'baz'.`,
-		},
-		{
-			about: "service changes multiple existing annotations",
-			current: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-					"foo":                              "bar",
-					"bar":                              "foo",
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			new: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-					"foo":                              "baz",
-					"bar":                              "fooz",
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			match: false,
-			// Test just the prefix to avoid flakiness and map sorting
-			reason: `new service's annotations does not match the current one:`,
-		},
-		{
-			about: "service adds a new custom annotation",
-			current: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			new: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-					"foo":                              "bar",
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			match:  false,
-			reason: `new service's annotations does not match the current one: Added 'foo' with value 'bar'.`,
-		},
-		{
-			about: "service removes a custom annotation",
-			current: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-					"foo":                              "bar",
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			new: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			match:  false,
-			reason: `new service's annotations does not match the current one: Removed 'foo'.`,
-		},
-		{
-			about: "service removes a custom annotation and adds a new one",
-			current: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-					"foo":                              "bar",
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			new: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-					"bar":                              "foo",
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			match:  false,
-			reason: `new service's annotations does not match the current one: Removed 'foo'. Added 'bar' with value 'foo'.`,
-		},
-		{
-			about: "service removes a custom annotation, adds a new one and change another",
-			current: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-					"foo":                              "bar",
-					"zalan":                            "do",
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			new: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-					"bar":                              "foo",
-					"zalan":                            "do.com",
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			match: false,
-			// Test just the prefix to avoid flakiness and map sorting
-			reason: `new service's annotations does not match the current one: Removed 'foo'.`,
-		},
-		{
-			about: "service add annotations",
-			current: newsService(
-				map[string]string{},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			new: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			match: false,
-			// Test just the prefix to avoid flakiness and map sorting
-			reason: `new service's annotations does not match the current one: Added `,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.about, func(t *testing.T) {
-			match, reason := SameService(tt.current, tt.new)
-			if match && !tt.match {
-				t.Errorf("expected services to do not match: '%q' and '%q'", tt.current, tt.new)
-				return
-			}
-			if !match && tt.match {
-				t.Errorf("expected services to be the same: '%q' and '%q'", tt.current, tt.new)
-				return
-			}
-			if !match && !tt.match {
-				if !strings.HasPrefix(reason, tt.reason) {
-					t.Errorf("expected reason prefix '%s', found '%s'", tt.reason, reason)
-					return
-				}
-			}
-		})
-	}
-}

From d6641a9b0fb83797bb17bc1f3dcb10635b2ac230 Mon Sep 17 00:00:00 2001
From: Felix Kunde <felix-kunde@gmx.de>
Date: Fri, 25 Mar 2022 10:03:20 +0100
Subject: [PATCH 40/58] [UI] support Postgres version and pod resource changes
 + adding IOPS and throughput config options (#1824)

---
 ui/app/src/edit.tag.pug         | 13 ++++++--
 ui/app/src/help-general.tag.pug |  4 +--
 ui/app/src/new.tag.pug          | 45 +++++++++++++++++++++++++++-
 ui/app/src/postgresqls.tag.pug  | 28 +++++++++++++----
 ui/manifests/deployment.yaml    |  4 ++-
 ui/operator_ui/main.py          | 53 +++++++++++++++++++++++++++++++--
 ui/run_local.sh                 |  4 ++-
 7 files changed, 137 insertions(+), 14 deletions(-)

diff --git a/ui/app/src/edit.tag.pug b/ui/app/src/edit.tag.pug
index c1d94e589..bb536a8e4 100644
--- a/ui/app/src/edit.tag.pug
+++ b/ui/app/src/edit.tag.pug
@@ -126,6 +126,7 @@ edit
         if (i.metadata.selfLink) { delete i.metadata.selfLink }
         if (i.metadata.uid) { delete i.metadata.uid }
         if (i.metadata.resourceVersion) { delete i.metadata.resourceVersion }
+        if (i.metadata.managedFields) { delete i.metadata.managedFields }
 
         this.update()
         this.refs.yamlNice.innerHTML = yamlParser.safeDump(i.postgresql, {sortKeys: true})
@@ -138,7 +139,15 @@ edit
         o.spec.enableMasterLoadBalancer = i.spec.enableMasterLoadBalancer || false
         o.spec.enableReplicaLoadBalancer = i.spec.enableReplicaLoadBalancer || false
         o.spec.enableConnectionPooler = i.spec.enableConnectionPooler || false
-        o.spec.volume = { size: i.spec.volume.size }
+
+        o.spec.volume = {
+          size: i.spec.volume.size,
+          throughput: i.spec.volume.throughput || 125,
+          iops: i.spec.volume.iops || 3000
+        }
+
+        o.spec.postgresql = {}
+        o.spec.postgresql.version = i.spec.postgresql.version
 
         if ('users' in i.spec && typeof i.spec.users === 'object') {
           o.spec.users = Object.mapValues(i.spec.users, roleFlags =>
@@ -166,7 +175,7 @@ edit
               ].forEach(resourceType => {
                 if (resourceType in resources) {
                   const resourceClaim = resources[resourceType]
-                  if (typeof resourceClaim === '') {
+                  if (typeof resourceClaim === 'string') {
                     o.spec.resources[section][resourceType] = resources[resourceType]
                   }
                 }
diff --git a/ui/app/src/help-general.tag.pug b/ui/app/src/help-general.tag.pug
index 1af25f1a8..7771fcebd 100644
--- a/ui/app/src/help-general.tag.pug
+++ b/ui/app/src/help-general.tag.pug
@@ -13,6 +13,6 @@ help-general
     h3 Basics
 
     p.
-      The PostgreSQL operator will use your definition to create a new
+      The Postgres Operator will use your definition to create a new
       PostgreSQL cluster for you. You can either copy the yaml definition
-      to the repositiory or you can just hit create cluster.
+      to a repositiory or hit create cluster (not available in prod).
diff --git a/ui/app/src/new.tag.pug b/ui/app/src/new.tag.pug
index 6293a6c7a..a2b0506da 100644
--- a/ui/app/src/new.tag.pug
+++ b/ui/app/src/new.tag.pug
@@ -266,6 +266,37 @@ new
                   )
                   .input-group-addon
                     .input-units Gi
+            tr
+              td 
+              td Specify Iops and Throughput only if you need more than the default 3000 Iops and 125Mb/s EBS provides.
+
+            tr
+              td Iops
+              td
+                .input-group
+                  input.form-control(
+                    ref='iops'
+                    type='number'
+                    value='{ iops }'
+                    onchange='{ iopsChange }'
+                    onkeyup='{ iopsChange }'
+                  )
+                  .input-group-addon
+                    .input-units
+
+            tr
+              td Througput
+              td
+                .input-group
+                  input.form-control(
+                    ref='throughput'
+                    type='number'
+                    value='{ throughput }'
+                    onchange='{ throughputChange }'
+                    onkeyup='{ throughputChange }'
+                  )
+                  .input-group-addon
+                    .input-units MB/s
 
             tr(if='{ config.users_visible }')
               td
@@ -509,7 +540,9 @@ new
         enableConnectionPooler: true
         {{/if}}
         volume:
-          size: "{{ volumeSize }}Gi"
+          size: "{{ volumeSize }}Gi"{{#if iops}}
+          iops: {{ iops }}{{/if}}{{#if throughput}}
+          throughput: {{ throughput }}{{/if}}
         {{#if users}}
         users:{{#each users}}
           {{ state }}: []{{/each}}{{/if}}
@@ -560,6 +593,8 @@ new
         enableReplicaLoadBalancer: this.enableReplicaLoadBalancer,
         enableConnectionPooler: this.enableConnectionPooler,
         volumeSize: this.volumeSize,
+        iops: this.iops,
+        throughput: this.throughput,
         users: this.users.valids,
         databases: this.databases.valids,
         ranges: this.ranges,
@@ -624,6 +659,14 @@ new
       this.volumeSize = +e.target.value
     }
 
+    this.iopsChange = e => {
+      this.iops = +e.target.value
+    }
+
+    this.throughputChange = e => {
+      this.throughput = +e.target.value
+    }
+
     this.updateDNSName = () => {
       this.dnsName = this.config.dns_format_string.format(
         this.name,
diff --git a/ui/app/src/postgresqls.tag.pug b/ui/app/src/postgresqls.tag.pug
index 38e5fcd9d..fabf769a0 100644
--- a/ui/app/src/postgresqls.tag.pug
+++ b/ui/app/src/postgresqls.tag.pug
@@ -69,7 +69,7 @@ postgresqls
             td { cpu } / { cpu_limit }
             td { memory } / { memory_limit }
             td { volume_size }
-            td { calcCosts(nodes, cpu, memory, volume_size) }$
+            td { calcCosts(nodes, cpu, memory, volume_size, iops, throughput) }$
 
             td
 
@@ -152,7 +152,7 @@ postgresqls
             td { cpu } / { cpu_limit }
             td { memory } / { memory_limit }
             td { volume_size }
-            td { calcCosts(nodes, cpu, memory, volume_size) }$
+            td { calcCosts(nodes, cpu, memory, volume_size, iops, throughput) }$
 
             td
 
@@ -227,9 +227,22 @@ postgresqls
       + '/' + encodeURI(cluster.name)
     )
 
-    const calcCosts = this.calcCosts = (nodes, cpu, memory, disk) => {
-        costs = Math.max(nodes, opts.config.min_pods) * (toCores(cpu) * opts.config.cost_core + toMemory(memory) * opts.config.cost_memory + toDisk(disk) * opts.config.cost_ebs)
-        return costs.toFixed(2)
+    const calcCosts = this.calcCosts = (nodes, cpu, memory, disk, iops, throughput) => {
+      podcount = Math.max(nodes, opts.config.min_pods)
+      corecost = toCores(cpu) * opts.config.cost_core
+      memorycost = toMemory(memory) * opts.config.cost_memory
+      diskcost = toDisk(disk) * opts.config.cost_ebs
+      iopscost = 0
+      if (iops !== undefined && iops > 3000) {
+        iopscost = (iops - 3000) * opts.config.cost_iops
+      }
+      throughputcost = 0
+      if (throughput !== undefined && throughput > 125) {
+        throughputcost = (throughput - 125) * opts.config.cost_throughput
+      }
+
+      costs = podcount * (corecost + memorycost + diskcost + iopscost + throughputcost)
+       return costs.toFixed(2)
     }
 
     const toDisk = this.toDisk = value => {
@@ -253,6 +266,11 @@ postgresqls
         value = Number(value)
         return value
       }
+      else if(value.endsWith("Ti")) {
+        value = value.substring(0, value.length-2)
+        value = Number(value) * 1000
+        return value
+      }
 
       return value
     }
diff --git a/ui/manifests/deployment.yaml b/ui/manifests/deployment.yaml
index 865acd6b9..b2af47790 100644
--- a/ui/manifests/deployment.yaml
+++ b/ui/manifests/deployment.yaml
@@ -62,7 +62,9 @@ spec:
                   "replica_load_balancer_visible": true,
                   "resources_visible": true,
                   "users_visible": true,
-                  "cost_ebs": 0.119,
+                  "cost_ebs": 0.0952,
+                  "cost_iops": 0.006,
+                  "cost_throughput": 0.0476,
                   "cost_core": 0.0575,
                   "cost_memory": 0.014375,
                   "postgresql_versions": [
diff --git a/ui/operator_ui/main.py b/ui/operator_ui/main.py
index dc207d35e..3e1e54969 100644
--- a/ui/operator_ui/main.py
+++ b/ui/operator_ui/main.py
@@ -89,8 +89,10 @@ TARGET_NAMESPACE = getenv('TARGET_NAMESPACE')
 GOOGLE_ANALYTICS = getenv('GOOGLE_ANALYTICS', False)
 MIN_PODS= getenv('MIN_PODS', 2)
 
-# storage pricing, i.e. https://aws.amazon.com/ebs/pricing/
-COST_EBS = float(getenv('COST_EBS', 0.119))  # GB per month
+# storage pricing, i.e. https://aws.amazon.com/ebs/pricing/ (e.g. Europe - Franfurt)
+COST_EBS = float(getenv('COST_EBS', 0.0952))  # GB per month
+COST_IOPS = float(getenv('COST_IOPS', 0.006))  # IOPS per month above 3000 baseline
+COST_THROUGHPUT = float(getenv('COST_THROUGHPUT', 0.0476))  # MB/s per month above 125 MB/s baseline
 
 # compute costs, i.e. https://www.ec2instances.info/?region=eu-central-1&selected=m5.2xlarge
 COST_CORE = 30.5 * 24 * float(getenv('COST_CORE', 0.0575))  # Core per hour m5.2xlarge / 8.
@@ -308,6 +310,8 @@ DEFAULT_UI_CONFIG = {
     'pgui_link': '',
     'static_network_whitelist': {},
     'cost_ebs': COST_EBS,
+    'cost_iops': COST_IOPS,
+    'cost_throughput': COST_THROUGHPUT,
     'cost_core': COST_CORE,
     'cost_memory': COST_MEMORY,
     'min_pods': MIN_PODS
@@ -487,6 +491,8 @@ def get_postgresqls():
             'cpu': spec.get('resources', {}).get('requests', {}).get('cpu', 0),
             'cpu_limit': spec.get('resources', {}).get('limits', {}).get('cpu', 0),
             'volume_size': spec.get('volume', {}).get('size', 0),
+            'iops': spec.get('volume', {}).get('iops', 3000),
+            'throughput': spec.get('volume', {}).get('throughput', 125),
             'team': (
                 spec.get('teamId') or
                 metadata.get('labels', {}).get('team', '')
@@ -614,6 +620,28 @@ def update_postgresql(namespace: str, cluster: str):
 
         spec['volume'] = {'size': size}
 
+    if (
+        'volume' in postgresql['spec']
+        and 'iops' in postgresql['spec']['volume']
+        and postgresql['spec']['volume']['iops'] != None
+    ):
+        iops = int(postgresql['spec']['volume']['iops'])
+        if not 'volume' in spec:
+            spec['volume'] = {}
+
+        spec['volume']['iops'] = iops
+
+    if (
+        'volume' in postgresql['spec']
+        and 'throughput' in postgresql['spec']['volume']
+        and postgresql['spec']['volume']['throughput'] != None
+    ):
+        throughput = int(postgresql['spec']['volume']['throughput'])
+        if not 'volume' in spec:
+            spec['volume'] = {}
+
+        spec['volume']['throughput'] = throughput
+
     if 'enableConnectionPooler' in postgresql['spec']:
         cp = postgresql['spec']['enableConnectionPooler']
         if not cp:
@@ -758,6 +786,27 @@ def update_postgresql(namespace: str, cluster: str):
                     owner_username=owner_username,
                 )
 
+    resource_types = ["cpu","memory"]
+    resource_constraints = ["requests","limits"]
+    if "resources" in postgresql["spec"]:
+        spec["resources"] = {}
+
+        res = postgresql["spec"]["resources"]
+        for rt in resource_types:
+            for rc in resource_constraints:
+                if rc in res:
+                    if rt in res[rc]:
+                        if not rc in spec["resources"]:
+                            spec["resources"][rc] = {}
+                        spec["resources"][rc][rt] = res[rc][rt]
+
+    if "postgresql" in postgresql["spec"]:
+        if "version" in postgresql["spec"]["postgresql"]:
+            if "postgresql" not in spec:
+                spec["postgresql"]={}
+
+            spec["postgresql"]["version"] = postgresql["spec"]["postgresql"]["version"]
+
     o['spec'].update(spec)
 
     apply_postgresql(get_cluster(), namespace, cluster, o)
diff --git a/ui/run_local.sh b/ui/run_local.sh
index 33c0abf27..e23b43423 100755
--- a/ui/run_local.sh
+++ b/ui/run_local.sh
@@ -19,7 +19,9 @@ default_operator_ui_config='{
   "nat_gateways_visible": false,
   "resources_visible": true,
   "users_visible": true,
-  "cost_ebs": 0.119,
+  "cost_ebs": 0.0952,
+  "cost_iops": 0.006,
+  "cost_throughput": 0.0476,
   "cost_core": 0.0575,
   "cost_memory": 0.014375,
   "postgresql_versions": [

From cc2686151976e11a01bd4a2dfcfdd89537b389a1 Mon Sep 17 00:00:00 2001
From: Matthias Teich <matthiasteich83@gmail.com>
Date: Fri, 25 Mar 2022 14:56:06 +0100
Subject: [PATCH 41/58] [UI] add ingressClassName (#1787)

* feat: add ingressClassName

Signed-off-by: Matthias Teich <matthias.teich@gdata.de>

* Update example manifest

Signed-off-by: Matthias Teich <matthias.teich@gdata.de>

* Update ui/manifests/ingress.yaml

Co-authored-by: Felix Kunde <felix-kunde@gmx.de>
---
 charts/postgres-operator-ui/templates/ingress.yaml | 3 +++
 charts/postgres-operator-ui/values.yaml            | 1 +
 ui/manifests/ingress.yaml                          | 1 +
 3 files changed, 5 insertions(+)

diff --git a/charts/postgres-operator-ui/templates/ingress.yaml b/charts/postgres-operator-ui/templates/ingress.yaml
index 21e7dbea2..37b47a6c5 100644
--- a/charts/postgres-operator-ui/templates/ingress.yaml
+++ b/charts/postgres-operator-ui/templates/ingress.yaml
@@ -23,6 +23,9 @@ metadata:
     {{- toYaml . | nindent 4 }}
   {{- end }}
 spec:
+{{- if .Values.ingress.ingressClassName }}
+  ingressClassName: {{ .Values.ingress.ingressClassName }}
+{{- end }}
 {{- if .Values.ingress.tls }}
   tls:
   {{- range .Values.ingress.tls }}
diff --git a/charts/postgres-operator-ui/values.yaml b/charts/postgres-operator-ui/values.yaml
index ff1555b13..c8353667e 100644
--- a/charts/postgres-operator-ui/values.yaml
+++ b/charts/postgres-operator-ui/values.yaml
@@ -93,6 +93,7 @@ ingress:
     {}
     # kubernetes.io/ingress.class: nginx
     # kubernetes.io/tls-acme: "true"
+  ingressClassName: ""
   hosts:
     - host: ui.example.org
       paths: [""]
diff --git a/ui/manifests/ingress.yaml b/ui/manifests/ingress.yaml
index a5e6f0fab..53be1f45b 100644
--- a/ui/manifests/ingress.yaml
+++ b/ui/manifests/ingress.yaml
@@ -6,6 +6,7 @@ metadata:
   labels:
     application: "postgres-operator-ui"
 spec:
+  # ingressClassName: "ingress-nginx"
   rules:
     - host: "ui.example.org"
       http:

From 7e92fa6cb010433853bcf338ed7f6d2a8fb20007 Mon Sep 17 00:00:00 2001
From: Aisuko <urakiny@gmail.com>
Date: Fri, 25 Mar 2022 21:58:16 +0800
Subject: [PATCH 42/58] Add the minimal master/replica svc-monitor example
 manifest for end user (#1452)

Signed-off-by: aisuko <urakiny@gmail.com>
---
 .../minimal-master-replica-svcmonitor.yaml    | 131 ++++++++++++++++++
 1 file changed, 131 insertions(+)
 create mode 100644 manifests/minimal-master-replica-svcmonitor.yaml

diff --git a/manifests/minimal-master-replica-svcmonitor.yaml b/manifests/minimal-master-replica-svcmonitor.yaml
new file mode 100644
index 000000000..67ed28c81
--- /dev/null
+++ b/manifests/minimal-master-replica-svcmonitor.yaml
@@ -0,0 +1,131 @@
+# Here we use https://github.com/prometheus-community/helm-charts/charts/kube-prometheus-stack
+# Please keep the ServiceMonitor's label same as the Helm release name of kube-prometheus-stack 
+
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: test-pg
+---
+apiVersion: "acid.zalan.do/v1"
+kind: postgresql
+metadata:
+  name: acid-minimal-cluster
+  namespace: test-pg
+  labels:
+    app: test-pg
+spec:
+  teamId: "acid"
+  volume:
+    size: 1Gi
+  numberOfInstances: 2
+  users:
+    zalando:  # database owner
+    - superuser
+    - createdb
+    foo_user: []  # role for application foo
+  databases:
+    foo: zalando  # dbname: owner
+  preparedDatabases:
+    bar: {}
+  postgresql:
+    version: "13"
+  sidecars:
+    - name: "exporter"
+      image: "wrouesnel/postgres_exporter"
+      ports:
+        - name: exporter
+          containerPort: 9187
+          protocol: TCP
+      resources:
+        limits:
+          cpu: 500m
+          memory: 256M
+        requests:
+          cpu: 100m
+          memory: 200M
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: acid-minimal-cluster-svc-metrics-master
+  namespace: test-pg
+  labels:
+    app: test-pg
+    spilo-role: master
+  annotations:
+      prometheus.io/scrape: "true"
+      prometheus.io/port: "9187"
+spec:
+  type: ClusterIP
+  ports:
+    - name: exporter
+      port: 9187
+      targetPort: exporter
+  selector:
+    application: spilo
+    cluster-name: acid-minimal-cluster
+    spilo-role: master
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: acid-minimal-cluster-svc-metrics-replica
+  namespace: test-pg
+  labels:
+    app: test-pg
+    spilo-role: replica
+  annotations:
+      prometheus.io/scrape: "true"
+      prometheus.io/port: "9187"
+spec:
+  type: ClusterIP
+  ports:
+    - name: exporter
+      port: 9187
+      targetPort: exporter
+  selector:
+    application: spilo
+    cluster-name: acid-minimal-cluster
+    spilo-role: replica
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: acid-minimal-cluster-svcm-master
+  namespace: test-pg
+  labels:
+    app: test-pg
+    spilo-role: master
+spec:
+  endpoints:
+    - port: exporter
+      interval: 15s
+      scrapeTimeout: 10s
+  namespaceSelector:
+    matchNames:
+      - test-pg
+  selector:
+    matchLabels:
+      app: test-pg
+      spilo-role: master
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: acid-minimal-cluster-svcm-replica
+  namespace: test-pg
+  labels:
+    app: test-pg
+    spilo-role: replica
+spec:
+  endpoints:
+    - port: exporter
+      interval: 15s
+      scrapeTimeout: 10s
+  namespaceSelector:
+    matchNames:
+      - test-pg
+  selector:
+    matchLabels:
+      app: test-pg
+      spilo-role: replica

From 30f2ba6525897aa78a55efb4131625fdf6ad71e2 Mon Sep 17 00:00:00 2001
From: evsasha <evsashadroid@gmail.com>
Date: Mon, 28 Mar 2022 11:09:26 +0300
Subject: [PATCH 43/58] do not create endpoints when use config maps (#1760)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* do not create endpoints when use config maps
* delete cluster objects with 'leader' suffix

Co-authored-by: Евграфов Александр Александрович <aevgrafov@cmx.ru>
---
 pkg/cluster/cluster.go | 29 ++++++++++++++++-------------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go
index e411c66e8..dcef602b9 100644
--- a/pkg/cluster/cluster.go
+++ b/pkg/cluster/cluster.go
@@ -43,7 +43,7 @@ var (
 	alphaNumericRegexp    = regexp.MustCompile("^[a-zA-Z][a-zA-Z0-9]*$")
 	databaseNameRegexp    = regexp.MustCompile("^[a-zA-Z_][a-zA-Z0-9_]*$")
 	userRegexp            = regexp.MustCompile(`^[a-z0-9]([-_a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-_a-z0-9]*[a-z0-9])?)*$`)
-	patroniObjectSuffixes = []string{"config", "failover", "sync"}
+	patroniObjectSuffixes = []string{"config", "failover", "sync", "leader"}
 )
 
 // Config contains operator-wide clients and configuration used from a cluster. TODO: remove struct duplication.
@@ -258,18 +258,20 @@ func (c *Cluster) Create() error {
 
 	for _, role := range []PostgresRole{Master, Replica} {
 
-		if c.Endpoints[role] != nil {
-			return fmt.Errorf("%s endpoint already exists in the cluster", role)
-		}
-		if role == Master {
-			// replica endpoint will be created by the replica service. Master endpoint needs to be created by us,
-			// since the corresponding master service does not define any selectors.
-			ep, err = c.createEndpoint(role)
-			if err != nil {
-				return fmt.Errorf("could not create %s endpoint: %v", role, err)
+		if !c.patroniKubernetesUseConfigMaps() {
+			if c.Endpoints[role] != nil {
+				return fmt.Errorf("%s endpoint already exists in the cluster", role)
+			}
+			if role == Master {
+				// replica endpoint will be created by the replica service. Master endpoint needs to be created by us,
+				// since the corresponding master service does not define any selectors.
+				ep, err = c.createEndpoint(role)
+				if err != nil {
+					return fmt.Errorf("could not create %s endpoint: %v", role, err)
+				}
+				c.logger.Infof("endpoint %q has been successfully created", util.NameFromMeta(ep.ObjectMeta))
+				c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Endpoints", "Endpoint %q has been successfully created", util.NameFromMeta(ep.ObjectMeta))
 			}
-			c.logger.Infof("endpoint %q has been successfully created", util.NameFromMeta(ep.ObjectMeta))
-			c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Endpoints", "Endpoint %q has been successfully created", util.NameFromMeta(ep.ObjectMeta))
 		}
 
 		if c.Services[role] != nil {
@@ -1576,8 +1578,9 @@ func (c *Cluster) deletePatroniClusterObjects() error {
 
 	if !c.patroniKubernetesUseConfigMaps() {
 		actionsList = append(actionsList, c.deletePatroniClusterEndpoints)
+	} else {
+		actionsList = append(actionsList, c.deletePatroniClusterServices, c.deletePatroniClusterConfigMaps)
 	}
-	actionsList = append(actionsList, c.deletePatroniClusterServices, c.deletePatroniClusterConfigMaps)
 
 	c.logger.Debugf("removing leftover Patroni objects (endpoints / services and configmaps)")
 	for _, deleter := range actionsList {

From a081173f26f2d6b7dd6408d7cee8b307fd8e6578 Mon Sep 17 00:00:00 2001
From: Marcel Fest <marcel@fest4.eu>
Date: Mon, 28 Mar 2022 16:40:22 +0200
Subject: [PATCH 44/58] Added IPv6 support K8s Service. (#1405)

---
 docker/logical-backup/dump.sh | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/docker/logical-backup/dump.sh b/docker/logical-backup/dump.sh
index 85e372664..2627ac8c9 100755
--- a/docker/logical-backup/dump.sh
+++ b/docker/logical-backup/dump.sh
@@ -12,7 +12,16 @@ DUMP_SIZE_COEFF=5
 ERRORCOUNT=0
 
 TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
-K8S_API_URL=https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1
+if [ "$KUBERNETES_SERVICE_HOST" != "${KUBERNETES_SERVICE_HOST#*[0-9].[0-9]}" ]; then
+  echo "IPv4"
+  K8S_API_URL=https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1
+elif [ "$KUBERNETES_SERVICE_HOST" != "${KUBERNETES_SERVICE_HOST#*:[0-9a-fA-F]}" ]; then
+  echo "IPv6"
+  K8S_API_URL=https://[$KUBERNETES_SERVICE_HOST]:$KUBERNETES_SERVICE_PORT/api/v1
+else
+  echo "Unrecognized IP format '$KUBERNETES_SERVICE_HOST'"
+fi
+echo "API Endpoint: ${K8S_API_URL}"
 CERT=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
 
 LOGICAL_BACKUP_PROVIDER=${LOGICAL_BACKUP_PROVIDER:="s3"}

From 60e0685c32acde6f67dccd7345a0a2b67f605643 Mon Sep 17 00:00:00 2001
From: Felix Kunde <felix-kunde@gmx.de>
Date: Wed, 30 Mar 2022 18:19:34 +0200
Subject: [PATCH 45/58] define readinessProbe on statefulSet (#1825)

* define readinessProbe on statefulSet
* do not error out on deleting Patroni cluster objects
* change delete order for patroni objects
---
 .../templates/clusterrole-postgres-pod.yaml   |   8 +-
 .../templates/clusterrole.yaml                |   6 -
 docs/quickstart.md                            |  14 +-
 e2e/tests/test_e2e.py                         |   2 +
 manifests/configmap.yaml                      |   2 +-
 ...erator-service-account-rbac-openshift.yaml | 283 ++++++++++++++++++
 pkg/cluster/cluster.go                        |  69 +++--
 pkg/cluster/k8sres.go                         |  36 ++-
 pkg/cluster/resources.go                      |   8 +-
 pkg/controller/postgresql.go                  |   3 +-
 pkg/util/patroni/patroni.go                   |   4 +-
 pkg/util/patroni/patroni_test.go              |   6 +-
 12 files changed, 379 insertions(+), 62 deletions(-)
 create mode 100644 manifests/operator-service-account-rbac-openshift.yaml

diff --git a/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml b/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml
index 33c43822f..fdccf16d3 100644
--- a/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml
+++ b/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml
@@ -9,7 +9,7 @@ metadata:
     app.kubernetes.io/managed-by: {{ .Release.Service }}
     app.kubernetes.io/instance: {{ .Release.Name }}
 rules:
-# Patroni needs to watch and manage endpoints
+# Patroni needs to watch and manage config maps or endpoints
 {{- if toString .Values.configGeneral.kubernetes_use_configmaps | eq "true" }}
 - apiGroups:
   - ""
@@ -24,12 +24,6 @@ rules:
   - patch
   - update
   - watch
-- apiGroups:
-  - ""
-  resources:
-  - endpoints
-  verbs:
-  - get
 {{- else }}
 - apiGroups:
   - ""
diff --git a/charts/postgres-operator/templates/clusterrole.yaml b/charts/postgres-operator/templates/clusterrole.yaml
index 8b2e9136e..199086acc 100644
--- a/charts/postgres-operator/templates/clusterrole.yaml
+++ b/charts/postgres-operator/templates/clusterrole.yaml
@@ -89,12 +89,6 @@ rules:
   - patch
   - update
   - watch
-- apiGroups:
-  - ""
-  resources:
-  - endpoints
-  verbs:
-  - get
 {{- else }}
 # to read configuration from ConfigMaps
 - apiGroups:
diff --git a/docs/quickstart.md b/docs/quickstart.md
index ed01367b7..7049a6ef9 100644
--- a/docs/quickstart.md
+++ b/docs/quickstart.md
@@ -37,7 +37,7 @@ The Postgres Operator can be deployed in the following ways:
 * Kustomization
 * Helm chart
 
-### Manual deployment setup
+### Manual deployment setup on Kubernetes
 
 The Postgres Operator can be installed simply by applying yaml manifests. Note,
 we provide the `/manifests` directory as an example only; you should consider
@@ -71,6 +71,18 @@ manifest.
 ./run_operator_locally.sh
 ```
 
+### Manual deployment setup on OpenShift
+
+To install the Postgres Operator in OpenShift you have to change the config
+parameter `kubernetes_use_configmaps` to `"true"`. Otherwise, the operator
+and Patroni will store leader and config keys in `Endpoints` that are not
+supported in OpenShift. This requires also a slightly different set of rules
+for the `postgres-operator` and `postgres-pod` cluster roles.
+
+```bash
+oc create -f manifests/operator-service-account-rbac-openshift.yaml
+```
+
 ### Helm chart
 
 Alternatively, the operator can be installed by using the provided [Helm](https://helm.sh/)
diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py
index b2977b687..0f9515896 100644
--- a/e2e/tests/test_e2e.py
+++ b/e2e/tests/test_e2e.py
@@ -1759,6 +1759,8 @@ class EndToEndTestCase(unittest.TestCase):
             self.eventuallyEqual(lambda: len(k8s.api.custom_objects_api.list_namespaced_custom_object(
                 "acid.zalan.do", "v1", "default", "postgresqls", label_selector="cluster-name=acid-minimal-cluster")["items"]), 0, "Manifest not deleted")
 
+            self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
+
             # check if everything has been deleted
             self.eventuallyEqual(lambda: k8s.count_pods_with_label(cluster_label), 0, "Pods not deleted")
             self.eventuallyEqual(lambda: k8s.count_services_with_label(cluster_label), 0, "Service not deleted")
diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml
index 130a35176..8b392544b 100644
--- a/manifests/configmap.yaml
+++ b/manifests/configmap.yaml
@@ -63,13 +63,13 @@ data:
   # etcd_host: ""
   external_traffic_policy: "Cluster"
   # gcp_credentials: ""
-  # kubernetes_use_configmaps: "false"
   # ignored_annotations: ""
   # infrastructure_roles_secret_name: "postgresql-infrastructure-roles"
   # infrastructure_roles_secrets: "secretname:monitoring-roles,userkey:user,passwordkey:password,rolekey:inrole"
   # inherited_annotations: owned-by
   # inherited_labels: application,environment
   # kube_iam_role: ""
+  # kubernetes_use_configmaps: "false"
   # log_s3_bucket: ""
   logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:v1.7.1"
   # logical_backup_google_application_credentials: ""
diff --git a/manifests/operator-service-account-rbac-openshift.yaml b/manifests/operator-service-account-rbac-openshift.yaml
new file mode 100644
index 000000000..e0e45cc54
--- /dev/null
+++ b/manifests/operator-service-account-rbac-openshift.yaml
@@ -0,0 +1,283 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: postgres-operator
+  namespace: default
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: postgres-operator
+rules:
+# all verbs allowed for custom operator resources
+- apiGroups:
+  - acid.zalan.do
+  resources:
+  - postgresqls
+  - postgresqls/status
+  - operatorconfigurations
+  verbs:
+  - create
+  - delete
+  - deletecollection
+  - get
+  - list
+  - patch
+  - update
+  - watch
+# operator only reads PostgresTeams
+- apiGroups:
+  - acid.zalan.do
+  resources:
+  - postgresteams
+  verbs:
+  - get
+  - list
+  - watch
+# all verbs allowed for event streams (Zalando-internal feature)
+# - apiGroups:
+#   - zalando.org
+#   resources:
+#   - fabriceventstreams
+#   verbs:
+#   - create
+#   - delete
+#   - deletecollection
+#   - get
+#   - list
+#   - patch
+#   - update
+#   - watch
+# to create or get/update CRDs when starting up
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - create
+  - get
+  - patch
+  - update
+# to read configuration and manage ConfigMaps used by Patroni
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+  - delete
+  - deletecollection
+  - get
+  - list
+  - patch
+  - update
+  - watch
+# to send events to the CRs
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - get
+  - list
+  - patch
+  - update
+  - watch
+# to CRUD secrets for database access
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - create
+  - delete
+  - get
+  - update
+# to check nodes for node readiness label
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+# to read or delete existing PVCs. Creation via StatefulSet
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims
+  verbs:
+  - delete
+  - get
+  - list
+  - patch
+  - update
+ # to read existing PVs. Creation should be done via dynamic provisioning
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - update  # only for resizing AWS volumes
+# to watch Spilo pods and do rolling updates. Creation via StatefulSet
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+# to resize the filesystem in Spilo pods when increasing volume size
+- apiGroups:
+  - ""
+  resources:
+  - pods/exec
+  verbs:
+  - create
+# to CRUD services to point to Postgres cluster instances
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - create
+  - delete
+  - get
+  - patch
+  - update
+# to CRUD the StatefulSet which controls the Postgres cluster instances
+- apiGroups:
+  - apps
+  resources:
+  - statefulsets
+  - deployments
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+# to CRUD cron jobs for logical backups
+- apiGroups:
+  - batch
+  resources:
+  - cronjobs
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+# to get namespaces operator resources can run in
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  verbs:
+  - get
+# to define PDBs. Update happens via delete/create
+- apiGroups:
+  - policy
+  resources:
+  - poddisruptionbudgets
+  verbs:
+  - create
+  - delete
+  - get
+# to create ServiceAccounts in each namespace the operator watches
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - get
+  - create
+# to create role bindings to the postgres-pod service account
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - rolebindings
+  verbs:
+  - get
+  - create
+# to grant privilege to run privileged pods (not needed by default)
+#- apiGroups:
+#  - extensions
+#  resources:
+#  - podsecuritypolicies
+#  resourceNames:
+#  - privileged
+#  verbs:
+#  - use
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: postgres-operator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: postgres-operator
+subjects:
+- kind: ServiceAccount
+  name: postgres-operator
+  namespace: default
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: postgres-pod
+rules:
+# Patroni needs to watch and manage config maps
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+  - delete
+  - deletecollection
+  - get
+  - list
+  - patch
+  - update
+  - watch
+# Patroni needs to watch pods
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
+# to let Patroni create a headless service
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - create
+# to grant privilege to run privileged pods (not needed by default)
+#- apiGroups:
+#  - extensions
+#  resources:
+#  - podsecuritypolicies
+#  resourceNames:
+#  - privileged
+#  verbs:
+#  - use
diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go
index dcef602b9..2a08857e3 100644
--- a/pkg/cluster/cluster.go
+++ b/pkg/cluster/cluster.go
@@ -43,7 +43,7 @@ var (
 	alphaNumericRegexp    = regexp.MustCompile("^[a-zA-Z][a-zA-Z0-9]*$")
 	databaseNameRegexp    = regexp.MustCompile("^[a-zA-Z_][a-zA-Z0-9_]*$")
 	userRegexp            = regexp.MustCompile(`^[a-z0-9]([-_a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-_a-z0-9]*[a-z0-9])?)*$`)
-	patroniObjectSuffixes = []string{"config", "failover", "sync", "leader"}
+	patroniObjectSuffixes = []string{"leader", "config", "sync", "failover"}
 )
 
 // Config contains operator-wide clients and configuration used from a cluster. TODO: remove struct duplication.
@@ -258,6 +258,8 @@ func (c *Cluster) Create() error {
 
 	for _, role := range []PostgresRole{Master, Replica} {
 
+		// if kubernetes_use_configmaps is set Patroni will create configmaps
+		// otherwise it will use endpoints
 		if !c.patroniKubernetesUseConfigMaps() {
 			if c.Endpoints[role] != nil {
 				return fmt.Errorf("%s endpoint already exists in the cluster", role)
@@ -1482,22 +1484,26 @@ func (c *Cluster) GetCurrentProcess() Process {
 
 // GetStatus provides status of the cluster
 func (c *Cluster) GetStatus() *ClusterStatus {
-	return &ClusterStatus{
-		Cluster: c.Spec.ClusterName,
-		Team:    c.Spec.TeamID,
-		Status:  c.Status,
-		Spec:    c.Spec,
-
+	status := &ClusterStatus{
+		Cluster:             c.Spec.ClusterName,
+		Team:                c.Spec.TeamID,
+		Status:              c.Status,
+		Spec:                c.Spec,
 		MasterService:       c.GetServiceMaster(),
 		ReplicaService:      c.GetServiceReplica(),
-		MasterEndpoint:      c.GetEndpointMaster(),
-		ReplicaEndpoint:     c.GetEndpointReplica(),
 		StatefulSet:         c.GetStatefulSet(),
 		PodDisruptionBudget: c.GetPodDisruptionBudget(),
 		CurrentProcess:      c.GetCurrentProcess(),
 
 		Error: fmt.Errorf("error: %s", c.Error),
 	}
+
+	if !c.patroniKubernetesUseConfigMaps() {
+		status.MasterEndpoint = c.GetEndpointMaster()
+		status.ReplicaEndpoint = c.GetEndpointReplica()
+	}
+
+	return status
 }
 
 // Switchover does a switchover (via Patroni) to a candidate pod
@@ -1562,7 +1568,7 @@ func (c *Cluster) Unlock() {
 	c.mu.Unlock()
 }
 
-type simpleActionWithResult func() error
+type simpleActionWithResult func()
 
 type clusterObjectGet func(name string) (spec.NamespacedName, error)
 
@@ -1576,46 +1582,47 @@ func (c *Cluster) deletePatroniClusterObjects() error {
 		c.logger.Infof("not cleaning up Etcd Patroni objects on cluster delete")
 	}
 
-	if !c.patroniKubernetesUseConfigMaps() {
-		actionsList = append(actionsList, c.deletePatroniClusterEndpoints)
+	actionsList = append(actionsList, c.deletePatroniClusterServices)
+	if c.patroniKubernetesUseConfigMaps() {
+		actionsList = append(actionsList, c.deletePatroniClusterConfigMaps)
 	} else {
-		actionsList = append(actionsList, c.deletePatroniClusterServices, c.deletePatroniClusterConfigMaps)
+		actionsList = append(actionsList, c.deletePatroniClusterEndpoints)
 	}
 
 	c.logger.Debugf("removing leftover Patroni objects (endpoints / services and configmaps)")
 	for _, deleter := range actionsList {
-		if err := deleter(); err != nil {
-			return err
-		}
+		deleter()
 	}
 	return nil
 }
 
-func (c *Cluster) deleteClusterObject(
+func deleteClusterObject(
 	get clusterObjectGet,
 	del clusterObjectDelete,
-	objType string) error {
+	objType string,
+	clusterName string,
+	logger *logrus.Entry) {
 	for _, suffix := range patroniObjectSuffixes {
-		name := fmt.Sprintf("%s-%s", c.Name, suffix)
+		name := fmt.Sprintf("%s-%s", clusterName, suffix)
 
-		if namespacedName, err := get(name); err == nil {
-			c.logger.Debugf("deleting Patroni cluster object %q with name %q",
+		namespacedName, err := get(name)
+		if err == nil {
+			logger.Debugf("deleting %s %q",
 				objType, namespacedName)
 
 			if err = del(name); err != nil {
-				return fmt.Errorf("could not delete Patroni cluster object %q with name %q: %v",
+				logger.Warningf("could not delete %s %q: %v",
 					objType, namespacedName, err)
 			}
 
 		} else if !k8sutil.ResourceNotFound(err) {
-			return fmt.Errorf("could not fetch Patroni Endpoint %q: %v",
-				namespacedName, err)
+			logger.Warningf("could not fetch %s %q: %v",
+				objType, namespacedName, err)
 		}
 	}
-	return nil
 }
 
-func (c *Cluster) deletePatroniClusterServices() error {
+func (c *Cluster) deletePatroniClusterServices() {
 	get := func(name string) (spec.NamespacedName, error) {
 		svc, err := c.KubeClient.Services(c.Namespace).Get(context.TODO(), name, metav1.GetOptions{})
 		return util.NameFromMeta(svc.ObjectMeta), err
@@ -1625,10 +1632,10 @@ func (c *Cluster) deletePatroniClusterServices() error {
 		return c.KubeClient.Services(c.Namespace).Delete(context.TODO(), name, c.deleteOptions)
 	}
 
-	return c.deleteClusterObject(get, deleteServiceFn, "service")
+	deleteClusterObject(get, deleteServiceFn, "service", c.Name, c.logger)
 }
 
-func (c *Cluster) deletePatroniClusterEndpoints() error {
+func (c *Cluster) deletePatroniClusterEndpoints() {
 	get := func(name string) (spec.NamespacedName, error) {
 		ep, err := c.KubeClient.Endpoints(c.Namespace).Get(context.TODO(), name, metav1.GetOptions{})
 		return util.NameFromMeta(ep.ObjectMeta), err
@@ -1638,10 +1645,10 @@ func (c *Cluster) deletePatroniClusterEndpoints() error {
 		return c.KubeClient.Endpoints(c.Namespace).Delete(context.TODO(), name, c.deleteOptions)
 	}
 
-	return c.deleteClusterObject(get, deleteEndpointFn, "endpoint")
+	deleteClusterObject(get, deleteEndpointFn, "endpoint", c.Name, c.logger)
 }
 
-func (c *Cluster) deletePatroniClusterConfigMaps() error {
+func (c *Cluster) deletePatroniClusterConfigMaps() {
 	get := func(name string) (spec.NamespacedName, error) {
 		cm, err := c.KubeClient.ConfigMaps(c.Namespace).Get(context.TODO(), name, metav1.GetOptions{})
 		return util.NameFromMeta(cm.ObjectMeta), err
@@ -1651,5 +1658,5 @@ func (c *Cluster) deletePatroniClusterConfigMaps() error {
 		return c.KubeClient.ConfigMaps(c.Namespace).Delete(context.TODO(), name, c.deleteOptions)
 	}
 
-	return c.deleteClusterObject(get, deleteConfigMapFn, "configmap")
+	deleteClusterObject(get, deleteConfigMapFn, "configmap", c.Name, c.logger)
 }
diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go
index e545be7ef..625c6a813 100644
--- a/pkg/cluster/k8sres.go
+++ b/pkg/cluster/k8sres.go
@@ -26,6 +26,7 @@ import (
 	"github.com/zalando/postgres-operator/pkg/util/config"
 	"github.com/zalando/postgres-operator/pkg/util/constants"
 	"github.com/zalando/postgres-operator/pkg/util/k8sutil"
+	"github.com/zalando/postgres-operator/pkg/util/patroni"
 	"github.com/zalando/postgres-operator/pkg/util/retryutil"
 	batchv1 "k8s.io/api/batch/v1"
 	batchv1beta1 "k8s.io/api/batch/v1beta1"
@@ -111,7 +112,7 @@ func (c *Cluster) servicePort(role PostgresRole) int32 {
 		return service.Spec.Ports[0].Port
 	}
 
-	c.logger.Warningf("No service for role %s - defaulting to port 5432", role)
+	c.logger.Warningf("No service for role %s - defaulting to port %d", role, pgPort)
 	return pgPort
 }
 
@@ -558,15 +559,15 @@ func generateContainer(
 		Resources:       *resourceRequirements,
 		Ports: []v1.ContainerPort{
 			{
-				ContainerPort: 8008,
+				ContainerPort: patroni.ApiPort,
 				Protocol:      v1.ProtocolTCP,
 			},
 			{
-				ContainerPort: 5432,
+				ContainerPort: pgPort,
 				Protocol:      v1.ProtocolTCP,
 			},
 			{
-				ContainerPort: 8080,
+				ContainerPort: patroni.ApiPort,
 				Protocol:      v1.ProtocolTCP,
 			},
 		},
@@ -1058,6 +1059,22 @@ func extractPgVersionFromBinPath(binPath string, template string) (string, error
 	return fmt.Sprintf("%v", pgVersion), nil
 }
 
+func generateSpiloReadinessProbe() *v1.Probe {
+	return &v1.Probe{
+		Handler: v1.Handler{
+			HTTPGet: &v1.HTTPGetAction{
+				Path: "/readiness",
+				Port: intstr.IntOrString{IntVal: patroni.ApiPort},
+			},
+		},
+		InitialDelaySeconds: 6,
+		PeriodSeconds:       10,
+		TimeoutSeconds:      5,
+		SuccessThreshold:    1,
+		FailureThreshold:    3,
+	}
+}
+
 func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.StatefulSet, error) {
 
 	var (
@@ -1239,6 +1256,9 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
 		generateCapabilities(c.OpConfig.AdditionalPodCapabilities),
 	)
 
+	// Patroni responds 200 to probe only if it either owns the leader lock or postgres is running and DCS is accessible
+	spiloContainer.ReadinessProbe = generateSpiloReadinessProbe()
+
 	// generate container specs for sidecars specified in the cluster manifest
 	clusterSpecificSidecars := []v1.Container{}
 	if spec.Sidecars != nil && len(spec.Sidecars) > 0 {
@@ -1708,10 +1728,12 @@ func (c *Cluster) shouldCreateLoadBalancerForService(role PostgresRole, spec *ac
 
 func (c *Cluster) generateService(role PostgresRole, spec *acidv1.PostgresSpec) *v1.Service {
 	serviceSpec := v1.ServiceSpec{
-		Ports: []v1.ServicePort{{Name: "postgresql", Port: 5432, TargetPort: intstr.IntOrString{IntVal: 5432}}},
+		Ports: []v1.ServicePort{{Name: "postgresql", Port: pgPort, TargetPort: intstr.IntOrString{IntVal: pgPort}}},
 		Type:  v1.ServiceTypeClusterIP,
 	}
 
+	// no selector for master, see https://github.com/zalando/postgres-operator/issues/340
+	// if kubernetes_use_configmaps is set master service needs a selector
 	if role == Replica || c.patroniKubernetesUseConfigMaps() {
 		serviceSpec.Selector = c.roleLabelsSet(false, role)
 	}
@@ -1989,7 +2011,7 @@ func (c *Cluster) generatePodDisruptionBudget() *policybeta1.PodDisruptionBudget
 // TODO: handle clusters in different namespaces
 func (c *Cluster) getClusterServiceConnectionParameters(clusterName string) (host string, port string) {
 	host = clusterName
-	port = "5432"
+	port = fmt.Sprintf("%d", pgPort)
 	return
 }
 
@@ -2170,7 +2192,7 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar {
 		},
 		{
 			Name:  "PGPORT",
-			Value: "5432",
+			Value: fmt.Sprintf("%d", pgPort),
 		},
 		{
 			Name:  "PGUSER",
diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go
index 5e5c6156e..9a57051df 100644
--- a/pkg/cluster/resources.go
+++ b/pkg/cluster/resources.go
@@ -35,8 +35,10 @@ func (c *Cluster) listResources() error {
 		c.logger.Infof("found secret: %q (uid: %q) namesapce: %s", util.NameFromMeta(obj.ObjectMeta), obj.UID, obj.ObjectMeta.Namespace)
 	}
 
-	for role, endpoint := range c.Endpoints {
-		c.logger.Infof("found %s endpoint: %q (uid: %q)", role, util.NameFromMeta(endpoint.ObjectMeta), endpoint.UID)
+	if !c.patroniKubernetesUseConfigMaps() {
+		for role, endpoint := range c.Endpoints {
+			c.logger.Infof("found %s endpoint: %q (uid: %q)", role, util.NameFromMeta(endpoint.ObjectMeta), endpoint.UID)
+		}
 	}
 
 	for role, service := range c.Services {
@@ -589,7 +591,7 @@ func (c *Cluster) GetEndpointMaster() *v1.Endpoints {
 	return c.Endpoints[Master]
 }
 
-// GetEndpointReplica returns cluster's kubernetes master Endpoint
+// GetEndpointReplica returns cluster's kubernetes replica Endpoint
 func (c *Cluster) GetEndpointReplica() *v1.Endpoints {
 	return c.Endpoints[Replica]
 }
diff --git a/pkg/controller/postgresql.go b/pkg/controller/postgresql.go
index 590494412..aac078933 100644
--- a/pkg/controller/postgresql.go
+++ b/pkg/controller/postgresql.go
@@ -544,7 +544,8 @@ func (c *Controller) postgresqlCheck(obj interface{}) *acidv1.Postgresql {
   Ensures the pod service account and role bindings exists in a namespace
   before a PG cluster is created there so that a user does not have to deploy
   these credentials manually.  StatefulSets require the service account to
-  create pods; Patroni requires relevant RBAC bindings to access endpoints.
+  create pods; Patroni requires relevant RBAC bindings to access endpoints
+  or config maps.
 
   The operator does not sync accounts/role bindings after creation.
 */
diff --git a/pkg/util/patroni/patroni.go b/pkg/util/patroni/patroni.go
index d3b2f28f0..8126eddc7 100644
--- a/pkg/util/patroni/patroni.go
+++ b/pkg/util/patroni/patroni.go
@@ -25,7 +25,7 @@ const (
 	clusterPath  = "/cluster"
 	statusPath   = "/patroni"
 	restartPath  = "/restart"
-	apiPort      = 8008
+	ApiPort      = 8008
 	timeout      = 30 * time.Second
 )
 
@@ -74,7 +74,7 @@ func apiURL(masterPod *v1.Pod) (string, error) {
 			return "", fmt.Errorf("%s is not a valid IPv4/IPv6 address", masterPod.Status.PodIP)
 		}
 	}
-	return fmt.Sprintf("http://%s", net.JoinHostPort(ip.String(), strconv.Itoa(apiPort))), nil
+	return fmt.Sprintf("http://%s", net.JoinHostPort(ip.String(), strconv.Itoa(ApiPort))), nil
 }
 
 func (p *Patroni) httpPostOrPatch(method string, url string, body *bytes.Buffer) (err error) {
diff --git a/pkg/util/patroni/patroni_test.go b/pkg/util/patroni/patroni_test.go
index 60f289c6f..aa6ad9206 100644
--- a/pkg/util/patroni/patroni_test.go
+++ b/pkg/util/patroni/patroni_test.go
@@ -36,17 +36,17 @@ func TestApiURL(t *testing.T) {
 	}{
 		{
 			"127.0.0.1",
-			fmt.Sprintf("http://127.0.0.1:%d", apiPort),
+			fmt.Sprintf("http://127.0.0.1:%d", ApiPort),
 			nil,
 		},
 		{
 			"0000:0000:0000:0000:0000:0000:0000:0001",
-			fmt.Sprintf("http://[::1]:%d", apiPort),
+			fmt.Sprintf("http://[::1]:%d", ApiPort),
 			nil,
 		},
 		{
 			"::1",
-			fmt.Sprintf("http://[::1]:%d", apiPort),
+			fmt.Sprintf("http://[::1]:%d", ApiPort),
 			nil,
 		},
 		{

From 2333d531d34643a01a364242d567925e1b29cd33 Mon Sep 17 00:00:00 2001
From: Felix Kunde <felix-kunde@gmx.de>
Date: Thu, 31 Mar 2022 11:48:37 +0200
Subject: [PATCH 46/58] Fix deletion of event streams resources (#1831)

* fix deletion of event streams
* create cluster field to store stream application ids
---
 pkg/cluster/cluster.go      |  1 +
 pkg/cluster/streams.go      | 23 +++++++++++++++++------
 pkg/cluster/streams_test.go |  7 +++++++
 3 files changed, 25 insertions(+), 6 deletions(-)

diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go
index 2a08857e3..4d0fcb1c0 100644
--- a/pkg/cluster/cluster.go
+++ b/pkg/cluster/cluster.go
@@ -91,6 +91,7 @@ type Cluster struct {
 	currentProcess      Process
 	processMu           sync.RWMutex // protects the current operation for reporting, no need to hold the master mutex
 	specMu              sync.RWMutex // protects the spec for reporting, no need to hold the master mutex
+	streamApplications  []string
 	ConnectionPooler    map[PostgresRole]*ConnectionPoolerObjects
 	EBSVolumes          map[string]volumes.VolumeProperties
 	VolumeResizer       volumes.VolumeResizer
diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go
index 7325fa857..0236925ca 100644
--- a/pkg/cluster/streams.go
+++ b/pkg/cluster/streams.go
@@ -45,9 +45,17 @@ func (c *Cluster) deleteStreams() error {
 		return nil
 	}
 
-	err = c.KubeClient.FabricEventStreams(c.Namespace).Delete(context.TODO(), c.Name, metav1.DeleteOptions{})
-	if err != nil {
-		return fmt.Errorf("could not delete event stream custom resource: %v", err)
+	errors := make([]string, 0)
+	for _, appId := range c.streamApplications {
+		fesName := fmt.Sprintf("%s-%s", c.Name, appId)
+		err = c.KubeClient.FabricEventStreams(c.Namespace).Delete(context.TODO(), fesName, metav1.DeleteOptions{})
+		if err != nil {
+			errors = append(errors, fmt.Sprintf("could not delete event stream %q: %v", fesName, err))
+		}
+	}
+
+	if len(errors) > 0 {
+		return fmt.Errorf("could not delete all event stream custom resources: %v", strings.Join(errors, `', '`))
 	}
 
 	return nil
@@ -265,6 +273,11 @@ func (c *Cluster) syncStreams() error {
 		return nil
 	}
 
+	// fetch different application IDs from streams section
+	// there will be a separate event stream resource for each ID
+	appIds := gatherApplicationIds(c.Spec.Streams)
+	c.streamApplications = appIds
+
 	slots := make(map[string]map[string]string)
 	publications := make(map[string]map[string]acidv1.StreamTable)
 
@@ -329,9 +342,7 @@ func (c *Cluster) syncStreams() error {
 }
 
 func (c *Cluster) createOrUpdateStreams() error {
-
-	appIds := gatherApplicationIds(c.Spec.Streams)
-	for _, appId := range appIds {
+	for _, appId := range c.streamApplications {
 		fesName := fmt.Sprintf("%s-%s", c.Name, appId)
 		effectiveStreams, err := c.KubeClient.FabricEventStreams(c.Namespace).Get(context.TODO(), fesName, metav1.GetOptions{})
 		if err != nil {
diff --git a/pkg/cluster/streams_test.go b/pkg/cluster/streams_test.go
index 7ebb1c89a..0094708a4 100644
--- a/pkg/cluster/streams_test.go
+++ b/pkg/cluster/streams_test.go
@@ -196,6 +196,9 @@ func TestGenerateFabricEventStream(t *testing.T) {
 	_, err := cluster.createStatefulSet()
 	assert.NoError(t, err)
 
+	// createOrUpdateStreams will loop over existing apps
+	cluster.streamApplications = []string{appId}
+
 	// create the streams
 	err = cluster.createOrUpdateStreams()
 	assert.NoError(t, err)
@@ -327,6 +330,10 @@ func TestUpdateFabricEventStream(t *testing.T) {
 	_, err := cluster.KubeClient.Postgresqls(namespace).Create(
 		context.TODO(), &pg, metav1.CreateOptions{})
 	assert.NoError(t, err)
+
+	// createOrUpdateStreams will loop over existing apps
+	cluster.streamApplications = []string{appId}
+
 	err = cluster.createOrUpdateStreams()
 	assert.NoError(t, err)
 

From f5cca1a093d90ec5874edc410447e3fae64ba1e4 Mon Sep 17 00:00:00 2001
From: neelasha-09 <66790082+neelasha-09@users.noreply.github.com>
Date: Thu, 31 Mar 2022 18:23:02 +0530
Subject: [PATCH 47/58] major version upgrade for rootless and ocp : solving
 #1689 (#1770)

* major version upgrade for rootless and ocp : solving #1689

Co-authored-by: Felix Kunde <felix-kunde@gmx.de>
---
 pkg/cluster/majorversionupgrade.go | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/pkg/cluster/majorversionupgrade.go b/pkg/cluster/majorversionupgrade.go
index 36def164c..d42f2c93d 100644
--- a/pkg/cluster/majorversionupgrade.go
+++ b/pkg/cluster/majorversionupgrade.go
@@ -2,6 +2,7 @@ package cluster
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/zalando/postgres-operator/pkg/spec"
 	"github.com/zalando/postgres-operator/pkg/util"
@@ -105,14 +106,28 @@ func (c *Cluster) majorVersionUpgrade() error {
 			c.logger.Infof("triggering major version upgrade on pod %s of %d pods", masterPod.Name, numberOfPods)
 			c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Major Version Upgrade", "Starting major version upgrade on pod %s of %d pods", masterPod.Name, numberOfPods)
 			upgradeCommand := fmt.Sprintf("/usr/bin/python3 /scripts/inplace_upgrade.py %d 2>&1 | tee last_upgrade.log", numberOfPods)
-
-			result, err := c.ExecCommand(podName, "/bin/su", "postgres", "-c", upgradeCommand)
-			if err != nil {
-				c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Major Version Upgrade", "Upgrade from %d to %d FAILED: %v", c.currentMajorVersion, desiredVersion, err)
-				return err
+			
+			c.logger.Debugf("checking if the spilo image runs with root or non-root (check for user id=0)")
+			resultIdCheck, errIdCheck := c.ExecCommand(podName, "/bin/bash", "-c", "/usr/bin/id -u")
+			if errIdCheck != nil {
+				c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "Major Version Upgrade", "Checking user id to run upgrade from %d to %d FAILED: %v", c.currentMajorVersion, desiredVersion, errIdCheck)
 			}
 
+			resultIdCheck = strings.TrimSuffix(resultIdCheck, "\n")
+			var result string
+			if resultIdCheck != "0" {
+				c.logger.Infof("User id was identified as: %s, hence default user is non-root already", resultIdCheck)
+				result, err = c.ExecCommand(podName, "/bin/bash", "-c", upgradeCommand)
+			} else {
+				c.logger.Infof("User id was identified as: %s, using su to reach the postgres user", resultIdCheck)
+				result, err = c.ExecCommand(podName, "/bin/su", "postgres", "-c", upgradeCommand)
+			}
+			if err != nil {
+				c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "Major Version Upgrade", "Upgrade from %d to %d FAILED: %v", c.currentMajorVersion, desiredVersion, err)
+				return err
+			}
 			c.logger.Infof("upgrade action triggered and command completed: %s", result[:100])
+
 			c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Major Version Upgrade", "Upgrade from %d to %d finished", c.currentMajorVersion, desiredVersion)
 		}
 	}

From b5d1f179294c9ed94642c3ed3aede466cec1050e Mon Sep 17 00:00:00 2001
From: Felix Kunde <felix-kunde@gmx.de>
Date: Thu, 31 Mar 2022 14:55:19 +0200
Subject: [PATCH 48/58] add chapter about restoring in place (#1833)

---
 docs/user.md                              | 50 +++++++++++++++++++++--
 manifests/complete-postgres-manifest.yaml |  2 +-
 2 files changed, 47 insertions(+), 5 deletions(-)

diff --git a/docs/user.md b/docs/user.md
index 20db45979..fbacdb94e 100644
--- a/docs/user.md
+++ b/docs/user.md
@@ -737,10 +737,14 @@ source cluster. If you create it in the same Kubernetes environment, use a
 different name.
 
 ```yaml
+apiVersion: "acid.zalan.do/v1"
+kind: postgresql
+metadata:
+  name: acid-minimal-cluster-clone
 spec:
   clone:
     uid: "efd12e58-5786-11e8-b5a7-06148230260c"
-    cluster: "acid-batman"
+    cluster: "acid-minimal-cluster"
     timestamp: "2017-12-19T12:40:33+01:00"
     s3_wal_path: "s3://<bucketname>/spilo/<source_db_cluster>/<UID>/wal/<PGVERSION>"
 ```
@@ -756,7 +760,7 @@ specified `uid`. You can find the UID of the source cluster in its metadata:
 apiVersion: acid.zalan.do/v1
 kind: postgresql
 metadata:
-  name: acid-batman
+  name: acid-minimal-cluster
   uid: efd12e58-5786-11e8-b5a7-06148230260c
 ```
 
@@ -767,7 +771,7 @@ implementations:
 spec:
   clone:
     uid: "efd12e58-5786-11e8-b5a7-06148230260c"
-    cluster: "acid-batman"
+    cluster: "acid-minimal-cluster"
     timestamp: "2017-12-19T12:40:33+01:00"
     s3_endpoint: https://s3.acme.org
     s3_access_key_id: 0123456789abcdef0123456789abcdef
@@ -788,11 +792,49 @@ namespace.
 ```yaml
 spec:
   clone:
-    cluster: "acid-batman"
+    cluster: "acid-minimal-cluster"
 ```
 
 Be aware that on a busy source database this can result in an elevated load!
 
+## Restore in place
+
+There is also a possibility to restore a database without cloning it. The
+advantage to this is that there is no need to change anything on the
+application side. However, as it involves deleting the database first, this
+process is of course riskier than cloning (which involves adjusting the
+connection parameters of the app).
+
+First, make sure there is no writing activity on your DB, and save the UID.
+Then delete the `postgresql` K8S resource:
+
+```bash
+zkubectl delete postgresql acid-test-restore
+```
+
+Then deploy a new manifest with the same name, referring to itself
+(both name and UID) in the `clone` section:
+
+```yaml
+metadata:
+  name: acid-minimal-cluster
+  # [...]
+spec:
+  # [...]
+  clone:
+    cluster: "acid-minimal-cluster"  # the same as metadata.name above!
+    uid: "<original_UID>"
+    timestamp: "2022-04-01T10:11:12.000+00:00"
+```
+
+This will create a new database cluster with the same name but different UID,
+whereas the database will be in the state it was at the specified time.
+
+:warning: The backups and WAL files for the original DB are retained under the
+original UID, making it possible retry restoring. However, it is probably
+better to create a temporary clone for experimenting or finding out to which
+point you should restore.
+
 ## Setting up a standby cluster
 
 Standby cluster is a [Patroni feature](https://github.com/zalando/patroni/blob/master/docs/replica_bootstrap.rst#standby-cluster)
diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml
index cb2a8ee1f..611e90387 100644
--- a/manifests/complete-postgres-manifest.yaml
+++ b/manifests/complete-postgres-manifest.yaml
@@ -132,7 +132,7 @@ spec:
 # with an empty/absent timestamp, clone from an existing alive cluster using pg_basebackup
 #  clone:
 #    uid: "efd12e58-5786-11e8-b5a7-06148230260c"
-#    cluster: "acid-batman"
+#    cluster: "acid-minimal-cluster"
 #    timestamp: "2017-12-19T12:40:33+01:00"  # timezone required (offset relative to UTC, see RFC 3339 section 5.6)
 #    s3_wal_path: "s3://custom/path/to/bucket"
 

From 2dfb11ad4c35f7287b39e0944195c6773f8c5420 Mon Sep 17 00:00:00 2001
From: Jociele Padilha <45459238+jopadi@users.noreply.github.com>
Date: Mon, 4 Apr 2022 13:08:03 +0200
Subject: [PATCH 49/58] update team message (#1818)

* return only warning if team can't be found

Co-authored-by: Jociele Padilha <jociele.padilha@zalando.de>
Co-authored-by: Felix Kunde <felix-kunde@gmx.de>
---
 pkg/cluster/util.go | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go
index 5f9a0b379..cf320e057 100644
--- a/pkg/cluster/util.go
+++ b/pkg/cluster/util.go
@@ -274,15 +274,14 @@ func (c *Cluster) getTeamMembers(teamID string) ([]string, error) {
 
 	teamInfo, err := c.teamsAPIClient.TeamInfo(teamID, token)
 	if err != nil {
-		return nil, fmt.Errorf("could not get team info for team %q: %v", teamID, err)
-	}
-
-	for _, member := range teamInfo.Members {
-		if !(util.SliceContains(members, member)) {
-			members = append(members, member)
+		c.logger.Warningf("could not get team info for team %q: %v", teamID, err)
+	} else {
+		for _, member := range teamInfo.Members {
+			if !(util.SliceContains(members, member)) {
+				members = append(members, member)
+			}
 		}
 	}
-
 	return members, nil
 }
 

From 0dc370f15d5fd348937dd4eb631852e55df33e57 Mon Sep 17 00:00:00 2001
From: Felix Kunde <felix-kunde@gmx.de>
Date: Mon, 4 Apr 2022 15:41:11 +0200
Subject: [PATCH 50/58] standby cluster that streams from a remote primary
 (#1830)

* add the possibility to create a standby cluster that streams from a remote primary
* extending unit tests
* add more docs and e2e test

Co-authored-by: machine424 <ayoubmrini424@gmail.com>
---
 .../postgres-operator/crds/postgresqls.yaml   |  11 ++
 docs/administrator.md                         |  16 ++-
 docs/reference/cluster_manifest.md            |  14 ++-
 docs/user.md                                  |  58 +++++----
 e2e/tests/k8s_api.py                          |  12 +-
 e2e/tests/test_e2e.py                         |  52 ++++++--
 manifests/postgresql.crd.yaml                 |  11 ++
 manifests/standby-manifest.yaml               |   6 +-
 pkg/apis/acid.zalan.do/v1/crds.go             |  11 ++
 pkg/apis/acid.zalan.do/v1/postgresql_type.go  |   8 +-
 pkg/cluster/k8sres.go                         |  77 ++++++------
 pkg/cluster/k8sres_test.go                    | 111 ++++++++++++++++++
 12 files changed, 303 insertions(+), 84 deletions(-)

diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml
index 143f95591..8aa6f70b3 100644
--- a/charts/postgres-operator/crds/postgresqls.yaml
+++ b/charts/postgres-operator/crds/postgresqls.yaml
@@ -460,6 +460,17 @@ spec:
                     type: string
                   gs_wal_path:
                     type: string
+                  standby_host:
+                    type: string
+                  standby_port:
+                    type: string
+                oneOf:
+                - required:
+                  - s3_wal_path
+                - required:
+                  - gs_wal_path
+                - required:
+                  - standby_host
               streams:
                 type: array
                 nullable: true
diff --git a/docs/administrator.md b/docs/administrator.md
index dd6bf59b0..cd4504c31 100644
--- a/docs/administrator.md
+++ b/docs/administrator.md
@@ -1087,12 +1087,16 @@ data:
 
 ### Standby clusters
 
-The setup for [standby clusters](user.md#setting-up-a-standby-cluster) is very
-similar to cloning. At the moment, the operator only allows for streaming from
-the S3 WAL archive of the master specified in the manifest. Like with cloning,
-if you are using [additional environment variables](#custom-pod-environment-variables)
-to access your backup location you have to copy those variables and prepend the
-`STANDBY_` prefix for Spilo to find the backups and WAL files to stream.
+The setup for [standby clusters](user.md#setting-up-a-standby-cluster) is
+similar to cloning when they stream changes from a WAL archive (S3 or GCS).
+If you are using [additional environment variables](#custom-pod-environment-variables)
+to access your backup location you have to copy those variables and prepend
+the `STANDBY_` prefix for Spilo to find the backups and WAL files to stream.
+
+Alternatively, standby clusters can also stream from a remote primary cluster.
+You have to specify the host address. Port is optional and defaults to 5432.
+Note, that only one of the options (`s3_wal_path`, `gs_wal_path`,
+`standby_host`) can be present under the `standby` top-level key.
 
 ## Logical backups
 
diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md
index c69de8d40..b41550e22 100644
--- a/docs/reference/cluster_manifest.md
+++ b/docs/reference/cluster_manifest.md
@@ -395,16 +395,22 @@ under the `clone` top-level key and do not affect the already running cluster.
 ## Standby cluster
 
 On startup, an existing `standby` top-level key creates a standby Postgres
-cluster streaming from a remote location. So far streaming from S3 and GCS WAL
-archives is supported.
+cluster streaming from a remote location - either from a S3 or GCS WAL
+archive or a remote primary. Only one of options is allowed and required
+if the `standby` key is present.
 
 * **s3_wal_path**
   the url to S3 bucket containing the WAL archive of the remote primary.
-  Optional, but `s3_wal_path`  or `gs_wal_path` is required.
 
 * **gs_wal_path**
   the url to GS bucket containing the WAL archive of the remote primary.
-  Optional, but `s3_wal_path`  or `gs_wal_path` is required.
+
+* **standby_host**
+  hostname or IP address of the primary to stream from.
+
+* **standby_port**
+  TCP port on which the primary is listening for connections. Patroni will
+  use `"5432"` if not set.
 
 ## Volume properties
 
diff --git a/docs/user.md b/docs/user.md
index fbacdb94e..9a38844cd 100644
--- a/docs/user.md
+++ b/docs/user.md
@@ -838,15 +838,15 @@ point you should restore.
 ## Setting up a standby cluster
 
 Standby cluster is a [Patroni feature](https://github.com/zalando/patroni/blob/master/docs/replica_bootstrap.rst#standby-cluster)
-that first clones a database, and keeps replicating changes afterwards. As the
-replication is happening by the means of archived WAL files (stored on S3 or
-the equivalent of other cloud providers), the standby cluster can exist in a
-different location than its source database. Unlike cloning, the PostgreSQL
-version between source and target cluster has to be the same.
+that first clones a database, and keeps replicating changes afterwards. It can
+exist in a different location than its source database, but unlike cloning,
+the PostgreSQL version between source and target cluster has to be the same.
 
 To start a cluster as standby, add the following `standby` section in the YAML
-file. Specify the S3/GS bucket path. Omitting both settings will result in an error
-and no statefulset will be created.
+file. You can stream changes from archived WAL files (AWS S3 or Google Cloud
+Storage) or from a remote primary where you specify the host address and port.
+If you leave out the port, Patroni will use `"5432"`. Only one option can be
+specfied in the manifest:
 
 ```yaml
 spec:
@@ -860,32 +860,42 @@ spec:
     gs_wal_path: "gs://<bucketname>/spilo/<source_db_cluster>/<UID>/wal/<PGVERSION>"
 ```
 
-At the moment, the operator only allows to stream from the WAL archive of the
-master. Thus, it is recommended to deploy standby clusters with only [one pod](https://github.com/zalando/postgres-operator/blob/master/manifests/standby-manifest.yaml#L10).
-You can raise the instance count when detaching. Note, that the same pod role
-labels like for normal clusters are used: The standby leader is labeled as
-`master`.
+```yaml
+spec:
+  standby:
+    standby_host: "acid-minimal-cluster.default"
+    standby_port: "5433"
+```
+
+Note, that the pods and services use the same role labels like for normal clusters:
+The standby leader is labeled as `master`. When using the `standby_host` option
+you have to copy the credentials from the source cluster's secrets to successfully
+bootstrap a standby cluster (see next chapter).
 
 ### Providing credentials of source cluster
 
 A standby cluster is replicating the data (including users and passwords) from
 the source database and is read-only. The system and application users (like
 standby, postgres etc.) all have a password that does not match the credentials
-stored in secrets which are created by the operator. One solution is to create
-secrets beforehand and paste in the credentials of the source cluster.
+stored in secrets which are created by the operator. You have two options:
+
+a. Create secrets manually beforehand and paste the credentials of the source
+   cluster
+b. Let the operator create the secrets when it bootstraps the standby cluster.
+   Patch the secrets with the credentials of the source cluster. Replace the
+   spilo pods.
+
 Otherwise, you will see errors in the Postgres logs saying users cannot log in
 and the operator logs will complain about not being able to sync resources.
+If you stream changes from a remote primary you have to align the secrets or
+the standby cluster will not start up.
 
-When you only run a standby leader, you can safely ignore this, as it will be
-sorted out once the cluster is detached from the source. It is also harmless if
-you don’t plan it. But, when you created a standby replica, too, fix the
-credentials right away. WAL files will pile up on the standby leader if no
-connection can be established between standby replica(s). You can also edit the
-secrets after their creation. Find them by:
-
-```bash
-kubectl get secrets --all-namespaces | grep <standby-cluster-name>
-```
+If you stream changes from WAL files and you only run a standby leader, you
+can safely ignore the secret mismatch, as it will be sorted out once the
+cluster is detached from the source. It is also harmless if you do not plan it.
+But, when you create a standby replica, too, fix the credentials right away.
+WAL files will pile up on the standby leader if no connection can be
+established between standby replica(s).
 
 ### Promote the standby
 
diff --git a/e2e/tests/k8s_api.py b/e2e/tests/k8s_api.py
index 4ae6493c5..cf8f47be4 100644
--- a/e2e/tests/k8s_api.py
+++ b/e2e/tests/k8s_api.py
@@ -321,9 +321,15 @@ class K8s:
     def get_cluster_replica_pod(self, labels='application=spilo,cluster-name=acid-minimal-cluster', namespace='default'):
         return self.get_cluster_pod('replica', labels, namespace)
 
-    def get_secret_data(self, username, clustername='acid-minimal-cluster', namespace='default'):
-        return self.api.core_v1.read_namespaced_secret(
-                "{}.{}.credentials.postgresql.acid.zalan.do".format(username.replace("_","-"), clustername), namespace).data
+    def get_secret(self, username, clustername='acid-minimal-cluster', namespace='default'):
+        secret = self.api.core_v1.read_namespaced_secret(
+                "{}.{}.credentials.postgresql.acid.zalan.do".format(username.replace("_","-"), clustername), namespace)
+        secret.metadata.resource_version = None
+        secret.metadata.uid = None
+        return secret
+
+    def create_secret(self, secret, namespace='default'):
+        return self.api.core_v1.create_namespaced_secret(namespace, secret)
 
 class K8sBase:
     '''
diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py
index 0f9515896..9b1cf50b4 100644
--- a/e2e/tests/test_e2e.py
+++ b/e2e/tests/test_e2e.py
@@ -1319,8 +1319,8 @@ class EndToEndTestCase(unittest.TestCase):
         self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
 
         # check if next rotation date was set in secret
-        secret_data = k8s.get_secret_data("zalando")
-        next_rotation_timestamp = datetime.strptime(str(base64.b64decode(secret_data["nextRotation"]), 'utf-8'), "%Y-%m-%dT%H:%M:%SZ")
+        zalando_secret = k8s.get_secret("zalando")
+        next_rotation_timestamp = datetime.strptime(str(base64.b64decode(zalando_secret.data["nextRotation"]), 'utf-8'), "%Y-%m-%dT%H:%M:%SZ")
         today90days = today+timedelta(days=90)
         self.assertEqual(today90days, next_rotation_timestamp.date(),
                         "Unexpected rotation date in secret of zalando user: expected {}, got {}".format(today90days, next_rotation_timestamp.date()))
@@ -1361,9 +1361,9 @@ class EndToEndTestCase(unittest.TestCase):
                              "Operator does not get in sync")
 
         # check if next rotation date and username have been replaced
-        secret_data = k8s.get_secret_data("foo_user")
-        secret_username = str(base64.b64decode(secret_data["username"]), 'utf-8')
-        next_rotation_timestamp = datetime.strptime(str(base64.b64decode(secret_data["nextRotation"]), 'utf-8'), "%Y-%m-%dT%H:%M:%SZ")
+        foo_user_secret = k8s.get_secret("foo_user")
+        secret_username = str(base64.b64decode(foo_user_secret.data["username"]), 'utf-8')
+        next_rotation_timestamp = datetime.strptime(str(base64.b64decode(foo_user_secret.data["nextRotation"]), 'utf-8'), "%Y-%m-%dT%H:%M:%SZ")
         rotation_user = "foo_user"+today.strftime("%y%m%d")
         today30days = today+timedelta(days=30)
 
@@ -1396,9 +1396,9 @@ class EndToEndTestCase(unittest.TestCase):
                              "Operator does not get in sync")
 
         # check if username in foo_user secret is reset
-        secret_data = k8s.get_secret_data("foo_user")
-        secret_username = str(base64.b64decode(secret_data["username"]), 'utf-8')
-        next_rotation_timestamp = str(base64.b64decode(secret_data["nextRotation"]), 'utf-8')
+        foo_user_secret = k8s.get_secret("foo_user")
+        secret_username = str(base64.b64decode(foo_user_secret.data["username"]), 'utf-8')
+        next_rotation_timestamp = str(base64.b64decode(foo_user_secret.data["nextRotation"]), 'utf-8')
         self.assertEqual("foo_user", secret_username,
                         "Unexpected username in secret of foo_user: expected {}, got {}".format("foo_user", secret_username))
         self.assertEqual('', next_rotation_timestamp,
@@ -1644,6 +1644,42 @@ class EndToEndTestCase(unittest.TestCase):
         self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
         self.eventuallyTrue(lambda: k8s.check_statefulset_annotations(cluster_label, annotations), "Annotations missing")
 
+    @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
+    def test_standby_cluster(self):
+        '''
+        Create standby cluster streaming from remote primary
+        '''
+        k8s = self.k8s
+        standby_cluster_name = 'acid-standby-cluster'
+        cluster_name_label = 'cluster-name'
+        cluster_label = 'application=spilo,{}={}'.format(cluster_name_label, standby_cluster_name)
+        superuser_name = 'postgres'
+        replication_user = 'standby'
+        secret_suffix = 'credentials.postgresql.acid.zalan.do'
+
+        # copy secrets from remote cluster before operator creates them when bootstrapping the standby cluster
+        postgres_secret = k8s.get_secret(superuser_name)
+        postgres_secret.metadata.name = '{}.{}.{}'.format(superuser_name, standby_cluster_name, secret_suffix)
+        postgres_secret.metadata.labels[cluster_name_label] = standby_cluster_name
+        k8s.create_secret(postgres_secret)
+        standby_secret = k8s.get_secret(replication_user)
+        standby_secret.metadata.name = '{}.{}.{}'.format(replication_user, standby_cluster_name, secret_suffix)
+        standby_secret.metadata.labels[cluster_name_label] = standby_cluster_name
+        k8s.create_secret(standby_secret)
+
+        try:
+            k8s.create_with_kubectl("manifests/standby-manifest.yaml")
+            k8s.wait_for_pod_start("spilo-role=master," + cluster_label)
+
+        except timeout_decorator.TimeoutError:
+            print('Operator log: {}'.format(k8s.get_operator_log()))
+            raise
+        finally:
+            # delete the standby cluster so that the k8s_api.get_operator_state works correctly in subsequent tests
+            k8s.api.custom_objects_api.delete_namespaced_custom_object(
+                "acid.zalan.do", "v1", "default", "postgresqls", "acid-standby-cluster")
+            time.sleep(5)
+
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
     def test_taint_based_eviction(self):
         '''
diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml
index 93f5c1325..e73dc9ad1 100644
--- a/manifests/postgresql.crd.yaml
+++ b/manifests/postgresql.crd.yaml
@@ -458,6 +458,17 @@ spec:
                     type: string
                   gs_wal_path:
                     type: string
+                  standby_host:
+                    type: string
+                  standby_port:
+                    type: string
+                oneOf:
+                - required:
+                  - s3_wal_path
+                - required:
+                  - gs_wal_path
+                - required:
+                  - standby_host
               streams:
                 type: array
                 nullable: true
diff --git a/manifests/standby-manifest.yaml b/manifests/standby-manifest.yaml
index 3ba8d6b9d..66f515518 100644
--- a/manifests/standby-manifest.yaml
+++ b/manifests/standby-manifest.yaml
@@ -10,6 +10,8 @@ spec:
   numberOfInstances: 1
   postgresql:
     version: "14"
-# Make this a standby cluster and provide the s3 bucket path of source cluster for continuous streaming.
+  # Make this a standby cluster and provide either the s3 bucket path of source cluster or the remote primary host for continuous streaming.
   standby:
-    s3_wal_path: "s3://path/to/bucket/containing/wal/of/source/cluster/"
+    # s3_wal_path: "s3://mybucket/spilo/acid-minimal-cluster/abcd1234-2a4b-4b2a-8c9c-c1234defg567/wal/14/"
+    standby_host: "acid-minimal-cluster.default"
+    # standby_port: "5432"
diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go
index 77828255d..7fd927bb0 100644
--- a/pkg/apis/acid.zalan.do/v1/crds.go
+++ b/pkg/apis/acid.zalan.do/v1/crds.go
@@ -714,6 +714,17 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 							"gs_wal_path": {
 								Type: "string",
 							},
+							"standby_host": {
+								Type: "string",
+							},
+							"standby_port": {
+								Type: "string",
+							},
+						},
+						OneOf: []apiextv1.JSONSchemaProps{
+							apiextv1.JSONSchemaProps{Required: []string{"s3_wal_path"}},
+							apiextv1.JSONSchemaProps{Required: []string{"gs_wal_path"}},
+							apiextv1.JSONSchemaProps{Required: []string{"standby_host"}},
 						},
 					},
 					"streams": {
diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go
index 14b16541c..b7c41be58 100644
--- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go
+++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go
@@ -170,10 +170,12 @@ type Patroni struct {
 	SynchronousNodeCount  uint32                       `json:"synchronous_node_count,omitempty" defaults:"1"`
 }
 
-// StandbyDescription contains s3 wal path
+// StandbyDescription contains remote primary config or s3/gs wal path
 type StandbyDescription struct {
-	S3WalPath string `json:"s3_wal_path,omitempty"`
-	GSWalPath string `json:"gs_wal_path,omitempty"`
+	S3WalPath   string `json:"s3_wal_path,omitempty"`
+	GSWalPath   string `json:"gs_wal_path,omitempty"`
+	StandbyHost string `json:"standby_host,omitempty"`
+	StandbyPort string `json:"standby_port,omitempty"`
 }
 
 // TLSDescription specs TLS properties
diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go
index 625c6a813..382abd1e2 100644
--- a/pkg/cluster/k8sres.go
+++ b/pkg/cluster/k8sres.go
@@ -763,7 +763,12 @@ func (c *Cluster) generatePodTemplate(
 }
 
 // generatePodEnvVars generates environment variables for the Spilo Pod
-func (c *Cluster) generateSpiloPodEnvVars(uid types.UID, spiloConfiguration string, cloneDescription *acidv1.CloneDescription, standbyDescription *acidv1.StandbyDescription, customPodEnvVarsList []v1.EnvVar) []v1.EnvVar {
+func (c *Cluster) generateSpiloPodEnvVars(
+	uid types.UID,
+	spiloConfiguration string,
+	cloneDescription *acidv1.CloneDescription,
+	standbyDescription *acidv1.StandbyDescription,
+	customPodEnvVarsList []v1.EnvVar) []v1.EnvVar {
 	envVars := []v1.EnvVar{
 		{
 			Name:  "SCOPE",
@@ -1128,11 +1133,6 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
 	sort.Slice(customPodEnvVarsList,
 		func(i, j int) bool { return customPodEnvVarsList[i].Name < customPodEnvVarsList[j].Name })
 
-	if spec.StandbyCluster != nil && spec.StandbyCluster.S3WalPath == "" &&
-		spec.StandbyCluster.GSWalPath == "" {
-		return nil, fmt.Errorf("one of s3_wal_path or gs_wal_path must be set for standby cluster")
-	}
-
 	// backward compatible check for InitContainers
 	if spec.InitContainersOld != nil {
 		msg := "manifest parameter init_containers is deprecated."
@@ -1944,40 +1944,49 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription)
 func (c *Cluster) generateStandbyEnvironment(description *acidv1.StandbyDescription) []v1.EnvVar {
 	result := make([]v1.EnvVar, 0)
 
-	if description.S3WalPath == "" && description.GSWalPath == "" {
-		return nil
-	}
-
-	if description.S3WalPath != "" {
-		// standby with S3, find out the bucket to setup standby
-		msg := "standby from S3 bucket using custom parsed S3WalPath from the manifest %s "
-		c.logger.Infof(msg, description.S3WalPath)
-
+	if description.StandbyHost != "" {
+		// standby from remote primary
 		result = append(result, v1.EnvVar{
-			Name:  "STANDBY_WALE_S3_PREFIX",
-			Value: description.S3WalPath,
+			Name:  "STANDBY_HOST",
+			Value: description.StandbyHost,
 		})
-	} else if description.GSWalPath != "" {
-		msg := "standby from GS bucket using custom parsed GSWalPath from the manifest %s "
-		c.logger.Infof(msg, description.GSWalPath)
-
-		envs := []v1.EnvVar{
-			{
-				Name:  "STANDBY_WALE_GS_PREFIX",
-				Value: description.GSWalPath,
-			},
-			{
-				Name:  "STANDBY_GOOGLE_APPLICATION_CREDENTIALS",
-				Value: c.OpConfig.GCPCredentials,
-			},
+		if description.StandbyPort != "" {
+			result = append(result, v1.EnvVar{
+				Name:  "STANDBY_PORT",
+				Value: description.StandbyPort,
+			})
 		}
-		result = append(result, envs...)
+	} else {
+		if description.S3WalPath != "" {
+			// standby with S3, find out the bucket to setup standby
+			msg := "Standby from S3 bucket using custom parsed S3WalPath from the manifest %s "
+			c.logger.Infof(msg, description.S3WalPath)
 
+			result = append(result, v1.EnvVar{
+				Name:  "STANDBY_WALE_S3_PREFIX",
+				Value: description.S3WalPath,
+			})
+		} else if description.GSWalPath != "" {
+			msg := "Standby from GS bucket using custom parsed GSWalPath from the manifest %s "
+			c.logger.Infof(msg, description.GSWalPath)
+
+			envs := []v1.EnvVar{
+				{
+					Name:  "STANDBY_WALE_GS_PREFIX",
+					Value: description.GSWalPath,
+				},
+				{
+					Name:  "STANDBY_GOOGLE_APPLICATION_CREDENTIALS",
+					Value: c.OpConfig.GCPCredentials,
+				},
+			}
+			result = append(result, envs...)
+		}
+
+		result = append(result, v1.EnvVar{Name: "STANDBY_METHOD", Value: "STANDBY_WITH_WALE"})
+		result = append(result, v1.EnvVar{Name: "STANDBY_WAL_BUCKET_SCOPE_PREFIX", Value: ""})
 	}
 
-	result = append(result, v1.EnvVar{Name: "STANDBY_METHOD", Value: "STANDBY_WITH_WALE"})
-	result = append(result, v1.EnvVar{Name: "STANDBY_WAL_BUCKET_SCOPE_PREFIX", Value: ""})
-
 	return result
 }
 
diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go
index f27f9b13b..13e5aca67 100644
--- a/pkg/cluster/k8sres_test.go
+++ b/pkg/cluster/k8sres_test.go
@@ -531,6 +531,117 @@ func TestCloneEnv(t *testing.T) {
 	}
 }
 
+func TestStandbyEnv(t *testing.T) {
+	testName := "TestStandbyEnv"
+	tests := []struct {
+		subTest     string
+		standbyOpts *acidv1.StandbyDescription
+		env         v1.EnvVar
+		envPos      int
+		envLen      int
+	}{
+		{
+			subTest: "from custom s3 path",
+			standbyOpts: &acidv1.StandbyDescription{
+				S3WalPath: "s3://some/path/",
+			},
+			env: v1.EnvVar{
+				Name:  "STANDBY_WALE_S3_PREFIX",
+				Value: "s3://some/path/",
+			},
+			envPos: 0,
+			envLen: 3,
+		},
+		{
+			subTest: "from custom gs path",
+			standbyOpts: &acidv1.StandbyDescription{
+				GSWalPath: "gs://some/path/",
+			},
+			env: v1.EnvVar{
+				Name:  "STANDBY_GOOGLE_APPLICATION_CREDENTIALS",
+				Value: "",
+			},
+			envPos: 1,
+			envLen: 4,
+		},
+		{
+			subTest: "ignore gs path if s3 is set",
+			standbyOpts: &acidv1.StandbyDescription{
+				S3WalPath: "s3://some/path/",
+				GSWalPath: "gs://some/path/",
+			},
+			env: v1.EnvVar{
+				Name:  "STANDBY_METHOD",
+				Value: "STANDBY_WITH_WALE",
+			},
+			envPos: 1,
+			envLen: 3,
+		},
+		{
+			subTest: "from remote primary",
+			standbyOpts: &acidv1.StandbyDescription{
+				StandbyHost: "remote-primary",
+			},
+			env: v1.EnvVar{
+				Name:  "STANDBY_HOST",
+				Value: "remote-primary",
+			},
+			envPos: 0,
+			envLen: 1,
+		},
+		{
+			subTest: "from remote primary with port",
+			standbyOpts: &acidv1.StandbyDescription{
+				StandbyHost: "remote-primary",
+				StandbyPort: "9876",
+			},
+			env: v1.EnvVar{
+				Name:  "STANDBY_PORT",
+				Value: "9876",
+			},
+			envPos: 1,
+			envLen: 2,
+		},
+		{
+			subTest: "from remote primary - ignore WAL path",
+			standbyOpts: &acidv1.StandbyDescription{
+				GSWalPath:   "gs://some/path/",
+				StandbyHost: "remote-primary",
+			},
+			env: v1.EnvVar{
+				Name:  "STANDBY_HOST",
+				Value: "remote-primary",
+			},
+			envPos: 0,
+			envLen: 1,
+		},
+	}
+
+	var cluster = New(
+		Config{}, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder)
+
+	for _, tt := range tests {
+		envs := cluster.generateStandbyEnvironment(tt.standbyOpts)
+
+		env := envs[tt.envPos]
+
+		if env.Name != tt.env.Name {
+			t.Errorf("%s %s: Expected env name %s, have %s instead",
+				testName, tt.subTest, tt.env.Name, env.Name)
+		}
+
+		if env.Value != tt.env.Value {
+			t.Errorf("%s %s: Expected env value %s, have %s instead",
+				testName, tt.subTest, tt.env.Value, env.Value)
+		}
+
+		if len(envs) != tt.envLen {
+			t.Errorf("%s %s: Expected number of env variables %d, have %d instead",
+				testName, tt.subTest, tt.envLen, len(envs))
+		}
+	}
+}
+
 func TestExtractPgVersionFromBinPath(t *testing.T) {
 	testName := "TestExtractPgVersionFromBinPath"
 	tests := []struct {

From 7ac9c2a98eb0abfa26fb1d31554560cea86e298f Mon Sep 17 00:00:00 2001
From: Oleg <71509190+Lxrdknows77@users.noreply.github.com>
Date: Tue, 5 Apr 2022 12:07:34 +0300
Subject: [PATCH 51/58] Bump spilo-14:2.1-p4 (#1836)

Co-authored-by: Oleg Galantsev <oleg.galantsev@t-systems.com>
---
 README.md                                                 | 2 +-
 charts/postgres-operator/crds/operatorconfigurations.yaml | 2 +-
 charts/postgres-operator/values.yaml                      | 2 +-
 manifests/complete-postgres-manifest.yaml                 | 2 +-
 manifests/configmap.yaml                                  | 2 +-
 manifests/operatorconfiguration.crd.yaml                  | 2 +-
 manifests/postgresql-operator-default-configuration.yaml  | 2 +-
 pkg/controller/operator_config.go                         | 2 +-
 pkg/util/config/config.go                                 | 2 +-
 9 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md
index c69e0ca29..d51814963 100644
--- a/README.md
+++ b/README.md
@@ -61,7 +61,7 @@ We introduce the major version into the backup path to smoothen the [major versi
 The new operator configuration can set a compatibility flag *enable_spilo_wal_path_compat* to make Spilo look for wal segments in the current path but also old format paths.
 This comes at potential performance costs and should be disabled after a few days.
 
-The newest Spilo image is: `registry.opensource.zalan.do/acid/spilo-14:2.1-p3`
+The newest Spilo image is: `registry.opensource.zalan.do/acid/spilo-14:2.1-p4`
 
 The last Spilo 12 image is: `registry.opensource.zalan.do/acid/spilo-12:1.6-p5`
 
diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml
index 283acee2e..8d4b5c56b 100644
--- a/charts/postgres-operator/crds/operatorconfigurations.yaml
+++ b/charts/postgres-operator/crds/operatorconfigurations.yaml
@@ -68,7 +68,7 @@ spec:
                   type: string
               docker_image:
                 type: string
-                default: "registry.opensource.zalan.do/acid/spilo-14:2.1-p3"
+                default: "registry.opensource.zalan.do/acid/spilo-14:2.1-p4"
               enable_crd_registration:
                 type: boolean
                 default: true
diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml
index 2ce26d18a..67bb94e22 100644
--- a/charts/postgres-operator/values.yaml
+++ b/charts/postgres-operator/values.yaml
@@ -38,7 +38,7 @@ configGeneral:
   # Select if setup uses endpoints (default), or configmaps to manage leader (DCS=k8s)
   # kubernetes_use_configmaps: false
   # Spilo docker image
-  docker_image: registry.opensource.zalan.do/acid/spilo-14:2.1-p3
+  docker_image: registry.opensource.zalan.do/acid/spilo-14:2.1-p4
   # min number of instances in Postgres cluster. -1 = no limit
   min_instances: -1
   # max number of instances in Postgres cluster. -1 = no limit
diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml
index 611e90387..54a3b4249 100644
--- a/manifests/complete-postgres-manifest.yaml
+++ b/manifests/complete-postgres-manifest.yaml
@@ -10,7 +10,7 @@ metadata:
 #    "delete-date": "2020-08-31"  # can only be deleted on that day if "delete-date "key is configured
 #    "delete-clustername": "acid-test-cluster"  # can only be deleted when name matches if "delete-clustername" key is configured
 spec:
-  dockerImage: registry.opensource.zalan.do/acid/spilo-14:2.1-p3
+  dockerImage: registry.opensource.zalan.do/acid/spilo-14:2.1-p4
   teamId: "acid"
   numberOfInstances: 2
   users:  # Application/Robot users
diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml
index 8b392544b..b7d581bc6 100644
--- a/manifests/configmap.yaml
+++ b/manifests/configmap.yaml
@@ -34,7 +34,7 @@ data:
   # default_memory_request: 100Mi
   # delete_annotation_date_key: delete-date
   # delete_annotation_name_key: delete-clustername
-  docker_image: registry.opensource.zalan.do/acid/spilo-14:2.1-p3
+  docker_image: registry.opensource.zalan.do/acid/spilo-14:2.1-p4
   # downscaler_annotations: "deployment-time,downscaler/*"
   # enable_admin_role_for_users: "true"
   # enable_crd_registration: "true"
diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml
index 173e83c51..f43c9ce3e 100644
--- a/manifests/operatorconfiguration.crd.yaml
+++ b/manifests/operatorconfiguration.crd.yaml
@@ -66,7 +66,7 @@ spec:
                   type: string
               docker_image:
                 type: string
-                default: "registry.opensource.zalan.do/acid/spilo-14:2.1-p3"
+                default: "registry.opensource.zalan.do/acid/spilo-14:2.1-p4"
               enable_crd_registration:
                 type: boolean
                 default: true
diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml
index a5ceb8d2a..d1e36545b 100644
--- a/manifests/postgresql-operator-default-configuration.yaml
+++ b/manifests/postgresql-operator-default-configuration.yaml
@@ -3,7 +3,7 @@ kind: OperatorConfiguration
 metadata:
   name: postgresql-operator-default-configuration
 configuration:
-  docker_image: registry.opensource.zalan.do/acid/spilo-14:2.1-p3
+  docker_image: registry.opensource.zalan.do/acid/spilo-14:2.1-p4
   # enable_crd_registration: true
   # crd_categories:
   # - all
diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go
index 8e4ad1f69..e64b290ec 100644
--- a/pkg/controller/operator_config.go
+++ b/pkg/controller/operator_config.go
@@ -38,7 +38,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	result.EnableSpiloWalPathCompat = fromCRD.EnableSpiloWalPathCompat
 	result.EtcdHost = fromCRD.EtcdHost
 	result.KubernetesUseConfigMaps = fromCRD.KubernetesUseConfigMaps
-	result.DockerImage = util.Coalesce(fromCRD.DockerImage, "registry.opensource.zalan.do/acid/spilo-14:2.1-p3")
+	result.DockerImage = util.Coalesce(fromCRD.DockerImage, "registry.opensource.zalan.do/acid/spilo-14:2.1-p4")
 	result.Workers = util.CoalesceUInt32(fromCRD.Workers, 8)
 	result.MinInstances = fromCRD.MinInstances
 	result.MaxInstances = fromCRD.MaxInstances
diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go
index eb2eb07cf..06b55bbd1 100644
--- a/pkg/util/config/config.go
+++ b/pkg/util/config/config.go
@@ -161,7 +161,7 @@ type Config struct {
 	WatchedNamespace        string            `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to'
 	KubernetesUseConfigMaps bool              `name:"kubernetes_use_configmaps" default:"false"`
 	EtcdHost                string            `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS
-	DockerImage             string            `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-14:2.1-p3"`
+	DockerImage             string            `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-14:2.1-p4"`
 	SidecarImages           map[string]string `name:"sidecar_docker_images"` // deprecated in favour of SidecarContainers
 	SidecarContainers       []v1.Container    `name:"sidecars"`
 	PodServiceAccountName   string            `name:"pod_service_account_name" default:"postgres-pod"`

From 43e18052c427e255afaa985a046130fc47cf9264 Mon Sep 17 00:00:00 2001
From: Felix Kunde <felix-kunde@gmx.de>
Date: Tue, 5 Apr 2022 11:08:30 +0200
Subject: [PATCH 52/58] bump pooler image with pgBouncer 1.17.0 and auth_type
 md5 (#1837)

* bump pooler image with pgBouncer 1.17.0 and auth_type md5

* add docs about scram hasher

* only one yaml
---
 .../crds/operatorconfigurations.yaml          |  2 +-
 charts/postgres-operator/values.yaml          |  2 +-
 docs/user.md                                  | 26 ++++++++++++++++---
 manifests/configmap.yaml                      |  2 +-
 manifests/minimal-fake-pooler-deployment.yaml |  2 +-
 manifests/operatorconfiguration.crd.yaml      |  2 +-
 ...gresql-operator-default-configuration.yaml |  2 +-
 7 files changed, 29 insertions(+), 9 deletions(-)

diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml
index 8d4b5c56b..03b5d7629 100644
--- a/charts/postgres-operator/crds/operatorconfigurations.yaml
+++ b/charts/postgres-operator/crds/operatorconfigurations.yaml
@@ -588,7 +588,7 @@ spec:
                     default: "pooler"
                   connection_pooler_image:
                     type: string
-                    default: "registry.opensource.zalan.do/acid/pgbouncer:master-19"
+                    default: "registry.opensource.zalan.do/acid/pgbouncer:master-22"
                   connection_pooler_max_db_connections:
                     type: integer
                     default: 60
diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml
index 67bb94e22..f6c61c89e 100644
--- a/charts/postgres-operator/values.yaml
+++ b/charts/postgres-operator/values.yaml
@@ -381,7 +381,7 @@ configConnectionPooler:
   # db user for pooler to use
   connection_pooler_user: "pooler"
   # docker image
-  connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-19"
+  connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-22"
   # max db connections the pooler should hold
   connection_pooler_max_db_connections: 60
   # default pooling mode
diff --git a/docs/user.md b/docs/user.md
index 9a38844cd..4e690fb3c 100644
--- a/docs/user.md
+++ b/docs/user.md
@@ -83,9 +83,9 @@ kubectl port-forward $PGMASTER 6432:5432 -n default
 ```
 
 Open another CLI and connect to the database using e.g. the psql client.
-When connecting with the `postgres` user read its password from the K8s secret
-which was generated when creating the `acid-minimal-cluster`. As non-encrypted
-connections are rejected by default set the SSL mode to `require`:
+When connecting with a manifest role like `foo_user` user, read its password
+from the K8s secret which was generated when creating `acid-minimal-cluster`.
+As non-encrypted connections are rejected by default set SSL mode to `require`:
 
 ```bash
 export PGPASSWORD=$(kubectl get secret postgres.acid-minimal-cluster.credentials.postgresql.acid.zalan.do -o 'jsonpath={.data.password}' | base64 -d)
@@ -93,6 +93,26 @@ export PGSSLMODE=require
 psql -U postgres -h localhost -p 6432
 ```
 
+## Password encryption
+
+Passwords are encrypted with `md5` hash generation by default. However, it is
+possible to use the more recent `scram-sha-256` method by changing the
+`password_encryption` parameter in the Postgres config. You can define it
+directly from the cluster manifest:
+
+```yaml
+apiVersion: "acid.zalan.do/v1"
+kind: postgresql
+metadata:
+  name: acid-minimal-cluster
+spec:
+  [...]
+  postgresql:
+    version: "14"
+    parameters:
+      password_encryption: scram-sha-256
+```
+
 ## Defining database roles in the operator
 
 Postgres Operator allows defining roles to be created in the resulting database
diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml
index b7d581bc6..36dd092c2 100644
--- a/manifests/configmap.yaml
+++ b/manifests/configmap.yaml
@@ -17,7 +17,7 @@ data:
   # connection_pooler_default_cpu_request: "500m"
   # connection_pooler_default_memory_limit: 100Mi
   # connection_pooler_default_memory_request: 100Mi
-  connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-19"
+  connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-22"
   # connection_pooler_max_db_connections: 60
   # connection_pooler_mode: "transaction"
   # connection_pooler_number_of_instances: 2
diff --git a/manifests/minimal-fake-pooler-deployment.yaml b/manifests/minimal-fake-pooler-deployment.yaml
index c432b4613..7bd661a87 100644
--- a/manifests/minimal-fake-pooler-deployment.yaml
+++ b/manifests/minimal-fake-pooler-deployment.yaml
@@ -23,7 +23,7 @@ spec:
       serviceAccountName: postgres-operator
       containers:
       - name: postgres-operator
-        image: registry.opensource.zalan.do/acid/pgbouncer:master-19
+        image: registry.opensource.zalan.do/acid/pgbouncer:master-22
         imagePullPolicy: IfNotPresent
         resources:
           requests:
diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml
index f43c9ce3e..99184dd17 100644
--- a/manifests/operatorconfiguration.crd.yaml
+++ b/manifests/operatorconfiguration.crd.yaml
@@ -586,7 +586,7 @@ spec:
                     default: "pooler"
                   connection_pooler_image:
                     type: string
-                    default: "registry.opensource.zalan.do/acid/pgbouncer:master-19"
+                    default: "registry.opensource.zalan.do/acid/pgbouncer:master-22"
                   connection_pooler_max_db_connections:
                     type: integer
                     default: 60
diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml
index d1e36545b..5cef7a353 100644
--- a/manifests/postgresql-operator-default-configuration.yaml
+++ b/manifests/postgresql-operator-default-configuration.yaml
@@ -187,7 +187,7 @@ configuration:
     connection_pooler_default_cpu_request: "500m"
     connection_pooler_default_memory_limit: 100Mi
     connection_pooler_default_memory_request: 100Mi
-    connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-19"
+    connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-22"
     # connection_pooler_max_db_connections: 60
     connection_pooler_mode: "transaction"
     connection_pooler_number_of_instances: 2

From 9bcb25ac7eee8cd538b6bc70a9379c18a735080b Mon Sep 17 00:00:00 2001
From: Dmitry Volodin <dmvolod@gmail.com>
Date: Mon, 11 Apr 2022 11:16:35 +0300
Subject: [PATCH 53/58] Ability to set pod environment variables on cluster
 resource (#1794)

* Ability to set pod environment variables on cluster resource

Co-authored-by: Felix Kunde <felix-kunde@gmx.de>
---
 .../postgres-operator/crds/postgresqls.yaml   |   6 +
 docs/administrator.md                         |  23 ++++
 manifests/complete-postgres-manifest.yaml     |   6 +-
 manifests/postgresql.crd.yaml                 |   6 +
 pkg/apis/acid.zalan.do/v1/crds.go             |  10 ++
 pkg/apis/acid.zalan.do/v1/postgresql_type.go  |   1 +
 .../acid.zalan.do/v1/zz_generated.deepcopy.go |   7 ++
 pkg/cluster/k8sres.go                         | 102 ++++++++--------
 pkg/cluster/k8sres_test.go                    | 109 ++++++++++++++++--
 9 files changed, 204 insertions(+), 66 deletions(-)

diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml
index 8aa6f70b3..8534650e1 100644
--- a/charts/postgres-operator/crds/postgresqls.yaml
+++ b/charts/postgres-operator/crds/postgresqls.yaml
@@ -196,6 +196,12 @@ spec:
                 type: boolean
               enableShmVolume:
                 type: boolean
+              env:
+                type: array
+                nullable: true
+                items:
+                  type: object
+                  x-kubernetes-preserve-unknown-fields: true
               init_containers:
                 type: array
                 description: deprecated
diff --git a/docs/administrator.md b/docs/administrator.md
index cd4504c31..061f9184e 100644
--- a/docs/administrator.md
+++ b/docs/administrator.md
@@ -706,6 +706,29 @@ data:
 The key-value pairs of the Secret are all accessible as environment variables
 to the Postgres StatefulSet/pods.
 
+### For individual cluster
+
+It is possible to define environment variables directly in the Postgres cluster
+manifest to configure it individually. The variables must be listed under the
+`env` section in the same way you would do for [containers](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/).
+Global parameters served from a custom config map or secret will be overridden.
+
+```yaml
+apiVersion: "acid.zalan.do/v1"
+kind: postgresql
+metadata:
+  name: acid-test-cluster
+spec:
+  env:
+  - name: wal_s3_bucket
+    value: my-custom-bucket
+  - name: minio_secret_key
+      valueFrom:
+        secretKeyRef:
+          name: my-custom-secret
+          key: minio_secret_key
+```
+
 ## Limiting the number of min and max instances in clusters
 
 As a preventive measure, one can restrict the minimum and the maximum number of
diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml
index 54a3b4249..26e404e6d 100644
--- a/manifests/complete-postgres-manifest.yaml
+++ b/manifests/complete-postgres-manifest.yaml
@@ -49,6 +49,10 @@ spec:
       shared_buffers: "32MB"
       max_connections: "10"
       log_statement: "all"
+#  env:
+#  - name: wal_s3_bucket
+#    value: my-custom-bucket
+
   volume:
     size: 1Gi
 #    storageClass: my-sc
@@ -120,7 +124,7 @@ spec:
 #        database: foo
 #        plugin: pgoutput
     ttl: 30
-    loop_wait: &loop_wait 10
+    loop_wait: 10
     retry_timeout: 10
     synchronous_mode: false
     synchronous_mode_strict: false
diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml
index e73dc9ad1..34a4298dd 100644
--- a/manifests/postgresql.crd.yaml
+++ b/manifests/postgresql.crd.yaml
@@ -194,6 +194,12 @@ spec:
                 type: boolean
               enableShmVolume:
                 type: boolean
+              env:
+                type: array
+                nullable: true
+                items:
+                  type: object
+                  x-kubernetes-preserve-unknown-fields: true
               init_containers:
                 type: array
                 description: deprecated
diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go
index 7fd927bb0..2851d31ac 100644
--- a/pkg/apis/acid.zalan.do/v1/crds.go
+++ b/pkg/apis/acid.zalan.do/v1/crds.go
@@ -311,6 +311,16 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 					"enableShmVolume": {
 						Type: "boolean",
 					},
+					"env": {
+						Type:     "array",
+						Nullable: true,
+						Items: &apiextv1.JSONSchemaPropsOrArray{
+							Schema: &apiextv1.JSONSchemaProps{
+								Type:                   "object",
+								XPreserveUnknownFields: util.True(),
+							},
+						},
+					},
 					"init_containers": {
 						Type:        "array",
 						Description: "deprecated",
diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go
index b7c41be58..3d37dacfb 100644
--- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go
+++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go
@@ -80,6 +80,7 @@ type PostgresSpec struct {
 	TLS                   *TLSDescription             `json:"tls,omitempty"`
 	AdditionalVolumes     []AdditionalVolume          `json:"additionalVolumes,omitempty"`
 	Streams               []Stream                    `json:"streams,omitempty"`
+	Env                   []v1.EnvVar                 `json:"env,omitempty"`
 
 	// deprecated json tags
 	InitContainersOld       []v1.Container `json:"init_containers,omitempty"`
diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
index 26d930f2b..b338421a2 100644
--- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
+++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
@@ -779,6 +779,13 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	if in.Env != nil {
+		in, out := &in.Env, &out.Env
+		*out = make([]corev1.EnvVar, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	if in.InitContainersOld != nil {
 		in, out := &in.InitContainersOld, &out.InitContainersOld
 		*out = make([]corev1.Container, len(*in))
diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go
index 382abd1e2..a2186caa8 100644
--- a/pkg/cluster/k8sres.go
+++ b/pkg/cluster/k8sres.go
@@ -649,8 +649,7 @@ func patchSidecarContainers(in []v1.Container, volumeMounts []v1.VolumeMount, su
 				},
 			},
 		}
-		mergedEnv := append(env, container.Env...)
-		container.Env = deduplicateEnvVars(mergedEnv, container.Name, logger)
+		container.Env = appendEnvVars(env, container.Env...)
 		result = append(result, container)
 	}
 
@@ -769,6 +768,7 @@ func (c *Cluster) generateSpiloPodEnvVars(
 	cloneDescription *acidv1.CloneDescription,
 	standbyDescription *acidv1.StandbyDescription,
 	customPodEnvVarsList []v1.EnvVar) []v1.EnvVar {
+
 	envVars := []v1.EnvVar{
 		{
 			Name:  "SCOPE",
@@ -843,6 +843,11 @@ func (c *Cluster) generateSpiloPodEnvVars(
 			Value: c.OpConfig.PamRoleName,
 		},
 	}
+
+	if c.OpConfig.EnableSpiloWalPathCompat {
+		envVars = append(envVars, v1.EnvVar{Name: "ENABLE_WAL_PATH_COMPAT", Value: "true"})
+	}
+
 	if c.OpConfig.EnablePgVersionEnvVar {
 		envVars = append(envVars, v1.EnvVar{Name: "PGVERSION", Value: c.GetDesiredMajorVersion()})
 	}
@@ -874,73 +879,67 @@ func (c *Cluster) generateSpiloPodEnvVars(
 		envVars = append(envVars, c.generateStandbyEnvironment(standbyDescription)...)
 	}
 
+	if len(c.Spec.Env) > 0 {
+		envVars = appendEnvVars(envVars, c.Spec.Env...)
+	}
+
 	// add vars taken from pod_environment_configmap and pod_environment_secret first
 	// (to allow them to override the globals set in the operator config)
 	if len(customPodEnvVarsList) > 0 {
-		envVars = append(envVars, customPodEnvVarsList...)
+		envVars = appendEnvVars(envVars, customPodEnvVarsList...)
 	}
 
 	if c.OpConfig.WALES3Bucket != "" {
-		envVars = append(envVars, v1.EnvVar{Name: "WAL_S3_BUCKET", Value: c.OpConfig.WALES3Bucket})
-		envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))})
-		envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""})
+		envVars = appendEnvVars(envVars, v1.EnvVar{Name: "WAL_S3_BUCKET", Value: c.OpConfig.WALES3Bucket})
+		envVars = appendEnvVars(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))})
+		envVars = appendEnvVars(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""})
 	}
 
 	if c.OpConfig.WALGSBucket != "" {
-		envVars = append(envVars, v1.EnvVar{Name: "WAL_GS_BUCKET", Value: c.OpConfig.WALGSBucket})
-		envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))})
-		envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""})
+		envVars = appendEnvVars(envVars, v1.EnvVar{Name: "WAL_GS_BUCKET", Value: c.OpConfig.WALGSBucket})
+		envVars = appendEnvVars(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))})
+		envVars = appendEnvVars(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""})
 	}
 
 	if c.OpConfig.WALAZStorageAccount != "" {
-		envVars = append(envVars, v1.EnvVar{Name: "AZURE_STORAGE_ACCOUNT", Value: c.OpConfig.WALAZStorageAccount})
-		envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))})
-		envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""})
+		envVars = appendEnvVars(envVars, v1.EnvVar{Name: "AZURE_STORAGE_ACCOUNT", Value: c.OpConfig.WALAZStorageAccount})
+		envVars = appendEnvVars(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))})
+		envVars = appendEnvVars(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""})
 	}
 
 	if c.OpConfig.GCPCredentials != "" {
-		envVars = append(envVars, v1.EnvVar{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: c.OpConfig.GCPCredentials})
+		envVars = appendEnvVars(envVars, v1.EnvVar{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: c.OpConfig.GCPCredentials})
 	}
 
 	if c.OpConfig.LogS3Bucket != "" {
-		envVars = append(envVars, v1.EnvVar{Name: "LOG_S3_BUCKET", Value: c.OpConfig.LogS3Bucket})
-		envVars = append(envVars, v1.EnvVar{Name: "LOG_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))})
-		envVars = append(envVars, v1.EnvVar{Name: "LOG_BUCKET_SCOPE_PREFIX", Value: ""})
+		envVars = appendEnvVars(envVars, v1.EnvVar{Name: "LOG_S3_BUCKET", Value: c.OpConfig.LogS3Bucket})
+		envVars = appendEnvVars(envVars, v1.EnvVar{Name: "LOG_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))})
+		envVars = appendEnvVars(envVars, v1.EnvVar{Name: "LOG_BUCKET_SCOPE_PREFIX", Value: ""})
 	}
 
 	return envVars
 }
 
-// deduplicateEnvVars makes sure there are no duplicate in the target envVar array. While Kubernetes already
-// deduplicates variables defined in a container, it leaves the last definition in the list and this behavior is not
-// well-documented, which means that the behavior can be reversed at some point (it may also start producing an error).
-// Therefore, the merge is done by the operator, the entries that are ahead in the passed list take priority over those
-// that are behind, and only the name is considered in order to eliminate duplicates.
-func deduplicateEnvVars(input []v1.EnvVar, containerName string, logger *logrus.Entry) []v1.EnvVar {
-	result := make([]v1.EnvVar, 0)
-	names := make(map[string]int)
-
-	for i, va := range input {
-		if names[va.Name] == 0 {
-			names[va.Name]++
-			result = append(result, input[i])
-		} else if names[va.Name] == 1 {
-			names[va.Name]++
-
-			// Some variables (those to configure the WAL_ and LOG_ shipping) may be overwritten, only log as info
-			if strings.HasPrefix(va.Name, "WAL_") || strings.HasPrefix(va.Name, "LOG_") {
-				logger.Infof("global variable %q has been overwritten by configmap/secret for container %q",
-					va.Name, containerName)
-			} else {
-				logger.Warningf("variable %q is defined in %q more than once, the subsequent definitions are ignored",
-					va.Name, containerName)
-			}
+func appendEnvVars(envs []v1.EnvVar, appEnv ...v1.EnvVar) []v1.EnvVar {
+	jenvs := envs
+	for _, env := range appEnv {
+		if !isEnvVarPresent(jenvs, env.Name) {
+			jenvs = append(jenvs, env)
 		}
 	}
-	return result
+	return jenvs
 }
 
-// Return list of variables the pod recieved from the configured ConfigMap
+func isEnvVarPresent(envs []v1.EnvVar, key string) bool {
+	for _, env := range envs {
+		if env.Name == key {
+			return true
+		}
+	}
+	return false
+}
+
+// Return list of variables the pod received from the configured ConfigMap
 func (c *Cluster) getPodEnvironmentConfigMapVariables() ([]v1.EnvVar, error) {
 	configMapPodEnvVarsList := make([]v1.EnvVar, 0)
 
@@ -1105,16 +1104,6 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
 		initContainers = spec.InitContainers
 	}
 
-	spiloCompathWalPathList := make([]v1.EnvVar, 0)
-	if c.OpConfig.EnableSpiloWalPathCompat {
-		spiloCompathWalPathList = append(spiloCompathWalPathList,
-			v1.EnvVar{
-				Name:  "ENABLE_WAL_PATH_COMPAT",
-				Value: "true",
-			},
-		)
-	}
-
 	// fetch env vars from custom ConfigMap
 	configMapEnvVarsList, err := c.getPodEnvironmentConfigMapVariables()
 	if err != nil {
@@ -1128,8 +1117,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
 	}
 
 	// concat all custom pod env vars and sort them
-	customPodEnvVarsList := append(spiloCompathWalPathList, configMapEnvVarsList...)
-	customPodEnvVarsList = append(customPodEnvVarsList, secretEnvVarsList...)
+	customPodEnvVarsList := append(configMapEnvVarsList, secretEnvVarsList...)
 	sort.Slice(customPodEnvVarsList,
 		func(i, j int) bool { return customPodEnvVarsList[i].Name < customPodEnvVarsList[j].Name })
 
@@ -1210,7 +1198,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
 		// use the same filenames as Secret resources by default
 		certFile := ensurePath(spec.TLS.CertificateFile, mountPath, "tls.crt")
 		privateKeyFile := ensurePath(spec.TLS.PrivateKeyFile, mountPath, "tls.key")
-		spiloEnvVars = append(
+		spiloEnvVars = appendEnvVars(
 			spiloEnvVars,
 			v1.EnvVar{Name: "SSL_CERTIFICATE_FILE", Value: certFile},
 			v1.EnvVar{Name: "SSL_PRIVATE_KEY_FILE", Value: privateKeyFile},
@@ -1224,7 +1212,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
 			}
 
 			caFile := ensurePath(spec.TLS.CAFile, mountPathCA, "")
-			spiloEnvVars = append(
+			spiloEnvVars = appendEnvVars(
 				spiloEnvVars,
 				v1.EnvVar{Name: "SSL_CA_FILE", Value: caFile},
 			)
@@ -1249,7 +1237,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
 	spiloContainer := generateContainer(constants.PostgresContainerName,
 		&effectiveDockerImage,
 		resourceRequirements,
-		deduplicateEnvVars(spiloEnvVars, constants.PostgresContainerName, c.logger),
+		spiloEnvVars,
 		volumeMounts,
 		c.OpConfig.Resources.SpiloPrivileged,
 		c.OpConfig.Resources.SpiloAllowPrivilegeEscalation,
diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go
index 13e5aca67..94e78572c 100644
--- a/pkg/cluster/k8sres_test.go
+++ b/pkg/cluster/k8sres_test.go
@@ -131,17 +131,17 @@ func TestGenerateSpiloPodEnvVars(t *testing.T) {
 		}, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder)
 
 	expectedValuesGSBucket := []ExpectedValue{
-		ExpectedValue{
+		{
 			envIndex:       15,
 			envVarConstant: "WAL_GS_BUCKET",
 			envVarValue:    "wale-gs-bucket",
 		},
-		ExpectedValue{
+		{
 			envIndex:       16,
 			envVarConstant: "WAL_BUCKET_SCOPE_SUFFIX",
 			envVarValue:    "/SomeUUID",
 		},
-		ExpectedValue{
+		{
 			envIndex:       17,
 			envVarConstant: "WAL_BUCKET_SCOPE_PREFIX",
 			envVarValue:    "",
@@ -149,27 +149,48 @@ func TestGenerateSpiloPodEnvVars(t *testing.T) {
 	}
 
 	expectedValuesGCPCreds := []ExpectedValue{
-		ExpectedValue{
+		{
 			envIndex:       15,
 			envVarConstant: "WAL_GS_BUCKET",
 			envVarValue:    "wale-gs-bucket",
 		},
-		ExpectedValue{
+		{
 			envIndex:       16,
 			envVarConstant: "WAL_BUCKET_SCOPE_SUFFIX",
 			envVarValue:    "/SomeUUID",
 		},
-		ExpectedValue{
+		{
 			envIndex:       17,
 			envVarConstant: "WAL_BUCKET_SCOPE_PREFIX",
 			envVarValue:    "",
 		},
-		ExpectedValue{
+		{
 			envIndex:       18,
 			envVarConstant: "GOOGLE_APPLICATION_CREDENTIALS",
 			envVarValue:    "some_path_to_credentials",
 		},
 	}
+	expectedClusterNameLabel := []ExpectedValue{
+		{
+			envIndex:       5,
+			envVarConstant: "KUBERNETES_SCOPE_LABEL",
+			envVarValue:    "cluster-name",
+		},
+	}
+	expectedCustomS3Bucket := []ExpectedValue{
+		{
+			envIndex:       15,
+			envVarConstant: "WAL_S3_BUCKET",
+			envVarValue:    "custom-s3-bucket",
+		},
+	}
+	expectedCustomVariable := []ExpectedValue{
+		{
+			envIndex:       15,
+			envVarConstant: "CUSTOM_VARIABLE",
+			envVarValue:    "cluster-variable",
+		},
+	}
 
 	testName := "TestGenerateSpiloPodEnvVars"
 	tests := []struct {
@@ -181,6 +202,7 @@ func TestGenerateSpiloPodEnvVars(t *testing.T) {
 		standbyDescription *acidv1.StandbyDescription
 		customEnvList      []v1.EnvVar
 		expectedValues     []ExpectedValue
+		pgsql              acidv1.Postgresql
 	}{
 		{
 			subTest: "Will set WAL_GS_BUCKET env",
@@ -207,10 +229,81 @@ func TestGenerateSpiloPodEnvVars(t *testing.T) {
 			customEnvList:      []v1.EnvVar{},
 			expectedValues:     expectedValuesGCPCreds,
 		},
+		{
+			subTest: "Will not overwrite global KUBERNETES_SCOPE_LABEL parameter from the cluster Env option",
+			opConfig: config.Config{
+				Resources: config.Resources{
+					ClusterNameLabel: "cluster-name",
+				},
+			},
+			uid:                "SomeUUID",
+			spiloConfig:        "someConfig",
+			cloneDescription:   &acidv1.CloneDescription{},
+			standbyDescription: &acidv1.StandbyDescription{},
+			customEnvList:      []v1.EnvVar{},
+			expectedValues:     expectedClusterNameLabel,
+			pgsql: acidv1.Postgresql{
+				Spec: acidv1.PostgresSpec{
+					Env: []v1.EnvVar{
+						{
+							Name:  "KUBERNETES_SCOPE_LABEL",
+							Value: "my-scope-label",
+						},
+					},
+				},
+			},
+		},
+		{
+			subTest: "Will overwrite global WAL_S3_BUCKET parameter from the cluster Env option",
+			opConfig: config.Config{
+				WALGSBucket: "global-s3-bucket",
+			},
+			uid:                "SomeUUID",
+			spiloConfig:        "someConfig",
+			cloneDescription:   &acidv1.CloneDescription{},
+			standbyDescription: &acidv1.StandbyDescription{},
+			customEnvList:      []v1.EnvVar{},
+			expectedValues:     expectedCustomS3Bucket,
+			pgsql: acidv1.Postgresql{
+				Spec: acidv1.PostgresSpec{
+					Env: []v1.EnvVar{
+						{
+							Name:  "WAL_S3_BUCKET",
+							Value: "custom-s3-bucket",
+						},
+					},
+				},
+			},
+		},
+		{
+			subTest:            "Will overwrite custom variable parameter from the cluster Env option",
+			uid:                "SomeUUID",
+			spiloConfig:        "someConfig",
+			cloneDescription:   &acidv1.CloneDescription{},
+			standbyDescription: &acidv1.StandbyDescription{},
+			customEnvList: []v1.EnvVar{
+				{
+					Name:  "CUSTOM_VARIABLE",
+					Value: "custom-variable",
+				},
+			},
+			expectedValues: expectedCustomVariable,
+			pgsql: acidv1.Postgresql{
+				Spec: acidv1.PostgresSpec{
+					Env: []v1.EnvVar{
+						{
+							Name:  "CUSTOM_VARIABLE",
+							Value: "cluster-variable",
+						},
+					},
+				},
+			},
+		},
 	}
 
 	for _, tt := range tests {
 		cluster.OpConfig = tt.opConfig
+		cluster.Postgresql = tt.pgsql
 
 		actualEnvs := cluster.generateSpiloPodEnvVars(tt.uid, tt.spiloConfig, tt.cloneDescription, tt.standbyDescription, tt.customEnvList)
 
@@ -853,7 +946,7 @@ func TestPodEnvironmentConfigMapVariables(t *testing.T) {
 			err: fmt.Errorf("could not read PodEnvironmentConfigMap: NotFound"),
 		},
 		{
-			subTest: "simple PodEnvironmentConfigMap",
+			subTest: "Pod environment vars configured by PodEnvironmentConfigMap",
 			opConfig: config.Config{
 				Resources: config.Resources{
 					PodEnvironmentConfigMap: spec.NamespacedName{

From 483bf624eece809e39eb5fd9cd59d493e3b10d56 Mon Sep 17 00:00:00 2001
From: Jociele Padilha <45459238+jopadi@users.noreply.github.com>
Date: Thu, 14 Apr 2022 10:02:54 +0200
Subject: [PATCH 54/58] add test team member (#1842)

* return err if teams API fails with StatusCode other than 404
* add unit test for 404 at team members

Co-authored-by: Jociele Padilha <jociele.padilha@zalando.de>
Co-authored-by: Felix Kunde <felix-kunde@gmx.de>
---
 pkg/cluster/cluster_test.go  |  10 +-
 pkg/cluster/util.go          |  10 +-
 pkg/util/teams/teams.go      |  21 +-
 pkg/util/teams/teams_test.go | 468 ++++++++++++++++++-----------------
 4 files changed, 267 insertions(+), 242 deletions(-)

diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go
index acfecc166..f531962d1 100644
--- a/pkg/cluster/cluster_test.go
+++ b/pkg/cluster/cluster_test.go
@@ -194,8 +194,8 @@ type mockTeamsAPIClient struct {
 	members []string
 }
 
-func (m *mockTeamsAPIClient) TeamInfo(teamID, token string) (tm *teams.Team, err error) {
-	return &teams.Team{Members: m.members}, nil
+func (m *mockTeamsAPIClient) TeamInfo(teamID, token string) (tm *teams.Team, statusCode int, err error) {
+	return &teams.Team{Members: m.members}, statusCode, nil
 }
 
 func (m *mockTeamsAPIClient) setMembers(members []string) {
@@ -260,15 +260,15 @@ type mockTeamsAPIClientMultipleTeams struct {
 	teams []mockTeam
 }
 
-func (m *mockTeamsAPIClientMultipleTeams) TeamInfo(teamID, token string) (tm *teams.Team, err error) {
+func (m *mockTeamsAPIClientMultipleTeams) TeamInfo(teamID, token string) (tm *teams.Team, statusCode int, err error) {
 	for _, team := range m.teams {
 		if team.teamID == teamID {
-			return &teams.Team{Members: team.members}, nil
+			return &teams.Team{Members: team.members}, statusCode, nil
 		}
 	}
 
 	// should not be reached if a slice with teams is populated correctly
-	return nil, nil
+	return nil, statusCode, nil
 }
 
 // Test adding members of maintenance teams that get superuser rights for all PG databases
diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go
index cf320e057..0f71d2d64 100644
--- a/pkg/cluster/util.go
+++ b/pkg/cluster/util.go
@@ -6,6 +6,7 @@ import (
 	"encoding/gob"
 	"encoding/json"
 	"fmt"
+	"net/http"
 	"reflect"
 	"sort"
 	"strings"
@@ -272,9 +273,14 @@ func (c *Cluster) getTeamMembers(teamID string) ([]string, error) {
 		return nil, fmt.Errorf("could not get oauth token to authenticate to team service API: %v", err)
 	}
 
-	teamInfo, err := c.teamsAPIClient.TeamInfo(teamID, token)
+	teamInfo, statusCode, err := c.teamsAPIClient.TeamInfo(teamID, token)
+
 	if err != nil {
-		c.logger.Warningf("could not get team info for team %q: %v", teamID, err)
+		if statusCode == http.StatusNotFound {
+			c.logger.Warningf("could not get team info for team %q: %v", teamID, err)
+		} else {
+			return nil, fmt.Errorf("could not get team info for team %q: %v", teamID, err)
+		}
 	} else {
 		for _, member := range teamInfo.Members {
 			if !(util.SliceContains(members, member)) {
diff --git a/pkg/util/teams/teams.go b/pkg/util/teams/teams.go
index d7413ab9c..bf1260d6e 100644
--- a/pkg/util/teams/teams.go
+++ b/pkg/util/teams/teams.go
@@ -45,7 +45,7 @@ type httpClient interface {
 
 // Interface to the TeamsAPIClient
 type Interface interface {
-	TeamInfo(teamID, token string) (tm *Team, err error)
+	TeamInfo(teamID, token string) (tm *Team, statusCode int, err error)
 }
 
 // API describes teams API
@@ -67,7 +67,7 @@ func NewTeamsAPI(url string, log *logrus.Entry) *API {
 }
 
 // TeamInfo returns information about a given team using its ID and a token to authenticate to the API service.
-func (t *API) TeamInfo(teamID, token string) (tm *Team, err error) {
+func (t *API) TeamInfo(teamID, token string) (tm *Team, statusCode int, err error) {
 	var (
 		req  *http.Request
 		resp *http.Response
@@ -77,37 +77,38 @@ func (t *API) TeamInfo(teamID, token string) (tm *Team, err error) {
 	t.logger.Debugf("request url: %s", url)
 	req, err = http.NewRequest("GET", url, nil)
 	if err != nil {
-		return nil, err
+		return nil, http.StatusBadRequest, err
 	}
 
 	req.Header.Add("Authorization", "Bearer "+token)
 	if resp, err = t.httpClient.Do(req); err != nil {
-		return nil, err
+		return nil, http.StatusUnauthorized, err
 	}
 	defer func() {
 		if closeErr := resp.Body.Close(); closeErr != nil {
 			err = fmt.Errorf("error when closing response: %v", closeErr)
 		}
 	}()
-	if resp.StatusCode != 200 {
+	statusCode = resp.StatusCode
+	if statusCode != http.StatusOK {
 		var raw map[string]json.RawMessage
 		d := json.NewDecoder(resp.Body)
 		err = d.Decode(&raw)
 		if err != nil {
-			return nil, fmt.Errorf("team API query failed with status code %d and malformed response: %v", resp.StatusCode, err)
+			return nil, statusCode, fmt.Errorf("team API query failed with status code %d and malformed response: %v", statusCode, err)
 		}
 
 		if errMessage, ok := raw["error"]; ok {
-			return nil, fmt.Errorf("team API query failed with status code %d and message: '%v'", resp.StatusCode, string(errMessage))
+			return nil, statusCode, fmt.Errorf("team API query failed with status code %d and message: '%v'", statusCode, string(errMessage))
 		}
-		return nil, fmt.Errorf("team API query failed with status code %d", resp.StatusCode)
+		return nil, statusCode, fmt.Errorf("team API query failed with status code %d", statusCode)
 	}
 
 	tm = &Team{}
 	d := json.NewDecoder(resp.Body)
 	if err = d.Decode(tm); err != nil {
-		return nil, fmt.Errorf("could not parse team API response: %v", err)
+		return nil, statusCode, fmt.Errorf("could not parse team API response: %v", err)
 	}
 
-	return tm, nil
+	return tm, statusCode, nil
 }
diff --git a/pkg/util/teams/teams_test.go b/pkg/util/teams/teams_test.go
index 33d01b75b..da9f497c1 100644
--- a/pkg/util/teams/teams_test.go
+++ b/pkg/util/teams/teams_test.go
@@ -1,225 +1,243 @@
-package teams
-
-import (
-	"fmt"
-	"net/http"
-	"net/http/httptest"
-	"reflect"
-	"testing"
-
-	"github.com/sirupsen/logrus"
-)
-
-var (
-	logger = logrus.New().WithField("pkg", "teamsapi")
-	token  = "ec45b1cfbe7100c6315d183a3eb6cec0M2U1LWJkMzEtZDgzNzNmZGQyNGM3IiwiYXV0aF90aW1lIjoxNDkzNzMwNzQ1LCJpc3MiOiJodHRwcz"
-)
-
-var teamsAPItc = []struct {
-	in     string
-	inCode int
-	out    *Team
-	err    error
-}{
-	{`{
-"dn": "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net",
-"id": "acid",
-"id_name": "acid",
-"team_id": "111222",
-"type": "official",
-"name": "Acid team name",
-"mail": [
-"email1@example.com",
-"email2@example.com"
-],
-"alias": [
-"acid"
-],
-"member": [
-  "member1",
-  "member2",
-  "member3"
-],
-"infrastructure-accounts": [
-{
-  "id": "1234512345",
-  "name": "acid",
-  "provider": "aws",
-  "type": "aws",
-  "description": "",
-  "owner": "acid",
-  "owner_dn": "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net",
-  "disabled": false
-},
-{
-  "id": "5432154321",
-  "name": "db",
-  "provider": "aws",
-  "type": "aws",
-  "description": "",
-  "owner": "acid",
-  "owner_dn": "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net",
-  "disabled": false
-}
-],
-"cost_center": "00099999",
-"delivery_lead": "member4",
-"parent_team_id": "111221"
-}`,
-		200,
-		&Team{
-			Dn:           "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net",
-			ID:           "acid",
-			TeamName:     "acid",
-			TeamID:       "111222",
-			Type:         "official",
-			FullName:     "Acid team name",
-			Aliases:      []string{"acid"},
-			Mails:        []string{"email1@example.com", "email2@example.com"},
-			Members:      []string{"member1", "member2", "member3"},
-			CostCenter:   "00099999",
-			DeliveryLead: "member4",
-			ParentTeamID: "111221",
-			InfrastructureAccounts: []infrastructureAccount{
-				{
-					ID:          "1234512345",
-					Name:        "acid",
-					Provider:    "aws",
-					Type:        "aws",
-					Description: "",
-					Owner:       "acid",
-					OwnerDn:     "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net",
-					Disabled:    false},
-				{
-					ID:          "5432154321",
-					Name:        "db",
-					Provider:    "aws",
-					Type:        "aws",
-					Description: "",
-					Owner:       "acid",
-					OwnerDn:     "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net",
-					Disabled:    false},
-			},
-		},
-		nil}, {
-		`{"error": "Access Token not valid"}`,
-		401,
-		nil,
-		fmt.Errorf(`team API query failed with status code 401 and message: '"Access Token not valid"'`),
-	},
-	{
-		`{"status": "I'm a teapot'"}`,
-		418,
-		nil,
-		fmt.Errorf(`team API query failed with status code 418`),
-	},
-	{
-		`{"status": "I'm a teapot`,
-		418,
-		nil,
-		fmt.Errorf(`team API query failed with status code 418 and malformed response: unexpected EOF`),
-	},
-	{
-		`{"status": "I'm a teapot`,
-		200,
-		nil,
-		fmt.Errorf(`could not parse team API response: unexpected EOF`),
-	},
-}
-
-var requestsURLtc = []struct {
-	url string
-	err error
-}{
-	{
-		"coffee://localhost/",
-		fmt.Errorf(`Get "coffee://localhost/teams/acid": unsupported protocol scheme "coffee"`),
-	},
-	{
-		"http://192.168.0.%31/",
-		fmt.Errorf(`parse "http://192.168.0.%%31/teams/acid": invalid URL escape "%%31"`),
-	},
-}
-
-func TestInfo(t *testing.T) {
-	for _, tc := range teamsAPItc {
-		func() {
-			ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-				if r.Header.Get("Authorization") != "Bearer "+token {
-					t.Errorf("authorization token is wrong or not provided")
-				}
-				w.WriteHeader(tc.inCode)
-				if _, err := fmt.Fprint(w, tc.in); err != nil {
-					t.Errorf("error writing teams api response %v", err)
-				}
-			}))
-			defer ts.Close()
-			api := NewTeamsAPI(ts.URL, logger)
-
-			actual, err := api.TeamInfo("acid", token)
-			if err != nil && err.Error() != tc.err.Error() {
-				t.Errorf("expected error: %v, got: %v", tc.err, err)
-				return
-			}
-
-			if !reflect.DeepEqual(actual, tc.out) {
-				t.Errorf("expected %#v, got: %#v", tc.out, actual)
-			}
-		}()
-	}
-}
-
-type mockHTTPClient struct {
-}
-
-type mockBody struct {
-}
-
-func (b *mockBody) Read(p []byte) (n int, err error) {
-	return 2, nil
-}
-
-func (b *mockBody) Close() error {
-	return fmt.Errorf("close error")
-}
-
-func (c *mockHTTPClient) Do(req *http.Request) (*http.Response, error) {
-	resp := http.Response{
-		Status:        "200 OK",
-		StatusCode:    200,
-		ContentLength: 2,
-		Close:         false,
-		Request:       req,
-	}
-	resp.Body = &mockBody{}
-
-	return &resp, nil
-}
-
-func TestHttpClientClose(t *testing.T) {
-	ts := httptest.NewServer(nil)
-
-	api := NewTeamsAPI(ts.URL, logger)
-	api.httpClient = &mockHTTPClient{}
-
-	_, err := api.TeamInfo("acid", token)
-	expError := fmt.Errorf("error when closing response: close error")
-	if err.Error() != expError.Error() {
-		t.Errorf("expected error: %v, got: %v", expError, err)
-	}
-}
-
-func TestRequest(t *testing.T) {
-	for _, tc := range requestsURLtc {
-		api := NewTeamsAPI(tc.url, logger)
-		resp, err := api.TeamInfo("acid", token)
-		if resp != nil {
-			t.Errorf("response expected to be nil")
-			continue
-		}
-
-		if err.Error() != tc.err.Error() {
-			t.Errorf("expected error: %v, got: %v", tc.err, err)
-		}
-	}
-}
+package teams
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"testing"
+
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	logger = logrus.New().WithField("pkg", "teamsapi")
+	token  = "ec45b1cfbe7100c6315d183a3eb6cec0M2U1LWJkMzEtZDgzNzNmZGQyNGM3IiwiYXV0aF90aW1lIjoxNDkzNzMwNzQ1LCJpc3MiOiJodHRwcz"
+	input  = `{
+	"dn": "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net",
+	"id": "acid",
+	"id_name": "acid",
+	"team_id": "111222",
+	"type": "official",
+	"name": "Acid team name",
+	"mail": [
+	"email1@example.com",
+	"email2@example.com"
+	],
+	"alias": [
+	"acid"
+	],
+	"member": [
+	  "member1",
+	  "member2",
+	  "member3"
+	],
+	"infrastructure-accounts": [
+	{
+	  "id": "1234512345",
+	  "name": "acid",
+	  "provider": "aws",
+	  "type": "aws",
+	  "description": "",
+	  "owner": "acid",
+	  "owner_dn": "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net",
+	  "disabled": false
+	},
+	{
+	  "id": "5432154321",
+	  "name": "db",
+	  "provider": "aws",
+	  "type": "aws",
+	  "description": "",
+	  "owner": "acid",
+	  "owner_dn": "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net",
+	  "disabled": false
+	}
+	],
+	"cost_center": "00099999",
+	"delivery_lead": "member4",
+	"parent_team_id": "111221"
+	}`
+)
+var teamsAPItc = []struct {
+	in     string
+	inCode int
+	inTeam string
+	out    *Team
+	err    error
+}{
+	{
+		input,
+		200,
+		"acid",
+		&Team{
+			Dn:           "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net",
+			ID:           "acid",
+			TeamName:     "acid",
+			TeamID:       "111222",
+			Type:         "official",
+			FullName:     "Acid team name",
+			Aliases:      []string{"acid"},
+			Mails:        []string{"email1@example.com", "email2@example.com"},
+			Members:      []string{"member1", "member2", "member3"},
+			CostCenter:   "00099999",
+			DeliveryLead: "member4",
+			ParentTeamID: "111221",
+			InfrastructureAccounts: []infrastructureAccount{
+				{
+					ID:          "1234512345",
+					Name:        "acid",
+					Provider:    "aws",
+					Type:        "aws",
+					Description: "",
+					Owner:       "acid",
+					OwnerDn:     "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net",
+					Disabled:    false},
+				{
+					ID:          "5432154321",
+					Name:        "db",
+					Provider:    "aws",
+					Type:        "aws",
+					Description: "",
+					Owner:       "acid",
+					OwnerDn:     "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net",
+					Disabled:    false},
+			},
+		},
+		nil}, {
+		`{"error": "Access Token not valid"}`,
+		401,
+		"acid",
+		nil,
+		fmt.Errorf(`team API query failed with status code 401 and message: '"Access Token not valid"'`),
+	},
+	{
+		`{"status": "I'm a teapot'"}`,
+		418,
+		"acid",
+		nil,
+		fmt.Errorf(`team API query failed with status code 418`),
+	},
+	{
+		`{"status": "I'm a teapot`,
+		418,
+		"acid",
+		nil,
+		fmt.Errorf(`team API query failed with status code 418 and malformed response: unexpected EOF`),
+	},
+	{
+		`{"status": "I'm a teapot`,
+		200,
+		"acid",
+		nil,
+		fmt.Errorf(`could not parse team API response: unexpected EOF`),
+	},
+	{
+		input,
+		404,
+		"banana",
+		nil,
+		fmt.Errorf(`team API query failed with status code 404`),
+	},
+}
+
+var requestsURLtc = []struct {
+	url string
+	err error
+}{
+	{
+		"coffee://localhost/",
+		fmt.Errorf(`Get "coffee://localhost/teams/acid": unsupported protocol scheme "coffee"`),
+	},
+	{
+		"http://192.168.0.%31/",
+		fmt.Errorf(`parse "http://192.168.0.%%31/teams/acid": invalid URL escape "%%31"`),
+	},
+}
+
+func TestInfo(t *testing.T) {
+	for _, tc := range teamsAPItc {
+		func() {
+			ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				if r.Header.Get("Authorization") != "Bearer "+token {
+					t.Errorf("authorization token is wrong or not provided")
+				}
+				w.WriteHeader(tc.inCode)
+				if _, err := fmt.Fprint(w, tc.in); err != nil {
+					t.Errorf("error writing teams api response %v", err)
+				}
+			}))
+			defer ts.Close()
+			api := NewTeamsAPI(ts.URL, logger)
+
+			actual, statusCode, err := api.TeamInfo(tc.inTeam, token)
+			if err != nil && err.Error() != tc.err.Error() {
+				t.Errorf("expected error: %v, got: %v", tc.err, err)
+				return
+			}
+
+			if !reflect.DeepEqual(actual, tc.out) {
+				t.Errorf("expected %#v, got: %#v", tc.out, actual)
+			}
+
+			if statusCode != tc.inCode {
+				t.Errorf("expected %d, got: %d", tc.inCode, statusCode)
+			}
+		}()
+	}
+}
+
+type mockHTTPClient struct {
+}
+
+type mockBody struct {
+}
+
+func (b *mockBody) Read(p []byte) (n int, err error) {
+	return 2, nil
+}
+
+func (b *mockBody) Close() error {
+	return fmt.Errorf("close error")
+}
+
+func (c *mockHTTPClient) Do(req *http.Request) (*http.Response, error) {
+	resp := http.Response{
+		Status:        "200 OK",
+		StatusCode:    200,
+		ContentLength: 2,
+		Close:         false,
+		Request:       req,
+	}
+	resp.Body = &mockBody{}
+
+	return &resp, nil
+}
+
+func TestHttpClientClose(t *testing.T) {
+	ts := httptest.NewServer(nil)
+
+	api := NewTeamsAPI(ts.URL, logger)
+	api.httpClient = &mockHTTPClient{}
+
+	_, _, err := api.TeamInfo("acid", token)
+	expError := fmt.Errorf("error when closing response: close error")
+	if err.Error() != expError.Error() {
+		t.Errorf("expected error: %v, got: %v", expError, err)
+	}
+}
+
+func TestRequest(t *testing.T) {
+	for _, tc := range requestsURLtc {
+		api := NewTeamsAPI(tc.url, logger)
+		resp, _, err := api.TeamInfo("acid", token)
+		if resp != nil {
+			t.Errorf("response expected to be nil")
+			continue
+		}
+
+		if err.Error() != tc.err.Error() {
+			t.Errorf("expected error: %v, got: %v", tc.err, err)
+		}
+	}
+}

From eecd13169cf27fab06cd51fdfc06aa035424122a Mon Sep 17 00:00:00 2001
From: Felix Kunde <felix-kunde@gmx.de>
Date: Thu, 14 Apr 2022 11:47:33 +0200
Subject: [PATCH 55/58] refactor spilo env var generation (#1848)

* refactor spilo env generation
* enhance docs on env vars
* add unit test for appendEnvVar
---
 docs/administrator.md                 |   46 +-
 docs/reference/operator_parameters.md |    5 +-
 docs/user.md                          |   31 +-
 pkg/cluster/cluster.go                |    2 +-
 pkg/cluster/cluster_test.go           |   12 +-
 pkg/cluster/k8sres.go                 |  182 +--
 pkg/cluster/k8sres_test.go            | 1863 ++++++++++++++-----------
 7 files changed, 1231 insertions(+), 910 deletions(-)

diff --git a/docs/administrator.md b/docs/administrator.md
index 061f9184e..aa08f549e 100644
--- a/docs/administrator.md
+++ b/docs/administrator.md
@@ -601,15 +601,39 @@ spec:
 
 ## Custom Pod Environment Variables
 
-It is possible to configure a ConfigMap as well as a Secret which are used by
-the Postgres pods as an additional provider for environment variables. One use
-case is a customized Spilo image configured by extra environment variables.
-Another case could be to provide custom cloud provider or backup settings.
+The operator will assign a set of environment variables to the database pods
+that cannot be overridden to guarantee core functionality. Only variables with
+'WAL_' and 'LOG_' prefixes can be customized to allow for backup and log
+shipping to be specified differently. There are three ways to specify extra
+environment variables (or override existing ones) for database pods:
 
-In general the Operator will give preference to the globally configured
-variables, to not have the custom ones interfere with core functionality.
-Variables with the 'WAL_' and 'LOG_' prefix can be overwritten though, to
-allow backup and log shipping to be specified differently.
+* [Via ConfigMap](#via-configmap)
+* [Via Secret](#via-secret)
+* [Via Postgres Cluster Manifest](#via-postgres-cluster-manifest)
+
+The first two options must be referenced from the operator configuration
+making them global settings for all Postgres cluster the operator watches.
+One use case is a customized Spilo image that must be configured by extra
+environment variables. Another case could be to provide custom cloud
+provider or backup settings.
+
+The last options allows for specifying environment variables individual to
+every cluster via the `env` section in the manifest. For example, if you use
+individual backup locations for each of your clusters. Or you want to disable
+WAL archiving for a certain cluster by setting `WAL_S3_BUCKET`, `WAL_GS_BUCKET`
+or `AZURE_STORAGE_ACCOUNT` to an empty string.
+
+The operator will give precedence to environment variables in the following
+order (e.g. a variable defined in 4. overrides a variable with the same name
+in 5.):
+
+1. Assigned by the operator
+2. Clone section (with WAL settings from operator config when `s3_wal_path` is empty)
+3. Standby section
+4. `env` section in cluster manifest
+5. Pod environment secret via operator config
+6. Pod environment config map via operator config
+7. WAL and logical backup settings from operator config
 
 ### Via ConfigMap
 
@@ -706,7 +730,7 @@ data:
 The key-value pairs of the Secret are all accessible as environment variables
 to the Postgres StatefulSet/pods.
 
-### For individual cluster
+### Via Postgres Cluster Manifest
 
 It is possible to define environment variables directly in the Postgres cluster
 manifest to configure it individually. The variables must be listed under the
@@ -951,6 +975,10 @@ When the `AWS_REGION` is set, `AWS_ENDPOINT` and `WALE_S3_ENDPOINT` are
 generated automatically. `WALG_S3_PREFIX` is identical to `WALE_S3_PREFIX`.
 `SCOPE` is the Postgres cluster name.
 
+:warning: If both `AWS_REGION` and `AWS_ENDPOINT` or `WALE_S3_ENDPOINT` are
+defined backups with WAL-E will fail. You can fix it by switching to WAL-G
+with `USE_WALG_BACKUP: "true"`.
+
 ### Google Cloud Platform setup
 
 To configure the operator on GCP these prerequisites that are needed:
diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md
index d245be58d..08cb37e03 100644
--- a/docs/reference/operator_parameters.md
+++ b/docs/reference/operator_parameters.md
@@ -645,7 +645,10 @@ yet officially supported.
   empty.
 
 * **aws_region**
-  AWS region used to store EBS volumes. The default is `eu-central-1`.
+  AWS region used to store EBS volumes. The default is `eu-central-1`. Note,
+  this option is not meant for specifying the AWS region for backups and
+  restore, since it can be separate from the EBS region. You have to define
+  AWS_REGION as a [custom environment variable](../administrator.md#custom-pod-environment-variables).
 
 * **additional_secret_mount**
   Additional Secret (aws or gcp credentials) to mount in the pod.
diff --git a/docs/user.md b/docs/user.md
index 4e690fb3c..0fc7c35f2 100644
--- a/docs/user.md
+++ b/docs/user.md
@@ -766,15 +766,15 @@ spec:
     uid: "efd12e58-5786-11e8-b5a7-06148230260c"
     cluster: "acid-minimal-cluster"
     timestamp: "2017-12-19T12:40:33+01:00"
-    s3_wal_path: "s3://<bucketname>/spilo/<source_db_cluster>/<UID>/wal/<PGVERSION>"
 ```
 
 Here `cluster` is a name of a source cluster that is going to be cloned. A new
 cluster will be cloned from S3, using the latest backup before the `timestamp`.
-Note, that a time zone is required for `timestamp` in the format of +00:00 which
-is UTC. You can specify the `s3_wal_path` of the source cluster or let the
-operator try to find it based on the configured `wal_[s3|gs]_bucket` and the
-specified `uid`. You can find the UID of the source cluster in its metadata:
+Note, a time zone is required for `timestamp` in the format of `+00:00` (UTC).
+
+The operator will try to find the WAL location based on the configured
+`wal_[s3|gs]_bucket` or `wal_az_storage_account` and the specified `uid`.
+You can find the UID of the source cluster in its metadata:
 
 ```yaml
 apiVersion: acid.zalan.do/v1
@@ -784,6 +784,14 @@ metadata:
   uid: efd12e58-5786-11e8-b5a7-06148230260c
 ```
 
+If your source cluster uses a WAL location different from the global
+configuration you can specify the full path under `s3_wal_path`. For
+[Google Cloud Platform](administrator.md#google-cloud-platform-setup)
+or [Azure](administrator.md#azure-setup)
+it can only be set globally with [custom Pod environment variables](administrator.md#custom-pod-environment-variables)
+or locally in the Postgres manifest's [`env`](administrator.md#via-postgres-cluster-manifest) section.
+
+
 For non AWS S3 following settings can be set to support cloning from other S3
 implementations:
 
@@ -793,6 +801,7 @@ spec:
     uid: "efd12e58-5786-11e8-b5a7-06148230260c"
     cluster: "acid-minimal-cluster"
     timestamp: "2017-12-19T12:40:33+01:00"
+    s3_wal_path: "s3://custom/path/to/bucket"
     s3_endpoint: https://s3.acme.org
     s3_access_key_id: 0123456789abcdef0123456789abcdef
     s3_secret_access_key: 0123456789abcdef0123456789abcdef
@@ -864,9 +873,8 @@ the PostgreSQL version between source and target cluster has to be the same.
 
 To start a cluster as standby, add the following `standby` section in the YAML
 file. You can stream changes from archived WAL files (AWS S3 or Google Cloud
-Storage) or from a remote primary where you specify the host address and port.
-If you leave out the port, Patroni will use `"5432"`. Only one option can be
-specfied in the manifest:
+Storage) or from a remote primary. Only one option can be specfied in the
+manifest:
 
 ```yaml
 spec:
@@ -874,12 +882,19 @@ spec:
     s3_wal_path: "s3://<bucketname>/spilo/<source_db_cluster>/<UID>/wal/<PGVERSION>"
 ```
 
+For GCS, you have to define STANDBY_GOOGLE_APPLICATION_CREDENTIALS as a
+[custom pod environment variable](administrator.md#custom-pod-environment-variables).
+It is not set from the config to allow for overridding.
+
 ```yaml
 spec:
   standby:
     gs_wal_path: "gs://<bucketname>/spilo/<source_db_cluster>/<UID>/wal/<PGVERSION>"
 ```
 
+For a remote primary you specify the host address and optionally the port.
+If you leave out the port Patroni will use `"5432"`.
+
 ```yaml
 spec:
   standby:
diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go
index 4d0fcb1c0..86a7df8dd 100644
--- a/pkg/cluster/cluster.go
+++ b/pkg/cluster/cluster.go
@@ -1318,7 +1318,7 @@ func (c *Cluster) initAdditionalOwnerRoles() {
 			}
 		}
 
-		if len(memberOf) > 1 {
+		if len(memberOf) > 0 {
 			namespace := c.Namespace
 			additionalOwnerPgUser := spec.PgUser{
 				Origin:    spec.RoleOriginSpilo,
diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go
index f531962d1..7581d1473 100644
--- a/pkg/cluster/cluster_test.go
+++ b/pkg/cluster/cluster_test.go
@@ -26,6 +26,8 @@ import (
 const (
 	superUserName       = "postgres"
 	replicationUserName = "standby"
+	exampleSpiloConfig  = `{"postgresql":{"bin_dir":"/usr/lib/postgresql/12/bin","parameters":{"autovacuum_analyze_scale_factor":"0.1"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"users":{"test":{"password":"","options":["CREATEDB","NOLOGIN"]}},"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"postgresql":{"parameters":{"max_connections":"100","max_locks_per_transaction":"64","max_worker_processes":"4"}}}}}`
+	spiloConfigDiff     = `{"postgresql":{"bin_dir":"/usr/lib/postgresql/12/bin","parameters":{"autovacuum_analyze_scale_factor":"0.1"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"users":{"test":{"password":"","options":["CREATEDB","NOLOGIN"]}},"dcs":{"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"postgresql":{"parameters":{"max_locks_per_transaction":"64","max_worker_processes":"4"}}}}}`
 )
 
 var logger = logrus.New().WithField("test", "cluster")
@@ -957,7 +959,7 @@ func TestCompareEnv(t *testing.T) {
 				},
 				{
 					Name:  "SPILO_CONFIGURATION",
-					Value: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/12/bin","parameters":{"autovacuum_analyze_scale_factor":"0.1"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"users":{"test":{"password":"","options":["CREATEDB","NOLOGIN"]}},"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"postgresql":{"parameters":{"max_connections":"100","max_locks_per_transaction":"64","max_worker_processes":"4"}}}}}`,
+					Value: exampleSpiloConfig,
 				},
 			},
 			ExpectedResult: true,
@@ -978,7 +980,7 @@ func TestCompareEnv(t *testing.T) {
 				},
 				{
 					Name:  "SPILO_CONFIGURATION",
-					Value: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/12/bin","parameters":{"autovacuum_analyze_scale_factor":"0.1"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"users":{"test":{"password":"","options":["CREATEDB","NOLOGIN"]}},"dcs":{"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"postgresql":{"parameters":{"max_locks_per_transaction":"64","max_worker_processes":"4"}}}}}`,
+					Value: spiloConfigDiff,
 				},
 			},
 			ExpectedResult: true,
@@ -999,7 +1001,7 @@ func TestCompareEnv(t *testing.T) {
 				},
 				{
 					Name:  "SPILO_CONFIGURATION",
-					Value: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/12/bin","parameters":{"autovacuum_analyze_scale_factor":"0.1"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"users":{"test":{"password":"","options":["CREATEDB","NOLOGIN"]}},"dcs":{"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"postgresql":{"parameters":{"max_locks_per_transaction":"64","max_worker_processes":"4"}}}}}`,
+					Value: exampleSpiloConfig,
 				},
 			},
 			ExpectedResult: false,
@@ -1024,7 +1026,7 @@ func TestCompareEnv(t *testing.T) {
 				},
 				{
 					Name:  "SPILO_CONFIGURATION",
-					Value: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/12/bin","parameters":{"autovacuum_analyze_scale_factor":"0.1"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"users":{"test":{"password":"","options":["CREATEDB","NOLOGIN"]}},"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"postgresql":{"parameters":{"max_connections":"100","max_locks_per_transaction":"64","max_worker_processes":"4"}}}}}`,
+					Value: exampleSpiloConfig,
 				},
 			},
 			ExpectedResult: false,
@@ -1041,7 +1043,7 @@ func TestCompareEnv(t *testing.T) {
 				},
 				{
 					Name:  "SPILO_CONFIGURATION",
-					Value: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/12/bin","parameters":{"autovacuum_analyze_scale_factor":"0.1"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"users":{"test":{"password":"","options":["CREATEDB","NOLOGIN"]}},"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"postgresql":{"parameters":{"max_connections":"100","max_locks_per_transaction":"64","max_worker_processes":"4"}}}}}`,
+					Value: exampleSpiloConfig,
 				},
 			},
 			ExpectedResult: false,
diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go
index a2186caa8..6cdf379bd 100644
--- a/pkg/cluster/k8sres.go
+++ b/pkg/cluster/k8sres.go
@@ -766,9 +766,10 @@ func (c *Cluster) generateSpiloPodEnvVars(
 	uid types.UID,
 	spiloConfiguration string,
 	cloneDescription *acidv1.CloneDescription,
-	standbyDescription *acidv1.StandbyDescription,
-	customPodEnvVarsList []v1.EnvVar) []v1.EnvVar {
+	standbyDescription *acidv1.StandbyDescription) []v1.EnvVar {
 
+	// hard-coded set of environment variables we need
+	// to guarantee core functionality of the operator
 	envVars := []v1.EnvVar{
 		{
 			Name:  "SCOPE",
@@ -875,59 +876,75 @@ func (c *Cluster) generateSpiloPodEnvVars(
 		envVars = append(envVars, c.generateCloneEnvironment(cloneDescription)...)
 	}
 
-	if c.Spec.StandbyCluster != nil {
+	if standbyDescription != nil {
 		envVars = append(envVars, c.generateStandbyEnvironment(standbyDescription)...)
 	}
 
+	// fetch cluster-specific variables that will override all subsequent global variables
 	if len(c.Spec.Env) > 0 {
 		envVars = appendEnvVars(envVars, c.Spec.Env...)
 	}
 
-	// add vars taken from pod_environment_configmap and pod_environment_secret first
-	// (to allow them to override the globals set in the operator config)
-	if len(customPodEnvVarsList) > 0 {
-		envVars = appendEnvVars(envVars, customPodEnvVarsList...)
+	// fetch variables from custom environment Secret
+	// that will override all subsequent global variables
+	secretEnvVarsList, err := c.getPodEnvironmentSecretVariables()
+	if err != nil {
+		c.logger.Warningf("%v", err)
 	}
+	envVars = appendEnvVars(envVars, secretEnvVarsList...)
 
+	// fetch variables from custom environment ConfigMap
+	// that will override all subsequent global variables
+	configMapEnvVarsList, err := c.getPodEnvironmentConfigMapVariables()
+	if err != nil {
+		c.logger.Warningf("%v", err)
+	}
+	envVars = appendEnvVars(envVars, configMapEnvVarsList...)
+
+	// global variables derived from operator configuration
+	opConfigEnvVars := make([]v1.EnvVar, 0)
 	if c.OpConfig.WALES3Bucket != "" {
-		envVars = appendEnvVars(envVars, v1.EnvVar{Name: "WAL_S3_BUCKET", Value: c.OpConfig.WALES3Bucket})
-		envVars = appendEnvVars(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))})
-		envVars = appendEnvVars(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""})
+		opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "WAL_S3_BUCKET", Value: c.OpConfig.WALES3Bucket})
+		opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))})
+		opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""})
 	}
 
 	if c.OpConfig.WALGSBucket != "" {
-		envVars = appendEnvVars(envVars, v1.EnvVar{Name: "WAL_GS_BUCKET", Value: c.OpConfig.WALGSBucket})
-		envVars = appendEnvVars(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))})
-		envVars = appendEnvVars(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""})
+		opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "WAL_GS_BUCKET", Value: c.OpConfig.WALGSBucket})
+		opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))})
+		opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""})
 	}
 
 	if c.OpConfig.WALAZStorageAccount != "" {
-		envVars = appendEnvVars(envVars, v1.EnvVar{Name: "AZURE_STORAGE_ACCOUNT", Value: c.OpConfig.WALAZStorageAccount})
-		envVars = appendEnvVars(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))})
-		envVars = appendEnvVars(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""})
+		opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "AZURE_STORAGE_ACCOUNT", Value: c.OpConfig.WALAZStorageAccount})
+		opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))})
+		opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""})
 	}
 
 	if c.OpConfig.GCPCredentials != "" {
-		envVars = appendEnvVars(envVars, v1.EnvVar{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: c.OpConfig.GCPCredentials})
+		opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: c.OpConfig.GCPCredentials})
 	}
 
 	if c.OpConfig.LogS3Bucket != "" {
-		envVars = appendEnvVars(envVars, v1.EnvVar{Name: "LOG_S3_BUCKET", Value: c.OpConfig.LogS3Bucket})
-		envVars = appendEnvVars(envVars, v1.EnvVar{Name: "LOG_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))})
-		envVars = appendEnvVars(envVars, v1.EnvVar{Name: "LOG_BUCKET_SCOPE_PREFIX", Value: ""})
+		opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "LOG_S3_BUCKET", Value: c.OpConfig.LogS3Bucket})
+		opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "LOG_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))})
+		opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "LOG_BUCKET_SCOPE_PREFIX", Value: ""})
 	}
 
+	envVars = appendEnvVars(envVars, opConfigEnvVars...)
+
 	return envVars
 }
 
 func appendEnvVars(envs []v1.EnvVar, appEnv ...v1.EnvVar) []v1.EnvVar {
-	jenvs := envs
+	collectedEnvs := envs
 	for _, env := range appEnv {
-		if !isEnvVarPresent(jenvs, env.Name) {
-			jenvs = append(jenvs, env)
+		env.Name = strings.ToUpper(env.Name)
+		if !isEnvVarPresent(collectedEnvs, env.Name) {
+			collectedEnvs = append(collectedEnvs, env)
 		}
 	}
-	return jenvs
+	return collectedEnvs
 }
 
 func isEnvVarPresent(envs []v1.EnvVar, key string) bool {
@@ -963,9 +980,11 @@ func (c *Cluster) getPodEnvironmentConfigMapVariables() ([]v1.EnvVar, error) {
 			return nil, fmt.Errorf("could not read PodEnvironmentConfigMap: %v", err)
 		}
 	}
+
 	for k, v := range cm.Data {
 		configMapPodEnvVarsList = append(configMapPodEnvVarsList, v1.EnvVar{Name: k, Value: v})
 	}
+	sort.Slice(configMapPodEnvVarsList, func(i, j int) bool { return configMapPodEnvVarsList[i].Name < configMapPodEnvVarsList[j].Name })
 	return configMapPodEnvVarsList, nil
 }
 
@@ -1015,6 +1034,7 @@ func (c *Cluster) getPodEnvironmentSecretVariables() ([]v1.EnvVar, error) {
 			}})
 	}
 
+	sort.Slice(secretPodEnvVarsList, func(i, j int) bool { return secretPodEnvVarsList[i].Name < secretPodEnvVarsList[j].Name })
 	return secretPodEnvVarsList, nil
 }
 
@@ -1104,23 +1124,6 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
 		initContainers = spec.InitContainers
 	}
 
-	// fetch env vars from custom ConfigMap
-	configMapEnvVarsList, err := c.getPodEnvironmentConfigMapVariables()
-	if err != nil {
-		return nil, err
-	}
-
-	// fetch env vars from custom ConfigMap
-	secretEnvVarsList, err := c.getPodEnvironmentSecretVariables()
-	if err != nil {
-		return nil, err
-	}
-
-	// concat all custom pod env vars and sort them
-	customPodEnvVarsList := append(configMapEnvVarsList, secretEnvVarsList...)
-	sort.Slice(customPodEnvVarsList,
-		func(i, j int) bool { return customPodEnvVarsList[i].Name < customPodEnvVarsList[j].Name })
-
 	// backward compatible check for InitContainers
 	if spec.InitContainersOld != nil {
 		msg := "manifest parameter init_containers is deprecated."
@@ -1153,9 +1156,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
 		c.Postgresql.GetUID(),
 		spiloConfiguration,
 		spec.Clone,
-		spec.StandbyCluster,
-		customPodEnvVarsList,
-	)
+		spec.StandbyCluster)
 
 	// pickup the docker image for the spilo container
 	effectiveDockerImage := util.Coalesce(spec.DockerImage, c.OpConfig.DockerImage)
@@ -1297,7 +1298,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
 
 	sidecarContainers, conflicts := mergeContainers(clusterSpecificSidecars, c.Config.OpConfig.SidecarContainers, globalSidecarContainersByDockerImage, scalyrSidecars)
 	for containerName := range conflicts {
-		c.logger.Warningf("a sidecar is specified twice. Ignoring sidecar %q in favor of %q with high a precendence",
+		c.logger.Warningf("a sidecar is specified twice. Ignoring sidecar %q in favor of %q with high a precedence",
 			containerName, containerName)
 	}
 
@@ -1819,6 +1820,7 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription)
 	cluster := description.ClusterName
 	result = append(result, v1.EnvVar{Name: "CLONE_SCOPE", Value: cluster})
 	if description.EndTimestamp == "" {
+		c.logger.Infof("cloning with basebackup from %s", cluster)
 		// cloning with basebackup, make a connection string to the cluster to clone from
 		host, port := c.getClusterServiceConnectionParameters(cluster)
 		// TODO: make some/all of those constants
@@ -1840,67 +1842,47 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription)
 				},
 			})
 	} else {
-		// cloning with S3, find out the bucket to clone
-		msg := "clone from S3 bucket"
-		c.logger.Info(msg, description.S3WalPath)
-
+		c.logger.Info("cloning from WAL location")
 		if description.S3WalPath == "" {
-			msg := "figure out which S3 bucket to use from env"
-			c.logger.Info(msg, description.S3WalPath)
+			c.logger.Info("no S3 WAL path defined - taking value from global config", description.S3WalPath)
 
 			if c.OpConfig.WALES3Bucket != "" {
-				envs := []v1.EnvVar{
-					{
-						Name:  "CLONE_WAL_S3_BUCKET",
-						Value: c.OpConfig.WALES3Bucket,
-					},
-				}
-				result = append(result, envs...)
+				c.logger.Debugf("found WALES3Bucket %s - will set CLONE_WAL_S3_BUCKET", c.OpConfig.WALES3Bucket)
+				result = append(result, v1.EnvVar{Name: "CLONE_WAL_S3_BUCKET", Value: c.OpConfig.WALES3Bucket})
 			} else if c.OpConfig.WALGSBucket != "" {
-				envs := []v1.EnvVar{
-					{
-						Name:  "CLONE_WAL_GS_BUCKET",
-						Value: c.OpConfig.WALGSBucket,
-					},
-					{
-						Name:  "CLONE_GOOGLE_APPLICATION_CREDENTIALS",
-						Value: c.OpConfig.GCPCredentials,
-					},
+				c.logger.Debugf("found WALGSBucket %s - will set CLONE_WAL_GS_BUCKET", c.OpConfig.WALGSBucket)
+				result = append(result, v1.EnvVar{Name: "CLONE_WAL_GS_BUCKET", Value: c.OpConfig.WALGSBucket})
+				if c.OpConfig.GCPCredentials != "" {
+					result = append(result, v1.EnvVar{Name: "CLONE_GOOGLE_APPLICATION_CREDENTIALS", Value: c.OpConfig.GCPCredentials})
 				}
-				result = append(result, envs...)
 			} else if c.OpConfig.WALAZStorageAccount != "" {
-				envs := []v1.EnvVar{
-					{
-						Name:  "CLONE_AZURE_STORAGE_ACCOUNT",
-						Value: c.OpConfig.WALAZStorageAccount,
-					},
-				}
-				result = append(result, envs...)
+				c.logger.Debugf("found WALAZStorageAccount %s - will set CLONE_AZURE_STORAGE_ACCOUNT", c.OpConfig.WALAZStorageAccount)
+				result = append(result, v1.EnvVar{Name: "CLONE_AZURE_STORAGE_ACCOUNT", Value: c.OpConfig.WALAZStorageAccount})
 			} else {
-				c.logger.Error("Cannot figure out S3 or GS bucket. Both are empty.")
+				c.logger.Error("cannot figure out S3 or GS bucket or AZ storage account. All options are empty in the config.")
 			}
 
+			// append suffix because WAL location name is not the whole path
+			result = append(result, v1.EnvVar{Name: "CLONE_WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(description.UID)})
+		} else {
+			c.logger.Debugf("use S3WalPath %s from the manifest", description.S3WalPath)
+
 			envs := []v1.EnvVar{
+				{
+					Name:  "CLONE_WALE_S3_PREFIX",
+					Value: description.S3WalPath,
+				},
 				{
 					Name:  "CLONE_WAL_BUCKET_SCOPE_SUFFIX",
-					Value: getBucketScopeSuffix(description.UID),
+					Value: "",
 				},
 			}
 
 			result = append(result, envs...)
-		} else {
-			msg := "use custom parsed S3WalPath %s from the manifest"
-			c.logger.Warningf(msg, description.S3WalPath)
-
-			result = append(result, v1.EnvVar{
-				Name:  "CLONE_WALE_S3_PREFIX",
-				Value: description.S3WalPath,
-			})
 		}
 
 		result = append(result, v1.EnvVar{Name: "CLONE_METHOD", Value: "CLONE_WITH_WALE"})
 		result = append(result, v1.EnvVar{Name: "CLONE_TARGET_TIME", Value: description.EndTimestamp})
-		result = append(result, v1.EnvVar{Name: "CLONE_WAL_BUCKET_SCOPE_PREFIX", Value: ""})
 
 		if description.S3Endpoint != "" {
 			result = append(result, v1.EnvVar{Name: "CLONE_AWS_ENDPOINT", Value: description.S3Endpoint})
@@ -1933,7 +1915,7 @@ func (c *Cluster) generateStandbyEnvironment(description *acidv1.StandbyDescript
 	result := make([]v1.EnvVar, 0)
 
 	if description.StandbyHost != "" {
-		// standby from remote primary
+		c.logger.Info("standby cluster streaming from remote primary")
 		result = append(result, v1.EnvVar{
 			Name:  "STANDBY_HOST",
 			Value: description.StandbyHost,
@@ -1945,30 +1927,20 @@ func (c *Cluster) generateStandbyEnvironment(description *acidv1.StandbyDescript
 			})
 		}
 	} else {
+		c.logger.Info("standby cluster streaming from WAL location")
 		if description.S3WalPath != "" {
-			// standby with S3, find out the bucket to setup standby
-			msg := "Standby from S3 bucket using custom parsed S3WalPath from the manifest %s "
-			c.logger.Infof(msg, description.S3WalPath)
-
 			result = append(result, v1.EnvVar{
 				Name:  "STANDBY_WALE_S3_PREFIX",
 				Value: description.S3WalPath,
 			})
 		} else if description.GSWalPath != "" {
-			msg := "Standby from GS bucket using custom parsed GSWalPath from the manifest %s "
-			c.logger.Infof(msg, description.GSWalPath)
-
-			envs := []v1.EnvVar{
-				{
-					Name:  "STANDBY_WALE_GS_PREFIX",
-					Value: description.GSWalPath,
-				},
-				{
-					Name:  "STANDBY_GOOGLE_APPLICATION_CREDENTIALS",
-					Value: c.OpConfig.GCPCredentials,
-				},
-			}
-			result = append(result, envs...)
+			result = append(result, v1.EnvVar{
+				Name:  "STANDBY_WALE_GS_PREFIX",
+				Value: description.GSWalPath,
+			})
+		} else {
+			c.logger.Error("no WAL path specified in standby section")
+			return result
 		}
 
 		result = append(result, v1.EnvVar{Name: "STANDBY_METHOD", Value: "STANDBY_WITH_WALE"})
diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go
index 94e78572c..7a1bdeaf9 100644
--- a/pkg/cluster/k8sres_test.go
+++ b/pkg/cluster/k8sres_test.go
@@ -49,6 +49,7 @@ type ExpectedValue struct {
 	envIndex       int
 	envVarConstant string
 	envVarValue    string
+	envVarValueRef *v1.EnvVarSource
 }
 
 func TestGenerateSpiloJSONConfiguration(t *testing.T) {
@@ -117,29 +118,360 @@ func TestGenerateSpiloJSONConfiguration(t *testing.T) {
 	}
 }
 
-func TestGenerateSpiloPodEnvVars(t *testing.T) {
-	var cluster = New(
-		Config{
-			OpConfig: config.Config{
-				WALGSBucket:    "wale-gs-bucket",
-				ProtectedRoles: []string{"admin"},
-				Auth: config.Auth{
-					SuperUsername:       superUserName,
-					ReplicationUsername: replicationUserName,
+func TestExtractPgVersionFromBinPath(t *testing.T) {
+	testName := "TestExtractPgVersionFromBinPath"
+	tests := []struct {
+		subTest  string
+		binPath  string
+		template string
+		expected string
+	}{
+		{
+			subTest:  "test current bin path with decimal against hard coded template",
+			binPath:  "/usr/lib/postgresql/9.6/bin",
+			template: pgBinariesLocationTemplate,
+			expected: "9.6",
+		},
+		{
+			subTest:  "test current bin path against hard coded template",
+			binPath:  "/usr/lib/postgresql/12/bin",
+			template: pgBinariesLocationTemplate,
+			expected: "12",
+		},
+		{
+			subTest:  "test alternative bin path against a matching template",
+			binPath:  "/usr/pgsql-12/bin",
+			template: "/usr/pgsql-%v/bin",
+			expected: "12",
+		},
+	}
+
+	for _, tt := range tests {
+		pgVersion, err := extractPgVersionFromBinPath(tt.binPath, tt.template)
+		if err != nil {
+			t.Errorf("Unexpected error: %v", err)
+		}
+		if pgVersion != tt.expected {
+			t.Errorf("%s %s: Expected version %s, have %s instead",
+				testName, tt.subTest, tt.expected, pgVersion)
+		}
+	}
+}
+
+const (
+	testPodEnvironmentConfigMapName      = "pod_env_cm"
+	testPodEnvironmentSecretName         = "pod_env_sc"
+	testPodEnvironmentObjectNotExists    = "idonotexist"
+	testPodEnvironmentSecretNameAPIError = "pod_env_sc_apierror"
+	testResourceCheckInterval            = 3
+	testResourceCheckTimeout             = 10
+)
+
+type mockSecret struct {
+	v1core.SecretInterface
+}
+
+type mockConfigMap struct {
+	v1core.ConfigMapInterface
+}
+
+func (c *mockSecret) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.Secret, error) {
+	if name == testPodEnvironmentSecretNameAPIError {
+		return nil, fmt.Errorf("Secret PodEnvironmentSecret API error")
+	}
+	if name != testPodEnvironmentSecretName {
+		return nil, k8serrors.NewNotFound(schema.GroupResource{Group: "core", Resource: "secret"}, name)
+	}
+	secret := &v1.Secret{}
+	secret.Name = testPodEnvironmentSecretName
+	secret.Data = map[string][]byte{
+		"clone_aws_access_key_id":                []byte("0123456789abcdef0123456789abcdef"),
+		"custom_variable":                        []byte("secret-test"),
+		"standby_google_application_credentials": []byte("0123456789abcdef0123456789abcdef"),
+	}
+	return secret, nil
+}
+
+func (c *mockConfigMap) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.ConfigMap, error) {
+	if name != testPodEnvironmentConfigMapName {
+		return nil, fmt.Errorf("NotFound")
+	}
+	configmap := &v1.ConfigMap{}
+	configmap.Name = testPodEnvironmentConfigMapName
+	configmap.Data = map[string]string{
+		// hard-coded clone env variable, can set when not specified in manifest
+		"clone_aws_endpoint": "s3.eu-west-1.amazonaws.com",
+		// custom variable, can be overridden by c.Spec.Env
+		"custom_variable": "configmap-test",
+		// hard-coded env variable, can not be overridden
+		"kubernetes_scope_label": "pgaas",
+		// hard-coded env variable, can be overridden
+		"wal_s3_bucket": "global-s3-bucket-configmap",
+	}
+	return configmap, nil
+}
+
+type MockSecretGetter struct {
+}
+
+type MockConfigMapsGetter struct {
+}
+
+func (c *MockSecretGetter) Secrets(namespace string) v1core.SecretInterface {
+	return &mockSecret{}
+}
+
+func (c *MockConfigMapsGetter) ConfigMaps(namespace string) v1core.ConfigMapInterface {
+	return &mockConfigMap{}
+}
+
+func newMockKubernetesClient() k8sutil.KubernetesClient {
+	return k8sutil.KubernetesClient{
+		SecretsGetter:    &MockSecretGetter{},
+		ConfigMapsGetter: &MockConfigMapsGetter{},
+	}
+}
+func newMockCluster(opConfig config.Config) *Cluster {
+	cluster := &Cluster{
+		Config:     Config{OpConfig: opConfig},
+		KubeClient: newMockKubernetesClient(),
+		logger:     logger,
+	}
+	return cluster
+}
+
+func TestPodEnvironmentConfigMapVariables(t *testing.T) {
+	testName := "TestPodEnvironmentConfigMapVariables"
+	tests := []struct {
+		subTest  string
+		opConfig config.Config
+		envVars  []v1.EnvVar
+		err      error
+	}{
+		{
+			subTest: "no PodEnvironmentConfigMap",
+			envVars: []v1.EnvVar{},
+		},
+		{
+			subTest: "missing PodEnvironmentConfigMap",
+			opConfig: config.Config{
+				Resources: config.Resources{
+					PodEnvironmentConfigMap: spec.NamespacedName{
+						Name: testPodEnvironmentObjectNotExists,
+					},
 				},
 			},
-		}, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder)
+			err: fmt.Errorf("could not read PodEnvironmentConfigMap: NotFound"),
+		},
+		{
+			subTest: "Pod environment vars configured by PodEnvironmentConfigMap",
+			opConfig: config.Config{
+				Resources: config.Resources{
+					PodEnvironmentConfigMap: spec.NamespacedName{
+						Name: testPodEnvironmentConfigMapName,
+					},
+				},
+			},
+			envVars: []v1.EnvVar{
+				{
+					Name:  "clone_aws_endpoint",
+					Value: "s3.eu-west-1.amazonaws.com",
+				},
+				{
+					Name:  "custom_variable",
+					Value: "configmap-test",
+				},
+				{
+					Name:  "kubernetes_scope_label",
+					Value: "pgaas",
+				},
+				{
+					Name:  "wal_s3_bucket",
+					Value: "global-s3-bucket-configmap",
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		c := newMockCluster(tt.opConfig)
+		vars, err := c.getPodEnvironmentConfigMapVariables()
+		if !reflect.DeepEqual(vars, tt.envVars) {
+			t.Errorf("%s %s: expected `%v` but got `%v`",
+				testName, tt.subTest, tt.envVars, vars)
+		}
+		if tt.err != nil {
+			if err.Error() != tt.err.Error() {
+				t.Errorf("%s %s: expected error `%v` but got `%v`",
+					testName, tt.subTest, tt.err, err)
+			}
+		} else {
+			if err != nil {
+				t.Errorf("%s %s: expected no error but got error: `%v`",
+					testName, tt.subTest, err)
+			}
+		}
+	}
+}
 
-	expectedValuesGSBucket := []ExpectedValue{
+// Test if the keys of an existing secret are properly referenced
+func TestPodEnvironmentSecretVariables(t *testing.T) {
+	maxRetries := int(testResourceCheckTimeout / testResourceCheckInterval)
+	testName := "TestPodEnvironmentSecretVariables"
+	tests := []struct {
+		subTest  string
+		opConfig config.Config
+		envVars  []v1.EnvVar
+		err      error
+	}{
+		{
+			subTest: "No PodEnvironmentSecret configured",
+			envVars: []v1.EnvVar{},
+		},
+		{
+			subTest: "Secret referenced by PodEnvironmentSecret does not exist",
+			opConfig: config.Config{
+				Resources: config.Resources{
+					PodEnvironmentSecret:  testPodEnvironmentObjectNotExists,
+					ResourceCheckInterval: time.Duration(testResourceCheckInterval),
+					ResourceCheckTimeout:  time.Duration(testResourceCheckTimeout),
+				},
+			},
+			err: fmt.Errorf("could not read Secret PodEnvironmentSecretName: still failing after %d retries: secret.core %q not found", maxRetries, testPodEnvironmentObjectNotExists),
+		},
+		{
+			subTest: "API error during PodEnvironmentSecret retrieval",
+			opConfig: config.Config{
+				Resources: config.Resources{
+					PodEnvironmentSecret:  testPodEnvironmentSecretNameAPIError,
+					ResourceCheckInterval: time.Duration(testResourceCheckInterval),
+					ResourceCheckTimeout:  time.Duration(testResourceCheckTimeout),
+				},
+			},
+			err: fmt.Errorf("could not read Secret PodEnvironmentSecretName: Secret PodEnvironmentSecret API error"),
+		},
+		{
+			subTest: "Pod environment vars reference all keys from secret configured by PodEnvironmentSecret",
+			opConfig: config.Config{
+				Resources: config.Resources{
+					PodEnvironmentSecret:  testPodEnvironmentSecretName,
+					ResourceCheckInterval: time.Duration(testResourceCheckInterval),
+					ResourceCheckTimeout:  time.Duration(testResourceCheckTimeout),
+				},
+			},
+			envVars: []v1.EnvVar{
+				{
+					Name: "clone_aws_access_key_id",
+					ValueFrom: &v1.EnvVarSource{
+						SecretKeyRef: &v1.SecretKeySelector{
+							LocalObjectReference: v1.LocalObjectReference{
+								Name: testPodEnvironmentSecretName,
+							},
+							Key: "clone_aws_access_key_id",
+						},
+					},
+				},
+				{
+					Name: "custom_variable",
+					ValueFrom: &v1.EnvVarSource{
+						SecretKeyRef: &v1.SecretKeySelector{
+							LocalObjectReference: v1.LocalObjectReference{
+								Name: testPodEnvironmentSecretName,
+							},
+							Key: "custom_variable",
+						},
+					},
+				},
+				{
+					Name: "standby_google_application_credentials",
+					ValueFrom: &v1.EnvVarSource{
+						SecretKeyRef: &v1.SecretKeySelector{
+							LocalObjectReference: v1.LocalObjectReference{
+								Name: testPodEnvironmentSecretName,
+							},
+							Key: "standby_google_application_credentials",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		c := newMockCluster(tt.opConfig)
+		vars, err := c.getPodEnvironmentSecretVariables()
+		sort.Slice(vars, func(i, j int) bool { return vars[i].Name < vars[j].Name })
+		if !reflect.DeepEqual(vars, tt.envVars) {
+			t.Errorf("%s %s: expected `%v` but got `%v`",
+				testName, tt.subTest, tt.envVars, vars)
+		}
+		if tt.err != nil {
+			if err.Error() != tt.err.Error() {
+				t.Errorf("%s %s: expected error `%v` but got `%v`",
+					testName, tt.subTest, tt.err, err)
+			}
+		} else {
+			if err != nil {
+				t.Errorf("%s %s: expected no error but got error: `%v`",
+					testName, tt.subTest, err)
+			}
+		}
+	}
+
+}
+
+func testEnvs(cluster *Cluster, podSpec *v1.PodTemplateSpec, role PostgresRole) error {
+	required := map[string]bool{
+		"PGHOST":                 false,
+		"PGPORT":                 false,
+		"PGUSER":                 false,
+		"PGSCHEMA":               false,
+		"PGPASSWORD":             false,
+		"CONNECTION_POOLER_MODE": false,
+		"CONNECTION_POOLER_PORT": false,
+	}
+
+	container := getPostgresContainer(&podSpec.Spec)
+	envs := container.Env
+	for _, env := range envs {
+		required[env.Name] = true
+	}
+
+	for env, value := range required {
+		if !value {
+			return fmt.Errorf("Environment variable %s is not present", env)
+		}
+	}
+
+	return nil
+}
+
+func TestGenerateSpiloPodEnvVars(t *testing.T) {
+	var dummyUUID = "efd12e58-5786-11e8-b5a7-06148230260c"
+
+	expectedClusterNameLabel := []ExpectedValue{
+		{
+			envIndex:       5,
+			envVarConstant: "KUBERNETES_SCOPE_LABEL",
+			envVarValue:    "cluster-name",
+		},
+	}
+	expectedSpiloWalPathCompat := []ExpectedValue{
+		{
+			envIndex:       12,
+			envVarConstant: "ENABLE_WAL_PATH_COMPAT",
+			envVarValue:    "true",
+		},
+	}
+	expectedValuesS3Bucket := []ExpectedValue{
 		{
 			envIndex:       15,
-			envVarConstant: "WAL_GS_BUCKET",
-			envVarValue:    "wale-gs-bucket",
+			envVarConstant: "WAL_S3_BUCKET",
+			envVarValue:    "global-s3-bucket",
 		},
 		{
 			envIndex:       16,
 			envVarConstant: "WAL_BUCKET_SCOPE_SUFFIX",
-			envVarValue:    "/SomeUUID",
+			envVarValue:    fmt.Sprintf("/%s", dummyUUID),
 		},
 		{
 			envIndex:       17,
@@ -147,17 +479,16 @@ func TestGenerateSpiloPodEnvVars(t *testing.T) {
 			envVarValue:    "",
 		},
 	}
-
 	expectedValuesGCPCreds := []ExpectedValue{
 		{
 			envIndex:       15,
 			envVarConstant: "WAL_GS_BUCKET",
-			envVarValue:    "wale-gs-bucket",
+			envVarValue:    "global-gs-bucket",
 		},
 		{
 			envIndex:       16,
 			envVarConstant: "WAL_BUCKET_SCOPE_SUFFIX",
-			envVarValue:    "/SomeUUID",
+			envVarValue:    fmt.Sprintf("/%s", dummyUUID),
 		},
 		{
 			envIndex:       17,
@@ -167,28 +498,116 @@ func TestGenerateSpiloPodEnvVars(t *testing.T) {
 		{
 			envIndex:       18,
 			envVarConstant: "GOOGLE_APPLICATION_CREDENTIALS",
-			envVarValue:    "some_path_to_credentials",
+			envVarValue:    "some-path-to-credentials",
 		},
 	}
-	expectedClusterNameLabel := []ExpectedValue{
+	expectedS3BucketConfigMap := []ExpectedValue{
 		{
-			envIndex:       5,
-			envVarConstant: "KUBERNETES_SCOPE_LABEL",
-			envVarValue:    "cluster-name",
+			envIndex:       17,
+			envVarConstant: "WAL_S3_BUCKET",
+			envVarValue:    "global-s3-bucket-configmap",
 		},
 	}
-	expectedCustomS3Bucket := []ExpectedValue{
+	expectedCustomS3BucketSpec := []ExpectedValue{
 		{
 			envIndex:       15,
 			envVarConstant: "WAL_S3_BUCKET",
 			envVarValue:    "custom-s3-bucket",
 		},
 	}
-	expectedCustomVariable := []ExpectedValue{
+	expectedCustomVariableSecret := []ExpectedValue{
+		{
+			envIndex:       16,
+			envVarConstant: "CUSTOM_VARIABLE",
+			envVarValueRef: &v1.EnvVarSource{
+				SecretKeyRef: &v1.SecretKeySelector{
+					LocalObjectReference: v1.LocalObjectReference{
+						Name: testPodEnvironmentSecretName,
+					},
+					Key: "custom_variable",
+				},
+			},
+		},
+	}
+	expectedCustomVariableConfigMap := []ExpectedValue{
+		{
+			envIndex:       16,
+			envVarConstant: "CUSTOM_VARIABLE",
+			envVarValue:    "configmap-test",
+		},
+	}
+	expectedCustomVariableSpec := []ExpectedValue{
 		{
 			envIndex:       15,
 			envVarConstant: "CUSTOM_VARIABLE",
-			envVarValue:    "cluster-variable",
+			envVarValue:    "spec-env-test",
+		},
+	}
+	expectedCloneEnvSpec := []ExpectedValue{
+		{
+			envIndex:       16,
+			envVarConstant: "CLONE_WALE_S3_PREFIX",
+			envVarValue:    "s3://another-bucket",
+		},
+		{
+			envIndex:       17,
+			envVarConstant: "CLONE_WAL_BUCKET_SCOPE_SUFFIX",
+			envVarValue:    "",
+		},
+		{
+			envIndex:       20,
+			envVarConstant: "CLONE_AWS_ENDPOINT",
+			envVarValue:    "s3.eu-central-1.amazonaws.com",
+		},
+	}
+	expectedCloneEnvConfigMap := []ExpectedValue{
+		{
+			envIndex:       16,
+			envVarConstant: "CLONE_WAL_S3_BUCKET",
+			envVarValue:    "global-s3-bucket",
+		},
+		{
+			envIndex:       17,
+			envVarConstant: "CLONE_WAL_BUCKET_SCOPE_SUFFIX",
+			envVarValue:    fmt.Sprintf("/%s", dummyUUID),
+		},
+		{
+			envIndex:       20,
+			envVarConstant: "CLONE_AWS_ENDPOINT",
+			envVarValue:    "s3.eu-west-1.amazonaws.com",
+		},
+	}
+	expectedCloneEnvSecret := []ExpectedValue{
+		{
+			envIndex:       20,
+			envVarConstant: "CLONE_AWS_ACCESS_KEY_ID",
+			envVarValueRef: &v1.EnvVarSource{
+				SecretKeyRef: &v1.SecretKeySelector{
+					LocalObjectReference: v1.LocalObjectReference{
+						Name: testPodEnvironmentSecretName,
+					},
+					Key: "clone_aws_access_key_id",
+				},
+			},
+		},
+	}
+	expectedStandbyEnvSecret := []ExpectedValue{
+		{
+			envIndex:       15,
+			envVarConstant: "STANDBY_WALE_GS_PREFIX",
+			envVarValue:    "gs://some/path/",
+		},
+		{
+			envIndex:       20,
+			envVarConstant: "STANDBY_GOOGLE_APPLICATION_CREDENTIALS",
+			envVarValueRef: &v1.EnvVarSource{
+				SecretKeyRef: &v1.SecretKeySelector{
+					LocalObjectReference: v1.LocalObjectReference{
+						Name: testPodEnvironmentSecretName,
+					},
+					Key: "standby_google_application_credentials",
+				},
+			},
 		},
 	}
 
@@ -196,51 +615,51 @@ func TestGenerateSpiloPodEnvVars(t *testing.T) {
 	tests := []struct {
 		subTest            string
 		opConfig           config.Config
-		uid                types.UID
-		spiloConfig        string
 		cloneDescription   *acidv1.CloneDescription
 		standbyDescription *acidv1.StandbyDescription
-		customEnvList      []v1.EnvVar
 		expectedValues     []ExpectedValue
 		pgsql              acidv1.Postgresql
 	}{
 		{
-			subTest: "Will set WAL_GS_BUCKET env",
+			subTest: "will set ENABLE_WAL_PATH_COMPAT env",
 			opConfig: config.Config{
-				WALGSBucket: "wale-gs-bucket",
+				EnableSpiloWalPathCompat: true,
 			},
-			uid:                "SomeUUID",
-			spiloConfig:        "someConfig",
 			cloneDescription:   &acidv1.CloneDescription{},
 			standbyDescription: &acidv1.StandbyDescription{},
-			customEnvList:      []v1.EnvVar{},
-			expectedValues:     expectedValuesGSBucket,
+			expectedValues:     expectedSpiloWalPathCompat,
 		},
 		{
-			subTest: "Will set GOOGLE_APPLICATION_CREDENTIALS env",
+			subTest: "will set WAL_S3_BUCKET env",
 			opConfig: config.Config{
-				WALGSBucket:    "wale-gs-bucket",
-				GCPCredentials: "some_path_to_credentials",
+				WALES3Bucket: "global-s3-bucket",
+			},
+			cloneDescription:   &acidv1.CloneDescription{},
+			standbyDescription: &acidv1.StandbyDescription{},
+			expectedValues:     expectedValuesS3Bucket,
+		},
+		{
+			subTest: "will set GOOGLE_APPLICATION_CREDENTIALS env",
+			opConfig: config.Config{
+				WALGSBucket:    "global-gs-bucket",
+				GCPCredentials: "some-path-to-credentials",
 			},
-			uid:                "SomeUUID",
-			spiloConfig:        "someConfig",
 			cloneDescription:   &acidv1.CloneDescription{},
 			standbyDescription: &acidv1.StandbyDescription{},
-			customEnvList:      []v1.EnvVar{},
 			expectedValues:     expectedValuesGCPCreds,
 		},
 		{
-			subTest: "Will not overwrite global KUBERNETES_SCOPE_LABEL parameter from the cluster Env option",
+			subTest: "will not override global config KUBERNETES_SCOPE_LABEL parameter",
 			opConfig: config.Config{
 				Resources: config.Resources{
 					ClusterNameLabel: "cluster-name",
+					PodEnvironmentConfigMap: spec.NamespacedName{
+						Name: testPodEnvironmentConfigMapName, // contains kubernetes_scope_label, too
+					},
 				},
 			},
-			uid:                "SomeUUID",
-			spiloConfig:        "someConfig",
 			cloneDescription:   &acidv1.CloneDescription{},
 			standbyDescription: &acidv1.StandbyDescription{},
-			customEnvList:      []v1.EnvVar{},
 			expectedValues:     expectedClusterNameLabel,
 			pgsql: acidv1.Postgresql{
 				Spec: acidv1.PostgresSpec{
@@ -254,16 +673,27 @@ func TestGenerateSpiloPodEnvVars(t *testing.T) {
 			},
 		},
 		{
-			subTest: "Will overwrite global WAL_S3_BUCKET parameter from the cluster Env option",
+			subTest: "will override global WAL_S3_BUCKET parameter from pod environment config map",
+			opConfig: config.Config{
+				Resources: config.Resources{
+					PodEnvironmentConfigMap: spec.NamespacedName{
+						Name: testPodEnvironmentConfigMapName,
+					},
+				},
+				WALES3Bucket: "global-s3-bucket",
+			},
+			cloneDescription:   &acidv1.CloneDescription{},
+			standbyDescription: &acidv1.StandbyDescription{},
+			expectedValues:     expectedS3BucketConfigMap,
+		},
+		{
+			subTest: "will override global WAL_S3_BUCKET parameter from manifest `env` section",
 			opConfig: config.Config{
 				WALGSBucket: "global-s3-bucket",
 			},
-			uid:                "SomeUUID",
-			spiloConfig:        "someConfig",
 			cloneDescription:   &acidv1.CloneDescription{},
 			standbyDescription: &acidv1.StandbyDescription{},
-			customEnvList:      []v1.EnvVar{},
-			expectedValues:     expectedCustomS3Bucket,
+			expectedValues:     expectedCustomS3BucketSpec,
 			pgsql: acidv1.Postgresql{
 				Spec: acidv1.PostgresSpec{
 					Env: []v1.EnvVar{
@@ -276,276 +706,164 @@ func TestGenerateSpiloPodEnvVars(t *testing.T) {
 			},
 		},
 		{
-			subTest:            "Will overwrite custom variable parameter from the cluster Env option",
-			uid:                "SomeUUID",
-			spiloConfig:        "someConfig",
-			cloneDescription:   &acidv1.CloneDescription{},
-			standbyDescription: &acidv1.StandbyDescription{},
-			customEnvList: []v1.EnvVar{
-				{
-					Name:  "CUSTOM_VARIABLE",
-					Value: "custom-variable",
+			subTest: "will set CUSTOM_VARIABLE from pod environment secret and not config map",
+			opConfig: config.Config{
+				Resources: config.Resources{
+					PodEnvironmentConfigMap: spec.NamespacedName{
+						Name: testPodEnvironmentConfigMapName,
+					},
+					PodEnvironmentSecret:  testPodEnvironmentSecretName,
+					ResourceCheckInterval: time.Duration(testResourceCheckInterval),
+					ResourceCheckTimeout:  time.Duration(testResourceCheckTimeout),
 				},
 			},
-			expectedValues: expectedCustomVariable,
+			cloneDescription:   &acidv1.CloneDescription{},
+			standbyDescription: &acidv1.StandbyDescription{},
+			expectedValues:     expectedCustomVariableSecret,
+		},
+		{
+			subTest: "will set CUSTOM_VARIABLE from pod environment config map",
+			opConfig: config.Config{
+				Resources: config.Resources{
+					PodEnvironmentConfigMap: spec.NamespacedName{
+						Name: testPodEnvironmentConfigMapName,
+					},
+				},
+			},
+			cloneDescription:   &acidv1.CloneDescription{},
+			standbyDescription: &acidv1.StandbyDescription{},
+			expectedValues:     expectedCustomVariableConfigMap,
+		},
+		{
+			subTest: "will override CUSTOM_VARIABLE of pod environment secret/configmap from manifest `env` section",
+			opConfig: config.Config{
+				Resources: config.Resources{
+					PodEnvironmentConfigMap: spec.NamespacedName{
+						Name: testPodEnvironmentConfigMapName,
+					},
+					PodEnvironmentSecret:  testPodEnvironmentSecretName,
+					ResourceCheckInterval: time.Duration(testResourceCheckInterval),
+					ResourceCheckTimeout:  time.Duration(testResourceCheckTimeout),
+				},
+			},
+			cloneDescription:   &acidv1.CloneDescription{},
+			standbyDescription: &acidv1.StandbyDescription{},
+			expectedValues:     expectedCustomVariableSpec,
 			pgsql: acidv1.Postgresql{
 				Spec: acidv1.PostgresSpec{
 					Env: []v1.EnvVar{
 						{
 							Name:  "CUSTOM_VARIABLE",
-							Value: "cluster-variable",
+							Value: "spec-env-test",
 						},
 					},
 				},
 			},
 		},
+		{
+			subTest: "will set CLONE_ parameters from spec and not global config or pod environment config map",
+			opConfig: config.Config{
+				Resources: config.Resources{
+					PodEnvironmentConfigMap: spec.NamespacedName{
+						Name: testPodEnvironmentConfigMapName,
+					},
+				},
+				WALES3Bucket: "global-s3-bucket",
+			},
+			cloneDescription: &acidv1.CloneDescription{
+				ClusterName:  "test-cluster",
+				EndTimestamp: "somewhen",
+				UID:          dummyUUID,
+				S3WalPath:    "s3://another-bucket",
+				S3Endpoint:   "s3.eu-central-1.amazonaws.com",
+			},
+			standbyDescription: &acidv1.StandbyDescription{},
+			expectedValues:     expectedCloneEnvSpec,
+		},
+		{
+			subTest: "will set CLONE_AWS_ENDPOINT parameter from pod environment config map",
+			opConfig: config.Config{
+				Resources: config.Resources{
+					PodEnvironmentConfigMap: spec.NamespacedName{
+						Name: testPodEnvironmentConfigMapName,
+					},
+				},
+				WALES3Bucket: "global-s3-bucket",
+			},
+			cloneDescription: &acidv1.CloneDescription{
+				ClusterName:  "test-cluster",
+				EndTimestamp: "somewhen",
+				UID:          dummyUUID,
+			},
+			standbyDescription: &acidv1.StandbyDescription{},
+			expectedValues:     expectedCloneEnvConfigMap,
+		},
+		{
+			subTest: "will set CLONE_AWS_ACCESS_KEY_ID parameter from pod environment secret",
+			opConfig: config.Config{
+				Resources: config.Resources{
+					PodEnvironmentSecret:  testPodEnvironmentSecretName,
+					ResourceCheckInterval: time.Duration(testResourceCheckInterval),
+					ResourceCheckTimeout:  time.Duration(testResourceCheckTimeout),
+				},
+				WALES3Bucket: "global-s3-bucket",
+			},
+			cloneDescription: &acidv1.CloneDescription{
+				ClusterName:  "test-cluster",
+				EndTimestamp: "somewhen",
+				UID:          dummyUUID,
+			},
+			standbyDescription: &acidv1.StandbyDescription{},
+			expectedValues:     expectedCloneEnvSecret,
+		},
+		{
+			subTest: "will set STANDBY_GOOGLE_APPLICATION_CREDENTIALS parameter from pod environment secret",
+			opConfig: config.Config{
+				Resources: config.Resources{
+					PodEnvironmentSecret:  testPodEnvironmentSecretName,
+					ResourceCheckInterval: time.Duration(testResourceCheckInterval),
+					ResourceCheckTimeout:  time.Duration(testResourceCheckTimeout),
+				},
+				WALES3Bucket: "global-s3-bucket",
+			},
+			cloneDescription: &acidv1.CloneDescription{},
+			standbyDescription: &acidv1.StandbyDescription{
+				GSWalPath: "gs://some/path/",
+			},
+			expectedValues: expectedStandbyEnvSecret,
+		},
 	}
 
 	for _, tt := range tests {
-		cluster.OpConfig = tt.opConfig
-		cluster.Postgresql = tt.pgsql
-
-		actualEnvs := cluster.generateSpiloPodEnvVars(tt.uid, tt.spiloConfig, tt.cloneDescription, tt.standbyDescription, tt.customEnvList)
+		c := newMockCluster(tt.opConfig)
+		c.Postgresql = tt.pgsql
+		actualEnvs := c.generateSpiloPodEnvVars(
+			types.UID(dummyUUID), exampleSpiloConfig, tt.cloneDescription, tt.standbyDescription)
 
 		for _, ev := range tt.expectedValues {
 			env := actualEnvs[ev.envIndex]
 
 			if env.Name != ev.envVarConstant {
-				t.Errorf("%s %s: Expected env name %s, have %s instead",
+				t.Errorf("%s %s: expected env name %s, have %s instead",
 					testName, tt.subTest, ev.envVarConstant, env.Name)
 			}
 
+			if ev.envVarValueRef != nil {
+				if !reflect.DeepEqual(env.ValueFrom, ev.envVarValueRef) {
+					t.Errorf("%s %s: expected env value reference %#v, have %#v instead",
+						testName, tt.subTest, ev.envVarValueRef, env.ValueFrom)
+				}
+				continue
+			}
+
 			if env.Value != ev.envVarValue {
-				t.Errorf("%s %s: Expected env value %s, have %s instead",
+				t.Errorf("%s %s: expected env value %s, have %s instead",
 					testName, tt.subTest, ev.envVarValue, env.Value)
 			}
 		}
 	}
 }
 
-func TestCreateLoadBalancerLogic(t *testing.T) {
-	var cluster = New(
-		Config{
-			OpConfig: config.Config{
-				ProtectedRoles: []string{"admin"},
-				Auth: config.Auth{
-					SuperUsername:       superUserName,
-					ReplicationUsername: replicationUserName,
-				},
-			},
-		}, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder)
-
-	testName := "TestCreateLoadBalancerLogic"
-	tests := []struct {
-		subtest  string
-		role     PostgresRole
-		spec     *acidv1.PostgresSpec
-		opConfig config.Config
-		result   bool
-	}{
-		{
-			subtest:  "new format, load balancer is enabled for replica",
-			role:     Replica,
-			spec:     &acidv1.PostgresSpec{EnableReplicaLoadBalancer: util.True()},
-			opConfig: config.Config{},
-			result:   true,
-		},
-		{
-			subtest:  "new format, load balancer is disabled for replica",
-			role:     Replica,
-			spec:     &acidv1.PostgresSpec{EnableReplicaLoadBalancer: util.False()},
-			opConfig: config.Config{},
-			result:   false,
-		},
-		{
-			subtest:  "new format, load balancer isn't specified for replica",
-			role:     Replica,
-			spec:     &acidv1.PostgresSpec{EnableReplicaLoadBalancer: nil},
-			opConfig: config.Config{EnableReplicaLoadBalancer: true},
-			result:   true,
-		},
-		{
-			subtest:  "new format, load balancer isn't specified for replica",
-			role:     Replica,
-			spec:     &acidv1.PostgresSpec{EnableReplicaLoadBalancer: nil},
-			opConfig: config.Config{EnableReplicaLoadBalancer: false},
-			result:   false,
-		},
-	}
-	for _, tt := range tests {
-		cluster.OpConfig = tt.opConfig
-		result := cluster.shouldCreateLoadBalancerForService(tt.role, tt.spec)
-		if tt.result != result {
-			t.Errorf("%s %s: Load balancer is %t, expect %t for role %#v and spec %#v",
-				testName, tt.subtest, result, tt.result, tt.role, tt.spec)
-		}
-	}
-}
-
-func TestGeneratePodDisruptionBudget(t *testing.T) {
-	tests := []struct {
-		c   *Cluster
-		out policyv1beta1.PodDisruptionBudget
-	}{
-		// With multiple instances.
-		{
-			New(
-				Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-pdb"}},
-				k8sutil.KubernetesClient{},
-				acidv1.Postgresql{
-					ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"},
-					Spec:       acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}},
-				logger,
-				eventRecorder),
-			policyv1beta1.PodDisruptionBudget{
-				ObjectMeta: metav1.ObjectMeta{
-					Name:      "postgres-myapp-database-pdb",
-					Namespace: "myapp",
-					Labels:    map[string]string{"team": "myapp", "cluster-name": "myapp-database"},
-				},
-				Spec: policyv1beta1.PodDisruptionBudgetSpec{
-					MinAvailable: util.ToIntStr(1),
-					Selector: &metav1.LabelSelector{
-						MatchLabels: map[string]string{"spilo-role": "master", "cluster-name": "myapp-database"},
-					},
-				},
-			},
-		},
-		// With zero instances.
-		{
-			New(
-				Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-pdb"}},
-				k8sutil.KubernetesClient{},
-				acidv1.Postgresql{
-					ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"},
-					Spec:       acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 0}},
-				logger,
-				eventRecorder),
-			policyv1beta1.PodDisruptionBudget{
-				ObjectMeta: metav1.ObjectMeta{
-					Name:      "postgres-myapp-database-pdb",
-					Namespace: "myapp",
-					Labels:    map[string]string{"team": "myapp", "cluster-name": "myapp-database"},
-				},
-				Spec: policyv1beta1.PodDisruptionBudgetSpec{
-					MinAvailable: util.ToIntStr(0),
-					Selector: &metav1.LabelSelector{
-						MatchLabels: map[string]string{"spilo-role": "master", "cluster-name": "myapp-database"},
-					},
-				},
-			},
-		},
-		// With PodDisruptionBudget disabled.
-		{
-			New(
-				Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-pdb", EnablePodDisruptionBudget: util.False()}},
-				k8sutil.KubernetesClient{},
-				acidv1.Postgresql{
-					ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"},
-					Spec:       acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}},
-				logger,
-				eventRecorder),
-			policyv1beta1.PodDisruptionBudget{
-				ObjectMeta: metav1.ObjectMeta{
-					Name:      "postgres-myapp-database-pdb",
-					Namespace: "myapp",
-					Labels:    map[string]string{"team": "myapp", "cluster-name": "myapp-database"},
-				},
-				Spec: policyv1beta1.PodDisruptionBudgetSpec{
-					MinAvailable: util.ToIntStr(0),
-					Selector: &metav1.LabelSelector{
-						MatchLabels: map[string]string{"spilo-role": "master", "cluster-name": "myapp-database"},
-					},
-				},
-			},
-		},
-		// With non-default PDBNameFormat and PodDisruptionBudget explicitly enabled.
-		{
-			New(
-				Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-databass-budget", EnablePodDisruptionBudget: util.True()}},
-				k8sutil.KubernetesClient{},
-				acidv1.Postgresql{
-					ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"},
-					Spec:       acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}},
-				logger,
-				eventRecorder),
-			policyv1beta1.PodDisruptionBudget{
-				ObjectMeta: metav1.ObjectMeta{
-					Name:      "postgres-myapp-database-databass-budget",
-					Namespace: "myapp",
-					Labels:    map[string]string{"team": "myapp", "cluster-name": "myapp-database"},
-				},
-				Spec: policyv1beta1.PodDisruptionBudgetSpec{
-					MinAvailable: util.ToIntStr(1),
-					Selector: &metav1.LabelSelector{
-						MatchLabels: map[string]string{"spilo-role": "master", "cluster-name": "myapp-database"},
-					},
-				},
-			},
-		},
-	}
-
-	for _, tt := range tests {
-		result := tt.c.generatePodDisruptionBudget()
-		if !reflect.DeepEqual(*result, tt.out) {
-			t.Errorf("Expected PodDisruptionBudget: %#v, got %#v", tt.out, *result)
-		}
-	}
-}
-
-func TestShmVolume(t *testing.T) {
-	testName := "TestShmVolume"
-	tests := []struct {
-		subTest string
-		podSpec *v1.PodSpec
-		shmPos  int
-	}{
-		{
-			subTest: "empty PodSpec",
-			podSpec: &v1.PodSpec{
-				Volumes: []v1.Volume{},
-				Containers: []v1.Container{
-					{
-						VolumeMounts: []v1.VolumeMount{},
-					},
-				},
-			},
-			shmPos: 0,
-		},
-		{
-			subTest: "non empty PodSpec",
-			podSpec: &v1.PodSpec{
-				Volumes: []v1.Volume{{}},
-				Containers: []v1.Container{
-					{
-						Name: "postgres",
-						VolumeMounts: []v1.VolumeMount{
-							{},
-						},
-					},
-				},
-			},
-			shmPos: 1,
-		},
-	}
-	for _, tt := range tests {
-		addShmVolume(tt.podSpec)
-		postgresContainer := getPostgresContainer(tt.podSpec)
-
-		volumeName := tt.podSpec.Volumes[tt.shmPos].Name
-		volumeMountName := postgresContainer.VolumeMounts[tt.shmPos].Name
-
-		if volumeName != constants.ShmVolumeName {
-			t.Errorf("%s %s: Expected volume %s was not created, have %s instead",
-				testName, tt.subTest, constants.ShmVolumeName, volumeName)
-		}
-		if volumeMountName != constants.ShmVolumeName {
-			t.Errorf("%s %s: Expected mount %s was not created, have %s instead",
-				testName, tt.subTest, constants.ShmVolumeName, volumeMountName)
-		}
-	}
-}
-
 func TestCloneEnv(t *testing.T) {
 	testName := "TestCloneEnv"
 	tests := []struct {
@@ -624,6 +942,79 @@ func TestCloneEnv(t *testing.T) {
 	}
 }
 
+func TestAppendEnvVar(t *testing.T) {
+	testName := "TestAppendEnvVar"
+	tests := []struct {
+		subTest      string
+		envs         []v1.EnvVar
+		envsToAppend []v1.EnvVar
+		expectedSize int
+	}{
+		{
+			subTest: "append two variables - one with same key that should get rejected",
+			envs: []v1.EnvVar{
+				{
+					Name:  "CUSTOM_VARIABLE",
+					Value: "test",
+				},
+			},
+			envsToAppend: []v1.EnvVar{
+				{
+					Name:  "CUSTOM_VARIABLE",
+					Value: "new-test",
+				},
+				{
+					Name:  "ANOTHER_CUSTOM_VARIABLE",
+					Value: "another-test",
+				},
+			},
+			expectedSize: 2,
+		},
+		{
+			subTest: "append empty slice",
+			envs: []v1.EnvVar{
+				{
+					Name:  "CUSTOM_VARIABLE",
+					Value: "test",
+				},
+			},
+			envsToAppend: []v1.EnvVar{},
+			expectedSize: 1,
+		},
+		{
+			subTest: "append nil",
+			envs: []v1.EnvVar{
+				{
+					Name:  "CUSTOM_VARIABLE",
+					Value: "test",
+				},
+			},
+			envsToAppend: nil,
+			expectedSize: 1,
+		},
+	}
+
+	for _, tt := range tests {
+		finalEnvs := appendEnvVars(tt.envs, tt.envsToAppend...)
+
+		if len(finalEnvs) != tt.expectedSize {
+			t.Errorf("%s %s: expected %d env variables, got %d",
+				testName, tt.subTest, tt.expectedSize, len(finalEnvs))
+		}
+
+		for _, env := range tt.envs {
+			for _, finalEnv := range finalEnvs {
+				if env.Name == finalEnv.Name {
+					if env.Value != finalEnv.Value {
+						t.Errorf("%s %s: expected env value %s of variable %s, got %s instead",
+							testName, tt.subTest, env.Value, env.Name, finalEnv.Value)
+					}
+				}
+			}
+		}
+	}
+}
+
 func TestStandbyEnv(t *testing.T) {
 	testName := "TestStandbyEnv"
 	tests := []struct {
@@ -645,18 +1036,6 @@ func TestStandbyEnv(t *testing.T) {
 			envPos: 0,
 			envLen: 3,
 		},
-		{
-			subTest: "from custom gs path",
-			standbyOpts: &acidv1.StandbyDescription{
-				GSWalPath: "gs://some/path/",
-			},
-			env: v1.EnvVar{
-				Name:  "STANDBY_GOOGLE_APPLICATION_CREDENTIALS",
-				Value: "",
-			},
-			envPos: 1,
-			envLen: 4,
-		},
 		{
 			subTest: "ignore gs path if s3 is set",
 			standbyOpts: &acidv1.StandbyDescription{
@@ -735,380 +1114,6 @@ func TestStandbyEnv(t *testing.T) {
 	}
 }
 
-func TestExtractPgVersionFromBinPath(t *testing.T) {
-	testName := "TestExtractPgVersionFromBinPath"
-	tests := []struct {
-		subTest  string
-		binPath  string
-		template string
-		expected string
-	}{
-		{
-			subTest:  "test current bin path with decimal against hard coded template",
-			binPath:  "/usr/lib/postgresql/9.6/bin",
-			template: pgBinariesLocationTemplate,
-			expected: "9.6",
-		},
-		{
-			subTest:  "test current bin path against hard coded template",
-			binPath:  "/usr/lib/postgresql/12/bin",
-			template: pgBinariesLocationTemplate,
-			expected: "12",
-		},
-		{
-			subTest:  "test alternative bin path against a matching template",
-			binPath:  "/usr/pgsql-12/bin",
-			template: "/usr/pgsql-%v/bin",
-			expected: "12",
-		},
-	}
-
-	for _, tt := range tests {
-		pgVersion, err := extractPgVersionFromBinPath(tt.binPath, tt.template)
-		if err != nil {
-			t.Errorf("Unexpected error: %v", err)
-		}
-		if pgVersion != tt.expected {
-			t.Errorf("%s %s: Expected version %s, have %s instead",
-				testName, tt.subTest, tt.expected, pgVersion)
-		}
-	}
-}
-
-func TestSecretVolume(t *testing.T) {
-	testName := "TestSecretVolume"
-	tests := []struct {
-		subTest   string
-		podSpec   *v1.PodSpec
-		secretPos int
-	}{
-		{
-			subTest: "empty PodSpec",
-			podSpec: &v1.PodSpec{
-				Volumes: []v1.Volume{},
-				Containers: []v1.Container{
-					{
-						VolumeMounts: []v1.VolumeMount{},
-					},
-				},
-			},
-			secretPos: 0,
-		},
-		{
-			subTest: "non empty PodSpec",
-			podSpec: &v1.PodSpec{
-				Volumes: []v1.Volume{{}},
-				Containers: []v1.Container{
-					{
-						VolumeMounts: []v1.VolumeMount{
-							{
-								Name:      "data",
-								ReadOnly:  false,
-								MountPath: "/data",
-							},
-						},
-					},
-				},
-			},
-			secretPos: 1,
-		},
-	}
-	for _, tt := range tests {
-		additionalSecretMount := "aws-iam-s3-role"
-		additionalSecretMountPath := "/meta/credentials"
-		postgresContainer := getPostgresContainer(tt.podSpec)
-
-		numMounts := len(postgresContainer.VolumeMounts)
-
-		addSecretVolume(tt.podSpec, additionalSecretMount, additionalSecretMountPath)
-
-		volumeName := tt.podSpec.Volumes[tt.secretPos].Name
-
-		if volumeName != additionalSecretMount {
-			t.Errorf("%s %s: Expected volume %s was not created, have %s instead",
-				testName, tt.subTest, additionalSecretMount, volumeName)
-		}
-
-		for i := range tt.podSpec.Containers {
-			volumeMountName := tt.podSpec.Containers[i].VolumeMounts[tt.secretPos].Name
-
-			if volumeMountName != additionalSecretMount {
-				t.Errorf("%s %s: Expected mount %s was not created, have %s instead",
-					testName, tt.subTest, additionalSecretMount, volumeMountName)
-			}
-		}
-
-		postgresContainer = getPostgresContainer(tt.podSpec)
-		numMountsCheck := len(postgresContainer.VolumeMounts)
-
-		if numMountsCheck != numMounts+1 {
-			t.Errorf("Unexpected number of VolumeMounts: got %v instead of %v",
-				numMountsCheck, numMounts+1)
-		}
-	}
-}
-
-const (
-	testPodEnvironmentConfigMapName      = "pod_env_cm"
-	testPodEnvironmentSecretName         = "pod_env_sc"
-	testPodEnvironmentObjectNotExists    = "idonotexist"
-	testPodEnvironmentSecretNameAPIError = "pod_env_sc_apierror"
-	testResourceCheckInterval            = 3
-	testResourceCheckTimeout             = 10
-)
-
-type mockSecret struct {
-	v1core.SecretInterface
-}
-
-type mockConfigMap struct {
-	v1core.ConfigMapInterface
-}
-
-func (c *mockSecret) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.Secret, error) {
-	if name == testPodEnvironmentSecretNameAPIError {
-		return nil, fmt.Errorf("Secret PodEnvironmentSecret API error")
-	}
-	if name != testPodEnvironmentSecretName {
-		return nil, k8serrors.NewNotFound(schema.GroupResource{Group: "core", Resource: "secret"}, name)
-	}
-	secret := &v1.Secret{}
-	secret.Name = testPodEnvironmentSecretName
-	secret.Data = map[string][]byte{
-		"minio_access_key": []byte("alpha"),
-		"minio_secret_key": []byte("beta"),
-	}
-	return secret, nil
-}
-
-func (c *mockConfigMap) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.ConfigMap, error) {
-	if name != testPodEnvironmentConfigMapName {
-		return nil, fmt.Errorf("NotFound")
-	}
-	configmap := &v1.ConfigMap{}
-	configmap.Name = testPodEnvironmentConfigMapName
-	configmap.Data = map[string]string{
-		"foo1": "bar1",
-		"foo2": "bar2",
-	}
-	return configmap, nil
-}
-
-type MockSecretGetter struct {
-}
-
-type MockConfigMapsGetter struct {
-}
-
-func (c *MockSecretGetter) Secrets(namespace string) v1core.SecretInterface {
-	return &mockSecret{}
-}
-
-func (c *MockConfigMapsGetter) ConfigMaps(namespace string) v1core.ConfigMapInterface {
-	return &mockConfigMap{}
-}
-
-func newMockKubernetesClient() k8sutil.KubernetesClient {
-	return k8sutil.KubernetesClient{
-		SecretsGetter:    &MockSecretGetter{},
-		ConfigMapsGetter: &MockConfigMapsGetter{},
-	}
-}
-func newMockCluster(opConfig config.Config) *Cluster {
-	cluster := &Cluster{
-		Config:     Config{OpConfig: opConfig},
-		KubeClient: newMockKubernetesClient(),
-	}
-	return cluster
-}
-
-func TestPodEnvironmentConfigMapVariables(t *testing.T) {
-	testName := "TestPodEnvironmentConfigMapVariables"
-	tests := []struct {
-		subTest  string
-		opConfig config.Config
-		envVars  []v1.EnvVar
-		err      error
-	}{
-		{
-			subTest: "no PodEnvironmentConfigMap",
-			envVars: []v1.EnvVar{},
-		},
-		{
-			subTest: "missing PodEnvironmentConfigMap",
-			opConfig: config.Config{
-				Resources: config.Resources{
-					PodEnvironmentConfigMap: spec.NamespacedName{
-						Name: testPodEnvironmentObjectNotExists,
-					},
-				},
-			},
-			err: fmt.Errorf("could not read PodEnvironmentConfigMap: NotFound"),
-		},
-		{
-			subTest: "Pod environment vars configured by PodEnvironmentConfigMap",
-			opConfig: config.Config{
-				Resources: config.Resources{
-					PodEnvironmentConfigMap: spec.NamespacedName{
-						Name: testPodEnvironmentConfigMapName,
-					},
-				},
-			},
-			envVars: []v1.EnvVar{
-				{
-					Name:  "foo1",
-					Value: "bar1",
-				},
-				{
-					Name:  "foo2",
-					Value: "bar2",
-				},
-			},
-		},
-	}
-	for _, tt := range tests {
-		c := newMockCluster(tt.opConfig)
-		vars, err := c.getPodEnvironmentConfigMapVariables()
-		sort.Slice(vars, func(i, j int) bool { return vars[i].Name < vars[j].Name })
-		if !reflect.DeepEqual(vars, tt.envVars) {
-			t.Errorf("%s %s: expected `%v` but got `%v`",
-				testName, tt.subTest, tt.envVars, vars)
-		}
-		if tt.err != nil {
-			if err.Error() != tt.err.Error() {
-				t.Errorf("%s %s: expected error `%v` but got `%v`",
-					testName, tt.subTest, tt.err, err)
-			}
-		} else {
-			if err != nil {
-				t.Errorf("%s %s: expected no error but got error: `%v`",
-					testName, tt.subTest, err)
-			}
-		}
-	}
-}
-
-// Test if the keys of an existing secret are properly referenced
-func TestPodEnvironmentSecretVariables(t *testing.T) {
-	maxRetries := int(testResourceCheckTimeout / testResourceCheckInterval)
-	testName := "TestPodEnvironmentSecretVariables"
-	tests := []struct {
-		subTest  string
-		opConfig config.Config
-		envVars  []v1.EnvVar
-		err      error
-	}{
-		{
-			subTest: "No PodEnvironmentSecret configured",
-			envVars: []v1.EnvVar{},
-		},
-		{
-			subTest: "Secret referenced by PodEnvironmentSecret does not exist",
-			opConfig: config.Config{
-				Resources: config.Resources{
-					PodEnvironmentSecret:  testPodEnvironmentObjectNotExists,
-					ResourceCheckInterval: time.Duration(testResourceCheckInterval),
-					ResourceCheckTimeout:  time.Duration(testResourceCheckTimeout),
-				},
-			},
-			err: fmt.Errorf("could not read Secret PodEnvironmentSecretName: still failing after %d retries: secret.core %q not found", maxRetries, testPodEnvironmentObjectNotExists),
-		},
-		{
-			subTest: "API error during PodEnvironmentSecret retrieval",
-			opConfig: config.Config{
-				Resources: config.Resources{
-					PodEnvironmentSecret:  testPodEnvironmentSecretNameAPIError,
-					ResourceCheckInterval: time.Duration(testResourceCheckInterval),
-					ResourceCheckTimeout:  time.Duration(testResourceCheckTimeout),
-				},
-			},
-			err: fmt.Errorf("could not read Secret PodEnvironmentSecretName: Secret PodEnvironmentSecret API error"),
-		},
-		{
-			subTest: "Pod environment vars reference all keys from secret configured by PodEnvironmentSecret",
-			opConfig: config.Config{
-				Resources: config.Resources{
-					PodEnvironmentSecret:  testPodEnvironmentSecretName,
-					ResourceCheckInterval: time.Duration(testResourceCheckInterval),
-					ResourceCheckTimeout:  time.Duration(testResourceCheckTimeout),
-				},
-			},
-			envVars: []v1.EnvVar{
-				{
-					Name: "minio_access_key",
-					ValueFrom: &v1.EnvVarSource{
-						SecretKeyRef: &v1.SecretKeySelector{
-							LocalObjectReference: v1.LocalObjectReference{
-								Name: testPodEnvironmentSecretName,
-							},
-							Key: "minio_access_key",
-						},
-					},
-				},
-				{
-					Name: "minio_secret_key",
-					ValueFrom: &v1.EnvVarSource{
-						SecretKeyRef: &v1.SecretKeySelector{
-							LocalObjectReference: v1.LocalObjectReference{
-								Name: testPodEnvironmentSecretName,
-							},
-							Key: "minio_secret_key",
-						},
-					},
-				},
-			},
-		},
-	}
-
-	for _, tt := range tests {
-		c := newMockCluster(tt.opConfig)
-		vars, err := c.getPodEnvironmentSecretVariables()
-		sort.Slice(vars, func(i, j int) bool { return vars[i].Name < vars[j].Name })
-		if !reflect.DeepEqual(vars, tt.envVars) {
-			t.Errorf("%s %s: expected `%v` but got `%v`",
-				testName, tt.subTest, tt.envVars, vars)
-		}
-		if tt.err != nil {
-			if err.Error() != tt.err.Error() {
-				t.Errorf("%s %s: expected error `%v` but got `%v`",
-					testName, tt.subTest, tt.err, err)
-			}
-		} else {
-			if err != nil {
-				t.Errorf("%s %s: expected no error but got error: `%v`",
-					testName, tt.subTest, err)
-			}
-		}
-	}
-
-}
-
-func testEnvs(cluster *Cluster, podSpec *v1.PodTemplateSpec, role PostgresRole) error {
-	required := map[string]bool{
-		"PGHOST":                 false,
-		"PGPORT":                 false,
-		"PGUSER":                 false,
-		"PGSCHEMA":               false,
-		"PGPASSWORD":             false,
-		"CONNECTION_POOLER_MODE": false,
-		"CONNECTION_POOLER_PORT": false,
-	}
-
-	container := getPostgresContainer(&podSpec.Spec)
-	envs := container.Env
-	for _, env := range envs {
-		required[env.Name] = true
-	}
-
-	for env, value := range required {
-		if !value {
-			return fmt.Errorf("Environment variable %s is not present", env)
-		}
-	}
-
-	return nil
-}
-
 func TestNodeAffinity(t *testing.T) {
 	var err error
 	var spec acidv1.PostgresSpec
@@ -1286,6 +1291,132 @@ func TestTLS(t *testing.T) {
 	assert.Contains(t, postgresContainer.Env, v1.EnvVar{Name: "SSL_CA_FILE", Value: "/tls/ca.crt"})
 }
 
+func TestShmVolume(t *testing.T) {
+	testName := "TestShmVolume"
+	tests := []struct {
+		subTest string
+		podSpec *v1.PodSpec
+		shmPos  int
+	}{
+		{
+			subTest: "empty PodSpec",
+			podSpec: &v1.PodSpec{
+				Volumes: []v1.Volume{},
+				Containers: []v1.Container{
+					{
+						VolumeMounts: []v1.VolumeMount{},
+					},
+				},
+			},
+			shmPos: 0,
+		},
+		{
+			subTest: "non empty PodSpec",
+			podSpec: &v1.PodSpec{
+				Volumes: []v1.Volume{{}},
+				Containers: []v1.Container{
+					{
+						Name: "postgres",
+						VolumeMounts: []v1.VolumeMount{
+							{},
+						},
+					},
+				},
+			},
+			shmPos: 1,
+		},
+	}
+	for _, tt := range tests {
+		addShmVolume(tt.podSpec)
+		postgresContainer := getPostgresContainer(tt.podSpec)
+
+		volumeName := tt.podSpec.Volumes[tt.shmPos].Name
+		volumeMountName := postgresContainer.VolumeMounts[tt.shmPos].Name
+
+		if volumeName != constants.ShmVolumeName {
+			t.Errorf("%s %s: Expected volume %s was not created, have %s instead",
+				testName, tt.subTest, constants.ShmVolumeName, volumeName)
+		}
+		if volumeMountName != constants.ShmVolumeName {
+			t.Errorf("%s %s: Expected mount %s was not created, have %s instead",
+				testName, tt.subTest, constants.ShmVolumeName, volumeMountName)
+		}
+	}
+}
+
+func TestSecretVolume(t *testing.T) {
+	testName := "TestSecretVolume"
+	tests := []struct {
+		subTest   string
+		podSpec   *v1.PodSpec
+		secretPos int
+	}{
+		{
+			subTest: "empty PodSpec",
+			podSpec: &v1.PodSpec{
+				Volumes: []v1.Volume{},
+				Containers: []v1.Container{
+					{
+						VolumeMounts: []v1.VolumeMount{},
+					},
+				},
+			},
+			secretPos: 0,
+		},
+		{
+			subTest: "non empty PodSpec",
+			podSpec: &v1.PodSpec{
+				Volumes: []v1.Volume{{}},
+				Containers: []v1.Container{
+					{
+						VolumeMounts: []v1.VolumeMount{
+							{
+								Name:      "data",
+								ReadOnly:  false,
+								MountPath: "/data",
+							},
+						},
+					},
+				},
+			},
+			secretPos: 1,
+		},
+	}
+	for _, tt := range tests {
+		additionalSecretMount := "aws-iam-s3-role"
+		additionalSecretMountPath := "/meta/credentials"
+		postgresContainer := getPostgresContainer(tt.podSpec)
+
+		numMounts := len(postgresContainer.VolumeMounts)
+
+		addSecretVolume(tt.podSpec, additionalSecretMount, additionalSecretMountPath)
+
+		volumeName := tt.podSpec.Volumes[tt.secretPos].Name
+
+		if volumeName != additionalSecretMount {
+			t.Errorf("%s %s: Expected volume %s was not created, have %s instead",
+				testName, tt.subTest, additionalSecretMount, volumeName)
+		}
+
+		for i := range tt.podSpec.Containers {
+			volumeMountName := tt.podSpec.Containers[i].VolumeMounts[tt.secretPos].Name
+
+			if volumeMountName != additionalSecretMount {
+				t.Errorf("%s %s: Expected mount %s was not created, have %s instead",
+					testName, tt.subTest, additionalSecretMount, volumeMountName)
+			}
+		}
+
+		postgresContainer = getPostgresContainer(tt.podSpec)
+		numMountsCheck := len(postgresContainer.VolumeMounts)
+
+		if numMountsCheck != numMounts+1 {
+			t.Errorf("Unexpected number of VolumeMounts: got %v instead of %v",
+				numMountsCheck, numMounts+1)
+		}
+	}
+}
+
 func TestAdditionalVolume(t *testing.T) {
 	testName := "TestAdditionalVolume"
 
@@ -1406,6 +1537,109 @@ func TestAdditionalVolume(t *testing.T) {
 	}
 }
 
+func TestVolumeSelector(t *testing.T) {
+	testName := "TestVolumeSelector"
+	makeSpec := func(volume acidv1.Volume) acidv1.PostgresSpec {
+		return acidv1.PostgresSpec{
+			TeamID:            "myapp",
+			NumberOfInstances: 0,
+			Resources: &acidv1.Resources{
+				ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"},
+				ResourceLimits:   acidv1.ResourceDescription{CPU: "1", Memory: "10"},
+			},
+			Volume: volume,
+		}
+	}
+
+	tests := []struct {
+		subTest      string
+		volume       acidv1.Volume
+		wantSelector *metav1.LabelSelector
+	}{
+		{
+			subTest: "PVC template has no selector",
+			volume: acidv1.Volume{
+				Size: "1G",
+			},
+			wantSelector: nil,
+		},
+		{
+			subTest: "PVC template has simple label selector",
+			volume: acidv1.Volume{
+				Size: "1G",
+				Selector: &metav1.LabelSelector{
+					MatchLabels: map[string]string{"environment": "unittest"},
+				},
+			},
+			wantSelector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{"environment": "unittest"},
+			},
+		},
+		{
+			subTest: "PVC template has full selector",
+			volume: acidv1.Volume{
+				Size: "1G",
+				Selector: &metav1.LabelSelector{
+					MatchLabels: map[string]string{"environment": "unittest"},
+					MatchExpressions: []metav1.LabelSelectorRequirement{
+						{
+							Key:      "flavour",
+							Operator: metav1.LabelSelectorOpIn,
+							Values:   []string{"banana", "chocolate"},
+						},
+					},
+				},
+			},
+			wantSelector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{"environment": "unittest"},
+				MatchExpressions: []metav1.LabelSelectorRequirement{
+					{
+						Key:      "flavour",
+						Operator: metav1.LabelSelectorOpIn,
+						Values:   []string{"banana", "chocolate"},
+					},
+				},
+			},
+		},
+	}
+
+	cluster := New(
+		Config{
+			OpConfig: config.Config{
+				PodManagementPolicy: "ordered_ready",
+				ProtectedRoles:      []string{"admin"},
+				Auth: config.Auth{
+					SuperUsername:       superUserName,
+					ReplicationUsername: replicationUserName,
+				},
+			},
+		}, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder)
+
+	for _, tt := range tests {
+		pgSpec := makeSpec(tt.volume)
+		sts, err := cluster.generateStatefulSet(&pgSpec)
+		if err != nil {
+			t.Fatalf("%s %s: no statefulset created %v", testName, tt.subTest, err)
+		}
+
+		volIdx := len(sts.Spec.VolumeClaimTemplates)
+		for i, ct := range sts.Spec.VolumeClaimTemplates {
+			if ct.ObjectMeta.Name == constants.DataVolumeName {
+				volIdx = i
+				break
+			}
+		}
+		if volIdx == len(sts.Spec.VolumeClaimTemplates) {
+			t.Errorf("%s %s: no datavolume found in sts", testName, tt.subTest)
+		}
+
+		selector := sts.Spec.VolumeClaimTemplates[volIdx].Spec.Selector
+		if !reflect.DeepEqual(selector, tt.wantSelector) {
+			t.Errorf("%s %s: expected: %#v but got: %#v", testName, tt.subTest, tt.wantSelector, selector)
+		}
+	}
+}
+
 // inject sidecars through all available mechanisms and check the resulting container specs
 func TestSidecars(t *testing.T) {
 	var err error
@@ -1461,7 +1695,7 @@ func TestSidecars(t *testing.T) {
 			},
 			acidv1.Sidecar{
 				Name:        "replace-sidecar",
-				DockerImage: "overwrite-image",
+				DockerImage: "override-image",
 			},
 		},
 	}
@@ -1589,7 +1823,7 @@ func TestSidecars(t *testing.T) {
 	// replaced sidecar
 	assert.Contains(t, s.Spec.Template.Spec.Containers, v1.Container{
 		Name:            "replace-sidecar",
-		Image:           "overwrite-image",
+		Image:           "override-image",
 		Resources:       generateKubernetesResources("200m", "500m", "0.7Gi", "1.3Gi"),
 		ImagePullPolicy: v1.PullIfNotPresent,
 		Env:             env,
@@ -1610,6 +1844,117 @@ func TestSidecars(t *testing.T) {
 
 }
 
+func TestGeneratePodDisruptionBudget(t *testing.T) {
+	tests := []struct {
+		c   *Cluster
+		out policyv1beta1.PodDisruptionBudget
+	}{
+		// With multiple instances.
+		{
+			New(
+				Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-pdb"}},
+				k8sutil.KubernetesClient{},
+				acidv1.Postgresql{
+					ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"},
+					Spec:       acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}},
+				logger,
+				eventRecorder),
+			policyv1beta1.PodDisruptionBudget{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "postgres-myapp-database-pdb",
+					Namespace: "myapp",
+					Labels:    map[string]string{"team": "myapp", "cluster-name": "myapp-database"},
+				},
+				Spec: policyv1beta1.PodDisruptionBudgetSpec{
+					MinAvailable: util.ToIntStr(1),
+					Selector: &metav1.LabelSelector{
+						MatchLabels: map[string]string{"spilo-role": "master", "cluster-name": "myapp-database"},
+					},
+				},
+			},
+		},
+		// With zero instances.
+		{
+			New(
+				Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-pdb"}},
+				k8sutil.KubernetesClient{},
+				acidv1.Postgresql{
+					ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"},
+					Spec:       acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 0}},
+				logger,
+				eventRecorder),
+			policyv1beta1.PodDisruptionBudget{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "postgres-myapp-database-pdb",
+					Namespace: "myapp",
+					Labels:    map[string]string{"team": "myapp", "cluster-name": "myapp-database"},
+				},
+				Spec: policyv1beta1.PodDisruptionBudgetSpec{
+					MinAvailable: util.ToIntStr(0),
+					Selector: &metav1.LabelSelector{
+						MatchLabels: map[string]string{"spilo-role": "master", "cluster-name": "myapp-database"},
+					},
+				},
+			},
+		},
+		// With PodDisruptionBudget disabled.
+		{
+			New(
+				Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-pdb", EnablePodDisruptionBudget: util.False()}},
+				k8sutil.KubernetesClient{},
+				acidv1.Postgresql{
+					ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"},
+					Spec:       acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}},
+				logger,
+				eventRecorder),
+			policyv1beta1.PodDisruptionBudget{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "postgres-myapp-database-pdb",
+					Namespace: "myapp",
+					Labels:    map[string]string{"team": "myapp", "cluster-name": "myapp-database"},
+				},
+				Spec: policyv1beta1.PodDisruptionBudgetSpec{
+					MinAvailable: util.ToIntStr(0),
+					Selector: &metav1.LabelSelector{
+						MatchLabels: map[string]string{"spilo-role": "master", "cluster-name": "myapp-database"},
+					},
+				},
+			},
+		},
+		// With non-default PDBNameFormat and PodDisruptionBudget explicitly enabled.
+		{
+			New(
+				Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-databass-budget", EnablePodDisruptionBudget: util.True()}},
+				k8sutil.KubernetesClient{},
+				acidv1.Postgresql{
+					ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"},
+					Spec:       acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}},
+				logger,
+				eventRecorder),
+			policyv1beta1.PodDisruptionBudget{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "postgres-myapp-database-databass-budget",
+					Namespace: "myapp",
+					Labels:    map[string]string{"team": "myapp", "cluster-name": "myapp-database"},
+				},
+				Spec: policyv1beta1.PodDisruptionBudgetSpec{
+					MinAvailable: util.ToIntStr(1),
+					Selector: &metav1.LabelSelector{
+						MatchLabels: map[string]string{"spilo-role": "master", "cluster-name": "myapp-database"},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		result := tt.c.generatePodDisruptionBudget()
+		if !reflect.DeepEqual(*result, tt.out) {
+			t.Errorf("Expected PodDisruptionBudget: %#v, got %#v", tt.out, *result)
+		}
+	}
+}
+
 func TestGenerateService(t *testing.T) {
 	var spec acidv1.PostgresSpec
 	var cluster *Cluster
@@ -1636,7 +1981,7 @@ func TestGenerateService(t *testing.T) {
 			},
 			acidv1.Sidecar{
 				Name:        "replace-sidecar",
-				DockerImage: "overwrite-image",
+				DockerImage: "override-image",
 			},
 		},
 		EnableMasterLoadBalancer: &enableLB,
@@ -1690,6 +2035,65 @@ func TestGenerateService(t *testing.T) {
 
 }
 
+func TestCreateLoadBalancerLogic(t *testing.T) {
+	var cluster = New(
+		Config{
+			OpConfig: config.Config{
+				ProtectedRoles: []string{"admin"},
+				Auth: config.Auth{
+					SuperUsername:       superUserName,
+					ReplicationUsername: replicationUserName,
+				},
+			},
+		}, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder)
+
+	testName := "TestCreateLoadBalancerLogic"
+	tests := []struct {
+		subtest  string
+		role     PostgresRole
+		spec     *acidv1.PostgresSpec
+		opConfig config.Config
+		result   bool
+	}{
+		{
+			subtest:  "new format, load balancer is enabled for replica",
+			role:     Replica,
+			spec:     &acidv1.PostgresSpec{EnableReplicaLoadBalancer: util.True()},
+			opConfig: config.Config{},
+			result:   true,
+		},
+		{
+			subtest:  "new format, load balancer is disabled for replica",
+			role:     Replica,
+			spec:     &acidv1.PostgresSpec{EnableReplicaLoadBalancer: util.False()},
+			opConfig: config.Config{},
+			result:   false,
+		},
+		{
+			subtest:  "new format, load balancer isn't specified for replica",
+			role:     Replica,
+			spec:     &acidv1.PostgresSpec{EnableReplicaLoadBalancer: nil},
+			opConfig: config.Config{EnableReplicaLoadBalancer: true},
+			result:   true,
+		},
+		{
+			subtest:  "new format, load balancer isn't specified for replica",
+			role:     Replica,
+			spec:     &acidv1.PostgresSpec{EnableReplicaLoadBalancer: nil},
+			opConfig: config.Config{EnableReplicaLoadBalancer: false},
+			result:   false,
+		},
+	}
+	for _, tt := range tests {
+		cluster.OpConfig = tt.opConfig
+		result := cluster.shouldCreateLoadBalancerForService(tt.role, tt.spec)
+		if tt.result != result {
+			t.Errorf("%s %s: Load balancer is %t, expect %t for role %#v and spec %#v",
+				testName, tt.subtest, result, tt.result, tt.role, tt.spec)
+		}
+	}
+}
+
 func newLBFakeClient() (k8sutil.KubernetesClient, *fake.Clientset) {
 	clientSet := fake.NewSimpleClientset()
 
@@ -2231,106 +2635,3 @@ func TestGenerateCapabilities(t *testing.T) {
 		}
 	}
 }
-
-func TestVolumeSelector(t *testing.T) {
-	testName := "TestVolumeSelector"
-	makeSpec := func(volume acidv1.Volume) acidv1.PostgresSpec {
-		return acidv1.PostgresSpec{
-			TeamID:            "myapp",
-			NumberOfInstances: 0,
-			Resources: &acidv1.Resources{
-				ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"},
-				ResourceLimits:   acidv1.ResourceDescription{CPU: "1", Memory: "10"},
-			},
-			Volume: volume,
-		}
-	}
-
-	tests := []struct {
-		subTest      string
-		volume       acidv1.Volume
-		wantSelector *metav1.LabelSelector
-	}{
-		{
-			subTest: "PVC template has no selector",
-			volume: acidv1.Volume{
-				Size: "1G",
-			},
-			wantSelector: nil,
-		},
-		{
-			subTest: "PVC template has simple label selector",
-			volume: acidv1.Volume{
-				Size: "1G",
-				Selector: &metav1.LabelSelector{
-					MatchLabels: map[string]string{"environment": "unittest"},
-				},
-			},
-			wantSelector: &metav1.LabelSelector{
-				MatchLabels: map[string]string{"environment": "unittest"},
-			},
-		},
-		{
-			subTest: "PVC template has full selector",
-			volume: acidv1.Volume{
-				Size: "1G",
-				Selector: &metav1.LabelSelector{
-					MatchLabels: map[string]string{"environment": "unittest"},
-					MatchExpressions: []metav1.LabelSelectorRequirement{
-						{
-							Key:      "flavour",
-							Operator: metav1.LabelSelectorOpIn,
-							Values:   []string{"banana", "chocolate"},
-						},
-					},
-				},
-			},
-			wantSelector: &metav1.LabelSelector{
-				MatchLabels: map[string]string{"environment": "unittest"},
-				MatchExpressions: []metav1.LabelSelectorRequirement{
-					{
-						Key:      "flavour",
-						Operator: metav1.LabelSelectorOpIn,
-						Values:   []string{"banana", "chocolate"},
-					},
-				},
-			},
-		},
-	}
-
-	cluster := New(
-		Config{
-			OpConfig: config.Config{
-				PodManagementPolicy: "ordered_ready",
-				ProtectedRoles:      []string{"admin"},
-				Auth: config.Auth{
-					SuperUsername:       superUserName,
-					ReplicationUsername: replicationUserName,
-				},
-			},
-		}, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder)
-
-	for _, tt := range tests {
-		pgSpec := makeSpec(tt.volume)
-		sts, err := cluster.generateStatefulSet(&pgSpec)
-		if err != nil {
-			t.Fatalf("%s %s: no statefulset created %v", testName, tt.subTest, err)
-		}
-
-		volIdx := len(sts.Spec.VolumeClaimTemplates)
-		for i, ct := range sts.Spec.VolumeClaimTemplates {
-			if ct.ObjectMeta.Name == constants.DataVolumeName {
-				volIdx = i
-				break
-			}
-		}
-		if volIdx == len(sts.Spec.VolumeClaimTemplates) {
-			t.Errorf("%s %s: no datavolume found in sts", testName, tt.subTest)
-		}
-
-		selector := sts.Spec.VolumeClaimTemplates[volIdx].Spec.Selector
-		if !reflect.DeepEqual(selector, tt.wantSelector) {
-			t.Errorf("%s %s: expected: %#v but got: %#v", testName, tt.subTest, tt.wantSelector, selector)
-		}
-	}
-}

From 532772c5cd64f3951b6003a7f7d9ae09005a186c Mon Sep 17 00:00:00 2001
From: Felix Kunde <felix-kunde@gmx.de>
Date: Wed, 20 Apr 2022 12:12:02 +0200
Subject: [PATCH 56/58] do not call EBS api when there are no pvs (#1851)

* do not call EBS api when there are no pvs
* no extra aws api call in executeEBSMigration, operate on fetched cluster.EBSVolumes
---
 pkg/cluster/cluster.go      |  1 -
 pkg/cluster/sync.go         |  2 +-
 pkg/cluster/volumes.go      | 40 ++++++++++++-------------------------
 pkg/cluster/volumes_test.go |  5 ++++-
 4 files changed, 18 insertions(+), 30 deletions(-)

diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go
index 86a7df8dd..a92b09094 100644
--- a/pkg/cluster/cluster.go
+++ b/pkg/cluster/cluster.go
@@ -149,7 +149,6 @@ func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgres
 	cluster.EBSVolumes = make(map[string]volumes.VolumeProperties)
 	if cfg.OpConfig.StorageResizeMode != "pvc" || cfg.OpConfig.EnableEBSGp3Migration {
 		cluster.VolumeResizer = &volumes.EBSVolumeResizer{AWSRegion: cfg.OpConfig.AWSRegion}
-
 	}
 
 	return cluster
diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go
index 7ff021ceb..a34279533 100644
--- a/pkg/cluster/sync.go
+++ b/pkg/cluster/sync.go
@@ -69,7 +69,7 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
 		return err
 	}
 
-	if c.OpConfig.EnableEBSGp3Migration {
+	if c.OpConfig.EnableEBSGp3Migration && len(c.EBSVolumes) > 0 {
 		err = c.executeEBSMigration()
 		if nil != err {
 			return err
diff --git a/pkg/cluster/volumes.go b/pkg/cluster/volumes.go
index 78533ef54..6b22ed54c 100644
--- a/pkg/cluster/volumes.go
+++ b/pkg/cluster/volumes.go
@@ -137,7 +137,6 @@ func (c *Cluster) syncUnderlyingEBSVolume() error {
 		for _, s := range errors {
 			c.logger.Warningf(s)
 		}
-		// c.logger.Errorf("failed to modify %d of %d volumes", len(c.EBSVolumes), len(errors))
 	}
 	return nil
 }
@@ -149,7 +148,11 @@ func (c *Cluster) populateVolumeMetaData() error {
 	if err != nil {
 		return fmt.Errorf("could not list persistent volumes: %v", err)
 	}
-	c.logger.Debugf("found %d volumes, size of known volumes %d", len(pvs), len(c.EBSVolumes))
+	if len(pvs) == 0 {
+		c.EBSVolumes = make(map[string]volumes.VolumeProperties)
+		return fmt.Errorf("no persistent volumes found")
+	}
+	c.logger.Debugf("found %d persistent volumes, size of known volumes %d", len(pvs), len(c.EBSVolumes))
 
 	volumeIds := []string{}
 	var volumeID string
@@ -167,7 +170,7 @@ func (c *Cluster) populateVolumeMetaData() error {
 		return err
 	}
 
-	if len(currentVolumes) != len(c.EBSVolumes) {
+	if len(currentVolumes) != len(c.EBSVolumes) && len(c.EBSVolumes) > 0 {
 		c.logger.Debugf("number of ebs volumes (%d) discovered differs from already known volumes (%d)", len(currentVolumes), len(c.EBSVolumes))
 	}
 
@@ -205,7 +208,7 @@ func (c *Cluster) syncVolumeClaims() error {
 
 // syncVolumes reads all persistent volumes and checks that their size matches the one declared in the statefulset.
 func (c *Cluster) syncEbsVolumes() error {
-	c.setProcessName("syncing EBS and Claims volumes")
+	c.setProcessName("syncing EBS volumes")
 
 	act, err := c.volumesNeedResizing()
 	if err != nil {
@@ -451,29 +454,17 @@ func quantityToGigabyte(q resource.Quantity) int64 {
 }
 
 func (c *Cluster) executeEBSMigration() error {
-	if !c.OpConfig.EnableEBSGp3Migration {
-		return nil
-	}
-	c.logger.Infof("starting EBS gp2 to gp3 migration")
-
 	pvs, err := c.listPersistentVolumes()
 	if err != nil {
 		return fmt.Errorf("could not list persistent volumes: %v", err)
 	}
+	if len(pvs) == 0 {
+		c.logger.Warningf("no persistent volumes found - skipping EBS migration")
+		return nil
+	}
 	c.logger.Debugf("found %d volumes, size of known volumes %d", len(pvs), len(c.EBSVolumes))
 
-	volumeIds := []string{}
-	var volumeID string
-	for _, pv := range pvs {
-		volumeID, err = c.VolumeResizer.ExtractVolumeID(pv.Spec.AWSElasticBlockStore.VolumeID)
-		if err != nil {
-			continue
-		}
-
-		volumeIds = append(volumeIds, volumeID)
-	}
-
-	if len(volumeIds) == len(c.EBSVolumes) {
+	if len(pvs) == len(c.EBSVolumes) {
 		hasGp2 := false
 		for _, v := range c.EBSVolumes {
 			if v.VolumeType == "gp2" {
@@ -487,15 +478,10 @@ func (c *Cluster) executeEBSMigration() error {
 		}
 	}
 
-	awsVolumes, err := c.VolumeResizer.DescribeVolumes(volumeIds)
-	if nil != err {
-		return err
-	}
-
 	var i3000 int64 = 3000
 	var i125 int64 = 125
 
-	for _, volume := range awsVolumes {
+	for _, volume := range c.EBSVolumes {
 		if volume.VolumeType == "gp2" && volume.Size < c.OpConfig.EnableEBSGp3MigrationMaxSize {
 			c.logger.Infof("modifying EBS volume %s to type gp3 migration (%d)", volume.VolumeID, volume.Size)
 			err = c.VolumeResizer.ModifyVolume(volume.VolumeID, aws.String("gp3"), &volume.Size, &i3000, &i125)
diff --git a/pkg/cluster/volumes_test.go b/pkg/cluster/volumes_test.go
index 204ea8aab..4ef94fcfb 100644
--- a/pkg/cluster/volumes_test.go
+++ b/pkg/cluster/volumes_test.go
@@ -224,7 +224,10 @@ func TestMigrateEBS(t *testing.T) {
 	resizer.EXPECT().ModifyVolume(gomock.Eq("ebs-volume-1"), gomock.Eq(aws.String("gp3")), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)
 
 	cluster.VolumeResizer = resizer
-	cluster.executeEBSMigration()
+	err := cluster.populateVolumeMetaData()
+	assert.NoError(t, err)
+	err = cluster.executeEBSMigration()
+	assert.NoError(t, err)
 }
 
 func initTestVolumesAndPods(client k8sutil.KubernetesClient, namespace, clustername string, labels labels.Set, volumes []testVolume) {

From cde448d12ed39fddbcd2e08b4a435815d84bf575 Mon Sep 17 00:00:00 2001
From: Felix Kunde <felix-kunde@gmx.de>
Date: Wed, 20 Apr 2022 18:28:46 +0200
Subject: [PATCH 57/58] [UI] logs for clusters where teamId has dash (#1859)

* [UI] logs for clusters where teamId has dash
---
 ui/operator_ui/main.py |  9 +++++++--
 ui/requirements.txt    | 10 +++++-----
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/ui/operator_ui/main.py b/ui/operator_ui/main.py
index 3e1e54969..579f0e7d8 100644
--- a/ui/operator_ui/main.py
+++ b/ui/operator_ui/main.py
@@ -961,8 +961,13 @@ def get_operator_get_logs(worker: int):
 @app.route('/operator/clusters/<namespace>/<cluster>/logs')
 @authorize
 def get_operator_get_logs_per_cluster(namespace: str, cluster: str):
-    team, clustername = cluster.split('-', 1)
-    return proxy_operator(f'/clusters/{team}/{namespace}/{clustername}/logs/')
+    user_teams = get_teams_for_user(session.get('user_name', ''))
+    for user_team in user_teams:
+        if cluster.find(user_team) == 0:
+            team = cluster[:len(user_team)]
+            cluster_name = cluster[len(user_team)+1:]
+            break
+    return proxy_operator(f'/clusters/{team}/{namespace}/{cluster_name}/logs/')
 
 
 @app.route('/login')
diff --git a/ui/requirements.txt b/ui/requirements.txt
index 8f612d554..c84450466 100644
--- a/ui/requirements.txt
+++ b/ui/requirements.txt
@@ -1,15 +1,15 @@
-Flask-OAuthlib==0.9.5
-Flask==1.1.2
+Flask-OAuthlib==0.9.6
+Flask==2.1.0
 backoff==1.10.0
 boto3==1.16.52
 boto==2.49.0
-click==7.1.2
+click==8.1.2
 furl==2.1.0
 gevent==20.12.1
-jq==1.1.1
+jq==1.2.2
 json_delta>=2.0
 kubernetes==3.0.0
 requests==2.25.1
 stups-tokens>=1.1.19
 wal_e==1.1.1
-werkzeug==0.16.1
+werkzeug==2.1.1

From cde88d3711d9ec395570ae8352b34632a2540ae3 Mon Sep 17 00:00:00 2001
From: Felix Kunde <felix-kunde@gmx.de>
Date: Wed, 20 Apr 2022 18:30:23 +0200
Subject: [PATCH 58/58] bump to v1.8.0 (#1852)

* bump to v1.8.0
* fix helm instructions for ui
* update UI version package.json
* update ui chart
---
 .../postgres-operator-issue-template.md       |   2 +-
 README.md                                     |  10 +--
 charts/postgres-operator-ui/Chart.yaml        |   4 +-
 charts/postgres-operator-ui/index.yaml        |  60 +++++++++---------
 .../postgres-operator-ui-1.5.0.tgz            | Bin 3512 -> 0 bytes
 .../postgres-operator-ui-1.8.0.tgz            | Bin 0 -> 4901 bytes
 .../templates/deployment.yaml                 |   5 ++
 charts/postgres-operator-ui/values.yaml       |   2 +-
 charts/postgres-operator/Chart.yaml           |   4 +-
 .../crds/operatorconfigurations.yaml          |   4 +-
 charts/postgres-operator/index.yaml           |  58 ++++++++---------
 .../postgres-operator-1.5.0.tgz               | Bin 15565 -> 0 bytes
 .../postgres-operator-1.8.0.tgz               | Bin 0 -> 16609 bytes
 charts/postgres-operator/values.yaml          |   6 +-
 docs/administrator.md                         |   2 +-
 docs/quickstart.md                            |  25 +++++---
 docs/reference/operator_parameters.md         |   2 +-
 go.mod                                        |   2 +-
 kubectl-pg/go.mod                             |   2 +-
 manifests/complete-postgres-manifest.yaml     |   2 +-
 manifests/configmap.yaml                      |   4 +-
 manifests/operatorconfiguration.crd.yaml      |   4 +-
 manifests/postgres-operator.yaml              |   2 +-
 ...gresql-operator-default-configuration.yaml |   4 +-
 pkg/controller/operator_config.go             |   4 +-
 pkg/util/config/config.go                     |   4 +-
 ui/app/package.json                           |   2 +-
 ui/manifests/deployment.yaml                  |   2 +-
 28 files changed, 113 insertions(+), 103 deletions(-)
 delete mode 100644 charts/postgres-operator-ui/postgres-operator-ui-1.5.0.tgz
 create mode 100644 charts/postgres-operator-ui/postgres-operator-ui-1.8.0.tgz
 delete mode 100644 charts/postgres-operator/postgres-operator-1.5.0.tgz
 create mode 100644 charts/postgres-operator/postgres-operator-1.8.0.tgz

diff --git a/.github/ISSUE_TEMPLATE/postgres-operator-issue-template.md b/.github/ISSUE_TEMPLATE/postgres-operator-issue-template.md
index 70d77b80a..a33b556cc 100644
--- a/.github/ISSUE_TEMPLATE/postgres-operator-issue-template.md
+++ b/.github/ISSUE_TEMPLATE/postgres-operator-issue-template.md
@@ -9,7 +9,7 @@ assignees: ''
 
 Please, answer some short questions which should help us to understand your problem / question better?
 
-- **Which image of the operator are you using?** e.g. registry.opensource.zalan.do/acid/postgres-operator:v1.7.1
+- **Which image of the operator are you using?** e.g. registry.opensource.zalan.do/acid/postgres-operator:v1.8.0
 - **Where do you run it - cloud or metal? Kubernetes or OpenShift?** [AWS K8s | GCP ... | Bare Metal K8s]
 - **Are you running Postgres Operator in production?** [yes | no]
 - **Type of issue?** [Bug report, question, feature request, etc.]
diff --git a/README.md b/README.md
index d51814963..137f46a44 100644
--- a/README.md
+++ b/README.md
@@ -17,15 +17,15 @@ pipelines with no access to Kubernetes API directly, promoting infrastructure as
 * Live volume resize without pod restarts (AWS EBS, PVC)
 * Database connection pooling with PGBouncer
 * Support fast in place major version upgrade to PG13. Supports global upgrade of all clusters.
-* Restore and cloning Postgres clusters (incl. major version upgrade)
-* Additionally logical backups to S3 bucket can be configured
-* Standby cluster from S3 WAL archive
+* Restore and cloning Postgres clusters on AWS, GCS and Azure
+* Additionally logical backups to S3 or GCS bucket can be configured
+* Standby cluster from S3 or GCS WAL archive
 * Configurable for non-cloud environments
 * Basic credential and user management on K8s, eases application deployments
 * Support for custom TLS certificates
 * UI to create and edit Postgres cluster manifests
-* Works well on Amazon AWS, Google Cloud, OpenShift and locally on Kind
 * Support for AWS EBS gp2 to gp3 migration, supporting iops and throughput configuration
+* Compatible with OpenShift
 
 ### PostgreSQL features
 
@@ -61,7 +61,7 @@ We introduce the major version into the backup path to smoothen the [major versi
 The new operator configuration can set a compatibility flag *enable_spilo_wal_path_compat* to make Spilo look for wal segments in the current path but also old format paths.
 This comes at potential performance costs and should be disabled after a few days.
 
-The newest Spilo image is: `registry.opensource.zalan.do/acid/spilo-14:2.1-p4`
+The newest Spilo image is: `registry.opensource.zalan.do/acid/spilo-14:2.1-p5`
 
 The last Spilo 12 image is: `registry.opensource.zalan.do/acid/spilo-12:1.6-p5`
 
diff --git a/charts/postgres-operator-ui/Chart.yaml b/charts/postgres-operator-ui/Chart.yaml
index 9e38f218a..467e19b45 100644
--- a/charts/postgres-operator-ui/Chart.yaml
+++ b/charts/postgres-operator-ui/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v1
 name: postgres-operator-ui
-version: 1.7.1
-appVersion: 1.7.1
+version: 1.8.0
+appVersion: 1.8.0
 home: https://github.com/zalando/postgres-operator
 description: Postgres Operator UI provides a graphical interface for a convenient database-as-a-service user experience
 keywords:
diff --git a/charts/postgres-operator-ui/index.yaml b/charts/postgres-operator-ui/index.yaml
index f0005e003..b7d845a73 100644
--- a/charts/postgres-operator-ui/index.yaml
+++ b/charts/postgres-operator-ui/index.yaml
@@ -1,9 +1,32 @@
 apiVersion: v1
 entries:
   postgres-operator-ui:
+  - apiVersion: v1
+    appVersion: 1.8.0
+    created: "2022-04-20T15:39:16.094386569+02:00"
+    description: Postgres Operator UI provides a graphical interface for a convenient
+      database-as-a-service user experience
+    digest: d4a7b40c23fd167841cc28342afdbd5ecc809181913a5c31061c83139187f148
+    home: https://github.com/zalando/postgres-operator
+    keywords:
+    - postgres
+    - operator
+    - ui
+    - cloud-native
+    - patroni
+    - spilo
+    maintainers:
+    - email: opensource@zalando.de
+      name: Zalando
+    name: postgres-operator-ui
+    sources:
+    - https://github.com/zalando/postgres-operator
+    urls:
+    - postgres-operator-ui-1.8.0.tgz
+    version: 1.8.0
   - apiVersion: v1
     appVersion: 1.7.1
-    created: "2021-11-03T11:44:00.75040818+01:00"
+    created: "2022-04-20T15:39:16.093853803+02:00"
     description: Postgres Operator UI provides a graphical interface for a convenient
       database-as-a-service user experience
     digest: 97aed1a1d37cd5f8441eea9522f38e56cc829786ad2134c437a5e6a15c995869
@@ -26,7 +49,7 @@ entries:
     version: 1.7.1
   - apiVersion: v1
     appVersion: 1.7.0
-    created: "2021-11-03T11:44:00.748544296+01:00"
+    created: "2022-04-20T15:39:16.093334397+02:00"
     description: Postgres Operator UI provides a graphical interface for a convenient
       database-as-a-service user experience
     digest: 37fba1968347daad393dbd1c6ee6e5b6a24d1095f972c0102197531c62dcada8
@@ -49,7 +72,7 @@ entries:
     version: 1.7.0
   - apiVersion: v1
     appVersion: 1.6.3
-    created: "2021-11-03T11:44:00.745261041+01:00"
+    created: "2022-04-20T15:39:16.092419178+02:00"
     description: Postgres Operator UI provides a graphical interface for a convenient
       database-as-a-service user experience
     digest: 08b810aa632dcc719e4785ef184e391267f7c460caa99677f2d00719075aac78
@@ -72,7 +95,7 @@ entries:
     version: 1.6.3
   - apiVersion: v1
     appVersion: 1.6.2
-    created: "2021-11-03T11:44:00.743617168+01:00"
+    created: "2022-04-20T15:39:16.091945123+02:00"
     description: Postgres Operator UI provides a graphical interface for a convenient
       database-as-a-service user experience
     digest: 14d1559bb0bd1e1e828f2daaaa6f6ac9ffc268d79824592c3589b55dd39241f6
@@ -95,7 +118,7 @@ entries:
     version: 1.6.2
   - apiVersion: v1
     appVersion: 1.6.1
-    created: "2021-11-03T11:44:00.741913772+01:00"
+    created: "2022-04-20T15:39:16.0914401+02:00"
     description: Postgres Operator UI provides a graphical interface for a convenient
       database-as-a-service user experience
     digest: 3d321352f2f1e7bb7450aa8876e3d818aa9f9da9bd4250507386f0490f2c1969
@@ -118,7 +141,7 @@ entries:
     version: 1.6.1
   - apiVersion: v1
     appVersion: 1.6.0
-    created: "2021-11-03T11:44:00.740290409+01:00"
+    created: "2022-04-20T15:39:16.090887513+02:00"
     description: Postgres Operator UI provides a graphical interface for a convenient
       database-as-a-service user experience
     digest: 1e0aa1e7db3c1daa96927ffbf6fdbcdb434562f961833cb5241ddbe132220ee4
@@ -139,27 +162,4 @@ entries:
     urls:
     - postgres-operator-ui-1.6.0.tgz
     version: 1.6.0
-  - apiVersion: v1
-    appVersion: 1.5.0
-    created: "2021-11-03T11:44:00.738607341+01:00"
-    description: Postgres Operator UI provides a graphical interface for a convenient
-      database-as-a-service user experience
-    digest: c91ea39e6d51d57f4048fb1b6ec53b40823f2690eb88e4e4f1a036367b9fdd61
-    home: https://github.com/zalando/postgres-operator
-    keywords:
-    - postgres
-    - operator
-    - ui
-    - cloud-native
-    - patroni
-    - spilo
-    maintainers:
-    - email: opensource@zalando.de
-      name: Zalando
-    name: postgres-operator-ui
-    sources:
-    - https://github.com/zalando/postgres-operator
-    urls:
-    - postgres-operator-ui-1.5.0.tgz
-    version: 1.5.0
-generated: "2021-11-03T11:44:00.736383918+01:00"
+generated: "2022-04-20T15:39:16.0877032+02:00"
diff --git a/charts/postgres-operator-ui/postgres-operator-ui-1.5.0.tgz b/charts/postgres-operator-ui/postgres-operator-ui-1.5.0.tgz
deleted file mode 100644
index d8527f293f9ad6864a4c19af3b7980964c1bb0f7..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 3512
zcmV;p4M*}HiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc
zVQyr3R8em|NM&qo0PH+}bKAO+`J10&OJ8T+OGAp3EhXhl?#|C~Gnyuj#&*)_b>3(o
zawXvz1Q-C6?dRw|`whS!q9`eH64z<2!XGwCEEd4-Vt;X|Oej5-s63IOBw9!>r=61-
zk=nl{DSPq|EeL|3e{^L32SL#OAN2d(C*fhg-|q&!UU>K<2z!TN{|N*SGW(8}D^28+
z;G5@aZ|(<aB%>ctDk^vc^Uxt#R^Ntx&kvjgRV-<y?W18SE%2_C7I;5^Oo};8PyvCd
zB-xC{gaPFm<%Glt6Cr^>EchHbMXnuyglIA*3O%Ac;whAK8YAQiCE!hlk|K}MnTZr5
zm}#A<sMDEJJ<G>_EYi;Jgb|*I&YJ8xS9p6ZWTGO+gR0l@pzik|r;Z0P6M5osqUjtB
zB+*iE)1flTgp(4=HK81(MT9A#EHX&A61j}=zsm*q2|56rm}UM?@yIFeHk~S`7;I5~
z(Wd6B40%}oZ#@5V!g5p(rT})F|HJTkuYLY|z5V%rly(5Ga6)pXp<uc_uA79ycMjlk
zMio$V(Eon?_Ki0YG9_AL0u##60KCGO5s5G-l9Dk)1zNxuAtOp*Vw8e_Taio9m}ZP<
zRKDX#%ovTyiO9K*AatAqqb(+MnoES70!_&j9s4(O03=SS((*Q{ul(iF^zAY76JY@}
zp{N!Hl*BYyKU(Hg0%$Tdy3ZUgFQHQ3IsjSD*ibMU-$vjLCTBtqB`W0FbsRY+v02Cg
zTx1y22}K3hGt@JbaQ@5j2}mrDI;fe*Sz;&<OC%Z-2g<7mw9L_QO67MP$9DT$&=R;<
zf!mGP;JKWk)lMKLpl8^|p@mVoz6BjjfC~)@H4vVZs0^zOvS&aFJXFCHxuG2<kSofk
z7Vjiy%=TAYt+e7QD~+iT$&xxwePy=IXiBxc0*JFbf^HC`_AbTL@~|5OZ>i<$#~jrP
zY}kekEo^)G)3XoCK|WUnod<7+@6Ioe&n_eQ+l(@-?F7We0kV{MKd5db0!@%7n3+FZ
zKLcGJ2SbCUy&I#xM&x#e_XCTh#(O**_%PrKrM9Cg)7Ruu6BuSF@edBv505kYUa|;W
zV?m<Mx=2Saf)|0?7;(aKrBR-d6yK0Bnz3>5$TPFLmY^jcK2Sx+3?p#=LYT6Bn#d_?
zGg@UNHeE@fFWfcimYb<W+R#|vW@bpahQZKa%!JetxG#bip<BtG*|&kMXi(9GDmW7f
z4uxTW9SJevM(kC2Mw)WSE6@o=mNap%_87EQ+AAE-KuDkypc-bx^cBj`a2Podkj65n
z#r#y6Q&V8)LR+*E9Cmx%aE&6A8{R129xucMlL%boMl&YGIdz>9#!##7nHcIj0EBa)
ziE+JD1a}K7;RkS)k5O_o%9DytHAl>dQW5Ye<u@AvHB%mmjd$V}GXu;-eT+PN6G2XW
zydi1E&=+!QyTCX|GZn!<T=!SQRA#P!wG;pk6vncU5Hs4-Ocgy1^alkRufU#?3j6mq
ze*3oB|EsNY=j`3(>4mRv^ta5hEB@>Dk6QlU^I#wUJx;q6Fh@xzx1eW4uX}!0AsP~C
zqb6m}BgX-&Sz^t=vo^$9hL?AD;GZKyqR_AH1uPaYMT0j{;4yGXmieoC$zU*pE1?5-
zqx)%%75K1NxZj&*|DW29^FNv)%TOv`XI~!*?l}M5qabXb|K~w>fBqk%-Q9Jb!kng&
z_5Dp8XZ*02sUb@|!w8;s77NEfIj1+7ab|TZs<7^{@04&JBsih`^QYKbg4)p2FA|PM
zYk>cuNIn?IcXO1ICJ3L5v&3WQAKFWrUgVPry#d!-B14|o>teAdMFd0YIZTjp3m<dB
z3b&DDncar({0$527G9e<%$5|4F($b}P$EV6Ypt`2&09xO*{HxqE|{2j%0>h~jc+Ys
zUY%VSSmT`<PxEKr8B9PH8nQqV3!7(BWr$J4SQ@S=W5&ldekKqTZkxrItGyqctB`C-
z9y+N>TXojttzfnS+>8E9EiwIv-AnZH{w!{+vnES{WyPv2*Shlbix-$l%Jl@?KPvB!
z%57mQ=KT_(+jiX&#wxBMWeENyEM3hMoHdwR6Zt2~W0ohog~I>2X2L33%M<wACK{q@
zv_YAiG-7JSwUqq=ikX1(zWcL7+^r01%1?1!CaU2RNX(dVHrzkC;Evoc;@tOEz~8y;
zu>WPMKnjLmpAOz(|HEFtmH+Jq{cgCo|Bun`?wXcCj!Ep7$-l+IX}%n?T_o2tA?fcn
zC*fbcunzn@bgn2*A}G^5=YpY=q9zH^B&y$#n3+DSo;E2s<L^6AD3UT$u{naU`0pJ;
zSVS{qsjp@ot9b4+zSK}Pg7uZ8gp(;I-uTwytF20zdEGvca-|86adp^Psf=aLiuXn&
zqpzjNGfN83X0=x{-i?In@(tKWo06HM9G4vhhwWv849m+YYWts2Ra|65$1{7CXNeiG
zwgA`NG;7kDZVmCZMSL%8rBHkh<2qTgOs!)|Fk|92MXptWSrd(uoGH{_gniR0>nQkh
znC|YZ(fQZ1IrrE9W6BfCr(a<PcIp4apq2mWcEh8?z5aiU_D|CPzZAl<Hv<o91_Z-%
zoS6MrI_WcfaRAoM^0#wA<>TKl*47*q_*Y*mJ_w`v@_o~`wBo;}I{o3=4*kD0n%__j
zvP1v(y4}OJ|9^D2_x~TIHC4Rvo0i)ARn7VpE&gw=gB#jy#}XUa7OOWijEnLWTkWDE
z2ps^03}Y>1LC@6q#cvt}en1j`%~q=!%GEWbz%;h9&<9=W*DSVZ8c7;~Ya$~T4q!+m
zNl~MuzvF*dpo$Ondd&iTo7uAUC}LSVSS`vcg~lz%x^)i})lCt&W>cjXW*_-gbrUQW
zoeiB;Z4)dO(OS1AQ&=q4rWrP>n%2fFOP-qaEb28Hdnxu)@3JU=coDn^S0Df~DYS?M
zi@?1+88(+9ktCF(QbQ@mxQcnEb@m!{)J8`1ECRPPL&EgTZTD_o1Lagx!d@XGw-*=-
zo~Q_VD<GPtC~{r*hAYc#gm}w#cjti?e<vwhdd+pc*C*iA76(DtU8N%Od7Fl%OdJn~
zqxa`;+MBW!D9b`0CSt;DiAGg8vjNq+;pzGD<-7CI@o?}NoDJ3^pDTa0xM+3Vo3Nj}
zd4F+vdOkWketSB4bNtKcmrm5KN!mK)`RT>G_va_47o!h@i@`5%PPb91JyW}+sdX9}
zQ(PXOzdpURd$|}MpKP0<JwdB3YU>UzPmkYTY{B`-Ydr_}$J!GpmDyUyuRFN(elR+D
zclMXT>-*1sw`ul7#A?JSzj7nDs#{)DjJ`S4dWHtrC~M>$>p<#``)ngpu0}=q=tyZv
z`P4N`3>V&Au<-A?i!Gzdx^Xp{7i@KlDyogZDKRoOVuB>2F=57Z$oq`rM31IK<2AW$
z&SPt`HV!B)`j^jC3+a7;bA|F=fXl4ksC+r2+-Rd<0k~mzBc)AGxV0zP*7Ik-zrCxs
zF2ZdgHf<lyJ!;z#|5eq6-}EWKuKJI|!(grcBYeJ({~n{YN&_l>m!+u7_xUE8IV<1k
z-}^1|a=4nuw*EGH*?#ft+p5~e+kb`W9Oa2fGa?b(|9lzx;gJ`(;Gd9;j47j<qVoTm
zkFktg{nu1iFN3{S+LEKb7V?Vn=`vk8Moq%il7QwAywS+5-du$(l1)W$JSgb2Ils9i
z);?=##u(~dQ$1UgvPOM#0og2gwv#oz?S^-Kv_xF8r3mY`NV8@oeqHdvfJwrqxE->p
zh_1c=#u)p&>G64X<M0VS=0Z1o!N&TE2hCb(Gh}(cwxN}=<qN>it$bxsRW-Mn!(6Q|
z&4v~A28sgj`nEBP?OFA1y)qoLdv5u?V-jB>PulTYdDhRioRAHmMO<k>E7j3}T%pZo
zntgA*`lGZR_P<in|4jQI9tG|1f1f`Oj`sHdF<R^U-<3By`@HW1=6xG!G_TB9HF63o
z&QPuCiX_8YimkMAgeUM5eq6(0KcsT2@21?fQa```8mgYCda`j%>)>m4ZJF%5JQ=Q~
zwZA5p`#>UAe1NuLQ<g}Jh2#92TZ;SX|FT~7E6u<T{U7v?TKc~mcKiG9e;%c+)_=5$
zfj3uzw9Q2I;(uT3@qqT`v8o$3o`2uaiB7qY_z-jKI{&?Hf9>}ly8G{c9;F??kZ6sP
z+n;JEcn;Sy<S@=DODLa0M&c_nMOFU-gvvAP&48LAV=!f6T)eubeEJL|GW)|SHcD(f
zCp>WufaBE0_wZ9DaYAn}DN+M}_}PbdoZW)pcBEMnWGI1Aj=tl+x)@z(A<;R26OpEZ
z!-ta#NT^hfKc%{3{}#)4{PFLyWB*nUX48)OQ{AdL?<^6Sm^IJrw@=D>>Z|L_dFqeJ
zmGjisY34lrU*`Zm5J^R@VDRcxIesR^Zy0OGrwNiy0Vl<8jz3qiNU-w*`4k(^KjjJD
zJl^j=g<<>q-`-(wAOAf{YZ-RrT2d^2ZJ_?Cj|kju&<(tx?*&JfVL$5iqTtXEUmOL`
m|NK1c{we52LE!#CQonEewr~5kZ~qqn0RR8=KA-*oQUCzn5Z8kM

diff --git a/charts/postgres-operator-ui/postgres-operator-ui-1.8.0.tgz b/charts/postgres-operator-ui/postgres-operator-ui-1.8.0.tgz
new file mode 100644
index 0000000000000000000000000000000000000000..e3563911c86384bfd3b371611ff109924082773f
GIT binary patch
literal 4901
zcmV+=6WZ(_iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc
zVQyr3R8em|NM&qo0PKA0a^p6#aDMYCdg<Nzl59xvu1jsUKF6b^Tr;s<wr6toY;CDP
zvMC{^K!yM{<IMOx`ze5=MCxK`F7eFn5q{Vd(P%UpjqXOHi9F+CS|B&F919|{!YFC)
zY(@%U-jjLy)u$+yWm$pessEN`t^c=z!1>Dd99KTNUhtJ=`>y4G1=c6&eNQR5AjMbK
zNB6ZpxPOyEa{3AjPBS)wg{_f1U;ebsfoW+m@~EJ>P_M3PX@SdHTHxh5<VCiiF>)X<
zEl5725lMkEfki?hgd{6~K$NisGKx%S05K6HBpeOG4PtOC7BoUAITnC#ax5sa2(?)@
z#}Ukg$oZ((n^G|=Lo>?ez28YnSe*4XWLLYx``1|!^O0siQ>z)UtT&*fngLOom9fEy
zpbM0cL=+j56+EYDrp*auf>4HqLWFZd(~(5Ncvcn>{{NZ)Ge!-75lQBMSFg0{*S1xq
zRgD$O_ZrlqJ0YKT{<p6G1xZWfpG*PlS^t3@c<bxm_k!d4e~NMf&oCinDxjKlwOqFe
zg{hsu?Tm7ua-sj@^ye2wk`;3z1jdli6eYkjj8ak{EJ#5~m?8&}L5PqO&M}rsA%pv@
zEI{BqPl-ToYFdGLN+WWXl}wDl*0d9;EfP8{3xtvaos%hQ>hDMcC~!)-DDFqgCv&wl
zO|^{7I8%Um#;M38D2ZsixwPb}1Q28@b)Q;X0C}0FS6ND<`w{5p$welv3gpOyE(yC-
zijbt~J@6bOn%o0nK#GFgLzYyeUX^KjgHeHkOP<t^FfX}~yqFVKDg;r1L@2I+n4!`m
zJSUN2WP!;__1%>r77Qh~Xx59fi1(t5MVJfqP^+q*>YVpZYQiX+8tRoC5wD0aGG-7X
z&tmOlgVvFD0suy(&or$FNhH~Q0yhdDMGmiLC}vo|^$(|KP++alfX}irjpf+tA!$%Q
zjX)G7YFe%0Poqdp@(#2FZaQ%L5lcL`HSZ+<B_G#th)fz5Q-Pi*z%l_G1rQe3sH8wk
zWZwa)@K6p@WKu#fhLTe@Rd|y!O;vp(yN&Me3am7US-j$owtSMqPU)Nq^#l;*<p><h
znyX)PoGVFnEbC{gr1!THc?Z^BhqV>#diUdtS6oB3;1!?Gf4;iBzCFFT9l<X%n&Q$L
zLL}Wh#fg{a&5r_svKVt@F*5NTh<eFO4y<f?h~hOOQ}|z=D-=ttr&s4DoHLGvP^}vC
z-kzg2FwC&PKWVUhd79IgMLGgq%Kxahxok%RYhdZE7H4V61r`@%jxR`vvTanoGGwp1
zVrM-IuPCQsiX+f}Bq>*Qf)rB}7tQ(`fgV>nQB@+D*OOp?F7=4s<v6|(1zGa_1jg#O
z{Gqt{x}XA?T;y^YGb}`<Wap$}J&9wCK_`;Sx|CMQpFE5I9|vTMl_u3r9@7Vc@H^h4
z>{pD0A>Av3<Vvebb=8Y8p$rXV3tD7sj!YO0USp|1dXM)|V1kA8%UYEN$!YI(R@|i-
ziH#+TdwCWc3%hr+twTAGm*=(iq)~T}5D_Mqdy<tbR)X<<4qz`#v#>WOa_V~}?KSTw
zCGAPC&dl#MHDBX;dHnNVN`}X8<eFCE$x#T(rpj{7$sGb`B<HhCz=GmykfW6vp;p=@
zmD-EdYU1>l8-=b`qVUV<i}B>fos2J@U0t4E+^R~bENwlozve{DM!g<)&5|2PxG-#!
z%*pQ=Bd@s}?a1@o4%VBi^B0#A?GLAC&tG0muCB*Fp8r=Dw{m2Aoo1)M+)Pf-&c-)4
zljq}qOwOORTl`pL^HKc*pt2Uv@%|c<)!W9^{aeHK)iSOcz-uaIvL6uH9kRc-THfLP
zE(+T8-Hgw!$G0t@&qCvND%x~D8~=Fv^2P1sdi>M*<;7<qatjH&1muU8XV1rL!uL1s
zLRdz3D5QCeA-wM>=D%-Pm0FEArd*Cj+~-nR*4p>{N~+5|E5r!&N<S-$E73~GM)x=N
zfO5FVV!X<v*-`D9)nDs|j`d{?XhJcK+qk<m60NfF4UX?1D}W|I1<Z)7t2iJn8!{kJ
zNKG=Y`lnnmLt$^wj=*(%$KK!vWpd`YshkSL7~>J>$fO#JEAK?tY8c6@WlbWjjs`#&
z%Y;(TwRL&>p#GgeSNS&mL@D8X1Z+y#o2`H%<pzo5{OeWk(9n1$n_ei<(B<n{w)86u
z^<xAjHSrCRI?&9DsaDCFR84#Y|I~G<B}FQqD}4z@EvXgOk{TlAjg&6Y*D`BV*yp6d
z{<-%&$FeG0_y5b?S?}WVc6?)sH{xUZ*cbnEHunF0>lpugnsS@L0t>m16*D3>cm7Ig
zN)`#JN{f<>G!2yIi9`y9vD~$O|Mo4I*O(&VXfAzeczA#*O1v=#BL$u0x!KiQ5`%2q
z2p#BK!w<@l2NND1^v}(+|D>{K{ZD3?=2&o3<R2ai+_U~2&$8Fof6%v&>;Ear+qb=M
zVL|636%|Qniqcp6elFP(&2R+Y_8uNI38jtS<Z{C**5WNHaA0aRoB`El^U+motUy)h
z(X0-fq}BlQRTX@eke3TA3K}E4lMWS&AaK<qo!^v6Lf?RHtdJp#)pJGcSrywM%RZ7a
zP4D5ilBAVuN%C9`!_<Dk3cG?AvJW{V4k1RQ<On>QBRpT~tZMMe*{pW>DyD;RwRfi+
zzOH@gXBRgTR(i0~8~w)A&J!pq4Ot<n_ujk?zEsgC>HVjv^n;aSPB;)nd`<0Pd~SkO
zEB4T2$=arJ$=(KLC%{4cPx%VdpVe4K-yc53EjpKMsjzgc%8J&`+xI3gFDMfU=zrnH
zU%0-8t?KuEgl^aKnlLtdA5sS=-)Bm@G1QCJ<Sv=~HDys+#`}fB{AM``U9@UW;ITop
zL{(~o`fy5$X?q`8L8T<b^w9CCCi>0^YRgacTpv`zJ18(urL&>`RR?{dzt_xtZUy|A
z%O3k*hf#|x#ScFY+++W3Kk&Qu-?0MQI@<rIC~x1kEkhBK$gGb=9v-xIa)7ELrI=*}
z{awW^%)5bd;1{-bM_D|AIuLZ7rKrtOkeCQETAq;0o*q?C+Z3Gi_cf>#NgcA-zJpNs
z58gs3L^Dk1CZF|`;yI*!t)Z3(HYu4CMy41W;l0AQv?}$P)b56qaY0yw-DVfHGA_!r
z${Qd#{i(>xT(QDXr<&ap&(=}XDg)v-6`NUL5!Mye46DZ&Q>-7SDAfOy^6DWcBAThE
zGLL104FYt1Td(n2cN>UbE5rw3tA*k*jGJu9bH0fw#yrjL=g0)FFbg6uDN~N(y|8bi
za=8nB4Aa}U%IJJqHs^5tA5s=mHvIrIuuuQH)>{0>actK)>i?%Ge~|wFp%Rv(8Th1T
zAWN}6NNRM_BlDsGY?|edp9NlqzhWemIjZn?6D~f<M6=ad{I0O#m#I$w?aChgzcQL1
zkqx*n|Ih6Q>;AuY)c;RY+A3c9O)G8wY&rEEE&hL82e-7_9uiy8mcx}A&oQc!998xX
zAAzj_a7-~0Sy6F)F1_a$t!em#6Y(JoX?m!sYsrc%X@}5HO6rFY3(&B*%FkFgK_6$g
z)H!}P*t(u(xdPlwz-$nJ$o@g*>24hFRFWPOFzEJM8^r3xDm8S){U$|3uLq$oYifE%
zpi9QJJ2sm<TX=ZrZK-UsY~kTyv{5a{6doQny18m)*>1F1rQo*by2>qSjinmTa+FoF
z#=sib9SDHD$V3)p=?L`Ov#T~K1(KLD<ov40LhNFm36cMVVziEoh}j7A-V8}9X8L;V
zb{Z(7f|B$ZrsVzxqm0FT1bzoZ&^czMSk~GdGFzL?Rgn|ECMc_%+;_lg5wV>v7m+R2
zd05ND>DATb<@Jm8LDd;UO*&?pMI=>OBXXSjc2t*F<LlGg%j?PM)%hbhTT_p0!Od<8
zD7=H;%1mIF{%yUSy?A+ZJHDP=oc=tXyg2<~{JtHoXOQmg`g(kG`SSW~d^34<esliA
zi}8USuIHid?dbOO`ls=&8rjX&>Di(EtmU-s?`C}Z^UV(2?~Jwg0RP;0Q^{jN*c896
z<$aAJBy;}FIsr{nFk2*kw*z}ey{)xR2^U*B+%<4n{<TM{UY<|RE-!vO|LJhPyxl^a
z9%m7sq?Fz1BfZIH-LNdCR7PTk5?C5CWIg4Ai=MuX9eT|8q)H>6a8Xb;)urmTAB?xw
zgZb8Z*wL!~@`_IumHg>duG&`MDvx%OW+a}3B$Z~hIHVmTVlpKHUz7WGKPvNWYlGU`
zdH<f4tLYHnl4EfYV3ct&!I0}CuuN;{JG-H1me*ibup3IuimaT@@={c=UjGr`C@Zi6
z^ZWj8u<9Ft3efi49iXca@T5MeljLq&y#TuHZ50|><L<0+cGlSI%LX>qoGcF;My=&N
zEt9p^*Khl{dIj{ZF^Id(Uq_RC?%mhtFMIa?P4?+WeG0f||L=O9yPp5kx4q;3|0&A)
zZl#{@I$e2{&mniS7j;g@!C3~Y=6n&U9PQEf>xl=Z%Dk@A3=}OK=L;;(vUyGl1pV*d
z+onA<EFH`<l9Q08R8ZvR^D@LbbYcEPMUxF+ti`Pviu#+4R=j$d4zi;=?~`^Hq9ArR
z1*P2t-w0$%HFsev6;%ae&Z~LZ9#3CQ&cShpV;o1B0rj25y6m?y^DD^BRQ=Y@t7U4u
z72<mGs#wF0HMX9eU$x%M*>CTUA8Fd^c1MRItp_BkiC;SGyCIt|1X{s~WgGRrZCs6Y
zKYmwaqW$fFY_j^j!NE@3`W*Z*$~#0gUpL4;<kKc;YEQcyK=r|c)^-%?UQ|ujkud65
zfEx3Li}9!;`}4u7rHxc}cdI_RIiKe#&XH9@S${|yb$~+>-64zD!xycuce-VlGU*l-
z-`L{Sv2q?_QMGSW{Wh<wQGbV1dT{_<HE0z8+DAg2_O<@*5xM)^Ci)L8d;Gt~NdJ-k
zpY8e9X8!l_{)Z<iYxlo&PIQmwK2Lt`(+Y7Lbu_tU{yQ%7Qi)ZezK!sA?F0}O!`JxR
z1`hQ}rrvyksc$3%o6GMimmRG>iQGcdCisTgUj^cC&#pF(wLau5=MxEH^&#t)K`AE5
zt;v6^5jaf$*ZJxnXa@G^f2(h=>3_#|{NwjOPg1)1Uuz!&Y|jK+Hxo_b&!^4xI__c*
z(19PH1GRPio00=`%CZ7Kg^cZ6|GpD!-2Y`C*Z-506SyKmV8PUdC)J!+SDJ(+O=HTY
zkdx?+Opz~d0OMt@{1o6bOjDSqSy&~dP&WMz3QX0-dTPJgdQVuaodCnB+O)#gd4UOi
zgK-sv`O`NhT(a~YGNxKek|4(dQp(WO%x5=~8<7>LoxoW(pJxnSo!vl83$B?{DthW~
zMZRW+zZX6Aw|Oy}_T-=DCttAM3X$A^mbv=&k!#<Y{B^E<Ylh@b`_>fmT>JKav=ewm
z3YwK1&Yz9BX68lqD@H;yX^f;-!4=uBnz`Un7Gv+<$XnRD{wa&`%~Rd~Y1^K=9{=~<
zqy2xHvS!$kiGo%!wr!Hpd|%ONQyZ?edqd^h`qs7)=#J$$hUFQycWe72%NcpL8Mw9|
zIF>i~tL2O=t2!<GBd!}&cn`XAR6)`yaxnrY46MK#1~Gb;@A`=!4x`BN95PJMx2(`l
zTss^P5)kx;UO$e4IF9Wgu`mpSz8h4o#QWc23e_z?^*yG|PO!eqwEEeBRd<?J51o5W
zp@~tdyG^TK)%~WRk_c%xJ8bVgr>iJobJywW&5oND0Xm6G?|JX3Y*H5K1{2zy7)sbo
zG5uWuE!Ji50AcN$_CdmG4Tpj2eVMX4>=TS_V#jVjiv48Zc{U;$`i_&h1N4I^8aTuL
zARw`0yI$nEed1$4Y(E+f{XsB1%Ied|YTvZC$!aBn2g&MTiq-N0-?qL`VRidSNbF%S
zaJ_y^NbC;XIE?KmKn&0i1LAnGZ4Lco*mok!wj6ud_gy;*oH!ye861W6sf2Z_V(lNQ
zSOe2NL|C1n@7ta|{4#~r8iWJeA|!B~IEwoA5WW7u$HX3Z=nics=qLTi3#^Ec;V|g;
z6DPK;zCE=1J|R*6D6CH;tbyrn6V^%w4-!`AFk!U<cjyIQrm)&xZ2SH&46QJ>(Z+!@
zNSv4uLV_e9(J)COCm6*2;lT0yp%c0OU>N#-9J@ouvy-FC`eefDY!_DNP+>jDWpxIA
zU|E6vWeTesyN>PpPU0js_QSC6`4%CAK|eq@9@qmyhRHA{!!Y(7-}0@#I|vfX8(N7I
z*~6pH`gFp&)n^S3^;v_%d{$>*`K~>1zf57Z(IUjgejK`y9TPGPhE6|8!X!xIFp5LZ
z_58p|hJihB-6-@O&yM2|ZP#%e3(-3Y>r)BqR-aYM;POuYof6o0gJIw=!%?3*G<Ph=
XaxBO4hnD{h00960=DBd<0B!&PKv%Pn

literal 0
HcmV?d00001

diff --git a/charts/postgres-operator-ui/templates/deployment.yaml b/charts/postgres-operator-ui/templates/deployment.yaml
index 8942539d6..c82d38c70 100644
--- a/charts/postgres-operator-ui/templates/deployment.yaml
+++ b/charts/postgres-operator-ui/templates/deployment.yaml
@@ -70,6 +70,11 @@ spec:
                   "replica_load_balancer_visible": true,
                   "resources_visible": true,
                   "users_visible": true,
+                  "cost_ebs": 0.0952,
+                  "cost_iops": 0.006,
+                  "cost_throughput": 0.0476,
+                  "cost_core": 0.0575,
+                  "cost_memory": 0.014375,
                   "postgresql_versions": [
                     "14",
                     "13",
diff --git a/charts/postgres-operator-ui/values.yaml b/charts/postgres-operator-ui/values.yaml
index c8353667e..6fa9e11cc 100644
--- a/charts/postgres-operator-ui/values.yaml
+++ b/charts/postgres-operator-ui/values.yaml
@@ -8,7 +8,7 @@ replicaCount: 1
 image:
   registry: registry.opensource.zalan.do
   repository: acid/postgres-operator-ui
-  tag: v1.7.1
+  tag: v1.8.0
   pullPolicy: "IfNotPresent"
 
 # Optionally specify an array of imagePullSecrets.
diff --git a/charts/postgres-operator/Chart.yaml b/charts/postgres-operator/Chart.yaml
index 7bc9ce866..7afbf2354 100644
--- a/charts/postgres-operator/Chart.yaml
+++ b/charts/postgres-operator/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v1
 name: postgres-operator
-version: 1.7.1
-appVersion: 1.7.1
+version: 1.8.0
+appVersion: 1.8.0
 home: https://github.com/zalando/postgres-operator
 description: Postgres Operator creates and manages PostgreSQL clusters running in Kubernetes
 keywords:
diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml
index 03b5d7629..4e708ace6 100644
--- a/charts/postgres-operator/crds/operatorconfigurations.yaml
+++ b/charts/postgres-operator/crds/operatorconfigurations.yaml
@@ -68,7 +68,7 @@ spec:
                   type: string
               docker_image:
                 type: string
-                default: "registry.opensource.zalan.do/acid/spilo-14:2.1-p4"
+                default: "registry.opensource.zalan.do/acid/spilo-14:2.1-p5"
               enable_crd_registration:
                 type: boolean
                 default: true
@@ -450,7 +450,7 @@ spec:
                 properties:
                   logical_backup_docker_image:
                     type: string
-                    default: "registry.opensource.zalan.do/acid/logical-backup:v1.7.1"
+                    default: "registry.opensource.zalan.do/acid/logical-backup:v1.8.0"
                   logical_backup_google_application_credentials:
                     type: string
                   logical_backup_job_prefix:
diff --git a/charts/postgres-operator/index.yaml b/charts/postgres-operator/index.yaml
index a5a5d4fc6..60a4fb8b9 100644
--- a/charts/postgres-operator/index.yaml
+++ b/charts/postgres-operator/index.yaml
@@ -1,9 +1,31 @@
 apiVersion: v1
 entries:
   postgres-operator:
+  - apiVersion: v1
+    appVersion: 1.8.0
+    created: "2022-04-14T15:02:07.818613578+02:00"
+    description: Postgres Operator creates and manages PostgreSQL clusters running
+      in Kubernetes
+    digest: 3ae232cf009e09aa2ad11c171484cd2f1b72e63c59735e58fbe2b6eb842f4c86
+    home: https://github.com/zalando/postgres-operator
+    keywords:
+    - postgres
+    - operator
+    - cloud-native
+    - patroni
+    - spilo
+    maintainers:
+    - email: opensource@zalando.de
+      name: Zalando
+    name: postgres-operator
+    sources:
+    - https://github.com/zalando/postgres-operator
+    urls:
+    - postgres-operator-1.8.0.tgz
+    version: 1.8.0
   - apiVersion: v1
     appVersion: 1.7.1
-    created: "2021-11-03T11:43:09.121092027+01:00"
+    created: "2022-04-14T15:02:07.817076674+02:00"
     description: Postgres Operator creates and manages PostgreSQL clusters running
       in Kubernetes
     digest: 7262563bec0b058e669ae6bcff0226e33fa9ece9c41ac46a53274046afe7700c
@@ -25,7 +47,7 @@ entries:
     version: 1.7.1
   - apiVersion: v1
     appVersion: 1.7.0
-    created: "2021-11-03T11:43:09.119630978+01:00"
+    created: "2022-04-14T15:02:07.815161671+02:00"
     description: Postgres Operator creates and manages PostgreSQL clusters running
       in Kubernetes
     digest: c3e99fb94305f81484b8b1af18eefb78681f3b5d057d5ad10565e4afb7c65ffe
@@ -47,7 +69,7 @@ entries:
     version: 1.7.0
   - apiVersion: v1
     appVersion: 1.6.3
-    created: "2021-11-03T11:43:09.118146221+01:00"
+    created: "2022-04-14T15:02:07.813087244+02:00"
     description: Postgres Operator creates and manages PostgreSQL clusters running
       in Kubernetes
     digest: ea08f991bf23c9ad114bca98ebcbe3e2fa15beab163061399394905eaee89b35
@@ -69,7 +91,7 @@ entries:
     version: 1.6.3
   - apiVersion: v1
     appVersion: 1.6.2
-    created: "2021-11-03T11:43:09.115637274+01:00"
+    created: "2022-04-14T15:02:07.8114121+02:00"
     description: Postgres Operator creates and manages PostgreSQL clusters running
       in Kubernetes
     digest: d886f8a0879ca07d1e5246ee7bc55710e1c872f3977280fe495db6fc2057a7f4
@@ -91,7 +113,7 @@ entries:
     version: 1.6.2
   - apiVersion: v1
     appVersion: 1.6.1
-    created: "2021-11-03T11:43:09.114107417+01:00"
+    created: "2022-04-14T15:02:07.809829808+02:00"
     description: Postgres Operator creates and manages PostgreSQL clusters running
       in Kubernetes
     digest: 4ba5972cd486dcaa2d11c5613a6f97f6b7b831822e610fe9e10a57ea1db23556
@@ -113,7 +135,7 @@ entries:
     version: 1.6.1
   - apiVersion: v1
     appVersion: 1.6.0
-    created: "2021-11-03T11:43:09.112550808+01:00"
+    created: "2022-04-14T15:02:07.808307624+02:00"
     description: Postgres Operator creates and manages PostgreSQL clusters running
       in Kubernetes
     digest: f52149718ea364f46b4b9eec9a65f6253ad182bb78df541d14cd5277b9c8a8c3
@@ -133,26 +155,4 @@ entries:
     urls:
     - postgres-operator-1.6.0.tgz
     version: 1.6.0
-  - apiVersion: v1
-    appVersion: 1.5.0
-    created: "2021-11-03T11:43:09.110907395+01:00"
-    description: Postgres Operator creates and manages PostgreSQL clusters running
-      in Kubernetes
-    digest: 198351d5db52e65cdf383d6f3e1745d91ac1e2a01121f8476f8b1be728b09531
-    home: https://github.com/zalando/postgres-operator
-    keywords:
-    - postgres
-    - operator
-    - cloud-native
-    - patroni
-    - spilo
-    maintainers:
-    - email: opensource@zalando.de
-      name: Zalando
-    name: postgres-operator
-    sources:
-    - https://github.com/zalando/postgres-operator
-    urls:
-    - postgres-operator-1.5.0.tgz
-    version: 1.5.0
-generated: "2021-11-03T11:43:09.109237601+01:00"
+generated: "2022-04-14T15:02:07.806370532+02:00"
diff --git a/charts/postgres-operator/postgres-operator-1.5.0.tgz b/charts/postgres-operator/postgres-operator-1.5.0.tgz
deleted file mode 100644
index 6e1a48ab772c479d991b518a7a5164311b93a632..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 15565
zcmYM*Q*b8H8ZO}2nb@}NWMbR4ZCihA+qP}noN$7PJ()Q9PxjuY&gqM;zF5^)UG=T?
zzE2ayz@UTt=K|7z&>2gqFq=xrbIN)1aG0=ZFq^4zSnH_ra4KqOaLQ}i*%&*Rd8;Wq
z@=KZ9*@Im5d3)|~rc>_)Zd435th(yB%<7QKde~5|2x_amTbt<OrX}a*lP5rn7gsNm
z!g7MhFf!e?47fHW{?^Dr=kaFnAgT}p!@}C$b#iu`2IM-E{!B-Wr#Sjd;W=sU7<7KD
zfcAk39r}GX2LEq=$Vln)_P5o`gW^bNNlD4)&_PVeK>^eV787(tph#$hh|#YU*r?Y6
zsAq?otuDdIP}MiGgOFx3fI|d9Zux7$$eyv|B5V{4w|tq8yR`;<H**anWUi4k#ISUv
zN$zhPl=jt^f&F6QX#lw%aXtmCyhS7>0ib_@5<n$}x|G$D8M~-Atrq+CiIeu%ZW&%{
z2N+3~3#c{JEzHAf?lk?0UJ@U1wGkija7zeBnjU5lLeWQ|1S30-x^<r@Wj)HvQXVzR
zPNFwN%u>Dtb3qki{*xFuVBLe2{6hp53IQlNaB((pA$6V&w<nR;#gOn+vJmJG+&`oy
zJ0yV}eE1Ax=ElcYdjcNZjs3O;3XY4K{v<2;A(A^=ndrwd9r!PrU`Z9e6N^=y**ruB
zO59m!BwRc$ag?o)%0`|n)k4{AiWHY#qS9Buf8m&dDuldw`~0d9UR=)!ogq<5TBrgZ
zfCk{1fFCWFGWN<6U&@5Ym0G&IBL0gumX&BF67*mO7epyZQVg5yTKrTm<t$W)O0^QQ
zymL%mDXA@J7JLRx4=Y8xh9pY(R_(mB&l0(0c2cFeNLG||r*&w|WNvI(5l)A2_2U5n
zF$zo5!fJpDUjL9kUb-Ts`HovUbCem`1R&ratm$|U5PNG-7)++g(>E0$a{+WyT`YKV
ztAlfs!5(mnbSsTxV5geIn}UsQtOk_CSm%cDn1&GZijzo_wMR;mGO*lQT31HCO|4Z4
zd8*9^?@>);oC*?tM3UiFL-E2SK`4?QLjQbp{QFF{n)Z`Se=Rl~poPk?3V?~C$g)TZ
z`UO5opAtFF0!x!lu2Yx@nh{i-D~|H9P8_{X`r&*)CvszX9($A+tK=C?W}FT9q(@P}
zq$csC9z|7bB&Ieu-ZK(2wSmHGf{5~tH9=+Tr`AFFyE$O44E6WqF=CP!;H)532&FCc
zgIfVdG9!EDaZx*W8gd!}<?LN^xRJ!ohxG>33kUF(3!5p9=psjoDwPQ<Y?gizO%AIn
zfkw0FAXUyCbG+vYaiuY57ONUe&W3;`4iNz;*<DGw_vC6A%?Hz^cM?P!Wui@&v_$}q
z62aU-uFr6U&Jy&O$4g%h?Jowg;p{#5vfAW(SO$mW9w$;t9zz>mqf`KlN((_;&FV!n
zFAF1ii#`&_Wael75Mh!3=$(df?<*STET7^JIz);SQXQs&Ujj_>iJ)Se2#3hY<5JO-
z<2yer#KO?9{~Jkt@1BPt7_SkE6h}=39n8TW91;?RaSBmUa9HCc$3H`YW2T4X<cL9Q
z(j5lJoKPEfSc}P%G7S7DkJoS3f8orsU?HFaLSZEZHp^iS;5b(3TV!@e+1G~@VbJ{7
zg&jE0DvbuhM%DEaCKi2*3d#G2o>=6#FTxp?bzYj3Y}3Xwc0ULqyqgJ~fQ}k-S>+Vs
zNkgD1i`wcJk_fD$&DHMhBEt(xHN%uN?vhZ7C?D4=++vFLz<c`>42%@20;c(IQd!st
zR-5BOPqbv$oGGv+z_in{aCad%LxvLKN!V4*64y3fUO!PuVz3pQ==*1lA4y3M&%yl5
zvR>v5vF`W=hve&Ff>sb2BvAy(+6RVsbP+K<L>>h>Kw)t9m&fKY5xJ7>sz@9H^su#n
z0vUq{xY@rph`O^J$deyQJu<lxMwxE1Ja2@(E1Ot$D8@9-%o_|S__MOD`i{kNQkd^F
z@cvg1D>sPRzmrhy?*HO7XP)h&wG@-lk|T6)YfWoMTf+ULv2+%YVo^9tg@Kjgo|QWn
zgrr(yo4G+n`5k!&$0^RExSaWu3GS%?CzTxw)kZt-_z;GDx&`7uh>V9sbv&x*K+S4h
zayTSL*nI%mQl?g}`4>Vq$ao~0G)lfg(Fd3o>8yJHy$WVDSUI7-wEaD+V3QJ)qG1gQ
zsZHtL((wzXYNk0mhyW-32z=G=EL8AMX-pfO3FB@HX8LQm3vkpXGpK@VtusQH&1xTK
zt@e81R7uhdRMEb!Uuqi~sM+@NbPcNUVaM9UZ1v>mqzDvYuZm5`GHmKfGhzPfn;XoZ
ztky4<$7%4s)lS~d*>iNKTC^4{B$1S^!`A&`yM1;URYRkxwp)t}(k`+*va8K&s#2z4
zDyT|INsvOr71{t3UbbWkC-U0t;ItyV1yoR9wiu&K<zW$%&iJh(<_VS_ivD1Jk@wI}
zQy#50U+xaf!OCHifi>c!%och$<KGa>n4)3YU@Ozok=#Q?lsig05rV<Ou(F0OgzY!6
zBik`Ya`VqgW?baTc8Hhd2RY>2m}f5!dco%0DXxYXc6sNyU^*8Cu1eB}>|oZ~(kT@U
zOE(noOV<4yi?(eXPwY>H8ewSi{1vei*iUlx(f+LP2QeApXnoVdKEE5q$z`=iE#Tcm
zF{|rvAwf3N^l!Fxz7T^M4A%>U3L)jvZz}YxmuSOUZWb9%)NY-z4bv@>)^U-$DEyLE
zJcXvKG+^4ID`^F`60q_u!1KRsEOu<tgJ2#tW9F;x)vU{9xLF|xX#N4@ZAHqM$y8UL
zRBJX^>(>i)++rNyBWjn_Tj6c>kOO4FN#DA51}t^JWQ;T$Fk24de(_YJ4hs)9i(Ryp
zdhri~eQvxQ`U7t#ZufKu#2qmy@Z>SWj7(|5NgjL>5v}%vp5Pb92*G|X0w_s9_8YGo
zqy21Ws)6Nc6716@Q9M^<gezh#4&&gNi&nvR=sq;3YFc2H65LET3*0}X)LWUZr}Yk%
zk_%Ztus?gWr=2H*9EIbQo>MT82Ob~lw&1y8VXwNR9tEaS_ZP`OiUa1jXO@|Kl&kj7
zKGbfD`G~IwHPmp!wJ&SIu0s~(yA0e1NjXeM!6@kOxlNa&Pkk){gGBf-;ND=|BSOL1
z20dL#Pi@ZVJ?CB<$|gVxLcT;Ct;H<J#hNTYaBc4U>FJJ1uL;tJa$!VcXZ>_wm+u~M
z`Y5D@b%gwt+C&5cxkPp?^=H^P>c*ATR#6Jgw8&toJ$14e9?`4@7+=qDpLQwW8>Aer
z!`2lT0~dKMKnpkZyDGtK+9;I+5-qx-E!>oXj2DGuICnu(tGGl)f|V)b`a4lnBx2aT
z2!24_R>Irj9ZyGB4KIM?nG#kDOt2R@W0HBcg`CNIwJs*>$3fx!4PuLYk0{@Vs}=V0
zo&1xuDF{)@*QEt<VOPq@7uiAY!OljOu4|e{BS;IY?&+Fb$R}6Efw#sJLYYny6`B4g
zEeIhn*b6}&p2o(CcMpz0s*ZGZ(44+%qd7ge_R9)X7EStvAs!bIPYaAcrFc0X!)<YD
z&01St1CHwlRyPa0FW`rm{FPcUeT*u0h)EKt7^JK&5rGlU2{gF!xE^cR{sW;@cnA-y
zL+d2GE8JU08Pd2-Nx*xx9^XUUN)D~5XCVYhuCC_}<s#$xoR1a#m}}z4(*;BNNx9fK
zrCGcqjVfkV<N>n06CkCqF|F5}+s}UQe9Y@DAXcDR?AEl0_IDZTh8mGfqEN>y9>T6p
z<2KcFzO)FDP`ncNc@h%#Nj+9$XZaYovcig0QJW|Ogqp5ecRDk#u5{bNSZK8mP`Kui
zsDQ7!-`GflqOER9T!!{jl{EuJ&1f9%-#A=@cwii*H9KC=kuLXK^&QkTZs_WAxO|p5
zLywNQK%EA<>mQto2J=z;=*o8-%HmqvkFrEvfndzXvS(=~a4kr~uVx;?_FMbN%>bsM
z(NNa8GdH0fY7JsUWhu&Ec2Jd>NsRP%Dv4G$9t$++cxjZ6>36VbVxmGQIZ+~@o??Rk
zekEkulrB9ccW}=o3*ntt>RC~2-Z7{}dQgPhQY@M1Z*uk5Cvj<U)*haxLfA~<jG|k1
z^5PkWV%m~Z4ZAKEP3xhMB&b$P7HO=1FmtQ?Ho*tj=Bf|{#4A0o{a!27Z6Zt&bCGd*
zolh>}!BBz4585WrQHZjoXE*d$)rPGO#XXa%;|1RYE#*Kq?=tPRLbX7~#ID3f$$72r
zhT3IV(Gh><$k{mLf>}Y-P;A4G55y%KU&=|5Wy_xMJPa9tRpF1H2s73wfGpPcqbeMy
zmr5#uzp5%BEIZ6%)b{Ueyw-6IrR5aSH!{f&IJF+rqm$K|hqnTOQ`}@|g2BF1#YJgC
zP9Eh!X+GSoSH{xOc~Yf6C0o7z9?*#{D~t$`RBYJo59qmZhPr8x8nbzK?oBA9Sy+Sg
zN0WG%Lh}jx*4|LWX3+`Qr8M!r-@791kXwLDH?vEEIXZ>Ez=nroGweN0d|I)#RoW3a
z`&0;?ecvpgT*X9T1zjVh1>U-~@zN4H;N@^|gJT2oY2{v*e3G6riU&1}YcFfP&&cl*
zWIs3d7TlQ<m)M|NR$+upMF(#)Cnko=apB`=W~SqmYGUIQ>-ICHre)>Xa^rNuaU!lA
zzpfUl>UMAqv71kZz|ASWFt9O-WqNh(EEjtTM6$uzDJ6@)biTZ{knYMR&P|wInF8Zj
zpDa1(m`REE?+CquGy0GX=a|3#4iiy;hh{-Q_FI0v;4;DrpWj+rN?@(@%d=VVyy7Kd
zG`MK<CtmZNwN@@d&!xmI)(VA!7-m&THgBz;%~4oIpf65>OQZ!}b5-yz%!tD<7>aGC
z2u?z4Z#r`nD=7T&ONBETy6!}nDYZNcc^n6#JWco*B<KVB!38@=68Z0#RcZ)CCkVNN
z`(IwbJE;vWaI4rg@>79Eh&G`mkV$mHqMIQQM5r^I4@CyETxrZubkl#R*qh#%M`A>p
z7uOlOQFZiI>SA66Y#P*mj*!)q#Jg{-(2IEbDZ=CCPIT0>oO7jIUyvKMtZ@e&8TXGZ
z0uOR?xQT_$R3ua4-IONDAg>M^(X@Dnruj|4cAF-=yLw&qS%Y#g)fHWc(?YH2KFq+L
z>VGx!WI~~3gR8x8mtlA!*@v*=!x=x0tm$UZ9?H!Jzh#>K-u+0$J|;A;-oR%VYcg~x
zr11;0Vl}T<Skd(O`s}YpBm&xCR{Pu&Ojq(oMIPS15g#{0HbwA+jlrK_Mij9s@sQb-
zugR!vtyd@r7t$lwze3QxQcCK`*9urLJA{`vb(|baHT3uJAfCMbq=e|a%jH(BTgR7w
zT(SwZF6BmGD%VhqcD+;bL)~S?EH^!Qfs32~RI)|!DU^}J%KcQ3p;GlR5+ZIW=&=TH
zr;vlJu@ONCNicYA$k-=b{kzz6lGkiEhwQfn%~{T(v+2pM$6dYn$@g2ex<^SnSf4^!
zihfOd1V%E+8WYpo{!;7EWT<-g(DLW9toQ1z;yvR1Diu;it0jUvzcb$BbUru*;q?n&
zS*$o<!+(6s>q*++pUN7$01}#bkST&_2|jBvjCd@ld3!bT<Q4)s5I>{=@7Z0w?d<n$
zJ!F+eQ#vGiayATm2(wugMBx>fdP7UUP~to#y{b-E5@akj@)1&yC8!RzwLy_B%&%N=
zW7Bt*uip`9`&IlRyCuRM$Kqh-!6ex4zToJXHVK@$Fm;n^X;h){msG0IB{A=IIuDgw
zOk_v6ix<SW@|zJ`fL(*`x$<88fT^v1)J~c@Vb2!`XtE<ZNibf>ytVu&y*>c}nr`cg
zMze?20STB*`wFzXM#S`XCk)D=!^Rq%yB}|`qs$B;bJFNUKR4vSzXWBmsSXBew8^>E
zb{a3e55vn121S?DN&g_q`*NToV(_d(JK1lVursFV?s^8f+WWaMTvrb+^0ik)=PRZG
zto?1-sn760eB&F#MR!-Y_|auT=ZdA!{7uc913soVJa!gG)-vndy)I1paKfuI^uJHg
zSW$MoWke4>&|^&ke%Ed)3OE%m&Zx#DdzYlU3K2aMDsNZN0VL?9+WjqwJVZ3{URUQC
zsvcSlIYc4&x<o}gjzR<ubf-xKZf(I4?n5VUxKM8MvRYDaeCy|fBhqIFVk%6}a@4)?
z0{)<YdSh_Bl?Ty7h~ZMT>zUrLfQlCeiL+Bf(BNl!@^xQxI*1@F@0Qr`Ql!3Tzyq|#
z+|d3Of9CxTlAa5uIG~`Q-EpoMombG^qr810ylZL=rMZX<^+_EaV}{SuLp9)uSn|}R
zU1x+odNZ$`<U|9rPqmz0HdanZyep6KZZgCav(rQJOfY(&_pfS#JU<Ur*UW4nFS)oi
zNE~0s&Z{Ibjo~}oL|^{fC(*XxQedF7Kj+O9(nG#|Z8xwZu+^{&==x~lbms;<Wy!Y<
zcYxi<SJql*YA@oM|CA{p%cmey5(<iiWEIKJ;M4SbK1aLIPl_}W_PM>*ef$HTjZ!t|
zY5?fwmiIL$g+VTKnTP4&AorOD#QB`fWZV$>_I%#to4@ZDq)=1nJ(~G6)CAEz0f+9)
z-GKu;fv*q$NoW3^s{(Pl%kHI`#%SUq^-7QVj9$1iQ&4=?fe&TiWxSR|D&}l854!G#
zy{6ON5|dW@V+P6!)=~Wa2l!G@7oU|(1tL@Mmkn<?FRrMCCE)*3#!V>*OX@X2$fPR+
zY~>W$EhoJ3wjl2caN>p3;7}A)*CGr~sfvqg9#FS}zF^o%K%vT{eM8Zy{Hjk?Wk`?g
zDv#|hx1}MsI6RPDcMz9DrKB4-E^`%8Hjb6SS&~y1!L7`Ysu)5wHU`0r!+M82;)L5>
zWLP-evJ+}PGZX5*C%94FP=Xlf`9d{W$?bZ9A@qWu9D6FNAdM44-!Ar0>WY|x^7B66
z*|AtRxO{upY^?3GzIL>_<!0-zw#LLhA6rhs9iw~*HM}N$#GZk`hk2$A=c3Cao}P!M
z2Z{NC&Snpq%eMXim;PdSjKUiFyZf8E3g&t?k|uZ$P_A=P<LFWeu5gsn14IedHfs{L
zSwni`5+(Jv{RYYN{bHX|1mdmM&qZkZGGIkqf1qb+{B<k$_WX4P?sY+{gF7Ztb88K6
zJY0(eh-lbfIT`r=E(twC^?J*auUMe>8l>p7$mWs6WXS8_g#8cJ(q*3evr>+~FGo`*
zxQD>Q!3s5gG0K2Rlb$p$QwrZCO0o+BO{ZFUf|C%m+5j4-C%ava(>5ON#;3X@gq2Ep
z2kmb^sSh0;-g4$f{gIKLM(>me=u8>xNc0xnmTQVj+jBVN9X+YZ>aQkc$kpJw#l2b9
zOuhq;XeCerq=VG{1A5uDMDH!jVl&LA+(RiI*6a<R<u%1?@0$;VAYsVysbO$`gHNtv
z5ug+(#Xq`)wcpl1xA_X`Pi#r->@I)Q=a3NNL`Tlv!_Z$Apy5l19e;IQ8k;)WaK!J$
z_t_oo0IYAQUx}qjSM7txf20r}sWGnpVbysM(k#bNWp*r!m8XP(p%}E%eE7w24(uBS
ztt$jdN;1*j4?ID?kCafo0kIb}%Ym|Mxm1)!nIQ3`eFFzrJuRBdkHO9kRSfalw#Vge
z$ID{QS9Ju+pFK?E=~)5xy0m$z{Fp>Zv&jzQMb)q>W+v!!(#3VFye3K`Fr}@9T^4`V
zp+%6Y{Lga-J3N*>io<(OAR!{s8s(;Es0ZNsD3R!kpI3Pf;`^mj08-)!Qw?yrdUbRt
zl{R#~olorQe^8Hn!g++bQRz3-nf^BV4Tv_0v%0zb8?g?U=x}@2?0K)FW2OeVEVO5F
z(@gBoNN-%!OHemPe&~eC=B$s13~3Tcbn$6OXLQp!h?s<?T|MdkxEFg|jB@wvE5nYP
zYkOr$vW<2dR0EJlt@Z<j0oHm$b;f$^*^nMT6c}aZy}4u~hne-rxywOiczVi&z^que
zf2^sKSyiHLEwL=Pp_2*=;tHOBG><5dz+K9&y}kLM7ALBU6zm3e+M>E${;qFgQ1g`f
zR}3S+6F}`Fho*IM0iyqmD?Ij3`6dO4+$y7G?rHU~GagU4twLeVd+px5XCQW;)o<?6
z@St)hDw>Dwcv0(1>=o<|7yF3aUI}++&<9HLvbalGkani<EUd4zXiDidS{ytbjg3%U
zVq6<c=q6ajJ=Ri;SMvjJlmvrOGJ;wwMFkTSJiDu|smvh1ACvzvbno6Zx11a$d#6(2
zQNF{*{vweiE~Xy_P<ljM$FEc$q{bN<vs1-)tNGep64l?hw8v}-!mzryTMk9Vr6YCC
zS+aqSQW~yM1En&7s?!ozC!yREcDnUQq!=|Y(Ng~92<6JhHc0z^`?X{&4kRKJ7z6s}
zo^mY%4KM#am_d2Luz|!(Vr0cXrd@FH3N5gjXmyP5xZa7E-*P1I=7Snousj#&QH3k!
z5dAV$8VT&Pr1|X7i1399HT3>^eyF?vy0iP;ivfv_SmL>k3+iP>2yJ%a*bU5KAb)j;
zXIFi)W?#IBeZ-)7^V!L>+_m!yhV?fO9C26A*1<v~2#{q1$djxBwXHF1yyu$ZOLlxP
z>=J7^VtR^V*K18e9cp(&(9DeHcxlCE-|q4vX)oiGyQumt?k~z_bT}G}N@dRlk%-8V
zCS`y?;j}Lz0z&>*pun_hIuM+Y_fU`?Jw63TI8dP}S5PuApWetep<lFbU@0<kFDFu7
z32R*%n59Fb-@*vAgGa8S8}VKEyECS#6QQ`U;hlaB90qs^(l(j^2e^G$D)WIzPtOHU
zK+~V~Vi7B9=vyDwD06#hmtW=-t=D2YNy$cPVhw0y7;grfvUzXm4CzB3b!_!W9L?Kq
ztVk+$j38KS4ZnGuvF2*#*kQ~F-Ro(ZH>tBI?lpV?C4$fOh>U$3TF4kZwYthOJ-(F@
z0%R-`?-7!uF)@@b1HcHY)}_D31Q6`LuHjSn8lEZw9I~!ytX=5O(_!cA%6qv;U9s)j
zQgeChGresCC$4=pZSz}kWdR%Pg}#}^t0FWi7MWZIP4D^%UX}nmKV)iS-co~qc7w=|
ze`E&^O6JPL`~7g%1Z01zW)RB8>G7Vg+Hgec*)qD2clGTDw0#P8c>z2Aa((q2$pSy`
zds;sDmXnIOHeR-$s_pL{O#D7iF6^!h_44%z3=Q>Oal?q}-LSi3Hh}IPE}ubOo$po6
z0kRz&a6DbcvGom%d;R=959+-gbIf#x%-Z4l=ZjVSdC}mDetd|~b^bNw^>lFSDO+pX
z)oTW(&?P_j#n487;>n|xsu%&nr*EDmKpco8!r$gkF`F+{+HAPvi?hO|2<F4>>s}(W
zmsnp1@-FSy$KhTBfsH4?o&n&)`PW3r&zjC)`A6Tw`rwqBm293))b%5eO%5M1G60q8
z{Zq6QkHk*6hC%6(O1oxFrmx|d3|CpLJCbcVlviC=3a2hc&SshX!oy@3?e;utz*atV
zDU{kt7tuPGUrzO(#v44HVMGK!D!=hh?ZkHQ%Iz^%h@IaRGt5@4sd&^yKXUrZr*?km
z5+1sOt9k`L#Zt0mige+N<-qQNREI^_Myxtm1>Q@tI58dpEhhHfVT{&ZZwoJgY<}L%
zUJpR;)QWqt4dzO!p8e130GFiX9jwz26&8F(S1RGNmztrPV2kSSLtUd4pk9%24X>)S
zQMN<tms@9?_Q_Yva~sz6%3q@e+{gV%&+V3A_0xgxv3&8muymmM(IM$QeL9df$UK){
zsLpAe2U6h2f4g8uxI59*5A<bL;|;n4Z)>sv68)%o*gtT(I|klUC%`Qd!)~1CJhon5
z>ZZg>(015HhvNMtE8h4eyy05yazIA{6~z-p${wS3fr&F~&FWjfLbj|MlDaK<Pe^5K
z2~A;Ufyz>vD|Zf!dV!7u+&4hdCtm_<5?g~vRjN*wr6}Dv&x`+TGRt-IScHu+O)n%;
z9H%bTbMs-NeZMe_3#j0Fo&0`yHxalT5qwf~&Uu6vqNIrIo?Z7|NAt8k)piVV<X!M#
zRp49rQnZBrjN~!T5<0E0EMAf2d54BItB@28dGE%z1ip|>i~sv<==IKS@NuL8qu&QY
zhn9red=wnS3>~a|1;$of0hI^H_6G<QB1gL#8oG%c5e+L|Azz_R_L~9KMKJ5e|Jua2
ztnb9mbT9=iugdvEK#}-UbV)bH<vfmPe?;}go8Kz1Z`U69iiLgj_s#11YoZjfPjkm_
zNeCAht(c{YXLg`3o1etaJVW{rnp81DiY8JS_Z-u*?ZU=1b2~e*#4mPDSsHUF=lLu5
z!ThIKbuUNp_trdJaS7S2QG0xQV^*>5V^R<4FnE=kl+XIpYTs(0NN~{*97<b}O<!i{
zt{Te?Zv-nakJ}stu0B3FNo<0`KAf$oXQ~}`=!^E!iVNR_-Q31v%T~nq(pM7Yd2@Ww
zxH)9D0Mk(NJWtNg*zA3b6@|fOFLl`h@hg+EKL={G*j-OY`6S*8HogpsKL_(Ndt6Kk
zzk&cDHdts7&;cDTXO^-IDFblZLBaS(_)986=2_|VjGeuL=;Dr;V(R|SYXY=s9?p3F
zJ?*2L%y>LM)5cVx<a`$YmDlOQchAUliHDYDww#PQu`i*FbUHfFp7|)6Oz#mRjCM^f
zsXCr3=Lp^J!>josmZq3eJt#Y}v2Ne?W4G12jH%A+q^b{$@NG@HZatqF6D5~jJ>hO5
ztU-0|Kb2x5WH1Hs=+7tj#1O>lX=1gigpC`AruR>|O*2Ing|WA|@t4pPmx`urS%Tc?
zZ1&hhE-@?Uug@Q0YMTf%Wg!F3@VVcG+Y8M>YM3n&;ddkFx1tlG<_h(ioP{80i_~36
z0?W))7H)@=y%i8VVZJ)oW9d!g_LqUdWd6SRfnur{W}IKK??^x`xwGw6;zQR*2|8fn
z1<_j6ZLAt6T_98SqD|S3Ocv%(f5(N?LOHM|Lx&Umu5#8xcfEnIV82IJqZU>>);i9l
z{wk)<;2)MgJX7mav;kg%2=8_;9WTpIDOJSzsf7Kq#9a!~L6n2bgOyf)<#$H?U;V~b
zg&D8zT<g4<ls>0}OGmWU2i2ZPLgH>{h7nx9aCPvc^$-hlWmU}e(J3-g<gtk5PXMaY
z!)ZO|MHO46ClN_;zZmZ0Hm1h;rL?3zewJ@m`@iTq7IJD9%KN5;qseQ!>Fm#4YHIW|
zX-|`K>Q?%VZVS0Kn_)HYYa~9Nhk})c!rCL0Ny|1SM0CeS08*6dqQyJv)S`LfNm19}
zL6X<|){sc)N7GX#<s3GyGeuM4^~K!%kezfrYv82ZS4MwzF(oJ8Wm0OP297%m=>SgB
z3UjYdGj1X6!Nh1RREz8#>)LY4M0NWjlUsSlm~y)Y2$fe@+C>H;U7M$v+h3{ccT>!Z
z&G;2BPFhFtH)9tKzn*WH4!G`54Tl{)0?$8D18@ZdUz(pc`UHXI);*kgmq+b-GS5D@
z<(^aD>wE*}P+lx<8&1|1EHgG7_xyo$`p`Mhc+vLvpr62|4Y1bNf0iY8TlO~IM{$rj
z>P-y5AF5q#Zsib`s3;Vi`|!yoC%6D!#?G-ES2gJICo3zklbXxSXW=pkOFTE)`E~g=
zPR+SCk?fY|{V)|?<!~9)HzMt39(XR9j3uu0AGkW2p==Ua?;>p6VNsnKjgC!SNcbC<
z{Z-TBbXA#tH85kG#{VSpY=e&~wC1^>J*fN?EViY}xqbm&&?jV8cRM{sE&93dsI;A0
zW}OykgT!^$>-Qq8s@v~Z(}Uk;ekiWvU}L3o-?{s_EUNl<cdZ&mbo|NO0R2UMg9KtT
z!=VLj)-J|kfds0$6NoEpIN82Q#i4xmmucb2<6Q^hP25k_yY2aWEF&7<?B<5Ott-rf
zWy+$zE4PKCr7`Jwj@}+>qD%deSf7>q0J&77tW7aX0DE(97LNPv0s#7Z2VTxpQ&UvR
z`KluyGDEl(wmQ7o(H513*FgZTn=8RmL7wM3Ya^k#9V2V6*AQwd=Oh%=7c9el<*<C{
zNay?e@66JV4RjzN&(Z!PBU+8--&$RchThhd(o?04LKp;p8hLUD!nBLfqB?P8fLMzB
zWekWUynu)mbQt*`^1)`8_Wpm9R*N?E?FYf-Z>D%ZaxI89!p(5^6inFC2KvWJ;%7Cv
z1aaJi{3-oRo0putJE~4jWc)GelNaC&TM);sC$K*xDr<z-6csP(7&71p+v~2^-Ktpd
z9x5kPlC9|E5B$Qxai3Ox0n7^@8bXz01z!$EnFQ>FJ@h}~p)T2{F9OVLV~3YbIfMa)
z^w#`JDfm=ibnMyF++@A2W_@g1B&(Jy9vh6U<H#rTyzjD2HGy@ek8SY;G9}!4mRJ3I
z!s4(SBSXY{5WD-(o-|0$MtAWJOA!t8sp<cQxoJJCx!1gAZL+#ovnINLOD@2PDG+ZU
z%;L-~uot_xc03TbAHUzP8~B%c!O;^qp)V2`DJuv$d|~dqQt_hHv!^<rd$D(3B~)X0
zEWOb;>?-fHV_oWA)<dL(;7qEL4)YzvcJ-Jt`sMc8_uhe;u7Wds{&drEV-liwTpc6=
zKI{5QmNbQ#F#Cu%{TwdoGgQkEnD%shkY!LZt$U}hc4m0jlUu;6Z~mzC``S1=%M4J%
z3OvpwPSg`kS!rw%|E#K5Cg9yDVhp^iXQ$FF5mrqGC^h`d2-CGRz3d8=TDX&6f?uBP
z(<tq{(CD-Z_cL^3PmSnNPTfCk?uyoQKO;ZNB}a`CN5zKvk)lh#^5N9APNCju$>&(#
z`3DgQddoRPo+U{1BBL4xQM~_1n1YAYX%5Jilpe7sMZI~>m8d^6>cJ@@{+E4b=agrf
zgNHXqFxL}vbkT<VDV&EeqT~2c4ZGdBiF4C38(;zmk?8XbgDEg=RktS)J~yi(`=Cwv
zihgH*(>8fFj4I`;ok6za*od)f9)rm;d+6S!%WXBV@!6!4sGE?iuYP(xEqcCg7yuMR
z3;@3G&w+$qOurXCM+%_#4jOz19tk^?_R+T@I-muSdOr<oqD_!FUe@WeI-dwG(vEQ~
zBQ^G6wstW66I5S)dvJeF-uH0dRm4Jzy|%oFFsA3_w-E$Jg9Cp^Zg-7J(M9j92;Ki3
ze1F<-eCpi|)`=^K5JU=pAHWO*{@ekorv*cDf4JYC+P+>gek?N?FA;upb8V^1YV^^4
z-oIKkcD&0jv_5R??iK(~j~aXnJOjVRZNZL&tXZN8FL{J>XD1`nCcH^_b2ug)2@&9y
z4b-fjFC3}Pe*gZh%f9YG=(fqXCF&AicDU^(6D6dDcr2NY41gSlHCd@5Zib;I!+AS+
zD@93ZLE=^j#zq|Ye450&@f~q&h>$D^b(D-?>Km$ADs;FHv$@TJ0OPCxlc$Li>7mj$
zv%zRN33K>4u9UuXk+ieyj;t0EDUf;;=0d}}Kb=l=gjCU>cw=f0+{Ib2eb@!h8PM%}
zShBh4lT%auRU5ar`k<}tl_S82HJMn|sb^s=?!$42%U9>q?mx#dRBnm!bq8U+{^k6~
zXGV8xn!AB$H!MQZ#8;PNT61<~=WBqYLzj@((a*>3^O}%lQg|r-8pAhf28tvbt-cg2
z2HV85aIT0O4OJcnP8!)*K|~CVB81xU=eIRjw}3AEeQVHFa4px2p?qzliTJsz*T;V-
zMdR~Up0-a@;mHN@QI1^SI`Hl<6nlD~fF~hWWgufXl5qZvz6-YK*g^jBbr-k8!Rc^M
zWzbA=dnt);id=@Or7yTdZsv(({-9$o`~u0pz3V@bys&}u`hf;3Xt>a_5P0<X7kg&m
zi2fJewGIC%8?`{-Cq-*E>c>a#@pk?*-q0r`e+|OEOp<$6XLd}r`54t$a?5d1FO9J%
zAf@5dAbbAf)eM%;&x%MNrO6`eBDkkVj9Hgxm9`{I?@)WA985g6?BZb`)1g7^VeCTI
zr*2~qOPcwV-Z)F^AZyL$pi1b@*eHZSByscw+>SwTn5}@~3w<SWml3To`%P{B`o+sc
zL)R#rL(7@1bw($;<saSlN(wY<_MXOkyBwQr_<4H(W*e&gCw-fm-!`9Bn~LAIZ`1Mi
zL9(M{-F|rJlOiIc!_!KD=+2K>XSw6$8u|eWbwwU&;dsH%IE=(}0s0M@IQUH@doF(s
zlpg@N=j;aguR6w8uSohf&RWwSTKpdluGc9_As%DJ=Jt%C<9KeB9FtaTm2;*(^d*2g
z%n?l6gXNm)I*no9Y>w?(Nm+Wyn)E5um?hlsYyvkKFx(>;3M~tKNvr8c6q;;$r4M|T
z_~9qiIwes`*~Dga>G0!RwB!-^^H`ZB+t4LAUC86_Z&@mcO*pk~WXi+S<yEx5a}ggN
zeTBLh%-CIJ1#T^pTPPkJ?y)*z)fnlASvXnVqz3V9ltNYAH6tkeHc9u3bFtqs@@t}t
z{feX~eT)b@iB<j_%(uZjiFN%m9-dNugpSBzjOz6QHYc5`&;qka{nHBhAlDfkjpQyi
zGH?T$pPogFYkLKhUbZ-Fcs17$2X8rcFL|`mJZE9`e?YUj!E_RA2}uhzQ7t;)d@Fy7
z<q(oyflazMSIVmkwrl_mEqBrEq^WAdA#QgMLI*zyh5u4LCzo|_h+PimO^Q2|pOpk=
z-Dd?hI-KTIE9V?+yltXbhTXVQ<L@FP*I0d94{)9p{r6}+-{ovL<S=y{vwUsok31zu
z^$<8kg7^-RHg{G~|Dd(9j2L1mGu0qsiL)!7Zol~Yf)cKUrJmEXa?2!Z$QZJr#9bB|
z1ghDlUQ?LK3x;?pQc!09c*b*AH~H_Ck32q-1tquVw~o$?f(TxnDGB>C$@(5(e@D3r
zUQW>V^*qbJ%3e-#=TCA}Nhpd1o93w6UokdMLhH10e%#q=i37P&p47Ql2jceGAyxHJ
zxkL`hb5|z^)7K#<G+n^?hT93#xBdp$*EH8k{@!cPnP&UvzpENx&Gc6W4=)Woh^GuR
z{n*hh28NlYqC+eBcFLsY0(7r=%d3YG%h-G1LOv32S<Il%rM&7wk~Po4LBQeO-e4+>
zfl^o9z(WO;pXc2&yyU5Y$f{+(=nz?#q%`7&>wL!7<0%*-%Pyt72;d%S#o#hEm3mhO
z8I#6G2`W2W#`#F;k_-EQtZ^joT52~;o|7=Ixf4pW+jMEr6j^8uu;h@e*GmN{ia1Jy
zLj6{8C@Fu-BZ=!y|LGDgds{V2H6j1B(b*>S`Fx%^x_`O2?!31hPTvR!Z0hZow&b$@
z@N#kY@bmY0>n{6K=Du%8UsreOvff#4Kck7|xRBnytFzy}2RB4MZS~>t^%a<>pJ&+7
z+FN__9%JrK##tgi9rE2=^zsDmFB|yQJ!4(}%Vigh+i#d>+|Mt-H!Lu0ZR>#>eveUS
ze)#PjXE-7F9XWk63-s7n2YO#WW6k}`_#?lqced<VzH3?4DK01=1eo;&K>kv*t+qWE
zVU4#GdsVaxc&_?9G^{eP{fO}SK*|Ojk9>v(a4%GU?2jA>Q+!RtWH3Dy90vXzSCzhU
zEWs^SH2l_VgUQ4PFz(OL@8FJ3$A|`cNC^@VOcKycK9Z8qp^k?AKi@1G1#uC&14pkm
z6czn}wq$KscXvgRgWs=(mJGG|0%j<_0*OwiLO*{c`jfTNdlk@TvI<j#7W1YWiWU+^
zSqdroMDz(m7qQ0_%jQ+Uxi3`QX7wJ6C5n0Q<q^0g*_4s%V4rjeaWBC|P8%U$8(1ip
zUL*V`bvdyN_#_*bE(h6ETROp&%6EXKb!H`~vu)xK`0~%YXc;g?E}5EDZ77u7S|$6L
z?zJhaz4@!^01mt-C4%bzQsJ0ko8@!0T;brCa(cg-Q1T%|0(-H3jUfby%I!@0m*Wey
zxQg;^lw|sUq%N|>F7bvF6AMB9C3IG1q5svvka6rLF`4Ac2sLMx){$%XoBz|nOJQ+}
z52gWrtJX7@wrS%8r3dJz;vy!}<=W&%Dxi!r<i*Z%BGD94nAB$MS?8k6nBsT3p=b26
zV6o-gsNKSw$EXKp!Kf~pJESt&nOqfe%RW%ZvWb}!W8OCWvQA4_q@jylqqgLX1f-I_
z+t2@{zGs(U{y-V8sQv{%A66*m#!0f+ZhgrANnh~)O5dmSexaUmPkY6@T1;&~l9^Ni
zQ(@%qM5k6?AJ(s%leA4$qwN(WKC&{jq01QMNeefoKU7*W#BOei97uakf$UcV-si#-
zxpAB)6cy48v8swxND7(xsR|r4EDzzh<nlM5&8gDALRPbu7yoB?_5NdcGZcib-qzDp
zQA<X=FkmnoD<KBw0I@%ya7BX9Qe9kX+6fPYV!sg@N@Ep&fN2K+nm)b?Kv0eKnyfcr
z5k4nZN*mE9@VF?3x*pGWE+T3t<5$~HY~r}%Imgt;JxheT(B0i+62Lt}E-x1gGDDXr
zb+2EXt;ho(iTwI9I0?zCqzaL<NQouE6f}*YHk+A8*?9S2t=Y~Zy6=*(l(%#GbHJ?9
z<*8=wf_ndTr|Z`(UJJ~2N*e2(UV?@xynjw&E{8nk$(~$Ax+-^s5$xA<S?>_vx;4hP
z(<RG$!-9Hx*L-D_^pv@rmCS$}Z6RN(Hl?t3>gOQEq{h38Hzi>wc|*nwnMXynS%6RO
zD8cYuvYw}iUGb}xc$<_7))bc^CdhyXUm~FLEONtbf!y}YhqY3>2ti{&-Wo2lNv58F
z&VW4Q-^iBqlH9(zm$H57^x|M?p!BJdRHH6{?$}LjL0?ddD9;^xeHr5}QY9rfI(8EL
zGpx*Pz<*wc6XuyZRZ3kS;(&hp*XPp`S9oq*><=kf;b05I`64Ws{A8-L2{tH3wkK_i
zo2<h@2$l?IjxhM3rZXu`^OS0E$*|qYJKLHtaH|=C4`FJ!_osgyR7@p{uVl)RuADJO
zA!y@}v88ymafE4(aU%qCC6f4PGLBz+NnsOKW6r;jsuL2%!c$huO3#75pxR^pZOp}L
z7fO>-RunuFgc{;8ksTI78=oFl`zF`CaXPFgwSxy#jvifcBTxr65^+M)KBe6`2dtW*
z()?R5NyrS>xQx5;DbE^Ru8slpE{v?`7q&T4^pz|W`QW~)o;e$TE3xL$(&1oG&8}!f
zG+Tv$*e7V`iRKX-&8D(&8-Vf9zD=UkY#NpNGysIqOo~(1=$i$)@A2^uF#JmP%pDBq
zezceY-7~<qbm>n}doUeCZ5oH{uhPT)FS50tNxotX&2`h+OL8^LnN^pq`hUFE@|)LM
ze)HNqAJ*^utF2$rBF*o9*{S%CW}-az6=4pm_U>pzyA7bIg?5ScB3+#WITbYJrg>@2
zXNfP@hGI-R@(k0px<fFQNw!7GmNcA6%VROW0mmH?>t+I8(G^iKkOMVA-J}g#B{JG<
zPKwG_V>qMeRjTSIZMn6!5k_9S+5+OQtQ;-dxSe8st;g{>iAPs|lLpn9mRj$2%XuFU
zI|?%wj1p3QmSK^wi7Vje%dT$C``K;@dCgQYatu8WOKYH|x{rlYgc{Evc-HD<PCJ1{
zlB0bW1E_1fgf}}&gumGXJsk>b5NsSKjABv7wDq!5+5B?rtcEl7x0e-Cb}4P*DBPjk
zCMM*tu$9<V)ad<Kp6Re*`b|#WcuKnTPOcgtnsOKOR#r?AxD+GvDREligS88~_Uu;3
ziEU8_3*TzH;7b3z`V$R4wQ=+)R+?_>HtX|%5+E@$6im3+!;|F7;go)T68kIJGDrbU
z%`8+SxSSVJyO;o!6f68zE|yh3a#l~LX{FKYsK#bWrh>+u?Ly#E$D;b2J-)a&9^~I>
zww~01MD8WWVt_vOGRIYqET%zM#;`}KY;%818n_gm=-oOSU8Wc`-*}yNk#T&@I<$G{
zUCF^<Wt9ZlE=-k*2ACT+c5ew|t$kt*IjqnK{u#b2fxxeCDtRNbHW9rf0`DhZR63KD
zxb#(Y-T`KdyhrHbP5y)gdal4o221dy0FF-Y;l@i=p_@~f(%PBWcymH7?3pQJ8(r-R
zE>Ep)VN9{paf%}$x^k~^CS-w;1ogvosu#1Dw=OEtwUHCtsbmU5w$NNfZEFUcqd+e&
z9pg&Dwb;~+*oL+8Nml&{&hp#7;+c1^WZq*FAc*0(PN4_P_!4J%6mC2)&C*!ibiq4S
z@N>9eypEsCr^)jxi6GBkOE{oeWjtE&v!@^ONcsEz!<2PZB{fB3o#;rVkyRLVinN<I
z=s~V&0(hjRiG;aAAY4}mWyq+h`cn&onX@@EyV<(i(8MZppX2DNG8y?0sQ>$9kHge4
zUj-9t6E6e6mQ1JBf-p?w#BBkGDnE&GV!<i$*4pW@XsRC7rwk;Ff`8FH64H%g`WvPP
zb_uKQ6{?%A>TKD$mRnTlYqm^QY($&Z%N;$o#HNel<ox9+aUkjO1Cu|Ky-hLr(TE${
z*^ENierg}tY7X#VzAuzx<0U}mi%ydg&4Y0-zu}de!BnD!j4HRl%G=th8FSl16)+C5
zf}u8HQ?@#bf6<(8)#;czzLswxU&Lgk#nNHo+=0g!){nZ@(QaMIPE_a8<eV}2b6(Z^
z$)OU>r<WJkARNQ7jCR2Ds$^atnK;3_d!*1;`)bTwIyl7*v*E2H`Dk-zqtwKQR{%wK
zpe<sC%2VwZYn3ctG1|tWvGOxSO&PKlm0PFr2X>u5^$6jScqp@Lp3avfh+cVX^o1>D
z1EeEz2_8b34GKPxT_U!Giz>qcKjC&1GO)xo@wV`27`GY;e~GalP019qbaDyoY4spU
z<wh=|mU1UO4BtIC%=;~`Lfxf~(wK*-5S03-amOgB(V(KW*r<pU>M39X&a+2CSu48(
z-~D|tK(KVD6<)NcMMRVGx827UEnS`pn5!5oQxML064$EGUansENO8vmK^&=O4ZZz-
zK+(c{$f_qTRyJ9oN5F@|j8u?jxmOAfT=WrP{JmuQZ$8kST1PXQ89$k);NCVD-haJy
z_n^zK-9<%mR?$=^?DEE8yuR5I@I92Cw7=F|jT{xmX9_>vJ}_ZfW28;y?GNHS%h!1D
zU)hvHJMsI}LFlNvw<YpI{-;x$x>nncbJtTtnX*%S>1N=1$mJpUOI=;)CVM0<Kn@HF
z-A*F3a}TLQD0=8?-g%`_<oy>)vp0Gd|3gyAJZsPY{$G;1e%N#hJ$+MnDR1088Z}Ak
z9>0?PMBx7aF=|88e=+J!>o-RA{0~NzN@upN2PFiuI{=a;D?NW>yQm;Yo6h-oc|!vf
zSgYqjk)^@<rQ&AybC(F*6Bfq5>+9}*9HE>FeoLlriCVh|G6`iYwi^nHi?GP{QlK-*
zok9-<I#x}{-1wJ%JPOvL+C$U<qFp3^4eY>wo{O%ll;bn|NY^zkbN)mJ@4p81?%SZ!
zpbe^h8&t|~gPO@+@sq1|0GR`Ba*T)i&Nd`Xatwmk6zSDP_++cef$LPv5!sCmG-v&4
z4PPYGx;C9~*Bx=GoMOZuSwc{+&)phr{um#on+JDqPBC6MTN%o0uI>VM)iqmFtI&wl
z37a-FiH+FeN3B}PwZlau*mO2`gv{QV+TU6-aS@Sns*5_F{iVxe2Zc~?ffHEJa&D^m
z*2*<uH2>G)A%&(Ec82%(6tIdH2Zd~U%5uw-!YMD27Tw`n?zCibW?gP)#f1FJB+#kO
z0Fia;g_RqUHUapUCVsz-80QiSt-0}=Adfg@RbE#L1I=?aQJuWQd0g);461Z3+8iK^
zMzjs4hPi#2E~ychf0C^}Pj}qgCrv*|{@d|E>Qn4oEtNy4!RyODa&md+I3IOxrjzj^
zTZ$plHy8zNy?~gkt8F1yZ_WwT3RaU<YSHs@<kTmFkq4aynP!x+_m1U~k+9RQKj>un
z0`cPYj)*g>UE~0m&KEZ}eTLVL@&0u>m;j{+zQCpPkVQSzu>seFWtr^Er@br#s!Jol
zsp+w;`tUd{6>&l|<n?oO%+{wMZf@3Y%sSc%{ZEBXSp1FNbPcQ7n4l8px_(SUh<M8|
zw+2PpHnQeZHrAqxkI4qTEi4dPUQ(IQ6}umDmA^Ouwg8+_zk0GoLqx~CyYs0WtS%sb
zqU%$%?Jl-ziY^&sZ%AtWKM!iw+|wCRje+gc@uar&w+i*J7CZ(D(jVMt0cG8((l?+i
zLc}^m8{JyP-j1C!tkNZxfa}redt?&`c569?7hRZdd>@iHmS`VZE5?Snzb)vpj-MI@
zW-94d7s*oy`;4asVLzE5HBAJDih0CZ^Q-c3lY9S^;@;YQ<HN+JuqIxoC?6<2JHlCR
zMq}cm=T%c*M9ShwRH;EgyJ6>Fzc(-#eJss>B6KR`ol={D&g|{mPORCbg#j?ITWzUY
z(Mb&$5TXYQ97Jg%pIg{JRW;e618ql)M_nuJ$Mn%Kf`az@+2S;A0T)+t_HziXp=D(A
z7rK@UY3Nx@w>u?rJY%7=*)y%Y<cjD`eD+~8*Kj@+a_R!A6dbC`4i&od9a_J)2#SEz
zUk~$&>X~UgQ+WzsD<m{wjrzM?f93_jq+hq58S?WhPS3r40+<$nh~IxM0||-Ket?L8
Gg8Uy=k@dC!

diff --git a/charts/postgres-operator/postgres-operator-1.8.0.tgz b/charts/postgres-operator/postgres-operator-1.8.0.tgz
new file mode 100644
index 0000000000000000000000000000000000000000..caecd2929b0f255635850a65f351e080c215c321
GIT binary patch
literal 16609
zcmV)UK(N0biwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc
zVQyr3R8em|NM&qo0PMYcciT9!I5>anQ(%?(ok_YwSr1FLwc9&C$4O^Bo!Gv%({tu_
z_ws@yB%w_bY=D%bJodBi!HWc+q9xf*&zOBqS|)+20#GPa6$*ujuy{@pvq*#}j0xR4
zSwI^1Rxt4YvZJTp@AoJB`{Muoe!uwt{$z0Qm%;vUIOz|@`v=3n^auNsgM+_-{;mjU
zdJ+~x`j`IJxQdPYlRPlOKO@R838!E==)fpS{~YuVd;N}wm`iaKi&qzlTfn*E7T{6@
zW5fUqJrKYU&Jok4uKw#CaQ%eEh%!KvFvQ^;;IIS0Pl<zQh+@P#3lg9ySj2I}rh9vH
z94`{5=aOLW6ZB!|k-ZXwc5cw>mQasPI~LGiI~GX4Tfp^6;#nb#@e=VuFs3BL9Sg7s
z`=k><9L5lbhzb}efY_h%`XM6;b<zJ;xO*Pz01!e>;eX3l9r;(8C|Og$yHjJ9`QCo;
z@h{K+CG-=-b}0aw=l}k2vOg-!|M6gd_%#0?;=uu&qiF{KisqQbbTv(X^|Be!6Aaw*
zh$t8lhGW8uLKk}#ObtK`=Uko$o(3RFeE)*@*j-IQ_jGnf;tPrx3gd341HdzIF2oD;
z{S{yla`9{hU<e?ka0SQ=2pTUE-@igGMKSA%O6m^~BrFCF;?gG(Af!n001iRCKw|!~
z2)amCqiB}No10V+^)T7<h|Bg|61pgg#SbMU_5=ZJ@0lVFhjUB3;_YIR42y&ov?AgO
zBt?p+;;p4*NwB(9UHlVT$r5KUK%GuRyyGw=G34FBrr`5e{^}ickT3u0xFnq6`8A|-
z6i-1{i{TUTI-#QC$>r;Ar}GT_<Ldkj_+&oka(6~(0OJk{q2r@}FcQ9#AEsaieTI1X
zISLVlK5$K7Ac7PIh|6$={H-h|o5VZVLy=Eb0SaTl79{aKH3EA{<iQw;GAEa>SqA`#
z-gc>HD=vd_VToyi1VBChE{xHfP>dKzLJv`JyTI-OxYT1Bi-UlVS&Vp>bBI|LPMZa~
z(D%Cns3L@f{z_28Vi==Y;xo=aw>VyqBnCJLkcVN6{1xyB?=xPM65r>)C5WsEKK!(@
z#d2aN(VRjL>7Ep=K(VNO@%HC8msh9fX8?suu!I!gki`gkoUJ2Dme|vX)ItE37^ch1
zAAZllq3vN_4_sZGzB{*1&d+{4ef#0^`1<txETboyD*!eMm-Z4;vrl74V*tYy2+=L@
zU<@6|5QvDEHr~epR%{1I2-qS30SX9RWf<5Zu$RP7I0vVo<u?W#=-%8yJ~%E3A{b-G
zM||B~00+gl2!-H^_Z1ing{GkX7=15iioqJPOK+iX^V+tAW_B%#U5~S9h+M&+ET)RT
z!G&Dsd*DqF#Ve37#K2F7Oeq53^~n_%0UUt6AdJtSu974^a$|sJeB6>qRz#r}5gf)0
zJonHHCVu?lCEyB=m;?cg7>J2dd4P`~=h5e{Pp*ErIb<&c*i5+C2}5>P(QJ<F36Hop
zh|13B0QILWWpkrE-V+j7bec8TpALHiD;f*H0S-Z!1P-D?goZ4Jp^F%2q*RMj+F1`+
z1MmX~iDZKg;FayP$`n`w2@-zX0wnxc0Wv}q6E9^FmxK(ti5M=5B8VxV=#K=kn8}p~
zVrt7eQ!ozJfncl9%>&?XQUIgAgy6yuL=-K#1alGPQXjaGGXa^YX#mI)QI`Y(j=7-I
zliZeHZH2>*i7ncjN>$ly5eLMuhg?YIdc=9h9CRij$|ANo{%Y+2=pzgwA5B45{gT|F
z{)pjRt)<xjQIf<s<hT|n)s6v+;B6@IN0hjTv9yDD*SQI$D2#nR?{5jc;VLpX<f}ur
z4ghHqIDFcr<DRf=h4{dPW-V4*;>*T52+S!A`MS)Tu4NFp<qNDR3Ox@?HNX~cZBbXR
zoGvAy4ZQ$|9YgSd2r|Zv0YKHNDFgwGB1sU47cd5pA`p_e0;?9D3W}|vm@_nmjKdhw
zCG-Ufu^#Xqa4GUo7-Q&*iSW}Q14<>}V~|q9wV)yvTSJ_JqrOs3Vh+nh787uTP?V8T
z4~~DaQxH=yIFwK$bC5Xu@gJUO21SvNT?tsvpmdhXDW|{Ny7rVWD52-9^1xXVA<D}L
z4p(EYbuniJaEN2i=>S>KfR@^DihbV0-w9P#=m)j73RGP(n{}sLsHi3PCFn}6uRG<E
z7|#%kK}_*{&c_~#1>z;>&Jtggz#$HxZ>+S$mn~1L#DoY@^*f=qUQZP+J0N_G7c?lP
zLcHe+gzrn;f;SjaijfESqRKlsHGs)XX)7Q`Fks+zfkMDx0?Z7k)wp>T8~Q%EwK)(y
zDY)?OXlXCnmd-~3CvB>j27*pL4`33L0LIuA>av4$Q#>j8IUqYn#}ssrdJ{=tEOgI0
zM50v^oR}QC^*NNnKHyMFSO)uD%>|i{K(>~*Tg<3bOaK=~(7`^Axt7Q~JLSbxcP~no
zh;ThItEPLhd-bm?`|R}Ojgq2j$p^2`u7F1Zz9NvBnB1yKyrt9-t2lLWaBWnUv1QF^
zx+wa@g?@n~;0yI0K*sPql;V*KY>kIFzav+yUK*;bz$ph@zUcWmRo)HJS-eONWY(P;
zqB%Fs6~e&8=ch!Mw!z}>icNr-(ZX>-PoP9gL{~X=Q?9lNi%B3x!l<WKIdFqka0-ME
z0%orq^~#Z>bAwh=$OvJd0+DPkR7S~}N+O7=Le&LoeFI9F=L7Ad7#Wn<{F8lyxTz{X
zTKr9SYs_jq)BY=C7Tul0tSa#jihy()7+oGw62Z9`%NZp>iYgO{#58$?2A&yhk{EIu
zG8g)YmIuW$3R^K&<Xc9az3(J(LPCjECS3#h#EBQG-NFnPi?TG$i&~>;Ari*yw2m#o
z<?H}udGKZBBsJA}Q)v<^+0TH3W`rW8Wn(5yAtC4#B3|9jOnt+&iOGD<5pXjJhA{?k
zHshsNfE1{Yh-WGjg0W+)i<?Aim=~{Ki9tbPY~JJWcnL9|XrRw+j6oinhZ#+Ti^O&k
zPgwpX159g|Zn-Ob63+Q&-3yX>#<^I|No{&b5V{ZKh~{Uug8-g~gd$H$ddcxz(wipr
zOTZ9aVm{O;cBR`zRjcBzn?u%H4tj2gdv55?dYJ5mDCXLy#i1rlDYlv!JUIi&OtwRC
zhHkfXG0!MuF-_b!p-8EqQtRfNuk|LWb><68ol|j}omWr(@U47dOBAk~mRwXQ{gc}l
zyywCNqBxzUrjXY&Rgt91Rdp6uT|LWL<<UxczXOcf+9LFeY;{p@P`JdDghFmej>ZAf
zx(-WRZcxk;9|U-={72k`f&wL<CX$dOWbOiai7#~cka$SxK4R73EFX|1vK8zp81{os
z>Q~Xt)yjRkwZI3L&-<kWsog{oVT_wON+Bg7=X1`%6hTkwu)Mg<|K#QpQ!AcYeIyG`
zRcWmp2)k-QZbj8&Ef_@OQmLD{sRBgjNIF|8an%xGi<(o=J-_VUaTf5m=f_FB7=oDG
zpio$RQ1%1=CbiJxi&FuXz$9MSqF^oerwT%@6C|u4@?k7jMK0T(Ul0$xPQ?~@CB)VX
zNj<k+JV&l}(k(d!-PDQvS#juBEAkwva&p_7asd{Ti1=i_63ehu_zJR_!!F2QTh~%c
zYu@L(AS{;4YeVmzD7z$*LugC_u!Iy#3C#PKM2RmwjY|8`N)IQ>sFpc{#9A+O&Anh{
zN!`ejUo(<49nx0}p5rj5P6&!$?2I<O)^;<3<SWI3+C|GL;sYRkU;<+#oczMuq1FSy
zlqFnT>zNv=thOELEdoNt3>C9s)veKJOG_|#&Lx0ediGKW-V23^i)*dTN{<<TqvVRH
z1fc7ZgsX>0@D68P$b=((?_bC7-)U!~6eYrlc2<Vfp~gh%1F~loXj_4{Gtk2Sn0iTo
zgF|66-4VA}B)%52XurZzNk+amA>|XB1~`<OC7~`KP=5vBjJGmgV5Y^t>fyAwJ*Ql2
zD|qU1Vj{H*a_XYogQ&1dUJ3}p=1RR7O4_mfHjh?tq-Zg=e{?;$<jEN&kYk~ydU!TN
z!Z^r42~AZGE9Wkuml_2W-BMSgNU2hu@Z$eZqb~bDVCKWQCN{U{!ZjvC2-~{frl9*-
zH{xqA6_h>01=af;NAw1V-dFK!PDv7d?WSv|XfAJu_#nnXtR6x!OjSqloT5JxOyxY|
zI;7MbAp;=^ElNnNorf=!M8H0=X%a%lF-*arKPY|>82Oix_#z27eJ!QdE!RFQDAWgz
zQ|>dc4GEc<$i(~G1rkPx5Tb>~s%4r($03L)USc2d<vNghiG-QGH}(0Ja77ypdpY9s
zgfY3>q5wd#)rP`YZ&Rv<J1gP)RQq2cLLL*_6xd5?la8U3LK|_bthBKf6AI@DP{i;j
z1jN=Gny+30C`}Ka&o!y&LyD#VIqW5fmhMYHW;2dT*0*JS<qeCLt`;{SCO%R+pDCuZ
zZ2~e=3r~gF_spnnxB{lYb1V}2EA>iCgDH6P5i>RK06?=Da^opDBUe(dRMt!Cow<dv
zyC^z|v>!xG!8u-{Fq_&PplW&9)SZIvcU`S_OO*=xzG2#SI?vLnEV^3k)?^-bJ>l}E
zqpYFdT@ddQrQswT+sVZTxs8$mD0{kiUnot&cB4e^Lv#n-6ow*0c_rF|QuX+JQadJU
zQ{+xWsjU)4*SOz*k4;#c5SQP55%ejb3rIcDX5r6F0$>V~h;qj8SJFJLV#ff7W+x8E
zs@(#tB6xGi`JNb#InkZelwyD)AF7QKMgqghsJt3F3P6>gDjQC17*uV9!I-ZhQV~m`
zxlnFx3x<?ir^+5s0aG@Nu<b68dt>ZW3|P+Y6K;T8h-1A8YZ!qPv4|5P?V@!kQfqI#
z218l#lRA;~nR!YV5r1J4+PrE8e4s|M7CIMYLLU>$y;}OE!Koli%d8q~yHUvF(*_ne
zxqO}9HW3>z&|67l*%*vi$@Z2?gxCr+W(|iZU`Y$!0G3id>a#rMF%k4Nw;(1!jG7#B
z&ZIvbvQ*%yy;kwN#6Y7K!yAe4AXgYZFx`_iQGn;DH}8R~n7!l{=ZiAN+}PygvV#_I
ziHX=G^$GNV1AV?kr5l*<2=rd5cXD>UKDz>+NQlfm9r02qqM{8sNL;5(5qf4VF1{QQ
ze}FujgXC*mPi^D&ToP3I-Cz{?E9o(mx_%L(@Ov?>ewu#v)C#EdwSQEO^-opq_nga8
zV~$r5DzsIYS;{uyqnG)6wBcIv?<!Xhhhu9vRgTh4uyRGB4RTp>r!=VT(njp6%st^<
z+jxzBUF)TP2cVB)|G@__g|iuUwTZ81YK|_BMU}Tz`OK(4k(6{x-lKFHX$#jh6OED0
zcY(2_%Hi}vR`4lh7SiV!!QgA}vz&H()_vW84&BRjgy^nqpv2;3s>$g*OBHZF<FN-k
z<RmhBPYq_a+ZZ$C&y=)!&5KQQtg}SaAq+_fz$bH-d4P?5KJnT1btGVE>m$WCOF~!L
zE;wFEzpGMvKxVY%^fhhXXN;%c*s_M+WynRAK%vO<7VEf0z6xKyxw__RgHpQ@O~te*
zbBS1d9AVi5rLyXWq7ZY9fyNyGxZ=ToPQhVc#YtqLGboHj=%KdGmB3$MZUwD?U=Jsm
z&5*fUs=nH)vQS5hfPKMj3akM}I&{Q79ibUQK60F~a43e!7S#pd{E8-4SE(IpFLP*B
zZ21Z%IR57qAQZekxlsLhe9O-1TQ`zDc%0d(+TRFo8Lu&SBQxCPg$RHX{xa1d?NKx~
z+?5|g6q|xBR~3{u(K2>5FJQdLpm9*pA<NX^sXaQ_M{LDn6r|EkTLsUR1E2=82qv6@
z?jFa#=V7Yw@~)t0uEJ`J0eExuN;@GquiP?Q7MlW;SS}x7=v#waC{h~YU~Ybd0)^=I
zc{Jqk=h4W3tmX!Cm_3h1c3QJ6i5UpsM`3Wt+CZ4`I0SEBfh3HvuRH{t41<CNj3N{=
zgU~WEJAfZ;4o<PGDE#*1A{X0~g^(ET$DB~w%N(C`H?ob&nTwJ4#_92UsbTZpPK0Ko
z1{lMj2Lt$tgz%P$gA+;<PdC*}M{<hoCUKCBVIVx58Lq1la1!@M?y)9J%5k>9QN-t+
zA*ryx(lhND|6#L{Eo-&~_Rr&YfT4MSLKRTV-&k*?kBIYP8Q|NKt8F08Sqb9tCoT%E
zl%MBVxw6{>f13h_pKR^!QO;wD#S}Rr7KV}#{7xLT1yBwpN#!(lq5lfHH;EE5B6v~T
zTTD-9UltV6#7QI|KUbPDx2PG2=C+pv5%hgN-Bod07Ps?Jv)%P!v#P!&>rdrz1*sQ^
z&}A|Mb3*1mk_U=Py9B>85(=TD8ONkX*0hZ+)an@#Q8dFJQys%-y7E;Q;Qme=Th^I^
zZoUOpH|Ob-@#TmD3m`1Kj9oVBazWmmyR2IVtt(7LLCyBZa)JK|t-$H)awRrW3%Y%S
zRyOvssA_Y@NU5qCM-fYW;k+xuN|`!XS@O4)P(OvJgf*Ed*p)D}sLlw{dqE(fzO<b}
z5`%>C$#DG7t4b)kjtNJtM_HFt)>JD`)-5Io&P0fccnyF;m#!jZoL9hMoR5#+Tn)z)
zqX%?|C^vBBypBaghO8*vaeCHthu{{W8#(F+z=Nx*ffs$IzzVxLC0LD^+OALcR>Z_t
zM@q`jRN*P;j{2bA2jB7kg!oWvBubqUd?Xi1099D46h`URw4QO=;l<ylnx2e_^a30T
zxi8nI1zaMfTg?~JPa(pNRTPLz0^w#5{;0R<R=u$jX8^hOE}~S;prA^RvR;umec<F$
z>hHn@ZaV#CtHe1;#MMFRM9DTo(|{idOvZe=)XSTOLLBjm!b4=j&*E<-CfUdWX$^Sn
z6aXt*qoPT<uu+PHtt9)E%mg<m+-U=4pjo&A@6O+z8i!Q`N?FMEP(+biQhGA9a{P?e
zkvtL|#+Vv7*G6Q6i7*9GWo##mi%hi+r4TA&BMpemh7ryk`%%ZAq*e+b_;C5oT!3K`
zFZe7|0oTvdolM?QBKYEkT73dLw<DOHSi|9rnBJm25xFtk6SeurzbmmLUi<(OECOBp
z;D<eK&XN_<Axjx6^N9mJPwZtFU7H2(3%g0~*ohz;UY4)L3kM2|M)-53`lZ4~)PYy2
zQoU1AN;>eFtEqbQLd4bX1u_O$%w`+{1l>iI3q9rBw2B-J84+t$e9x1inx*2T_(I?y
z5o2;oaU7$N4>}CZ;HQ_eTe#s}jxFO9Q4%9JMqbthDK+xUg8Xe9Vi3|inenAxMa<$^
zHgCk{Q9zcOt)=9<I0a>C!<S@%HFbD10d@+yw(^K~vmgT<nlC5Sou*Pj-cCW*5ObGi
zhUcb5Z#pl<fEF1ng&bkodwIqeJtuVX7{($@WXkzepyjixG-LIWp_d3TN$rICat=-R
z_Ga(~S>lrgBivJ)Nj<i7d-*`^oW5%uIa6LX<J@CiD7CD9l2*MCHRbx{IbvB2V7g1{
z`@koJueaLY!VP=U0S0pIx06*x;1vJ6t3iqxB=u@Rq?A6DAfzbF1IV94E7xhSTt{<<
zB*LjE{d3lEs82Y2<b-ChnerV7dX)%W5Z|jLfSanGV;i-nVA9uivWVH|CHX3^_U|iT
z$deo~g&~tplWxIHqlht2!j!?t;x?I?=h}ujQp^pJGJwiG*0J1UtuI}->DdfFw;j9|
zxGF6{cb@6&2Y+MI)5O({lh-GJ#gxzH8G>9Is>8)s;$^Bsbf^Pe$?;bra10}s#!Xzl
zIzGud$%vTDHo)XBLj@~CQJ&??Os;bQ3g>f*=ISK4a!+!jH60sKyo9m`o;REhr%1#I
zxoA%=vld@{L-FsYJ>Js2X9~WI4z^(EiC>lqyb^vYiRxGrXwsi{o04P7jIZlJ2(EJF
z+I8WzikHgb3Ue@|%#d*?r!!$H>0<**?Ziv~dkLU~gU!e-X;uk`UE=$4#Zj(wolZj>
zb-$`;;d;ti1>Tud-;F}i%B~d=uM6Ztnx+8BVpRE8;kl-mP#mv->q91OfmExCh}Y5I
zPx$9EsO+lhism^P#=GaDx-vCmLu3GkTqNBnNu>u+K?+%%g$_Y6$_|QURU`sKZXytq
zFh(E6z|bT-W*<~&mvCx}H)kZgBqTeIKoRuLLx06z{+K#z)6XJK=|YDoy)mMc(i{X?
zD7B966?T+}H-82$?~uD8(;@aG3_faBNIbOnOos-G*$KlKTWS1oc537ES5D9w@sN%;
z--sr@%3AXTduWd%x0pC=&dKWVzvbC;LN&hYGz@A(d^LT*4U$;2Hw(1d>tw&#6#Vj=
zIC~dU;`@l6zOI)!St)S(+7LO?Q|+bBEW>Q#JsER>;rrRVO+)2H4v$!#D3P|4jNs%9
zBX@zalU70m%t#U{bqljRo-S8EGG!#AwL1m>SN4~6`r0BhD_)>9u>JowWMyX56-83M
z@ed#??=}BRIscEw&-TvFuisqt;*ar`IM(I=8cv3V{6CY?!T2fv&qF-d1S}Di;ruWz
zC$<6$n4X$@$)*0Z(*a_~#f^PnS!wL!51&7S-X-!8WT<zR21tMX3g#%5UVh65T^L2Z
zywu_lc;$2j?v{w?iLFA=17E*(A3jCazxC5J|Lq0xBScv*j<!k#wr2kC4=4MD`F}7x
z*ngV;5Al5dy!Rbg;$SM4tr_-Fyo%5dd_jip0!_hpdtbkHc&W~tkCE_G7S}_OFS4V`
zSvqj2b{1P%IbnzOWWq@<GC=QVnS73yJlChbzVP)kbirgKe&Rs+&AL_w427QfEh#-w
z&hZq7Jh$>k0)4K^cp@za$I<Kj6G`Zze9Uq1_AuZe7bXm;*v2f9LLUCc$X6$1ga)n;
zBw+@gD;JOoCK5+uBjmp5bxvo1O7RHtr&mM=D&3&4#SR&Of_O!3i%!;hMAkQWmiDXu
znD(tMQ(K@_=ld*!^jFc4=@0G4*Pt`)6A2}^o@BJ<-rkwOh~hAwf$sle*8gJN0yK&D
zE*w|)yTFOXz2ny7nfC*n86%ta@N*oxe&VfRhu({{`|~ixXn<z&E6$m%Z9+u6R@9pV
zvh22RO_!VDS))oGg*NTVDf6wy!?cjL9|;rMdU_2>!6mUyx&m1)AV-!GHfy^U@$%i;
z`C(39`CH}Y0bfMKBiHh}-*-XR?rtEefApc)a|w~ijiWXYbReYIs?Sc}VwQ@<6C)8~
zxsfvoRd(`^IzQrj7<G03QW(w=`0EW?y##-iXHKR+2n|!XM3g+b!J&7`K$rR0MO|G`
zjybOjzJ8s8&!55X1c#veGA;21{E?6tnb3p`GC&5Lm<%9Lj}W2aSpW0~dAz^yQ4!7(
z=O;lsOPrrI`DJi<kUszs4d$sojhQ#)L#=>jFkdt_Rlq2;G;M9@l^S1$)zDkAe)Z4C
z)|vjDo+kZYxmAVxIw#fdm>YPF{y*G5DCGYh4kx3*lm7n@&*#sken=hY_LSf1>(`E%
z99(>YNxUEwf0D<PZw`e4zZ`V9P@9_jFr5I!(1S6YrtWi&skA!Q=?F>?7gAkScn$>I
zcCG^fWr6&l#}<2H(OX+tEsbgIG7T#7(Q;M-SgK7bRiwV6-ZRWsJ5wqEO+@a)XX&Xq
zdGJ@9&5GTHk#1ctnIa=8p;_Kity9P3N}gvhbdD^IZ?85>Dfj}=AD|o41c_DC0x-gF
zDM=!oU$<)<x>E15>K!0XA?Wvlr^HVglwkaHx&Gs-C^!quMO2?97C-SL;w=`x(j>f=
z?1qlHwzQ%z>zxN9Pp(<DBPq6fTG6g{w+yoHLd!Fb(noPhhmIfv3j{qMvGl-EJ8E>E
zjmnN}X1&au?iCN$7n$yvKKO4GsqQiKF-D;{QeWM3YEw%bN5Z)iif1}p$1<?wo0Tw>
z*G2wqRFRhJ|5n!lG_U`I{-j^<{}20vgQxue5Ar<n`rqj0cv1qC5&(T)N+Y?2t`8K8
z1HM_rK^AOO5{|ho(JJ|<&!Ds7-+L+rAAhYe<#v`IW+gSkRYj&<ilE1Gp^%%mT7&|w
zSy(j{)gvKOhs$UvR`j{%qM|Al6@A?dq4P48Gw4w43Je;-gR3rCyg8v9ioV(5<np!h
z(TE*Fc~3&cI7B#Got4z6P#pz3>z7%6s*0g$r`gtAHc?bD3Y}Q$lVVRW{)Ze7ed_h(
zvZV?qtJMzRo%K0o0;F2)o}s?0<A4+a=rfXf2$lNxpJp9MMG`@;aJ`8Lbxz%bEr+qL
z9h|st-3*BdIJx}rI?cG2i#?M;jcpA8o%(ET3pBBcvS(TPkdS!l5RFLLnWn##i+(#o
z1dNvQTQdP~w4e{5kC+L`b@B5F>wyz}FlmV)xKc?BuiB1l#I+?BqcE0-bX_0fqQI``
z7eQuwo2i_MOU=e;>82vBNsfW6BaG&otv6!2NQgvdc&iWw*p<f$xhTise6wC@8mt}k
z0&-Z|3u>VP&!pNQuRLJ*lREg9p2RLr5ow#tlx{K`V(0;n6s`l>o?l8PeI0kSSq)Tb
zx580*9GdMAt2GY!JQbm5#$odIB=4b*$gQ0CrhHK;phfOcEQ5E8r+JJB`|X_$HQiV0
zwTwm)VVsSbzGF)6$+5~5<xLdBD5`Kin3=H^jr!cko!U`ggRB+$6t<M3`7_iNR643Z
zifbArAqTPrNyR>%qu5OQUN%%jo@$wRv8AZbGVPT_h(M_}054v@Vm<IdPJWdFK?*U^
zh&|P;5>A<{rjqHiB<wN=d+Uxe)#v;MHA_#qnHfh>>5%E2=7!pxuaU9lG*vX2$!AM>
zT&$`Dvn6tBeFJ4GIW~=~k;Eq#;)6Ub{eNo6Hgf~6+5a01C*y+uZ!jJ``Tri|xhMbM
zE6FcUZopl*0f~<;(QGP{R=t&}jhjSn*&DlM=d(oSkD7KY1w6lyZ9lk1iru<6@^40D
z{#!kb@?X0-w!92_jr<?=4~pkM4#rRCKOW{$Y8LvVpkT?!I?&BW&J}KpSAs?tsh0Ll
zGS!HQY%d4?gO>u`xrHr;s?(wpMMOMXA2zkcVN*Npp7phJk$hEGN^Bb0Bk0#v8iF+|
zozmb7z@dl27z{UuCGIHQ38aeaDt7|cICQWJa5`J~PJo#Sl6L}7r=Z*sbw<9qoj?>R
zR(D04k+XC+BdgM4?F3jcS4joXCnfCP=xNgbGDqch3g8<3Z#?W5<A3}8!PEJ_2YF15
z4n`5ntohg34WXu9_aCc!-KR>+A|!I;s0`w4Dy{%!IyZ#O(g9<4@y_V|UG@JaU6Ec%
zr4oo_{2}1w?;0=;)U>oS^idMANz9T!nW8yWQ=VP{f9+i>T6(V_Ln4!Ow_v(<Uw|I?
z0@wl$#}k$W845E*U~8n6@lkCE$RC!;EB436O%T!v5|e+yz^^NluGfbZ=V>HPKgUG7
zW1~$Nsk+gU&1>7MK9`$XYOJ1E;&;g!*}`EZ7$v(lSiP|_8DXBAGkQoTP&4c=@v5gY
zs)4Ux_bNU{gkh0;{_EFi>2nO{A~iw|$wlJ(7wYnbQjIA6XcC9Q<+OTCWGvM`7zvdE
zYgW^(esgyG>fIarA6MsR_Pg`9Z%@zO8VAwDS+A+un5iDEO~r&J;N<-5$J4j=`{N7y
z?D+kgGV;61?m7{n5nh~MUBA72b7h}jytzESKEJdj2_G(xuTRg<?DJRucyn^SLpmz@
zs`4ml=EJY(qwCA_ckkX@+9I9v-6+eSer=*Iw}V!m#n~QkI*SK=vvw&9RT)O1NHwHF
zFQrrCO2t2=1&d#*FqUe!5J4cloVE&LdZ~C7q+Eiiipu8qN&selr;7T<=bClH6wB2<
zBK}n5IWzrBbrqkC$;W@1?7w<_=59FF`Tqx_;`xu^aQw9Y`ykJ{1@~O>z=_ENCzr2F
z+X02NYs*1b*^!kHp0*0^+gvTAQ7#MrCZ9Tx>G<ob72N0hvQhwlC)8GHDeOc#!|Q=G
zdd?NdtjW_F1F2^3V%xos(mKA&{SOYW_R9UcO@zA6t=c}=*E=~XLAtmVy-7=}znQiK
zG)`|LtpiiuflyWRchJ?=uikV}FyJk-3AN{LR1cuM$&eCYUSQEgLFLARg5t@0*y<`5
z_7HTprS}Nb_gl%opOMW?F+PNGGl;rf2L-5<54s*w{r<rI)BeDp@M+TjOj%}WNoom6
zdM7P#UHso*Fe&JNqv7b_N&kC@r{#KTYI^E^9~{n&2tW@pm*PmKbN<^0hOz?}`g9S?
zvTezeXz4}liuL}cG{zhSPn!>g&4=}>OB>BNq6h_%l;hGzcp=}sBe#fZz@52I9O%E6
z>CJRjWaTOvNKSu}`G4l8N&YMKb<60#7V>{ID&>EjOrGTbLp+rceM(q8iQb3W_l;K(
zN+Y&UFNBRo<Mil>aZ9nhY>K$VpMvX?3-KlvzM2cOSbpnxlWn=}$M~^X`{`-%w2=SG
z*|GUDutxs(C#C#<{mJ-A|9_AtfBvVC7`XZ*P;r^ihd_3C*5}E$+bExRcoL>*{)?+?
zX3)jq+%nDr-p%-LvH!EbU$p;+<0t*^VV>Iaf0ehB6-B`oNx{s?TMs6^#LbF{B`%!I
zMUebQ&H9ZL(W%_Xk#}{fgktxS9_;?p*mNw*dPMF+p145qiTUo#e7xs&!+X_CW^hWz
zK8o`mnQ4|7-99nRCQQS7?x6%$O}eQ6gagNA{jbNF3(rGB(T;Gev;W3}LjH%rc+h{!
z|NbD4$V(j~8j92}a@b{(C?~<bhr>CDpnC)7i1j+p!1V$%z>-L~rvO_Z-v@J_IP!D?
z4(BfcMLvx25^-6)Fy6z^>pTM?7mecb`-q|${)jwqi{l0OKVI~}dFZbI2}Mm#5{M8L
z_sjP>z1LUvRZJ-AJOd{r2uKKiKDh!OrmWMO<9JW}E6MNloKJL5{HtFq=6n1f{U=+7
zdl?|P>n~Ggb-wGd+o<zh&w)3c?|N|%b-w%mI?uq*kYbWBaQgZU>+~W@en)QH>0uAS
zJz0*D-~WtaqCEdy>eYn)-P!O{PxJgA@An7&V*Z!W!RTrJKg5$i|5eV@YG_3wKdTM^
zdY$HHt?>KQa8Gg7ZsWfx@n`BYpUXrjMWCTh#ia9~RL39k?|ixcslLoRsSN-S`3Z%7
zU9;pl6CBPHAJUpavdn_e_$(vO0wj{9<ekhgMsq@O_8Izq2LR@h2u(ri`%-zOm_A*l
z1N)Z)i3QOm^3pq(0Z@q&%{PgYQ<!8at`><~h{bSto)}!{Ur=%7`w8)rAWTV<C{F`8
zM>^D8Y+*2wtV_Mh#RSM4gd%|6yvBP*D7cVW5c%^vBjE*%7gNy7M;&{_y+PE*DaQ0X
zgH+ZnaVDrGEuySI{nH^c)6vN*(}A-xuM}>i%ivk%A`?-3wx5!t301nZmF3K7L+Y8j
zX|Z_uTGoYkps+54F-PZn99pMol0%b9&IofvtH^whLmVUl2$R4;loOohNffJiF^%Fn
zq5~Y-I7fsg@3_Em%JeHh(=tgqvoHy-ae!D1gGkN*cY%WRxFU&AczkjC^XMw~T4Nw0
z`H^v~<&OkY<V}skGcqkw?nNe8bFa%{LG9^9h{hN(6RMD3#^06PA|DhcLB6QfAo%2C
zi2luy6r;Y4&xHu&GKxaS)^~(}Mm@VIBFi8lrEpdH7AC$A9p99J=7VDtR6uO#2>?ua
zR7Xr@MW8PWoPw@C^Vj1XD1}hgm%L~9L~=uGus<F4239og7J${HIIknfA;d>8tf#1=
zdkhpy=woj~1ct2eP(%?QN^cWFeE4Z)%bbsCfcb{4)pxvCV2S2BiXDYZdkN`1bSYZ|
zwoJc!AHYQq-9q2yW|-}gAcFC3dn$H4dqG$&)V1ti<mTFB^E;(GfFHA^r4Dj_sWTxP
zjNG*b;0NGg#(Uf=wg!MeE{aoN4eCiOeakI<-ynSyu~q2W5u%uQ8;|{{9~23qD1w-7
z3wT_D%TR0wC?IrY^R*kX*p7)UE^gi_V=?R@7gF0?8fpiyV(kM3y}NEDT+<nfj0asJ
zYgz-qM=P7uRz#7oPOK!n3CV3}$$PXbQQi`IgX$Ym$NwoyH*oY};gdS*bf=cp>nR{N
ziZ;0o5w*p(c~!~gK>_Or1~d&0ST}95^$s$2OSl@L5=k@>TQ@tvx*5Ws5++;NmrLk3
zp{yJ49Ll49D|~#sZHi(Pu4`y3tb@ZUv?`NJiclCY)kCyqgiKs~Rq&iDMqx>W=ngSy
zze{QHtUTdYjcB8}Q;9)b@6a?-4Q<Mn1~QwK7r{D=F@k|DE_t$jT&3iI7L#|InJGjW
z^z9sZTlc|HZ&K9<GK6%;xCZ<68s2@$+lX*8F@g^Eab5BI9XIU;@<;&j_MND}^?c$&
zziQ|-zG7CNGQQeXy1m|UC*o`B!zdMvQy}b=hE`?Hv|aV2CAONYPzndP;Gl{GR#j*p
zrbA^UgF%z+!5G;$Xthz7l);HU-WVLY4VeplMBDb-{Y<;mPeIf<Z{aLgC#&yGn^9PC
zh-15)4r*g`HL9Lg)&z&fILj+@zpdl=vg=Q5dH=~hx3S)e*$o=G72JBJ-w)5?a7H1E
zY2wBSMbb*h<pS7bLN)+>R1>5@SXMjL^N+r8F5R!FF2pt98&us6Swb5G^`)AX4f$lm
z)R$?aa4Y+>R{Gunj?u8KVR5e61spXEc@yQS(H#Ng^z8~@3~`;_bO$*3jF?S;>ctig
z7l`8c@x_uRW4%s!Pqq&f${$~N0<uK5N-nas3OFo#9f8WTW~X;gu2(l7M(?Yl$_@~%
z+Hh*s59R&4?6E-i`0RC6=cupemsKC@lGtoJP+7!g8u(F#KJpx?z0L>?s`fWFL~DIk
z#==)C@|;S{1s%+eNkn`yUu~>Nn{?zOVHSkNbw&%P1BFXWNf@9o&NfXpqM-<Fz4qJ)
zN`@5+Nb+XA`(a!{J;Zf6K37%^EF3`!eIHd?(K$cSI(T)Nr8ey+zMAdG$dwz$9AF*c
z#Q>1(rtPhfw?(;A5n)B7VsEP&vFK=Zm)%1&JVrFYA-C4&lv|^9ds}xzSKi|SsY)v4
zVY4=J2UMT6OZjUrm5Du<A`fxl=d<4DIHET=^uCH;A}H@`C3Rv$S~^tII}&ZgT<Gs+
z!=c7Dk$7nmLdKi2nY-TrBEt^1fHK=2M0UsIy_x<M*DK$T@J0@664^_);cJaUKPM%s
z0c4qPA79Utx4e3p)9TJ0U6;`BPOWOTdR33wZj{t*owsNz(4qZP@VEc>`LEvh!IxjX
z@1OtL`_=nAcscs|;_om1`nNSBI;a{?lRUM_|9AxUtE~)8ql!h?VJ|*Mzk1)l_<Ks~
z_b>jgUj6ds;`;6TpMJwF=pOg`?{N#lciO+ZA$?FsdVs@kfMti{YL4L4eXHCs99MK{
zmgKXV$c;ox1Y=4<Y{LlK;_i)B5drOXAS>6FGQwT?7sl22i1MV^w-CoWLpG>pD&eh1
zJ`%qAhd?OIB#CKf1Git<kp`w+wQ1C^!6X;aT{GmU4Yqq6uR^NbYTz5qVzqmkT3p}M
z4yq#pRgq_ZBH{Knk>~W#M;Jst>bazHN~ldN<;Gmo>0p=pW~t|tE4Vwxyk?tFg*-hD
z(uZyEN50-d-;ODq&9Hl~-Ys>&slg0-R~H&$GHnAG#WP;WcIr&c`~{z51cR@=&vFv-
zIaui;&|`UTgjgIy8*0pK;?;<^jZ;$2nt&wXW(}-?CBpr;?y<c9#c<DX^HdEN?<9)m
zyYQc(#Bxy>Q|Mcj=AD}C$YJ(88rcD!uQ%RzAzexc?ErqX8UEC`h_nkE?f0wVM(1v1
z8~wgf&)y^svM~&VyK^H5KAGEWWIKs_v!VO{7Wy{)WNSB)a=>m3YR<MLX7lD0ZwIKZ
zx{*N0gSMLwm9=Q--42&2?3QZSa?m^M^{b{|fy_A}b06789N*R~Tfp!;acr3%V(V_m
zH_~d+yoiz|_L|j)c6%|S7TDN`PY%SGeS=mu-fVNA40DUwQU-=XFCuuu>01W3#iCIL
zrWU`f4|fHZZ7e!V%~eMYgkiT6pfUshrZS@7FZ%N9cQ2mrNk7rA?EAv27oYdPu5}iT
z`k>zj-|_#7EU#`wsU~tmSG1GNZFTv{J)2Yd?gGVM_kqp00j3)8MxCVS7kU5Ww&#q<
z^C9;iovJn?#0d(QZofx$M|`$83$vT1uF($#$^ZopH=oq8<_Azq1=_Tg4}q<QLGOsD
zW*{RzZuQ9m@`zfkyo)%F*mQ3XhcjaOYW76D(r`}%Eb)(jSL=p&@dHS(2PlGm@WUPo
zBA=`fs_`I8<$5=f3_>gONR97Uad$f0ay%YBCNH8SMsAF}tmThD#hTM47Lb`vtG~IF
zgmzs4(W=yO&XU<Kv`-1J>hQG`{Dy$u8kYc7k0`=&;9AeLZ{)a;utj4Si#W2WN{Up~
zL?NT>gl_Mp*WyD!&B+fK;k~j?y?6~<HR{b=MiI8f1rXc%8`AqD@{c}ewZPm|U)d;(
ztMjYe5nYXYgLghx-E1F+yTMW@z+C9B=oElwO?gJP7!Zj-u-c}^h63^U`_h%ZHiNov
zHIhP+xVfK10r*z;-fJT<@^pWLff!Xm0|wzD{R7>pj7}n-K(B|Ap2ShLC0B93Wi{r_
zc0P;HZC1!5wqX+`w92EZC6wz7ft@3KC?{n!cSs_3^W-*_H5|4hFn}Lz&#|))+b=hh
z$`!PX`QCyh8Db2RN+nN~h$##i6y&Vi5nNNW+%B2i&nB~`{6wHtmp0^S$q_khMc6kU
zoY+k8V=uDfnnkvr>Th+~tc?O(+mvr`B`q~ZWoYe`Nvl}3+8AiLc0G0;;-}|vo|^pM
z#*GPEU;nu_|M$UUzj*)W;3@z2!#wwr|J&rA$4UB~Q6OGsnZC0RrS#tUTSLs8-UVpo
zMBdS#Hih>!f<dQtoy(<@<V68REFvMpj*nFC?sbWs`Ox)Z1cOsAhdG}ec^z_v=jy-S
z>7J{>l#Nx-EU2*>xyGxs)O_ykG?_K8)iRH@AVO-&=d;tw6xKhBq^~U-euY1Y1mN)k
z0d5I-U};t_VwN9!41X%5Nx%4DT}mydC1O<?dAx36tkX<4HB_X$*;ASo>$y>qBj~-*
zyf&suDsJ18CE?^K+ewD;H0!&*Zm3Q3P7+E{7z4HnT{WHg&}S^2Mv7N7P!#X!&QRf=
z2KvX#dCvPjeQ^(|&huA18J~8L;e1VwbG<GVVOOZ+bt!HvP%SOY5{CF(+i3YICnV%k
z@5RcN3MrcIm^*gU<gm?2f8-3R`6OapIp^1Wm(9O2=Zg8P%ka~b*JrIcn9{Z}1euNT
zWKBM?^&73NQa7#Hk8MG&+1bscHs*lZI9ITISmJEh>Ecj2@?--pVc3wi`N8fN042Y|
zmtO|f(Qm(cUk1N=zj|MWzx1tr{^xM~OWzv*_Ttyx!xq|8g@*j?=$F1V{H^TOaQ(^y
zH;vrTBo?fK;$?JJvyWcZ+HA9E%K*roM0oBj(vgkq8KEwc_X4nZ)f{<HAJ@Z1>Gc8|
z5fW>CX$M}nU;5V3Z{NSLp7#aUei{7I9}IuX{{Ce!{7sbMj5_%3#V^<KZ_oKJ`I0mD
z=(iWs=YqAxyWAgNCjA%Bf9)wOe}De`m+!5r`bPCS2j4Sry+DAXIr<pM%kdBsy1gfE
z48?&ks{$B<=a)a8fYE4l1g@m-5{!G37a(51Sg0L%h7ki$^y4QIA^;hHAt19ELveNs
zUrfM<>ysK<-pJ5~DHt3bO{~Gt8XR2@CezW;bbMg-52uxT_`Kp7iMg~}U<QO>)|1vG
zsG@38$NlQ04@eSR-mPE54w-lNV)vVSG9>?W<IS!BetiAvWZONcZ+J(kq`cnCmQX`d
z3(!WpSfA2S>}IUhf*KW=c9|mqaHFK@bBjSu@ol*KBOht|-VZ6;Y=TG?fScz}!?7b$
zHXn{HVi`7yNzf7Zk8pEJwy^1>S1es=J+MGjtg)IvAg=acAV^pY90WY%`j8?I;7}~+
zA^;uSpq2LCaZvp4Ua`eZOH2c~GCcdF<?2_h$VnKN^5E?*K)sVERbDlom1_d<UPO7k
zBhY)bCQZS5D6$#b!cx7nBQU$6>1q-DyusCBC|o|IWxM-pfz~EkZ)!~2Tj;*UMmITw
z{!9btoATkR_8eQIl-~aQS^&8w8uID)PaJyWcDEkSHXfZ{!FSJ}zbE0B>jZuIC-S~r
zFOn}mQvBr##$T?I@Wu4GG%LSwJM-_qdfFo8Kfg45dtu!Z++V?WrH%}VhmLEnA=tw1
ziYS^PN|E<E;eDw%1&8xfnYdrRe)AEz$+~k4TOCB%Dr>!^nt{SfFSIOji|~9=hoOxI
zo8f5E9a|~PZB=F<T^DaY@~O}2?b;&xCReq7)_9}y0N1uhKI^X+(2oV0ZV$*x74C<w
zf7*7aeAeXL&z{zXfP6H$_X1ob;&0WwyYPX0T5^L4yanBYdw-Ox)Dz=YKTnKX0s1i+
zS1uGyAhj12hw-Q?aypN$-WjL4Zu_;jfz8+M{t^#eAs@Lhq1T88#a(gvtmo1aoOMp6
z<}qG7u1^c?le~J8SL%6MXrCDOPr|q@HnN)6&2$5+xM`~L4e!)G<`%c-Y`FC9@c6z@
zNF*}yZn&`tKk`1aeK@yCXwM+_$#T<M8lt(qa2mHN?k6V&MKN8q$=cg+uY*sTZ|B?T
zXg12kyMS9CZL$j81tlR#*xF-}4WT9lC^pCEZg>Ty2~1qEwKg+h{jkOH2Dg?+#5-=v
zj(p$ST&-^}Y8=~k&h&_Q7Yfx0$8>gAL|KUIPNT?{r7&fMuCY_MyBK>-B*sy1+z^FP
zb8ol$I;c$Bs6SR7O4T1LPgCoUl?e>>$I|WIK&*8X1}I*Wd+Yw!rih{la$EI5wv%hy
zvre~D=RH5vCo5Yo&}2U9u+chvXt}TB1|FKAodkrm859#~)6nD~UJ`$u7Q7y!3<EuD
zzAUJ5fatax+HIx({ai)V6sfg?K(NMepjLLzGw>R@IDkF~VVqFtgBg(_=D=Bj;3Xgw
zc=aV3;Cdv^!Ac?!6A&*bLMcQK&vDF(K_&*22Kd(t%m8M<K`u-f5@9Ppr5nBCAebIg
zuz(B%*!M95P=v5Hz*TBKm&z{Yf~p5Z94vI>1p3N61dt*iZorU1G%oR)V45^iBu;7q
zl={iQElGS2I0!6~IRY^Bz=BXJ&Mm^QI>uDO4N1%w`W09pKbj>zCuIqJ><N7hAW8`Z
z0b&f!5&J)73@iy1<<(9~37CZbN>Hl;d{tP`94}F*tk2CxO{c=iAEot(%l2Fnx+sdp
zk1Xrkp1hdTa!C*+F|yKmYNeqSd(Q#{LmbX6GPCsAy{c!Mn71sB7)c%|hFgi)Dn1|%
z0-q2ieC9X^aQ?6Nt2_ho0?HUHmr!|N)x+UjM-|p}^1ArDmUI7KBmpiAH81LuijV;a
zA|K*Vn+3cv;_VE7M4lxM7&O4uwGzhL!Z}Vd5JNyFD@1Gx-n;}CFTwRo@b)Em{}SBv
zG&v1`q0b1=Gb!R-%f~#vC4j5YOX#C8=AiLQaDiWfYy1+t#V^5o{1W_xzfFcUPIXp2
zJB?M{oQ)TFzSxA1#R6Ex#m!-}-^ALuxwzQL^xVnF>-5|7&ep8D9qCcce}+Be!asW)
znYak`G6|2_hsr49CI?Jb_ign}tZOU(c(qB*7>1s++c2qZ4`I#OcJ)=8#PwQR0a%yg
zv<3hP(fO=g&syXDF}m%}b&W2&J6@x^O58!K^EyAmtX#J>Z`}M*I+d===rH-3&oL*v
zN?TsH2<32Z=&%RCfw8-|s=ddj&3Pr=z>R5`VZZiB>SK6p<dA$@4x2XHvKBx~6vo%9
zsNP&`=lDtCwDHlb-TcI!%($}?U=dvT1a1pa=8?FQ$`V)I=(C0AJ#$~(b*W1$ZnV(2
z9P*4y4)i15Fxz@W@e;=9C$!oagbTN*dbN2MWEY-4P;!rTGp-?yVeOPy7pu@>PBm@D
za5jDOM*^!7&8Oc#Vis3>dxQzNF==QvLv>r5J89V&k&%1#!jhNKM>84<U5L2zHAOSN
zyFxAr>%)MW7TkoP`*&8?To|)MbF41<U~Q6?hTw!Pb{#F?tRl!5^{e^V0<ZpcaeQ@k
z`R=Obdq~znI6GelL!{3_o>TMr<nqn&^_$nPYAIM#|LlAX1X-Vw8vfN3T%NysvxbHS
z7@CR(C^%doimNpt3!I+)KW{EiuWKo2u6=giT;C`2+B;L;oxeRjtHs+?^X$B-uG)l(
zt9IdKiOV+^?@mvSuTRhG@vnz+cD^1?dbdu^_p1*VZ!SMvy}7KVV;zLE^Ewzp?EDkQ
ziwnruEur3}at%GCC0HpxQp!%l3m>}Zn<H@9cmj4a?NsMYG@mt*cqG~|=LJ(y>G$TY
zZ67=SEk?e#h;(fohT1l|wKc$Y5v}(W)7(g(81#2Wmfd<dwL`V;hU14xhJNRLm8v}v
zl{;5NpKa!$Y|1OQrf$5TB$+RwyM~26T(0*u`M>22SX*WQZ_fYS-|vqG#r)s<lc)UO
z5Axh={%^@WkCOphC&En=TxU7J%}=#izy<B%OF0jCtYKDV0#84FQ`x}(y|e$823twS
zZz=z8oAkR^+Kjkv+6$^#wsUTd?t2)C?AX%fD*~V(W5mVWS~gH-#*e&2u^iO<Ku(T4
z`FviA02fXZhzW3zO9GT7`_3-;tqMcl%wv+Z^a1<-+wJw+1z`xt>x~(>#lA0Ng&f4G
zL!OL1M5Uzo_c0*vB^3`dBsFaHs6!&tiQO*90}aGuD<)6RgFI#ZABP_Lxc#L+#?!3-
z5Bdj_Q9=LTKN$C)^#6x=O#Od3=or`kmN-M|riBVXD)H&7LGPd^-zvxa6m*CE;lLUU
ztp4bFFqrO-rv0PdU^wU>4g14`@B4#kUmahny77g9bA?|)WI}MufKX=M3!{V{12hT4
zY%Rf|5yCFf;5lOP6dVjE!|`O~Ah+-I$A@S#If7{7xU*S*IGmu-Xa<jvi;mp=0d)5#
za6B3w?DzL4aE1;J`hBU@E(q7vbwukN$BV=fS4FE^-S^6-43N&PTSC3eZcwGvpSiLY
zaD9?^Rwyz$>k9HvvHD9yrHS7nwV9bI&H^Za*q?&zip~G6XzQf`_R`|`Z~3~Dy{RTX
zXKGk?Z+t@iQp3{{mSCY?JpUw0EQ_4giUs>^Sa5JO8TF424j+*P-4QxEnmI@NqyBhy
zIM_ejcMhEaoDB{UnmGrDlf%Jm<cz)k_`n-OZ_po4#%LeToC9|<p3Tq`3*H+G_N!U2
z-;M<*y-_O`JRIy#hQq-lvLJ%}!`ac%z?lt4?h*6`gMG(^M~BF99W+A2861oq1f9WT
z)SnDSM@OTh{iFUEL4*#EoYDA+1@DOkC%sV>3rfyv#e&0DEI6Kw4u%K&kH~`F;o)R<
z2>XWzM=tCicmp&Z?oSXpaNP0uV9-Yc_wZmi8yy`S3=jJ=w0|`AoXN}`_QwbCV7C9n
zg7?OP!)g{Bwq?Oqy5L}c&_CE8Jf<$V@4)fV!O->g4=0`r;m{il-0@^Ef|J?N!EE9j
zIESOb;c$p1gZ>O1p+O&x4-g!9&TupyPo7xt-dM0w7o4=!1t%?a!Qpt^KkPrIE;t(x
z2m40{gF^&Ill|HL#MyU_5OR;;WIUS;$0O(s4u_6&aOlm(`vU=z0B!$ou<v@~;lY7(
z<Q~F9cl5-9_r`*iy5OX(E;w$<g8ifZ!RToGh%7ibIvkA$-q>@-Lo^w?-fVO@@+Pwp
z8XWA8y`uqi2WSZUgTZhxJKR5*%nqG_gARs=PXA~;8a%P!J+a`pnguy$%~8mU)$>*V
kXgE5U^ivz-v0YA2&(rhtd?U~Q9{>RV|G7UQ*#MdW0M7yyzW@LL

literal 0
HcmV?d00001

diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml
index f6c61c89e..3878cc0cd 100644
--- a/charts/postgres-operator/values.yaml
+++ b/charts/postgres-operator/values.yaml
@@ -1,7 +1,7 @@
 image:
   registry: registry.opensource.zalan.do
   repository: acid/postgres-operator
-  tag: v1.7.1
+  tag: v1.8.0
   pullPolicy: "IfNotPresent"
 
   # Optionally specify an array of imagePullSecrets.
@@ -38,7 +38,7 @@ configGeneral:
   # Select if setup uses endpoints (default), or configmaps to manage leader (DCS=k8s)
   # kubernetes_use_configmaps: false
   # Spilo docker image
-  docker_image: registry.opensource.zalan.do/acid/spilo-14:2.1-p4
+  docker_image: registry.opensource.zalan.do/acid/spilo-14:2.1-p5
   # min number of instances in Postgres cluster. -1 = no limit
   min_instances: -1
   # max number of instances in Postgres cluster. -1 = no limit
@@ -313,7 +313,7 @@ configAwsOrGcp:
 # configure K8s cron job managed by the operator
 configLogicalBackup:
   # image for pods of the logical backup job (example runs pg_dumpall)
-  logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:v1.7.1"
+  logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:v1.8.0"
   # path of google cloud service account json file
   # logical_backup_google_application_credentials: ""
 
diff --git a/docs/administrator.md b/docs/administrator.md
index aa08f549e..aea4ec605 100644
--- a/docs/administrator.md
+++ b/docs/administrator.md
@@ -1286,7 +1286,7 @@ make docker
 
 # build in image in minikube docker env
 eval $(minikube docker-env)
-docker build -t registry.opensource.zalan.do/acid/postgres-operator-ui:v1.7.1 .
+docker build -t registry.opensource.zalan.do/acid/postgres-operator-ui:v1.8.0 .
 
 # apply UI manifests next to a running Postgres Operator
 kubectl apply -f manifests/
diff --git a/docs/quickstart.md b/docs/quickstart.md
index 7049a6ef9..00ba2f3aa 100644
--- a/docs/quickstart.md
+++ b/docs/quickstart.md
@@ -85,20 +85,25 @@ oc create -f manifests/operator-service-account-rbac-openshift.yaml
 
 ### Helm chart
 
-Alternatively, the operator can be installed by using the provided [Helm](https://helm.sh/)
-chart which saves you the manual steps. Clone this repo and change directory to
-the repo root. With Helm v3 installed you should be able to run:
+Alternatively, the operator can be installed by using the provided
+[Helm](https://helm.sh/) chart which saves you the manual steps. The charts
+for both the Postgres Operator and its UI are hosted via the `gh-pages` branch.
+They only work only with Helm v3. Helm v2 support was dropped with v1.8.0.
 
 ```bash
-helm install postgres-operator ./charts/postgres-operator
+# add repo for postgres-operator
+helm repo add postgres-operator-charts https://opensource.zalando.com/postgres-operator/charts/postgres-operator
+
+# install the postgres-operator
+helm install postgres-operator postgres-operator-charts/postgres-operator
+
+# add repo for postgres-operator-ui
+helm repo add postgres-operator-ui-charts https://opensource.zalando.com/postgres-operator/charts/postgres-operator-ui
+
+# install the postgres-operator-ui
+helm install postgres-operator-ui postgres-operator-ui-charts/postgres-operator-ui
 ```
 
-The chart works with both Helm 2 and Helm 3. The `crd-install` hook from v2 will
-be skipped with warning when using v3. Documentation for installing applications
-with Helm 2 can be found in the [v2 docs](https://v2.helm.sh/docs/).
-
-The chart is also hosted at: https://opensource.zalando.com/postgres-operator/charts/postgres-operator/
-
 ## Check if Postgres Operator is running
 
 Starting the operator may take a few seconds. Check if the operator pod is
diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md
index 08cb37e03..e4e64cc32 100644
--- a/docs/reference/operator_parameters.md
+++ b/docs/reference/operator_parameters.md
@@ -679,7 +679,7 @@ grouped under the `logical_backup` key.
   runs `pg_dumpall` on a replica if possible and uploads compressed results to
   an S3 bucket under the key `/spilo/pg_cluster_name/cluster_k8s_uuid/logical_backups`.
   The default image is the same image built with the Zalando-internal CI
-  pipeline. Default: "registry.opensource.zalan.do/acid/logical-backup:v1.7.1"
+  pipeline. Default: "registry.opensource.zalan.do/acid/logical-backup:v1.8.0"
 
 * **logical_backup_google_application_credentials**
   Specifies the path of the google cloud service account json file. Default is empty.
diff --git a/go.mod b/go.mod
index 0d46daeec..26e629beb 100644
--- a/go.mod
+++ b/go.mod
@@ -7,6 +7,7 @@ require (
 	github.com/golang/mock v1.6.0
 	github.com/lib/pq v1.10.4
 	github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d
+	github.com/pkg/errors v0.9.1
 	github.com/r3labs/diff v1.1.0
 	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.7.0
@@ -44,7 +45,6 @@ require (
 	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.1 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	golang.org/x/mod v0.5.1 // indirect
diff --git a/kubectl-pg/go.mod b/kubectl-pg/go.mod
index 954b99940..d82b2392f 100644
--- a/kubectl-pg/go.mod
+++ b/kubectl-pg/go.mod
@@ -5,7 +5,7 @@ go 1.17
 require (
 	github.com/spf13/cobra v1.2.1
 	github.com/spf13/viper v1.9.0
-	github.com/zalando/postgres-operator v1.7.1
+	github.com/zalando/postgres-operator v1.8.0
 	k8s.io/api v0.22.4
 	k8s.io/apiextensions-apiserver v0.22.4
 	k8s.io/apimachinery v0.22.4
diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml
index 26e404e6d..e2d798a94 100644
--- a/manifests/complete-postgres-manifest.yaml
+++ b/manifests/complete-postgres-manifest.yaml
@@ -10,7 +10,7 @@ metadata:
 #    "delete-date": "2020-08-31"  # can only be deleted on that day if "delete-date "key is configured
 #    "delete-clustername": "acid-test-cluster"  # can only be deleted when name matches if "delete-clustername" key is configured
 spec:
-  dockerImage: registry.opensource.zalan.do/acid/spilo-14:2.1-p4
+  dockerImage: registry.opensource.zalan.do/acid/spilo-14:2.1-p5
   teamId: "acid"
   numberOfInstances: 2
   users:  # Application/Robot users
diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml
index 36dd092c2..64e01fb50 100644
--- a/manifests/configmap.yaml
+++ b/manifests/configmap.yaml
@@ -34,7 +34,7 @@ data:
   # default_memory_request: 100Mi
   # delete_annotation_date_key: delete-date
   # delete_annotation_name_key: delete-clustername
-  docker_image: registry.opensource.zalan.do/acid/spilo-14:2.1-p4
+  docker_image: registry.opensource.zalan.do/acid/spilo-14:2.1-p5
   # downscaler_annotations: "deployment-time,downscaler/*"
   # enable_admin_role_for_users: "true"
   # enable_crd_registration: "true"
@@ -71,7 +71,7 @@ data:
   # kube_iam_role: ""
   # kubernetes_use_configmaps: "false"
   # log_s3_bucket: ""
-  logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:v1.7.1"
+  logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:v1.8.0"
   # logical_backup_google_application_credentials: ""
   logical_backup_job_prefix: "logical-backup-"
   logical_backup_provider: "s3"
diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml
index 99184dd17..d5594f6d4 100644
--- a/manifests/operatorconfiguration.crd.yaml
+++ b/manifests/operatorconfiguration.crd.yaml
@@ -66,7 +66,7 @@ spec:
                   type: string
               docker_image:
                 type: string
-                default: "registry.opensource.zalan.do/acid/spilo-14:2.1-p4"
+                default: "registry.opensource.zalan.do/acid/spilo-14:2.1-p5"
               enable_crd_registration:
                 type: boolean
                 default: true
@@ -448,7 +448,7 @@ spec:
                 properties:
                   logical_backup_docker_image:
                     type: string
-                    default: "registry.opensource.zalan.do/acid/logical-backup:v1.7.1"
+                    default: "registry.opensource.zalan.do/acid/logical-backup:v1.8.0"
                   logical_backup_google_application_credentials:
                     type: string
                   logical_backup_job_prefix:
diff --git a/manifests/postgres-operator.yaml b/manifests/postgres-operator.yaml
index eddd44650..f06ba50e6 100644
--- a/manifests/postgres-operator.yaml
+++ b/manifests/postgres-operator.yaml
@@ -19,7 +19,7 @@ spec:
       serviceAccountName: postgres-operator
       containers:
       - name: postgres-operator
-        image: registry.opensource.zalan.do/acid/postgres-operator:v1.7.1
+        image: registry.opensource.zalan.do/acid/postgres-operator:v1.8.0
         imagePullPolicy: IfNotPresent
         resources:
           requests:
diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml
index 5cef7a353..d2e42e4f8 100644
--- a/manifests/postgresql-operator-default-configuration.yaml
+++ b/manifests/postgresql-operator-default-configuration.yaml
@@ -3,7 +3,7 @@ kind: OperatorConfiguration
 metadata:
   name: postgresql-operator-default-configuration
 configuration:
-  docker_image: registry.opensource.zalan.do/acid/spilo-14:2.1-p4
+  docker_image: registry.opensource.zalan.do/acid/spilo-14:2.1-p5
   # enable_crd_registration: true
   # crd_categories:
   # - all
@@ -144,7 +144,7 @@ configuration:
     # wal_gs_bucket: ""
     # wal_s3_bucket: ""
   logical_backup:
-    logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:v1.7.1"
+    logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:v1.8.0"
     # logical_backup_google_application_credentials: ""
     logical_backup_job_prefix: "logical-backup-"
     logical_backup_provider: "s3"
diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go
index e64b290ec..029685729 100644
--- a/pkg/controller/operator_config.go
+++ b/pkg/controller/operator_config.go
@@ -38,7 +38,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	result.EnableSpiloWalPathCompat = fromCRD.EnableSpiloWalPathCompat
 	result.EtcdHost = fromCRD.EtcdHost
 	result.KubernetesUseConfigMaps = fromCRD.KubernetesUseConfigMaps
-	result.DockerImage = util.Coalesce(fromCRD.DockerImage, "registry.opensource.zalan.do/acid/spilo-14:2.1-p4")
+	result.DockerImage = util.Coalesce(fromCRD.DockerImage, "registry.opensource.zalan.do/acid/spilo-14:2.1-p5")
 	result.Workers = util.CoalesceUInt32(fromCRD.Workers, 8)
 	result.MinInstances = fromCRD.MinInstances
 	result.MaxInstances = fromCRD.MaxInstances
@@ -165,7 +165,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 
 	// logical backup config
 	result.LogicalBackupSchedule = util.Coalesce(fromCRD.LogicalBackup.Schedule, "30 00 * * *")
-	result.LogicalBackupDockerImage = util.Coalesce(fromCRD.LogicalBackup.DockerImage, "registry.opensource.zalan.do/acid/logical-backup:v1.7.1")
+	result.LogicalBackupDockerImage = util.Coalesce(fromCRD.LogicalBackup.DockerImage, "registry.opensource.zalan.do/acid/logical-backup:v1.8.0")
 	result.LogicalBackupProvider = util.Coalesce(fromCRD.LogicalBackup.BackupProvider, "s3")
 	result.LogicalBackupS3Bucket = fromCRD.LogicalBackup.S3Bucket
 	result.LogicalBackupS3Region = fromCRD.LogicalBackup.S3Region
diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go
index 06b55bbd1..4d88db85d 100644
--- a/pkg/util/config/config.go
+++ b/pkg/util/config/config.go
@@ -122,7 +122,7 @@ type Scalyr struct {
 // LogicalBackup defines configuration for logical backup
 type LogicalBackup struct {
 	LogicalBackupSchedule                     string `name:"logical_backup_schedule" default:"30 00 * * *"`
-	LogicalBackupDockerImage                  string `name:"logical_backup_docker_image" default:"registry.opensource.zalan.do/acid/logical-backup:v1.7.1"`
+	LogicalBackupDockerImage                  string `name:"logical_backup_docker_image" default:"registry.opensource.zalan.do/acid/logical-backup:v1.8.0"`
 	LogicalBackupProvider                     string `name:"logical_backup_provider" default:"s3"`
 	LogicalBackupS3Bucket                     string `name:"logical_backup_s3_bucket" default:""`
 	LogicalBackupS3Region                     string `name:"logical_backup_s3_region" default:""`
@@ -161,7 +161,7 @@ type Config struct {
 	WatchedNamespace        string            `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to'
 	KubernetesUseConfigMaps bool              `name:"kubernetes_use_configmaps" default:"false"`
 	EtcdHost                string            `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS
-	DockerImage             string            `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-14:2.1-p4"`
+	DockerImage             string            `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-14:2.1-p5"`
 	SidecarImages           map[string]string `name:"sidecar_docker_images"` // deprecated in favour of SidecarContainers
 	SidecarContainers       []v1.Container    `name:"sidecars"`
 	PodServiceAccountName   string            `name:"pod_service_account_name" default:"postgres-pod"`
diff --git a/ui/app/package.json b/ui/app/package.json
index d0528e0bd..a1838df0f 100644
--- a/ui/app/package.json
+++ b/ui/app/package.json
@@ -1,6 +1,6 @@
 {
   "name": "postgres-operator-ui",
-  "version": "1.3.0",
+  "version": "1.8.0",
   "description": "PostgreSQL Operator UI",
   "main": "src/app.js",
   "config": {
diff --git a/ui/manifests/deployment.yaml b/ui/manifests/deployment.yaml
index b2af47790..18d33d323 100644
--- a/ui/manifests/deployment.yaml
+++ b/ui/manifests/deployment.yaml
@@ -18,7 +18,7 @@ spec:
       serviceAccountName: postgres-operator-ui
       containers:
         - name: "service"
-          image: registry.opensource.zalan.do/acid/postgres-operator-ui:v1.7.1
+          image: registry.opensource.zalan.do/acid/postgres-operator-ui:v1.8.0
           ports:
             - containerPort: 8081
               protocol: "TCP"