Merge branch 'master' into gh-pages

.gitignore

@@ -7,6 +7,8 @@
 _obj
 _test
 _manifests
+_tmp
+github.com
 
 # Architecture specific extensions/prefixes
 *.[568vq]
@@ -26,6 +28,7 @@ _testmain.go
 /vendor/
 /build/
 /docker/build/
+/github.com/
 .idea
 
 scm-source.json

.travis.yml

@@ -8,7 +8,7 @@ branches:
 language: go
 
 go:
-  - "1.12.x"
+  - "1.14.x"
 
 before_install:
   - go get github.com/mattn/goveralls

Makefile

@@ -97,4 +97,4 @@ test:
 	GO111MODULE=on go test ./...
 
 e2e: docker # build operator image to be tested
-	cd e2e; make tools test clean
+	cd e2e; make tools e2etest clean

README.md

@@ -17,8 +17,9 @@ pipelines with no access to Kubernetes directly.
 
 * Rolling updates on Postgres cluster changes
 * Volume resize without Pod restarts
+* Database connection pooler
 * Cloning Postgres clusters
-* Logical Backups to S3 Bucket
+* Logical backups to S3 Bucket
 * Standby cluster from S3 WAL archive
 * Configurable for non-cloud environments
 * UI to create and edit Postgres cluster manifests

charts/postgres-operator-ui/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 name: postgres-operator-ui
-version: 1.4.0
-appVersion: 1.4.0
+version: 1.5.0
+appVersion: 1.5.0
 home: https://github.com/zalando/postgres-operator
 description: Postgres Operator UI provides a graphical interface for a convenient database-as-a-service user experience
 keywords:
@@ -14,8 +14,6 @@ keywords:
 maintainers:
 - name: Zalando
   email: opensource@zalando.de
-- name: siku4
-  email: sk@sik-net.de
 sources:
 - https://github.com/zalando/postgres-operator
 engine: gotpl

charts/postgres-operator-ui/index.yaml

@@ -1,9 +1,32 @@
 apiVersion: v1
 entries:
   postgres-operator-ui:
+  - apiVersion: v1
+    appVersion: 1.5.0
+    created: "2020-05-08T12:07:31.651762139+02:00"
+    description: Postgres Operator UI provides a graphical interface for a convenient
+      database-as-a-service user experience
+    digest: d7a36de8a3716f7b7954e2e51ed07863ea764dbb129a2fd3ac6a453f9e98a115
+    home: https://github.com/zalando/postgres-operator
+    keywords:
+    - postgres
+    - operator
+    - ui
+    - cloud-native
+    - patroni
+    - spilo
+    maintainers:
+    - email: opensource@zalando.de
+      name: Zalando
+    name: postgres-operator-ui
+    sources:
+    - https://github.com/zalando/postgres-operator
+    urls:
+    - postgres-operator-ui-1.5.0.tgz
+    version: 1.5.0
   - apiVersion: v1
     appVersion: 1.4.0
-    created: "2020-02-24T15:32:47.610967635+01:00"
+    created: "2020-05-08T12:07:31.651223951+02:00"
     description: Postgres Operator UI provides a graphical interface for a convenient
       database-as-a-service user experience
     digest: 00e0eff7056d56467cd5c975657fbb76c8d01accd25a4b7aca81bc42aeac961d
@@ -26,4 +49,4 @@ entries:
     urls:
     - postgres-operator-ui-1.4.0.tgz
     version: 1.4.0
-generated: "2020-02-24T15:32:47.610348278+01:00"
+generated: "2020-05-08T12:07:31.650495247+02:00"

charts/postgres-operator-ui/templates/clusterrole.yaml

@@ -38,6 +38,7 @@ rules:
 - apiGroups:
   - apps
   resources:
+  - deployments
   - statefulsets
   verbs:
   - get

charts/postgres-operator-ui/templates/deployment.yaml

@@ -1,5 +1,5 @@
-apiVersion: "apps/v1"
-kind: "Deployment"
+apiVersion: apps/v1
+kind: Deployment
 metadata:
   labels:
     app.kubernetes.io/name: {{ template "postgres-operator-ui.name" . }}
@@ -41,6 +41,10 @@ spec:
               value: "http://localhost:8081"
             - name: "OPERATOR_API_URL"
               value: {{ .Values.envs.operatorApiUrl }}
+            - name: "OPERATOR_CLUSTER_NAME_LABEL"
+              value: {{ .Values.envs.operatorClusterNameLabel }}
+            - name: "RESOURCES_VISIBLE"
+              value: "{{ .Values.envs.resourcesVisible }}"
             - name: "TARGET_NAMESPACE"
               value: {{ .Values.envs.targetNamespace }}
             - name: "TEAMS"

charts/postgres-operator-ui/templates/service.yaml

@@ -11,6 +11,9 @@ spec:
   ports:
     - port: {{ .Values.service.port }}
       targetPort: 8081
+      {{- if and (eq .Values.service.type "NodePort") .Values.service.nodePort }}
+      nodePort: {{ .Values.service.nodePort }}
+      {{- end }}
       protocol: TCP
   selector:
     app.kubernetes.io/instance: {{ .Release.Name }}

charts/postgres-operator-ui/values.yaml

@@ -8,7 +8,7 @@ replicaCount: 1
 image:
   registry: registry.opensource.zalan.do
   repository: acid/postgres-operator-ui
-  tag: v1.4.0
+  tag: v1.5.0-dirty
   pullPolicy: "IfNotPresent"
 
 rbac:
@@ -25,8 +25,8 @@ serviceAccount:
 # configure UI pod resources
 resources:
   limits:
-    cpu: 300m
-    memory: 3000Mi
+    cpu: 200m
+    memory: 200Mi
   requests:
     cpu: 100m
     memory: 100Mi
@@ -36,12 +36,17 @@ envs:
   # IMPORTANT: While operator chart and UI chart are idendependent, this is the interface between
   # UI and operator API. Insert the service name of the operator API here!
   operatorApiUrl: "http://postgres-operator:8080"
+  operatorClusterNameLabel: "cluster-name"
+  resourcesVisible: "False"
   targetNamespace: "default"
 
 # configure UI service
 service:
   type: "ClusterIP"
-  port: "8080"
+  port: "8081"
+  # If the type of the service is NodePort a port can be specified using the nodePort field
+  # If the nodePort field is not specified, or if it has no value, then a random port is used
+  # notePort: 32521
 
 # configure UI ingress. If needed: "enabled: true"
 ingress:

charts/postgres-operator/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 name: postgres-operator
-version: 1.4.0
-appVersion: 1.4.0
+version: 1.5.0
+appVersion: 1.5.0
 home: https://github.com/zalando/postgres-operator
 description: Postgres Operator creates and manages PostgreSQL clusters running in Kubernetes
 keywords:

charts/postgres-operator/crds/operatorconfigurations.yaml

@@ -62,10 +62,14 @@ spec:
               type: string
             enable_crd_validation:
               type: boolean
+            enable_lazy_spilo_upgrade:
+              type: boolean
             enable_shm_volume:
               type: boolean
             etcd_host:
               type: string
+            kubernetes_use_configmaps:
+              type: boolean
             max_instances:
               type: integer
               minimum: -1  # -1 = disabled
@@ -82,6 +86,12 @@ spec:
               type: object
               additionalProperties:
                 type: string
+            sidecars:
+              type: array
+              nullable: true
+              items:
+                type: object
+                additionalProperties: true
             workers:
               type: integer
               minimum: 1
@@ -107,6 +117,10 @@ spec:
                   type: object
                   additionalProperties:
                     type: string
+                downscaler_annotations:
+                  type: array
+                  items:
+                    type: string
                 enable_init_containers:
                   type: boolean
                 enable_pod_antiaffinity:
@@ -297,7 +311,7 @@ spec:
                   type: integer
                 ring_log_lines:
                   type: integer
-            scalyr:
+            scalyr:  # deprecated
               type: object
               properties:
                 scalyr_api_key:
@@ -318,6 +332,47 @@ spec:
                   pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$'
                 scalyr_server_url:
                   type: string
+            connection_pooler:
+              type: object
+              properties:
+                connection_pooler_schema:
+                  type: string
+                  #default: "pooler"
+                connection_pooler_user:
+                  type: string
+                  #default: "pooler"
+                connection_pooler_image:
+                  type: string
+                  #default: "registry.opensource.zalan.do/acid/pgbouncer"
+                connection_pooler_max_db_connections:
+                  type: integer
+                  #default: 60
+                connection_pooler_mode:
+                  type: string
+                  enum:
+                    - "session"
+                    - "transaction"
+                  #default: "transaction"
+                connection_pooler_number_of_instances:
+                  type: integer
+                  minimum: 2
+                  #default: 2
+                connection_pooler_default_cpu_limit:
+                  type: string
+                  pattern: '^(\d+m|\d+(\.\d{1,3})?)$'
+                  #default: "1"
+                connection_pooler_default_cpu_request:
+                  type: string
+                  pattern: '^(\d+m|\d+(\.\d{1,3})?)$'
+                  #default: "500m"
+                connection_pooler_default_memory_limit:
+                  type: string
+                  pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$'
+                  #default: "100Mi"
+                connection_pooler_default_memory_request:
+                  type: string
+                  pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$'
+                  #default: "100Mi"
         status:
           type: object
           additionalProperties:

charts/postgres-operator/crds/postgresqls.yaml

@@ -74,6 +74,28 @@ spec:
             - teamId
             - postgresql
           properties:
+            additionalVolumes:
+              type: array
+              items:
+                type: object
+                required:
+                  - name
+                  - mountPath
+                  - volumeSource
+                properties:
+                  name:
+                    type: string
+                  mountPath:
+                    type: string
+                  targetContainers:
+                    type: array
+                    nullable: true
+                    items:
+                      type: string
+                  volumeSource:
+                    type: object
+                  subPath:
+                    type: string
             allowedSourceRanges:
               type: array
               nullable: true
@@ -106,6 +128,55 @@ spec:
                 uid:
                   format: uuid
                   type: string
+            connectionPooler:
+              type: object
+              properties:
+                dockerImage:
+                  type: string
+                maxDBConnections:
+                  type: integer
+                mode:
+                  type: string
+                  enum:
+                    - "session"
+                    - "transaction"
+                numberOfInstances:
+                  type: integer
+                  minimum: 2
+                resources:
+                  type: object
+                  required:
+                    - requests
+                    - limits
+                  properties:
+                    limits:
+                      type: object
+                      required:
+                        - cpu
+                        - memory
+                      properties:
+                        cpu:
+                          type: string
+                          pattern: '^(\d+m|\d+(\.\d{1,3})?)$'
+                        memory:
+                          type: string
+                          pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$'
+                    requests:
+                      type: object
+                      required:
+                        - cpu
+                        - memory
+                      properties:
+                        cpu:
+                          type: string
+                          pattern: '^(\d+m|\d+(\.\d{1,3})?)$'
+                        memory:
+                          type: string
+                          pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$'
+                schema:
+                  type: string
+                user:
+                  type: string
             databases:
               type: object
               additionalProperties:
@@ -113,6 +184,8 @@ spec:
               # Note: usernames specified here as database owners must be declared in the users key of the spec key.
             dockerImage:
               type: string
+            enableConnectionPooler:
+              type: boolean
             enableLogicalBackup:
               type: boolean
             enableMasterLoadBalancer:
@@ -167,6 +240,10 @@ spec:
                   type: integer
                 retry_timeout:
                   type: integer
+                synchronous_mode:
+                  type: boolean
+                synchronous_mode_strict:
+                  type: boolean
                 maximum_lag_on_failover:
                   type: integer
             podAnnotations:
@@ -196,6 +273,26 @@ spec:
                   type: object
                   additionalProperties:
                     type: string
+            preparedDatabases:
+              type: object
+              additionalProperties:
+                type: object
+                properties:
+                  defaultUsers:
+                    type: boolean
+                  extensions:
+                    type: object
+                    additionalProperties:
+                      type: string
+                  schemas:
+                    type: object
+                    additionalProperties:
+                      type: object
+                      properties:
+                        defaultUsers:
+                          type: boolean
+                        defaultRoles:
+                          type: boolean
             replicaLoadBalancer:  # deprecated
               type: boolean
             resources:
@@ -287,6 +384,21 @@ spec:
                   type: string
             teamId:
               type: string
+            tls:
+              type: object
+              required:
+                - secretName
+              properties:
+                secretName:
+                  type: string
+                certificateFile:
+                  type: string
+                privateKeyFile:
+                  type: string
+                caFile:
+                  type: string
+                caSecretName:
+                  type: string
             tolerations:
               type: array
               items:

charts/postgres-operator/index.yaml

@@ -2,11 +2,33 @@ apiVersion: v1
 entries:
   postgres-operator:
   - apiVersion: v1
-    appVersion: 1.4.0
-    created: "2020-02-20T17:39:25.443276193+01:00"
+    appVersion: 1.5.0
+    created: "2020-05-04T16:36:19.646719041+02:00"
     description: Postgres Operator creates and manages PostgreSQL clusters running
       in Kubernetes
-    digest: b93ccde5581deb8ed0857136b8ce74ca3f1b7240438fa4415f705764a1300bed
+    digest: 43510e4ed7005b2b80708df24cfbb0099b263b4a2954cff4e8f305543760be6d
+    home: https://github.com/zalando/postgres-operator
+    keywords:
+    - postgres
+    - operator
+    - cloud-native
+    - patroni
+    - spilo
+    maintainers:
+    - email: opensource@zalando.de
+      name: Zalando
+    name: postgres-operator
+    sources:
+    - https://github.com/zalando/postgres-operator
+    urls:
+    - postgres-operator-1.5.0.tgz
+    version: 1.5.0
+  - apiVersion: v1
+    appVersion: 1.4.0
+    created: "2020-05-04T16:36:19.645338751+02:00"
+    description: Postgres Operator creates and manages PostgreSQL clusters running
+      in Kubernetes
+    digest: f8b90fecfc3cb825b94ed17edd9d5cefc36ae61801d4568597b4a79bcd73b2e9
     home: https://github.com/zalando/postgres-operator
     keywords:
     - postgres
@@ -23,50 +45,4 @@ entries:
     urls:
     - postgres-operator-1.4.0.tgz
     version: 1.4.0
-  - apiVersion: v1
-    appVersion: 1.3.0
-    created: "2020-02-20T17:39:25.441532163+01:00"
-    description: Postgres Operator creates and manages PostgreSQL clusters running
-      in Kubernetes
-    digest: 7e788fd37daec76a01f6d6f9fe5be5b54f5035e4eba0041e80a760d656537325
-    home: https://github.com/zalando/postgres-operator
-    keywords:
-    - postgres
-    - operator
-    - cloud-native
-    - patroni
-    - spilo
-    maintainers:
-    - email: opensource@zalando.de
-      name: Zalando
-    name: postgres-operator
-    sources:
-    - https://github.com/zalando/postgres-operator
-    urls:
-    - postgres-operator-1.3.0.tgz
-    version: 1.3.0
-  - apiVersion: v1
-    appVersion: 1.2.0
-    created: "2020-02-20T17:39:25.440278302+01:00"
-    description: Postgres Operator creates and manages PostgreSQL clusters running
-      in Kubernetes
-    digest: d10710c7cf19f4e266e7704f5d1e98dcfc61bee3919522326c35c22ca7d2f2bf
-    home: https://github.com/zalando/postgres-operator
-    keywords:
-    - postgres
-    - operator
-    - cloud-native
-    - patroni
-    - spilo
-    maintainers:
-    - email: opensource@zalando.de
-      name: Zalando
-    - email: kgyoo8232@gmail.com
-      name: kimxogus
-    name: postgres-operator
-    sources:
-    - https://github.com/zalando/postgres-operator
-    urls:
-    - postgres-operator-1.2.0.tgz
-    version: 1.2.0
-generated: "2020-02-20T17:39:25.439168098+01:00"
+generated: "2020-05-04T16:36:19.643857452+02:00"

charts/postgres-operator/templates/_helpers.tpl

@@ -31,6 +31,20 @@ Create a service account name.
 {{ default (include "postgres-operator.fullname" .) .Values.serviceAccount.name }}
 {{- end -}}
 
+{{/*
+Create a pod service account name.
+*/}}
+{{- define "postgres-pod.serviceAccountName" -}}
+{{ default (printf "%s-%v" (include "postgres-operator.fullname" .) "pod") .Values.podServiceAccount.name }}
+{{- end -}}
+
+{{/*
+Create a controller ID.
+*/}}
+{{- define "postgres-operator.controllerID" -}}
+{{ default (include "postgres-operator.fullname" .) .Values.controllerID.name }}
+{{- end -}}
+
 {{/*
 Create chart name and version as used by the chart label.
 */}}

charts/postgres-operator/templates/clusterrole-postgres-pod.yaml

@@ -2,7 +2,7 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: postgres-pod
+  name: {{ include "postgres-pod.serviceAccountName" . }}
   labels:
     app.kubernetes.io/name: {{ template "postgres-operator.name" . }}
     helm.sh/chart: {{ template "postgres-operator.chart" . }}

charts/postgres-operator/templates/clusterrole.yaml

@@ -42,6 +42,18 @@ rules:
   - configmaps
   verbs:
   - get
+# to send events to the CRs
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - get
+  - list
+  - patch
+  - update
+  - watch
 # to manage endpoints which are also used by Patroni
 - apiGroups:
   - ""
@@ -128,6 +140,7 @@ rules:
   - apps
   resources:
   - statefulsets
+  - deployments
   verbs:
   - create
   - delete

charts/postgres-operator/templates/configmap.yaml

@@ -9,6 +9,7 @@ metadata:
     app.kubernetes.io/managed-by: {{ .Release.Service }}
     app.kubernetes.io/instance: {{ .Release.Name }}
 data:
+  pod_service_account_name: {{ include "postgres-pod.serviceAccountName" . }}
 {{ toYaml .Values.configGeneral | indent 2 }}
 {{ toYaml .Values.configUsers | indent 2 }}
 {{ toYaml .Values.configKubernetes | indent 2 }}
@@ -19,4 +20,5 @@ data:
 {{ toYaml .Values.configDebug | indent 2 }}
 {{ toYaml .Values.configLoggingRestApi | indent 2 }}
 {{ toYaml .Values.configTeamsApi | indent 2 }}
+{{ toYaml .Values.configConnectionPooler | indent 2 }}
 {{- end }}

charts/postgres-operator/templates/deployment.yaml

@@ -43,6 +43,10 @@ spec:
       {{- else }}
         - name: POSTGRES_OPERATOR_CONFIGURATION_OBJECT
           value: {{ template "postgres-operator.fullname" . }}
+      {{- end }}
+      {{- if .Values.controllerID.create }}
+        - name: CONTROLLER_ID
+          value: {{ template "postgres-operator.controllerID" . }}
       {{- end }}
         resources:
 {{ toYaml .Values.resources | indent 10 }}

charts/postgres-operator/templates/operatorconfiguration.yaml

@@ -13,6 +13,7 @@ configuration:
   users:
 {{ toYaml .Values.configUsers | indent 4 }}
   kubernetes:
+    pod_service_account_name: {{ include "postgres-pod.serviceAccountName" . }}
     oauth_token_secret_name: {{ template "postgres-operator.fullname" . }}
 {{ toYaml .Values.configKubernetes | indent 4 }}
   postgres_pod_resources:
@@ -31,6 +32,6 @@ configuration:
 {{ toYaml .Values.configTeamsApi | indent 4 }}
   logging_rest_api:
 {{ toYaml .Values.configLoggingRestApi | indent 4 }}
-  scalyr:
-{{ toYaml .Values.configScalyr | indent 4 }}
+  connection_pooler:
+{{ toYaml .Values.configConnectionPooler | indent 4 }}
 {{- end }}

charts/postgres-operator/values-crd.yaml

@@ -1,7 +1,7 @@
 image:
   registry: registry.opensource.zalan.do
   repository: acid/postgres-operator
-  tag: v1.4.0
+  tag: v1.5.0
   pullPolicy: "IfNotPresent"
 
 # Optionally specify an array of imagePullSecrets.
@@ -19,12 +19,16 @@ configTarget: "OperatorConfigurationCRD"
 configGeneral:
   # choose if deployment creates/updates CRDs with OpenAPIV3Validation
   enable_crd_validation: true
+  # update only the statefulsets without immediately doing the rolling update
+  enable_lazy_spilo_upgrade: false
   # start any new database pod without limitations on shm memory
   enable_shm_volume: true
   # etcd connection string for Patroni. Empty uses K8s-native DCS.
   etcd_host: ""
+  # Select if setup uses endpoints (default), or configmaps to manage leader (DCS=k8s)
+  # kubernetes_use_configmaps: false
   # Spilo docker image
-  docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p2
+  docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p3
   # max number of instances in Postgres cluster. -1 = no limit
   min_instances: -1
   # min number of instances in Postgres cluster. -1 = no limit
@@ -63,6 +67,11 @@ configKubernetes:
   #   keya: valuea
   #   keyb: valueb
 
+  # list of annotations propagated from cluster manifest to statefulset and deployment
+  # downscaler_annotations:
+  #   - deployment-time
+  #   - downscaler/*
+
   # enables initContainers to run actions before Spilo is started
   enable_init_containers: true
   # toggles pod anti affinity on the Postgres pods
@@ -71,7 +80,7 @@ configKubernetes:
   enable_pod_disruption_budget: true
   # enables sidecar containers to run alongside Spilo in the same pod
   enable_sidecars: true
-  # name of the secret containing infrastructure roles names and passwords
+  # namespaced name of the secret containing infrastructure roles names and passwords
   # infrastructure_roles_secret_name: postgresql-infrastructure-roles
 
   # list of labels that can be inherited from the cluster manifest
@@ -86,15 +95,15 @@ configKubernetes:
   # node_readiness_label:
   #   status: ready
 
-  # name of the secret containing the OAuth2 token to pass to the teams API
+  # namespaced name of the secret containing the OAuth2 token to pass to the teams API
   # oauth_token_secret_name: postgresql-operator
 
   # defines the template for PDB (Pod Disruption Budget) names
   pdb_name_format: "postgres-{cluster}-pdb"
   # override topology key for pod anti affinity
   pod_antiaffinity_topology_key: "kubernetes.io/hostname"
-  # name of the ConfigMap with environment variables to populate on every pod
-  # pod_environment_configmap: ""
+  # namespaced name of the ConfigMap with environment variables to populate on every pod
+  # pod_environment_configmap: "default/my-custom-config"
 
   # specify the pod management policy of stateful sets of Postgres clusters
   pod_management_policy: "ordered_ready"
@@ -103,8 +112,6 @@ configKubernetes:
   # service account definition as JSON/YAML string to be used by postgres cluster pods
   # pod_service_account_definition: ""
 
-  # name of service account to be used by postgres cluster pods
-  pod_service_account_name: "postgres-pod"
   # role binding definition as JSON/YAML string to be used by pod service account
   # pod_service_account_role_binding_definition: ""
 
@@ -207,7 +214,7 @@ configAwsOrGcp:
 # configure K8s cron job managed by the operator
 configLogicalBackup:
   # image for pods of the logical backup job (example runs pg_dumpall)
-  logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup"
+  logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:master-58"
   # S3 Access Key ID
   logical_backup_s3_access_key_id: ""
   # S3 bucket to store backup results
@@ -218,7 +225,7 @@ configLogicalBackup:
   logical_backup_s3_endpoint: ""
   # S3 Secret Access Key
   logical_backup_s3_secret_access_key: ""
-  # S3 server side encription
+  # S3 server side encryption
   logical_backup_s3_sse: "AES256"
   # backup schedule in the cron format
   logical_backup_schedule: "30 00 * * *"
@@ -252,22 +259,24 @@ configTeamsApi:
   # URL of the Teams API service
   # teams_api_url: http://fake-teams-api.default.svc.cluster.local
 
-# Scalyr is a log management tool that Zalando uses as a sidecar
-configScalyr:
-  # API key for the Scalyr sidecar
-  # scalyr_api_key: ""
-
-  # Docker image for the Scalyr sidecar
-  # scalyr_image: ""
-
-  # CPU limit value for the Scalyr sidecar
-  scalyr_cpu_limit: "1"
-  # CPU rquest value for the Scalyr sidecar
-  scalyr_cpu_request: 100m
-  # Memory limit value for the Scalyr sidecar
-  scalyr_memory_limit: 500Mi
-  # Memory request value for the Scalyr sidecar
-  scalyr_memory_request: 50Mi
+configConnectionPooler:
+  # db schema to install lookup function into
+  connection_pooler_schema: "pooler"
+  # db user for pooler to use
+  connection_pooler_user: "pooler"
+  # docker image
+  connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-7"
+  # max db connections the pooler should hold
+  connection_pooler_max_db_connections: 60
+  # default pooling mode
+  connection_pooler_mode: "transaction"
+  # number of pooler instances
+  connection_pooler_number_of_instances: 2
+  # default resources
+  connection_pooler_default_cpu_request: 500m
+  connection_pooler_default_memory_request: 100Mi
+  connection_pooler_default_cpu_limit: "1"
+  connection_pooler_default_memory_limit: 100Mi
 
 rbac:
   # Specifies whether RBAC resources should be created
@@ -275,6 +284,7 @@ rbac:
 
 crd:
   # Specifies whether custom resource definitions should be created
+  # When using helm3, this is ignored; instead use "--skip-crds" to skip.
   create: true
 
 serviceAccount:
@@ -284,6 +294,11 @@ serviceAccount:
   # If not set and create is true, a name is generated using the fullname template
   name:
 
+podServiceAccount:
+  # The name of the ServiceAccount to be used by postgres cluster pods
+  # If not set a name is generated using the fullname template and "-pod" suffix
+  name: "postgres-pod"
+
 priorityClassName: ""
 
 resources:
@@ -305,3 +320,12 @@ tolerations: []
 # Node labels for pod assignment
 # Ref: https://kubernetes.io/docs/user-guide/node-selection/
 nodeSelector: {}
+
+controllerID:
+  # Specifies whether a controller ID should be defined for the operator
+  # Note, all postgres manifest must then contain the following annotation to be found by this operator
+  # "acid.zalan.do/controller": <controller-ID-of-the-operator>
+  create: false
+  # The name of the controller ID to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name:

charts/postgres-operator/values.yaml

@@ -1,7 +1,7 @@
 image:
   registry: registry.opensource.zalan.do
   repository: acid/postgres-operator
-  tag: v1.4.0
+  tag: v1.5.0
   pullPolicy: "IfNotPresent"
 
 # Optionally specify an array of imagePullSecrets.
@@ -19,12 +19,16 @@ configTarget: "ConfigMap"
 configGeneral:
   # choose if deployment creates/updates CRDs with OpenAPIV3Validation
   enable_crd_validation: "true"
+  # update only the statefulsets without immediately doing the rolling update
+  enable_lazy_spilo_upgrade: "false"
   # start any new database pod without limitations on shm memory
   enable_shm_volume: "true"
   # etcd connection string for Patroni. Empty uses K8s-native DCS.
   etcd_host: ""
+  # Select if setup uses endpoints (default), or configmaps to manage leader (DCS=k8s)
+  # kubernetes_use_configmaps: "false"
   # Spilo docker image
-  docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p2
+  docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p3
   # max number of instances in Postgres cluster. -1 = no limit
   min_instances: "-1"
   # min number of instances in Postgres cluster. -1 = no limit
@@ -59,6 +63,9 @@ configKubernetes:
   # annotations attached to each database pod
   # custom_pod_annotations: "keya:valuea,keyb:valueb"
 
+  # list of annotations propagated from cluster manifest to statefulset and deployment
+  # downscaler_annotations: "deployment-time,downscaler/*"
+
   # enables initContainers to run actions before Spilo is started
   enable_init_containers: "true"
   # toggles pod anti affinity on the Postgres pods
@@ -67,7 +74,7 @@ configKubernetes:
   enable_pod_disruption_budget: "true"
   # enables sidecar containers to run alongside Spilo in the same pod
   enable_sidecars: "true"
-  # name of the secret containing infrastructure roles names and passwords
+  # namespaced name of the secret containing infrastructure roles names and passwords
   # infrastructure_roles_secret_name: postgresql-infrastructure-roles
 
   # list of labels that can be inherited from the cluster manifest
@@ -79,15 +86,15 @@ configKubernetes:
   # set of labels that a running and active node should possess to be considered ready
   # node_readiness_label: ""
 
-  # name of the secret containing the OAuth2 token to pass to the teams API
+  # namespaced name of the secret containing the OAuth2 token to pass to the teams API
   # oauth_token_secret_name: postgresql-operator
 
   # defines the template for PDB (Pod Disruption Budget) names
   pdb_name_format: "postgres-{cluster}-pdb"
   # override topology key for pod anti affinity
   pod_antiaffinity_topology_key: "kubernetes.io/hostname"
-  # name of the ConfigMap with environment variables to populate on every pod
-  # pod_environment_configmap: ""
+  # namespaced name of the ConfigMap with environment variables to populate on every pod
+  # pod_environment_configmap: "default/my-custom-config"
 
   # specify the pod management policy of stateful sets of Postgres clusters
   pod_management_policy: "ordered_ready"
@@ -96,15 +103,13 @@ configKubernetes:
   # service account definition as JSON/YAML string to be used by postgres cluster pods
   # pod_service_account_definition: ""
 
-  # name of service account to be used by postgres cluster pods
-  pod_service_account_name: "postgres-pod"
   # role binding definition as JSON/YAML string to be used by pod service account
   # pod_service_account_role_binding_definition: ""
 
   # Postgres pods are terminated forcefully after this timeout
   pod_terminate_grace_period: 5m
   # template for database user secrets generated by the operator
-  secret_name_template: '{username}.{cluster}.credentials'
+  secret_name_template: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}"
   # group ID with write-access to volumes (required to run Spilo as non-root process)
   # spilo_fsgroup: "103"
 
@@ -198,7 +203,7 @@ configAwsOrGcp:
 # configure K8s cron job managed by the operator
 configLogicalBackup:
   # image for pods of the logical backup job (example runs pg_dumpall)
-  logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup"
+  logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:master-58"
   # S3 Access Key ID
   logical_backup_s3_access_key_id: ""
   # S3 bucket to store backup results
@@ -209,7 +214,7 @@ configLogicalBackup:
   logical_backup_s3_endpoint: ""
   # S3 Secret Access Key
   logical_backup_s3_secret_access_key: ""
-  # S3 server side encription
+  # S3 server side encryption
   logical_backup_s3_sse: "AES256"
   # backup schedule in the cron format
   logical_backup_schedule: "30 00 * * *"
@@ -245,12 +250,33 @@ configTeamsApi:
   # URL of the Teams API service
   # teams_api_url: http://fake-teams-api.default.svc.cluster.local
 
+# configure connection pooler deployment created by the operator
+configConnectionPooler:
+  # db schema to install lookup function into
+  connection_pooler_schema: "pooler"
+  # db user for pooler to use
+  connection_pooler_user: "pooler"
+  # docker image
+  connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-7"
+  # max db connections the pooler should hold
+  connection_pooler_max_db_connections: "60"
+  # default pooling mode
+  connection_pooler_mode: "transaction"
+  # number of pooler instances
+  connection_pooler_number_of_instances: "2"
+  # default resources
+  connection_pooler_default_cpu_request: 500m
+  connection_pooler_default_memory_request: 100Mi
+  connection_pooler_default_cpu_limit: "1"
+  connection_pooler_default_memory_limit: 100Mi
+
 rbac:
   # Specifies whether RBAC resources should be created
   create: true
 
 crd:
   # Specifies whether custom resource definitions should be created
+  # When using helm3, this is ignored; instead use "--skip-crds" to skip.
   create: true
 
 serviceAccount:
@@ -260,6 +286,11 @@ serviceAccount:
   # If not set and create is true, a name is generated using the fullname template
   name:
 
+podServiceAccount:
+  # The name of the ServiceAccount to be used by postgres cluster pods
+  # If not set a name is generated using the fullname template and "-pod" suffix
+  name: "postgres-pod"
+
 priorityClassName: ""
 
 resources:
@@ -281,3 +312,12 @@ tolerations: []
 # Node labels for pod assignment
 # Ref: https://kubernetes.io/docs/user-guide/node-selection/
 nodeSelector: {}
+
+controllerID:
+  # Specifies whether a controller ID should be defined for the operator
+  # Note, all postgres manifest must then contain the following annotation to be found by this operator
+  # "acid.zalan.do/controller": <controller-ID-of-the-operator>
+  create: false
+  # The name of the controller ID to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name:

cmd/main.go

@@ -77,7 +77,7 @@ func main() {
 		log.Fatalf("couldn't get REST config: %v", err)
 	}
 
-	c := controller.NewController(&config)
+	c := controller.NewController(&config, "")
 
 	c.Run(stop, wg)
 

delivery.yaml

@@ -12,7 +12,7 @@ pipeline:
         - desc: 'Install go'
           cmd: |
             cd /tmp
-            wget -q https://storage.googleapis.com/golang/go1.12.linux-amd64.tar.gz -O go.tar.gz
+            wget -q https://storage.googleapis.com/golang/go1.14.linux-amd64.tar.gz -O go.tar.gz
             tar -xf go.tar.gz
             mv go /usr/local
             ln -s /usr/local/go/bin/go /usr/bin/go

docker/DebugDockerfile

@@ -3,8 +3,17 @@ MAINTAINER Team ACID @ Zalando <team-acid@zalando.de>
 
 # We need root certificates to deal with teams api over https
 RUN apk --no-cache add ca-certificates go git musl-dev
-RUN go get github.com/derekparker/delve/cmd/dlv
 
 COPY build/* /
 
-CMD ["/root/go/bin/dlv", "--listen=:7777", "--headless=true", "--api-version=2", "exec", "/postgres-operator"]
+RUN addgroup -g 1000 pgo
+RUN adduser -D -u 1000 -G pgo -g 'Postgres Operator' pgo
+
+RUN go get github.com/derekparker/delve/cmd/dlv
+RUN cp /root/go/bin/dlv /dlv
+RUN chown -R pgo:pgo /dlv
+
+USER pgo:pgo
+RUN ls -l /
+
+CMD ["/dlv", "--listen=:7777", "--headless=true", "--api-version=2", "exec", "/postgres-operator"]

docs/administrator.md

@@ -95,6 +95,34 @@ lacks access rights to any of them (except K8s system namespaces like
 'list pods' execute at the cluster scope and fail at the first violation of
 access rights.
 
+## Operators with defined ownership of certain Postgres clusters
+
+By default, multiple operators can only run together in one K8s cluster when
+isolated into their [own namespaces](administrator.md#specify-the-namespace-to-watch).
+But, it is also possible to define ownership between operator instances and
+Postgres clusters running all in the same namespace or K8s cluster without
+interfering.
+
+First, define the [`CONTROLLER_ID`](../../manifests/postgres-operator.yaml#L38)
+environment variable in the operator deployment manifest. Then specify the ID
+in every Postgres cluster manifest you want this operator to watch using the
+`"acid.zalan.do/controller"` annotation:
+
+```yaml
+apiVersion: "acid.zalan.do/v1"
+kind: postgresql
+metadata:
+  name: demo-cluster
+  annotations:
+    "acid.zalan.do/controller": "second-operator"
+spec:
+  ...
+```
+
+Every other Postgres cluster which lacks the annotation will be ignored by this
+operator. Conversely, operators without a defined `CONTROLLER_ID` will ignore
+clusters with defined ownership of another operator.
+
 ## Role-based access control for the operator
 
 The manifest [`operator-service-account-rbac.yaml`](../manifests/operator-service-account-rbac.yaml)
@@ -293,11 +321,12 @@ spec:
 ## Custom Pod Environment Variables
 
 It is possible to configure a ConfigMap which is used by the Postgres pods as
-an additional provider for environment variables.
-
-One use case is to customize the Spilo image and configure it with environment
-variables. The ConfigMap with the additional settings is configured in the
-operator's main ConfigMap:
+an additional provider for environment variables. One use case is to customize
+the Spilo image and configure it with environment variables. The ConfigMap with
+the additional settings is referenced in the operator's main configuration.
+A namespace can be specified along with the name. If left out, the configured
+default namespace of your K8s client will be used and if the ConfigMap is not
+found there, the Postgres cluster's namespace is taken when different:
 
 **postgres-operator ConfigMap**
 
@@ -308,7 +337,7 @@ metadata:
   name: postgres-operator
 data:
   # referencing config map with custom settings
-  pod_environment_configmap: postgres-pod-config
+  pod_environment_configmap: default/postgres-pod-config
 ```
 
 **OperatorConfiguration**
@@ -321,7 +350,7 @@ metadata:
 configuration:
   kubernetes:
     # referencing config map with custom settings
-    pod_environment_configmap: postgres-pod-config
+    pod_environment_configmap: default/postgres-pod-config
 ```
 
 **referenced ConfigMap `postgres-pod-config`**
@@ -429,6 +458,17 @@ from numerous escape characters in the latter log entry, view it in CLI with
 `PodTemplate` used by the operator is yet to be updated with the default values
 used internally in K8s.
 
+The operator also support lazy updates of the Spilo image. That means the pod
+template of a PG cluster's stateful set is updated immediately with the new
+image, but no rolling update follows. This feature saves you a switchover - and
+hence downtime - when you know pods are re-started later anyway, for instance
+due to the node rotation. To force a rolling update, disable this mode by
+setting the `enable_lazy_spilo_upgrade` to `false` in the operator configuration
+and restart the operator pod. With the standard eager rolling updates the
+operator checks during Sync all pods run images specified in their respective
+statefulsets. The operator triggers a rolling upgrade for PG clusters that
+violate this condition.
+
 ## Logical backups
 
 The operator can manage K8s cron jobs to run logical backups of Postgres
@@ -478,6 +518,33 @@ A secret can be pre-provisioned in different ways:
 * Automatically provisioned via a custom K8s controller like
   [kube-aws-iam-controller](https://github.com/mikkeloscar/kube-aws-iam-controller)
 
+## Sidecars for Postgres clusters
+
+A list of sidecars is added to each cluster created by the operator. The default
+is empty.
+
+```yaml
+kind: OperatorConfiguration
+configuration:
+  sidecars:
+  - image: image:123
+    name: global-sidecar
+    ports:
+    - containerPort: 80
+    volumeMounts:
+    - mountPath: /custom-pgdata-mountpoint
+      name: pgdata
+  - ...
+```
+
+In addition to any environment variables you specify, the following environment
+variables are always passed to sidecars:
+
+  - `POD_NAME` - field reference to `metadata.name`
+  - `POD_NAMESPACE` - field reference to `metadata.namespace`
+  - `POSTGRES_USER` - the superuser that can be used to connect to the database
+  - `POSTGRES_PASSWORD` - the password for the superuser
+
 ## Setting up the Postgres Operator UI
 
 Since the v1.2 release the Postgres Operator is shipped with a browser-based

docs/index.md

@@ -37,9 +37,10 @@ in some overarching orchestration, like rolling updates to improve the user
 experience.
 
 Monitoring or tuning Postgres is not in scope of the operator in the current
-state. Other tools like [ZMON](https://opensource.zalando.com/zmon/),
-[Prometheus](https://prometheus.io/) or more Postgres specific options can be
-used to complement it.
+state. However, with globally configurable sidecars we provide enough
+flexibility to complement it with other tools like [ZMON](https://opensource.zalando.com/zmon/),
+[Prometheus](https://prometheus.io/) or more Postgres specific options.
+
 
 ## Overview of involved entities
 
@@ -70,12 +71,26 @@ Please, report any issues discovered to https://github.com/zalando/postgres-oper
 
 ## Talks
 
-1. "Building your own PostgreSQL-as-a-Service on Kubernetes" talk by Alexander Kukushkin, KubeCon NA 2018: [video](https://www.youtube.com/watch?v=G8MnpkbhClc) | [slides](https://static.sched.com/hosted_files/kccna18/1d/Building%20your%20own%20PostgreSQL-as-a-Service%20on%20Kubernetes.pdf)
+- "PostgreSQL on K8S at Zalando: Two years in production" talk by Alexander Kukushkin, FOSSDEM 2020: [video](https://fosdem.org/2020/schedule/event/postgresql_postgresql_on_k8s_at_zalando_two_years_in_production/) | [slides](https://fosdem.org/2020/schedule/event/postgresql_postgresql_on_k8s_at_zalando_two_years_in_production/attachments/slides/3883/export/events/attachments/postgresql_postgresql_on_k8s_at_zalando_two_years_in_production/slides/3883/PostgreSQL_on_K8s_at_Zalando_Two_years_in_production.pdf)
 
-2. "PostgreSQL and Kubernetes: DBaaS without a vendor-lock" talk by Oleksii Kliukin, PostgreSQL Sessions 2018: [video](https://www.youtube.com/watch?v=q26U2rQcqMw) | [slides](https://speakerdeck.com/alexeyklyukin/postgresql-and-kubernetes-dbaas-without-a-vendor-lock)
+- "Postgres as a Service at Zalando" talk by Jan Mußler, DevOpsDays Poznań 2019: [video](https://www.youtube.com/watch?v=FiWS5m72XI8)
 
-3. "PostgreSQL High Availability on Kubernetes with Patroni" talk by Oleksii Kliukin, Atmosphere 2018: [video](https://www.youtube.com/watch?v=cFlwQOPPkeg) | [slides](https://speakerdeck.com/alexeyklyukin/postgresql-high-availability-on-kubernetes-with-patroni)
+- "Building your own PostgreSQL-as-a-Service on Kubernetes" talk by Alexander Kukushkin, KubeCon NA 2018: [video](https://www.youtube.com/watch?v=G8MnpkbhClc) | [slides](https://static.sched.com/hosted_files/kccna18/1d/Building%20your%20own%20PostgreSQL-as-a-Service%20on%20Kubernetes.pdf)
 
-4. "Blue elephant on-demand: Postgres + Kubernetes" talk by Oleksii Kliukin and Jan Mussler, FOSDEM 2018: [video](https://fosdem.org/2018/schedule/event/blue_elephant_on_demand_postgres_kubernetes/) | [slides (pdf)](https://www.postgresql.eu/events/fosdem2018/sessions/session/1735/slides/59/FOSDEM%202018_%20Blue_Elephant_On_Demand.pdf)
+- "PostgreSQL and Kubernetes: DBaaS without a vendor-lock" talk by Oleksii Kliukin, PostgreSQL Sessions 2018: [video](https://www.youtube.com/watch?v=q26U2rQcqMw) | [slides](https://speakerdeck.com/alexeyklyukin/postgresql-and-kubernetes-dbaas-without-a-vendor-lock)
 
-5. "Kube-Native Postgres" talk by Josh Berkus, KubeCon 2017: [video](https://www.youtube.com/watch?v=Zn1vd7sQ_bc)
+- "PostgreSQL High Availability on Kubernetes with Patroni" talk by Oleksii Kliukin, Atmosphere 2018: [video](https://www.youtube.com/watch?v=cFlwQOPPkeg) | [slides](https://speakerdeck.com/alexeyklyukin/postgresql-high-availability-on-kubernetes-with-patroni)
+
+- "Blue elephant on-demand: Postgres + Kubernetes" talk by Oleksii Kliukin and Jan Mussler, FOSDEM 2018: [video](https://fosdem.org/2018/schedule/event/blue_elephant_on_demand_postgres_kubernetes/) | [slides (pdf)](https://www.postgresql.eu/events/fosdem2018/sessions/session/1735/slides/59/FOSDEM%202018_%20Blue_Elephant_On_Demand.pdf)
+
+- "Kube-Native Postgres" talk by Josh Berkus, KubeCon 2017: [video](https://www.youtube.com/watch?v=Zn1vd7sQ_bc)
+
+## Posts
+
+- "How to set up continuous backups and monitoring" by Pål Kristensen on [GitHub](https://github.com/zalando/postgres-operator/issues/858#issuecomment-608136253), Mar. 2020.
+
+- "Postgres on Kubernetes with the Zalando operator" by Vito Botta on [has_many :code](https://vitobotta.com/2020/02/05/postgres-kubernetes-zalando-operator/), Feb. 2020.
+
+- "Running PostgreSQL in Google Kubernetes Engine" by Kenneth Rørvik on [Repill Linpro](https://www.redpill-linpro.com/techblog/2019/09/28/postgres-in-kubernetes.html), Sep. 2019.
+
+- "Zalando Postgres Operator: One Year Later" by Sergey Dudoladov on [Open Source Zalando](https://opensource.zalando.com/blog/2018/11/postgres-operator/), Nov. 2018

docs/reference/cluster_manifest.md

@@ -111,12 +111,12 @@ These parameters are grouped directly under  the `spec` key in the manifest.
   value overrides the `pod_toleration` setting from the operator. Optional.
 
 * **podPriorityClassName**
-   a name of the [priority
-   class](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass)
-   that should be assigned to the cluster pods. When not specified, the value
-   is taken from the `pod_priority_class_name` operator parameter, if not set
-   then the default priority class is taken. The priority class itself must be
-   defined in advance. Optional.
+  a name of the [priority
+  class](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass)
+  that should be assigned to the cluster pods. When not specified, the value
+  is taken from the `pod_priority_class_name` operator parameter, if not set
+  then the default priority class is taken. The priority class itself must be
+  defined in advance. Optional.
 
 * **podAnnotations**
   A map of key value pairs that gets attached as [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/)
@@ -140,6 +140,11 @@ These parameters are grouped directly under  the `spec` key in the manifest.
   is `false`, then no volume will be mounted no matter how operator was
   configured (so you can override the operator configuration). Optional.
 
+* **enableConnectionPooler**
+  Tells the operator to create a connection pooler with a database. If this
+  field is true, a connection pooler deployment will be created even if
+  `connectionPooler` section is empty. Optional, not set by default.
+
 * **enableLogicalBackup**
   Determines if the logical backup of this cluster should be taken and uploaded
   to S3. Default: false. Optional.
@@ -149,6 +154,18 @@ These parameters are grouped directly under  the `spec` key in the manifest.
   [the reference schedule format](https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#schedule)
   into account. Optional. Default is: "30 00 \* \* \*"
 
+* **additionalVolumes**
+  List of additional volumes to mount in each container of the statefulset pod.
+  Each item must contain a `name`, `mountPath`, and `volumeSource` which is a
+  [kubernetes volumeSource](https://godoc.org/k8s.io/api/core/v1#VolumeSource).
+  It allows you to mount existing PersistentVolumeClaims, ConfigMaps and Secrets inside the StatefulSet.
+  Also an `emptyDir` volume can be shared between initContainer and statefulSet.
+  Additionaly, you can provide a `SubPath` for volume mount (a file in a configMap source volume, for example).
+  You can also specify in which container the additional Volumes will be mounted with the `targetContainers` array option.
+  If `targetContainers` is empty, additional volumes will be mounted only in the `postgres` container.
+  If you set the `all` special item, it will be mounted in all containers (postgres + sidecars).
+  Else you can set the list of target containers in which the additional volumes will be mounted (eg : postgres, telegraf)
+
 ## Postgres parameters
 
 Those parameters are grouped under the `postgresql` top-level key, which is
@@ -184,9 +201,9 @@ explanation of `ttl` and `loop_wait` parameters.
   ```
   hostssl all +pamrole all pam
   ```
-  , where pamrole is the name of the role for the pam authentication; any
-    custom `pg_hba` should include the pam line to avoid breaking pam
-    authentication. Optional.
+  where pamrole is the name of the role for the pam authentication; any
+  custom `pg_hba` should include the pam line to avoid breaking pam
+  authentication. Optional.
 
 * **ttl**
   Patroni `ttl` parameter value, optional. The default is set by the Spilo
@@ -212,6 +229,12 @@ explanation of `ttl` and `loop_wait` parameters.
   automatically created by Patroni for cluster members and permanent replication
   slots. Optional.
 
+* **synchronous_mode**
+  Patroni `synchronous_mode` parameter value. The default is set to `false`. Optional.
+  
+* **synchronous_mode_strict**
+  Patroni `synchronous_mode_strict` parameter value. Can be used in addition to `synchronous_mode`. The default is set to `false`. Optional.
+  
 ## Postgres container resources
 
 Those parameters define [CPU and memory requests and limits](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/)
@@ -359,3 +382,65 @@ CPU and memory limits for the sidecar container.
 * **memory**
   memory limits for the sidecar container. Optional, overrides the
   `default_memory_limits` operator configuration parameter. Optional.
+
+## Connection pooler
+
+Parameters are grouped under the `connectionPooler` top-level key and specify
+configuration for connection pooler. If this section is not empty, a connection
+pooler will be created for a database even if `enableConnectionPooler` is not
+present.
+
+* **numberOfInstances**
+  How many instances of connection pooler to create.
+
+* **schema**
+  Database schema to create for credentials lookup function.
+
+* **user**
+  User to create for connection pooler to be able to connect to a database.
+  You can also choose a role from the `users` section or a system user role.
+
+* **dockerImage**
+  Which docker image to use for connection pooler deployment.
+
+* **maxDBConnections**
+  How many connections the pooler can max hold. This value is divided among the
+  pooler pods.
+
+* **mode**
+  In which mode to run connection pooler, transaction or session.
+
+* **resources**
+  Resource configuration for connection pooler deployment.
+
+## Custom TLS certificates
+
+Those parameters are grouped under the `tls` top-level key.
+
+* **secretName**
+  By setting the `secretName` value, the cluster will switch to load the given
+  Kubernetes Secret into the container as a volume and uses that as the
+  certificate instead. It is up to the user to create and manage the
+  Kubernetes Secret either by hand or using a tool like the CertManager
+  operator.
+
+* **certificateFile**
+  Filename of the certificate. Defaults to "tls.crt".
+
+* **privateKeyFile**
+  Filename of the private key. Defaults to "tls.key".
+
+* **caFile**
+  Optional filename to the CA certificate (e.g. "ca.crt"). Useful when the
+  client connects with `sslmode=verify-ca` or `sslmode=verify-full`.
+  Default is empty.
+
+* **caSecretName**
+  By setting the `caSecretName` value, the ca certificate file defined by the
+  `caFile` will be fetched from this secret instead of `secretName` above.
+  This secret has to hold a file with that name in its root.
+
+  Optionally one can provide full path for any of them. By default it is
+  relative to the "/tls/", which is mount path of the tls secret.
+  If `caSecretName` is defined, the ca.crt path is relative to "/tlsca/",
+  otherwise to the same "/tls/".

docs/reference/command_line_and_environment.md

@@ -45,7 +45,7 @@ The following environment variables are accepted by the operator:
   all namespaces. Empty value defaults to the operator namespace. Overrides the
   `watched_namespace` operator parameter.
 
-* **SCALYR_API_KEY**
+* **SCALYR_API_KEY** (*deprecated*)
   the value of the Scalyr API key to supply to the pods. Overrides the
   `scalyr_api_key` operator parameter.
 

docs/reference/operator_parameters.md

@@ -75,11 +75,21 @@ Those are top-level keys, containing both leaf keys and groups.
   [OpenAPI v3 schema validation](https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#validation)
   The default is `true`.
 
+* **enable_lazy_spilo_upgrade**
+  Instruct operator to update only the statefulsets with the new image without immediately doing the rolling update. The assumption is pods will be re-started later with the new image, for example due to the node rotation.
+  The default is `false`.
+
 * **etcd_host**
   Etcd connection string for Patroni defined as `host:port`. Not required when
   Patroni native Kubernetes support is used. The default is empty (use
   Kubernetes-native DCS).
 
+* **kubernetes_use_configmaps**
+  Select if setup uses endpoints (default), or configmaps to manage leader when
+  DCS is kubernetes (not etcd or similar). In OpenShift it is not possible to
+  use endpoints option, and configmaps is required. By default,
+  `kubernetes_use_configmaps: false`, meaning endpoints will be used.
+
 * **docker_image**
   Spilo Docker image for Postgres instances. For production, don't rely on the
   default image, as it might be not the most up-to-date one. Instead, build
@@ -87,9 +97,18 @@ Those are top-level keys, containing both leaf keys and groups.
   repository](https://github.com/zalando/spilo).
 
 * **sidecar_docker_images**
-  a map of sidecar names to Docker images to run with Spilo. In case of the name
-  conflict with the definition in the cluster manifest the cluster-specific one
-  is preferred.
+  *deprecated*: use **sidecars** instead. A map of sidecar names to Docker
+  images to run with Spilo. In case of the name conflict with the definition in
+  the cluster manifest the cluster-specific one is preferred.
+
+* **sidecars**
+  a list of sidecars to run with Spilo, for any cluster (i.e. globally defined
+  sidecars). Each item in the list is of type
+  [Container](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#container-v1-core).
+  Globally defined sidecars can be overwritten by specifying a sidecar in the
+  Postgres manifest with the same name.
+  Note: This field is not part of the schema validation. If the container
+  specification is invalid, then the operator fails to create the statefulset.
 
 * **enable_shm_volume**
   Instruct operator to start any new database pod without limitations on shm
@@ -127,8 +146,9 @@ Those are top-level keys, containing both leaf keys and groups.
   at the cost of overprovisioning memory and potential scheduling problems for
   containers with high memory limits due to the lack of memory on Kubernetes
   cluster nodes. This affects all containers created by the operator (Postgres,
-  Scalyr sidecar, and other sidecars); to set resources for the operator's own
-  container, change the [operator deployment manually](../../manifests/postgres-operator.yaml#L20).
+  Scalyr sidecar, and other sidecars except **sidecars** defined in the operator
+  configuration); to set resources for the operator's own container, change the
+  [operator deployment manually](../../manifests/postgres-operator.yaml#L20).
   The default is `false`.
 
 ## Postgres users
@@ -180,6 +200,12 @@ configuration they are grouped under the `kubernetes` key.
   of a database created by the operator. If the annotation key is also provided
   by the database definition, the database definition value is used.
 
+* **downscaler_annotations**
+  An array of annotations that should be passed from Postgres CRD on to the
+  statefulset and, if exists, to the connection pooler deployment as well.
+  Regular expressions like `downscaler/*` etc. are also accepted. Can be used
+  with [kube-downscaler](https://github.com/hjacobs/kube-downscaler).
+
 * **watched_namespace**
   The operator watches for Postgres objects in the given namespace. If not
   specified, the value is taken from the operator namespace. A special `*`
@@ -200,12 +226,13 @@ configuration they are grouped under the `kubernetes` key.
   Default is true.
 
 * **enable_init_containers**
-  global option to allow for creating init containers to run actions before
-  Spilo is started. Default is true.
+  global option to allow for creating init containers in the cluster manifest to
+  run actions before Spilo is started. Default is true.
 
 * **enable_sidecars**
-  global option to allow for creating sidecar containers to run alongside Spilo
-  on the same pod. Default is true.
+  global option to allow for creating sidecar containers in the cluster manifest
+  to run alongside Spilo on the same pod. Globally defined sidecars are always
+  enabled. Default is true.
 
 * **secret_name_template**
   a template for the name of the database user secrets generated by the
@@ -221,11 +248,12 @@ configuration they are grouped under the `kubernetes` key.
   to the Postgres clusters after creation.
 
 * **oauth_token_secret_name**
-  a name of the secret containing the `OAuth2` token to pass to the teams API.
-  The default is `postgresql-operator`.
+  namespaced name of the secret containing the `OAuth2` token to pass to the
+  teams API. The default is `postgresql-operator`.
 
 * **infrastructure_roles_secret_name**
-  name of the secret containing infrastructure roles names and passwords.
+  namespaced name of the secret containing infrastructure roles names and
+  passwords.
 
 * **pod_role_label**
   name of the label assigned to the Postgres pods (and services/endpoints) by
@@ -262,11 +290,11 @@ configuration they are grouped under the `kubernetes` key.
   for details on taints and tolerations. The default is empty.
 
 * **pod_environment_configmap**
-  a name of the ConfigMap with environment variables to populate on every pod.
-  Right now this ConfigMap is searched in the namespace of the Postgres cluster.
-  All variables from that ConfigMap are injected to the pod's environment, on
-  conflicts they are overridden by the environment variables generated by the
-  operator. The default is empty.
+  namespaced name of the ConfigMap with environment variables to populate on
+  every pod. Right now this ConfigMap is searched in the namespace of the
+  Postgres cluster. All variables from that ConfigMap are injected to the pod's
+  environment, on conflicts they are overridden by the environment variables
+  generated by the operator. The default is empty.
 
 * **pod_priority_class_name**
   a name of the [priority class](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass)
@@ -284,12 +312,12 @@ configuration they are grouped under the `kubernetes` key.
   used for AWS volume resizing and not required if you don't need that
   capability. The default is `false`.
 
- * **master_pod_move_timeout**
-   The period of time to wait for the success of migration of master pods from
-   an unschedulable node. The migration includes Patroni switchovers to
-   respective replicas on healthy nodes. The situation where master pods still
-   exist on the old node after this timeout expires has to be fixed manually.
-   The default is 20 minutes.
+* **master_pod_move_timeout**
+  The period of time to wait for the success of migration of master pods from
+  an unschedulable node. The migration includes Patroni switchovers to
+  respective replicas on healthy nodes. The situation where master pods still
+  exist on the old node after this timeout expires has to be fixed manually.
+  The default is 20 minutes.
 
 * **enable_pod_antiaffinity**
   toggles [pod anti affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/)
@@ -472,7 +500,7 @@ grouped under the `logical_backup` key.
   When using non-AWS S3 storage, endpoint can be set as a ENV variable. The default is empty.
 
 * **logical_backup_s3_sse**
-  Specify server side encription that S3 storage is using. If empty string
+  Specify server side encryption that S3 storage is using. If empty string
   is specified, no argument will be passed to `aws s3` command. Default: "AES256".
 
 * **logical_backup_s3_access_key_id**
@@ -569,11 +597,12 @@ configuration they are grouped under the `logging_rest_api` key.
 * **cluster_history_entries**
   number of entries in the cluster history ring buffer. The default is `1000`.
 
-## Scalyr options
+## Scalyr options (*deprecated*)
 
 Those parameters define the resource requests/limits and properties of the
 scalyr sidecar. In the CRD-based configuration they are grouped under the
-`scalyr` key.
+`scalyr` key. Note, that this section is deprecated. Instead, define Scalyr as
+a global sidecar under the `sidecars` key in the configuration.
 
 * **scalyr_api_key**
   API key for the Scalyr sidecar. The default is empty.
@@ -595,3 +624,43 @@ scalyr sidecar. In the CRD-based configuration they are grouped under the
 
 * **scalyr_memory_limit**
   Memory limit value for the Scalyr sidecar. The default is `500Mi`.
+
+## Connection pooler configuration
+
+Parameters are grouped under the `connection_pooler` top-level key and specify
+default configuration for connection pooler, if a postgres manifest requests it
+but do not specify some of the parameters. All of them are optional with the
+operator being able to provide some reasonable defaults.
+
+* **connection_pooler_number_of_instances**
+  How many instances of connection pooler to create. Default is 2 which is also
+  the required minimum.
+
+* **connection_pooler_schema**
+  Database schema to create for credentials lookup function to be used by the
+  connection pooler. Is is created in every database of the Postgres cluster.
+  You can also choose an existing schema. Default schema is `pooler`.
+
+* **connection_pooler_user**
+  User to create for connection pooler to be able to connect to a database.
+  You can also choose an existing role, but make sure it has the `LOGIN`
+  privilege. Default role is `pooler`.
+
+* **connection_pooler_image**
+  Docker image to use for connection pooler deployment.
+  Default: "registry.opensource.zalan.do/acid/pgbouncer"
+
+* **connection_pooler_max_db_connections**
+  How many connections the pooler can max hold. This value is divided among the
+  pooler pods. Default is 60 which will make up 30 connections per pod for the
+  default setup with two instances.
+
+* **connection_pooler_mode**
+  Default pooler mode, `session` or `transaction`. Default is `transaction`.
+
+* **connection_pooler_default_cpu_request**
+  **connection_pooler_default_memory_reques**
+  **connection_pooler_default_cpu_limit**
+  **connection_pooler_default_memory_limit**
+  Default resource configuration for connection pooler deployment. The internal
+  default for memory request and limit is `100Mi`, for CPU it is `500m` and `1`.

docs/user.md

@@ -30,7 +30,7 @@ spec:
   databases:
     foo: zalando
   postgresql:
-    version: "11"
+    version: "12"
 ```
 
 Once you cloned the Postgres Operator [repository](https://github.com/zalando/postgres-operator)
@@ -53,8 +53,19 @@ them.
 
 ## Watch pods being created
 
+Check if the database pods are coming up. Use the label `application=spilo` to
+filter and list the label `spilo-role` to see when the master is promoted and
+replicas get their labels.
+
 ```bash
-kubectl get pods -w --show-labels
+kubectl get pods -l application=spilo -L spilo-role -w
+```
+
+The operator also emits K8s events to the Postgresql CRD which can be inspected
+in the operator logs or with:
+
+```bash
+kubectl describe postgresql acid-minimal-cluster
 ```
 
 ## Connect to PostgreSQL
@@ -94,7 +105,10 @@ created on every cluster managed by the operator.
 * `teams API roles`: automatically create users for every member of the team
 owning the database cluster.
 
-In the next sections, we will cover those use cases in more details.
+In the next sections, we will cover those use cases in more details. Note, that
+the Postgres Operator can also create databases with pre-defined owner, reader
+and writer roles which saves you the manual setup. Read more in the next
+chapter.
 
 ### Manifest roles
 
@@ -216,6 +230,166 @@ to choose superusers, group roles, [PAM configuration](https://github.com/CyberD
 etc. An OAuth2 token can be passed to the Teams API via a secret. The name for
 this secret is configurable with the `oauth_token_secret_name` parameter.
 
+## Prepared databases with roles and default privileges
+
+The `users` section in the manifests only allows for creating database roles
+with global privileges. Fine-grained data access control or role membership can
+not be defined and must be set up by the user in the database. But, the Postgres
+Operator offers a separate section to specify `preparedDatabases` that will be
+created with pre-defined owner, reader and writer roles for each individual
+database and, optionally, for each database schema, too. `preparedDatabases`
+also enable users to specify PostgreSQL extensions that shall be created in a
+given database schema.
+
+### Default database and schema
+
+A prepared database is already created by adding an empty `preparedDatabases`
+section to the manifest. The database will then be called like the Postgres
+cluster manifest (`-` are replaced with `_`) and will also contain a schema
+called `data`.
+
+```yaml
+spec:
+  preparedDatabases: {}
+```
+
+### Default NOLOGIN roles
+
+Given an example with a specified database and schema:
+
+```yaml
+spec:
+  preparedDatabases:
+    foo:
+      schemas:
+        bar: {}
+```
+
+Postgres Operator will create the following NOLOGIN roles:
+
+| Role name      | Member of      | Admin         |
+| -------------- | -------------- | ------------- |
+| foo_owner      |                | admin         |
+| foo_reader     |                | foo_owner     |
+| foo_writer     | foo_reader     | foo_owner     |
+| foo_bar_owner  |                | foo_owner     |
+| foo_bar_reader |                | foo_bar_owner |
+| foo_bar_writer | foo_bar_reader | foo_bar_owner |
+
+The `<dbname>_owner` role is the database owner and should be used when creating
+new database objects. All members of the `admin` role, e.g. teams API roles, can
+become the owner with the `SET ROLE` command. [Default privileges](https://www.postgresql.org/docs/12/sql-alterdefaultprivileges.html)
+are configured for the owner role so that the `<dbname>_reader` role
+automatically gets read-access (SELECT) to new tables and sequences and the
+`<dbname>_writer` receives write-access (INSERT, UPDATE, DELETE on tables,
+USAGE and UPDATE on sequences). Both get USAGE on types and EXECUTE on
+functions.
+
+The same principle applies for database schemas which are owned by the
+`<dbname>_<schema>_owner` role. `<dbname>_<schema>_reader` is read-only,
+`<dbname>_<schema>_writer` has write access and inherit reading from the reader
+role. Note, that the `<dbname>_*` roles have access incl. default privileges on
+all schemas, too. If you don't need the dedicated schema roles - i.e. you only
+use one schema - you can disable the creation like this:
+
+```yaml
+spec:
+  preparedDatabases:
+    foo:
+      schemas:
+        bar:
+          defaultRoles: false
+```
+
+Then, the schemas are owned by the database owner, too.
+
+### Default LOGIN roles
+
+The roles described in the previous paragraph can be granted to LOGIN roles from
+the `users` section in the manifest. Optionally, the Postgres Operator can also
+create default LOGIN roles for the database an each schema individually. These
+roles will get the `_user` suffix and they inherit all rights from their NOLOGIN
+counterparts.
+
+| Role name           | Member of      | Admin         |
+| ------------------- | -------------- | ------------- |
+| foo_owner_user      | foo_owner      | admin         |
+| foo_reader_user     | foo_reader     | foo_owner     |
+| foo_writer_user     | foo_writer     | foo_owner     |
+| foo_bar_owner_user  | foo_bar_owner  | foo_owner     |
+| foo_bar_reader_user | foo_bar_reader | foo_bar_owner |
+| foo_bar_writer_user | foo_bar_writer | foo_bar_owner |
+
+These default users are enabled in the manifest with the `defaultUsers` flag:
+
+```yaml
+spec:
+  preparedDatabases:
+    foo:
+      defaultUsers: true
+      schemas:
+        bar:
+          defaultUsers: true
+```
+
+### Database extensions
+
+Prepared databases also allow for creating Postgres extensions. They will be
+created by the database owner in the specified schema.
+
+```yaml
+spec:
+  preparedDatabases:
+    foo:
+      extensions:
+        pg_partman: public
+        postgis: data
+```
+
+Some extensions require SUPERUSER rights on creation unless they are not
+whitelisted by the [pgextwlist](https://github.com/dimitri/pgextwlist)
+extension, that is shipped with the Spilo image. To see which extensions are
+on the list check the `extwlist.extension` parameter in the postgresql.conf
+file.
+
+```bash
+SHOW extwlist.extensions;
+```
+
+Make sure that `pgextlist` is also listed under `shared_preload_libraries` in
+the PostgreSQL configuration. Then the database owner should be able to create
+the extension specified in the manifest.
+
+### From `databases` to `preparedDatabases`
+
+If you wish to create the role setup described above for databases listed under
+the `databases` key, you have to make sure that the owner role follows the
+`<dbname>_owner` naming convention of `preparedDatabases`. As roles are synced
+first, this can be done with one edit:
+
+```yaml
+# before
+spec:
+  databases:
+    foo: db_owner
+
+# after
+spec:
+  databases:
+    foo: foo_owner
+  preparedDatabases:
+    foo:
+      schemas:
+        my_existing_schema: {}
+```
+
+Adding existing database schemas to the manifest to create roles for them as
+well is up the user and not done by the operator. Remember that if you don't
+specify any schema a new database schema called `data` will be created. When
+everything got synced (roles, schemas, extensions), you are free to remove the
+database from the `databases` section. Note, that the operator does not delete
+database objects or revoke privileges when removed from the manifest.
+
 ## Resource definition
 
 The compute resources to be used for the Postgres containers in the pods can be
@@ -442,6 +616,8 @@ The PostgreSQL volume is shared with sidecars and is mounted at
 specified but globally disabled in the configuration. The `enable_sidecars`
 option must be set to `true`.
 
+If you want to add a sidecar to every cluster managed by the operator, you can specify it in the [operator configuration](administrator.md#sidecars-for-postgres-clusters) instead.
+
 ## InitContainers Support
 
 Each cluster can specify arbitrary init containers to run. These containers can
@@ -511,3 +687,134 @@ monitoring is outside the scope of operator responsibilities. See
 [configuration reference](reference/cluster_manifest.md) and
 [administrator documentation](administrator.md) for details on how backups are
 executed.
+
+## Connection pooler
+
+The operator can create a database side connection pooler for those applications
+where an application side pooler is not feasible, but a number of connections is
+high. To create a connection pooler together with a database, modify the
+manifest:
+
+```yaml
+spec:
+  enableConnectionPooler: true
+```
+
+This will tell the operator to create a connection pooler with default
+configuration, through which one can access the master via a separate service
+`{cluster-name}-pooler`. In most of the cases the
+[default configuration](reference/operator_parameters.md#connection-pooler-configuration)
+should be good enough. To configure a new connection pooler individually for
+each Postgres cluster, specify:
+
+```
+spec:
+  connectionPooler:
+    # how many instances of connection pooler to create
+    numberOfInstances: 2
+
+    # in which mode to run, session or transaction
+    mode: "transaction"
+
+    # schema, which operator will create in each database
+    # to install credentials lookup function for connection pooler
+    schema: "pooler"
+
+    # user, which operator will create for connection pooler
+    user: "pooler"
+
+    # resources for each instance
+    resources:
+      requests:
+        cpu: 500m
+        memory: 100Mi
+      limits:
+        cpu: "1"
+        memory: 100Mi
+```
+
+The `enableConnectionPooler` flag is not required when the `connectionPooler`
+section is present in the manifest. But, it can be used to disable/remove the
+pooler while keeping its configuration.
+
+By default, [`PgBouncer`](https://www.pgbouncer.org/) is used as connection pooler.
+To find out about pool modes read the `PgBouncer` [docs](https://www.pgbouncer.org/config.html#pooler_mode)
+(but it should be the general approach between different implementation).
+
+Note, that using `PgBouncer` a meaningful resource CPU limit should be 1 core
+or less (there is a way to utilize more than one, but in K8s it's easier just to
+spin up more instances).
+
+## Custom TLS certificates
+
+By default, the Spilo image generates its own TLS certificate during startup.
+However, this certificate cannot be verified and thus doesn't protect from
+active MITM attacks. In this section we show how to specify a custom TLS
+certificate which is mounted in the database pods via a K8s Secret.
+
+Before applying these changes, in k8s the operator must also be configured with
+the `spilo_fsgroup` set to the GID matching the postgres user group. If you
+don't know the value, use `103` which is the GID from the default Spilo image
+(`spilo_fsgroup=103` in the cluster request spec).
+
+OpenShift allocates the users and groups dynamically (based on scc), and their
+range is different in every namespace. Due to this dynamic behaviour, it's not
+trivial to know at deploy time the uid/gid of the user in the cluster.
+Therefore, instead of using a global `spilo_fsgroup` setting, use the
+`spiloFSGroup` field per Postgres cluster.
+
+Upload the cert as a kubernetes secret:
+```sh
+kubectl create secret tls pg-tls \
+  --key pg-tls.key \
+  --cert pg-tls.crt
+```
+
+When doing client auth, CA can come optionally from the same secret:
+```sh
+kubectl create secret generic pg-tls \
+  --from-file=tls.crt=server.crt \
+  --from-file=tls.key=server.key \
+  --from-file=ca.crt=ca.crt
+```
+
+Then configure the postgres resource with the TLS secret:
+
+```yaml
+apiVersion: "acid.zalan.do/v1"
+kind: postgresql
+
+metadata:
+  name: acid-test-cluster
+spec:
+  tls:
+    secretName: "pg-tls"
+    caFile: "ca.crt" # add this if the secret is configured with a CA
+```
+
+Optionally, the CA can be provided by a different secret:
+```sh
+kubectl create secret generic pg-tls-ca \
+  --from-file=ca.crt=ca.crt
+```
+
+Then configure the postgres resource with the TLS secret:
+
+```yaml
+apiVersion: "acid.zalan.do/v1"
+kind: postgresql
+
+metadata:
+  name: acid-test-cluster
+spec:
+  tls:
+    secretName: "pg-tls"    # this should hold tls.key and tls.crt
+    caSecretName: "pg-tls-ca" # this should hold ca.crt
+    caFile: "ca.crt" # add this if the secret is configured with a CA
+```
+
+Alternatively, it is also possible to use
+[cert-manager](https://cert-manager.io/docs/) to generate these secrets.
+
+Certificate rotation is handled in the Spilo image which checks every 5
+minutes if the certificates have changed and reloads postgres accordingly.

e2e/Makefile

@@ -44,5 +44,5 @@ tools: docker
 	# install pinned version of 'kind'
 	GO111MODULE=on go get sigs.k8s.io/kind@v0.5.1
 
-test:
+e2etest:
 	./run.sh

e2e/tests/test_e2e.py

@@ -9,6 +9,10 @@ import yaml
 from kubernetes import client, config
 
 
+def to_selector(labels):
+    return ",".join(["=".join(l) for l in labels.items()])
+
+
 class EndToEndTestCase(unittest.TestCase):
     '''
     Test interaction of the operator with multiple K8s components.
@@ -47,7 +51,8 @@ class EndToEndTestCase(unittest.TestCase):
         for filename in ["operator-service-account-rbac.yaml",
                          "configmap.yaml",
                          "postgres-operator.yaml"]:
-            k8s.create_with_kubectl("manifests/" + filename)
+            result = k8s.create_with_kubectl("manifests/" + filename)
+            print("stdout: {}, stderr: {}".format(result.stdout, result.stderr))
 
         k8s.wait_for_operator_pod_start()
 
@@ -55,8 +60,91 @@ class EndToEndTestCase(unittest.TestCase):
             'default', label_selector='name=postgres-operator').items[0].spec.containers[0].image
         print("Tested operator image: {}".format(actual_operator_image))  # shows up after tests finish
 
-        k8s.create_with_kubectl("manifests/minimal-postgres-manifest.yaml")
-        k8s.wait_for_pod_start('spilo-role=master')
+        result = k8s.create_with_kubectl("manifests/minimal-postgres-manifest.yaml")
+        print('stdout: {}, stderr: {}'.format(result.stdout, result.stderr))
+        try:
+            k8s.wait_for_pod_start('spilo-role=master')
+            k8s.wait_for_pod_start('spilo-role=replica')
+        except timeout_decorator.TimeoutError:
+            print('Operator log: {}'.format(k8s.get_operator_log()))
+            raise
+
+    @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
+    def test_enable_disable_connection_pooler(self):
+        '''
+        For a database without connection pooler, then turns it on, scale up,
+        turn off and on again. Test with different ways of doing this (via
+        enableConnectionPooler or connectionPooler configuration section). At
+        the end turn connection pooler off to not interfere with other tests.
+        '''
+        k8s = self.k8s
+        service_labels = {
+            'cluster-name': 'acid-minimal-cluster',
+        }
+        pod_labels = dict({
+            'connection-pooler': 'acid-minimal-cluster-pooler',
+        })
+
+        pod_selector = to_selector(pod_labels)
+        service_selector = to_selector(service_labels)
+
+        try:
+            # enable connection pooler
+            k8s.api.custom_objects_api.patch_namespaced_custom_object(
+                'acid.zalan.do', 'v1', 'default',
+                'postgresqls', 'acid-minimal-cluster',
+                {
+                    'spec': {
+                        'enableConnectionPooler': True,
+                    }
+                })
+            k8s.wait_for_pod_start(pod_selector)
+
+            pods = k8s.api.core_v1.list_namespaced_pod(
+                'default', label_selector=pod_selector
+            ).items
+
+            self.assertTrue(pods, 'No connection pooler pods')
+
+            k8s.wait_for_service(service_selector)
+            services = k8s.api.core_v1.list_namespaced_service(
+                'default', label_selector=service_selector
+            ).items
+            services = [
+                s for s in services
+                if s.metadata.name.endswith('pooler')
+            ]
+
+            self.assertTrue(services, 'No connection pooler service')
+
+            # scale up connection pooler deployment
+            k8s.api.custom_objects_api.patch_namespaced_custom_object(
+                'acid.zalan.do', 'v1', 'default',
+                'postgresqls', 'acid-minimal-cluster',
+                {
+                    'spec': {
+                        'connectionPooler': {
+                            'numberOfInstances': 2,
+                        },
+                    }
+                })
+
+            k8s.wait_for_running_pods(pod_selector, 2)
+
+            # turn it off, keeping configuration section
+            k8s.api.custom_objects_api.patch_namespaced_custom_object(
+                'acid.zalan.do', 'v1', 'default',
+                'postgresqls', 'acid-minimal-cluster',
+                {
+                    'spec': {
+                        'enableConnectionPooler': False,
+                    }
+                })
+            k8s.wait_for_pods_to_stop(pod_selector)
+
+        except timeout_decorator.TimeoutError:
+            print('Operator log: {}'.format(k8s.get_operator_log()))
+            raise
 
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
     def test_enable_load_balancer(self):
@@ -65,7 +153,7 @@ class EndToEndTestCase(unittest.TestCase):
         '''
 
         k8s = self.k8s
-        cluster_label = 'cluster-name=acid-minimal-cluster'
+        cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster'
 
         # enable load balancer services
         pg_patch_enable_lbs = {
@@ -108,139 +196,64 @@ class EndToEndTestCase(unittest.TestCase):
                          "Expected ClusterIP service type for replica, found {}".format(repl_svc_type))
 
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
-    def test_min_resource_limits(self):
+    def test_lazy_spilo_upgrade(self):
         '''
-        Lower resource limits below configured minimum and let operator fix it
-        '''
-        k8s = self.k8s
-        cluster_label = 'cluster-name=acid-minimal-cluster'
-        _, failover_targets = k8s.get_pg_nodes(cluster_label)
+        Test lazy upgrade for the Spilo image: operator changes a stateful set but lets pods run with the old image
+        until they are recreated for reasons other than operator's activity. That works because the operator configures
+        stateful sets to use "onDelete" pod update policy.
 
-        # configure minimum boundaries for CPU and memory limits
-        minCPULimit = '500m'
-        minMemoryLimit = '500Mi'
-        patch_min_resource_limits = {
+        The test covers:
+        1) enabling lazy upgrade in existing operator deployment
+        2) forcing the normal rolling upgrade by changing the operator configmap and restarting its pod
+        '''
+
+        k8s = self.k8s
+
+        # update docker image in config and enable the lazy upgrade
+        conf_image = "registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p114"
+        patch_lazy_spilo_upgrade = {
             "data": {
-                "min_cpu_limit": minCPULimit,
-                "min_memory_limit": minMemoryLimit
+                "docker_image": conf_image,
+                "enable_lazy_spilo_upgrade": "true"
             }
         }
-        k8s.update_config(patch_min_resource_limits)
+        k8s.update_config(patch_lazy_spilo_upgrade)
 
-        # lower resource limits below minimum
-        pg_patch_resources = {
-            "spec": {
-                "resources": {
-                    "requests": {
-                        "cpu": "10m",
-                        "memory": "50Mi"
-                    },
-                    "limits": {
-                        "cpu": "200m",
-                        "memory": "200Mi"
-                    }
-                }
+        pod0 = 'acid-minimal-cluster-0'
+        pod1 = 'acid-minimal-cluster-1'
+
+        # restart the pod to get a container with the new image
+        k8s.api.core_v1.delete_namespaced_pod(pod0, 'default')
+        time.sleep(60)
+
+        # lazy update works if the restarted pod and older pods run different Spilo versions
+        new_image = k8s.get_effective_pod_image(pod0)
+        old_image = k8s.get_effective_pod_image(pod1)
+        self.assertNotEqual(new_image, old_image, "Lazy updated failed: pods have the same image {}".format(new_image))
+
+        # sanity check
+        assert_msg = "Image {} of a new pod differs from {} in operator conf".format(new_image, conf_image)
+        self.assertEqual(new_image, conf_image, assert_msg)
+
+        # clean up
+        unpatch_lazy_spilo_upgrade = {
+            "data": {
+                "enable_lazy_spilo_upgrade": "false",
             }
         }
-        k8s.api.custom_objects_api.patch_namespaced_custom_object(
-            "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_resources)
-        k8s.wait_for_master_failover(failover_targets)
+        k8s.update_config(unpatch_lazy_spilo_upgrade)
 
-        pods = k8s.api.core_v1.list_namespaced_pod(
-            'default', label_selector='spilo-role=master,' + cluster_label).items
-        self.assert_master_is_unique()
-        masterPod = pods[0]
+        # at this point operator will complete the normal rolling upgrade
+        # so we additonally test if disabling the lazy upgrade - forcing the normal rolling upgrade - works
 
-        self.assertEqual(masterPod.spec.containers[0].resources.limits['cpu'], minCPULimit,
-                         "Expected CPU limit {}, found {}"
-                         .format(minCPULimit, masterPod.spec.containers[0].resources.limits['cpu']))
-        self.assertEqual(masterPod.spec.containers[0].resources.limits['memory'], minMemoryLimit,
-                         "Expected memory limit {}, found {}"
-                         .format(minMemoryLimit, masterPod.spec.containers[0].resources.limits['memory']))
+        # XXX there is no easy way to wait until the end of Sync()
+        time.sleep(60)
 
-    @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
-    def test_multi_namespace_support(self):
-        '''
-        Create a customized Postgres cluster in a non-default namespace.
-        '''
-        k8s = self.k8s
+        image0 = k8s.get_effective_pod_image(pod0)
+        image1 = k8s.get_effective_pod_image(pod1)
 
-        with open("manifests/complete-postgres-manifest.yaml", 'r+') as f:
-            pg_manifest = yaml.safe_load(f)
-            pg_manifest["metadata"]["namespace"] = self.namespace
-            yaml.dump(pg_manifest, f, Dumper=yaml.Dumper)
-
-        k8s.create_with_kubectl("manifests/complete-postgres-manifest.yaml")
-        k8s.wait_for_pod_start("spilo-role=master", self.namespace)
-        self.assert_master_is_unique(self.namespace, "acid-test-cluster")
-
-    @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
-    def test_scaling(self):
-        '''
-           Scale up from 2 to 3 and back to 2 pods by updating the Postgres manifest at runtime.
-        '''
-        k8s = self.k8s
-        labels = "cluster-name=acid-minimal-cluster"
-
-        k8s.wait_for_pg_to_scale(3)
-        self.assertEqual(3, k8s.count_pods_with_label(labels))
-        self.assert_master_is_unique()
-
-        k8s.wait_for_pg_to_scale(2)
-        self.assertEqual(2, k8s.count_pods_with_label(labels))
-        self.assert_master_is_unique()
-
-    @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
-    def test_taint_based_eviction(self):
-        '''
-           Add taint "postgres=:NoExecute" to node with master. This must cause a failover.
-        '''
-        k8s = self.k8s
-        cluster_label = 'cluster-name=acid-minimal-cluster'
-
-        # get nodes of master and replica(s) (expected target of new master)
-        current_master_node, failover_targets = k8s.get_pg_nodes(cluster_label)
-        num_replicas = len(failover_targets)
-
-        # if all pods live on the same node, failover will happen to other worker(s)
-        failover_targets = [x for x in failover_targets if x != current_master_node]
-        if len(failover_targets) == 0:
-            nodes = k8s.api.core_v1.list_node()
-            for n in nodes.items:
-                if "node-role.kubernetes.io/master" not in n.metadata.labels and n.metadata.name != current_master_node:
-                    failover_targets.append(n.metadata.name)
-
-        # taint node with postgres=:NoExecute to force failover
-        body = {
-            "spec": {
-                "taints": [
-                    {
-                        "effect": "NoExecute",
-                        "key": "postgres"
-                    }
-                ]
-            }
-        }
-
-        # patch node and test if master is failing over to one of the expected nodes
-        k8s.api.core_v1.patch_node(current_master_node, body)
-        k8s.wait_for_master_failover(failover_targets)
-        k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label)
-
-        new_master_node, new_replica_nodes = k8s.get_pg_nodes(cluster_label)
-        self.assertNotEqual(current_master_node, new_master_node,
-                            "Master on {} did not fail over to one of {}".format(current_master_node, failover_targets))
-        self.assertEqual(num_replicas, len(new_replica_nodes),
-                         "Expected {} replicas, found {}".format(num_replicas, len(new_replica_nodes)))
-        self.assert_master_is_unique()
-
-        # undo the tainting
-        body = {
-            "spec": {
-                "taints": []
-            }
-        }
-        k8s.api.core_v1.patch_node(new_master_node, body)
+        assert_msg = "Disabling lazy upgrade failed: pods still have different images {} and {}".format(image0, image1)
+        self.assertEqual(image0, image1, assert_msg)
 
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
     def test_logical_backup_cron_job(self):
@@ -306,6 +319,136 @@ class EndToEndTestCase(unittest.TestCase):
         self.assertEqual(0, len(jobs),
                          "Expected 0 logical backup jobs, found {}".format(len(jobs)))
 
+    @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
+    def test_min_resource_limits(self):
+        '''
+        Lower resource limits below configured minimum and let operator fix it
+        '''
+        k8s = self.k8s
+        cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster'
+        labels = 'spilo-role=master,' + cluster_label
+        _, failover_targets = k8s.get_pg_nodes(cluster_label)
+
+        # configure minimum boundaries for CPU and memory limits
+        minCPULimit = '500m'
+        minMemoryLimit = '500Mi'
+        patch_min_resource_limits = {
+            "data": {
+                "min_cpu_limit": minCPULimit,
+                "min_memory_limit": minMemoryLimit
+            }
+        }
+        k8s.update_config(patch_min_resource_limits)
+
+        # lower resource limits below minimum
+        pg_patch_resources = {
+            "spec": {
+                "resources": {
+                    "requests": {
+                        "cpu": "10m",
+                        "memory": "50Mi"
+                    },
+                    "limits": {
+                        "cpu": "200m",
+                        "memory": "200Mi"
+                    }
+                }
+            }
+        }
+        k8s.api.custom_objects_api.patch_namespaced_custom_object(
+            "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_resources)
+        k8s.wait_for_pod_failover(failover_targets, labels)
+        k8s.wait_for_pod_start('spilo-role=replica')
+
+        pods = k8s.api.core_v1.list_namespaced_pod(
+            'default', label_selector=labels).items
+        self.assert_master_is_unique()
+        masterPod = pods[0]
+
+        self.assertEqual(masterPod.spec.containers[0].resources.limits['cpu'], minCPULimit,
+                         "Expected CPU limit {}, found {}"
+                         .format(minCPULimit, masterPod.spec.containers[0].resources.limits['cpu']))
+        self.assertEqual(masterPod.spec.containers[0].resources.limits['memory'], minMemoryLimit,
+                         "Expected memory limit {}, found {}"
+                         .format(minMemoryLimit, masterPod.spec.containers[0].resources.limits['memory']))
+
+    @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
+    def test_multi_namespace_support(self):
+        '''
+        Create a customized Postgres cluster in a non-default namespace.
+        '''
+        k8s = self.k8s
+
+        with open("manifests/complete-postgres-manifest.yaml", 'r+') as f:
+            pg_manifest = yaml.safe_load(f)
+            pg_manifest["metadata"]["namespace"] = self.namespace
+            yaml.dump(pg_manifest, f, Dumper=yaml.Dumper)
+
+        k8s.create_with_kubectl("manifests/complete-postgres-manifest.yaml")
+        k8s.wait_for_pod_start("spilo-role=master", self.namespace)
+        self.assert_master_is_unique(self.namespace, "acid-test-cluster")
+
+    @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
+    def test_node_readiness_label(self):
+        '''
+           Remove node readiness label from master node. This must cause a failover.
+        '''
+        k8s = self.k8s
+        cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster'
+        readiness_label = 'lifecycle-status'
+        readiness_value = 'ready'
+
+        # get nodes of master and replica(s) (expected target of new master)
+        current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label)
+        num_replicas = len(current_replica_nodes)
+        failover_targets = self.get_failover_targets(current_master_node, current_replica_nodes)
+
+        # add node_readiness_label to potential failover nodes
+        patch_readiness_label = {
+            "metadata": {
+                "labels": {
+                    readiness_label: readiness_value
+                }
+            }
+        }
+        for failover_target in failover_targets:
+            k8s.api.core_v1.patch_node(failover_target, patch_readiness_label)
+
+        # define node_readiness_label in config map which should trigger a failover of the master
+        patch_readiness_label_config = {
+            "data": {
+                "node_readiness_label": readiness_label + ':' + readiness_value,
+            }
+        }
+        k8s.update_config(patch_readiness_label_config)
+        new_master_node, new_replica_nodes = self.assert_failover(
+            current_master_node, num_replicas, failover_targets, cluster_label)
+
+        # patch also node where master ran before
+        k8s.api.core_v1.patch_node(current_master_node, patch_readiness_label)
+
+        # wait a little before proceeding with the pod distribution test
+        time.sleep(30)
+
+        # toggle pod anti affinity to move replica away from master node
+        self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label)
+
+    @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
+    def test_scaling(self):
+        '''
+           Scale up from 2 to 3 and back to 2 pods by updating the Postgres manifest at runtime.
+        '''
+        k8s = self.k8s
+        labels = "application=spilo,cluster-name=acid-minimal-cluster"
+
+        k8s.wait_for_pg_to_scale(3)
+        self.assertEqual(3, k8s.count_pods_with_label(labels))
+        self.assert_master_is_unique()
+
+        k8s.wait_for_pg_to_scale(2)
+        self.assertEqual(2, k8s.count_pods_with_label(labels))
+        self.assert_master_is_unique()
+
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
     def test_service_annotations(self):
         '''
@@ -322,21 +465,24 @@ class EndToEndTestCase(unittest.TestCase):
         pg_patch_custom_annotations = {
             "spec": {
                 "serviceAnnotations": {
-                    "annotation.key": "value"
+                    "annotation.key": "value",
+                    "foo": "bar",
                 }
             }
         }
         k8s.api.custom_objects_api.patch_namespaced_custom_object(
             "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_custom_annotations)
 
+        # wait a little before proceeding
+        time.sleep(30)
         annotations = {
             "annotation.key": "value",
             "foo": "bar",
         }
         self.assertTrue(k8s.check_service_annotations(
-            "cluster-name=acid-service-annotations,spilo-role=master", annotations))
+            "cluster-name=acid-minimal-cluster,spilo-role=master", annotations))
         self.assertTrue(k8s.check_service_annotations(
-            "cluster-name=acid-service-annotations,spilo-role=replica", annotations))
+            "cluster-name=acid-minimal-cluster,spilo-role=replica", annotations))
 
         # clean up
         unpatch_custom_service_annotations = {
@@ -346,18 +492,155 @@ class EndToEndTestCase(unittest.TestCase):
         }
         k8s.update_config(unpatch_custom_service_annotations)
 
+    @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
+    def test_statefulset_annotation_propagation(self):
+        '''
+           Inject annotation to Postgresql CRD and check it's propagation to stateful set
+        '''
+        k8s = self.k8s
+        cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster'
+
+        patch_sset_propagate_annotations = {
+            "data": {
+                "downscaler_annotations": "deployment-time,downscaler/*",
+            }
+        }
+        k8s.update_config(patch_sset_propagate_annotations)
+
+        pg_crd_annotations = {
+            "metadata": {
+                "annotations": {
+                    "deployment-time": "2020-04-30 12:00:00",
+                    "downscaler/downtime_replicas": "0",
+                },
+            }
+        }
+        k8s.api.custom_objects_api.patch_namespaced_custom_object(
+            "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_crd_annotations)
+
+        # wait a little before proceeding
+        time.sleep(60)
+        annotations = {
+            "deployment-time": "2020-04-30 12:00:00",
+            "downscaler/downtime_replicas": "0",
+        }
+        self.assertTrue(k8s.check_statefulset_annotations(cluster_label, annotations))
+
+    @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
+    def test_taint_based_eviction(self):
+        '''
+           Add taint "postgres=:NoExecute" to node with master. This must cause a failover.
+        '''
+        k8s = self.k8s
+        cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster'
+
+        # get nodes of master and replica(s) (expected target of new master)
+        current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label)
+        num_replicas = len(current_replica_nodes)
+        failover_targets = self.get_failover_targets(current_master_node, current_replica_nodes)
+
+        # taint node with postgres=:NoExecute to force failover
+        body = {
+            "spec": {
+                "taints": [
+                    {
+                        "effect": "NoExecute",
+                        "key": "postgres"
+                    }
+                ]
+            }
+        }
+
+        # patch node and test if master is failing over to one of the expected nodes
+        k8s.api.core_v1.patch_node(current_master_node, body)
+        new_master_node, new_replica_nodes = self.assert_failover(
+            current_master_node, num_replicas, failover_targets, cluster_label)
+
+        # add toleration to pods
+        patch_toleration_config = {
+            "data": {
+                "toleration": "key:postgres,operator:Exists,effect:NoExecute"
+            }
+        }
+        k8s.update_config(patch_toleration_config)
+
+        # wait a little before proceeding with the pod distribution test
+        time.sleep(30)
+
+        # toggle pod anti affinity to move replica away from master node
+        self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label)
+
+    def get_failover_targets(self, master_node, replica_nodes):
+        '''
+           If all pods live on the same node, failover will happen to other worker(s)
+        '''
+        k8s = self.k8s
+
+        failover_targets = [x for x in replica_nodes if x != master_node]
+        if len(failover_targets) == 0:
+            nodes = k8s.api.core_v1.list_node()
+            for n in nodes.items:
+                if "node-role.kubernetes.io/master" not in n.metadata.labels and n.metadata.name != master_node:
+                    failover_targets.append(n.metadata.name)
+
+        return failover_targets
+
+    def assert_failover(self, current_master_node, num_replicas, failover_targets, cluster_label):
+        '''
+           Check if master is failing over. The replica should move first to be the switchover target
+        '''
+        k8s = self.k8s
+        k8s.wait_for_pod_failover(failover_targets, 'spilo-role=master,' + cluster_label)
+        k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label)
+
+        new_master_node, new_replica_nodes = k8s.get_pg_nodes(cluster_label)
+        self.assertNotEqual(current_master_node, new_master_node,
+                            "Master on {} did not fail over to one of {}".format(current_master_node, failover_targets))
+        self.assertEqual(num_replicas, len(new_replica_nodes),
+                         "Expected {} replicas, found {}".format(num_replicas, len(new_replica_nodes)))
+        self.assert_master_is_unique()
+
+        return new_master_node, new_replica_nodes
+
     def assert_master_is_unique(self, namespace='default', clusterName="acid-minimal-cluster"):
         '''
            Check that there is a single pod in the k8s cluster with the label "spilo-role=master"
            To be called manually after operations that affect pods
         '''
-
         k8s = self.k8s
         labels = 'spilo-role=master,cluster-name=' + clusterName
 
         num_of_master_pods = k8s.count_pods_with_label(labels, namespace)
         self.assertEqual(num_of_master_pods, 1, "Expected 1 master pod, found {}".format(num_of_master_pods))
 
+    def assert_distributed_pods(self, master_node, replica_nodes, cluster_label):
+        '''
+           Other tests can lead to the situation that master and replica are on the same node.
+           Toggle pod anti affinty to distribute pods accross nodes (replica in particular).
+        '''
+        k8s = self.k8s
+        failover_targets = self.get_failover_targets(master_node, replica_nodes)
+
+        # enable pod anti affintiy in config map which should trigger movement of replica
+        patch_enable_antiaffinity = {
+            "data": {
+                "enable_pod_antiaffinity": "true"
+            }
+        }
+        k8s.update_config(patch_enable_antiaffinity)
+        self.assert_failover(
+            master_node, len(replica_nodes), failover_targets, cluster_label)
+
+        # now disable pod anti affintiy again which will cause yet another failover
+        patch_disable_antiaffinity = {
+            "data": {
+                "enable_pod_antiaffinity": "false"
+            }
+        }
+        k8s.update_config(patch_disable_antiaffinity)
+        k8s.wait_for_pod_start('spilo-role=master')
+        k8s.wait_for_pod_start('spilo-role=replica')
+
 
 class K8sApi:
 
@@ -380,7 +663,7 @@ class K8s:
     Wraps around K8 api client and helper methods.
     '''
 
-    RETRY_TIMEOUT_SEC = 5
+    RETRY_TIMEOUT_SEC = 10
 
     def __init__(self):
         self.api = K8sApi()
@@ -399,16 +682,43 @@ class K8s:
 
     def wait_for_operator_pod_start(self):
         self. wait_for_pod_start("name=postgres-operator")
-        # HACK operator must register CRD / add existing PG clusters after pod start up
+        # HACK operator must register CRD and/or Sync existing PG clusters after start up
         # for local execution ~ 10 seconds suffices
         time.sleep(60)
 
+    def get_operator_pod(self):
+        pods = self.api.core_v1.list_namespaced_pod(
+            'default', label_selector='name=postgres-operator'
+        ).items
+
+        if pods:
+            return pods[0]
+
+        return None
+
+    def get_operator_log(self):
+        operator_pod = self.get_operator_pod()
+        pod_name = operator_pod.metadata.name
+        return self.api.core_v1.read_namespaced_pod_log(
+            name=pod_name,
+            namespace='default'
+        )
+
     def wait_for_pod_start(self, pod_labels, namespace='default'):
         pod_phase = 'No pod running'
         while pod_phase != 'Running':
             pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=pod_labels).items
             if pods:
                 pod_phase = pods[0].status.phase
+
+            if pods and pod_phase != 'Running':
+                pod_name = pods[0].metadata.name
+                response = self.api.core_v1.read_namespaced_pod(
+                    name=pod_name,
+                    namespace=namespace
+                )
+                print("Pod description {}".format(response))
+
             time.sleep(self.RETRY_TIMEOUT_SEC)
 
     def get_service_type(self, svc_labels, namespace='default'):
@@ -421,10 +731,18 @@ class K8s:
     def check_service_annotations(self, svc_labels, annotations, namespace='default'):
         svcs = self.api.core_v1.list_namespaced_service(namespace, label_selector=svc_labels, limit=1).items
         for svc in svcs:
-            if len(svc.metadata.annotations) != len(annotations):
-                return False
-            for key in svc.metadata.annotations:
-                if svc.metadata.annotations[key] != annotations[key]:
+            for key, value in annotations.items():
+                if key not in svc.metadata.annotations or svc.metadata.annotations[key] != value:
+                    print("Expected key {} not found in annotations {}".format(key, svc.metadata.annotation))
+                    return False
+        return True
+
+    def check_statefulset_annotations(self, sset_labels, annotations, namespace='default'):
+        ssets = self.api.apps_v1.list_namespaced_stateful_set(namespace, label_selector=sset_labels, limit=1).items
+        for sset in ssets:
+            for key, value in annotations.items():
+                if key not in sset.metadata.annotations or sset.metadata.annotations[key] != value:
+                    print("Expected key {} not found in annotations {}".format(key, sset.metadata.annotation))
                     return False
         return True
 
@@ -438,22 +756,38 @@ class K8s:
         _ = self.api.custom_objects_api.patch_namespaced_custom_object(
             "acid.zalan.do", "v1", namespace, "postgresqls", "acid-minimal-cluster", body)
 
-        labels = 'cluster-name=acid-minimal-cluster'
+        labels = 'application=spilo,cluster-name=acid-minimal-cluster'
         while self.count_pods_with_label(labels) != number_of_instances:
             time.sleep(self.RETRY_TIMEOUT_SEC)
 
+    def wait_for_running_pods(self, labels, number, namespace=''):
+        while self.count_pods_with_label(labels) != number:
+            time.sleep(self.RETRY_TIMEOUT_SEC)
+
+    def wait_for_pods_to_stop(self, labels, namespace=''):
+        while self.count_pods_with_label(labels) != 0:
+            time.sleep(self.RETRY_TIMEOUT_SEC)
+
+    def wait_for_service(self, labels, namespace='default'):
+        def get_services():
+            return self.api.core_v1.list_namespaced_service(
+                namespace, label_selector=labels
+            ).items
+
+        while not get_services():
+            time.sleep(self.RETRY_TIMEOUT_SEC)
+
     def count_pods_with_label(self, labels, namespace='default'):
         return len(self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items)
 
-    def wait_for_master_failover(self, expected_master_nodes, namespace='default'):
+    def wait_for_pod_failover(self, failover_targets, labels, namespace='default'):
         pod_phase = 'Failing over'
-        new_master_node = ''
-        labels = 'spilo-role=master,cluster-name=acid-minimal-cluster'
+        new_pod_node = ''
 
-        while (pod_phase != 'Running') or (new_master_node not in expected_master_nodes):
+        while (pod_phase != 'Running') or (new_pod_node not in failover_targets):
             pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items
             if pods:
-                new_master_node = pods[0].spec.node_name
+                new_pod_node = pods[0].spec.node_name
                 pod_phase = pods[0].status.phase
             time.sleep(self.RETRY_TIMEOUT_SEC)
 
@@ -470,16 +804,30 @@ class K8s:
     def wait_for_logical_backup_job_creation(self):
         self.wait_for_logical_backup_job(expected_num_of_jobs=1)
 
-    def update_config(self, config_map_patch):
-        self.api.core_v1.patch_namespaced_config_map("postgres-operator", "default", config_map_patch)
-
+    def delete_operator_pod(self):
         operator_pod = self.api.core_v1.list_namespaced_pod(
             'default', label_selector="name=postgres-operator").items[0].metadata.name
         self.api.core_v1.delete_namespaced_pod(operator_pod, "default")  # restart reloads the conf
         self.wait_for_operator_pod_start()
 
+    def update_config(self, config_map_patch):
+        self.api.core_v1.patch_namespaced_config_map("postgres-operator", "default", config_map_patch)
+        self.delete_operator_pod()
+
     def create_with_kubectl(self, path):
-        subprocess.run(["kubectl", "create", "-f", path])
+        return subprocess.run(
+            ["kubectl", "create", "-f", path],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE)
+
+    def get_effective_pod_image(self, pod_name, namespace='default'):
+        '''
+        Get the Spilo image pod currently uses. In case of lazy rolling updates
+        it may differ from the one specified in the stateful set.
+        '''
+        pod = self.api.core_v1.list_namespaced_pod(
+            namespace, label_selector="statefulset.kubernetes.io/pod-name=" + pod_name)
+        return pod.items[0].spec.containers[0].image
 
 
 if __name__ == '__main__':

go.mod

@@ -1,25 +1,23 @@
 module github.com/zalando/postgres-operator
 
-go 1.12
+go 1.14
 
 require (
-	github.com/aws/aws-sdk-go v1.25.44
+	github.com/aws/aws-sdk-go v1.29.33
 	github.com/emicklei/go-restful v2.9.6+incompatible // indirect
 	github.com/evanphx/json-patch v4.5.0+incompatible // indirect
 	github.com/googleapis/gnostic v0.3.0 // indirect
-	github.com/imdario/mergo v0.3.8 // indirect
-	github.com/lib/pq v1.2.0
+	github.com/lib/pq v1.3.0
 	github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d
-	github.com/sirupsen/logrus v1.4.2
-	golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 // indirect
-	golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect
-	golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 // indirect
-	golang.org/x/tools v0.0.0-20191209225234-22774f7dae43 // indirect
-	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
-	gopkg.in/yaml.v2 v2.2.4
-	k8s.io/api v0.0.0-20191121015604-11707872ac1c
-	k8s.io/apiextensions-apiserver v0.0.0-20191204090421-cd61debedab5
-	k8s.io/apimachinery v0.0.0-20191203211716-adc6f4cd9e7d
-	k8s.io/client-go v0.0.0-20191204082520-bc9b51d240b2
-	k8s.io/code-generator v0.0.0-20191121015212-c4c8f8345c7e
+	github.com/r3labs/diff v0.0.0-20191120142937-b4ed99a31f5a
+	github.com/sirupsen/logrus v1.5.0
+	github.com/stretchr/testify v1.4.0
+	golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b // indirect
+	gopkg.in/yaml.v2 v2.2.8
+	k8s.io/api v0.18.2
+	k8s.io/apiextensions-apiserver v0.18.2
+	k8s.io/apimachinery v0.18.2
+	k8s.io/client-go v11.0.0+incompatible
+	k8s.io/code-generator v0.18.2
+	sigs.k8s.io/kind v0.5.1 // indirect
 )

go.sum

@@ -11,7 +11,6 @@ github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6L
 github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
@@ -27,12 +26,13 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/aws/aws-sdk-go v1.25.44 h1:n9ahFoiyn66smjF34hYr3tb6/ZdBcLuFz7BCDhHyJ7I=
-github.com/aws/aws-sdk-go v1.25.44/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/aws/aws-sdk-go v1.29.33 h1:WP85+WHalTFQR2wYp5xR2sjiVAZXew2bBQXGU1QJBXI=
+github.com/aws/aws-sdk-go v1.29.33/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -58,12 +58,16 @@ github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QL
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e h1:p1yVGRW3nmb85p1Sh1ZJSDm4A4iKLS5QNbvUHMgGu/M=
-github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
+github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.6+incompatible h1:tfrHha8zJ01ywiOEC1miGY8st1/igzWB8OmvPgoYX7w=
 github.com/emicklei/go-restful v2.9.6+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I=
 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M=
 github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
@@ -124,11 +128,12 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh
 github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4=
 github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA=
 github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
-github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
+github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -136,6 +141,7 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -147,13 +153,19 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
 github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
+github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gnostic v0.0.0-20170426233943-68f4ded48ba9/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
+github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
+github.com/googleapis/gnostic v0.1.0 h1:rVsPeBmXbYv4If/cumu1AzZPwV58q433hvONV1UEZoI=
+github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0=
 github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
@@ -169,15 +181,16 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
-github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ=
-github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -195,13 +208,14 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
-github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU=
+github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190620125010-da37f6c1e481/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM=
 github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
@@ -227,17 +241,20 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+
 github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
-github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw=
+github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -246,26 +263,33 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
+github.com/r3labs/diff v0.0.0-20191120142937-b4ed99a31f5a h1:2v4Ipjxa3sh+xn6GvtgrMub2ci4ZLQMvTaYIba2lfdc=
+github.com/r3labs/diff v0.0.0-20191120142937-b4ed99a31f5a/go.mod h1:ozniNEFS3j1qCwHKdvraMn1WJOsUxHd7lYfukEIS4cs=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q=
+github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
@@ -275,6 +299,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -285,6 +310,7 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
 go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
@@ -301,19 +327,17 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g=
-golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 h1:/Tl7pH94bvbAAHBdZJT947M/+gp0+CqQXDtMRC0fseo=
+golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495 h1:I6A9Ag9FpEKOjcKrRNjQkPHawoXIhKyTGfvvjFAiiAk=
-golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -331,8 +355,9 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8=
-golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
@@ -342,6 +367,7 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -351,20 +377,22 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f h1:25KHgbfyiSm6vwQLbM3zZIe1v9p/3ea4Rz+nnM5K/i4=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190621203818-d432491b9138/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 h1:gSbV7h1NRL2G1xTg/owz62CST1oJBmxy4QpMMregXVQ=
-golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 h1:HmbHVPwrPEKPGLAcHSrMe6+hqSUlvZU0rab6x5EXfGU=
+golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -374,23 +402,20 @@ golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191209225234-22774f7dae43 h1:NfPq5mgc5ArFgVLCpeS4z07IoxSAqVfV/gQ5vxdgaxI=
-golang.org/x/tools v0.0.0-20191209225234-22774f7dae43/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b h1:zSzQJAznWxAh9fZxiPy2FZo+ZZEYoYFYYDYdOrU7AaM=
+golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485 h1:OB/uP/Puiu5vS5QMRPrXCDWUPb+kt8f1KW8oQzFejQw=
-gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0=
-gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
-gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e h1:jRyg0XfpwWlhEV8mDfdNGBeSJM2fuyh9Yjrnd8kF2Ts=
-gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -399,14 +424,15 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
@@ -422,45 +448,55 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-k8s.io/api v0.0.0-20191121015604-11707872ac1c h1:Z87my3sF4WhG0OMxzARkWY/IKBtOr+MhXZAb4ts6qFc=
-k8s.io/api v0.0.0-20191121015604-11707872ac1c/go.mod h1:R/s4gKT0V/cWEnbQa9taNRJNbWUK57/Dx6cPj6MD3A0=
-k8s.io/apiextensions-apiserver v0.0.0-20191204090421-cd61debedab5 h1:g+GvnbGqLU1Jxb/9iFm/BFcmkqG9HdsGh52+wHirpsM=
-k8s.io/apiextensions-apiserver v0.0.0-20191204090421-cd61debedab5/go.mod h1:CPw0IHz1YrWGy0+8mG/76oTHXvChlgCb3EAezKQKB2I=
-k8s.io/apimachinery v0.0.0-20191121015412-41065c7a8c2a/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg=
-k8s.io/apimachinery v0.0.0-20191123233150-4c4803ed55e3/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg=
-k8s.io/apimachinery v0.0.0-20191128180518-03184f823e28/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg=
-k8s.io/apimachinery v0.0.0-20191203211716-adc6f4cd9e7d h1:q+OZmYewHJeMCzwpHkXlNTtk5bvaUMPCikKvf77RBlo=
-k8s.io/apimachinery v0.0.0-20191203211716-adc6f4cd9e7d/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg=
-k8s.io/apiserver v0.0.0-20191204084332-137a9d3b886b/go.mod h1:itgfam5HJbT/4b2BGfpUkkxfheMmDH+Ix+tEAP3uqZk=
-k8s.io/client-go v0.0.0-20191204082517-8c19b9f4a642/go.mod h1:HMVIZ0dPop3WCrPEaJ+v5/94cjt56avdDFshpX0Fjvo=
-k8s.io/client-go v0.0.0-20191204082519-e9644b2e3edc/go.mod h1:5lSG1yeDZVwDYAHe9VK48SCe5zmcnkAcf2Mx59TuhmM=
-k8s.io/client-go v0.0.0-20191204082520-bc9b51d240b2 h1:T2HGghBOPAOEjWuIyFSeCsWEwsxa6unkBvy3PHfqonM=
-k8s.io/client-go v0.0.0-20191204082520-bc9b51d240b2/go.mod h1:5lSG1yeDZVwDYAHe9VK48SCe5zmcnkAcf2Mx59TuhmM=
-k8s.io/code-generator v0.0.0-20191121015212-c4c8f8345c7e h1:HB9Zu5ZUvJfNpLiTPhz+CebVKV8C39qTBMQkAgAZLNw=
-k8s.io/code-generator v0.0.0-20191121015212-c4c8f8345c7e/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s=
-k8s.io/component-base v0.0.0-20191204083903-0d4d24e738e4/go.mod h1:8VIh1jErItC4bg9hLBkPneyS77Tin8KwSzbYepHJnQI=
-k8s.io/component-base v0.0.0-20191204083906-3ac1376c73aa/go.mod h1:mECWvHCPhJudDVDMtBl+AIf/YnTMp5r1F947OYFUwP0=
+k8s.io/api v0.0.0-20190313235455-40a48860b5ab/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA=
+k8s.io/api v0.0.0-20190409021203-6e4e0e4f393b/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA=
+k8s.io/api v0.18.2 h1:wG5g5ZmSVgm5B+eHMIbI9EGATS2L8Z72rda19RIEgY8=
+k8s.io/api v0.18.2/go.mod h1:SJCWI7OLzhZSvbY7U8zwNl9UA4o1fizoug34OV/2r78=
+k8s.io/apiextensions-apiserver v0.18.2 h1:I4v3/jAuQC+89L3Z7dDgAiN4EOjN6sbm6iBqQwHTah8=
+k8s.io/apiextensions-apiserver v0.18.2/go.mod h1:q3faSnRGmYimiocj6cHQ1I3WpLqmDgJFlKL37fC4ZvY=
+k8s.io/apimachinery v0.0.0-20190313205120-d7deff9243b1/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0=
+k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0=
+k8s.io/apimachinery v0.18.2 h1:44CmtbmkzVDAhCpRVSiP2R5PPrC2RtlIv/MoB8xpdRA=
+k8s.io/apimachinery v0.18.2/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA=
+k8s.io/apiserver v0.18.2/go.mod h1:Xbh066NqrZO8cbsoenCwyDJ1OSi8Ag8I2lezeHxzwzw=
+k8s.io/client-go v0.18.2 h1:aLB0iaD4nmwh7arT2wIn+lMnAq7OswjaejkQ8p9bBYE=
+k8s.io/client-go v0.18.2/go.mod h1:Xcm5wVGXX9HAA2JJ2sSBUn3tCJ+4SVlCbl2MNNv+CIU=
+k8s.io/client-go v11.0.0+incompatible h1:LBbX2+lOwY9flffWlJM7f1Ct8V2SRNiMRDFeiwnJo9o=
+k8s.io/client-go v11.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s=
+k8s.io/code-generator v0.18.2 h1:C1Nn2JiMf244CvBDKVPX0W2mZFJkVBg54T8OV7/Imso=
+k8s.io/code-generator v0.18.2/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc=
+k8s.io/component-base v0.18.2/go.mod h1:kqLlMuhJNHQ9lz8Z7V5bxUUtjFZnrypArGl58gmDfUM=
 k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
-k8s.io/gengo v0.0.0-20190822140433-26a664648505 h1:ZY6yclUKVbZ+SdWnkfY+Je5vrMpKOxmGeKRbsXVmqYM=
-k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/gengo v0.0.0-20200114144118-36b2048a9120 h1:RPscN6KhmG54S33L+lr3GS+oD1jmchIU0ll519K6FA4=
+k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
+k8s.io/klog v0.3.3/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
 k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
-k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU=
-k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
-k8s.io/utils v0.0.0-20191114184206-e782cd3c129f h1:GiPwtSzdP43eI1hpPCbROQCCIgCuiMMNF8YUVLF3vJo=
-k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
-modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw=
-modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk=
-modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
-modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
-modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I=
+k8s.io/kube-openapi v0.0.0-20190603182131-db7b694dc208/go.mod h1:nfDlWeOsu3pUf4yWGL+ERqohP4YsZcBJXWMK+gkzOA4=
+k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c h1:/KUFqjjqAcY4Us6luF5RDNZ16KJtb49HfR3ZHB9qYXM=
+k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E=
+k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89 h1:d4vVOjXm687F1iLSP2q3lyPPuyvTUt3aVoBpi2DqRsU=
+k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0=
+sigs.k8s.io/kind v0.5.1 h1:BYnHEJ9DC+0Yjlyyehqd3xnKtEmFdLKU8QxqOqvQzdw=
+sigs.k8s.io/kind v0.5.1/go.mod h1:L+Kcoo83/D1+ryU5P2VFbvYm0oqbkJn9zTZq0KNxW68=
+sigs.k8s.io/kustomize/v3 v3.1.1-0.20190821175718-4b67a6de1296 h1:iQaIG5Dq+3qSiaFrJ/l/0MjjxKmdwyVNpKRYJwUe/+0=
+sigs.k8s.io/kustomize/v3 v3.1.1-0.20190821175718-4b67a6de1296/go.mod h1:ztX4zYc/QIww3gSripwF7TBOarBTm5BvyAMem0kCzOE=
+sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e h1:4Z09Hglb792X0kfOBBJUPFEyvVfQWrYT/l8h5EKA6JQ=
 sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
-sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06/go.mod h1:/ULNhyfzRopfcjskuui0cTITekDduZ7ycKN3oUT9R18=
+sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=
+sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E=
+sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=
 sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
+sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
+sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

kubectl-pg/cmd/list.go

@@ -24,13 +24,14 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/spf13/cobra"
-	"github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
-	PostgresqlLister "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"log"
 	"strconv"
 	"time"
+
+	"github.com/spf13/cobra"
+	v1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
+	PostgresqlLister "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 const (
@@ -95,8 +96,12 @@ func listAll(listPostgres *v1.PostgresqlList) {
 	template := "%-32s%-16s%-12s%-12s%-12s%-12s%-12s\n"
 	fmt.Printf(template, "NAME", "STATUS", "INSTANCES", "VERSION", "AGE", "VOLUME", "NAMESPACE")
 	for _, pgObjs := range listPostgres.Items {
-		fmt.Printf(template, pgObjs.Name, pgObjs.Status.PostgresClusterStatus, strconv.Itoa(int(pgObjs.Spec.NumberOfInstances)),
-			pgObjs.Spec.PgVersion, time.Since(pgObjs.CreationTimestamp.Time).Truncate(TrimCreateTimestamp), pgObjs.Spec.Size, pgObjs.Namespace)
+		fmt.Printf(template, pgObjs.Name,
+			pgObjs.Status.PostgresClusterStatus,
+			strconv.Itoa(int(pgObjs.Spec.NumberOfInstances)),
+			pgObjs.Spec.PostgresqlParam.PgVersion,
+			time.Since(pgObjs.CreationTimestamp.Time).Truncate(TrimCreateTimestamp),
+			pgObjs.Spec.Size, pgObjs.Namespace)
 	}
 }
 
@@ -104,8 +109,12 @@ func listWithNamespace(listPostgres *v1.PostgresqlList) {
 	template := "%-32s%-16s%-12s%-12s%-12s%-12s\n"
 	fmt.Printf(template, "NAME", "STATUS", "INSTANCES", "VERSION", "AGE", "VOLUME")
 	for _, pgObjs := range listPostgres.Items {
-		fmt.Printf(template, pgObjs.Name, pgObjs.Status.PostgresClusterStatus, strconv.Itoa(int(pgObjs.Spec.NumberOfInstances)),
-			pgObjs.Spec.PgVersion, time.Since(pgObjs.CreationTimestamp.Time).Truncate(TrimCreateTimestamp), pgObjs.Spec.Size)
+		fmt.Printf(template, pgObjs.Name,
+			pgObjs.Status.PostgresClusterStatus,
+			strconv.Itoa(int(pgObjs.Spec.NumberOfInstances)),
+			pgObjs.Spec.PostgresqlParam.PgVersion,
+			time.Since(pgObjs.CreationTimestamp.Time).Truncate(TrimCreateTimestamp),
+			pgObjs.Spec.Size)
 	}
 }
 

manifests/complete-postgres-manifest.yaml

@@ -4,12 +4,11 @@ metadata:
   name: acid-test-cluster
 #  labels:
 #    environment: demo
+#  annotations:
+#    "acid.zalan.do/controller": "second-operator"
 spec:
-  dockerImage: registry.opensource.zalan.do/acid/spilo-12:1.6-p2
+  dockerImage: registry.opensource.zalan.do/acid/spilo-12:1.6-p3
   teamId: "acid"
-  volume:
-    size: 1Gi
-#    storageClass: my-sc
   numberOfInstances: 2
   users:  # Application/Robot users
     zalando:
@@ -17,16 +16,54 @@ spec:
     - createdb
   enableMasterLoadBalancer: false
   enableReplicaLoadBalancer: false
+#  enableConnectionPooler: true  # not needed when connectionPooler section is present (see below)
   allowedSourceRanges:  # load balancers' source ranges for both master and replica services
   - 127.0.0.1/32
   databases:
     foo: zalando
+  preparedDatabases:
+    bar:
+      defaultUsers: true
+      extensions:
+        pg_partman: public
+        pgcrypto: public
+      schemas:
+        data: {}
+        history:
+          defaultRoles: true
+          defaultUsers: false
   postgresql:
-    version: "11"
+    version: "12"
     parameters: # Expert section
       shared_buffers: "32MB"
       max_connections: "10"
       log_statement: "all"
+  volume:
+    size: 1Gi
+#    storageClass: my-sc
+  additionalVolumes:
+    - name: empty
+      mountPath: /opt/empty
+      targetContainers:
+        - all
+      volumeSource:
+        emptyDir: {}
+#    - name: data
+#      mountPath: /home/postgres/pgdata/partitions
+#      targetContainers:
+#        - postgres
+#      volumeSource:
+#        PersistentVolumeClaim:
+#          claimName: pvc-postgresql-data-partitions
+#          readyOnly: false
+#    - name: conf
+#      mountPath: /etc/telegraf
+#      subPath: telegraf.conf
+#      targetContainers:
+#        - telegraf-sidecar
+#      volumeSource:
+#        configMap:
+#          name: my-config-map
 
   enableShmVolume: true
 #  spiloFSGroup: 103
@@ -64,6 +101,8 @@ spec:
     ttl: 30
     loop_wait: &loop_wait 10
     retry_timeout: 10
+    synchronous_mode: false
+    synchronous_mode_strict: false
     maximum_lag_on_failover: 33554432
 
 # restore a Postgres DB with point-in-time-recovery
@@ -83,6 +122,19 @@ spec:
 #  - 01:00-06:00  #UTC
 #  - Sat:00:00-04:00
 
+  connectionPooler:
+    numberOfInstances: 2
+    mode: "transaction"
+    schema: "pooler"
+    user: "pooler"
+    resources:
+      requests:
+        cpu: 300m
+        memory: 100Mi
+      limits:
+        cpu: "1"
+        memory: 100Mi
+
   initContainers:
   - name: date
     image: busybox
@@ -100,3 +152,17 @@ spec:
 #      env:
 #        - name: "USEFUL_VAR"
 #          value: "perhaps-true"
+
+# Custom TLS certificate. Disabled unless tls.secretName has a value.
+  tls:
+    secretName: ""  # should correspond to a Kubernetes Secret resource to load
+    certificateFile: "tls.crt"
+    privateKeyFile: "tls.key"
+    caFile: ""  # optionally configure Postgres with a CA certificate
+    caSecretName: "" # optionally the ca.crt can come from this secret instead.
+# file names can be also defined with absolute path, and will no longer be relative
+# to the "/tls/" path where the secret is being mounted by default, and "/tlsca/"
+# where the caSecret is mounted by default.
+# When TLS is enabled, also set spiloFSGroup parameter above to the relevant value.
+# if unknown, set it to 103 which is the usual value in the default spilo images.
+# In Openshift, there is no need to set spiloFSGroup/spilo_fsgroup.

manifests/configmap.yaml

@@ -11,6 +11,16 @@ data:
   cluster_history_entries: "1000"
   cluster_labels: application:spilo
   cluster_name_label: cluster-name
+  # connection_pooler_default_cpu_limit: "1"
+  # connection_pooler_default_cpu_request: "500m"
+  # connection_pooler_default_memory_limit: 100Mi
+  # connection_pooler_default_memory_request: 100Mi
+  connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-7"
+  # connection_pooler_max_db_connections: 60
+  # connection_pooler_mode: "transaction"
+  # connection_pooler_number_of_instances: 2
+  # connection_pooler_schema: "pooler"
+  # connection_pooler_user: "pooler"
   # custom_service_annotations: "keyx:valuez,keya:valuea"
   # custom_pod_annotations: "keya:valuea,keyb:valueb"
   db_hosted_zone: db.example.com
@@ -19,11 +29,13 @@ data:
   # default_cpu_request: 100m
   # default_memory_limit: 500Mi
   # default_memory_request: 100Mi
-  docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p2
+  docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p3
+  # downscaler_annotations: "deployment-time,downscaler/*"
   # enable_admin_role_for_users: "true"
   # enable_crd_validation: "true"
   # enable_database_access: "true"
   # enable_init_containers: "true"
+  # enable_lazy_spilo_upgrade: "false"
   enable_master_load_balancer: "false"
   # enable_pod_antiaffinity: "false"
   # enable_pod_disruption_budget: "true"
@@ -33,20 +45,21 @@ data:
   # enable_team_superuser: "false"
   enable_teams_api: "false"
   # etcd_host: ""
+  # kubernetes_use_configmaps: "false"
   # infrastructure_roles_secret_name: postgresql-infrastructure-roles
   # inherited_labels: application,environment
   # kube_iam_role: ""
   # log_s3_bucket: ""
-  # logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup"
+  logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup"
   # logical_backup_s3_access_key_id: ""
-  # logical_backup_s3_bucket: "my-bucket-url"
+  logical_backup_s3_bucket: "my-bucket-url"
   # logical_backup_s3_region: ""
   # logical_backup_s3_endpoint: ""
   # logical_backup_s3_secret_access_key: ""
-  # logical_backup_s3_sse: "AES256"
-  # logical_backup_schedule: "30 00 * * *"
+  logical_backup_s3_sse: "AES256"
+  logical_backup_schedule: "30 00 * * *"
   master_dns_name_format: "{cluster}.{team}.{hostedzone}"
-  # master_pod_move_timeout: 10m
+  # master_pod_move_timeout: 20m
   # max_instances: "-1"
   # min_instances: "-1"
   # min_cpu_limit: 250m
@@ -59,7 +72,7 @@ data:
   pdb_name_format: "postgres-{cluster}-pdb"
   # pod_antiaffinity_topology_key: "kubernetes.io/hostname"
   pod_deletion_wait_timeout: 10m
-  # pod_environment_configmap: ""
+  # pod_environment_configmap: "default/my-custom-config"
   pod_label_wait_timeout: 10m
   pod_management_policy: "ordered_ready"
   pod_role_label: spilo-role

manifests/minimal-postgres-manifest.yaml

@@ -15,5 +15,7 @@ spec:
     foo_user: []  # role for application foo
   databases:
     foo: zalando  # dbname: owner
+  preparedDatabases:
+    bar: {}
   postgresql:
-    version: "11"
+    version: "12"

manifests/operator-service-account-rbac.yaml

@@ -43,6 +43,18 @@ rules:
   - configmaps
   verbs:
   - get
+# to send events to the CRs
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - get
+  - list
+  - patch
+  - update
+  - watch
 # to manage endpoints which are also used by Patroni
 - apiGroups:
   - ""
@@ -129,6 +141,7 @@ rules:
   - apps
   resources:
   - statefulsets
+  - deployments
   verbs:
   - create
   - delete

manifests/operatorconfiguration.crd.yaml

@@ -38,10 +38,14 @@ spec:
               type: string
             enable_crd_validation:
               type: boolean
+            enable_lazy_spilo_upgrade:
+              type: boolean
             enable_shm_volume:
               type: boolean
             etcd_host:
               type: string
+            kubernetes_use_configmaps:
+              type: boolean
             max_instances:
               type: integer
               minimum: -1  # -1 = disabled
@@ -58,6 +62,12 @@ spec:
               type: object
               additionalProperties:
                 type: string
+            sidecars:
+              type: array
+              nullable: true
+              items:
+                type: object
+                additionalProperties: true
             workers:
               type: integer
               minimum: 1
@@ -83,6 +93,10 @@ spec:
                   type: object
                   additionalProperties:
                     type: string
+                downscaler_annotations:
+                  type: array
+                  items:
+                    type: string
                 enable_init_containers:
                   type: boolean
                 enable_pod_antiaffinity:
@@ -273,7 +287,7 @@ spec:
                   type: integer
                 ring_log_lines:
                   type: integer
-            scalyr:
+            scalyr:  # deprecated
               type: object
               properties:
                 scalyr_api_key:
@@ -294,6 +308,47 @@ spec:
                   pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$'
                 scalyr_server_url:
                   type: string
+            connection_pooler:
+              type: object
+              properties:
+                connection_pooler_schema:
+                  type: string
+                  #default: "pooler"
+                connection_pooler_user:
+                  type: string
+                  #default: "pooler"
+                connection_pooler_image:
+                  type: string
+                  #default: "registry.opensource.zalan.do/acid/pgbouncer"
+                connection_pooler_max_db_connections:
+                  type: integer
+                  #default: 60
+                connection_pooler_mode:
+                  type: string
+                  enum:
+                    - "session"
+                    - "transaction"
+                  #default: "transaction"
+                connection_pooler_number_of_instances:
+                  type: integer
+                  minimum: 2
+                  #default: 2
+                connection_pooler_default_cpu_limit:
+                  type: string
+                  pattern: '^(\d+m|\d+(\.\d{1,3})?)$'
+                  #default: "1"
+                connection_pooler_default_cpu_request:
+                  type: string
+                  pattern: '^(\d+m|\d+(\.\d{1,3})?)$'
+                  #default: "500m"
+                connection_pooler_default_memory_limit:
+                  type: string
+                  pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$'
+                  #default: "100Mi"
+                connection_pooler_default_memory_request:
+                  type: string
+                  pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$'
+                  #default: "100Mi"
         status:
           type: object
           additionalProperties:

manifests/postgres-operator.yaml

@@ -15,7 +15,7 @@ spec:
       serviceAccountName: postgres-operator
       containers:
       - name: postgres-operator
-        image: registry.opensource.zalan.do/acid/postgres-operator:v1.4.0
+        image: registry.opensource.zalan.do/acid/postgres-operator:v1.5.0
         imagePullPolicy: IfNotPresent
         resources:
           requests:
@@ -35,3 +35,6 @@ spec:
         # In order to use the CRD OperatorConfiguration instead, uncomment these lines and comment out the two lines above
         # - name: POSTGRES_OPERATOR_CONFIGURATION_OBJECT
         #  value: postgresql-operator-default-configuration
+        # Define an ID to isolate controllers from each other
+        # - name: CONTROLLER_ID
+        #   value: "second-operator"

manifests/postgresql-operator-default-configuration.yaml

@@ -3,17 +3,22 @@ kind: OperatorConfiguration
 metadata:
   name: postgresql-operator-default-configuration
 configuration:
+  docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p3
   # enable_crd_validation: true
-  etcd_host: ""
-  docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p2
+  # enable_lazy_spilo_upgrade: false
   # enable_shm_volume: true
+  etcd_host: ""
+  # kubernetes_use_configmaps: false
   max_instances: -1
   min_instances: -1
   resync_period: 30m
   repair_period: 5m
   # set_memory_request_to_limit: false
-  # sidecar_docker_images:
-  #   example: "exampleimage:exampletag"
+  # sidecars:
+  # - image: image:123
+  #   name: global-sidecar-1
+  #   ports:
+  #   - containerPort: 80
   workers: 4
   users:
     replication_username: standby
@@ -26,6 +31,9 @@ configuration:
     # custom_pod_annotations:
     #   keya: valuea
     #   keyb: valueb
+    # downscaler_annotations:
+    # - deployment-time
+    # - downscaler/*
     enable_init_containers: true
     enable_pod_antiaffinity: false
     enable_pod_disruption_budget: true
@@ -40,7 +48,7 @@ configuration:
     oauth_token_secret_name: postgresql-operator
     pdb_name_format: "postgres-{cluster}-pdb"
     pod_antiaffinity_topology_key: "kubernetes.io/hostname"
-    # pod_environment_configmap: ""
+    # pod_environment_configmap: "default/my-custom-config"
     pod_management_policy: "ordered_ready"
     # pod_priority_class_name: ""
     pod_role_label: spilo-role
@@ -84,7 +92,7 @@ configuration:
     # log_s3_bucket: ""
     # wal_s3_bucket: ""
   logical_backup:
-    logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup"
+    logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:master-58"
     # logical_backup_s3_access_key_id: ""
     logical_backup_s3_bucket: "my-bucket-url"
     # logical_backup_s3_endpoint: ""
@@ -110,14 +118,17 @@ configuration:
       log_statement: all
     # teams_api_url: ""
   logging_rest_api:
-    api_port: 8008
+    api_port: 8080
     cluster_history_entries: 1000
     ring_log_lines: 100
-  scalyr:
-    # scalyr_api_key: ""
-    scalyr_cpu_limit: "1"
-    scalyr_cpu_request: 100m
-    # scalyr_image: ""
-    scalyr_memory_limit: 500Mi
-    scalyr_memory_request: 50Mi
-    # scalyr_server_url: ""
+  connection_pooler:
+    connection_pooler_default_cpu_limit: "1"
+    connection_pooler_default_cpu_request: "500m"
+    connection_pooler_default_memory_limit: 100Mi
+    connection_pooler_default_memory_request: 100Mi
+    connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-7"
+    # connection_pooler_max_db_connections: 60
+    connection_pooler_mode: "transaction"
+    connection_pooler_number_of_instances: 2
+    # connection_pooler_schema: "pooler"
+    # connection_pooler_user: "pooler"

manifests/postgresql.crd.yaml

@@ -38,6 +38,28 @@ spec:
             - teamId
             - postgresql
           properties:
+            additionalVolumes:
+              type: array
+              items:
+                type: object
+                required:
+                  - name
+                  - mountPath
+                  - volumeSource
+                properties:
+                  name:
+                    type: string
+                  mountPath:
+                    type: string
+                  targetContainers:
+                    type: array
+                    nullable: true
+                    items:
+                      type: string
+                  volumeSource:
+                    type: object
+                  subPath:
+                    type: string
             allowedSourceRanges:
               type: array
               nullable: true
@@ -70,6 +92,55 @@ spec:
                 uid:
                   format: uuid
                   type: string
+            connectionPooler:
+              type: object
+              properties:
+                dockerImage:
+                  type: string
+                maxDBConnections:
+                  type: integer
+                mode:
+                  type: string
+                  enum:
+                    - "session"
+                    - "transaction"
+                numberOfInstances:
+                  type: integer
+                  minimum: 2
+                resources:
+                  type: object
+                  required:
+                    - requests
+                    - limits
+                  properties:
+                    limits:
+                      type: object
+                      required:
+                        - cpu
+                        - memory
+                      properties:
+                        cpu:
+                          type: string
+                          pattern: '^(\d+m|\d+(\.\d{1,3})?)$'
+                        memory:
+                          type: string
+                          pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$'
+                    requests:
+                      type: object
+                      required:
+                        - cpu
+                        - memory
+                      properties:
+                        cpu:
+                          type: string
+                          pattern: '^(\d+m|\d+(\.\d{1,3})?)$'
+                        memory:
+                          type: string
+                          pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$'
+                schema:
+                  type: string
+                user:
+                  type: string
             databases:
               type: object
               additionalProperties:
@@ -77,6 +148,8 @@ spec:
               # Note: usernames specified here as database owners must be declared in the users key of the spec key.
             dockerImage:
               type: string
+            enableConnectionPooler:
+              type: boolean
             enableLogicalBackup:
               type: boolean
             enableMasterLoadBalancer:
@@ -133,6 +206,10 @@ spec:
                   type: integer
                 maximum_lag_on_failover:
                   type: integer
+                synchronous_mode:
+                  type: boolean
+                synchronous_mode_strict:
+                  type: boolean
             podAnnotations:
               type: object
               additionalProperties:
@@ -160,6 +237,26 @@ spec:
                   type: object
                   additionalProperties:
                     type: string
+            preparedDatabases:
+              type: object
+              additionalProperties:
+                type: object
+                properties:
+                  defaultUsers:
+                    type: boolean
+                  extensions:
+                    type: object
+                    additionalProperties:
+                      type: string
+                  schemas:
+                    type: object
+                    additionalProperties:
+                      type: object
+                      properties:
+                        defaultUsers:
+                          type: boolean
+                        defaultRoles:
+                          type: boolean
             replicaLoadBalancer:  # deprecated
               type: boolean
             resources:
@@ -251,6 +348,21 @@ spec:
                   type: string
             teamId:
               type: string
+            tls:
+              type: object
+              required:
+                - secretName
+              properties:
+                secretName:
+                  type: string
+                certificateFile:
+                  type: string
+                privateKeyFile:
+                  type: string
+                caFile:
+                  type: string
+                caSecretName:
+                  type: string
             tolerations:
               type: array
               items:

manifests/standby-manifest.yaml

@@ -9,7 +9,7 @@ spec:
     size: 1Gi
   numberOfInstances: 1
   postgresql:
-    version: "11"
+    version: "12"
 # Make this a standby cluster and provide the s3 bucket path of source cluster for continuous streaming.
   standby:
     s3_wal_path: "s3://path/to/bucket/containing/wal/of/source/cluster/"

pkg/apis/acid.zalan.do/v1/crds.go

@@ -21,48 +21,48 @@ const (
 
 // PostgresCRDResourceColumns definition of AdditionalPrinterColumns for postgresql CRD
 var PostgresCRDResourceColumns = []apiextv1beta1.CustomResourceColumnDefinition{
-	apiextv1beta1.CustomResourceColumnDefinition{
+	{
 		Name:        "Team",
 		Type:        "string",
 		Description: "Team responsible for Postgres cluster",
 		JSONPath:    ".spec.teamId",
 	},
-	apiextv1beta1.CustomResourceColumnDefinition{
+	{
 		Name:        "Version",
 		Type:        "string",
 		Description: "PostgreSQL version",
 		JSONPath:    ".spec.postgresql.version",
 	},
-	apiextv1beta1.CustomResourceColumnDefinition{
+	{
 		Name:        "Pods",
 		Type:        "integer",
 		Description: "Number of Pods per Postgres cluster",
 		JSONPath:    ".spec.numberOfInstances",
 	},
-	apiextv1beta1.CustomResourceColumnDefinition{
+	{
 		Name:        "Volume",
 		Type:        "string",
 		Description: "Size of the bound volume",
 		JSONPath:    ".spec.volume.size",
 	},
-	apiextv1beta1.CustomResourceColumnDefinition{
+	{
 		Name:        "CPU-Request",
 		Type:        "string",
 		Description: "Requested CPU for Postgres containers",
 		JSONPath:    ".spec.resources.requests.cpu",
 	},
-	apiextv1beta1.CustomResourceColumnDefinition{
+	{
 		Name:        "Memory-Request",
 		Type:        "string",
 		Description: "Requested memory for Postgres containers",
 		JSONPath:    ".spec.resources.requests.memory",
 	},
-	apiextv1beta1.CustomResourceColumnDefinition{
+	{
 		Name:     "Age",
 		Type:     "date",
 		JSONPath: ".metadata.creationTimestamp",
 	},
-	apiextv1beta1.CustomResourceColumnDefinition{
+	{
 		Name:        "Status",
 		Type:        "string",
 		Description: "Current sync status of postgresql resource",
@@ -72,31 +72,31 @@ var PostgresCRDResourceColumns = []apiextv1beta1.CustomResourceColumnDefinition{
 
 // OperatorConfigCRDResourceColumns definition of AdditionalPrinterColumns for OperatorConfiguration CRD
 var OperatorConfigCRDResourceColumns = []apiextv1beta1.CustomResourceColumnDefinition{
-	apiextv1beta1.CustomResourceColumnDefinition{
+	{
 		Name:        "Image",
 		Type:        "string",
 		Description: "Spilo image to be used for Pods",
 		JSONPath:    ".configuration.docker_image",
 	},
-	apiextv1beta1.CustomResourceColumnDefinition{
+	{
 		Name:        "Cluster-Label",
 		Type:        "string",
 		Description: "Label for K8s resources created by operator",
 		JSONPath:    ".configuration.kubernetes.cluster_name_label",
 	},
-	apiextv1beta1.CustomResourceColumnDefinition{
+	{
 		Name:        "Service-Account",
 		Type:        "string",
 		Description: "Name of service account to be used",
 		JSONPath:    ".configuration.kubernetes.pod_service_account_name",
 	},
-	apiextv1beta1.CustomResourceColumnDefinition{
+	{
 		Name:        "Min-Instances",
 		Type:        "integer",
 		Description: "Minimum number of instances per Postgres cluster",
 		JSONPath:    ".configuration.min_instances",
 	},
-	apiextv1beta1.CustomResourceColumnDefinition{
+	{
 		Name:     "Age",
 		Type:     "date",
 		JSONPath: ".metadata.creationTimestamp",
@@ -105,6 +105,7 @@ var OperatorConfigCRDResourceColumns = []apiextv1beta1.CustomResourceColumnDefin
 
 var min0 = 0.0
 var min1 = 1.0
+var min2 = 2.0
 var minDisable = -1.0
 
 // PostgresCRDResourceValidation to check applied manifest parameters
@@ -176,6 +177,76 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{
 							},
 						},
 					},
+					"connectionPooler": {
+						Type: "object",
+						Properties: map[string]apiextv1beta1.JSONSchemaProps{
+							"dockerImage": {
+								Type: "string",
+							},
+							"maxDBConnections": {
+								Type: "integer",
+							},
+							"mode": {
+								Type: "string",
+								Enum: []apiextv1beta1.JSON{
+									{
+										Raw: []byte(`"session"`),
+									},
+									{
+										Raw: []byte(`"transaction"`),
+									},
+								},
+							},
+							"numberOfInstances": {
+								Type:    "integer",
+								Minimum: &min2,
+							},
+							"resources": {
+								Type:     "object",
+								Required: []string{"requests", "limits"},
+								Properties: map[string]apiextv1beta1.JSONSchemaProps{
+									"limits": {
+										Type:     "object",
+										Required: []string{"cpu", "memory"},
+										Properties: map[string]apiextv1beta1.JSONSchemaProps{
+											"cpu": {
+												Type:        "string",
+												Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0",
+												Pattern:     "^(\\d+m|\\d+(\\.\\d{1,3})?)$",
+											},
+											"memory": {
+												Type:        "string",
+												Description: "Plain integer or fixed-point integer using one of these suffixes: E, P, T, G, M, k (with or without a tailing i). Must be greater than 0",
+												Pattern:     "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$",
+											},
+										},
+									},
+									"requests": {
+										Type:     "object",
+										Required: []string{"cpu", "memory"},
+										Properties: map[string]apiextv1beta1.JSONSchemaProps{
+											"cpu": {
+												Type:        "string",
+												Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0",
+												Pattern:     "^(\\d+m|\\d+(\\.\\d{1,3})?)$",
+											},
+											"memory": {
+												Type:        "string",
+												Description: "Plain integer or fixed-point integer using one of these suffixes: E, P, T, G, M, k (with or without a tailing i). Must be greater than 0",
+												Pattern:     "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$",
+											},
+										},
+									},
+								},
+							},
+							"schema": {
+								Type: "string",
+							},
+							"user": {
+								Type: "string",
+							},
+						},
+					},
 					"databases": {
 						Type: "object",
 						AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{
@@ -188,6 +259,9 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{
 					"dockerImage": {
 						Type: "string",
 					},
+					"enableConnectionPooler": {
+						Type: "boolean",
+					},
 					"enableLogicalBackup": {
 						Type: "boolean",
 					},
@@ -284,6 +358,12 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{
 							"maximum_lag_on_failover": {
 								Type: "integer",
 							},
+							"synchronous_mode": {
+								Type: "boolean",
+							},
+							"synchronous_mode_strict": {
+								Type: "boolean",
+							},
 						},
 					},
 					"podAnnotations": {
@@ -341,6 +421,43 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{
 							},
 						},
 					},
+					"preparedDatabases": {
+						Type: "object",
+						AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{
+							Schema: &apiextv1beta1.JSONSchemaProps{
+								Type: "object",
+								Properties: map[string]apiextv1beta1.JSONSchemaProps{
+									"defaultUsers": {
+										Type: "boolean",
+									},
+									"extensions": {
+										Type: "object",
+										AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{
+											Schema: &apiextv1beta1.JSONSchemaProps{
+												Type: "string",
+											},
+										},
+									},
+									"schemas": {
+										Type: "object",
+										AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{
+											Schema: &apiextv1beta1.JSONSchemaProps{
+												Type: "object",
+												Properties: map[string]apiextv1beta1.JSONSchemaProps{
+													"defaultUsers": {
+														Type: "boolean",
+													},
+													"defaultRoles": {
+														Type: "boolean",
+													},
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
 					"replicaLoadBalancer": {
 						Type:        "boolean",
 						Description: "Deprecated",
@@ -417,6 +534,27 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{
 					"teamId": {
 						Type: "string",
 					},
+					"tls": {
+						Type:     "object",
+						Required: []string{"secretName"},
+						Properties: map[string]apiextv1beta1.JSONSchemaProps{
+							"secretName": {
+								Type: "string",
+							},
+							"certificateFile": {
+								Type: "string",
+							},
+							"privateKeyFile": {
+								Type: "string",
+							},
+							"caFile": {
+								Type: "string",
+							},
+							"caSecretName": {
+								Type: "string",
+							},
+						},
+					},
 					"tolerations": {
 						Type: "array",
 						Items: &apiextv1beta1.JSONSchemaPropsOrArray{
@@ -584,6 +722,37 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{
 							},
 						},
 					},
+					"additionalVolumes": {
+						Type: "array",
+						Items: &apiextv1beta1.JSONSchemaPropsOrArray{
+							Schema: &apiextv1beta1.JSONSchemaProps{
+								Type:     "object",
+								Required: []string{"name", "mountPath", "volumeSource"},
+								Properties: map[string]apiextv1beta1.JSONSchemaProps{
+									"name": {
+										Type: "string",
+									},
+									"mountPath": {
+										Type: "string",
+									},
+									"targetContainers": {
+										Type: "array",
+										Items: &apiextv1beta1.JSONSchemaPropsOrArray{
+											Schema: &apiextv1beta1.JSONSchemaProps{
+												Type: "string",
+											},
+										},
+									},
+									"volumeSource": {
+										Type: "object",
+									},
+									"subPath": {
+										Type: "string",
+									},
+								},
+							},
+						},
+					},
 				},
 			},
 			"status": {
@@ -629,12 +798,18 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation
 					"enable_crd_validation": {
 						Type: "boolean",
 					},
+					"enable_lazy_spilo_upgrade": {
+						Type: "boolean",
+					},
 					"enable_shm_volume": {
 						Type: "boolean",
 					},
 					"etcd_host": {
 						Type: "string",
 					},
+					"kubernetes_use_configmaps": {
+						Type: "boolean",
+					},
 					"max_instances": {
 						Type:        "integer",
 						Description: "-1 = disabled",
@@ -662,6 +837,17 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation
 							},
 						},
 					},
+					"sidecars": {
+						Type: "array",
+						Items: &apiextv1beta1.JSONSchemaPropsOrArray{
+							Schema: &apiextv1beta1.JSONSchemaProps{
+								Type: "object",
+								AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{
+									Allows: true,
+								},
+							},
+						},
+					},
 					"workers": {
 						Type:    "integer",
 						Minimum: &min1,
@@ -702,6 +888,14 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation
 									},
 								},
 							},
+							"downscaler_annotations": {
+								Type: "array",
+								Items: &apiextv1beta1.JSONSchemaPropsOrArray{
+									Schema: &apiextv1beta1.JSONSchemaProps{
+										Type: "string",
+									},
+								},
+							},
 							"enable_init_containers": {
 								Type: "boolean",
 							},
@@ -1037,6 +1231,54 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation
 							},
 						},
 					},
+					"connection_pooler": {
+						Type: "object",
+						Properties: map[string]apiextv1beta1.JSONSchemaProps{
+							"connection_pooler_default_cpu_limit": {
+								Type:    "string",
+								Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$",
+							},
+							"connection_pooler_default_cpu_request": {
+								Type:    "string",
+								Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$",
+							},
+							"connection_pooler_default_memory_limit": {
+								Type:    "string",
+								Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$",
+							},
+							"connection_pooler_default_memory_request": {
+								Type:    "string",
+								Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$",
+							},
+							"connection_pooler_image": {
+								Type: "string",
+							},
+							"connection_pooler_max_db_connections": {
+								Type: "integer",
+							},
+							"connection_pooler_mode": {
+								Type: "string",
+								Enum: []apiextv1beta1.JSON{
+									{
+										Raw: []byte(`"session"`),
+									},
+									{
+										Raw: []byte(`"transaction"`),
+									},
+								},
+							},
+							"connection_pooler_number_of_instances": {
+								Type:    "integer",
+								Minimum: &min2,
+							},
+							"connection_pooler_schema": {
+								Type: "string",
+							},
+							"connection_pooler_user": {
+								Type: "string",
+							},
+						},
+					},
 				},
 			},
 			"status": {

pkg/apis/acid.zalan.do/v1/operator_configuration_type.go

@@ -1,11 +1,14 @@
 package v1
 
+// Operator configuration CRD definition, please use snake_case for field names.
+
 import (
 	"github.com/zalando/postgres-operator/pkg/util/config"
 
 	"time"
 
 	"github.com/zalando/postgres-operator/pkg/spec"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -53,24 +56,24 @@ type KubernetesMetaConfiguration struct {
 	EnableInitContainers                   *bool                 `json:"enable_init_containers,omitempty"`
 	EnableSidecars                         *bool                 `json:"enable_sidecars,omitempty"`
 	SecretNameTemplate                     config.StringTemplate `json:"secret_name_template,omitempty"`
-	ClusterDomain                          string                `json:"cluster_domain"`
+	ClusterDomain                          string                `json:"cluster_domain,omitempty"`
 	OAuthTokenSecretName                   spec.NamespacedName   `json:"oauth_token_secret_name,omitempty"`
 	InfrastructureRolesSecretName          spec.NamespacedName   `json:"infrastructure_roles_secret_name,omitempty"`
 	PodRoleLabel                           string                `json:"pod_role_label,omitempty"`
 	ClusterLabels                          map[string]string     `json:"cluster_labels,omitempty"`
 	InheritedLabels                        []string              `json:"inherited_labels,omitempty"`
+	DownscalerAnnotations                  []string              `json:"downscaler_annotations,omitempty"`
 	ClusterNameLabel                       string                `json:"cluster_name_label,omitempty"`
 	NodeReadinessLabel                     map[string]string     `json:"node_readiness_label,omitempty"`
 	CustomPodAnnotations                   map[string]string     `json:"custom_pod_annotations,omitempty"`
 	// TODO: use a proper toleration structure?
-	PodToleration map[string]string `json:"toleration,omitempty"`
-	// TODO: use namespacedname
-	PodEnvironmentConfigMap    string        `json:"pod_environment_configmap,omitempty"`
-	PodPriorityClassName       string        `json:"pod_priority_class_name,omitempty"`
-	MasterPodMoveTimeout       Duration      `json:"master_pod_move_timeout,omitempty"`
-	EnablePodAntiAffinity      bool          `json:"enable_pod_antiaffinity,omitempty"`
-	PodAntiAffinityTopologyKey string        `json:"pod_antiaffinity_topology_key,omitempty"`
-	PodManagementPolicy        string        `json:"pod_management_policy,omitempty"`
+	PodToleration              map[string]string   `json:"toleration,omitempty"`
+	PodEnvironmentConfigMap    spec.NamespacedName `json:"pod_environment_configmap,omitempty"`
+	PodPriorityClassName       string              `json:"pod_priority_class_name,omitempty"`
+	MasterPodMoveTimeout       Duration            `json:"master_pod_move_timeout,omitempty"`
+	EnablePodAntiAffinity      bool                `json:"enable_pod_antiaffinity,omitempty"`
+	PodAntiAffinityTopologyKey string              `json:"pod_antiaffinity_topology_key,omitempty"`
+	PodManagementPolicy        string              `json:"pod_management_policy,omitempty"`
 }
 
 // PostgresPodResourcesDefaults defines the spec of default resources
@@ -152,6 +155,20 @@ type ScalyrConfiguration struct {
 	ScalyrMemoryLimit   string `json:"scalyr_memory_limit,omitempty"`
 }
 
+// Defines default configuration for connection pooler
+type ConnectionPoolerConfiguration struct {
+	NumberOfInstances    *int32 `json:"connection_pooler_number_of_instances,omitempty"`
+	Schema               string `json:"connection_pooler_schema,omitempty"`
+	User                 string `json:"connection_pooler_user,omitempty"`
+	Image                string `json:"connection_pooler_image,omitempty"`
+	Mode                 string `json:"connection_pooler_mode,omitempty"`
+	MaxDBConnections     *int32 `json:"connection_pooler_max_db_connections,omitempty"`
+	DefaultCPURequest    string `json:"connection_pooler_default_cpu_request,omitempty"`
+	DefaultMemoryRequest string `json:"connection_pooler_default_memory_request,omitempty"`
+	DefaultCPULimit      string `json:"connection_pooler_default_cpu_limit,omitempty"`
+	DefaultMemoryLimit   string `json:"connection_pooler_default_memory_limit,omitempty"`
+}
+
 // OperatorLogicalBackupConfiguration defines configuration for logical backup
 type OperatorLogicalBackupConfiguration struct {
 	Schedule          string `json:"logical_backup_schedule,omitempty"`
@@ -166,17 +183,21 @@ type OperatorLogicalBackupConfiguration struct {
 
 // OperatorConfigurationData defines the operation config
 type OperatorConfigurationData struct {
-	EnableCRDValidation        *bool                              `json:"enable_crd_validation,omitempty"`
-	EtcdHost                   string                             `json:"etcd_host,omitempty"`
-	DockerImage                string                             `json:"docker_image,omitempty"`
-	Workers                    uint32                             `json:"workers,omitempty"`
-	MinInstances               int32                              `json:"min_instances,omitempty"`
-	MaxInstances               int32                              `json:"max_instances,omitempty"`
-	ResyncPeriod               Duration                           `json:"resync_period,omitempty"`
-	RepairPeriod               Duration                           `json:"repair_period,omitempty"`
-	SetMemoryRequestToLimit    bool                               `json:"set_memory_request_to_limit,omitempty"`
-	ShmVolume                  *bool                              `json:"enable_shm_volume,omitempty"`
-	Sidecars                   map[string]string                  `json:"sidecar_docker_images,omitempty"`
+	EnableCRDValidation     *bool    `json:"enable_crd_validation,omitempty"`
+	EnableLazySpiloUpgrade  bool     `json:"enable_lazy_spilo_upgrade,omitempty"`
+	EtcdHost                string   `json:"etcd_host,omitempty"`
+	KubernetesUseConfigMaps bool     `json:"kubernetes_use_configmaps,omitempty"`
+	DockerImage             string   `json:"docker_image,omitempty"`
+	Workers                 uint32   `json:"workers,omitempty"`
+	MinInstances            int32    `json:"min_instances,omitempty"`
+	MaxInstances            int32    `json:"max_instances,omitempty"`
+	ResyncPeriod            Duration `json:"resync_period,omitempty"`
+	RepairPeriod            Duration `json:"repair_period,omitempty"`
+	SetMemoryRequestToLimit bool     `json:"set_memory_request_to_limit,omitempty"`
+	ShmVolume               *bool    `json:"enable_shm_volume,omitempty"`
+	// deprecated in favour of SidecarContainers
+	SidecarImages              map[string]string                  `json:"sidecar_docker_images,omitempty"`
+	SidecarContainers          []v1.Container                     `json:"sidecars,omitempty"`
 	PostgresUsersConfiguration PostgresUsersConfiguration         `json:"users"`
 	Kubernetes                 KubernetesMetaConfiguration        `json:"kubernetes"`
 	PostgresPodResources       PostgresPodResourcesDefaults       `json:"postgres_pod_resources"`
@@ -188,6 +209,7 @@ type OperatorConfigurationData struct {
 	LoggingRESTAPI             LoggingRESTAPIConfiguration        `json:"logging_rest_api"`
 	Scalyr                     ScalyrConfiguration                `json:"scalyr"`
 	LogicalBackup              OperatorLogicalBackupConfiguration `json:"logical_backup"`
+	ConnectionPooler           ConnectionPoolerConfiguration      `json:"connection_pooler"`
 }
 
 //Duration shortens this frequently used name

pkg/apis/acid.zalan.do/v1/postgresql_type.go

@@ -1,5 +1,7 @@
 package v1
 
+// Postgres CRD definition, please use CamelCase for field names.
+
 import (
 	"time"
 
@@ -27,6 +29,9 @@ type PostgresSpec struct {
 	Patroni         `json:"patroni,omitempty"`
 	Resources       `json:"resources,omitempty"`
 
+	EnableConnectionPooler *bool             `json:"enableConnectionPooler,omitempty"`
+	ConnectionPooler       *ConnectionPooler `json:"connectionPooler,omitempty"`
+
 	TeamID      string `json:"teamId"`
 	DockerImage string `json:"dockerImage,omitempty"`
 
@@ -45,22 +50,25 @@ type PostgresSpec struct {
 	// load balancers' source ranges are the same for master and replica services
 	AllowedSourceRanges []string `json:"allowedSourceRanges"`
 
-	NumberOfInstances     int32                `json:"numberOfInstances"`
-	Users                 map[string]UserFlags `json:"users"`
-	MaintenanceWindows    []MaintenanceWindow  `json:"maintenanceWindows,omitempty"`
-	Clone                 CloneDescription     `json:"clone"`
-	ClusterName           string               `json:"-"`
-	Databases             map[string]string    `json:"databases,omitempty"`
-	Tolerations           []v1.Toleration      `json:"tolerations,omitempty"`
-	Sidecars              []Sidecar            `json:"sidecars,omitempty"`
-	InitContainers        []v1.Container       `json:"initContainers,omitempty"`
-	PodPriorityClassName  string               `json:"podPriorityClassName,omitempty"`
-	ShmVolume             *bool                `json:"enableShmVolume,omitempty"`
-	EnableLogicalBackup   bool                 `json:"enableLogicalBackup,omitempty"`
-	LogicalBackupSchedule string               `json:"logicalBackupSchedule,omitempty"`
-	StandbyCluster        *StandbyDescription  `json:"standby"`
-	PodAnnotations        map[string]string    `json:"podAnnotations"`
-	ServiceAnnotations    map[string]string    `json:"serviceAnnotations"`
+	NumberOfInstances     int32                       `json:"numberOfInstances"`
+	Users                 map[string]UserFlags        `json:"users"`
+	MaintenanceWindows    []MaintenanceWindow         `json:"maintenanceWindows,omitempty"`
+	Clone                 CloneDescription            `json:"clone"`
+	ClusterName           string                      `json:"-"`
+	Databases             map[string]string           `json:"databases,omitempty"`
+	PreparedDatabases     map[string]PreparedDatabase `json:"preparedDatabases,omitempty"`
+	Tolerations           []v1.Toleration             `json:"tolerations,omitempty"`
+	Sidecars              []Sidecar                   `json:"sidecars,omitempty"`
+	InitContainers        []v1.Container              `json:"initContainers,omitempty"`
+	PodPriorityClassName  string                      `json:"podPriorityClassName,omitempty"`
+	ShmVolume             *bool                       `json:"enableShmVolume,omitempty"`
+	EnableLogicalBackup   bool                        `json:"enableLogicalBackup,omitempty"`
+	LogicalBackupSchedule string                      `json:"logicalBackupSchedule,omitempty"`
+	StandbyCluster        *StandbyDescription         `json:"standby"`
+	PodAnnotations        map[string]string           `json:"podAnnotations"`
+	ServiceAnnotations    map[string]string           `json:"serviceAnnotations"`
+	TLS                   *TLSDescription             `json:"tls"`
+	AdditionalVolumes     []AdditionalVolume          `json:"additionalVolumes,omitempty"`
 
 	// deprecated json tags
 	InitContainersOld       []v1.Container `json:"init_containers,omitempty"`
@@ -77,6 +85,19 @@ type PostgresqlList struct {
 	Items []Postgresql `json:"items"`
 }
 
+// PreparedDatabase describes elements to be bootstrapped
+type PreparedDatabase struct {
+	PreparedSchemas map[string]PreparedSchema `json:"schemas,omitempty"`
+	DefaultUsers    bool                      `json:"defaultUsers,omitempty" defaults:"false"`
+	Extensions      map[string]string         `json:"extensions,omitempty"`
+}
+
+// PreparedSchema describes elements to be bootstrapped per schema
+type PreparedSchema struct {
+	DefaultRoles *bool `json:"defaultRoles,omitempty" defaults:"true"`
+	DefaultUsers bool  `json:"defaultUsers,omitempty" defaults:"false"`
+}
+
 // MaintenanceWindow describes the time window when the operator is allowed to do maintenance on a cluster.
 type MaintenanceWindow struct {
 	Everyday  bool
@@ -92,6 +113,14 @@ type Volume struct {
 	SubPath      string `json:"subPath,omitempty"`
 }
 
+type AdditionalVolume struct {
+	Name             string          `json:"name"`
+	MountPath        string          `json:"mountPath"`
+	SubPath          string          `json:"subPath"`
+	TargetContainers []string        `json:"targetContainers"`
+	VolumeSource     v1.VolumeSource `json:"volumeSource"`
+}
+
 // PostgresqlParam describes PostgreSQL version and pairs of configuration parameter name - values.
 type PostgresqlParam struct {
 	PgVersion  string            `json:"version"`
@@ -112,13 +141,15 @@ type Resources struct {
 
 // Patroni contains Patroni-specific configuration
 type Patroni struct {
-	InitDB               map[string]string            `json:"initdb"`
-	PgHba                []string                     `json:"pg_hba"`
-	TTL                  uint32                       `json:"ttl"`
-	LoopWait             uint32                       `json:"loop_wait"`
-	RetryTimeout         uint32                       `json:"retry_timeout"`
-	MaximumLagOnFailover float32                      `json:"maximum_lag_on_failover"` // float32 because https://github.com/kubernetes/kubernetes/issues/30213
-	Slots                map[string]map[string]string `json:"slots"`
+	InitDB                map[string]string            `json:"initdb"`
+	PgHba                 []string                     `json:"pg_hba"`
+	TTL                   uint32                       `json:"ttl"`
+	LoopWait              uint32                       `json:"loop_wait"`
+	RetryTimeout          uint32                       `json:"retry_timeout"`
+	MaximumLagOnFailover  float32                      `json:"maximum_lag_on_failover"` // float32 because https://github.com/kubernetes/kubernetes/issues/30213
+	Slots                 map[string]map[string]string `json:"slots"`
+	SynchronousMode       bool                         `json:"synchronous_mode"`
+	SynchronousModeStrict bool                         `json:"synchronous_mode_strict"`
 }
 
 //StandbyCluster
@@ -126,6 +157,14 @@ type StandbyDescription struct {
 	S3WalPath string `json:"s3_wal_path,omitempty"`
 }
 
+type TLSDescription struct {
+	SecretName      string `json:"secretName,omitempty"`
+	CertificateFile string `json:"certificateFile,omitempty"`
+	PrivateKeyFile  string `json:"privateKeyFile,omitempty"`
+	CAFile          string `json:"caFile,omitempty"`
+	CASecretName    string `json:"caSecretName,omitempty"`
+}
+
 // CloneDescription describes which cluster the new should clone and up to which point in time
 type CloneDescription struct {
 	ClusterName       string `json:"cluster,omitempty"`
@@ -154,3 +193,24 @@ type UserFlags []string
 type PostgresStatus struct {
 	PostgresClusterStatus string `json:"PostgresClusterStatus"`
 }
+
+// Options for connection pooler
+//
+// TODO: prepared snippets of configuration, one can choose via type, e.g.
+// pgbouncer-large (with higher resources) or odyssey-small (with smaller
+// resources)
+// Type              string `json:"type,omitempty"`
+//
+// TODO: figure out what other important parameters of the connection pooler it
+// makes sense to expose. E.g. pool size (min/max boundaries), max client
+// connections etc.
+type ConnectionPooler struct {
+	NumberOfInstances *int32 `json:"numberOfInstances,omitempty"`
+	Schema            string `json:"schema,omitempty"`
+	User              string `json:"user,omitempty"`
+	Mode              string `json:"mode,omitempty"`
+	DockerImage       string `json:"dockerImage,omitempty"`
+	MaxDBConnections  *int32 `json:"maxDBConnections,omitempty"`
+
+	Resources `json:"resources,omitempty"`
+}

pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go

@@ -47,6 +47,28 @@ func (in *AWSGCPConfiguration) DeepCopy() *AWSGCPConfiguration {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AdditionalVolume) DeepCopyInto(out *AdditionalVolume) {
+	*out = *in
+	if in.TargetContainers != nil {
+		in, out := &in.TargetContainers, &out.TargetContainers
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	in.VolumeSource.DeepCopyInto(&out.VolumeSource)
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AdditionalVolume.
+func (in *AdditionalVolume) DeepCopy() *AdditionalVolume {
+	if in == nil {
+		return nil
+	}
+	out := new(AdditionalVolume)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *CloneDescription) DeepCopyInto(out *CloneDescription) {
 	*out = *in
@@ -68,6 +90,59 @@ func (in *CloneDescription) DeepCopy() *CloneDescription {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ConnectionPooler) DeepCopyInto(out *ConnectionPooler) {
+	*out = *in
+	if in.NumberOfInstances != nil {
+		in, out := &in.NumberOfInstances, &out.NumberOfInstances
+		*out = new(int32)
+		**out = **in
+	}
+	if in.MaxDBConnections != nil {
+		in, out := &in.MaxDBConnections, &out.MaxDBConnections
+		*out = new(int32)
+		**out = **in
+	}
+	out.Resources = in.Resources
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectionPooler.
+func (in *ConnectionPooler) DeepCopy() *ConnectionPooler {
+	if in == nil {
+		return nil
+	}
+	out := new(ConnectionPooler)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ConnectionPoolerConfiguration) DeepCopyInto(out *ConnectionPoolerConfiguration) {
+	*out = *in
+	if in.NumberOfInstances != nil {
+		in, out := &in.NumberOfInstances, &out.NumberOfInstances
+		*out = new(int32)
+		**out = **in
+	}
+	if in.MaxDBConnections != nil {
+		in, out := &in.MaxDBConnections, &out.MaxDBConnections
+		*out = new(int32)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectionPoolerConfiguration.
+func (in *ConnectionPoolerConfiguration) DeepCopy() *ConnectionPoolerConfiguration {
+	if in == nil {
+		return nil
+	}
+	out := new(ConnectionPoolerConfiguration)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfiguration) {
 	*out = *in
@@ -105,6 +180,11 @@ func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfigura
 		*out = make([]string, len(*in))
 		copy(*out, *in)
 	}
+	if in.DownscalerAnnotations != nil {
+		in, out := &in.DownscalerAnnotations, &out.DownscalerAnnotations
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	if in.NodeReadinessLabel != nil {
 		in, out := &in.NodeReadinessLabel, &out.NodeReadinessLabel
 		*out = make(map[string]string, len(*in))
@@ -126,6 +206,7 @@ func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfigura
 			(*out)[key] = val
 		}
 	}
+	out.PodEnvironmentConfigMap = in.PodEnvironmentConfigMap
 	return
 }
 
@@ -236,13 +317,20 @@ func (in *OperatorConfigurationData) DeepCopyInto(out *OperatorConfigurationData
 		*out = new(bool)
 		**out = **in
 	}
-	if in.Sidecars != nil {
-		in, out := &in.Sidecars, &out.Sidecars
+	if in.SidecarImages != nil {
+		in, out := &in.SidecarImages, &out.SidecarImages
 		*out = make(map[string]string, len(*in))
 		for key, val := range *in {
 			(*out)[key] = val
 		}
 	}
+	if in.SidecarContainers != nil {
+		in, out := &in.SidecarContainers, &out.SidecarContainers
+		*out = make([]corev1.Container, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	out.PostgresUsersConfiguration = in.PostgresUsersConfiguration
 	in.Kubernetes.DeepCopyInto(&out.Kubernetes)
 	out.PostgresPodResources = in.PostgresPodResources
@@ -254,6 +342,7 @@ func (in *OperatorConfigurationData) DeepCopyInto(out *OperatorConfigurationData
 	out.LoggingRESTAPI = in.LoggingRESTAPI
 	out.Scalyr = in.Scalyr
 	out.LogicalBackup = in.LogicalBackup
+	in.ConnectionPooler.DeepCopyInto(&out.ConnectionPooler)
 	return
 }
 
@@ -416,6 +505,16 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) {
 	out.Volume = in.Volume
 	in.Patroni.DeepCopyInto(&out.Patroni)
 	out.Resources = in.Resources
+	if in.EnableConnectionPooler != nil {
+		in, out := &in.EnableConnectionPooler, &out.EnableConnectionPooler
+		*out = new(bool)
+		**out = **in
+	}
+	if in.ConnectionPooler != nil {
+		in, out := &in.ConnectionPooler, &out.ConnectionPooler
+		*out = new(ConnectionPooler)
+		(*in).DeepCopyInto(*out)
+	}
 	if in.SpiloFSGroup != nil {
 		in, out := &in.SpiloFSGroup, &out.SpiloFSGroup
 		*out = new(int64)
@@ -476,6 +575,13 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) {
 			(*out)[key] = val
 		}
 	}
+	if in.PreparedDatabases != nil {
+		in, out := &in.PreparedDatabases, &out.PreparedDatabases
+		*out = make(map[string]PreparedDatabase, len(*in))
+		for key, val := range *in {
+			(*out)[key] = *val.DeepCopy()
+		}
+	}
 	if in.Tolerations != nil {
 		in, out := &in.Tolerations, &out.Tolerations
 		*out = make([]corev1.Toleration, len(*in))
@@ -521,6 +627,18 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) {
 			(*out)[key] = val
 		}
 	}
+	if in.TLS != nil {
+		in, out := &in.TLS, &out.TLS
+		*out = new(TLSDescription)
+		**out = **in
+	}
+	if in.AdditionalVolumes != nil {
+		in, out := &in.AdditionalVolumes, &out.AdditionalVolumes
+		*out = make([]AdditionalVolume, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	if in.InitContainersOld != nil {
 		in, out := &in.InitContainersOld, &out.InitContainersOld
 		*out = make([]corev1.Container, len(*in))
@@ -657,6 +775,57 @@ func (in *PostgresqlParam) DeepCopy() *PostgresqlParam {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PreparedDatabase) DeepCopyInto(out *PreparedDatabase) {
+	*out = *in
+	if in.PreparedSchemas != nil {
+		in, out := &in.PreparedSchemas, &out.PreparedSchemas
+		*out = make(map[string]PreparedSchema, len(*in))
+		for key, val := range *in {
+			(*out)[key] = *val.DeepCopy()
+		}
+	}
+	if in.Extensions != nil {
+		in, out := &in.Extensions, &out.Extensions
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PreparedDatabase.
+func (in *PreparedDatabase) DeepCopy() *PreparedDatabase {
+	if in == nil {
+		return nil
+	}
+	out := new(PreparedDatabase)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PreparedSchema) DeepCopyInto(out *PreparedSchema) {
+	*out = *in
+	if in.DefaultRoles != nil {
+		in, out := &in.DefaultRoles, &out.DefaultRoles
+		*out = new(bool)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PreparedSchema.
+func (in *PreparedSchema) DeepCopy() *PreparedSchema {
+	if in == nil {
+		return nil
+	}
+	out := new(PreparedSchema)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ResourceDescription) DeepCopyInto(out *ResourceDescription) {
 	*out = *in
@@ -752,6 +921,22 @@ func (in *StandbyDescription) DeepCopy() *StandbyDescription {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *TLSDescription) DeepCopyInto(out *TLSDescription) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSDescription.
+func (in *TLSDescription) DeepCopy() *TLSDescription {
+	if in == nil {
+		return nil
+	}
+	out := new(TLSDescription)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *TeamsAPIConfiguration) DeepCopyInto(out *TeamsAPIConfiguration) {
 	*out = *in

pkg/cluster/cluster.go

@@ -3,14 +3,17 @@ package cluster
 // Postgres CustomResourceDefinition object i.e. Spilo
 
 import (
+	"context"
 	"database/sql"
 	"encoding/json"
 	"fmt"
 	"reflect"
 	"regexp"
+	"strings"
 	"sync"
 	"time"
 
+	"github.com/r3labs/diff"
 	"github.com/sirupsen/logrus"
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
@@ -19,8 +22,11 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/client-go/tools/reference"
 
 	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
+	"github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/scheme"
 	"github.com/zalando/postgres-operator/pkg/spec"
 	"github.com/zalando/postgres-operator/pkg/util"
 	"github.com/zalando/postgres-operator/pkg/util/config"
@@ -48,11 +54,26 @@ type Config struct {
 	PodServiceAccountRoleBinding *rbacv1.RoleBinding
 }
 
+// K8S objects that are belongs to a connection pooler
+type ConnectionPoolerObjects struct {
+	Deployment *appsv1.Deployment
+	Service    *v1.Service
+
+	// It could happen that a connection pooler was enabled, but the operator
+	// was not able to properly process a corresponding event or was restarted.
+	// In this case we will miss missing/require situation and a lookup function
+	// will not be installed. To avoid synchronizing it all the time to prevent
+	// this, we can remember the result in memory at least until the next
+	// restart.
+	LookupFunction bool
+}
+
 type kubeResources struct {
 	Services            map[PostgresRole]*v1.Service
 	Endpoints           map[PostgresRole]*v1.Endpoints
 	Secrets             map[types.UID]*v1.Secret
 	Statefulset         *appsv1.StatefulSet
+	ConnectionPooler    *ConnectionPoolerObjects
 	PodDisruptionBudget *policybeta1.PodDisruptionBudget
 	//Pods are treated separately
 	//PVCs are treated separately
@@ -64,6 +85,7 @@ type Cluster struct {
 	acidv1.Postgresql
 	Config
 	logger           *logrus.Entry
+	eventRecorder    record.EventRecorder
 	patroni          patroni.Interface
 	pgUsers          map[string]spec.PgUser
 	systemUsers      map[string]spec.PgUser
@@ -72,7 +94,7 @@ type Cluster struct {
 	pgDb             *sql.DB
 	mu               sync.Mutex
 	userSyncStrategy spec.UserSyncer
-	deleteOptions    *metav1.DeleteOptions
+	deleteOptions    metav1.DeleteOptions
 	podEventsQueue   *cache.FIFO
 
 	teamsAPIClient   teams.Interface
@@ -92,7 +114,7 @@ type compareStatefulsetResult struct {
 }
 
 // New creates a new cluster. This function should be called from a controller.
-func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgresql, logger *logrus.Entry) *Cluster {
+func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgresql, logger *logrus.Entry, eventRecorder record.EventRecorder) *Cluster {
 	deletePropagationPolicy := metav1.DeletePropagationOrphan
 
 	podEventsQueue := cache.NewFIFO(func(obj interface{}) (string, error) {
@@ -115,7 +137,7 @@ func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgres
 			Services:  make(map[PostgresRole]*v1.Service),
 			Endpoints: make(map[PostgresRole]*v1.Endpoints)},
 		userSyncStrategy: users.DefaultUserSyncStrategy{},
-		deleteOptions:    &metav1.DeleteOptions{PropagationPolicy: &deletePropagationPolicy},
+		deleteOptions:    metav1.DeleteOptions{PropagationPolicy: &deletePropagationPolicy},
 		podEventsQueue:   podEventsQueue,
 		KubeClient:       kubeClient,
 	}
@@ -123,7 +145,7 @@ func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgres
 	cluster.teamsAPIClient = teams.NewTeamsAPI(cfg.OpConfig.TeamsAPIUrl, logger)
 	cluster.oauthTokenGetter = newSecretOauthTokenGetter(&kubeClient, cfg.OpConfig.OAuthTokenSecretName)
 	cluster.patroni = patroni.New(cluster.logger)
-
+	cluster.eventRecorder = eventRecorder
 	return cluster
 }
 
@@ -149,6 +171,16 @@ func (c *Cluster) setProcessName(procName string, args ...interface{}) {
 	}
 }
 
+// GetReference of Postgres CR object
+// i.e. required to emit events to this resource
+func (c *Cluster) GetReference() *v1.ObjectReference {
+	ref, err := reference.GetReference(scheme.Scheme, &c.Postgresql)
+	if err != nil {
+		c.logger.Errorf("could not get reference for Postgresql CR %v/%v: %v", c.Postgresql.Namespace, c.Postgresql.Name, err)
+	}
+	return ref
+}
+
 // SetStatus of Postgres cluster
 // TODO: eventually switch to updateStatus() for kubernetes 1.11 and above
 func (c *Cluster) setStatus(status string) {
@@ -166,7 +198,8 @@ func (c *Cluster) setStatus(status string) {
 	// we cannot do a full scale update here without fetching the previous manifest (as the resourceVersion may differ),
 	// however, we could do patch without it. In the future, once /status subresource is there (starting Kubernetes 1.11)
 	// we should take advantage of it.
-	newspec, err := c.KubeClient.AcidV1ClientSet.AcidV1().Postgresqls(c.clusterNamespace()).Patch(c.Name, types.MergePatchType, patch, "status")
+	newspec, err := c.KubeClient.AcidV1ClientSet.AcidV1().Postgresqls(c.clusterNamespace()).Patch(
+		context.TODO(), c.Name, types.MergePatchType, patch, metav1.PatchOptions{}, "status")
 	if err != nil {
 		c.logger.Errorf("could not update status: %v", err)
 		// return as newspec is empty, see PR654
@@ -184,7 +217,8 @@ func (c *Cluster) isNewCluster() bool {
 func (c *Cluster) initUsers() error {
 	c.setProcessName("initializing users")
 
-	// clear our the previous state of the cluster users (in case we are running a sync).
+	// clear our the previous state of the cluster users (in case we are
+	// running a sync).
 	c.systemUsers = map[string]spec.PgUser{}
 	c.pgUsers = map[string]spec.PgUser{}
 
@@ -194,6 +228,10 @@ func (c *Cluster) initUsers() error {
 		return fmt.Errorf("could not init infrastructure roles: %v", err)
 	}
 
+	if err := c.initPreparedDatabaseRoles(); err != nil {
+		return fmt.Errorf("could not init default users: %v", err)
+	}
+
 	if err := c.initRobotUsers(); err != nil {
 		return fmt.Errorf("could not init robot users: %v", err)
 	}
@@ -226,6 +264,7 @@ func (c *Cluster) Create() error {
 	}()
 
 	c.setStatus(acidv1.ClusterStatusCreating)
+	c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Create", "Started creation of new cluster resources")
 
 	if err = c.enforceMinResourceLimits(&c.Spec); err != nil {
 		return fmt.Errorf("could not enforce minimum resource limits: %v", err)
@@ -244,6 +283,7 @@ func (c *Cluster) Create() error {
 				return fmt.Errorf("could not create %s endpoint: %v", role, err)
 			}
 			c.logger.Infof("endpoint %q has been successfully created", util.NameFromMeta(ep.ObjectMeta))
+			c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Endpoints", "Endpoint %q has been successfully created", util.NameFromMeta(ep.ObjectMeta))
 		}
 
 		if c.Services[role] != nil {
@@ -254,6 +294,7 @@ func (c *Cluster) Create() error {
 			return fmt.Errorf("could not create %s service: %v", role, err)
 		}
 		c.logger.Infof("%s service %q has been successfully created", role, util.NameFromMeta(service.ObjectMeta))
+		c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Services", "The service %q for role %s has been successfully created", util.NameFromMeta(service.ObjectMeta), role)
 	}
 
 	if err = c.initUsers(); err != nil {
@@ -265,6 +306,7 @@ func (c *Cluster) Create() error {
 		return fmt.Errorf("could not create secrets: %v", err)
 	}
 	c.logger.Infof("secrets have been successfully created")
+	c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Secrets", "The secrets have been successfully created")
 
 	if c.PodDisruptionBudget != nil {
 		return fmt.Errorf("pod disruption budget already exists in the cluster")
@@ -283,6 +325,7 @@ func (c *Cluster) Create() error {
 		return fmt.Errorf("could not create statefulset: %v", err)
 	}
 	c.logger.Infof("statefulset %q has been successfully created", util.NameFromMeta(ss.ObjectMeta))
+	c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "StatefulSet", "Statefulset %q has been successfully created", util.NameFromMeta(ss.ObjectMeta))
 
 	c.logger.Info("waiting for the cluster being ready")
 
@@ -291,9 +334,12 @@ func (c *Cluster) Create() error {
 		return err
 	}
 	c.logger.Infof("pods are ready")
+	c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "StatefulSet", "Pods are ready")
 
-	// create database objects unless we are running without pods or disabled that feature explicitly
+	// create database objects unless we are running without pods or disabled
+	// that feature explicitly
 	if !(c.databaseAccessDisabled() || c.getNumberOfInstances(&c.Spec) <= 0 || c.Spec.StandbyCluster != nil) {
+		c.logger.Infof("Create roles")
 		if err = c.createRoles(); err != nil {
 			return fmt.Errorf("could not create users: %v", err)
 		}
@@ -302,6 +348,9 @@ func (c *Cluster) Create() error {
 		if err = c.syncDatabases(); err != nil {
 			return fmt.Errorf("could not sync databases: %v", err)
 		}
+		if err = c.syncPreparedDatabases(); err != nil {
+			return fmt.Errorf("could not sync prepared databases: %v", err)
+		}
 		c.logger.Infof("databases have been successfully created")
 	}
 
@@ -316,6 +365,26 @@ func (c *Cluster) Create() error {
 		c.logger.Errorf("could not list resources: %v", err)
 	}
 
+	// Create connection pooler deployment and services if necessary. Since we
+	// need to perform some operations with the database itself (e.g. install
+	// lookup function), do it as the last step, when everything is available.
+	//
+	// Do not consider connection pooler as a strict requirement, and if
+	// something fails, report warning
+	if c.needConnectionPooler() {
+		if c.ConnectionPooler != nil {
+			c.logger.Warning("Connection pooler already exists in the cluster")
+			return nil
+		}
+		connectionPooler, err := c.createConnectionPooler(c.installLookupFunction)
+		if err != nil {
+			c.logger.Warningf("could not create connection pooler: %v", err)
+			return nil
+		}
+		c.logger.Infof("connection pooler %q has been successfully created",
+			util.NameFromMeta(connectionPooler.Deployment.ObjectMeta))
+	}
+
 	return nil
 }
 
@@ -409,6 +478,14 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa
 		}
 	}
 
+	// lazy Spilo update: modify the image in the statefulset itself but let its pods run with the old image
+	// until they are re-created for other reasons, for example node rotation
+	if c.OpConfig.EnableLazySpiloUpgrade && !reflect.DeepEqual(c.Statefulset.Spec.Template.Spec.Containers[0].Image, statefulSet.Spec.Template.Spec.Containers[0].Image) {
+		needsReplace = true
+		needsRollUpdate = false
+		reasons = append(reasons, "lazy Spilo update: new statefulset's pod image doesn't match the current one")
+	}
+
 	if needsRollUpdate || needsReplace {
 		match = false
 	}
@@ -440,8 +517,6 @@ func (c *Cluster) compareContainers(description string, setA, setB []v1.Containe
 	checks := []containerCheck{
 		newCheck("new statefulset %s's %s (index %d) name doesn't match the current one",
 			func(a, b v1.Container) bool { return a.Name != b.Name }),
-		newCheck("new statefulset %s's %s (index %d) image doesn't match the current one",
-			func(a, b v1.Container) bool { return a.Image != b.Image }),
 		newCheck("new statefulset %s's %s (index %d) ports don't match the current one",
 			func(a, b v1.Container) bool { return !reflect.DeepEqual(a.Ports, b.Ports) }),
 		newCheck("new statefulset %s's %s (index %d) resources don't match the current ones",
@@ -452,6 +527,11 @@ func (c *Cluster) compareContainers(description string, setA, setB []v1.Containe
 			func(a, b v1.Container) bool { return !reflect.DeepEqual(a.EnvFrom, b.EnvFrom) }),
 	}
 
+	if !c.OpConfig.EnableLazySpiloUpgrade {
+		checks = append(checks, newCheck("new statefulset %s's %s (index %d) image doesn't match the current one",
+			func(a, b v1.Container) bool { return a.Image != b.Image }))
+	}
+
 	for index, containerA := range setA {
 		containerB := setB[index]
 		for _, check := range checks {
@@ -514,6 +594,7 @@ func (c *Cluster) enforceMinResourceLimits(spec *acidv1.PostgresSpec) error {
 		}
 		if isSmaller {
 			c.logger.Warningf("defined CPU limit %s is below required minimum %s and will be set to it", cpuLimit, minCPULimit)
+			c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "ResourceLimits", "defined CPU limit %s is below required minimum %s and will be set to it", cpuLimit, minCPULimit)
 			spec.Resources.ResourceLimits.CPU = minCPULimit
 		}
 	}
@@ -526,6 +607,7 @@ func (c *Cluster) enforceMinResourceLimits(spec *acidv1.PostgresSpec) error {
 		}
 		if isSmaller {
 			c.logger.Warningf("defined memory limit %s is below required minimum %s and will be set to it", memoryLimit, minMemoryLimit)
+			c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "ResourceLimits", "defined memory limit %s is below required minimum %s and will be set to it", memoryLimit, minMemoryLimit)
 			spec.Resources.ResourceLimits.Memory = minMemoryLimit
 		}
 	}
@@ -554,10 +636,13 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
 		}
 	}()
 
-	if oldSpec.Spec.PgVersion != newSpec.Spec.PgVersion { // PG versions comparison
-		c.logger.Warningf("postgresql version change(%q -> %q) has no effect", oldSpec.Spec.PgVersion, newSpec.Spec.PgVersion)
+	if oldSpec.Spec.PostgresqlParam.PgVersion != newSpec.Spec.PostgresqlParam.PgVersion { // PG versions comparison
+		c.logger.Warningf("postgresql version change(%q -> %q) has no effect",
+			oldSpec.Spec.PostgresqlParam.PgVersion, newSpec.Spec.PostgresqlParam.PgVersion)
+		c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "PostgreSQL", "postgresql version change(%q -> %q) has no effect",
+			oldSpec.Spec.PostgresqlParam.PgVersion, newSpec.Spec.PostgresqlParam.PgVersion)
 		//we need that hack to generate statefulset with the old version
-		newSpec.Spec.PgVersion = oldSpec.Spec.PgVersion
+		newSpec.Spec.PostgresqlParam.PgVersion = oldSpec.Spec.PostgresqlParam.PgVersion
 	}
 
 	// Service
@@ -570,7 +655,12 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
 		}
 	}
 
-	if !reflect.DeepEqual(oldSpec.Spec.Users, newSpec.Spec.Users) {
+	// connection pooler needs one system user created, which is done in
+	// initUsers. Check if it needs to be called.
+	sameUsers := reflect.DeepEqual(oldSpec.Spec.Users, newSpec.Spec.Users) &&
+		reflect.DeepEqual(oldSpec.Spec.PreparedDatabases, newSpec.Spec.PreparedDatabases)
+	needConnectionPooler := c.needConnectionPoolerWorker(&newSpec.Spec)
+	if !sameUsers || needConnectionPooler {
 		c.logger.Debugf("syncing secrets")
 		if err := c.initUsers(); err != nil {
 			c.logger.Errorf("could not init users: %v", err)
@@ -621,8 +711,7 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
 			updateFailed = true
 			return
 		}
-
-		if !reflect.DeepEqual(oldSs, newSs) {
+		if !reflect.DeepEqual(oldSs, newSs) || !reflect.DeepEqual(oldSpec.Annotations, newSpec.Annotations) {
 			c.logger.Debugf("syncing statefulsets")
 			// TODO: avoid generating the StatefulSet object twice by passing it to syncStatefulSet
 			if err := c.syncStatefulSet(); err != nil {
@@ -685,13 +774,28 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
 			c.logger.Errorf("could not sync roles: %v", err)
 			updateFailed = true
 		}
-		if !reflect.DeepEqual(oldSpec.Spec.Databases, newSpec.Spec.Databases) {
+		if !reflect.DeepEqual(oldSpec.Spec.Databases, newSpec.Spec.Databases) ||
+			!reflect.DeepEqual(oldSpec.Spec.PreparedDatabases, newSpec.Spec.PreparedDatabases) {
 			c.logger.Infof("syncing databases")
 			if err := c.syncDatabases(); err != nil {
 				c.logger.Errorf("could not sync databases: %v", err)
 				updateFailed = true
 			}
 		}
+		if !reflect.DeepEqual(oldSpec.Spec.PreparedDatabases, newSpec.Spec.PreparedDatabases) {
+			c.logger.Infof("syncing prepared databases")
+			if err := c.syncPreparedDatabases(); err != nil {
+				c.logger.Errorf("could not sync prepared databases: %v", err)
+				updateFailed = true
+			}
+		}
+	}
+
+	// sync connection pooler
+	if _, err := c.syncConnectionPooler(oldSpec, newSpec,
+		c.installLookupFunction); err != nil {
+		c.logger.Errorf("could not sync connection pooler: %v", err)
+		updateFailed = true
 	}
 
 	return nil
@@ -705,6 +809,7 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
 func (c *Cluster) Delete() {
 	c.mu.Lock()
 	defer c.mu.Unlock()
+	c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Delete", "Started deletion of new cluster resources")
 
 	// delete the backup job before the stateful set of the cluster to prevent connections to non-existing pods
 	// deleting the cron job also removes pods and batch jobs it created
@@ -732,8 +837,10 @@ func (c *Cluster) Delete() {
 
 	for _, role := range []PostgresRole{Master, Replica} {
 
-		if err := c.deleteEndpoint(role); err != nil {
-			c.logger.Warningf("could not delete %s endpoint: %v", role, err)
+		if !c.patroniKubernetesUseConfigMaps() {
+			if err := c.deleteEndpoint(role); err != nil {
+				c.logger.Warningf("could not delete %s endpoint: %v", role, err)
+			}
 		}
 
 		if err := c.deleteService(role); err != nil {
@@ -745,6 +852,12 @@ func (c *Cluster) Delete() {
 		c.logger.Warningf("could not remove leftover patroni objects; %v", err)
 	}
 
+	// Delete connection pooler objects anyway, even if it's not mentioned in the
+	// manifest, just to not keep orphaned components in case if something went
+	// wrong
+	if err := c.deleteConnectionPooler(); err != nil {
+		c.logger.Warningf("could not remove connection pooler: %v", err)
+	}
 }
 
 //NeedsRepair returns true if the cluster should be included in the repair scan (based on its in-memory status).
@@ -811,6 +924,140 @@ func (c *Cluster) initSystemUsers() {
 		Name:     c.OpConfig.ReplicationUsername,
 		Password: util.RandomPassword(constants.PasswordLength),
 	}
+
+	// Connection pooler user is an exception, if requested it's going to be
+	// created by operator as a normal pgUser
+	if c.needConnectionPooler() {
+		// initialize empty connection pooler if not done yet
+		if c.Spec.ConnectionPooler == nil {
+			c.Spec.ConnectionPooler = &acidv1.ConnectionPooler{}
+		}
+
+		// Using superuser as pooler user is not a good idea. First of all it's
+		// not going to be synced correctly with the current implementation,
+		// and second it's a bad practice.
+		username := c.OpConfig.ConnectionPooler.User
+
+		isSuperUser := c.Spec.ConnectionPooler.User == c.OpConfig.SuperUsername
+		isProtectedUser := c.shouldAvoidProtectedOrSystemRole(
+			c.Spec.ConnectionPooler.User, "connection pool role")
+
+		if !isSuperUser && !isProtectedUser {
+			username = util.Coalesce(
+				c.Spec.ConnectionPooler.User,
+				c.OpConfig.ConnectionPooler.User)
+		}
+
+		// connection pooler application should be able to login with this role
+		connectionPoolerUser := spec.PgUser{
+			Origin:   spec.RoleConnectionPooler,
+			Name:     username,
+			Flags:    []string{constants.RoleFlagLogin},
+			Password: util.RandomPassword(constants.PasswordLength),
+		}
+
+		if _, exists := c.pgUsers[username]; !exists {
+			c.pgUsers[username] = connectionPoolerUser
+		}
+
+		if _, exists := c.systemUsers[constants.ConnectionPoolerUserKeyName]; !exists {
+			c.systemUsers[constants.ConnectionPoolerUserKeyName] = connectionPoolerUser
+		}
+	}
+}
+
+func (c *Cluster) initPreparedDatabaseRoles() error {
+
+	if c.Spec.PreparedDatabases != nil && len(c.Spec.PreparedDatabases) == 0 { // TODO: add option to disable creating such a default DB
+		c.Spec.PreparedDatabases = map[string]acidv1.PreparedDatabase{strings.Replace(c.Name, "-", "_", -1): {}}
+	}
+
+	// create maps with default roles/users as keys and their membership as values
+	defaultRoles := map[string]string{
+		constants.OwnerRoleNameSuffix:  "",
+		constants.ReaderRoleNameSuffix: "",
+		constants.WriterRoleNameSuffix: constants.ReaderRoleNameSuffix,
+	}
+	defaultUsers := map[string]string{
+		constants.OwnerRoleNameSuffix + constants.UserRoleNameSuffix:  constants.OwnerRoleNameSuffix,
+		constants.ReaderRoleNameSuffix + constants.UserRoleNameSuffix: constants.ReaderRoleNameSuffix,
+		constants.WriterRoleNameSuffix + constants.UserRoleNameSuffix: constants.WriterRoleNameSuffix,
+	}
+
+	for preparedDbName, preparedDB := range c.Spec.PreparedDatabases {
+		// default roles per database
+		if err := c.initDefaultRoles(defaultRoles, "admin", preparedDbName); err != nil {
+			return fmt.Errorf("could not initialize default roles for database %s: %v", preparedDbName, err)
+		}
+		if preparedDB.DefaultUsers {
+			if err := c.initDefaultRoles(defaultUsers, "admin", preparedDbName); err != nil {
+				return fmt.Errorf("could not initialize default roles for database %s: %v", preparedDbName, err)
+			}
+		}
+
+		// default roles per database schema
+		preparedSchemas := preparedDB.PreparedSchemas
+		if len(preparedDB.PreparedSchemas) == 0 {
+			preparedSchemas = map[string]acidv1.PreparedSchema{"data": {DefaultRoles: util.True()}}
+		}
+		for preparedSchemaName, preparedSchema := range preparedSchemas {
+			if preparedSchema.DefaultRoles == nil || *preparedSchema.DefaultRoles {
+				if err := c.initDefaultRoles(defaultRoles,
+					preparedDbName+constants.OwnerRoleNameSuffix,
+					preparedDbName+"_"+preparedSchemaName); err != nil {
+					return fmt.Errorf("could not initialize default roles for database schema %s: %v", preparedSchemaName, err)
+				}
+				if preparedSchema.DefaultUsers {
+					if err := c.initDefaultRoles(defaultUsers,
+						preparedDbName+constants.OwnerRoleNameSuffix,
+						preparedDbName+"_"+preparedSchemaName); err != nil {
+						return fmt.Errorf("could not initialize default users for database schema %s: %v", preparedSchemaName, err)
+					}
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func (c *Cluster) initDefaultRoles(defaultRoles map[string]string, admin, prefix string) error {
+
+	for defaultRole, inherits := range defaultRoles {
+
+		roleName := prefix + defaultRole
+
+		flags := []string{constants.RoleFlagNoLogin}
+		if defaultRole[len(defaultRole)-5:] == constants.UserRoleNameSuffix {
+			flags = []string{constants.RoleFlagLogin}
+		}
+
+		memberOf := make([]string, 0)
+		if inherits != "" {
+			memberOf = append(memberOf, prefix+inherits)
+		}
+
+		adminRole := ""
+		if strings.Contains(defaultRole, constants.OwnerRoleNameSuffix) {
+			adminRole = admin
+		} else {
+			adminRole = prefix + constants.OwnerRoleNameSuffix
+		}
+
+		newRole := spec.PgUser{
+			Origin:    spec.RoleOriginBootstrap,
+			Name:      roleName,
+			Password:  util.RandomPassword(constants.PasswordLength),
+			Flags:     flags,
+			MemberOf:  memberOf,
+			AdminRole: adminRole,
+		}
+		if currentRole, present := c.pgUsers[roleName]; present {
+			c.pgUsers[roleName] = c.resolveNameConflict(&currentRole, &newRole)
+		} else {
+			c.pgUsers[roleName] = newRole
+		}
+	}
+	return nil
 }
 
 func (c *Cluster) initRobotUsers() error {
@@ -995,6 +1242,7 @@ func (c *Cluster) Switchover(curMaster *v1.Pod, candidate spec.NamespacedName) e
 
 	var err error
 	c.logger.Debugf("switching over from %q to %q", curMaster.Name, candidate)
+	c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Switchover", "Switching over from %q to %q", curMaster.Name, candidate)
 
 	var wg sync.WaitGroup
 
@@ -1021,6 +1269,7 @@ func (c *Cluster) Switchover(curMaster *v1.Pod, candidate spec.NamespacedName) e
 
 	if err = c.patroni.Switchover(curMaster, candidate.Name); err == nil {
 		c.logger.Debugf("successfully switched over from %q to %q", curMaster.Name, candidate)
+		c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Switchover", "Successfully switched over from %q to %q", curMaster.Name, candidate)
 		if err = <-podLabelErr; err != nil {
 			err = fmt.Errorf("could not get master pod label: %v", err)
 		}
@@ -1036,6 +1285,7 @@ func (c *Cluster) Switchover(curMaster *v1.Pod, candidate spec.NamespacedName) e
 	// close the label waiting channel no sooner than the waiting goroutine terminates.
 	close(podLabelErr)
 
+	c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Switchover", "Switchover from %q to %q FAILED: %v", curMaster.Name, candidate, err)
 	return err
 
 }
@@ -1063,11 +1313,19 @@ type clusterObjectDelete func(name string) error
 
 func (c *Cluster) deletePatroniClusterObjects() error {
 	// TODO: figure out how to remove leftover patroni objects in other cases
+	var actionsList []simpleActionWithResult
+
 	if !c.patroniUsesKubernetes() {
 		c.logger.Infof("not cleaning up Etcd Patroni objects on cluster delete")
 	}
-	c.logger.Debugf("removing leftover Patroni objects (endpoints, services and configmaps)")
-	for _, deleter := range []simpleActionWithResult{c.deletePatroniClusterEndpoints, c.deletePatroniClusterServices, c.deletePatroniClusterConfigMaps} {
+
+	if !c.patroniKubernetesUseConfigMaps() {
+		actionsList = append(actionsList, c.deletePatroniClusterEndpoints)
+	}
+	actionsList = append(actionsList, c.deletePatroniClusterServices, c.deletePatroniClusterConfigMaps)
+
+	c.logger.Debugf("removing leftover Patroni objects (endpoints / services and configmaps)")
+	for _, deleter := range actionsList {
 		if err := deleter(); err != nil {
 			return err
 		}
@@ -1101,12 +1359,12 @@ func (c *Cluster) deleteClusterObject(
 
 func (c *Cluster) deletePatroniClusterServices() error {
 	get := func(name string) (spec.NamespacedName, error) {
-		svc, err := c.KubeClient.Services(c.Namespace).Get(name, metav1.GetOptions{})
+		svc, err := c.KubeClient.Services(c.Namespace).Get(context.TODO(), name, metav1.GetOptions{})
 		return util.NameFromMeta(svc.ObjectMeta), err
 	}
 
 	deleteServiceFn := func(name string) error {
-		return c.KubeClient.Services(c.Namespace).Delete(name, c.deleteOptions)
+		return c.KubeClient.Services(c.Namespace).Delete(context.TODO(), name, c.deleteOptions)
 	}
 
 	return c.deleteClusterObject(get, deleteServiceFn, "service")
@@ -1114,12 +1372,12 @@ func (c *Cluster) deletePatroniClusterServices() error {
 
 func (c *Cluster) deletePatroniClusterEndpoints() error {
 	get := func(name string) (spec.NamespacedName, error) {
-		ep, err := c.KubeClient.Endpoints(c.Namespace).Get(name, metav1.GetOptions{})
+		ep, err := c.KubeClient.Endpoints(c.Namespace).Get(context.TODO(), name, metav1.GetOptions{})
 		return util.NameFromMeta(ep.ObjectMeta), err
 	}
 
 	deleteEndpointFn := func(name string) error {
-		return c.KubeClient.Endpoints(c.Namespace).Delete(name, c.deleteOptions)
+		return c.KubeClient.Endpoints(c.Namespace).Delete(context.TODO(), name, c.deleteOptions)
 	}
 
 	return c.deleteClusterObject(get, deleteEndpointFn, "endpoint")
@@ -1127,13 +1385,129 @@ func (c *Cluster) deletePatroniClusterEndpoints() error {
 
 func (c *Cluster) deletePatroniClusterConfigMaps() error {
 	get := func(name string) (spec.NamespacedName, error) {
-		cm, err := c.KubeClient.ConfigMaps(c.Namespace).Get(name, metav1.GetOptions{})
+		cm, err := c.KubeClient.ConfigMaps(c.Namespace).Get(context.TODO(), name, metav1.GetOptions{})
 		return util.NameFromMeta(cm.ObjectMeta), err
 	}
 
 	deleteConfigMapFn := func(name string) error {
-		return c.KubeClient.ConfigMaps(c.Namespace).Delete(name, c.deleteOptions)
+		return c.KubeClient.ConfigMaps(c.Namespace).Delete(context.TODO(), name, c.deleteOptions)
 	}
 
 	return c.deleteClusterObject(get, deleteConfigMapFn, "configmap")
 }
+
+// Test if two connection pooler configuration needs to be synced. For simplicity
+// compare not the actual K8S objects, but the configuration itself and request
+// sync if there is any difference.
+func (c *Cluster) needSyncConnectionPoolerSpecs(oldSpec, newSpec *acidv1.ConnectionPooler) (sync bool, reasons []string) {
+	reasons = []string{}
+	sync = false
+
+	changelog, err := diff.Diff(oldSpec, newSpec)
+	if err != nil {
+		c.logger.Infof("Cannot get diff, do not do anything, %+v", err)
+		return false, reasons
+	}
+
+	if len(changelog) > 0 {
+		sync = true
+	}
+
+	for _, change := range changelog {
+		msg := fmt.Sprintf("%s %+v from '%+v' to '%+v'",
+			change.Type, change.Path, change.From, change.To)
+		reasons = append(reasons, msg)
+	}
+
+	return sync, reasons
+}
+
+func syncResources(a, b *v1.ResourceRequirements) bool {
+	for _, res := range []v1.ResourceName{
+		v1.ResourceCPU,
+		v1.ResourceMemory,
+	} {
+		if !a.Limits[res].Equal(b.Limits[res]) ||
+			!a.Requests[res].Equal(b.Requests[res]) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// Check if we need to synchronize connection pooler deployment due to new
+// defaults, that are different from what we see in the DeploymentSpec
+func (c *Cluster) needSyncConnectionPoolerDefaults(
+	spec *acidv1.ConnectionPooler,
+	deployment *appsv1.Deployment) (sync bool, reasons []string) {
+
+	reasons = []string{}
+	sync = false
+
+	config := c.OpConfig.ConnectionPooler
+	podTemplate := deployment.Spec.Template
+	poolerContainer := podTemplate.Spec.Containers[constants.ConnectionPoolerContainer]
+
+	if spec == nil {
+		spec = &acidv1.ConnectionPooler{}
+	}
+
+	if spec.NumberOfInstances == nil &&
+		*deployment.Spec.Replicas != *config.NumberOfInstances {
+
+		sync = true
+		msg := fmt.Sprintf("NumberOfInstances is different (having %d, required %d)",
+			*deployment.Spec.Replicas, *config.NumberOfInstances)
+		reasons = append(reasons, msg)
+	}
+
+	if spec.DockerImage == "" &&
+		poolerContainer.Image != config.Image {
+
+		sync = true
+		msg := fmt.Sprintf("DockerImage is different (having %s, required %s)",
+			poolerContainer.Image, config.Image)
+		reasons = append(reasons, msg)
+	}
+
+	expectedResources, err := generateResourceRequirements(spec.Resources,
+		c.makeDefaultConnectionPoolerResources())
+
+	// An error to generate expected resources means something is not quite
+	// right, but for the purpose of robustness do not panic here, just report
+	// and ignore resources comparison (in the worst case there will be no
+	// updates for new resource values).
+	if err == nil && syncResources(&poolerContainer.Resources, expectedResources) {
+		sync = true
+		msg := fmt.Sprintf("Resources are different (having %+v, required %+v)",
+			poolerContainer.Resources, expectedResources)
+		reasons = append(reasons, msg)
+	}
+
+	if err != nil {
+		c.logger.Warningf("Cannot generate expected resources, %v", err)
+	}
+
+	for _, env := range poolerContainer.Env {
+		if spec.User == "" && env.Name == "PGUSER" {
+			ref := env.ValueFrom.SecretKeyRef.LocalObjectReference
+
+			if ref.Name != c.credentialSecretName(config.User) {
+				sync = true
+				msg := fmt.Sprintf("pooler user is different (having %s, required %s)",
+					ref.Name, config.User)
+				reasons = append(reasons, msg)
+			}
+		}
+
+		if spec.Schema == "" && env.Name == "PGSCHEMA" && env.Value != config.Schema {
+			sync = true
+			msg := fmt.Sprintf("pooler schema is different (having %s, required %s)",
+				env.Value, config.Schema)
+			reasons = append(reasons, msg)
+		}
+	}
+
+	return sync, reasons
+}

pkg/cluster/cluster_test.go

@@ -9,9 +9,12 @@ import (
 	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
 	"github.com/zalando/postgres-operator/pkg/spec"
 	"github.com/zalando/postgres-operator/pkg/util/config"
+	"github.com/zalando/postgres-operator/pkg/util/constants"
 	"github.com/zalando/postgres-operator/pkg/util/k8sutil"
 	"github.com/zalando/postgres-operator/pkg/util/teams"
 	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/tools/record"
 )
 
 const (
@@ -20,21 +23,53 @@ const (
 )
 
 var logger = logrus.New().WithField("test", "cluster")
+var eventRecorder = record.NewFakeRecorder(1)
+
 var cl = New(
 	Config{
 		OpConfig: config.Config{
-			ProtectedRoles: []string{"admin"},
+			PodManagementPolicy: "ordered_ready",
+			ProtectedRoles:      []string{"admin"},
 			Auth: config.Auth{
 				SuperUsername:       superUserName,
 				ReplicationUsername: replicationUserName,
 			},
+			Resources: config.Resources{
+				DownscalerAnnotations: []string{"downscaler/*"},
+			},
 		},
 	},
 	k8sutil.NewMockKubernetesClient(),
-	acidv1.Postgresql{},
+	acidv1.Postgresql{ObjectMeta: metav1.ObjectMeta{Name: "acid-test", Namespace: "test", Annotations: map[string]string{"downscaler/downtime_replicas": "0"}}},
 	logger,
+	eventRecorder,
 )
 
+func TestStatefulSetAnnotations(t *testing.T) {
+	testName := "CheckStatefulsetAnnotations"
+	spec := acidv1.PostgresSpec{
+		TeamID: "myapp", NumberOfInstances: 1,
+		Resources: acidv1.Resources{
+			ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"},
+			ResourceLimits:   acidv1.ResourceDescription{CPU: "1", Memory: "10"},
+		},
+		Volume: acidv1.Volume{
+			Size: "1G",
+		},
+	}
+	ss, err := cl.generateStatefulSet(&spec)
+	if err != nil {
+		t.Errorf("in %s no statefulset created %v", testName, err)
+	}
+	if ss != nil {
+		annotation := ss.ObjectMeta.GetAnnotations()
+		if _, ok := annotation["downscaler/downtime_replicas"]; !ok {
+			t.Errorf("in %s respective annotation not found on sts", testName)
+		}
+	}
+
+}
+
 func TestInitRobotUsers(t *testing.T) {
 	testName := "TestInitRobotUsers"
 	tests := []struct {
@@ -704,3 +739,140 @@ func TestServiceAnnotations(t *testing.T) {
 		})
 	}
 }
+
+func TestInitSystemUsers(t *testing.T) {
+	testName := "Test system users initialization"
+
+	// default cluster without connection pooler
+	cl.initSystemUsers()
+	if _, exist := cl.systemUsers[constants.ConnectionPoolerUserKeyName]; exist {
+		t.Errorf("%s, connection pooler user is present", testName)
+	}
+
+	// cluster with connection pooler
+	cl.Spec.EnableConnectionPooler = boolToPointer(true)
+	cl.initSystemUsers()
+	if _, exist := cl.systemUsers[constants.ConnectionPoolerUserKeyName]; !exist {
+		t.Errorf("%s, connection pooler user is not present", testName)
+	}
+
+	// superuser is not allowed as connection pool user
+	cl.Spec.ConnectionPooler = &acidv1.ConnectionPooler{
+		User: "postgres",
+	}
+	cl.OpConfig.SuperUsername = "postgres"
+	cl.OpConfig.ConnectionPooler.User = "pooler"
+
+	cl.initSystemUsers()
+	if _, exist := cl.pgUsers["pooler"]; !exist {
+		t.Errorf("%s, Superuser is not allowed to be a connection pool user", testName)
+	}
+
+	// neither protected users are
+	delete(cl.pgUsers, "pooler")
+	cl.Spec.ConnectionPooler = &acidv1.ConnectionPooler{
+		User: "admin",
+	}
+	cl.OpConfig.ProtectedRoles = []string{"admin"}
+
+	cl.initSystemUsers()
+	if _, exist := cl.pgUsers["pooler"]; !exist {
+		t.Errorf("%s, Protected user are not allowed to be a connection pool user", testName)
+	}
+
+	delete(cl.pgUsers, "pooler")
+	cl.Spec.ConnectionPooler = &acidv1.ConnectionPooler{
+		User: "standby",
+	}
+
+	cl.initSystemUsers()
+	if _, exist := cl.pgUsers["pooler"]; !exist {
+		t.Errorf("%s, System users are not allowed to be a connection pool user", testName)
+	}
+}
+
+func TestPreparedDatabases(t *testing.T) {
+	testName := "TestDefaultPreparedDatabase"
+
+	cl.Spec.PreparedDatabases = map[string]acidv1.PreparedDatabase{}
+	cl.initPreparedDatabaseRoles()
+
+	for _, role := range []string{"acid_test_owner", "acid_test_reader", "acid_test_writer",
+		"acid_test_data_owner", "acid_test_data_reader", "acid_test_data_writer"} {
+		if _, exist := cl.pgUsers[role]; !exist {
+			t.Errorf("%s, default role %q for prepared database not present", testName, role)
+		}
+	}
+
+	testName = "TestPreparedDatabaseWithSchema"
+
+	cl.Spec.PreparedDatabases = map[string]acidv1.PreparedDatabase{
+		"foo": {
+			DefaultUsers: true,
+			PreparedSchemas: map[string]acidv1.PreparedSchema{
+				"bar": {
+					DefaultUsers: true,
+				},
+			},
+		},
+	}
+	cl.initPreparedDatabaseRoles()
+
+	for _, role := range []string{
+		"foo_owner", "foo_reader", "foo_writer",
+		"foo_owner_user", "foo_reader_user", "foo_writer_user",
+		"foo_bar_owner", "foo_bar_reader", "foo_bar_writer",
+		"foo_bar_owner_user", "foo_bar_reader_user", "foo_bar_writer_user"} {
+		if _, exist := cl.pgUsers[role]; !exist {
+			t.Errorf("%s, default role %q for prepared database not present", testName, role)
+		}
+	}
+
+	roleTests := []struct {
+		subTest  string
+		role     string
+		memberOf string
+		admin    string
+	}{
+		{
+			subTest:  "Test admin role of owner",
+			role:     "foo_owner",
+			memberOf: "",
+			admin:    "admin",
+		},
+		{
+			subTest:  "Test writer is a member of reader",
+			role:     "foo_writer",
+			memberOf: "foo_reader",
+			admin:    "foo_owner",
+		},
+		{
+			subTest:  "Test reader LOGIN role",
+			role:     "foo_reader_user",
+			memberOf: "foo_reader",
+			admin:    "foo_owner",
+		},
+		{
+			subTest:  "Test schema owner",
+			role:     "foo_bar_owner",
+			memberOf: "",
+			admin:    "foo_owner",
+		},
+		{
+			subTest:  "Test schema writer LOGIN role",
+			role:     "foo_bar_writer_user",
+			memberOf: "foo_bar_writer",
+			admin:    "foo_bar_owner",
+		},
+	}
+
+	for _, tt := range roleTests {
+		user := cl.pgUsers[tt.role]
+		if (tt.memberOf == "" && len(user.MemberOf) > 0) || (tt.memberOf != "" && user.MemberOf[0] != tt.memberOf) {
+			t.Errorf("%s, incorrect membership for default role %q. Expected %q, got %q", tt.subTest, tt.role, tt.memberOf, user.MemberOf[0])
+		}
+		if user.AdminRole != tt.admin {
+			t.Errorf("%s, incorrect admin role for default role %q. Expected %q, got %q", tt.subTest, tt.role, tt.admin, user.AdminRole)
+		}
+	}
+}

pkg/cluster/database.go

@@ -1,10 +1,12 @@
 package cluster
 
 import (
+	"bytes"
 	"database/sql"
 	"fmt"
 	"net"
 	"strings"
+	"text/template"
 	"time"
 
 	"github.com/lib/pq"
@@ -25,16 +27,66 @@ const (
 	 WHERE a.rolname = ANY($1)
 	 ORDER BY 1;`
 
-	getDatabasesSQL       = `SELECT datname, pg_get_userbyid(datdba) AS owner FROM pg_database;`
-	createDatabaseSQL     = `CREATE DATABASE "%s" OWNER "%s";`
-	alterDatabaseOwnerSQL = `ALTER DATABASE "%s" OWNER TO "%s";`
+	getDatabasesSQL = `SELECT datname, pg_get_userbyid(datdba) AS owner FROM pg_database;`
+	getSchemasSQL   = `SELECT n.nspname AS dbschema FROM pg_catalog.pg_namespace n
+			WHERE n.nspname !~ '^pg_' AND n.nspname <> 'information_schema' ORDER BY 1`
+	getExtensionsSQL = `SELECT e.extname, n.nspname FROM pg_catalog.pg_extension e
+	        LEFT JOIN pg_catalog.pg_namespace n ON n.oid = e.extnamespace ORDER BY 1;`
+
+	createDatabaseSQL       = `CREATE DATABASE "%s" OWNER "%s";`
+	createDatabaseSchemaSQL = `SET ROLE TO "%s"; CREATE SCHEMA IF NOT EXISTS "%s" AUTHORIZATION "%s"`
+	alterDatabaseOwnerSQL   = `ALTER DATABASE "%s" OWNER TO "%s";`
+	createExtensionSQL      = `CREATE EXTENSION IF NOT EXISTS "%s" SCHEMA "%s"`
+	alterExtensionSQL       = `ALTER EXTENSION "%s" SET SCHEMA "%s"`
+
+	globalDefaultPrivilegesSQL = `SET ROLE TO "%s";
+			ALTER DEFAULT PRIVILEGES GRANT USAGE ON SCHEMAS TO "%s","%s";
+			ALTER DEFAULT PRIVILEGES GRANT SELECT ON TABLES TO "%s";
+			ALTER DEFAULT PRIVILEGES GRANT SELECT ON SEQUENCES TO "%s";
+			ALTER DEFAULT PRIVILEGES GRANT INSERT, UPDATE, DELETE ON TABLES TO "%s";
+			ALTER DEFAULT PRIVILEGES GRANT USAGE, UPDATE ON SEQUENCES TO "%s";
+			ALTER DEFAULT PRIVILEGES GRANT EXECUTE ON FUNCTIONS TO "%s","%s";
+			ALTER DEFAULT PRIVILEGES GRANT USAGE ON TYPES TO "%s","%s";`
+	schemaDefaultPrivilegesSQL = `SET ROLE TO "%s";
+			GRANT USAGE ON SCHEMA "%s" TO "%s","%s";
+			ALTER DEFAULT PRIVILEGES IN SCHEMA "%s" GRANT SELECT ON TABLES TO "%s";
+			ALTER DEFAULT PRIVILEGES IN SCHEMA "%s" GRANT SELECT ON SEQUENCES TO "%s";
+			ALTER DEFAULT PRIVILEGES IN SCHEMA "%s" GRANT INSERT, UPDATE, DELETE ON TABLES TO "%s";
+			ALTER DEFAULT PRIVILEGES IN SCHEMA "%s" GRANT USAGE, UPDATE ON SEQUENCES TO "%s";
+			ALTER DEFAULT PRIVILEGES IN SCHEMA "%s" GRANT EXECUTE ON FUNCTIONS TO "%s","%s";
+			ALTER DEFAULT PRIVILEGES IN SCHEMA "%s" GRANT USAGE ON TYPES TO "%s","%s";`
+
+	connectionPoolerLookup = `
+		CREATE SCHEMA IF NOT EXISTS {{.pooler_schema}};
+
+		CREATE OR REPLACE FUNCTION {{.pooler_schema}}.user_lookup(
+			in i_username text, out uname text, out phash text)
+		RETURNS record AS $$
+		BEGIN
+			SELECT usename, passwd FROM pg_catalog.pg_shadow
+			WHERE usename = i_username INTO uname, phash;
+			RETURN;
+		END;
+		$$ LANGUAGE plpgsql SECURITY DEFINER;
+
+		REVOKE ALL ON FUNCTION {{.pooler_schema}}.user_lookup(text)
+			FROM public, {{.pooler_user}};
+		GRANT EXECUTE ON FUNCTION {{.pooler_schema}}.user_lookup(text)
+			TO {{.pooler_user}};
+		GRANT USAGE ON SCHEMA {{.pooler_schema}} TO {{.pooler_user}};
+	`
 )
 
-func (c *Cluster) pgConnectionString() string {
+func (c *Cluster) pgConnectionString(dbname string) string {
 	password := c.systemUsers[constants.SuperuserKeyName].Password
 
-	return fmt.Sprintf("host='%s' dbname=postgres sslmode=require user='%s' password='%s' connect_timeout='%d'",
+	if dbname == "" {
+		dbname = "postgres"
+	}
+
+	return fmt.Sprintf("host='%s' dbname='%s' sslmode=require user='%s' password='%s' connect_timeout='%d'",
 		fmt.Sprintf("%s.%s.svc.%s", c.Name, c.Namespace, c.OpConfig.ClusterDomain),
+		dbname,
 		c.systemUsers[constants.SuperuserKeyName].Name,
 		strings.Replace(password, "$", "\\$", -1),
 		constants.PostgresConnectTimeout/time.Second)
@@ -49,13 +101,17 @@ func (c *Cluster) databaseAccessDisabled() bool {
 }
 
 func (c *Cluster) initDbConn() error {
+	return c.initDbConnWithName("")
+}
+
+func (c *Cluster) initDbConnWithName(dbname string) error {
 	c.setProcessName("initializing db connection")
 	if c.pgDb != nil {
 		return nil
 	}
 
 	var conn *sql.DB
-	connstring := c.pgConnectionString()
+	connstring := c.pgConnectionString(dbname)
 
 	finalerr := retryutil.Retry(constants.PostgresConnectTimeout, constants.PostgresConnectRetryTimeout,
 		func() (bool, error) {
@@ -94,6 +150,10 @@ func (c *Cluster) initDbConn() error {
 	return nil
 }
 
+func (c *Cluster) connectionIsClosed() bool {
+	return c.pgDb == nil
+}
+
 func (c *Cluster) closeDbConn() (err error) {
 	c.setProcessName("closing db connection")
 	if c.pgDb != nil {
@@ -187,43 +247,141 @@ func (c *Cluster) getDatabases() (dbs map[string]string, err error) {
 }
 
 // executeCreateDatabase creates new database with the given owner.
-// The caller is responsible for openinging and closing the database connection.
-func (c *Cluster) executeCreateDatabase(datname, owner string) error {
-	return c.execCreateOrAlterDatabase(datname, owner, createDatabaseSQL,
+// The caller is responsible for opening and closing the database connection.
+func (c *Cluster) executeCreateDatabase(databaseName, owner string) error {
+	return c.execCreateOrAlterDatabase(databaseName, owner, createDatabaseSQL,
 		"creating database", "create database")
 }
 
-// executeCreateDatabase changes the owner of the given database.
-// The caller is responsible for openinging and closing the database connection.
-func (c *Cluster) executeAlterDatabaseOwner(datname string, owner string) error {
-	return c.execCreateOrAlterDatabase(datname, owner, alterDatabaseOwnerSQL,
+// executeAlterDatabaseOwner changes the owner of the given database.
+// The caller is responsible for opening and closing the database connection.
+func (c *Cluster) executeAlterDatabaseOwner(databaseName string, owner string) error {
+	return c.execCreateOrAlterDatabase(databaseName, owner, alterDatabaseOwnerSQL,
 		"changing owner for database", "alter database owner")
 }
 
-func (c *Cluster) execCreateOrAlterDatabase(datname, owner, statement, doing, operation string) error {
-	if !c.databaseNameOwnerValid(datname, owner) {
+func (c *Cluster) execCreateOrAlterDatabase(databaseName, owner, statement, doing, operation string) error {
+	if !c.databaseNameOwnerValid(databaseName, owner) {
 		return nil
 	}
-	c.logger.Infof("%s %q owner %q", doing, datname, owner)
-	if _, err := c.pgDb.Exec(fmt.Sprintf(statement, datname, owner)); err != nil {
+	c.logger.Infof("%s %q owner %q", doing, databaseName, owner)
+	if _, err := c.pgDb.Exec(fmt.Sprintf(statement, databaseName, owner)); err != nil {
 		return fmt.Errorf("could not execute %s: %v", operation, err)
 	}
 	return nil
 }
 
-func (c *Cluster) databaseNameOwnerValid(datname, owner string) bool {
+func (c *Cluster) databaseNameOwnerValid(databaseName, owner string) bool {
 	if _, ok := c.pgUsers[owner]; !ok {
-		c.logger.Infof("skipping creation of the %q database, user %q does not exist", datname, owner)
+		c.logger.Infof("skipping creation of the %q database, user %q does not exist", databaseName, owner)
 		return false
 	}
 
-	if !databaseNameRegexp.MatchString(datname) {
-		c.logger.Infof("database %q has invalid name", datname)
+	if !databaseNameRegexp.MatchString(databaseName) {
+		c.logger.Infof("database %q has invalid name", databaseName)
 		return false
 	}
 	return true
 }
 
+// getSchemas returns the list of current database schemas
+// The caller is responsible for opening and closing the database connection
+func (c *Cluster) getSchemas() (schemas []string, err error) {
+	var (
+		rows      *sql.Rows
+		dbschemas []string
+	)
+
+	if rows, err = c.pgDb.Query(getSchemasSQL); err != nil {
+		return nil, fmt.Errorf("could not query database schemas: %v", err)
+	}
+
+	defer func() {
+		if err2 := rows.Close(); err2 != nil {
+			if err != nil {
+				err = fmt.Errorf("error when closing query cursor: %v, previous error: %v", err2, err)
+			} else {
+				err = fmt.Errorf("error when closing query cursor: %v", err2)
+			}
+		}
+	}()
+
+	for rows.Next() {
+		var dbschema string
+
+		if err = rows.Scan(&dbschema); err != nil {
+			return nil, fmt.Errorf("error when processing row: %v", err)
+		}
+		dbschemas = append(dbschemas, dbschema)
+	}
+
+	return dbschemas, err
+}
+
+// executeCreateDatabaseSchema creates new database schema with the given owner.
+// The caller is responsible for opening and closing the database connection.
+func (c *Cluster) executeCreateDatabaseSchema(databaseName, schemaName, dbOwner string, schemaOwner string) error {
+	return c.execCreateDatabaseSchema(databaseName, schemaName, dbOwner, schemaOwner, createDatabaseSchemaSQL,
+		"creating database schema", "create database schema")
+}
+
+func (c *Cluster) execCreateDatabaseSchema(databaseName, schemaName, dbOwner, schemaOwner, statement, doing, operation string) error {
+	if !c.databaseSchemaNameValid(schemaName) {
+		return nil
+	}
+	c.logger.Infof("%s %q owner %q", doing, schemaName, schemaOwner)
+	if _, err := c.pgDb.Exec(fmt.Sprintf(statement, dbOwner, schemaName, schemaOwner)); err != nil {
+		return fmt.Errorf("could not execute %s: %v", operation, err)
+	}
+
+	// set default privileges for schema
+	c.execAlterSchemaDefaultPrivileges(schemaName, schemaOwner, databaseName)
+	if schemaOwner != dbOwner {
+		c.execAlterSchemaDefaultPrivileges(schemaName, dbOwner, databaseName+"_"+schemaName)
+		c.execAlterSchemaDefaultPrivileges(schemaName, schemaOwner, databaseName+"_"+schemaName)
+	}
+
+	return nil
+}
+
+func (c *Cluster) databaseSchemaNameValid(schemaName string) bool {
+	if !databaseNameRegexp.MatchString(schemaName) {
+		c.logger.Infof("database schema %q has invalid name", schemaName)
+		return false
+	}
+	return true
+}
+
+func (c *Cluster) execAlterSchemaDefaultPrivileges(schemaName, owner, rolePrefix string) error {
+	if _, err := c.pgDb.Exec(fmt.Sprintf(schemaDefaultPrivilegesSQL, owner,
+		schemaName, rolePrefix+constants.ReaderRoleNameSuffix, rolePrefix+constants.WriterRoleNameSuffix, // schema
+		schemaName, rolePrefix+constants.ReaderRoleNameSuffix, // tables
+		schemaName, rolePrefix+constants.ReaderRoleNameSuffix, // sequences
+		schemaName, rolePrefix+constants.WriterRoleNameSuffix, // tables
+		schemaName, rolePrefix+constants.WriterRoleNameSuffix, // sequences
+		schemaName, rolePrefix+constants.ReaderRoleNameSuffix, rolePrefix+constants.WriterRoleNameSuffix, // types
+		schemaName, rolePrefix+constants.ReaderRoleNameSuffix, rolePrefix+constants.WriterRoleNameSuffix)); err != nil { // functions
+		return fmt.Errorf("could not alter default privileges for database schema %s: %v", schemaName, err)
+	}
+
+	return nil
+}
+
+func (c *Cluster) execAlterGlobalDefaultPrivileges(owner, rolePrefix string) error {
+	if _, err := c.pgDb.Exec(fmt.Sprintf(globalDefaultPrivilegesSQL, owner,
+		rolePrefix+constants.WriterRoleNameSuffix, rolePrefix+constants.ReaderRoleNameSuffix, // schemas
+		rolePrefix+constants.ReaderRoleNameSuffix,                                            // tables
+		rolePrefix+constants.ReaderRoleNameSuffix,                                            // sequences
+		rolePrefix+constants.WriterRoleNameSuffix,                                            // tables
+		rolePrefix+constants.WriterRoleNameSuffix,                                            // sequences
+		rolePrefix+constants.ReaderRoleNameSuffix, rolePrefix+constants.WriterRoleNameSuffix, // types
+		rolePrefix+constants.ReaderRoleNameSuffix, rolePrefix+constants.WriterRoleNameSuffix)); err != nil { // functions
+		return fmt.Errorf("could not alter default privileges for database %s: %v", rolePrefix, err)
+	}
+
+	return nil
+}
+
 func makeUserFlags(rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin bool) (result []string) {
 	if rolsuper {
 		result = append(result, constants.RoleFlagSuperuser)
@@ -243,3 +401,147 @@ func makeUserFlags(rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin
 
 	return result
 }
+
+// getExtension returns the list of current database extensions
+// The caller is responsible for opening and closing the database connection
+func (c *Cluster) getExtensions() (dbExtensions map[string]string, err error) {
+	var (
+		rows *sql.Rows
+	)
+
+	if rows, err = c.pgDb.Query(getExtensionsSQL); err != nil {
+		return nil, fmt.Errorf("could not query database extensions: %v", err)
+	}
+
+	defer func() {
+		if err2 := rows.Close(); err2 != nil {
+			if err != nil {
+				err = fmt.Errorf("error when closing query cursor: %v, previous error: %v", err2, err)
+			} else {
+				err = fmt.Errorf("error when closing query cursor: %v", err2)
+			}
+		}
+	}()
+
+	dbExtensions = make(map[string]string)
+
+	for rows.Next() {
+		var extension, schema string
+
+		if err = rows.Scan(&extension, &schema); err != nil {
+			return nil, fmt.Errorf("error when processing row: %v", err)
+		}
+		dbExtensions[extension] = schema
+	}
+
+	return dbExtensions, err
+}
+
+// executeCreateExtension creates new extension in the given schema.
+// The caller is responsible for opening and closing the database connection.
+func (c *Cluster) executeCreateExtension(extName, schemaName string) error {
+	return c.execCreateOrAlterExtension(extName, schemaName, createExtensionSQL,
+		"creating extension", "create extension")
+}
+
+// executeAlterExtension changes the schema of the given extension.
+// The caller is responsible for opening and closing the database connection.
+func (c *Cluster) executeAlterExtension(extName, schemaName string) error {
+	return c.execCreateOrAlterExtension(extName, schemaName, alterExtensionSQL,
+		"changing schema for extension", "alter extension schema")
+}
+
+func (c *Cluster) execCreateOrAlterExtension(extName, schemaName, statement, doing, operation string) error {
+
+	c.logger.Infof("%s %q schema %q", doing, extName, schemaName)
+	if _, err := c.pgDb.Exec(fmt.Sprintf(statement, extName, schemaName)); err != nil {
+		return fmt.Errorf("could not execute %s: %v", operation, err)
+	}
+
+	return nil
+}
+
+// Creates a connection pool credentials lookup function in every database to
+// perform remote authentification.
+func (c *Cluster) installLookupFunction(poolerSchema, poolerUser string) error {
+	var stmtBytes bytes.Buffer
+	c.logger.Info("Installing lookup function")
+
+	if err := c.initDbConn(); err != nil {
+		return fmt.Errorf("could not init database connection")
+	}
+	defer func() {
+		if c.connectionIsClosed() {
+			return
+		}
+
+		if err := c.closeDbConn(); err != nil {
+			c.logger.Errorf("could not close database connection: %v", err)
+		}
+	}()
+
+	currentDatabases, err := c.getDatabases()
+	if err != nil {
+		msg := "could not get databases to install pooler lookup function: %v"
+		return fmt.Errorf(msg, err)
+	}
+
+	templater := template.Must(template.New("sql").Parse(connectionPoolerLookup))
+
+	for dbname := range currentDatabases {
+		if dbname == "template0" || dbname == "template1" {
+			continue
+		}
+
+		if err := c.initDbConnWithName(dbname); err != nil {
+			return fmt.Errorf("could not init database connection to %s", dbname)
+		}
+
+		c.logger.Infof("Install pooler lookup function into %s", dbname)
+
+		params := TemplateParams{
+			"pooler_schema": poolerSchema,
+			"pooler_user":   poolerUser,
+		}
+
+		if err := templater.Execute(&stmtBytes, params); err != nil {
+			c.logger.Errorf("could not prepare sql statement %+v: %v",
+				params, err)
+			// process other databases
+			continue
+		}
+
+		// golang sql will do retries couple of times if pq driver reports
+		// connections issues (driver.ErrBadConn), but since our query is
+		// idempotent, we can retry in a view of other errors (e.g. due to
+		// failover a db is temporary in a read-only mode or so) to make sure
+		// it was applied.
+		execErr := retryutil.Retry(
+			constants.PostgresConnectTimeout,
+			constants.PostgresConnectRetryTimeout,
+			func() (bool, error) {
+				if _, err := c.pgDb.Exec(stmtBytes.String()); err != nil {
+					msg := fmt.Errorf("could not execute sql statement %s: %v",
+						stmtBytes.String(), err)
+					return false, msg
+				}
+
+				return true, nil
+			})
+
+		if execErr != nil {
+			c.logger.Errorf("could not execute after retries %s: %v",
+				stmtBytes.String(), err)
+			// process other databases
+			continue
+		}
+
+		c.logger.Infof("pooler lookup function installed into %s", dbname)
+		if err := c.closeDbConn(); err != nil {
+			c.logger.Errorf("could not close database connection: %v", err)
+		}
+	}
+
+	c.ConnectionPooler.LookupFunction = true
+	return nil
+}

pkg/cluster/exec.go

@@ -2,10 +2,11 @@ package cluster
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"strings"
 
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/remotecommand"
@@ -23,7 +24,7 @@ func (c *Cluster) ExecCommand(podName *spec.NamespacedName, command ...string) (
 		execErr bytes.Buffer
 	)
 
-	pod, err := c.KubeClient.Pods(podName.Namespace).Get(podName.Name, metav1.GetOptions{})
+	pod, err := c.KubeClient.Pods(podName.Namespace).Get(context.TODO(), podName.Name, metav1.GetOptions{})
 	if err != nil {
 		return "", fmt.Errorf("could not get pod info: %v", err)
 	}

pkg/cluster/pod.go

@@ -1,6 +1,7 @@
 package cluster
 
 import (
+	"context"
 	"fmt"
 	"math/rand"
 
@@ -17,7 +18,7 @@ func (c *Cluster) listPods() ([]v1.Pod, error) {
 		LabelSelector: c.labelsSet(false).String(),
 	}
 
-	pods, err := c.KubeClient.Pods(c.Namespace).List(listOptions)
+	pods, err := c.KubeClient.Pods(c.Namespace).List(context.TODO(), listOptions)
 	if err != nil {
 		return nil, fmt.Errorf("could not get list of pods: %v", err)
 	}
@@ -30,7 +31,7 @@ func (c *Cluster) getRolePods(role PostgresRole) ([]v1.Pod, error) {
 		LabelSelector: c.roleLabelsSet(false, role).String(),
 	}
 
-	pods, err := c.KubeClient.Pods(c.Namespace).List(listOptions)
+	pods, err := c.KubeClient.Pods(c.Namespace).List(context.TODO(), listOptions)
 	if err != nil {
 		return nil, fmt.Errorf("could not get list of pods: %v", err)
 	}
@@ -73,7 +74,7 @@ func (c *Cluster) deletePod(podName spec.NamespacedName) error {
 	ch := c.registerPodSubscriber(podName)
 	defer c.unregisterPodSubscriber(podName)
 
-	if err := c.KubeClient.Pods(podName.Namespace).Delete(podName.Name, c.deleteOptions); err != nil {
+	if err := c.KubeClient.Pods(podName.Namespace).Delete(context.TODO(), podName.Name, c.deleteOptions); err != nil {
 		return err
 	}
 
@@ -183,7 +184,7 @@ func (c *Cluster) MigrateMasterPod(podName spec.NamespacedName) error {
 		eol                bool
 	)
 
-	oldMaster, err := c.KubeClient.Pods(podName.Namespace).Get(podName.Name, metav1.GetOptions{})
+	oldMaster, err := c.KubeClient.Pods(podName.Namespace).Get(context.TODO(), podName.Name, metav1.GetOptions{})
 
 	if err != nil {
 		return fmt.Errorf("could not get pod: %v", err)
@@ -206,7 +207,9 @@ func (c *Cluster) MigrateMasterPod(podName spec.NamespacedName) error {
 	// we must have a statefulset in the cluster for the migration to work
 	if c.Statefulset == nil {
 		var sset *appsv1.StatefulSet
-		if sset, err = c.KubeClient.StatefulSets(c.Namespace).Get(c.statefulSetName(),
+		if sset, err = c.KubeClient.StatefulSets(c.Namespace).Get(
+			context.TODO(),
+			c.statefulSetName(),
 			metav1.GetOptions{}); err != nil {
 			return fmt.Errorf("could not retrieve cluster statefulset: %v", err)
 		}
@@ -247,7 +250,7 @@ func (c *Cluster) MigrateMasterPod(podName spec.NamespacedName) error {
 
 // MigrateReplicaPod recreates pod on a new node
 func (c *Cluster) MigrateReplicaPod(podName spec.NamespacedName, fromNodeName string) error {
-	replicaPod, err := c.KubeClient.Pods(podName.Namespace).Get(podName.Name, metav1.GetOptions{})
+	replicaPod, err := c.KubeClient.Pods(podName.Namespace).Get(context.TODO(), podName.Name, metav1.GetOptions{})
 	if err != nil {
 		return fmt.Errorf("could not get pod: %v", err)
 	}
@@ -276,7 +279,7 @@ func (c *Cluster) recreatePod(podName spec.NamespacedName) (*v1.Pod, error) {
 	defer c.unregisterPodSubscriber(podName)
 	stopChan := make(chan struct{})
 
-	if err := c.KubeClient.Pods(podName.Namespace).Delete(podName.Name, c.deleteOptions); err != nil {
+	if err := c.KubeClient.Pods(podName.Namespace).Delete(context.TODO(), podName.Name, c.deleteOptions); err != nil {
 		return nil, fmt.Errorf("could not delete pod: %v", err)
 	}
 
@@ -291,6 +294,27 @@ func (c *Cluster) recreatePod(podName spec.NamespacedName) (*v1.Pod, error) {
 	return pod, nil
 }
 
+func (c *Cluster) isSafeToRecreatePods(pods *v1.PodList) bool {
+
+	/*
+	 Operator should not re-create pods if there is at least one replica being bootstrapped
+	 because Patroni might use other replicas to take basebackup from (see Patroni's "clonefrom" tag).
+
+	 XXX operator cannot forbid replica re-init, so we might still fail if re-init is started
+	 after this check succeeds but before a pod is re-created
+	*/
+
+	for _, pod := range pods.Items {
+		state, err := c.patroni.GetPatroniMemberState(&pod)
+		if err != nil || state == "creating replica" {
+			c.logger.Warningf("cannot re-create replica %s: it is currently being initialized", pod.Name)
+			return false
+		}
+
+	}
+	return true
+}
+
 func (c *Cluster) recreatePods() error {
 	c.setProcessName("starting to recreate pods")
 	ls := c.labelsSet(false)
@@ -300,12 +324,16 @@ func (c *Cluster) recreatePods() error {
 		LabelSelector: ls.String(),
 	}
 
-	pods, err := c.KubeClient.Pods(namespace).List(listOptions)
+	pods, err := c.KubeClient.Pods(namespace).List(context.TODO(), listOptions)
 	if err != nil {
 		return fmt.Errorf("could not get the list of pods: %v", err)
 	}
 	c.logger.Infof("there are %d pods in the cluster to recreate", len(pods.Items))
 
+	if !c.isSafeToRecreatePods(pods) {
+		return fmt.Errorf("postpone pod recreation until next Sync: recreation is unsafe because pods are being initialized")
+	}
+
 	var (
 		masterPod, newMasterPod, newPod *v1.Pod
 	)
@@ -349,7 +377,7 @@ func (c *Cluster) recreatePods() error {
 }
 
 func (c *Cluster) podIsEndOfLife(pod *v1.Pod) (bool, error) {
-	node, err := c.KubeClient.Nodes().Get(pod.Spec.NodeName, metav1.GetOptions{})
+	node, err := c.KubeClient.Nodes().Get(context.TODO(), pod.Spec.NodeName, metav1.GetOptions{})
 	if err != nil {
 		return false, err
 	}

pkg/cluster/resources.go

@@ -1,6 +1,7 @@
 package cluster
 
 import (
+	"context"
 	"fmt"
 	"strconv"
 	"strings"
@@ -80,7 +81,10 @@ func (c *Cluster) createStatefulSet() (*appsv1.StatefulSet, error) {
 	if err != nil {
 		return nil, fmt.Errorf("could not generate statefulset: %v", err)
 	}
-	statefulSet, err := c.KubeClient.StatefulSets(statefulSetSpec.Namespace).Create(statefulSetSpec)
+	statefulSet, err := c.KubeClient.StatefulSets(statefulSetSpec.Namespace).Create(
+		context.TODO(),
+		statefulSetSpec,
+		metav1.CreateOptions{})
 	if err != nil {
 		return nil, err
 	}
@@ -90,6 +94,137 @@ func (c *Cluster) createStatefulSet() (*appsv1.StatefulSet, error) {
 	return statefulSet, nil
 }
 
+// Prepare the database for connection pooler to be used, i.e. install lookup
+// function (do it first, because it should be fast and if it didn't succeed,
+// it doesn't makes sense to create more K8S objects. At this moment we assume
+// that necessary connection pooler user exists.
+//
+// After that create all the objects for connection pooler, namely a deployment
+// with a chosen pooler and a service to expose it.
+func (c *Cluster) createConnectionPooler(lookup InstallFunction) (*ConnectionPoolerObjects, error) {
+	var msg string
+	c.setProcessName("creating connection pooler")
+
+	if c.ConnectionPooler == nil {
+		c.ConnectionPooler = &ConnectionPoolerObjects{}
+	}
+
+	schema := c.Spec.ConnectionPooler.Schema
+
+	if schema == "" {
+		schema = c.OpConfig.ConnectionPooler.Schema
+	}
+
+	user := c.Spec.ConnectionPooler.User
+	if user == "" {
+		user = c.OpConfig.ConnectionPooler.User
+	}
+
+	err := lookup(schema, user)
+
+	if err != nil {
+		msg = "could not prepare database for connection pooler: %v"
+		return nil, fmt.Errorf(msg, err)
+	}
+
+	deploymentSpec, err := c.generateConnectionPoolerDeployment(&c.Spec)
+	if err != nil {
+		msg = "could not generate deployment for connection pooler: %v"
+		return nil, fmt.Errorf(msg, err)
+	}
+
+	// client-go does retry 10 times (with NoBackoff by default) when the API
+	// believe a request can be retried and returns Retry-After header. This
+	// should be good enough to not think about it here.
+	deployment, err := c.KubeClient.
+		Deployments(deploymentSpec.Namespace).
+		Create(context.TODO(), deploymentSpec, metav1.CreateOptions{})
+
+	if err != nil {
+		return nil, err
+	}
+
+	serviceSpec := c.generateConnectionPoolerService(&c.Spec)
+	service, err := c.KubeClient.
+		Services(serviceSpec.Namespace).
+		Create(context.TODO(), serviceSpec, metav1.CreateOptions{})
+
+	if err != nil {
+		return nil, err
+	}
+
+	c.ConnectionPooler = &ConnectionPoolerObjects{
+		Deployment: deployment,
+		Service:    service,
+	}
+	c.logger.Debugf("created new connection pooler %q, uid: %q",
+		util.NameFromMeta(deployment.ObjectMeta), deployment.UID)
+
+	return c.ConnectionPooler, nil
+}
+
+func (c *Cluster) deleteConnectionPooler() (err error) {
+	c.setProcessName("deleting connection pooler")
+	c.logger.Debugln("deleting connection pooler")
+
+	// Lack of connection pooler objects is not a fatal error, just log it if
+	// it was present before in the manifest
+	if c.ConnectionPooler == nil {
+		c.logger.Infof("No connection pooler to delete")
+		return nil
+	}
+
+	// Clean up the deployment object. If deployment resource we've remembered
+	// is somehow empty, try to delete based on what would we generate
+	deploymentName := c.connectionPoolerName()
+	deployment := c.ConnectionPooler.Deployment
+
+	if deployment != nil {
+		deploymentName = deployment.Name
+	}
+
+	// set delete propagation policy to foreground, so that replica set will be
+	// also deleted.
+	policy := metav1.DeletePropagationForeground
+	options := metav1.DeleteOptions{PropagationPolicy: &policy}
+	err = c.KubeClient.
+		Deployments(c.Namespace).
+		Delete(context.TODO(), deploymentName, options)
+
+	if !k8sutil.ResourceNotFound(err) {
+		c.logger.Debugf("Connection pooler deployment was already deleted")
+	} else if err != nil {
+		return fmt.Errorf("could not delete deployment: %v", err)
+	}
+
+	c.logger.Infof("Connection pooler deployment %q has been deleted", deploymentName)
+
+	// Repeat the same for the service object
+	service := c.ConnectionPooler.Service
+	serviceName := c.connectionPoolerName()
+
+	if service != nil {
+		serviceName = service.Name
+	}
+
+	// set delete propagation policy to foreground, so that all the dependant
+	// will be deleted.
+	err = c.KubeClient.
+		Services(c.Namespace).
+		Delete(context.TODO(), serviceName, options)
+
+	if !k8sutil.ResourceNotFound(err) {
+		c.logger.Debugf("Connection pooler service was already deleted")
+	} else if err != nil {
+		return fmt.Errorf("could not delete service: %v", err)
+	}
+
+	c.logger.Infof("Connection pooler service %q has been deleted", serviceName)
+
+	c.ConnectionPooler = nil
+	return nil
+}
+
 func getPodIndex(podName string) (int32, error) {
 	parts := strings.Split(podName, "-")
 	if len(parts) == 0 {
@@ -125,7 +260,7 @@ func (c *Cluster) preScaleDown(newStatefulSet *appsv1.StatefulSet) error {
 	}
 
 	podName := fmt.Sprintf("%s-0", c.Statefulset.Name)
-	masterCandidatePod, err := c.KubeClient.Pods(c.clusterNamespace()).Get(podName, metav1.GetOptions{})
+	masterCandidatePod, err := c.KubeClient.Pods(c.clusterNamespace()).Get(context.TODO(), podName, metav1.GetOptions{})
 	if err != nil {
 		return fmt.Errorf("could not get master candidate pod: %v", err)
 	}
@@ -224,9 +359,12 @@ func (c *Cluster) updateStatefulSetAnnotations(annotations map[string]string) (*
 		return nil, fmt.Errorf("could not form patch for the statefulset metadata: %v", err)
 	}
 	result, err := c.KubeClient.StatefulSets(c.Statefulset.Namespace).Patch(
+		context.TODO(),
 		c.Statefulset.Name,
 		types.MergePatchType,
-		[]byte(patchData), "")
+		[]byte(patchData),
+		metav1.PatchOptions{},
+		"")
 	if err != nil {
 		return nil, fmt.Errorf("could not patch statefulset annotations %q: %v", patchData, err)
 	}
@@ -254,9 +392,12 @@ func (c *Cluster) updateStatefulSet(newStatefulSet *appsv1.StatefulSet) error {
 	}
 
 	statefulSet, err := c.KubeClient.StatefulSets(c.Statefulset.Namespace).Patch(
+		context.TODO(),
 		c.Statefulset.Name,
 		types.MergePatchType,
-		patchData, "")
+		patchData,
+		metav1.PatchOptions{},
+		"")
 	if err != nil {
 		return fmt.Errorf("could not patch statefulset spec %q: %v", statefulSetName, err)
 	}
@@ -288,7 +429,7 @@ func (c *Cluster) replaceStatefulSet(newStatefulSet *appsv1.StatefulSet) error {
 	oldStatefulset := c.Statefulset
 
 	options := metav1.DeleteOptions{PropagationPolicy: &deletePropagationPolicy}
-	err := c.KubeClient.StatefulSets(oldStatefulset.Namespace).Delete(oldStatefulset.Name, &options)
+	err := c.KubeClient.StatefulSets(oldStatefulset.Namespace).Delete(context.TODO(), oldStatefulset.Name, options)
 	if err != nil {
 		return fmt.Errorf("could not delete statefulset %q: %v", statefulSetName, err)
 	}
@@ -299,7 +440,7 @@ func (c *Cluster) replaceStatefulSet(newStatefulSet *appsv1.StatefulSet) error {
 
 	err = retryutil.Retry(c.OpConfig.ResourceCheckInterval, c.OpConfig.ResourceCheckTimeout,
 		func() (bool, error) {
-			_, err2 := c.KubeClient.StatefulSets(oldStatefulset.Namespace).Get(oldStatefulset.Name, metav1.GetOptions{})
+			_, err2 := c.KubeClient.StatefulSets(oldStatefulset.Namespace).Get(context.TODO(), oldStatefulset.Name, metav1.GetOptions{})
 			if err2 == nil {
 				return false, nil
 			}
@@ -313,7 +454,7 @@ func (c *Cluster) replaceStatefulSet(newStatefulSet *appsv1.StatefulSet) error {
 	}
 
 	// create the new statefulset with the desired spec. It would take over the remaining pods.
-	createdStatefulset, err := c.KubeClient.StatefulSets(newStatefulSet.Namespace).Create(newStatefulSet)
+	createdStatefulset, err := c.KubeClient.StatefulSets(newStatefulSet.Namespace).Create(context.TODO(), newStatefulSet, metav1.CreateOptions{})
 	if err != nil {
 		return fmt.Errorf("could not create statefulset %q: %v", statefulSetName, err)
 	}
@@ -334,7 +475,7 @@ func (c *Cluster) deleteStatefulSet() error {
 		return fmt.Errorf("there is no statefulset in the cluster")
 	}
 
-	err := c.KubeClient.StatefulSets(c.Statefulset.Namespace).Delete(c.Statefulset.Name, c.deleteOptions)
+	err := c.KubeClient.StatefulSets(c.Statefulset.Namespace).Delete(context.TODO(), c.Statefulset.Name, c.deleteOptions)
 	if err != nil {
 		return err
 	}
@@ -356,7 +497,7 @@ func (c *Cluster) createService(role PostgresRole) (*v1.Service, error) {
 	c.setProcessName("creating %v service", role)
 
 	serviceSpec := c.generateService(role, &c.Spec)
-	service, err := c.KubeClient.Services(serviceSpec.Namespace).Create(serviceSpec)
+	service, err := c.KubeClient.Services(serviceSpec.Namespace).Create(context.TODO(), serviceSpec, metav1.CreateOptions{})
 	if err != nil {
 		return nil, err
 	}
@@ -383,9 +524,12 @@ func (c *Cluster) updateService(role PostgresRole, newService *v1.Service) error
 	if len(newService.ObjectMeta.Annotations) > 0 {
 		if annotationsPatchData, err := metaAnnotationsPatch(newService.ObjectMeta.Annotations); err == nil {
 			_, err = c.KubeClient.Services(serviceName.Namespace).Patch(
+				context.TODO(),
 				serviceName.Name,
 				types.MergePatchType,
-				[]byte(annotationsPatchData), "")
+				[]byte(annotationsPatchData),
+				metav1.PatchOptions{},
+				"")
 
 			if err != nil {
 				return fmt.Errorf("could not replace annotations for the service %q: %v", serviceName, err)
@@ -402,7 +546,7 @@ func (c *Cluster) updateService(role PostgresRole, newService *v1.Service) error
 	if newServiceType == "ClusterIP" && newServiceType != oldServiceType {
 		newService.ResourceVersion = c.Services[role].ResourceVersion
 		newService.Spec.ClusterIP = c.Services[role].Spec.ClusterIP
-		svc, err = c.KubeClient.Services(serviceName.Namespace).Update(newService)
+		svc, err = c.KubeClient.Services(serviceName.Namespace).Update(context.TODO(), newService, metav1.UpdateOptions{})
 		if err != nil {
 			return fmt.Errorf("could not update service %q: %v", serviceName, err)
 		}
@@ -413,9 +557,7 @@ func (c *Cluster) updateService(role PostgresRole, newService *v1.Service) error
 		}
 
 		svc, err = c.KubeClient.Services(serviceName.Namespace).Patch(
-			serviceName.Name,
-			types.MergePatchType,
-			patchData, "")
+			context.TODO(), serviceName.Name, types.MergePatchType, patchData, metav1.PatchOptions{}, "")
 		if err != nil {
 			return fmt.Errorf("could not patch service %q: %v", serviceName, err)
 		}
@@ -434,7 +576,7 @@ func (c *Cluster) deleteService(role PostgresRole) error {
 		return nil
 	}
 
-	if err := c.KubeClient.Services(service.Namespace).Delete(service.Name, c.deleteOptions); err != nil {
+	if err := c.KubeClient.Services(service.Namespace).Delete(context.TODO(), service.Name, c.deleteOptions); err != nil {
 		return err
 	}
 
@@ -458,7 +600,7 @@ func (c *Cluster) createEndpoint(role PostgresRole) (*v1.Endpoints, error) {
 	}
 	endpointsSpec := c.generateEndpoint(role, subsets)
 
-	endpoints, err := c.KubeClient.Endpoints(endpointsSpec.Namespace).Create(endpointsSpec)
+	endpoints, err := c.KubeClient.Endpoints(endpointsSpec.Namespace).Create(context.TODO(), endpointsSpec, metav1.CreateOptions{})
 	if err != nil {
 		return nil, fmt.Errorf("could not create %s endpoint: %v", role, err)
 	}
@@ -500,7 +642,7 @@ func (c *Cluster) createPodDisruptionBudget() (*policybeta1.PodDisruptionBudget,
 	podDisruptionBudgetSpec := c.generatePodDisruptionBudget()
 	podDisruptionBudget, err := c.KubeClient.
 		PodDisruptionBudgets(podDisruptionBudgetSpec.Namespace).
-		Create(podDisruptionBudgetSpec)
+		Create(context.TODO(), podDisruptionBudgetSpec, metav1.CreateOptions{})
 
 	if err != nil {
 		return nil, err
@@ -521,7 +663,7 @@ func (c *Cluster) updatePodDisruptionBudget(pdb *policybeta1.PodDisruptionBudget
 
 	newPdb, err := c.KubeClient.
 		PodDisruptionBudgets(pdb.Namespace).
-		Create(pdb)
+		Create(context.TODO(), pdb, metav1.CreateOptions{})
 	if err != nil {
 		return fmt.Errorf("could not create pod disruption budget: %v", err)
 	}
@@ -539,7 +681,7 @@ func (c *Cluster) deletePodDisruptionBudget() error {
 	pdbName := util.NameFromMeta(c.PodDisruptionBudget.ObjectMeta)
 	err := c.KubeClient.
 		PodDisruptionBudgets(c.PodDisruptionBudget.Namespace).
-		Delete(c.PodDisruptionBudget.Name, c.deleteOptions)
+		Delete(context.TODO(), c.PodDisruptionBudget.Name, c.deleteOptions)
 	if err != nil {
 		return fmt.Errorf("could not delete pod disruption budget: %v", err)
 	}
@@ -548,7 +690,7 @@ func (c *Cluster) deletePodDisruptionBudget() error {
 
 	err = retryutil.Retry(c.OpConfig.ResourceCheckInterval, c.OpConfig.ResourceCheckTimeout,
 		func() (bool, error) {
-			_, err2 := c.KubeClient.PodDisruptionBudgets(pdbName.Namespace).Get(pdbName.Name, metav1.GetOptions{})
+			_, err2 := c.KubeClient.PodDisruptionBudgets(pdbName.Namespace).Get(context.TODO(), pdbName.Name, metav1.GetOptions{})
 			if err2 == nil {
 				return false, nil
 			}
@@ -571,7 +713,8 @@ func (c *Cluster) deleteEndpoint(role PostgresRole) error {
 		return fmt.Errorf("there is no %s endpoint in the cluster", role)
 	}
 
-	if err := c.KubeClient.Endpoints(c.Endpoints[role].Namespace).Delete(c.Endpoints[role].Name, c.deleteOptions); err != nil {
+	if err := c.KubeClient.Endpoints(c.Endpoints[role].Namespace).Delete(
+		context.TODO(), c.Endpoints[role].Name, c.deleteOptions); err != nil {
 		return fmt.Errorf("could not delete endpoint: %v", err)
 	}
 
@@ -585,7 +728,7 @@ func (c *Cluster) deleteEndpoint(role PostgresRole) error {
 func (c *Cluster) deleteSecret(secret *v1.Secret) error {
 	c.setProcessName("deleting secret %q", util.NameFromMeta(secret.ObjectMeta))
 	c.logger.Debugf("deleting secret %q", util.NameFromMeta(secret.ObjectMeta))
-	err := c.KubeClient.Secrets(secret.Namespace).Delete(secret.Name, c.deleteOptions)
+	err := c.KubeClient.Secrets(secret.Namespace).Delete(context.TODO(), secret.Name, c.deleteOptions)
 	if err != nil {
 		return err
 	}
@@ -610,7 +753,7 @@ func (c *Cluster) createLogicalBackupJob() (err error) {
 	}
 	c.logger.Debugf("Generated cronJobSpec: %v", logicalBackupJobSpec)
 
-	_, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Create(logicalBackupJobSpec)
+	_, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Create(context.TODO(), logicalBackupJobSpec, metav1.CreateOptions{})
 	if err != nil {
 		return fmt.Errorf("could not create k8s cron job: %v", err)
 	}
@@ -628,9 +771,12 @@ func (c *Cluster) patchLogicalBackupJob(newJob *batchv1beta1.CronJob) error {
 
 	// update the backup job spec
 	_, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Patch(
+		context.TODO(),
 		c.getLogicalBackupJobName(),
 		types.MergePatchType,
-		patchData, "")
+		patchData,
+		metav1.PatchOptions{},
+		"")
 	if err != nil {
 		return fmt.Errorf("could not patch logical backup job: %v", err)
 	}
@@ -642,7 +788,7 @@ func (c *Cluster) deleteLogicalBackupJob() error {
 
 	c.logger.Info("removing the logical backup job")
 
-	return c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Delete(c.getLogicalBackupJobName(), c.deleteOptions)
+	return c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Delete(context.TODO(), c.getLogicalBackupJobName(), c.deleteOptions)
 }
 
 // GetServiceMaster returns cluster's kubernetes master Service
@@ -674,3 +820,57 @@ func (c *Cluster) GetStatefulSet() *appsv1.StatefulSet {
 func (c *Cluster) GetPodDisruptionBudget() *policybeta1.PodDisruptionBudget {
 	return c.PodDisruptionBudget
 }
+
+// Perform actual patching of a connection pooler deployment, assuming that all
+// the check were already done before.
+func (c *Cluster) updateConnectionPoolerDeployment(oldDeploymentSpec, newDeployment *appsv1.Deployment) (*appsv1.Deployment, error) {
+	c.setProcessName("updating connection pooler")
+	if c.ConnectionPooler == nil || c.ConnectionPooler.Deployment == nil {
+		return nil, fmt.Errorf("there is no connection pooler in the cluster")
+	}
+
+	patchData, err := specPatch(newDeployment.Spec)
+	if err != nil {
+		return nil, fmt.Errorf("could not form patch for the deployment: %v", err)
+	}
+
+	// An update probably requires RetryOnConflict, but since only one operator
+	// worker at one time will try to update it chances of conflicts are
+	// minimal.
+	deployment, err := c.KubeClient.
+		Deployments(c.ConnectionPooler.Deployment.Namespace).Patch(
+		context.TODO(),
+		c.ConnectionPooler.Deployment.Name,
+		types.MergePatchType,
+		patchData,
+		metav1.PatchOptions{},
+		"")
+	if err != nil {
+		return nil, fmt.Errorf("could not patch deployment: %v", err)
+	}
+
+	c.ConnectionPooler.Deployment = deployment
+
+	return deployment, nil
+}
+
+//updateConnectionPoolerAnnotations updates the annotations of connection pooler deployment
+func (c *Cluster) updateConnectionPoolerAnnotations(annotations map[string]string) (*appsv1.Deployment, error) {
+	c.logger.Debugf("updating connection pooler annotations")
+	patchData, err := metaAnnotationsPatch(annotations)
+	if err != nil {
+		return nil, fmt.Errorf("could not form patch for the deployment metadata: %v", err)
+	}
+	result, err := c.KubeClient.Deployments(c.ConnectionPooler.Deployment.Namespace).Patch(
+		context.TODO(),
+		c.ConnectionPooler.Deployment.Name,
+		types.MergePatchType,
+		[]byte(patchData),
+		metav1.PatchOptions{},
+		"")
+	if err != nil {
+		return nil, fmt.Errorf("could not patch connection pooler annotations %q: %v", patchData, err)
+	}
+	return result, nil
+
+}

pkg/cluster/resources_test.go

@@ -0,0 +1,127 @@
+package cluster
+
+import (
+	"testing"
+
+	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
+	"github.com/zalando/postgres-operator/pkg/util/config"
+	"github.com/zalando/postgres-operator/pkg/util/k8sutil"
+
+	appsv1 "k8s.io/api/apps/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func mockInstallLookupFunction(schema string, user string) error {
+	return nil
+}
+
+func boolToPointer(value bool) *bool {
+	return &value
+}
+
+func TestConnectionPoolerCreationAndDeletion(t *testing.T) {
+	testName := "Test connection pooler creation"
+	var cluster = New(
+		Config{
+			OpConfig: config.Config{
+				ProtectedRoles: []string{"admin"},
+				Auth: config.Auth{
+					SuperUsername:       superUserName,
+					ReplicationUsername: replicationUserName,
+				},
+				ConnectionPooler: config.ConnectionPooler{
+					ConnectionPoolerDefaultCPURequest:    "100m",
+					ConnectionPoolerDefaultCPULimit:      "100m",
+					ConnectionPoolerDefaultMemoryRequest: "100Mi",
+					ConnectionPoolerDefaultMemoryLimit:   "100Mi",
+				},
+			},
+		}, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger, eventRecorder)
+
+	cluster.Statefulset = &appsv1.StatefulSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test-sts",
+		},
+	}
+
+	cluster.Spec = acidv1.PostgresSpec{
+		ConnectionPooler: &acidv1.ConnectionPooler{},
+	}
+	poolerResources, err := cluster.createConnectionPooler(mockInstallLookupFunction)
+
+	if err != nil {
+		t.Errorf("%s: Cannot create connection pooler, %s, %+v",
+			testName, err, poolerResources)
+	}
+
+	if poolerResources.Deployment == nil {
+		t.Errorf("%s: Connection pooler deployment is empty", testName)
+	}
+
+	if poolerResources.Service == nil {
+		t.Errorf("%s: Connection pooler service is empty", testName)
+	}
+
+	err = cluster.deleteConnectionPooler()
+	if err != nil {
+		t.Errorf("%s: Cannot delete connection pooler, %s", testName, err)
+	}
+}
+
+func TestNeedConnectionPooler(t *testing.T) {
+	testName := "Test how connection pooler can be enabled"
+	var cluster = New(
+		Config{
+			OpConfig: config.Config{
+				ProtectedRoles: []string{"admin"},
+				Auth: config.Auth{
+					SuperUsername:       superUserName,
+					ReplicationUsername: replicationUserName,
+				},
+				ConnectionPooler: config.ConnectionPooler{
+					ConnectionPoolerDefaultCPURequest:    "100m",
+					ConnectionPoolerDefaultCPULimit:      "100m",
+					ConnectionPoolerDefaultMemoryRequest: "100Mi",
+					ConnectionPoolerDefaultMemoryLimit:   "100Mi",
+				},
+			},
+		}, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger, eventRecorder)
+
+	cluster.Spec = acidv1.PostgresSpec{
+		ConnectionPooler: &acidv1.ConnectionPooler{},
+	}
+
+	if !cluster.needConnectionPooler() {
+		t.Errorf("%s: Connection pooler is not enabled with full definition",
+			testName)
+	}
+
+	cluster.Spec = acidv1.PostgresSpec{
+		EnableConnectionPooler: boolToPointer(true),
+	}
+
+	if !cluster.needConnectionPooler() {
+		t.Errorf("%s: Connection pooler is not enabled with flag",
+			testName)
+	}
+
+	cluster.Spec = acidv1.PostgresSpec{
+		EnableConnectionPooler: boolToPointer(false),
+		ConnectionPooler:       &acidv1.ConnectionPooler{},
+	}
+
+	if cluster.needConnectionPooler() {
+		t.Errorf("%s: Connection pooler is still enabled with flag being false",
+			testName)
+	}
+
+	cluster.Spec = acidv1.PostgresSpec{
+		EnableConnectionPooler: boolToPointer(true),
+		ConnectionPooler:       &acidv1.ConnectionPooler{},
+	}
+
+	if !cluster.needConnectionPooler() {
+		t.Errorf("%s: Connection pooler is not enabled with flag and full",
+			testName)
+	}
+}

pkg/cluster/sync.go

@@ -1,12 +1,10 @@
 package cluster
 
 import (
+	"context"
 	"fmt"
-
-	batchv1beta1 "k8s.io/api/batch/v1beta1"
-	v1 "k8s.io/api/core/v1"
-	policybeta1 "k8s.io/api/policy/v1beta1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"regexp"
+	"strings"
 
 	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
 	"github.com/zalando/postgres-operator/pkg/spec"
@@ -14,6 +12,11 @@ import (
 	"github.com/zalando/postgres-operator/pkg/util/constants"
 	"github.com/zalando/postgres-operator/pkg/util/k8sutil"
 	"github.com/zalando/postgres-operator/pkg/util/volumes"
+	appsv1 "k8s.io/api/apps/v1"
+	batchv1beta1 "k8s.io/api/batch/v1beta1"
+	v1 "k8s.io/api/core/v1"
+	policybeta1 "k8s.io/api/policy/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 // Sync syncs the cluster, making sure the actual Kubernetes objects correspond to what is defined in the manifest.
@@ -23,6 +26,7 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
+	oldSpec := c.Postgresql
 	c.setSpec(newSpec)
 
 	defer func() {
@@ -106,6 +110,16 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
 			err = fmt.Errorf("could not sync databases: %v", err)
 			return err
 		}
+		c.logger.Debugf("syncing prepared databases with schemas")
+		if err = c.syncPreparedDatabases(); err != nil {
+			err = fmt.Errorf("could not sync prepared database: %v", err)
+			return err
+		}
+	}
+
+	// sync connection pooler
+	if _, err = c.syncConnectionPooler(&oldSpec, newSpec, c.installLookupFunction); err != nil {
+		return fmt.Errorf("could not sync connection pooler: %v", err)
 	}
 
 	return err
@@ -115,10 +129,11 @@ func (c *Cluster) syncServices() error {
 	for _, role := range []PostgresRole{Master, Replica} {
 		c.logger.Debugf("syncing %s service", role)
 
-		if err := c.syncEndpoint(role); err != nil {
-			return fmt.Errorf("could not sync %s endpoint: %v", role, err)
+		if !c.patroniKubernetesUseConfigMaps() {
+			if err := c.syncEndpoint(role); err != nil {
+				return fmt.Errorf("could not sync %s endpoint: %v", role, err)
+			}
 		}
-
 		if err := c.syncService(role); err != nil {
 			return fmt.Errorf("could not sync %s service: %v", role, err)
 		}
@@ -134,7 +149,7 @@ func (c *Cluster) syncService(role PostgresRole) error {
 	)
 	c.setProcessName("syncing %s service", role)
 
-	if svc, err = c.KubeClient.Services(c.Namespace).Get(c.serviceName(role), metav1.GetOptions{}); err == nil {
+	if svc, err = c.KubeClient.Services(c.Namespace).Get(context.TODO(), c.serviceName(role), metav1.GetOptions{}); err == nil {
 		c.Services[role] = svc
 		desiredSvc := c.generateService(role, &c.Spec)
 		if match, reason := k8sutil.SameService(svc, desiredSvc); !match {
@@ -160,7 +175,7 @@ func (c *Cluster) syncService(role PostgresRole) error {
 			return fmt.Errorf("could not create missing %s service: %v", role, err)
 		}
 		c.logger.Infof("%s service %q already exists", role, util.NameFromMeta(svc.ObjectMeta))
-		if svc, err = c.KubeClient.Services(c.Namespace).Get(c.serviceName(role), metav1.GetOptions{}); err != nil {
+		if svc, err = c.KubeClient.Services(c.Namespace).Get(context.TODO(), c.serviceName(role), metav1.GetOptions{}); err != nil {
 			return fmt.Errorf("could not fetch existing %s service: %v", role, err)
 		}
 	}
@@ -175,7 +190,7 @@ func (c *Cluster) syncEndpoint(role PostgresRole) error {
 	)
 	c.setProcessName("syncing %s endpoint", role)
 
-	if ep, err = c.KubeClient.Endpoints(c.Namespace).Get(c.endpointName(role), metav1.GetOptions{}); err == nil {
+	if ep, err = c.KubeClient.Endpoints(c.Namespace).Get(context.TODO(), c.endpointName(role), metav1.GetOptions{}); err == nil {
 		// TODO: No syncing of endpoints here, is this covered completely by updateService?
 		c.Endpoints[role] = ep
 		return nil
@@ -194,7 +209,7 @@ func (c *Cluster) syncEndpoint(role PostgresRole) error {
 			return fmt.Errorf("could not create missing %s endpoint: %v", role, err)
 		}
 		c.logger.Infof("%s endpoint %q already exists", role, util.NameFromMeta(ep.ObjectMeta))
-		if ep, err = c.KubeClient.Endpoints(c.Namespace).Get(c.endpointName(role), metav1.GetOptions{}); err != nil {
+		if ep, err = c.KubeClient.Endpoints(c.Namespace).Get(context.TODO(), c.endpointName(role), metav1.GetOptions{}); err != nil {
 			return fmt.Errorf("could not fetch existing %s endpoint: %v", role, err)
 		}
 	}
@@ -207,7 +222,7 @@ func (c *Cluster) syncPodDisruptionBudget(isUpdate bool) error {
 		pdb *policybeta1.PodDisruptionBudget
 		err error
 	)
-	if pdb, err = c.KubeClient.PodDisruptionBudgets(c.Namespace).Get(c.podDisruptionBudgetName(), metav1.GetOptions{}); err == nil {
+	if pdb, err = c.KubeClient.PodDisruptionBudgets(c.Namespace).Get(context.TODO(), c.podDisruptionBudgetName(), metav1.GetOptions{}); err == nil {
 		c.PodDisruptionBudget = pdb
 		newPDB := c.generatePodDisruptionBudget()
 		if match, reason := k8sutil.SamePDB(pdb, newPDB); !match {
@@ -233,7 +248,7 @@ func (c *Cluster) syncPodDisruptionBudget(isUpdate bool) error {
 			return fmt.Errorf("could not create pod disruption budget: %v", err)
 		}
 		c.logger.Infof("pod disruption budget %q already exists", util.NameFromMeta(pdb.ObjectMeta))
-		if pdb, err = c.KubeClient.PodDisruptionBudgets(c.Namespace).Get(c.podDisruptionBudgetName(), metav1.GetOptions{}); err != nil {
+		if pdb, err = c.KubeClient.PodDisruptionBudgets(c.Namespace).Get(context.TODO(), c.podDisruptionBudgetName(), metav1.GetOptions{}); err != nil {
 			return fmt.Errorf("could not fetch existing %q pod disruption budget", util.NameFromMeta(pdb.ObjectMeta))
 		}
 	}
@@ -244,12 +259,34 @@ func (c *Cluster) syncPodDisruptionBudget(isUpdate bool) error {
 	return nil
 }
 
+func (c *Cluster) mustUpdatePodsAfterLazyUpdate(desiredSset *appsv1.StatefulSet) (bool, error) {
+
+	pods, err := c.listPods()
+	if err != nil {
+		return false, fmt.Errorf("could not list pods of the statefulset: %v", err)
+	}
+
+	for _, pod := range pods {
+
+		effectivePodImage := pod.Spec.Containers[0].Image
+		ssImage := desiredSset.Spec.Template.Spec.Containers[0].Image
+
+		if ssImage != effectivePodImage {
+			c.logger.Infof("not all pods were re-started when the lazy upgrade was enabled; forcing the rolling upgrade now")
+			return true, nil
+		}
+
+	}
+
+	return false, nil
+}
+
 func (c *Cluster) syncStatefulSet() error {
 	var (
 		podsRollingUpdateRequired bool
 	)
 	// NB: Be careful to consider the codepath that acts on podsRollingUpdateRequired before returning early.
-	sset, err := c.KubeClient.StatefulSets(c.Namespace).Get(c.statefulSetName(), metav1.GetOptions{})
+	sset, err := c.KubeClient.StatefulSets(c.Namespace).Get(context.TODO(), c.statefulSetName(), metav1.GetOptions{})
 	if err != nil {
 		if !k8sutil.ResourceNotFound(err) {
 			return fmt.Errorf("could not get statefulset: %v", err)
@@ -285,6 +322,18 @@ func (c *Cluster) syncStatefulSet() error {
 		// statefulset is already there, make sure we use its definition in order to compare with the spec.
 		c.Statefulset = sset
 
+		// check if there is no Postgres version mismatch
+		for _, container := range c.Statefulset.Spec.Template.Spec.Containers {
+			if container.Name != "postgres" {
+				continue
+			}
+			pgVersion, err := c.getNewPgVersion(container, c.Spec.PostgresqlParam.PgVersion)
+			if err != nil {
+				return fmt.Errorf("could not parse current Postgres version: %v", err)
+			}
+			c.Spec.PostgresqlParam.PgVersion = pgVersion
+		}
+
 		desiredSS, err := c.generateStatefulSet(&c.Spec)
 		if err != nil {
 			return fmt.Errorf("could not generate statefulset: %v", err)
@@ -310,6 +359,21 @@ func (c *Cluster) syncStatefulSet() error {
 				}
 			}
 		}
+		annotations := c.AnnotationsToPropagate(c.Statefulset.Annotations)
+		c.updateStatefulSetAnnotations(annotations)
+
+		if !podsRollingUpdateRequired && !c.OpConfig.EnableLazySpiloUpgrade {
+			// even if desired and actual statefulsets match
+			// there still may be not up-to-date pods on condition
+			//  (a) the lazy update was just disabled
+			// and
+			//  (b) some of the pods were not restarted when the lazy update was still in place
+			podsRollingUpdateRequired, err = c.mustUpdatePodsAfterLazyUpdate(desiredSS)
+			if err != nil {
+				return fmt.Errorf("could not list pods of the statefulset: %v", err)
+			}
+		}
+
 	}
 
 	// Apply special PostgreSQL parameters that can only be set via the Patroni API.
@@ -323,10 +387,12 @@ func (c *Cluster) syncStatefulSet() error {
 	// statefulset or those that got their configuration from the outdated statefulset)
 	if podsRollingUpdateRequired {
 		c.logger.Debugln("performing rolling update")
+		c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", "Performing rolling update")
 		if err := c.recreatePods(); err != nil {
 			return fmt.Errorf("could not recreate pods: %v", err)
 		}
 		c.logger.Infof("pods have been recreated")
+		c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", "Rolling update done - pods have been recreated")
 		if err := c.applyRollingUpdateFlagforStatefulSet(false); err != nil {
 			c.logger.Warningf("could not clear rolling update for the statefulset: %v", err)
 		}
@@ -334,6 +400,30 @@ func (c *Cluster) syncStatefulSet() error {
 	return nil
 }
 
+// AnnotationsToPropagate get the annotations to update if required
+// based on the annotations in postgres CRD
+func (c *Cluster) AnnotationsToPropagate(annotations map[string]string) map[string]string {
+	toPropagateAnnotations := c.OpConfig.DownscalerAnnotations
+	pgCRDAnnotations := c.Postgresql.ObjectMeta.GetAnnotations()
+
+	if toPropagateAnnotations != nil && pgCRDAnnotations != nil {
+		for _, anno := range toPropagateAnnotations {
+			for k, v := range pgCRDAnnotations {
+				matched, err := regexp.MatchString(anno, k)
+				if err != nil {
+					c.logger.Errorf("annotations matching issue: %v", err)
+					return nil
+				}
+				if matched {
+					annotations[k] = v
+				}
+			}
+		}
+	}
+
+	return annotations
+}
+
 // checkAndSetGlobalPostgreSQLConfiguration checks whether cluster-wide API parameters
 // (like max_connections) has changed and if necessary sets it via the Patroni API
 func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration() error {
@@ -386,14 +476,14 @@ func (c *Cluster) syncSecrets() error {
 	secrets := c.generateUserSecrets()
 
 	for secretUsername, secretSpec := range secrets {
-		if secret, err = c.KubeClient.Secrets(secretSpec.Namespace).Create(secretSpec); err == nil {
+		if secret, err = c.KubeClient.Secrets(secretSpec.Namespace).Create(context.TODO(), secretSpec, metav1.CreateOptions{}); err == nil {
 			c.Secrets[secret.UID] = secret
 			c.logger.Debugf("created new secret %q, uid: %q", util.NameFromMeta(secret.ObjectMeta), secret.UID)
 			continue
 		}
 		if k8sutil.ResourceAlreadyExists(err) {
 			var userMap map[string]spec.PgUser
-			if secret, err = c.KubeClient.Secrets(secretSpec.Namespace).Get(secretSpec.Name, metav1.GetOptions{}); err != nil {
+			if secret, err = c.KubeClient.Secrets(secretSpec.Namespace).Get(context.TODO(), secretSpec.Name, metav1.GetOptions{}); err != nil {
 				return fmt.Errorf("could not get current secret: %v", err)
 			}
 			if secretUsername != string(secret.Data["username"]) {
@@ -412,9 +502,11 @@ func (c *Cluster) syncSecrets() error {
 			}
 			pwdUser := userMap[secretUsername]
 			// if this secret belongs to the infrastructure role and the password has changed - replace it in the secret
-			if pwdUser.Password != string(secret.Data["password"]) && pwdUser.Origin == spec.RoleOriginInfrastructure {
+			if pwdUser.Password != string(secret.Data["password"]) &&
+				pwdUser.Origin == spec.RoleOriginInfrastructure {
+
 				c.logger.Debugf("updating the secret %q from the infrastructure roles", secretSpec.Name)
-				if _, err = c.KubeClient.Secrets(secretSpec.Namespace).Update(secretSpec); err != nil {
+				if _, err = c.KubeClient.Secrets(secretSpec.Namespace).Update(context.TODO(), secretSpec, metav1.UpdateOptions{}); err != nil {
 					return fmt.Errorf("could not update infrastructure role secret for role %q: %v", secretUsername, err)
 				}
 			} else {
@@ -456,6 +548,16 @@ func (c *Cluster) syncRoles() (err error) {
 	for _, u := range c.pgUsers {
 		userNames = append(userNames, u.Name)
 	}
+
+	if c.needConnectionPooler() {
+		connectionPoolerUser := c.systemUsers[constants.ConnectionPoolerUserKeyName]
+		userNames = append(userNames, connectionPoolerUser.Name)
+
+		if _, exists := c.pgUsers[connectionPoolerUser.Name]; !exists {
+			c.pgUsers[connectionPoolerUser.Name] = connectionPoolerUser
+		}
+	}
+
 	dbUsers, err = c.readPgUsersFromDatabase(userNames)
 	if err != nil {
 		return fmt.Errorf("error getting users from the database: %v", err)
@@ -494,6 +596,7 @@ func (c *Cluster) syncDatabases() error {
 
 	createDatabases := make(map[string]string)
 	alterOwnerDatabases := make(map[string]string)
+	preparedDatabases := make([]string, 0)
 
 	if err := c.initDbConn(); err != nil {
 		return fmt.Errorf("could not init database connection")
@@ -509,12 +612,24 @@ func (c *Cluster) syncDatabases() error {
 		return fmt.Errorf("could not get current databases: %v", err)
 	}
 
-	for datname, newOwner := range c.Spec.Databases {
-		currentOwner, exists := currentDatabases[datname]
+	// if no prepared databases are specified create a database named like the cluster
+	if c.Spec.PreparedDatabases != nil && len(c.Spec.PreparedDatabases) == 0 { // TODO: add option to disable creating such a default DB
+		c.Spec.PreparedDatabases = map[string]acidv1.PreparedDatabase{strings.Replace(c.Name, "-", "_", -1): {}}
+	}
+	for preparedDatabaseName := range c.Spec.PreparedDatabases {
+		_, exists := currentDatabases[preparedDatabaseName]
 		if !exists {
-			createDatabases[datname] = newOwner
+			createDatabases[preparedDatabaseName] = preparedDatabaseName + constants.OwnerRoleNameSuffix
+			preparedDatabases = append(preparedDatabases, preparedDatabaseName)
+		}
+	}
+
+	for databaseName, newOwner := range c.Spec.Databases {
+		currentOwner, exists := currentDatabases[databaseName]
+		if !exists {
+			createDatabases[databaseName] = newOwner
 		} else if currentOwner != newOwner {
-			alterOwnerDatabases[datname] = newOwner
+			alterOwnerDatabases[databaseName] = newOwner
 		}
 	}
 
@@ -522,13 +637,116 @@ func (c *Cluster) syncDatabases() error {
 		return nil
 	}
 
-	for datname, owner := range createDatabases {
-		if err = c.executeCreateDatabase(datname, owner); err != nil {
+	for databaseName, owner := range createDatabases {
+		if err = c.executeCreateDatabase(databaseName, owner); err != nil {
 			return err
 		}
 	}
-	for datname, owner := range alterOwnerDatabases {
-		if err = c.executeAlterDatabaseOwner(datname, owner); err != nil {
+	for databaseName, owner := range alterOwnerDatabases {
+		if err = c.executeAlterDatabaseOwner(databaseName, owner); err != nil {
+			return err
+		}
+	}
+
+	// set default privileges for prepared database
+	for _, preparedDatabase := range preparedDatabases {
+		if err = c.execAlterGlobalDefaultPrivileges(preparedDatabase+constants.OwnerRoleNameSuffix, preparedDatabase); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (c *Cluster) syncPreparedDatabases() error {
+	c.setProcessName("syncing prepared databases")
+	for preparedDbName, preparedDB := range c.Spec.PreparedDatabases {
+		if err := c.initDbConnWithName(preparedDbName); err != nil {
+			return fmt.Errorf("could not init connection to database %s: %v", preparedDbName, err)
+		}
+		defer func() {
+			if err := c.closeDbConn(); err != nil {
+				c.logger.Errorf("could not close database connection: %v", err)
+			}
+		}()
+
+		// now, prepare defined schemas
+		preparedSchemas := preparedDB.PreparedSchemas
+		if len(preparedDB.PreparedSchemas) == 0 {
+			preparedSchemas = map[string]acidv1.PreparedSchema{"data": {DefaultRoles: util.True()}}
+		}
+		if err := c.syncPreparedSchemas(preparedDbName, preparedSchemas); err != nil {
+			return err
+		}
+
+		// install extensions
+		if err := c.syncExtensions(preparedDB.Extensions); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (c *Cluster) syncPreparedSchemas(databaseName string, preparedSchemas map[string]acidv1.PreparedSchema) error {
+	c.setProcessName("syncing prepared schemas")
+
+	currentSchemas, err := c.getSchemas()
+	if err != nil {
+		return fmt.Errorf("could not get current schemas: %v", err)
+	}
+
+	var schemas []string
+
+	for schema := range preparedSchemas {
+		schemas = append(schemas, schema)
+	}
+
+	if createPreparedSchemas, equal := util.SubstractStringSlices(schemas, currentSchemas); !equal {
+		for _, schemaName := range createPreparedSchemas {
+			owner := constants.OwnerRoleNameSuffix
+			dbOwner := databaseName + owner
+			if preparedSchemas[schemaName].DefaultRoles == nil || *preparedSchemas[schemaName].DefaultRoles {
+				owner = databaseName + "_" + schemaName + owner
+			} else {
+				owner = dbOwner
+			}
+			if err = c.executeCreateDatabaseSchema(databaseName, schemaName, dbOwner, owner); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (c *Cluster) syncExtensions(extensions map[string]string) error {
+	c.setProcessName("syncing database extensions")
+
+	createExtensions := make(map[string]string)
+	alterExtensions := make(map[string]string)
+
+	currentExtensions, err := c.getExtensions()
+	if err != nil {
+		return fmt.Errorf("could not get current database extensions: %v", err)
+	}
+
+	for extName, newSchema := range extensions {
+		currentSchema, exists := currentExtensions[extName]
+		if !exists {
+			createExtensions[extName] = newSchema
+		} else if currentSchema != newSchema {
+			alterExtensions[extName] = newSchema
+		}
+	}
+
+	for extName, schema := range createExtensions {
+		if err = c.executeCreateExtension(extName, schema); err != nil {
+			return err
+		}
+	}
+	for extName, schema := range alterExtensions {
+		if err = c.executeAlterExtension(extName, schema); err != nil {
 			return err
 		}
 	}
@@ -547,7 +765,7 @@ func (c *Cluster) syncLogicalBackupJob() error {
 	// sync the job if it exists
 
 	jobName := c.getLogicalBackupJobName()
-	if job, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Get(jobName, metav1.GetOptions{}); err == nil {
+	if job, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Get(context.TODO(), jobName, metav1.GetOptions{}); err == nil {
 
 		desiredJob, err = c.generateLogicalBackupJob()
 		if err != nil {
@@ -581,10 +799,203 @@ func (c *Cluster) syncLogicalBackupJob() error {
 			return fmt.Errorf("could not create missing logical backup job: %v", err)
 		}
 		c.logger.Infof("logical backup job %q already exists", jobName)
-		if _, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Get(jobName, metav1.GetOptions{}); err != nil {
+		if _, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Get(context.TODO(), jobName, metav1.GetOptions{}); err != nil {
 			return fmt.Errorf("could not fetch existing logical backup job: %v", err)
 		}
 	}
 
 	return nil
 }
+
+func (c *Cluster) syncConnectionPooler(oldSpec,
+	newSpec *acidv1.Postgresql,
+	lookup InstallFunction) (SyncReason, error) {
+
+	var reason SyncReason
+	var err error
+
+	if c.ConnectionPooler == nil {
+		c.ConnectionPooler = &ConnectionPoolerObjects{}
+	}
+
+	newNeedConnectionPooler := c.needConnectionPoolerWorker(&newSpec.Spec)
+	oldNeedConnectionPooler := c.needConnectionPoolerWorker(&oldSpec.Spec)
+
+	if newNeedConnectionPooler {
+		// Try to sync in any case. If we didn't needed connection pooler before,
+		// it means we want to create it. If it was already present, still sync
+		// since it could happen that there is no difference in specs, and all
+		// the resources are remembered, but the deployment was manually deleted
+		// in between
+		c.logger.Debug("syncing connection pooler")
+
+		// in this case also do not forget to install lookup function as for
+		// creating cluster
+		if !oldNeedConnectionPooler || !c.ConnectionPooler.LookupFunction {
+			newConnectionPooler := newSpec.Spec.ConnectionPooler
+
+			specSchema := ""
+			specUser := ""
+
+			if newConnectionPooler != nil {
+				specSchema = newConnectionPooler.Schema
+				specUser = newConnectionPooler.User
+			}
+
+			schema := util.Coalesce(
+				specSchema,
+				c.OpConfig.ConnectionPooler.Schema)
+
+			user := util.Coalesce(
+				specUser,
+				c.OpConfig.ConnectionPooler.User)
+
+			if err = lookup(schema, user); err != nil {
+				return NoSync, err
+			}
+		}
+
+		if reason, err = c.syncConnectionPoolerWorker(oldSpec, newSpec); err != nil {
+			c.logger.Errorf("could not sync connection pooler: %v", err)
+			return reason, err
+		}
+	}
+
+	if oldNeedConnectionPooler && !newNeedConnectionPooler {
+		// delete and cleanup resources
+		if err = c.deleteConnectionPooler(); err != nil {
+			c.logger.Warningf("could not remove connection pooler: %v", err)
+		}
+	}
+
+	if !oldNeedConnectionPooler && !newNeedConnectionPooler {
+		// delete and cleanup resources if not empty
+		if c.ConnectionPooler != nil &&
+			(c.ConnectionPooler.Deployment != nil ||
+				c.ConnectionPooler.Service != nil) {
+
+			if err = c.deleteConnectionPooler(); err != nil {
+				c.logger.Warningf("could not remove connection pooler: %v", err)
+			}
+		}
+	}
+
+	return reason, nil
+}
+
+// Synchronize connection pooler resources. Effectively we're interested only in
+// synchronizing the corresponding deployment, but in case of deployment or
+// service is missing, create it. After checking, also remember an object for
+// the future references.
+func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql) (
+	SyncReason, error) {
+
+	deployment, err := c.KubeClient.
+		Deployments(c.Namespace).
+		Get(context.TODO(), c.connectionPoolerName(), metav1.GetOptions{})
+
+	if err != nil && k8sutil.ResourceNotFound(err) {
+		msg := "Deployment %s for connection pooler synchronization is not found, create it"
+		c.logger.Warningf(msg, c.connectionPoolerName())
+
+		deploymentSpec, err := c.generateConnectionPoolerDeployment(&newSpec.Spec)
+		if err != nil {
+			msg = "could not generate deployment for connection pooler: %v"
+			return NoSync, fmt.Errorf(msg, err)
+		}
+
+		deployment, err := c.KubeClient.
+			Deployments(deploymentSpec.Namespace).
+			Create(context.TODO(), deploymentSpec, metav1.CreateOptions{})
+
+		if err != nil {
+			return NoSync, err
+		}
+
+		c.ConnectionPooler.Deployment = deployment
+	} else if err != nil {
+		msg := "could not get connection pooler deployment to sync: %v"
+		return NoSync, fmt.Errorf(msg, err)
+	} else {
+		c.ConnectionPooler.Deployment = deployment
+
+		// actual synchronization
+		oldConnectionPooler := oldSpec.Spec.ConnectionPooler
+		newConnectionPooler := newSpec.Spec.ConnectionPooler
+
+		// sync implementation below assumes that both old and new specs are
+		// not nil, but it can happen. To avoid any confusion like updating a
+		// deployment because the specification changed from nil to an empty
+		// struct (that was initialized somewhere before) replace any nil with
+		// an empty spec.
+		if oldConnectionPooler == nil {
+			oldConnectionPooler = &acidv1.ConnectionPooler{}
+		}
+
+		if newConnectionPooler == nil {
+			newConnectionPooler = &acidv1.ConnectionPooler{}
+		}
+
+		c.logger.Infof("Old: %+v, New %+v", oldConnectionPooler, newConnectionPooler)
+
+		specSync, specReason := c.needSyncConnectionPoolerSpecs(oldConnectionPooler, newConnectionPooler)
+		defaultsSync, defaultsReason := c.needSyncConnectionPoolerDefaults(newConnectionPooler, deployment)
+		reason := append(specReason, defaultsReason...)
+		if specSync || defaultsSync {
+			c.logger.Infof("Update connection pooler deployment %s, reason: %+v",
+				c.connectionPoolerName(), reason)
+
+			newDeploymentSpec, err := c.generateConnectionPoolerDeployment(&newSpec.Spec)
+			if err != nil {
+				msg := "could not generate deployment for connection pooler: %v"
+				return reason, fmt.Errorf(msg, err)
+			}
+
+			oldDeploymentSpec := c.ConnectionPooler.Deployment
+
+			deployment, err := c.updateConnectionPoolerDeployment(
+				oldDeploymentSpec,
+				newDeploymentSpec)
+
+			if err != nil {
+				return reason, err
+			}
+
+			c.ConnectionPooler.Deployment = deployment
+			return reason, nil
+		}
+	}
+
+	newAnnotations := c.AnnotationsToPropagate(c.ConnectionPooler.Deployment.Annotations)
+	if newAnnotations != nil {
+		c.updateConnectionPoolerAnnotations(newAnnotations)
+	}
+
+	service, err := c.KubeClient.
+		Services(c.Namespace).
+		Get(context.TODO(), c.connectionPoolerName(), metav1.GetOptions{})
+
+	if err != nil && k8sutil.ResourceNotFound(err) {
+		msg := "Service %s for connection pooler synchronization is not found, create it"
+		c.logger.Warningf(msg, c.connectionPoolerName())
+
+		serviceSpec := c.generateConnectionPoolerService(&newSpec.Spec)
+		service, err := c.KubeClient.
+			Services(serviceSpec.Namespace).
+			Create(context.TODO(), serviceSpec, metav1.CreateOptions{})
+
+		if err != nil {
+			return NoSync, err
+		}
+
+		c.ConnectionPooler.Service = service
+	} else if err != nil {
+		msg := "could not get connection pooler service to sync: %v"
+		return NoSync, fmt.Errorf(msg, err)
+	} else {
+		// Service updates are not supported and probably not that useful anyway
+		c.ConnectionPooler.Service = service
+	}
+
+	return NoSync, nil
+}

pkg/cluster/sync_test.go

@@ -0,0 +1,261 @@
+package cluster
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
+	"github.com/zalando/postgres-operator/pkg/util/config"
+	"github.com/zalando/postgres-operator/pkg/util/k8sutil"
+
+	appsv1 "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func int32ToPointer(value int32) *int32 {
+	return &value
+}
+
+func deploymentUpdated(cluster *Cluster, err error, reason SyncReason) error {
+	if cluster.ConnectionPooler.Deployment.Spec.Replicas == nil ||
+		*cluster.ConnectionPooler.Deployment.Spec.Replicas != 2 {
+		return fmt.Errorf("Wrong nubmer of instances")
+	}
+
+	return nil
+}
+
+func objectsAreSaved(cluster *Cluster, err error, reason SyncReason) error {
+	if cluster.ConnectionPooler == nil {
+		return fmt.Errorf("Connection pooler resources are empty")
+	}
+
+	if cluster.ConnectionPooler.Deployment == nil {
+		return fmt.Errorf("Deployment was not saved")
+	}
+
+	if cluster.ConnectionPooler.Service == nil {
+		return fmt.Errorf("Service was not saved")
+	}
+
+	return nil
+}
+
+func objectsAreDeleted(cluster *Cluster, err error, reason SyncReason) error {
+	if cluster.ConnectionPooler != nil {
+		return fmt.Errorf("Connection pooler was not deleted")
+	}
+
+	return nil
+}
+
+func noEmptySync(cluster *Cluster, err error, reason SyncReason) error {
+	for _, msg := range reason {
+		if strings.HasPrefix(msg, "update [] from '<nil>' to '") {
+			return fmt.Errorf("There is an empty reason, %s", msg)
+		}
+	}
+
+	return nil
+}
+
+func TestConnectionPoolerSynchronization(t *testing.T) {
+	testName := "Test connection pooler synchronization"
+	var cluster = New(
+		Config{
+			OpConfig: config.Config{
+				ProtectedRoles: []string{"admin"},
+				Auth: config.Auth{
+					SuperUsername:       superUserName,
+					ReplicationUsername: replicationUserName,
+				},
+				ConnectionPooler: config.ConnectionPooler{
+					ConnectionPoolerDefaultCPURequest:    "100m",
+					ConnectionPoolerDefaultCPULimit:      "100m",
+					ConnectionPoolerDefaultMemoryRequest: "100Mi",
+					ConnectionPoolerDefaultMemoryLimit:   "100Mi",
+					NumberOfInstances:                    int32ToPointer(1),
+				},
+			},
+		}, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder)
+
+	cluster.Statefulset = &appsv1.StatefulSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test-sts",
+		},
+	}
+
+	clusterMissingObjects := *cluster
+	clusterMissingObjects.KubeClient = k8sutil.ClientMissingObjects()
+
+	clusterMock := *cluster
+	clusterMock.KubeClient = k8sutil.NewMockKubernetesClient()
+
+	clusterDirtyMock := *cluster
+	clusterDirtyMock.KubeClient = k8sutil.NewMockKubernetesClient()
+	clusterDirtyMock.ConnectionPooler = &ConnectionPoolerObjects{
+		Deployment: &appsv1.Deployment{},
+		Service:    &v1.Service{},
+	}
+
+	clusterNewDefaultsMock := *cluster
+	clusterNewDefaultsMock.KubeClient = k8sutil.NewMockKubernetesClient()
+
+	tests := []struct {
+		subTest          string
+		oldSpec          *acidv1.Postgresql
+		newSpec          *acidv1.Postgresql
+		cluster          *Cluster
+		defaultImage     string
+		defaultInstances int32
+		check            func(cluster *Cluster, err error, reason SyncReason) error
+	}{
+		{
+			subTest: "create if doesn't exist",
+			oldSpec: &acidv1.Postgresql{
+				Spec: acidv1.PostgresSpec{
+					ConnectionPooler: &acidv1.ConnectionPooler{},
+				},
+			},
+			newSpec: &acidv1.Postgresql{
+				Spec: acidv1.PostgresSpec{
+					ConnectionPooler: &acidv1.ConnectionPooler{},
+				},
+			},
+			cluster:          &clusterMissingObjects,
+			defaultImage:     "pooler:1.0",
+			defaultInstances: 1,
+			check:            objectsAreSaved,
+		},
+		{
+			subTest: "create if doesn't exist with a flag",
+			oldSpec: &acidv1.Postgresql{
+				Spec: acidv1.PostgresSpec{},
+			},
+			newSpec: &acidv1.Postgresql{
+				Spec: acidv1.PostgresSpec{
+					EnableConnectionPooler: boolToPointer(true),
+				},
+			},
+			cluster:          &clusterMissingObjects,
+			defaultImage:     "pooler:1.0",
+			defaultInstances: 1,
+			check:            objectsAreSaved,
+		},
+		{
+			subTest: "create from scratch",
+			oldSpec: &acidv1.Postgresql{
+				Spec: acidv1.PostgresSpec{},
+			},
+			newSpec: &acidv1.Postgresql{
+				Spec: acidv1.PostgresSpec{
+					ConnectionPooler: &acidv1.ConnectionPooler{},
+				},
+			},
+			cluster:          &clusterMissingObjects,
+			defaultImage:     "pooler:1.0",
+			defaultInstances: 1,
+			check:            objectsAreSaved,
+		},
+		{
+			subTest: "delete if not needed",
+			oldSpec: &acidv1.Postgresql{
+				Spec: acidv1.PostgresSpec{
+					ConnectionPooler: &acidv1.ConnectionPooler{},
+				},
+			},
+			newSpec: &acidv1.Postgresql{
+				Spec: acidv1.PostgresSpec{},
+			},
+			cluster:          &clusterMock,
+			defaultImage:     "pooler:1.0",
+			defaultInstances: 1,
+			check:            objectsAreDeleted,
+		},
+		{
+			subTest: "cleanup if still there",
+			oldSpec: &acidv1.Postgresql{
+				Spec: acidv1.PostgresSpec{},
+			},
+			newSpec: &acidv1.Postgresql{
+				Spec: acidv1.PostgresSpec{},
+			},
+			cluster:          &clusterDirtyMock,
+			defaultImage:     "pooler:1.0",
+			defaultInstances: 1,
+			check:            objectsAreDeleted,
+		},
+		{
+			subTest: "update deployment",
+			oldSpec: &acidv1.Postgresql{
+				Spec: acidv1.PostgresSpec{
+					ConnectionPooler: &acidv1.ConnectionPooler{
+						NumberOfInstances: int32ToPointer(1),
+					},
+				},
+			},
+			newSpec: &acidv1.Postgresql{
+				Spec: acidv1.PostgresSpec{
+					ConnectionPooler: &acidv1.ConnectionPooler{
+						NumberOfInstances: int32ToPointer(2),
+					},
+				},
+			},
+			cluster:          &clusterMock,
+			defaultImage:     "pooler:1.0",
+			defaultInstances: 1,
+			check:            deploymentUpdated,
+		},
+		{
+			subTest: "update image from changed defaults",
+			oldSpec: &acidv1.Postgresql{
+				Spec: acidv1.PostgresSpec{
+					ConnectionPooler: &acidv1.ConnectionPooler{},
+				},
+			},
+			newSpec: &acidv1.Postgresql{
+				Spec: acidv1.PostgresSpec{
+					ConnectionPooler: &acidv1.ConnectionPooler{},
+				},
+			},
+			cluster:          &clusterNewDefaultsMock,
+			defaultImage:     "pooler:2.0",
+			defaultInstances: 2,
+			check:            deploymentUpdated,
+		},
+		{
+			subTest: "there is no sync from nil to an empty spec",
+			oldSpec: &acidv1.Postgresql{
+				Spec: acidv1.PostgresSpec{
+					EnableConnectionPooler: boolToPointer(true),
+					ConnectionPooler:       nil,
+				},
+			},
+			newSpec: &acidv1.Postgresql{
+				Spec: acidv1.PostgresSpec{
+					EnableConnectionPooler: boolToPointer(true),
+					ConnectionPooler:       &acidv1.ConnectionPooler{},
+				},
+			},
+			cluster:          &clusterMock,
+			defaultImage:     "pooler:1.0",
+			defaultInstances: 1,
+			check:            noEmptySync,
+		},
+	}
+	for _, tt := range tests {
+		tt.cluster.OpConfig.ConnectionPooler.Image = tt.defaultImage
+		tt.cluster.OpConfig.ConnectionPooler.NumberOfInstances =
+			int32ToPointer(tt.defaultInstances)
+
+		reason, err := tt.cluster.syncConnectionPooler(tt.oldSpec,
+			tt.newSpec, mockInstallLookupFunction)
+
+		if err := tt.check(tt.cluster, err, reason); err != nil {
+			t.Errorf("%s [%s]: Could not synchronize, %+v",
+				testName, tt.subTest, err)
+		}
+	}
+}

pkg/cluster/types.go

@@ -69,3 +69,12 @@ type ClusterStatus struct {
 	Spec           acidv1.PostgresSpec
 	Error          error
 }
+
+type TemplateParams map[string]interface{}
+
+type InstallFunction func(schema string, user string) error
+
+type SyncReason []string
+
+// no sync happened, empty value
+var NoSync SyncReason = []string{}

pkg/cluster/util.go

@@ -2,6 +2,7 @@ package cluster
 
 import (
 	"bytes"
+	"context"
 	"encoding/gob"
 	"encoding/json"
 	"fmt"
@@ -47,7 +48,7 @@ func (g *SecretOauthTokenGetter) getOAuthToken() (string, error) {
 	// Temporary getting postgresql-operator secret from the NamespaceDefault
 	credentialsSecret, err := g.kubeClient.
 		Secrets(g.OAuthTokenSecretName.Namespace).
-		Get(g.OAuthTokenSecretName.Name, metav1.GetOptions{})
+		Get(context.TODO(), g.OAuthTokenSecretName.Name, metav1.GetOptions{})
 
 	if err != nil {
 		return "", fmt.Errorf("could not get credentials secret: %v", err)
@@ -278,7 +279,7 @@ func (c *Cluster) waitStatefulsetReady() error {
 			listOptions := metav1.ListOptions{
 				LabelSelector: c.labelsSet(false).String(),
 			}
-			ss, err := c.KubeClient.StatefulSets(c.Namespace).List(listOptions)
+			ss, err := c.KubeClient.StatefulSets(c.Namespace).List(context.TODO(), listOptions)
 			if err != nil {
 				return false, err
 			}
@@ -313,7 +314,7 @@ func (c *Cluster) _waitPodLabelsReady(anyReplica bool) error {
 	}
 	podsNumber = 1
 	if !anyReplica {
-		pods, err := c.KubeClient.Pods(namespace).List(listOptions)
+		pods, err := c.KubeClient.Pods(namespace).List(context.TODO(), listOptions)
 		if err != nil {
 			return err
 		}
@@ -327,7 +328,7 @@ func (c *Cluster) _waitPodLabelsReady(anyReplica bool) error {
 		func() (bool, error) {
 			masterCount := 0
 			if !anyReplica {
-				masterPods, err2 := c.KubeClient.Pods(namespace).List(masterListOption)
+				masterPods, err2 := c.KubeClient.Pods(namespace).List(context.TODO(), masterListOption)
 				if err2 != nil {
 					return false, err2
 				}
@@ -337,7 +338,7 @@ func (c *Cluster) _waitPodLabelsReady(anyReplica bool) error {
 				}
 				masterCount = len(masterPods.Items)
 			}
-			replicaPods, err2 := c.KubeClient.Pods(namespace).List(replicaListOption)
+			replicaPods, err2 := c.KubeClient.Pods(namespace).List(context.TODO(), replicaListOption)
 			if err2 != nil {
 				return false, err2
 			}
@@ -408,7 +409,32 @@ func (c *Cluster) labelsSet(shouldAddExtraLabels bool) labels.Set {
 }
 
 func (c *Cluster) labelsSelector() *metav1.LabelSelector {
-	return &metav1.LabelSelector{MatchLabels: c.labelsSet(false), MatchExpressions: nil}
+	return &metav1.LabelSelector{
+		MatchLabels:      c.labelsSet(false),
+		MatchExpressions: nil,
+	}
+}
+
+// Return connection pooler labels selector, which should from one point of view
+// inherit most of the labels from the cluster itself, but at the same time
+// have e.g. different `application` label, so that recreatePod operation will
+// not interfere with it (it lists all the pods via labels, and if there would
+// be no difference, it will recreate also pooler pods).
+func (c *Cluster) connectionPoolerLabelsSelector() *metav1.LabelSelector {
+	connectionPoolerLabels := labels.Set(map[string]string{})
+
+	extraLabels := labels.Set(map[string]string{
+		"connection-pooler": c.connectionPoolerName(),
+		"application":       "db-connection-pooler",
+	})
+
+	connectionPoolerLabels = labels.Merge(connectionPoolerLabels, c.labelsSet(false))
+	connectionPoolerLabels = labels.Merge(connectionPoolerLabels, extraLabels)
+
+	return &metav1.LabelSelector{
+		MatchLabels:      connectionPoolerLabels,
+		MatchExpressions: nil,
+	}
 }
 
 func (c *Cluster) roleLabelsSet(shouldAddExtraLabels bool, role PostgresRole) labels.Set {
@@ -483,3 +509,43 @@ func (c *Cluster) GetSpec() (*acidv1.Postgresql, error) {
 func (c *Cluster) patroniUsesKubernetes() bool {
 	return c.OpConfig.EtcdHost == ""
 }
+
+func (c *Cluster) patroniKubernetesUseConfigMaps() bool {
+	if !c.patroniUsesKubernetes() {
+		return false
+	}
+
+	// otherwise, follow the operator configuration
+	return c.OpConfig.KubernetesUseConfigMaps
+}
+
+func (c *Cluster) needConnectionPoolerWorker(spec *acidv1.PostgresSpec) bool {
+	if spec.EnableConnectionPooler == nil {
+		return spec.ConnectionPooler != nil
+	} else {
+		return *spec.EnableConnectionPooler
+	}
+}
+
+func (c *Cluster) needConnectionPooler() bool {
+	return c.needConnectionPoolerWorker(&c.Spec)
+}
+
+// Earlier arguments take priority
+func mergeContainers(containers ...[]v1.Container) ([]v1.Container, []string) {
+	containerNameTaken := map[string]bool{}
+	result := make([]v1.Container, 0)
+	conflicts := make([]string, 0)
+
+	for _, containerArray := range containers {
+		for _, container := range containerArray {
+			if _, taken := containerNameTaken[container.Name]; taken {
+				conflicts = append(conflicts, container.Name)
+			} else {
+				containerNameTaken[container.Name] = true
+				result = append(result, container)
+			}
+		}
+	}
+	return result, conflicts
+}

pkg/cluster/volumes.go

@@ -1,11 +1,12 @@
 package cluster
 
 import (
+	"context"
 	"fmt"
 	"strconv"
 	"strings"
 
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
@@ -23,7 +24,7 @@ func (c *Cluster) listPersistentVolumeClaims() ([]v1.PersistentVolumeClaim, erro
 		LabelSelector: c.labelsSet(false).String(),
 	}
 
-	pvcs, err := c.KubeClient.PersistentVolumeClaims(ns).List(listOptions)
+	pvcs, err := c.KubeClient.PersistentVolumeClaims(ns).List(context.TODO(), listOptions)
 	if err != nil {
 		return nil, fmt.Errorf("could not list of PersistentVolumeClaims: %v", err)
 	}
@@ -38,7 +39,7 @@ func (c *Cluster) deletePersistentVolumeClaims() error {
 	}
 	for _, pvc := range pvcs {
 		c.logger.Debugf("deleting PVC %q", util.NameFromMeta(pvc.ObjectMeta))
-		if err := c.KubeClient.PersistentVolumeClaims(pvc.Namespace).Delete(pvc.Name, c.deleteOptions); err != nil {
+		if err := c.KubeClient.PersistentVolumeClaims(pvc.Namespace).Delete(context.TODO(), pvc.Name, c.deleteOptions); err != nil {
 			c.logger.Warningf("could not delete PersistentVolumeClaim: %v", err)
 		}
 	}
@@ -78,7 +79,7 @@ func (c *Cluster) listPersistentVolumes() ([]*v1.PersistentVolume, error) {
 				continue
 			}
 		}
-		pv, err := c.KubeClient.PersistentVolumes().Get(pvc.Spec.VolumeName, metav1.GetOptions{})
+		pv, err := c.KubeClient.PersistentVolumes().Get(context.TODO(), pvc.Spec.VolumeName, metav1.GetOptions{})
 		if err != nil {
 			return nil, fmt.Errorf("could not get PersistentVolume: %v", err)
 		}
@@ -143,7 +144,7 @@ func (c *Cluster) resizeVolumes(newVolume acidv1.Volume, resizers []volumes.Volu
 			c.logger.Debugf("filesystem resize successful on volume %q", pv.Name)
 			pv.Spec.Capacity[v1.ResourceStorage] = newQuantity
 			c.logger.Debugf("updating persistent volume definition for volume %q", pv.Name)
-			if _, err := c.KubeClient.PersistentVolumes().Update(pv); err != nil {
+			if _, err := c.KubeClient.PersistentVolumes().Update(context.TODO(), pv, metav1.UpdateOptions{}); err != nil {
 				return fmt.Errorf("could not update persistent volume: %q", err)
 			}
 			c.logger.Debugf("successfully updated persistent volume %q", pv.Name)

pkg/controller/controller.go

@@ -1,28 +1,30 @@
 package controller
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"sync"
 
 	"github.com/sirupsen/logrus"
-	v1 "k8s.io/api/core/v1"
-	rbacv1 "k8s.io/api/rbac/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/client-go/kubernetes/scheme"
-	"k8s.io/client-go/tools/cache"
-
+	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
 	"github.com/zalando/postgres-operator/pkg/apiserver"
 	"github.com/zalando/postgres-operator/pkg/cluster"
+	acidv1informer "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/acid.zalan.do/v1"
 	"github.com/zalando/postgres-operator/pkg/spec"
 	"github.com/zalando/postgres-operator/pkg/util"
 	"github.com/zalando/postgres-operator/pkg/util/config"
 	"github.com/zalando/postgres-operator/pkg/util/constants"
 	"github.com/zalando/postgres-operator/pkg/util/k8sutil"
 	"github.com/zalando/postgres-operator/pkg/util/ringlog"
-
-	acidv1informer "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/acid.zalan.do/v1"
+	v1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes/scheme"
+	typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/record"
 )
 
 // Controller represents operator controller
@@ -34,8 +36,12 @@ type Controller struct {
 	KubeClient k8sutil.KubernetesClient
 	apiserver  *apiserver.Server
 
+	eventRecorder    record.EventRecorder
+	eventBroadcaster record.EventBroadcaster
+
 	stopCh chan struct{}
 
+	controllerID     string
 	curWorkerID      uint32 //initialized with 0
 	curWorkerCluster sync.Map
 	clusterWorkers   map[spec.NamespacedName]uint32
@@ -61,13 +67,25 @@ type Controller struct {
 }
 
 // NewController creates a new controller
-func NewController(controllerConfig *spec.ControllerConfig) *Controller {
+func NewController(controllerConfig *spec.ControllerConfig, controllerId string) *Controller {
 	logger := logrus.New()
 
+	var myComponentName = "postgres-operator"
+	if controllerId != "" {
+		myComponentName += "/" + controllerId
+	}
+
+	eventBroadcaster := record.NewBroadcaster()
+	eventBroadcaster.StartLogging(logger.Infof)
+	recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: myComponentName})
+
 	c := &Controller{
 		config:           *controllerConfig,
 		opConfig:         &config.Config{},
 		logger:           logger.WithField("pkg", "controller"),
+		eventRecorder:    recorder,
+		eventBroadcaster: eventBroadcaster,
+		controllerID:     controllerId,
 		curWorkerCluster: sync.Map{},
 		clusterWorkers:   make(map[spec.NamespacedName]uint32),
 		clusters:         make(map[spec.NamespacedName]*cluster.Cluster),
@@ -89,6 +107,11 @@ func (c *Controller) initClients() {
 	if err != nil {
 		c.logger.Fatalf("could not create kubernetes clients: %v", err)
 	}
+	c.eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: c.KubeClient.EventsGetter.Events("")})
+	if err != nil {
+		c.logger.Fatalf("could not setup kubernetes event sink: %v", err)
+	}
+
 }
 
 func (c *Controller) initOperatorConfig() {
@@ -96,7 +119,7 @@ func (c *Controller) initOperatorConfig() {
 
 	if c.config.ConfigMapName != (spec.NamespacedName{}) {
 		configMap, err := c.KubeClient.ConfigMaps(c.config.ConfigMapName.Namespace).
-			Get(c.config.ConfigMapName.Name, metav1.GetOptions{})
+			Get(context.TODO(), c.config.ConfigMapName.Name, metav1.GetOptions{})
 		if err != nil {
 			panic(err)
 		}
@@ -155,6 +178,11 @@ func (c *Controller) warnOnDeprecatedOperatorParameters() {
 		c.logger.Warningf("Operator configuration parameter 'enable_load_balancer' is deprecated and takes no effect. " +
 			"Consider using the 'enable_master_load_balancer' or 'enable_replica_load_balancer' instead.")
 	}
+
+	if len(c.opConfig.SidecarImages) > 0 {
+		c.logger.Warningf("Operator configuration parameter 'sidecar_docker_images' is deprecated. " +
+			"Consider using 'sidecars' instead.")
+	}
 }
 
 func (c *Controller) initPodServiceAccount() {
@@ -239,6 +267,7 @@ func (c *Controller) initRoleBinding() {
 
 func (c *Controller) initController() {
 	c.initClients()
+	c.controllerID = os.Getenv("CONTROLLER_ID")
 
 	if configObjectName := os.Getenv("POSTGRES_OPERATOR_CONFIGURATION_OBJECT"); configObjectName != "" {
 		if err := c.createConfigurationCRD(c.opConfig.EnableCRDValidation); err != nil {
@@ -402,7 +431,7 @@ func (c *Controller) getEffectiveNamespace(namespaceFromEnvironment, namespaceFr
 
 	} else {
 
-		if _, err := c.KubeClient.Namespaces().Get(namespace, metav1.GetOptions{}); err != nil {
+		if _, err := c.KubeClient.Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{}); err != nil {
 			c.logger.Fatalf("Could not find the watched namespace %q", namespace)
 		} else {
 			c.logger.Infof("Listenting to the specific namespace %q", namespace)
@@ -412,3 +441,16 @@ func (c *Controller) getEffectiveNamespace(namespaceFromEnvironment, namespaceFr
 
 	return namespace
 }
+
+// hasOwnership returns true if the controller is the "owner" of the postgresql.
+// Whether it's owner is determined by the value of 'acid.zalan.do/controller'
+// annotation. If the value matches the controllerID then it owns it, or if the
+// controllerID is "" and there's no annotation set.
+func (c *Controller) hasOwnership(postgresql *acidv1.Postgresql) bool {
+	if postgresql.Annotations != nil {
+		if owner, ok := postgresql.Annotations[constants.PostgresqlControllerAnnotationKey]; ok {
+			return owner == c.controllerID
+		}
+	}
+	return c.controllerID == ""
+}

pkg/controller/node.go

@@ -1,11 +1,12 @@
 package controller
 
 import (
+	"context"
 	"fmt"
 	"time"
 
 	"github.com/zalando/postgres-operator/pkg/util/retryutil"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -22,7 +23,7 @@ func (c *Controller) nodeListFunc(options metav1.ListOptions) (runtime.Object, e
 		TimeoutSeconds:  options.TimeoutSeconds,
 	}
 
-	return c.KubeClient.Nodes().List(opts)
+	return c.KubeClient.Nodes().List(context.TODO(), opts)
 }
 
 func (c *Controller) nodeWatchFunc(options metav1.ListOptions) (watch.Interface, error) {
@@ -32,7 +33,7 @@ func (c *Controller) nodeWatchFunc(options metav1.ListOptions) (watch.Interface,
 		TimeoutSeconds:  options.TimeoutSeconds,
 	}
 
-	return c.KubeClient.Nodes().Watch(opts)
+	return c.KubeClient.Nodes().Watch(context.TODO(), opts)
 }
 
 func (c *Controller) nodeAdd(obj interface{}) {
@@ -87,7 +88,7 @@ func (c *Controller) attemptToMoveMasterPodsOffNode(node *v1.Node) error {
 	opts := metav1.ListOptions{
 		LabelSelector: labels.Set(c.opConfig.ClusterLabels).String(),
 	}
-	podList, err := c.KubeClient.Pods(c.opConfig.WatchedNamespace).List(opts)
+	podList, err := c.KubeClient.Pods(c.opConfig.WatchedNamespace).List(context.TODO(), opts)
 	if err != nil {
 		c.logger.Errorf("could not fetch list of the pods: %v", err)
 		return err
@@ -172,19 +173,19 @@ func (c *Controller) nodeDelete(obj interface{}) {
 }
 
 func (c *Controller) moveMasterPodsOffNode(node *v1.Node) {
-
+	// retry to move master until configured timeout is reached
 	err := retryutil.Retry(1*time.Minute, c.opConfig.MasterPodMoveTimeout,
 		func() (bool, error) {
 			err := c.attemptToMoveMasterPodsOffNode(node)
 			if err != nil {
-				return false, fmt.Errorf("unable to move master pods off the unschedulable node; will retry after delay of 1 minute")
+				return false, err
 			}
 			return true, nil
 		},
 	)
 
 	if err != nil {
-		c.logger.Warningf("failed to move master pods from the node %q: timeout of %v minutes expired", node.Name, c.opConfig.MasterPodMoveTimeout)
+		c.logger.Warningf("failed to move master pods from the node %q: %v", node.Name, err)
 	}
 
 }

pkg/controller/node_test.go

@@ -4,7 +4,7 @@ import (
 	"testing"
 
 	"github.com/zalando/postgres-operator/pkg/spec"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -13,10 +13,10 @@ const (
 	readyValue = "ready"
 )
 
-func initializeController() *Controller {
-	var c = NewController(&spec.ControllerConfig{})
-	c.opConfig.NodeReadinessLabel = map[string]string{readyLabel: readyValue}
-	return c
+func newNodeTestController() *Controller {
+	var controller = NewController(&spec.ControllerConfig{}, "node-test")
+	controller.opConfig.NodeReadinessLabel = map[string]string{readyLabel: readyValue}
+	return controller
 }
 
 func makeNode(labels map[string]string, isSchedulable bool) *v1.Node {
@@ -31,7 +31,7 @@ func makeNode(labels map[string]string, isSchedulable bool) *v1.Node {
 	}
 }
 
-var c = initializeController()
+var nodeTestController = newNodeTestController()
 
 func TestNodeIsReady(t *testing.T) {
 	testName := "TestNodeIsReady"
@@ -57,7 +57,7 @@ func TestNodeIsReady(t *testing.T) {
 		},
 	}
 	for _, tt := range testTable {
-		if isReady := c.nodeIsReady(tt.in); isReady != tt.out {
+		if isReady := nodeTestController.nodeIsReady(tt.in); isReady != tt.out {
 			t.Errorf("%s: expected response %t doesn't match the actual %t for the node %#v",
 				testName, tt.out, isReady, tt.in)
 		}

pkg/controller/operator_config.go

@@ -1,18 +1,22 @@
 package controller
 
 import (
+	"context"
 	"fmt"
 
 	"time"
 
 	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
+	"github.com/zalando/postgres-operator/pkg/util"
 	"github.com/zalando/postgres-operator/pkg/util/config"
+	"github.com/zalando/postgres-operator/pkg/util/constants"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 func (c *Controller) readOperatorConfigurationFromCRD(configObjectNamespace, configObjectName string) (*acidv1.OperatorConfiguration, error) {
 
-	config, err := c.KubeClient.AcidV1ClientSet.AcidV1().OperatorConfigurations(configObjectNamespace).Get(configObjectName, metav1.GetOptions{})
+	config, err := c.KubeClient.AcidV1ClientSet.AcidV1().OperatorConfigurations(configObjectNamespace).Get(
+		context.TODO(), configObjectName, metav1.GetOptions{})
 	if err != nil {
 		return nil, fmt.Errorf("could not get operator configuration object %q: %v", configObjectName, err)
 	}
@@ -20,63 +24,71 @@ func (c *Controller) readOperatorConfigurationFromCRD(configObjectNamespace, con
 	return config, nil
 }
 
+func int32ToPointer(value int32) *int32 {
+	return &value
+}
+
 // importConfigurationFromCRD is a transitional function that converts CRD configuration to the one based on the configmap
 func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigurationData) *config.Config {
 	result := &config.Config{}
 
 	// general config
-	result.EnableCRDValidation = fromCRD.EnableCRDValidation
+	result.EnableCRDValidation = util.CoalesceBool(fromCRD.EnableCRDValidation, util.True())
+	result.EnableLazySpiloUpgrade = fromCRD.EnableLazySpiloUpgrade
 	result.EtcdHost = fromCRD.EtcdHost
-	result.DockerImage = fromCRD.DockerImage
+	result.KubernetesUseConfigMaps = fromCRD.KubernetesUseConfigMaps
+	result.DockerImage = util.Coalesce(fromCRD.DockerImage, "registry.opensource.zalan.do/acid/spilo-12:1.6-p3")
 	result.Workers = fromCRD.Workers
 	result.MinInstances = fromCRD.MinInstances
 	result.MaxInstances = fromCRD.MaxInstances
 	result.ResyncPeriod = time.Duration(fromCRD.ResyncPeriod)
 	result.RepairPeriod = time.Duration(fromCRD.RepairPeriod)
 	result.SetMemoryRequestToLimit = fromCRD.SetMemoryRequestToLimit
-	result.ShmVolume = fromCRD.ShmVolume
-	result.Sidecars = fromCRD.Sidecars
+	result.ShmVolume = util.CoalesceBool(fromCRD.ShmVolume, util.True())
+	result.SidecarImages = fromCRD.SidecarImages
+	result.SidecarContainers = fromCRD.SidecarContainers
 
 	// user config
-	result.SuperUsername = fromCRD.PostgresUsersConfiguration.SuperUsername
-	result.ReplicationUsername = fromCRD.PostgresUsersConfiguration.ReplicationUsername
+	result.SuperUsername = util.Coalesce(fromCRD.PostgresUsersConfiguration.SuperUsername, "postgres")
+	result.ReplicationUsername = util.Coalesce(fromCRD.PostgresUsersConfiguration.ReplicationUsername, "standby")
 
 	// kubernetes config
 	result.CustomPodAnnotations = fromCRD.Kubernetes.CustomPodAnnotations
-	result.PodServiceAccountName = fromCRD.Kubernetes.PodServiceAccountName
+	result.PodServiceAccountName = util.Coalesce(fromCRD.Kubernetes.PodServiceAccountName, "postgres-pod")
 	result.PodServiceAccountDefinition = fromCRD.Kubernetes.PodServiceAccountDefinition
 	result.PodServiceAccountRoleBindingDefinition = fromCRD.Kubernetes.PodServiceAccountRoleBindingDefinition
 	result.PodEnvironmentConfigMap = fromCRD.Kubernetes.PodEnvironmentConfigMap
 	result.PodTerminateGracePeriod = time.Duration(fromCRD.Kubernetes.PodTerminateGracePeriod)
 	result.SpiloPrivileged = fromCRD.Kubernetes.SpiloPrivileged
 	result.SpiloFSGroup = fromCRD.Kubernetes.SpiloFSGroup
-	result.ClusterDomain = fromCRD.Kubernetes.ClusterDomain
+	result.ClusterDomain = util.Coalesce(fromCRD.Kubernetes.ClusterDomain, "cluster.local")
 	result.WatchedNamespace = fromCRD.Kubernetes.WatchedNamespace
 	result.PDBNameFormat = fromCRD.Kubernetes.PDBNameFormat
-	result.EnablePodDisruptionBudget = fromCRD.Kubernetes.EnablePodDisruptionBudget
-	result.EnableInitContainers = fromCRD.Kubernetes.EnableInitContainers
-	result.EnableSidecars = fromCRD.Kubernetes.EnableSidecars
+	result.EnablePodDisruptionBudget = util.CoalesceBool(fromCRD.Kubernetes.EnablePodDisruptionBudget, util.True())
+	result.EnableInitContainers = util.CoalesceBool(fromCRD.Kubernetes.EnableInitContainers, util.True())
+	result.EnableSidecars = util.CoalesceBool(fromCRD.Kubernetes.EnableSidecars, util.True())
 	result.SecretNameTemplate = fromCRD.Kubernetes.SecretNameTemplate
 	result.OAuthTokenSecretName = fromCRD.Kubernetes.OAuthTokenSecretName
 	result.InfrastructureRolesSecretName = fromCRD.Kubernetes.InfrastructureRolesSecretName
-	result.PodRoleLabel = fromCRD.Kubernetes.PodRoleLabel
+	result.PodRoleLabel = util.Coalesce(fromCRD.Kubernetes.PodRoleLabel, "spilo-role")
 	result.ClusterLabels = fromCRD.Kubernetes.ClusterLabels
 	result.InheritedLabels = fromCRD.Kubernetes.InheritedLabels
-	result.ClusterNameLabel = fromCRD.Kubernetes.ClusterNameLabel
+	result.DownscalerAnnotations = fromCRD.Kubernetes.DownscalerAnnotations
+	result.ClusterNameLabel = util.Coalesce(fromCRD.Kubernetes.ClusterNameLabel, "cluster-name")
 	result.NodeReadinessLabel = fromCRD.Kubernetes.NodeReadinessLabel
 	result.PodPriorityClassName = fromCRD.Kubernetes.PodPriorityClassName
-	result.PodManagementPolicy = fromCRD.Kubernetes.PodManagementPolicy
+	result.PodManagementPolicy = util.Coalesce(fromCRD.Kubernetes.PodManagementPolicy, "ordered_ready")
 	result.MasterPodMoveTimeout = time.Duration(fromCRD.Kubernetes.MasterPodMoveTimeout)
 	result.EnablePodAntiAffinity = fromCRD.Kubernetes.EnablePodAntiAffinity
-	result.PodAntiAffinityTopologyKey = fromCRD.Kubernetes.PodAntiAffinityTopologyKey
+	result.PodAntiAffinityTopologyKey = util.Coalesce(fromCRD.Kubernetes.PodAntiAffinityTopologyKey, "kubernetes.io/hostname")
 
 	// Postgres Pod resources
-	result.DefaultCPURequest = fromCRD.PostgresPodResources.DefaultCPURequest
-	result.DefaultMemoryRequest = fromCRD.PostgresPodResources.DefaultMemoryRequest
-	result.DefaultCPULimit = fromCRD.PostgresPodResources.DefaultCPULimit
-	result.DefaultMemoryLimit = fromCRD.PostgresPodResources.DefaultMemoryLimit
-	result.MinCPULimit = fromCRD.PostgresPodResources.MinCPULimit
-	result.MinMemoryLimit = fromCRD.PostgresPodResources.MinMemoryLimit
+	result.DefaultCPURequest = util.Coalesce(fromCRD.PostgresPodResources.DefaultCPURequest, "100m")
+	result.DefaultMemoryRequest = util.Coalesce(fromCRD.PostgresPodResources.DefaultMemoryRequest, "100Mi")
+	result.DefaultCPULimit = util.Coalesce(fromCRD.PostgresPodResources.DefaultCPULimit, "1")
+	result.DefaultMemoryLimit = util.Coalesce(fromCRD.PostgresPodResources.DefaultMemoryLimit, "500Mi")
+	result.MinCPULimit = util.Coalesce(fromCRD.PostgresPodResources.MinCPULimit, "250m")
+	result.MinMemoryLimit = util.Coalesce(fromCRD.PostgresPodResources.MinMemoryLimit, "250Mi")
 
 	// timeout config
 	result.ResourceCheckInterval = time.Duration(fromCRD.Timeouts.ResourceCheckInterval)
@@ -103,8 +115,8 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	result.AdditionalSecretMountPath = fromCRD.AWSGCP.AdditionalSecretMountPath
 
 	// logical backup config
-	result.LogicalBackupSchedule = fromCRD.LogicalBackup.Schedule
-	result.LogicalBackupDockerImage = fromCRD.LogicalBackup.DockerImage
+	result.LogicalBackupSchedule = util.Coalesce(fromCRD.LogicalBackup.Schedule, "30 00 * * *")
+	result.LogicalBackupDockerImage = util.Coalesce(fromCRD.LogicalBackup.DockerImage, "registry.opensource.zalan.do/acid/logical-backup")
 	result.LogicalBackupS3Bucket = fromCRD.LogicalBackup.S3Bucket
 	result.LogicalBackupS3Region = fromCRD.LogicalBackup.S3Region
 	result.LogicalBackupS3Endpoint = fromCRD.LogicalBackup.S3Endpoint
@@ -142,5 +154,56 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	result.ScalyrCPULimit = fromCRD.Scalyr.ScalyrCPULimit
 	result.ScalyrMemoryLimit = fromCRD.Scalyr.ScalyrMemoryLimit
 
+	// Connection pooler. Looks like we can't use defaulting in CRD before 1.17,
+	// so ensure default values here.
+	result.ConnectionPooler.NumberOfInstances = util.CoalesceInt32(
+		fromCRD.ConnectionPooler.NumberOfInstances,
+		int32ToPointer(2))
+
+	result.ConnectionPooler.NumberOfInstances = util.MaxInt32(
+		result.ConnectionPooler.NumberOfInstances,
+		int32ToPointer(2))
+
+	result.ConnectionPooler.Schema = util.Coalesce(
+		fromCRD.ConnectionPooler.Schema,
+		constants.ConnectionPoolerSchemaName)
+
+	result.ConnectionPooler.User = util.Coalesce(
+		fromCRD.ConnectionPooler.User,
+		constants.ConnectionPoolerUserName)
+
+	if result.ConnectionPooler.User == result.SuperUsername {
+		msg := "Connection pool user is not allowed to be the same as super user, username: %s"
+		panic(fmt.Errorf(msg, result.ConnectionPooler.User))
+	}
+
+	result.ConnectionPooler.Image = util.Coalesce(
+		fromCRD.ConnectionPooler.Image,
+		"registry.opensource.zalan.do/acid/pgbouncer")
+
+	result.ConnectionPooler.Mode = util.Coalesce(
+		fromCRD.ConnectionPooler.Mode,
+		constants.ConnectionPoolerDefaultMode)
+
+	result.ConnectionPooler.ConnectionPoolerDefaultCPURequest = util.Coalesce(
+		fromCRD.ConnectionPooler.DefaultCPURequest,
+		constants.ConnectionPoolerDefaultCpuRequest)
+
+	result.ConnectionPooler.ConnectionPoolerDefaultMemoryRequest = util.Coalesce(
+		fromCRD.ConnectionPooler.DefaultMemoryRequest,
+		constants.ConnectionPoolerDefaultMemoryRequest)
+
+	result.ConnectionPooler.ConnectionPoolerDefaultCPULimit = util.Coalesce(
+		fromCRD.ConnectionPooler.DefaultCPULimit,
+		constants.ConnectionPoolerDefaultCpuLimit)
+
+	result.ConnectionPooler.ConnectionPoolerDefaultMemoryLimit = util.Coalesce(
+		fromCRD.ConnectionPooler.DefaultMemoryLimit,
+		constants.ConnectionPoolerDefaultMemoryLimit)
+
+	result.ConnectionPooler.MaxDBConnections = util.CoalesceInt32(
+		fromCRD.ConnectionPooler.MaxDBConnections,
+		int32ToPointer(constants.ConnectionPoolerMaxDBConnections))
+
 	return result
 }

pkg/controller/pod.go

@@ -1,7 +1,9 @@
 package controller
 
 import (
-	"k8s.io/api/core/v1"
+	"context"
+
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/watch"
@@ -19,7 +21,7 @@ func (c *Controller) podListFunc(options metav1.ListOptions) (runtime.Object, er
 		TimeoutSeconds:  options.TimeoutSeconds,
 	}
 
-	return c.KubeClient.Pods(c.opConfig.WatchedNamespace).List(opts)
+	return c.KubeClient.Pods(c.opConfig.WatchedNamespace).List(context.TODO(), opts)
 }
 
 func (c *Controller) podWatchFunc(options metav1.ListOptions) (watch.Interface, error) {
@@ -29,7 +31,7 @@ func (c *Controller) podWatchFunc(options metav1.ListOptions) (watch.Interface,
 		TimeoutSeconds:  options.TimeoutSeconds,
 	}
 
-	return c.KubeClient.Pods(c.opConfig.WatchedNamespace).Watch(opts)
+	return c.KubeClient.Pods(c.opConfig.WatchedNamespace).Watch(context.TODO(), opts)
 }
 
 func (c *Controller) dispatchPodEvent(clusterName spec.NamespacedName, event cluster.PodEvent) {

pkg/controller/postgresql.go

@@ -1,6 +1,7 @@
 package controller
 
 import (
+	"context"
 	"fmt"
 	"reflect"
 	"strings"
@@ -10,6 +11,7 @@ import (
 
 	"github.com/sirupsen/logrus"
 
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/cache"
@@ -40,12 +42,23 @@ func (c *Controller) clusterResync(stopCh <-chan struct{}, wg *sync.WaitGroup) {
 
 // clusterListFunc obtains a list of all PostgreSQL clusters
 func (c *Controller) listClusters(options metav1.ListOptions) (*acidv1.PostgresqlList, error) {
+	var pgList acidv1.PostgresqlList
+
 	// TODO: use the SharedInformer cache instead of quering Kubernetes API directly.
-	list, err := c.KubeClient.AcidV1ClientSet.AcidV1().Postgresqls(c.opConfig.WatchedNamespace).List(options)
+	list, err := c.KubeClient.AcidV1ClientSet.AcidV1().Postgresqls(c.opConfig.WatchedNamespace).List(context.TODO(), options)
 	if err != nil {
 		c.logger.Errorf("could not list postgresql objects: %v", err)
 	}
-	return list, err
+	if c.controllerID != "" {
+		c.logger.Debugf("watch only clusters with controllerID %q", c.controllerID)
+	}
+	for _, pg := range list.Items {
+		if pg.Error == "" && c.hasOwnership(&pg) {
+			pgList.Items = append(pgList.Items, pg)
+		}
+	}
+
+	return &pgList, err
 }
 
 // clusterListAndSync lists all manifests and decides whether to run the sync or repair.
@@ -145,7 +158,7 @@ func (c *Controller) acquireInitialListOfClusters() error {
 }
 
 func (c *Controller) addCluster(lg *logrus.Entry, clusterName spec.NamespacedName, pgSpec *acidv1.Postgresql) *cluster.Cluster {
-	cl := cluster.New(c.makeClusterConfig(), c.KubeClient, *pgSpec, lg)
+	cl := cluster.New(c.makeClusterConfig(), c.KubeClient, *pgSpec, lg, c.eventRecorder)
 	cl.Run(c.stopCh)
 	teamName := strings.ToLower(cl.Spec.TeamID)
 
@@ -224,6 +237,7 @@ func (c *Controller) processEvent(event ClusterEvent) {
 		if err := cl.Create(); err != nil {
 			cl.Error = fmt.Sprintf("could not create cluster: %v", err)
 			lg.Error(cl.Error)
+			c.eventRecorder.Eventf(cl.GetReference(), v1.EventTypeWarning, "Create", "%v", cl.Error)
 
 			return
 		}
@@ -262,6 +276,8 @@ func (c *Controller) processEvent(event ClusterEvent) {
 
 		c.curWorkerCluster.Store(event.WorkerID, cl)
 		cl.Delete()
+		// Fixme - no error handling for delete ?
+		// c.eventRecorder.Eventf(cl.GetReference, v1.EventTypeWarning, "Delete", "%v", cl.Error)
 
 		func() {
 			defer c.clustersMu.Unlock()
@@ -292,6 +308,7 @@ func (c *Controller) processEvent(event ClusterEvent) {
 		c.curWorkerCluster.Store(event.WorkerID, cl)
 		if err := cl.Sync(event.NewSpec); err != nil {
 			cl.Error = fmt.Sprintf("could not sync cluster: %v", err)
+			c.eventRecorder.Eventf(cl.GetReference(), v1.EventTypeWarning, "Sync", "%v", cl.Error)
 			lg.Error(cl.Error)
 			return
 		}
@@ -455,41 +472,50 @@ func (c *Controller) queueClusterEvent(informerOldSpec, informerNewSpec *acidv1.
 }
 
 func (c *Controller) postgresqlAdd(obj interface{}) {
-	pg, ok := obj.(*acidv1.Postgresql)
-	if !ok {
-		c.logger.Errorf("could not cast to postgresql spec")
-		return
+	pg := c.postgresqlCheck(obj)
+	if pg != nil {
+		// We will not get multiple Add events for the same cluster
+		c.queueClusterEvent(nil, pg, EventAdd)
 	}
 
-	// We will not get multiple Add events for the same cluster
-	c.queueClusterEvent(nil, pg, EventAdd)
+	return
 }
 
 func (c *Controller) postgresqlUpdate(prev, cur interface{}) {
-	pgOld, ok := prev.(*acidv1.Postgresql)
-	if !ok {
-		c.logger.Errorf("could not cast to postgresql spec")
-	}
-	pgNew, ok := cur.(*acidv1.Postgresql)
-	if !ok {
-		c.logger.Errorf("could not cast to postgresql spec")
-	}
-	// Avoid the inifinite recursion for status updates
-	if reflect.DeepEqual(pgOld.Spec, pgNew.Spec) {
-		return
+	pgOld := c.postgresqlCheck(prev)
+	pgNew := c.postgresqlCheck(cur)
+	if pgOld != nil && pgNew != nil {
+		// Avoid the inifinite recursion for status updates
+		if reflect.DeepEqual(pgOld.Spec, pgNew.Spec) {
+			if reflect.DeepEqual(pgNew.Annotations, pgOld.Annotations) {
+				return
+			}
+		}
+		c.queueClusterEvent(pgOld, pgNew, EventUpdate)
 	}
 
-	c.queueClusterEvent(pgOld, pgNew, EventUpdate)
+	return
 }
 
 func (c *Controller) postgresqlDelete(obj interface{}) {
+	pg := c.postgresqlCheck(obj)
+	if pg != nil {
+		c.queueClusterEvent(pg, nil, EventDelete)
+	}
+
+	return
+}
+
+func (c *Controller) postgresqlCheck(obj interface{}) *acidv1.Postgresql {
 	pg, ok := obj.(*acidv1.Postgresql)
 	if !ok {
 		c.logger.Errorf("could not cast to postgresql spec")
-		return
+		return nil
 	}
-
-	c.queueClusterEvent(pg, nil, EventDelete)
+	if !c.hasOwnership(pg) {
+		return nil
+	}
+	return pg
 }
 
 /*
@@ -517,7 +543,7 @@ func (c *Controller) submitRBACCredentials(event ClusterEvent) error {
 func (c *Controller) createPodServiceAccount(namespace string) error {
 
 	podServiceAccountName := c.opConfig.PodServiceAccountName
-	_, err := c.KubeClient.ServiceAccounts(namespace).Get(podServiceAccountName, metav1.GetOptions{})
+	_, err := c.KubeClient.ServiceAccounts(namespace).Get(context.TODO(), podServiceAccountName, metav1.GetOptions{})
 	if k8sutil.ResourceNotFound(err) {
 
 		c.logger.Infof(fmt.Sprintf("creating pod service account %q in the %q namespace", podServiceAccountName, namespace))
@@ -525,7 +551,7 @@ func (c *Controller) createPodServiceAccount(namespace string) error {
 		// get a separate copy of service account
 		// to prevent a race condition when setting a namespace for many clusters
 		sa := *c.PodServiceAccount
-		if _, err = c.KubeClient.ServiceAccounts(namespace).Create(&sa); err != nil {
+		if _, err = c.KubeClient.ServiceAccounts(namespace).Create(context.TODO(), &sa, metav1.CreateOptions{}); err != nil {
 			return fmt.Errorf("cannot deploy the pod service account %q defined in the configuration to the %q namespace: %v", podServiceAccountName, namespace, err)
 		}
 
@@ -542,7 +568,7 @@ func (c *Controller) createRoleBindings(namespace string) error {
 	podServiceAccountName := c.opConfig.PodServiceAccountName
 	podServiceAccountRoleBindingName := c.PodServiceAccountRoleBinding.Name
 
-	_, err := c.KubeClient.RoleBindings(namespace).Get(podServiceAccountRoleBindingName, metav1.GetOptions{})
+	_, err := c.KubeClient.RoleBindings(namespace).Get(context.TODO(), podServiceAccountRoleBindingName, metav1.GetOptions{})
 	if k8sutil.ResourceNotFound(err) {
 
 		c.logger.Infof("Creating the role binding %q in the %q namespace", podServiceAccountRoleBindingName, namespace)
@@ -550,7 +576,7 @@ func (c *Controller) createRoleBindings(namespace string) error {
 		// get a separate copy of role binding
 		// to prevent a race condition when setting a namespace for many clusters
 		rb := *c.PodServiceAccountRoleBinding
-		_, err = c.KubeClient.RoleBindings(namespace).Create(&rb)
+		_, err = c.KubeClient.RoleBindings(namespace).Create(context.TODO(), &rb, metav1.CreateOptions{})
 		if err != nil {
 			return fmt.Errorf("cannot bind the pod service account %q defined in the configuration to the cluster role in the %q namespace: %v", podServiceAccountName, namespace, err)
 		}

pkg/controller/postgresql_test.go

@@ -1,10 +1,12 @@
 package controller
 
 import (
-	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
-	"github.com/zalando/postgres-operator/pkg/spec"
 	"reflect"
 	"testing"
+
+	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
+	"github.com/zalando/postgres-operator/pkg/spec"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 var (
@@ -12,9 +14,55 @@ var (
 	False = false
 )
 
-func TestMergeDeprecatedPostgreSQLSpecParameters(t *testing.T) {
-	c := NewController(&spec.ControllerConfig{})
+func newPostgresqlTestController() *Controller {
+	controller := NewController(&spec.ControllerConfig{}, "postgresql-test")
+	return controller
+}
 
+var postgresqlTestController = newPostgresqlTestController()
+
+func TestControllerOwnershipOnPostgresql(t *testing.T) {
+	tests := []struct {
+		name  string
+		pg    *acidv1.Postgresql
+		owned bool
+		error string
+	}{
+		{
+			"Postgres cluster with defined ownership of mocked controller",
+			&acidv1.Postgresql{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{"acid.zalan.do/controller": "postgresql-test"},
+				},
+			},
+			True,
+			"Postgres cluster should be owned by operator, but controller says no",
+		},
+		{
+			"Postgres cluster with defined ownership of another controller",
+			&acidv1.Postgresql{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{"acid.zalan.do/controller": "stups-test"},
+				},
+			},
+			False,
+			"Postgres cluster should be owned by another operator, but controller say yes",
+		},
+		{
+			"Test Postgres cluster without defined ownership",
+			&acidv1.Postgresql{},
+			False,
+			"Postgres cluster should be owned by operator with empty controller ID, but controller says yes",
+		},
+	}
+	for _, tt := range tests {
+		if postgresqlTestController.hasOwnership(tt.pg) != tt.owned {
+			t.Errorf("%s: %v", tt.name, tt.error)
+		}
+	}
+}
+
+func TestMergeDeprecatedPostgreSQLSpecParameters(t *testing.T) {
 	tests := []struct {
 		name  string
 		in    *acidv1.PostgresSpec
@@ -36,7 +84,7 @@ func TestMergeDeprecatedPostgreSQLSpecParameters(t *testing.T) {
 		},
 	}
 	for _, tt := range tests {
-		result := c.mergeDeprecatedPostgreSQLSpecParameters(tt.in)
+		result := postgresqlTestController.mergeDeprecatedPostgreSQLSpecParameters(tt.in)
 		if !reflect.DeepEqual(result, tt.out) {
 			t.Errorf("%s: %v", tt.name, tt.error)
 		}

pkg/controller/types.go

@@ -1,9 +1,10 @@
 package controller
 
 import (
-	"k8s.io/apimachinery/pkg/types"
 	"time"
 
+	"k8s.io/apimachinery/pkg/types"
+
 	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
 )
 

pkg/controller/util.go

@@ -1,6 +1,7 @@
 package controller
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 
@@ -50,7 +51,7 @@ func (c *Controller) clusterWorkerID(clusterName spec.NamespacedName) uint32 {
 }
 
 func (c *Controller) createOperatorCRD(crd *apiextv1beta1.CustomResourceDefinition) error {
-	if _, err := c.KubeClient.CustomResourceDefinitions().Create(crd); err != nil {
+	if _, err := c.KubeClient.CustomResourceDefinitions().Create(context.TODO(), crd, metav1.CreateOptions{}); err != nil {
 		if k8sutil.ResourceAlreadyExists(err) {
 			c.logger.Infof("customResourceDefinition %q is already registered and will only be updated", crd.Name)
 
@@ -58,7 +59,8 @@ func (c *Controller) createOperatorCRD(crd *apiextv1beta1.CustomResourceDefiniti
 			if err != nil {
 				return fmt.Errorf("could not marshal new customResourceDefintion: %v", err)
 			}
-			if _, err := c.KubeClient.CustomResourceDefinitions().Patch(crd.Name, types.MergePatchType, patch); err != nil {
+			if _, err := c.KubeClient.CustomResourceDefinitions().Patch(
+				context.TODO(), crd.Name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil {
 				return fmt.Errorf("could not update customResourceDefinition: %v", err)
 			}
 		} else {
@@ -69,7 +71,7 @@ func (c *Controller) createOperatorCRD(crd *apiextv1beta1.CustomResourceDefiniti
 	}
 
 	return wait.Poll(c.config.CRDReadyWaitInterval, c.config.CRDReadyWaitTimeout, func() (bool, error) {
-		c, err := c.KubeClient.CustomResourceDefinitions().Get(crd.Name, metav1.GetOptions{})
+		c, err := c.KubeClient.CustomResourceDefinitions().Get(context.TODO(), crd.Name, metav1.GetOptions{})
 		if err != nil {
 			return false, err
 		}
@@ -115,7 +117,7 @@ func (c *Controller) getInfrastructureRoles(rolesSecret *spec.NamespacedName) (m
 
 	infraRolesSecret, err := c.KubeClient.
 		Secrets(rolesSecret.Namespace).
-		Get(rolesSecret.Name, metav1.GetOptions{})
+		Get(context.TODO(), rolesSecret.Name, metav1.GetOptions{})
 	if err != nil {
 		c.logger.Debugf("infrastructure roles secret name: %q", *rolesSecret)
 		return nil, fmt.Errorf("could not get infrastructure roles secret: %v", err)
@@ -161,7 +163,8 @@ Users:
 	}
 
 	// perhaps we have some map entries with usernames, passwords, let's check if we have those users in the configmap
-	if infraRolesMap, err := c.KubeClient.ConfigMaps(rolesSecret.Namespace).Get(rolesSecret.Name, metav1.GetOptions{}); err == nil {
+	if infraRolesMap, err := c.KubeClient.ConfigMaps(rolesSecret.Namespace).Get(
+		context.TODO(), rolesSecret.Name, metav1.GetOptions{}); err == nil {
 		// we have a configmap with username - json description, let's read and decode it
 		for role, s := range infraRolesMap.Data {
 			roleDescr, err := readDecodedRole(s)

pkg/controller/util_test.go

@@ -17,8 +17,8 @@ const (
 	testInfrastructureRolesSecretName = "infrastructureroles-test"
 )
 
-func newMockController() *Controller {
-	controller := NewController(&spec.ControllerConfig{})
+func newUtilTestController() *Controller {
+	controller := NewController(&spec.ControllerConfig{}, "util-test")
 	controller.opConfig.ClusterNameLabel = "cluster-name"
 	controller.opConfig.InfrastructureRolesSecretName =
 		spec.NamespacedName{Namespace: v1.NamespaceDefault, Name: testInfrastructureRolesSecretName}
@@ -27,7 +27,7 @@ func newMockController() *Controller {
 	return controller
 }
 
-var mockController = newMockController()
+var utilTestController = newUtilTestController()
 
 func TestPodClusterName(t *testing.T) {
 	var testTable = []struct {
@@ -43,7 +43,7 @@ func TestPodClusterName(t *testing.T) {
 				ObjectMeta: metav1.ObjectMeta{
 					Namespace: v1.NamespaceDefault,
 					Labels: map[string]string{
-						mockController.opConfig.ClusterNameLabel: "testcluster",
+						utilTestController.opConfig.ClusterNameLabel: "testcluster",
 					},
 				},
 			},
@@ -51,7 +51,7 @@ func TestPodClusterName(t *testing.T) {
 		},
 	}
 	for _, test := range testTable {
-		resp := mockController.podClusterName(test.in)
+		resp := utilTestController.podClusterName(test.in)
 		if resp != test.expected {
 			t.Errorf("expected response %v does not match the actual %v", test.expected, resp)
 		}
@@ -73,7 +73,7 @@ func TestClusterWorkerID(t *testing.T) {
 		},
 	}
 	for _, test := range testTable {
-		resp := mockController.clusterWorkerID(test.in)
+		resp := utilTestController.clusterWorkerID(test.in)
 		if resp != test.expected {
 			t.Errorf("expected response %v does not match the actual %v", test.expected, resp)
 		}
@@ -116,7 +116,7 @@ func TestGetInfrastructureRoles(t *testing.T) {
 		},
 	}
 	for _, test := range testTable {
-		roles, err := mockController.getInfrastructureRoles(&test.secretName)
+		roles, err := utilTestController.getInfrastructureRoles(&test.secretName)
 		if err != test.expectedError {
 			if err != nil && test.expectedError != nil && err.Error() == test.expectedError.Error() {
 				continue

pkg/generated/clientset/versioned/clientset.go

@@ -65,7 +65,7 @@ func NewForConfig(c *rest.Config) (*Clientset, error) {
 	configShallowCopy := *c
 	if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 {
 		if configShallowCopy.Burst <= 0 {
-			return nil, fmt.Errorf("Burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0")
+			return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0")
 		}
 		configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst)
 	}

pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go

@@ -25,6 +25,8 @@ SOFTWARE.
 package fake
 
 import (
+	"context"
+
 	acidzalandov1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	schema "k8s.io/apimachinery/pkg/runtime/schema"
@@ -42,7 +44,7 @@ var operatorconfigurationsResource = schema.GroupVersionResource{Group: "acid.za
 var operatorconfigurationsKind = schema.GroupVersionKind{Group: "acid.zalan.do", Version: "v1", Kind: "OperatorConfiguration"}
 
 // Get takes name of the operatorConfiguration, and returns the corresponding operatorConfiguration object, and an error if there is any.
-func (c *FakeOperatorConfigurations) Get(name string, options v1.GetOptions) (result *acidzalandov1.OperatorConfiguration, err error) {
+func (c *FakeOperatorConfigurations) Get(ctx context.Context, name string, options v1.GetOptions) (result *acidzalandov1.OperatorConfiguration, err error) {
 	obj, err := c.Fake.
 		Invokes(testing.NewGetAction(operatorconfigurationsResource, c.ns, name), &acidzalandov1.OperatorConfiguration{})
 

pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go

@@ -25,6 +25,8 @@ SOFTWARE.
 package fake
 
 import (
+	"context"
+
 	acidzalandov1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	labels "k8s.io/apimachinery/pkg/labels"
@@ -45,7 +47,7 @@ var postgresqlsResource = schema.GroupVersionResource{Group: "acid.zalan.do", Ve
 var postgresqlsKind = schema.GroupVersionKind{Group: "acid.zalan.do", Version: "v1", Kind: "Postgresql"}
 
 // Get takes name of the postgresql, and returns the corresponding postgresql object, and an error if there is any.
-func (c *FakePostgresqls) Get(name string, options v1.GetOptions) (result *acidzalandov1.Postgresql, err error) {
+func (c *FakePostgresqls) Get(ctx context.Context, name string, options v1.GetOptions) (result *acidzalandov1.Postgresql, err error) {
 	obj, err := c.Fake.
 		Invokes(testing.NewGetAction(postgresqlsResource, c.ns, name), &acidzalandov1.Postgresql{})
 
@@ -56,7 +58,7 @@ func (c *FakePostgresqls) Get(name string, options v1.GetOptions) (result *acidz
 }
 
 // List takes label and field selectors, and returns the list of Postgresqls that match those selectors.
-func (c *FakePostgresqls) List(opts v1.ListOptions) (result *acidzalandov1.PostgresqlList, err error) {
+func (c *FakePostgresqls) List(ctx context.Context, opts v1.ListOptions) (result *acidzalandov1.PostgresqlList, err error) {
 	obj, err := c.Fake.
 		Invokes(testing.NewListAction(postgresqlsResource, postgresqlsKind, c.ns, opts), &acidzalandov1.PostgresqlList{})
 
@@ -78,14 +80,14 @@ func (c *FakePostgresqls) List(opts v1.ListOptions) (result *acidzalandov1.Postg
 }
 
 // Watch returns a watch.Interface that watches the requested postgresqls.
-func (c *FakePostgresqls) Watch(opts v1.ListOptions) (watch.Interface, error) {
+func (c *FakePostgresqls) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) {
 	return c.Fake.
 		InvokesWatch(testing.NewWatchAction(postgresqlsResource, c.ns, opts))
 
 }
 
 // Create takes the representation of a postgresql and creates it.  Returns the server's representation of the postgresql, and an error, if there is any.
-func (c *FakePostgresqls) Create(postgresql *acidzalandov1.Postgresql) (result *acidzalandov1.Postgresql, err error) {
+func (c *FakePostgresqls) Create(ctx context.Context, postgresql *acidzalandov1.Postgresql, opts v1.CreateOptions) (result *acidzalandov1.Postgresql, err error) {
 	obj, err := c.Fake.
 		Invokes(testing.NewCreateAction(postgresqlsResource, c.ns, postgresql), &acidzalandov1.Postgresql{})
 
@@ -96,7 +98,7 @@ func (c *FakePostgresqls) Create(postgresql *acidzalandov1.Postgresql) (result *
 }
 
 // Update takes the representation of a postgresql and updates it. Returns the server's representation of the postgresql, and an error, if there is any.
-func (c *FakePostgresqls) Update(postgresql *acidzalandov1.Postgresql) (result *acidzalandov1.Postgresql, err error) {
+func (c *FakePostgresqls) Update(ctx context.Context, postgresql *acidzalandov1.Postgresql, opts v1.UpdateOptions) (result *acidzalandov1.Postgresql, err error) {
 	obj, err := c.Fake.
 		Invokes(testing.NewUpdateAction(postgresqlsResource, c.ns, postgresql), &acidzalandov1.Postgresql{})
 
@@ -108,7 +110,7 @@ func (c *FakePostgresqls) Update(postgresql *acidzalandov1.Postgresql) (result *
 
 // UpdateStatus was generated because the type contains a Status member.
 // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus().
-func (c *FakePostgresqls) UpdateStatus(postgresql *acidzalandov1.Postgresql) (*acidzalandov1.Postgresql, error) {
+func (c *FakePostgresqls) UpdateStatus(ctx context.Context, postgresql *acidzalandov1.Postgresql, opts v1.UpdateOptions) (*acidzalandov1.Postgresql, error) {
 	obj, err := c.Fake.
 		Invokes(testing.NewUpdateSubresourceAction(postgresqlsResource, "status", c.ns, postgresql), &acidzalandov1.Postgresql{})
 
@@ -119,7 +121,7 @@ func (c *FakePostgresqls) UpdateStatus(postgresql *acidzalandov1.Postgresql) (*a
 }
 
 // Delete takes name of the postgresql and deletes it. Returns an error if one occurs.
-func (c *FakePostgresqls) Delete(name string, options *v1.DeleteOptions) error {
+func (c *FakePostgresqls) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error {
 	_, err := c.Fake.
 		Invokes(testing.NewDeleteAction(postgresqlsResource, c.ns, name), &acidzalandov1.Postgresql{})
 
@@ -127,15 +129,15 @@ func (c *FakePostgresqls) Delete(name string, options *v1.DeleteOptions) error {
 }
 
 // DeleteCollection deletes a collection of objects.
-func (c *FakePostgresqls) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error {
-	action := testing.NewDeleteCollectionAction(postgresqlsResource, c.ns, listOptions)
+func (c *FakePostgresqls) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error {
+	action := testing.NewDeleteCollectionAction(postgresqlsResource, c.ns, listOpts)
 
 	_, err := c.Fake.Invokes(action, &acidzalandov1.PostgresqlList{})
 	return err
 }
 
 // Patch applies the patch and returns the patched postgresql.
-func (c *FakePostgresqls) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *acidzalandov1.Postgresql, err error) {
+func (c *FakePostgresqls) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *acidzalandov1.Postgresql, err error) {
 	obj, err := c.Fake.
 		Invokes(testing.NewPatchSubresourceAction(postgresqlsResource, c.ns, name, pt, data, subresources...), &acidzalandov1.Postgresql{})
 

pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go

@@ -25,6 +25,8 @@ SOFTWARE.
 package v1
 
 import (
+	"context"
+
 	acidzalandov1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
 	scheme "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/scheme"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -39,7 +41,7 @@ type OperatorConfigurationsGetter interface {
 
 // OperatorConfigurationInterface has methods to work with OperatorConfiguration resources.
 type OperatorConfigurationInterface interface {
-	Get(name string, options v1.GetOptions) (*acidzalandov1.OperatorConfiguration, error)
+	Get(ctx context.Context, name string, opts v1.GetOptions) (*acidzalandov1.OperatorConfiguration, error)
 	OperatorConfigurationExpansion
 }
 
@@ -58,14 +60,14 @@ func newOperatorConfigurations(c *AcidV1Client, namespace string) *operatorConfi
 }
 
 // Get takes name of the operatorConfiguration, and returns the corresponding operatorConfiguration object, and an error if there is any.
-func (c *operatorConfigurations) Get(name string, options v1.GetOptions) (result *acidzalandov1.OperatorConfiguration, err error) {
+func (c *operatorConfigurations) Get(ctx context.Context, name string, options v1.GetOptions) (result *acidzalandov1.OperatorConfiguration, err error) {
 	result = &acidzalandov1.OperatorConfiguration{}
 	err = c.client.Get().
 		Namespace(c.ns).
 		Resource("operatorconfigurations").
 		Name(name).
 		VersionedParams(&options, scheme.ParameterCodec).
-		Do().
+		Do(ctx).
 		Into(result)
 	return
 }

pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go

@@ -25,6 +25,7 @@ SOFTWARE.
 package v1
 
 import (
+	"context"
 	"time"
 
 	v1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
@@ -43,15 +44,15 @@ type PostgresqlsGetter interface {
 
 // PostgresqlInterface has methods to work with Postgresql resources.
 type PostgresqlInterface interface {
-	Create(*v1.Postgresql) (*v1.Postgresql, error)
-	Update(*v1.Postgresql) (*v1.Postgresql, error)
-	UpdateStatus(*v1.Postgresql) (*v1.Postgresql, error)
-	Delete(name string, options *metav1.DeleteOptions) error
-	DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error
-	Get(name string, options metav1.GetOptions) (*v1.Postgresql, error)
-	List(opts metav1.ListOptions) (*v1.PostgresqlList, error)
-	Watch(opts metav1.ListOptions) (watch.Interface, error)
-	Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.Postgresql, err error)
+	Create(ctx context.Context, postgresql *v1.Postgresql, opts metav1.CreateOptions) (*v1.Postgresql, error)
+	Update(ctx context.Context, postgresql *v1.Postgresql, opts metav1.UpdateOptions) (*v1.Postgresql, error)
+	UpdateStatus(ctx context.Context, postgresql *v1.Postgresql, opts metav1.UpdateOptions) (*v1.Postgresql, error)
+	Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error
+	DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error
+	Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.Postgresql, error)
+	List(ctx context.Context, opts metav1.ListOptions) (*v1.PostgresqlList, error)
+	Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error)
+	Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.Postgresql, err error)
 	PostgresqlExpansion
 }
 
@@ -70,20 +71,20 @@ func newPostgresqls(c *AcidV1Client, namespace string) *postgresqls {
 }
 
 // Get takes name of the postgresql, and returns the corresponding postgresql object, and an error if there is any.
-func (c *postgresqls) Get(name string, options metav1.GetOptions) (result *v1.Postgresql, err error) {
+func (c *postgresqls) Get(ctx context.Context, name string, options metav1.GetOptions) (result *v1.Postgresql, err error) {
 	result = &v1.Postgresql{}
 	err = c.client.Get().
 		Namespace(c.ns).
 		Resource("postgresqls").
 		Name(name).
 		VersionedParams(&options, scheme.ParameterCodec).
-		Do().
+		Do(ctx).
 		Into(result)
 	return
 }
 
 // List takes label and field selectors, and returns the list of Postgresqls that match those selectors.
-func (c *postgresqls) List(opts metav1.ListOptions) (result *v1.PostgresqlList, err error) {
+func (c *postgresqls) List(ctx context.Context, opts metav1.ListOptions) (result *v1.PostgresqlList, err error) {
 	var timeout time.Duration
 	if opts.TimeoutSeconds != nil {
 		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
@@ -94,13 +95,13 @@ func (c *postgresqls) List(opts metav1.ListOptions) (result *v1.PostgresqlList,
 		Resource("postgresqls").
 		VersionedParams(&opts, scheme.ParameterCodec).
 		Timeout(timeout).
-		Do().
+		Do(ctx).
 		Into(result)
 	return
 }
 
 // Watch returns a watch.Interface that watches the requested postgresqls.
-func (c *postgresqls) Watch(opts metav1.ListOptions) (watch.Interface, error) {
+func (c *postgresqls) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
 	var timeout time.Duration
 	if opts.TimeoutSeconds != nil {
 		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
@@ -111,87 +112,90 @@ func (c *postgresqls) Watch(opts metav1.ListOptions) (watch.Interface, error) {
 		Resource("postgresqls").
 		VersionedParams(&opts, scheme.ParameterCodec).
 		Timeout(timeout).
-		Watch()
+		Watch(ctx)
 }
 
 // Create takes the representation of a postgresql and creates it.  Returns the server's representation of the postgresql, and an error, if there is any.
-func (c *postgresqls) Create(postgresql *v1.Postgresql) (result *v1.Postgresql, err error) {
+func (c *postgresqls) Create(ctx context.Context, postgresql *v1.Postgresql, opts metav1.CreateOptions) (result *v1.Postgresql, err error) {
 	result = &v1.Postgresql{}
 	err = c.client.Post().
 		Namespace(c.ns).
 		Resource("postgresqls").
+		VersionedParams(&opts, scheme.ParameterCodec).
 		Body(postgresql).
-		Do().
+		Do(ctx).
 		Into(result)
 	return
 }
 
 // Update takes the representation of a postgresql and updates it. Returns the server's representation of the postgresql, and an error, if there is any.
-func (c *postgresqls) Update(postgresql *v1.Postgresql) (result *v1.Postgresql, err error) {
+func (c *postgresqls) Update(ctx context.Context, postgresql *v1.Postgresql, opts metav1.UpdateOptions) (result *v1.Postgresql, err error) {
 	result = &v1.Postgresql{}
 	err = c.client.Put().
 		Namespace(c.ns).
 		Resource("postgresqls").
 		Name(postgresql.Name).
+		VersionedParams(&opts, scheme.ParameterCodec).
 		Body(postgresql).
-		Do().
+		Do(ctx).
 		Into(result)
 	return
 }
 
 // UpdateStatus was generated because the type contains a Status member.
 // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus().
-
-func (c *postgresqls) UpdateStatus(postgresql *v1.Postgresql) (result *v1.Postgresql, err error) {
+func (c *postgresqls) UpdateStatus(ctx context.Context, postgresql *v1.Postgresql, opts metav1.UpdateOptions) (result *v1.Postgresql, err error) {
 	result = &v1.Postgresql{}
 	err = c.client.Put().
 		Namespace(c.ns).
 		Resource("postgresqls").
 		Name(postgresql.Name).
 		SubResource("status").
+		VersionedParams(&opts, scheme.ParameterCodec).
 		Body(postgresql).
-		Do().
+		Do(ctx).
 		Into(result)
 	return
 }
 
 // Delete takes name of the postgresql and deletes it. Returns an error if one occurs.
-func (c *postgresqls) Delete(name string, options *metav1.DeleteOptions) error {
+func (c *postgresqls) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error {
 	return c.client.Delete().
 		Namespace(c.ns).
 		Resource("postgresqls").
 		Name(name).
-		Body(options).
-		Do().
+		Body(&opts).
+		Do(ctx).
 		Error()
 }
 
 // DeleteCollection deletes a collection of objects.
-func (c *postgresqls) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error {
+func (c *postgresqls) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error {
 	var timeout time.Duration
-	if listOptions.TimeoutSeconds != nil {
-		timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second
+	if listOpts.TimeoutSeconds != nil {
+		timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second
 	}
 	return c.client.Delete().
 		Namespace(c.ns).
 		Resource("postgresqls").
-		VersionedParams(&listOptions, scheme.ParameterCodec).
+		VersionedParams(&listOpts, scheme.ParameterCodec).
 		Timeout(timeout).
-		Body(options).
-		Do().
+		Body(&opts).
+		Do(ctx).
 		Error()
 }
 
 // Patch applies the patch and returns the patched postgresql.
-func (c *postgresqls) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.Postgresql, err error) {
+func (c *postgresqls) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.Postgresql, err error) {
 	result = &v1.Postgresql{}
 	err = c.client.Patch(pt).
 		Namespace(c.ns).
 		Resource("postgresqls").
-		SubResource(subresources...).
 		Name(name).
+		SubResource(subresources...).
+		VersionedParams(&opts, scheme.ParameterCodec).
 		Body(data).
-		Do().
+		Do(ctx).
 		Into(result)
 	return
 }

pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go

@@ -25,6 +25,7 @@ SOFTWARE.
 package v1
 
 import (
+	"context"
 	time "time"
 
 	acidzalandov1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
@@ -67,13 +68,13 @@ func NewFilteredPostgresqlInformer(client versioned.Interface, namespace string,
 				if tweakListOptions != nil {
 					tweakListOptions(&options)
 				}
-				return client.AcidV1().Postgresqls(namespace).List(options)
+				return client.AcidV1().Postgresqls(namespace).List(context.TODO(), options)
 			},
 			WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
 				if tweakListOptions != nil {
 					tweakListOptions(&options)
 				}
-				return client.AcidV1().Postgresqls(namespace).Watch(options)
+				return client.AcidV1().Postgresqls(namespace).Watch(context.TODO(), options)
 			},
 		},
 		&acidzalandov1.Postgresql{},

pkg/spec/types.go

@@ -23,13 +23,16 @@ const fileWithNamespace = "/var/run/secrets/kubernetes.io/serviceaccount/namespa
 // RoleOrigin contains the code of the origin of a role
 type RoleOrigin int
 
-// The rolesOrigin constant values must be sorted by the role priority for resolveNameConflict(...) to work.
+// The rolesOrigin constant values must be sorted by the role priority for
+// resolveNameConflict(...) to work.
 const (
 	RoleOriginUnknown RoleOrigin = iota
 	RoleOriginManifest
 	RoleOriginInfrastructure
 	RoleOriginTeamsAPI
 	RoleOriginSystem
+	RoleOriginBootstrap
+	RoleConnectionPooler
 )
 
 type syncUserOperation int
@@ -178,6 +181,10 @@ func (r RoleOrigin) String() string {
 		return "teams API role"
 	case RoleOriginSystem:
 		return "system role"
+	case RoleOriginBootstrap:
+		return "bootstrapped role"
+	case RoleConnectionPooler:
+		return "connection pooler role"
 	default:
 		panic(fmt.Sprintf("bogus role origin value %d", r))
 	}

pkg/util/config/config.go

@@ -8,6 +8,8 @@ import (
 	"fmt"
 
 	"github.com/zalando/postgres-operator/pkg/spec"
+	"github.com/zalando/postgres-operator/pkg/util/constants"
+	v1 "k8s.io/api/core/v1"
 )
 
 // CRD describes CustomResourceDefinition specific configuration parameters
@@ -21,31 +23,32 @@ type CRD struct {
 
 // Resources describes kubernetes resource specific configuration parameters
 type Resources struct {
-	ResourceCheckInterval   time.Duration     `name:"resource_check_interval" default:"3s"`
-	ResourceCheckTimeout    time.Duration     `name:"resource_check_timeout" default:"10m"`
-	PodLabelWaitTimeout     time.Duration     `name:"pod_label_wait_timeout" default:"10m"`
-	PodDeletionWaitTimeout  time.Duration     `name:"pod_deletion_wait_timeout" default:"10m"`
-	PodTerminateGracePeriod time.Duration     `name:"pod_terminate_grace_period" default:"5m"`
-	SpiloFSGroup            *int64            `name:"spilo_fsgroup"`
-	PodPriorityClassName    string            `name:"pod_priority_class_name"`
-	ClusterDomain           string            `name:"cluster_domain" default:"cluster.local"`
-	SpiloPrivileged         bool              `name:"spilo_privileged" default:"false"`
-	ClusterLabels           map[string]string `name:"cluster_labels" default:"application:spilo"`
-	InheritedLabels         []string          `name:"inherited_labels" default:""`
-	ClusterNameLabel        string            `name:"cluster_name_label" default:"cluster-name"`
-	PodRoleLabel            string            `name:"pod_role_label" default:"spilo-role"`
-	PodToleration           map[string]string `name:"toleration" default:""`
-	DefaultCPURequest       string            `name:"default_cpu_request" default:"100m"`
-	DefaultMemoryRequest    string            `name:"default_memory_request" default:"100Mi"`
-	DefaultCPULimit         string            `name:"default_cpu_limit" default:"1"`
-	DefaultMemoryLimit      string            `name:"default_memory_limit" default:"500Mi"`
-	MinCPULimit             string            `name:"min_cpu_limit" default:"250m"`
-	MinMemoryLimit          string            `name:"min_memory_limit" default:"250Mi"`
-	PodEnvironmentConfigMap string            `name:"pod_environment_configmap" default:""`
-	NodeReadinessLabel      map[string]string `name:"node_readiness_label" default:""`
-	MaxInstances            int32             `name:"max_instances" default:"-1"`
-	MinInstances            int32             `name:"min_instances" default:"-1"`
-	ShmVolume               *bool             `name:"enable_shm_volume" default:"true"`
+	ResourceCheckInterval   time.Duration       `name:"resource_check_interval" default:"3s"`
+	ResourceCheckTimeout    time.Duration       `name:"resource_check_timeout" default:"10m"`
+	PodLabelWaitTimeout     time.Duration       `name:"pod_label_wait_timeout" default:"10m"`
+	PodDeletionWaitTimeout  time.Duration       `name:"pod_deletion_wait_timeout" default:"10m"`
+	PodTerminateGracePeriod time.Duration       `name:"pod_terminate_grace_period" default:"5m"`
+	SpiloFSGroup            *int64              `name:"spilo_fsgroup"`
+	PodPriorityClassName    string              `name:"pod_priority_class_name"`
+	ClusterDomain           string              `name:"cluster_domain" default:"cluster.local"`
+	SpiloPrivileged         bool                `name:"spilo_privileged" default:"false"`
+	ClusterLabels           map[string]string   `name:"cluster_labels" default:"application:spilo"`
+	InheritedLabels         []string            `name:"inherited_labels" default:""`
+	DownscalerAnnotations   []string            `name:"downscaler_annotations"`
+	ClusterNameLabel        string              `name:"cluster_name_label" default:"cluster-name"`
+	PodRoleLabel            string              `name:"pod_role_label" default:"spilo-role"`
+	PodToleration           map[string]string   `name:"toleration" default:""`
+	DefaultCPURequest       string              `name:"default_cpu_request" default:"100m"`
+	DefaultMemoryRequest    string              `name:"default_memory_request" default:"100Mi"`
+	DefaultCPULimit         string              `name:"default_cpu_limit" default:"1"`
+	DefaultMemoryLimit      string              `name:"default_memory_limit" default:"500Mi"`
+	MinCPULimit             string              `name:"min_cpu_limit" default:"250m"`
+	MinMemoryLimit          string              `name:"min_memory_limit" default:"250Mi"`
+	PodEnvironmentConfigMap spec.NamespacedName `name:"pod_environment_configmap"`
+	NodeReadinessLabel      map[string]string   `name:"node_readiness_label" default:""`
+	MaxInstances            int32               `name:"max_instances" default:"-1"`
+	MinInstances            int32               `name:"min_instances" default:"-1"`
+	ShmVolume               *bool               `name:"enable_shm_volume" default:"true"`
 }
 
 // Auth describes authentication specific configuration parameters
@@ -80,7 +83,21 @@ type LogicalBackup struct {
 	LogicalBackupS3Endpoint        string `name:"logical_backup_s3_endpoint" default:""`
 	LogicalBackupS3AccessKeyID     string `name:"logical_backup_s3_access_key_id" default:""`
 	LogicalBackupS3SecretAccessKey string `name:"logical_backup_s3_secret_access_key" default:""`
-	LogicalBackupS3SSE             string `name:"logical_backup_s3_sse" default:"AES256"`
+	LogicalBackupS3SSE             string `name:"logical_backup_s3_sse" default:""`
+}
+
+// Operator options for connection pooler
+type ConnectionPooler struct {
+	NumberOfInstances                    *int32 `name:"connection_pooler_number_of_instances" default:"2"`
+	Schema                               string `name:"connection_pooler_schema" default:"pooler"`
+	User                                 string `name:"connection_pooler_user" default:"pooler"`
+	Image                                string `name:"connection_pooler_image" default:"registry.opensource.zalan.do/acid/pgbouncer"`
+	Mode                                 string `name:"connection_pooler_mode" default:"transaction"`
+	MaxDBConnections                     *int32 `name:"connection_pooler_max_db_connections" default:"60"`
+	ConnectionPoolerDefaultCPURequest    string `name:"connection_pooler_default_cpu_request" default:"500m"`
+	ConnectionPoolerDefaultMemoryRequest string `name:"connection_pooler_default_memory_request" default:"100Mi"`
+	ConnectionPoolerDefaultCPULimit      string `name:"connection_pooler_default_cpu_limit" default:"1"`
+	ConnectionPoolerDefaultMemoryLimit   string `name:"connection_pooler_default_memory_limit" default:"100Mi"`
 }
 
 // Config describes operator config
@@ -90,11 +107,15 @@ type Config struct {
 	Auth
 	Scalyr
 	LogicalBackup
+	ConnectionPooler
 
-	WatchedNamespace      string            `name:"watched_namespace"`    // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to'
-	EtcdHost              string            `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS
-	DockerImage           string            `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-12:1.6-p2"`
-	Sidecars              map[string]string `name:"sidecar_docker_images"`
+	WatchedNamespace        string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to'
+	KubernetesUseConfigMaps bool   `name:"kubernetes_use_configmaps" default:"false"`
+	EtcdHost                string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS
+	DockerImage             string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-12:1.6-p3"`
+	// deprecated in favour of SidecarContainers
+	SidecarImages         map[string]string `name:"sidecar_docker_images"`
+	SidecarContainers     []v1.Container    `name:"sidecars"`
 	PodServiceAccountName string            `name:"pod_service_account_name" default:"postgres-pod"`
 	// value of this string must be valid JSON or YAML; see initPodServiceAccount
 	PodServiceAccountDefinition            string            `name:"pod_service_account_definition" default:""`
@@ -137,6 +158,7 @@ type Config struct {
 	ProtectedRoles            []string          `name:"protected_role_names" default:"admin"`
 	PostgresSuperuserTeams    []string          `name:"postgres_superuser_teams" default:""`
 	SetMemoryRequestToLimit   bool              `name:"set_memory_request_to_limit" default:"false"`
+	EnableLazySpiloUpgrade    bool              `name:"enable_lazy_spilo_upgrade" default:"false"`
 }
 
 // MustMarshal marshals the config or panics
@@ -196,5 +218,16 @@ func validate(cfg *Config) (err error) {
 	if cfg.Workers == 0 {
 		err = fmt.Errorf("number of workers should be higher than 0")
 	}
+
+	if *cfg.ConnectionPooler.NumberOfInstances < constants.ConnectionPoolerMinInstances {
+		msg := "number of connection pooler instances should be higher than %d"
+		err = fmt.Errorf(msg, constants.ConnectionPoolerMinInstances)
+	}
+
+	if cfg.ConnectionPooler.User == cfg.SuperUsername {
+		msg := "Connection pool user is not allowed to be the same as super user, username: %s"
+		err = fmt.Errorf(msg, cfg.ConnectionPooler.User)
+	}
+
 	return
 }

pkg/util/constants/annotations.go

@@ -7,4 +7,5 @@ const (
 	ElbTimeoutAnnotationValue          = "3600"
 	KubeIAmAnnotation                  = "iam.amazonaws.com/role"
 	VolumeStorateProvisionerAnnotation = "pv.kubernetes.io/provisioned-by"
+	PostgresqlControllerAnnotationKey  = "acid.zalan.do/controller"
 )

pkg/util/constants/pooler.go

@@ -0,0 +1,18 @@
+package constants
+
+// Connection pooler specific constants
+const (
+	ConnectionPoolerUserName             = "pooler"
+	ConnectionPoolerSchemaName           = "pooler"
+	ConnectionPoolerDefaultType          = "pgbouncer"
+	ConnectionPoolerDefaultMode          = "transaction"
+	ConnectionPoolerDefaultCpuRequest    = "500m"
+	ConnectionPoolerDefaultCpuLimit      = "1"
+	ConnectionPoolerDefaultMemoryRequest = "100Mi"
+	ConnectionPoolerDefaultMemoryLimit   = "100Mi"
+
+	ConnectionPoolerContainer            = 0
+	ConnectionPoolerMaxDBConnections     = 60
+	ConnectionPoolerMaxClientConnections = 10000
+	ConnectionPoolerMinInstances         = 2
+)

pkg/util/constants/roles.go

@@ -2,15 +2,20 @@ package constants
 
 // Roles specific constants
 const (
-	PasswordLength         = 64
-	SuperuserKeyName       = "superuser"
-	ReplicationUserKeyName = "replication"
-	RoleFlagSuperuser      = "SUPERUSER"
-	RoleFlagInherit        = "INHERIT"
-	RoleFlagLogin          = "LOGIN"
-	RoleFlagNoLogin        = "NOLOGIN"
-	RoleFlagCreateRole     = "CREATEROLE"
-	RoleFlagCreateDB       = "CREATEDB"
-	RoleFlagReplication    = "REPLICATION"
-	RoleFlagByPassRLS      = "BYPASSRLS"
+	PasswordLength            = 64
+	SuperuserKeyName          = "superuser"
+	ConnectionPoolerUserKeyName = "pooler"
+	ReplicationUserKeyName    = "replication"
+	RoleFlagSuperuser         = "SUPERUSER"
+	RoleFlagInherit           = "INHERIT"
+	RoleFlagLogin             = "LOGIN"
+	RoleFlagNoLogin           = "NOLOGIN"
+	RoleFlagCreateRole        = "CREATEROLE"
+	RoleFlagCreateDB          = "CREATEDB"
+	RoleFlagReplication       = "REPLICATION"
+	RoleFlagByPassRLS         = "BYPASSRLS"
+	OwnerRoleNameSuffix       = "_owner"
+	ReaderRoleNameSuffix      = "_reader"
+	WriterRoleNameSuffix      = "_writer"
+	UserRoleNameSuffix        = "_user"
 )

pkg/util/k8sutil/k8sutil.go

@@ -1,6 +1,7 @@
 package k8sutil
 
 import (
+	"context"
 	"fmt"
 	"reflect"
 
@@ -9,11 +10,13 @@ import (
 	batchv1beta1 "k8s.io/api/batch/v1beta1"
 	clientbatchv1beta1 "k8s.io/client-go/kubernetes/typed/batch/v1beta1"
 
+	apiappsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	policybeta1 "k8s.io/api/policy/v1beta1"
 	apiextclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	apiextbeta1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
 	appsv1 "k8s.io/client-go/kubernetes/typed/apps/v1"
 	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
@@ -26,6 +29,10 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+func Int32ToPointer(value int32) *int32 {
+	return &value
+}
+
 // KubernetesClient describes getters for Kubernetes objects
 type KubernetesClient struct {
 	corev1.SecretsGetter
@@ -38,7 +45,9 @@ type KubernetesClient struct {
 	corev1.NodesGetter
 	corev1.NamespacesGetter
 	corev1.ServiceAccountsGetter
+	corev1.EventsGetter
 	appsv1.StatefulSetsGetter
+	appsv1.DeploymentsGetter
 	rbacv1.RoleBindingsGetter
 	policyv1beta1.PodDisruptionBudgetsGetter
 	apiextbeta1.CustomResourceDefinitionsGetter
@@ -55,6 +64,34 @@ type mockSecret struct {
 type MockSecretGetter struct {
 }
 
+type mockDeployment struct {
+	appsv1.DeploymentInterface
+}
+
+type mockDeploymentNotExist struct {
+	appsv1.DeploymentInterface
+}
+
+type MockDeploymentGetter struct {
+}
+
+type MockDeploymentNotExistGetter struct {
+}
+
+type mockService struct {
+	corev1.ServiceInterface
+}
+
+type mockServiceNotExist struct {
+	corev1.ServiceInterface
+}
+
+type MockServiceGetter struct {
+}
+
+type MockServiceNotExistGetter struct {
+}
+
 type mockConfigMap struct {
 	corev1.ConfigMapInterface
 }
@@ -101,10 +138,12 @@ func NewFromConfig(cfg *rest.Config) (KubernetesClient, error) {
 	kubeClient.NodesGetter = client.CoreV1()
 	kubeClient.NamespacesGetter = client.CoreV1()
 	kubeClient.StatefulSetsGetter = client.AppsV1()
+	kubeClient.DeploymentsGetter = client.AppsV1()
 	kubeClient.PodDisruptionBudgetsGetter = client.PolicyV1beta1()
 	kubeClient.RESTClient = client.CoreV1().RESTClient()
 	kubeClient.RoleBindingsGetter = client.RbacV1()
 	kubeClient.CronJobsGetter = client.BatchV1beta1()
+	kubeClient.EventsGetter = client.CoreV1()
 
 	apiextClient, err := apiextclient.NewForConfig(cfg)
 	if err != nil {
@@ -201,7 +240,7 @@ func SameLogicalBackupJob(cur, new *batchv1beta1.CronJob) (match bool, reason st
 	return true, ""
 }
 
-func (c *mockSecret) Get(name string, options metav1.GetOptions) (*v1.Secret, error) {
+func (c *mockSecret) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.Secret, error) {
 	if name != "infrastructureroles-test" {
 		return nil, fmt.Errorf("NotFound")
 	}
@@ -217,7 +256,7 @@ func (c *mockSecret) Get(name string, options metav1.GetOptions) (*v1.Secret, er
 
 }
 
-func (c *mockConfigMap) Get(name string, options metav1.GetOptions) (*v1.ConfigMap, error) {
+func (c *mockConfigMap) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.ConfigMap, error) {
 	if name != "infrastructureroles-test" {
 		return nil, fmt.Errorf("NotFound")
 	}
@@ -230,19 +269,145 @@ func (c *mockConfigMap) Get(name string, options metav1.GetOptions) (*v1.ConfigM
 }
 
 // Secrets to be mocked
-func (c *MockSecretGetter) Secrets(namespace string) corev1.SecretInterface {
+func (mock *MockSecretGetter) Secrets(namespace string) corev1.SecretInterface {
 	return &mockSecret{}
 }
 
 // ConfigMaps to be mocked
-func (c *MockConfigMapsGetter) ConfigMaps(namespace string) corev1.ConfigMapInterface {
+func (mock *MockConfigMapsGetter) ConfigMaps(namespace string) corev1.ConfigMapInterface {
 	return &mockConfigMap{}
 }
 
+func (mock *MockDeploymentGetter) Deployments(namespace string) appsv1.DeploymentInterface {
+	return &mockDeployment{}
+}
+
+func (mock *MockDeploymentNotExistGetter) Deployments(namespace string) appsv1.DeploymentInterface {
+	return &mockDeploymentNotExist{}
+}
+
+func (mock *mockDeployment) Create(context.Context, *apiappsv1.Deployment, metav1.CreateOptions) (*apiappsv1.Deployment, error) {
+	return &apiappsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test-deployment",
+		},
+		Spec: apiappsv1.DeploymentSpec{
+			Replicas: Int32ToPointer(1),
+		},
+	}, nil
+}
+
+func (mock *mockDeployment) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error {
+	return nil
+}
+
+func (mock *mockDeployment) Get(ctx context.Context, name string, opts metav1.GetOptions) (*apiappsv1.Deployment, error) {
+	return &apiappsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test-deployment",
+		},
+		Spec: apiappsv1.DeploymentSpec{
+			Replicas: Int32ToPointer(1),
+			Template: v1.PodTemplateSpec{
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{
+						v1.Container{
+							Image: "pooler:1.0",
+						},
+					},
+				},
+			},
+		},
+	}, nil
+}
+
+func (mock *mockDeployment) Patch(ctx context.Context, name string, t types.PatchType, data []byte, opts metav1.PatchOptions, subres ...string) (*apiappsv1.Deployment, error) {
+	return &apiappsv1.Deployment{
+		Spec: apiappsv1.DeploymentSpec{
+			Replicas: Int32ToPointer(2),
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test-deployment",
+		},
+	}, nil
+}
+
+func (mock *mockDeploymentNotExist) Get(ctx context.Context, name string, opts metav1.GetOptions) (*apiappsv1.Deployment, error) {
+	return nil, &apierrors.StatusError{
+		ErrStatus: metav1.Status{
+			Reason: metav1.StatusReasonNotFound,
+		},
+	}
+}
+
+func (mock *mockDeploymentNotExist) Create(context.Context, *apiappsv1.Deployment, metav1.CreateOptions) (*apiappsv1.Deployment, error) {
+	return &apiappsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test-deployment",
+		},
+		Spec: apiappsv1.DeploymentSpec{
+			Replicas: Int32ToPointer(1),
+		},
+	}, nil
+}
+
+func (mock *MockServiceGetter) Services(namespace string) corev1.ServiceInterface {
+	return &mockService{}
+}
+
+func (mock *MockServiceNotExistGetter) Services(namespace string) corev1.ServiceInterface {
+	return &mockServiceNotExist{}
+}
+
+func (mock *mockService) Create(context.Context, *v1.Service, metav1.CreateOptions) (*v1.Service, error) {
+	return &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test-service",
+		},
+	}, nil
+}
+
+func (mock *mockService) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error {
+	return nil
+}
+
+func (mock *mockService) Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.Service, error) {
+	return &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test-service",
+		},
+	}, nil
+}
+
+func (mock *mockServiceNotExist) Create(context.Context, *v1.Service, metav1.CreateOptions) (*v1.Service, error) {
+	return &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test-service",
+		},
+	}, nil
+}
+
+func (mock *mockServiceNotExist) Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.Service, error) {
+	return nil, &apierrors.StatusError{
+		ErrStatus: metav1.Status{
+			Reason: metav1.StatusReasonNotFound,
+		},
+	}
+}
+
 // NewMockKubernetesClient for other tests
 func NewMockKubernetesClient() KubernetesClient {
 	return KubernetesClient{
-		SecretsGetter:    &MockSecretGetter{},
-		ConfigMapsGetter: &MockConfigMapsGetter{},
+		SecretsGetter:     &MockSecretGetter{},
+		ConfigMapsGetter:  &MockConfigMapsGetter{},
+		DeploymentsGetter: &MockDeploymentGetter{},
+		ServicesGetter:    &MockServiceGetter{},
+	}
+}
+
+func ClientMissingObjects() KubernetesClient {
+	return KubernetesClient{
+		DeploymentsGetter: &MockDeploymentNotExistGetter{},
+		ServicesGetter:    &MockServiceNotExistGetter{},
 	}
 }

pkg/util/patroni/patroni.go

@@ -3,6 +3,7 @@ package patroni
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"net"
@@ -11,7 +12,7 @@ import (
 	"time"
 
 	"github.com/sirupsen/logrus"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 )
 
 const (
@@ -25,6 +26,7 @@ const (
 type Interface interface {
 	Switchover(master *v1.Pod, candidate string) error
 	SetPostgresParameters(server *v1.Pod, options map[string]string) error
+	GetPatroniMemberState(pod *v1.Pod) (string, error)
 }
 
 // Patroni API client
@@ -123,3 +125,36 @@ func (p *Patroni) SetPostgresParameters(server *v1.Pod, parameters map[string]st
 	}
 	return p.httpPostOrPatch(http.MethodPatch, apiURLString+configPath, buf)
 }
+
+//GetPatroniMemberState returns a state of member of a Patroni cluster
+func (p *Patroni) GetPatroniMemberState(server *v1.Pod) (string, error) {
+
+	apiURLString, err := apiURL(server)
+	if err != nil {
+		return "", err
+	}
+	response, err := p.httpClient.Get(apiURLString)
+	if err != nil {
+		return "", fmt.Errorf("could not perform Get request: %v", err)
+	}
+	defer response.Body.Close()
+
+	body, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		return "", fmt.Errorf("could not read response: %v", err)
+	}
+
+	data := make(map[string]interface{})
+	err = json.Unmarshal(body, &data)
+	if err != nil {
+		return "", err
+	}
+
+	state, ok := data["state"].(string)
+	if !ok {
+		return "", errors.New("Patroni Get call response contains wrong type for 'state' field")
+	}
+
+	return state, nil
+
+}

pkg/util/retryutil/retry_util.go

@@ -27,7 +27,7 @@ func (t *Ticker) Tick() { <-t.ticker.C }
 func Retry(interval time.Duration, timeout time.Duration, f func() (bool, error)) error {
 	//TODO: make the retry exponential
 	if timeout < interval {
-		return fmt.Errorf("timout(%s) should be greater than interval(%v)", timeout, interval)
+		return fmt.Errorf("timeout(%s) should be greater than interval(%v)", timeout, interval)
 	}
 	tick := &Ticker{time.NewTicker(interval)}
 	return RetryWorker(interval, timeout, tick, f)

pkg/util/teams/teams_test.go

@@ -133,11 +133,11 @@ var requestsURLtc = []struct {
 }{
 	{
 		"coffee://localhost/",
-		fmt.Errorf(`Get coffee://localhost/teams/acid: unsupported protocol scheme "coffee"`),
+		fmt.Errorf(`Get "coffee://localhost/teams/acid": unsupported protocol scheme "coffee"`),
 	},
 	{
 		"http://192.168.0.%31/",
-		fmt.Errorf(`parse http://192.168.0.%%31/teams/acid: invalid URL escape "%%31"`),
+		fmt.Errorf(`parse "http://192.168.0.%%31/teams/acid": invalid URL escape "%%31"`),
 	},
 }
 

pkg/util/users/users.go

@@ -73,26 +73,44 @@ func (strategy DefaultUserSyncStrategy) ProduceSyncRequests(dbUsers spec.PgUserM
 }
 
 // ExecuteSyncRequests makes actual database changes from the requests passed in its arguments.
-func (strategy DefaultUserSyncStrategy) ExecuteSyncRequests(reqs []spec.PgSyncUserRequest, db *sql.DB) error {
-	for _, r := range reqs {
-		switch r.Kind {
+func (strategy DefaultUserSyncStrategy) ExecuteSyncRequests(requests []spec.PgSyncUserRequest, db *sql.DB) error {
+	var reqretries []spec.PgSyncUserRequest
+	var errors []string
+	for _, request := range requests {
+		switch request.Kind {
 		case spec.PGSyncUserAdd:
-			if err := strategy.createPgUser(r.User, db); err != nil {
-				return fmt.Errorf("could not create user %q: %v", r.User.Name, err)
+			if err := strategy.createPgUser(request.User, db); err != nil {
+				reqretries = append(reqretries, request)
+				errors = append(errors, fmt.Sprintf("could not create user %q: %v", request.User.Name, err))
 			}
 		case spec.PGsyncUserAlter:
-			if err := strategy.alterPgUser(r.User, db); err != nil {
-				return fmt.Errorf("could not alter user %q: %v", r.User.Name, err)
+			if err := strategy.alterPgUser(request.User, db); err != nil {
+				reqretries = append(reqretries, request)
+				errors = append(errors, fmt.Sprintf("could not alter user %q: %v", request.User.Name, err))
 			}
 		case spec.PGSyncAlterSet:
-			if err := strategy.alterPgUserSet(r.User, db); err != nil {
-				return fmt.Errorf("could not set custom user %q parameters: %v", r.User.Name, err)
+			if err := strategy.alterPgUserSet(request.User, db); err != nil {
+				reqretries = append(reqretries, request)
+				errors = append(errors, fmt.Sprintf("could not set custom user %q parameters: %v", request.User.Name, err))
 			}
 		default:
-			return fmt.Errorf("unrecognized operation: %v", r.Kind)
+			return fmt.Errorf("unrecognized operation: %v", request.Kind)
 		}
 
 	}
+
+	// creating roles might fail if group role members are created before the parent role
+	// retry adding roles as long as the number of failed attempts is shrinking
+	if len(reqretries) > 0 {
+		if len(reqretries) < len(requests) {
+			if err := strategy.ExecuteSyncRequests(reqretries, db); err != nil {
+				return err
+			}
+		} else {
+			return fmt.Errorf("could not execute sync requests for users: %v", errors)
+		}
+	}
+
 	return nil
 }
 func (strategy DefaultUserSyncStrategy) alterPgUserSet(user spec.PgUser, db *sql.DB) (err error) {

pkg/util/util.go

@@ -2,8 +2,10 @@ package util
 
 import (
 	"crypto/md5" // #nosec we need it to for PostgreSQL md5 passwords
+	cryptoRand "crypto/rand"
 	"encoding/hex"
 	"fmt"
+	"math/big"
 	"math/rand"
 	"regexp"
 	"strings"
@@ -37,13 +39,17 @@ func False() *bool {
 	return &b
 }
 
-// RandomPassword generates random alphanumeric password of a given length.
+// RandomPassword generates a secure, random alphanumeric password of a given length.
 func RandomPassword(n int) string {
 	b := make([]byte, n)
 	for i := range b {
-		b[i] = passwordChars[rand.Intn(len(passwordChars))]
+		maxN := big.NewInt(int64(len(passwordChars)))
+		if n, err := cryptoRand.Int(cryptoRand.Reader, maxN); err != nil {
+			panic(fmt.Errorf("Unable to generate secure, random password: %v", err))
+		} else {
+			b[i] = passwordChars[n.Int64()]
+		}
 	}
-
 	return string(b)
 }
 
@@ -141,6 +147,48 @@ func Coalesce(val, defaultVal string) string {
 	return val
 }
 
+// CoalesceInt32 works like coalesce but for *int32
+func CoalesceInt32(val, defaultVal *int32) *int32 {
+	if val == nil {
+		return defaultVal
+	}
+	return val
+}
+
+// CoalesceBool works like coalesce but for *bool
+func CoalesceBool(val, defaultVal *bool) *bool {
+	if val == nil {
+		return defaultVal
+	}
+	return val
+}
+
+// Test if any of the values is nil
+func testNil(values ...*int32) bool {
+	for _, v := range values {
+		if v == nil {
+			return true
+		}
+	}
+
+	return false
+}
+
+// MaxInt32 : Return maximum of two integers provided via pointers. If one value
+// is not defined, return the other one. If both are not defined, result is also
+// undefined, caller needs to check for that.
+func MaxInt32(a, b *int32) *int32 {
+	if testNil(a, b) {
+		return nil
+	}
+
+	if *a > *b {
+		return a
+	}
+
+	return b
+}
+
 // IsSmallerQuantity : checks if first resource is of a smaller quantity than the second
 func IsSmallerQuantity(requestStr, limitStr string) (bool, error) {
 

ui/Dockerfile

@@ -1,7 +1,7 @@
 FROM alpine:3.6
 MAINTAINER team-acid@zalando.de
 
-EXPOSE 8080
+EXPOSE 8081
 
 RUN \
   apk add --no-cache \
@@ -29,6 +29,7 @@ RUN \
     /var/cache/apk/*
 
 COPY requirements.txt /
+COPY start_server.sh /
 RUN pip3 install -r /requirements.txt
 
 COPY operator_ui /operator_ui
@@ -37,4 +38,4 @@ ARG VERSION=dev
 RUN sed -i "s/__version__ = .*/__version__ = '${VERSION}'/" /operator_ui/__init__.py
 
 WORKDIR /
-ENTRYPOINT ["/usr/bin/python3", "-m", "operator_ui"]
+CMD ["/usr/bin/python3", "-m", "operator_ui"]

ui/Makefile

@@ -36,4 +36,4 @@ push:
 	docker push "$(IMAGE):$(TAG)$(CDP_TAG)"
 
 mock:
-	docker run -it -p 8080:8080 "$(IMAGE):$(TAG)" --mock
+	docker run -it -p 8081:8081 "$(IMAGE):$(TAG)" --mock

ui/app/src/edit.tag.pug

@@ -137,6 +137,7 @@ edit
         o.spec.numberOfInstances = i.spec.numberOfInstances
         o.spec.enableMasterLoadBalancer = i.spec.enableMasterLoadBalancer || false
         o.spec.enableReplicaLoadBalancer = i.spec.enableReplicaLoadBalancer || false
+        o.spec.enableConnectionPooler = i.spec.enableConnectionPooler || false
         o.spec.volume = { size: i.spec.volume.size }
 
         if ('users' in i.spec && typeof i.spec.users === 'object') {

ui/app/src/new.tag.pug

@@ -239,6 +239,18 @@ new
                   |
                   | Enable replica ELB
 
+            tr
+              td Enable Connection Pool
+              td
+                label
+                  input(
+                    type='checkbox'
+                    value='{ enableConnectionPooler }'
+                    onchange='{ toggleEnableConnectionPooler }'
+                  )
+                  |
+                  | Enable Connection Pool (using PGBouncer)
+
             tr
               td Volume size
               td
@@ -493,6 +505,9 @@ new
         {{#if enableReplicaLoadBalancer}}
         enableReplicaLoadBalancer: true
         {{/if}}
+        {{#if enableConnectionPooler}}
+        enableConnectionPooler: true
+        {{/if}}
         volume:
           size: "{{ volumeSize }}Gi"
         {{#if users}}
@@ -516,13 +531,14 @@ new
           - {{ odd }}/32
         {{/if}}
 
+        {{#if resourcesVisible}}
         resources:
           requests:
             cpu: {{ cpu.state.request.state }}m
             memory: {{ memory.state.request.state }}Mi
           limits:
             cpu: {{ cpu.state.limit.state }}m
-            memory: {{ memory.state.limit.state }}Mi{{#if restoring}}
+            memory: {{ memory.state.limit.state }}Mi{{/if}}{{#if restoring}}
 
         clone:
           cluster: "{{ backup.state.name.state }}"
@@ -542,6 +558,7 @@ new
         instanceCount: this.instanceCount,
         enableMasterLoadBalancer: this.enableMasterLoadBalancer,
         enableReplicaLoadBalancer: this.enableReplicaLoadBalancer,
+        enableConnectionPooler: this.enableConnectionPooler,
         volumeSize: this.volumeSize,
         users: this.users.valids,
         databases: this.databases.valids,
@@ -552,6 +569,7 @@ new
         memory: this.memory,
         backup: this.backup,
         namespace: this.namespace,
+        resourcesVisible: this.config.resources_visible,
         restoring: this.backup.state.type.state !== 'empty',
         pitr: this.backup.state.type.state === 'pitr',
       }
@@ -598,6 +616,10 @@ new
       this.enableReplicaLoadBalancer = !this.enableReplicaLoadBalancer
     }
 
+    this.toggleEnableConnectionPooler = e => {
+      this.enableConnectionPooler = !this.enableConnectionPooler
+    }
+
     this.volumeChange = e => {
       this.volumeSize = +e.target.value
     }
@@ -892,6 +914,7 @@ new
       this.odd = ''
       this.enableMasterLoadBalancer = false
       this.enableReplicaLoadBalancer = false
+      this.enableConnectionPooler = false
 
       this.postgresqlVersion = this.postgresqlVersion = (
         this.config.postgresql_versions[0]

ui/app/src/postgresql.tag.pug

@@ -92,6 +92,8 @@ postgresql
         .alert.alert-success(if='{ progress.masterLabel }') PostgreSQL master available, label is attached
         .alert.alert-success(if='{ progress.masterLabel && progress.dnsName }') PostgreSQL ready: <strong>{ progress.dnsName }</strong>
 
+        .alert.alert-success(if='{ progress.pooler }') Connection pooler deployment created
+
       .col-lg-3
         help-general(config='{ opts.config }')
 
@@ -122,9 +124,11 @@ postgresql
       jQuery.get(
         '/postgresqls/' + this.cluster_path,
       ).done(data => {
+        this.progress.pooler = false
         this.progress.postgresql = true
         this.progress.postgresqlManifest = data
         this.progress.createdTimestamp = data.metadata.creationTimestamp
+        this.progress.poolerEnabled = data.spec.enableConnectionPooler
         this.uid = this.progress.postgresqlManifest.metadata.uid
         this.update()
 
@@ -160,6 +164,11 @@ postgresql
                 this.progress.dnsName = data.metadata.name + '.' + data.metadata.namespace
               }
 
+              jQuery.get('/pooler/' + this.cluster_path).done(data => {
+                this.progress.pooler = {"url": ""}
+                this.update()
+              })
+
               this.update()
             })
           })