End 2 End tests speedup (#1180)

* Improving end 2 end tests, especially speed of execution and error, by implementing proper eventual asserts and timeouts.
* Add documentation for running individual tests
* Fixed String encoding in Patorni state check and error case
* Printing config as multi log line entity, makes it readable and grepable on startup
* Cosmetic changes to logs. Removed quotes from diff. Move all object diffs to text diff. Enabled padding for log level.
* Mount script with tools for easy logaccess and watching objects.
* Set proper update strategy for Postgres operator deployment.
* Move long running test to end. Move pooler test to new functions.
* Remove quote from valid K8s identifiers.

Makefile

@@ -24,12 +24,16 @@ PKG := `go list ./... | grep -v /vendor/`
 
 ifeq ($(DEBUG),1)
 	DOCKERFILE = DebugDockerfile
-	DEBUG_POSTFIX := -debug
+	DEBUG_POSTFIX := -debug-$(shell date hhmmss)
 	BUILD_FLAGS += -gcflags "-N -l"
 else
 	DOCKERFILE = Dockerfile
 endif
 
+ifeq ($(FRESH),1)
+  DEBUG_FRESH=$(shell date +"%H-%M-%S")
+endif
+
 ifdef CDP_PULL_REQUEST_NUMBER
 	CDP_TAG := -${CDP_BUILD_VERSION}
 endif
@@ -66,7 +70,7 @@ docker: ${DOCKERDIR}/${DOCKERFILE} docker-context
 	echo "Version ${VERSION}"
 	echo "CDP tag ${CDP_TAG}"
 	echo "git describe $(shell git describe --tags --always --dirty)"
-	cd "${DOCKERDIR}" && docker build --rm -t "$(IMAGE):$(TAG)$(CDP_TAG)$(DEBUG_POSTFIX)" -f "${DOCKERFILE}" .
+	cd "${DOCKERDIR}" && docker build --rm -t "$(IMAGE):$(TAG)$(CDP_TAG)$(DEBUG_FRESH)$(DEBUG_POSTFIX)" -f "${DOCKERFILE}" .
 
 indocker-race:
 	docker run --rm -v "${GOPATH}":"${GOPATH}" -e GOPATH="${GOPATH}" -e RACE=1 -w ${PWD} golang:1.8.1 bash -c "make linux"

e2e/Dockerfile

@@ -14,6 +14,7 @@ RUN apt-get update \
            python3-setuptools \
            python3-pip \
            curl \
+           vim \
     && pip3 install --no-cache-dir -r requirements.txt \
     && curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl \
     && chmod +x ./kubectl \
@@ -21,4 +22,7 @@ RUN apt-get update \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-ENTRYPOINT ["python3", "-m", "unittest", "discover", "--start-directory", ".", "-v"]
+# working line
+# python3 -m unittest discover -v --failfast -k test_e2e.EndToEndTestCase.test_lazy_spilo_upgrade --start-directory tests
+ENTRYPOINT ["python3", "-m", "unittest"]
+CMD ["discover","-v","--failfast","--start-directory","/tests"]

e2e/README.md

@@ -12,6 +12,10 @@ Docker.
 Docker
 Go
 
+# Notice
+
+The `manifest` folder in e2e tests folder is not commited to git, it comes from `/manifests`
+
 ## Build test runner
 
 In the directory of the cloned Postgres Operator repository change to the e2e
@@ -35,6 +39,46 @@ In the e2e folder you can invoke tests either with `make test` or with:
 To run both the build and test step you can invoke `make e2e` from the parent
 directory.
 
+To run the end 2 end test and keep the kind state execute:
+```bash
+NOCLEANUP=True ./run.sh
+```
+
+## Run indidual test
+
+After having executed a normal E2E run with `NOCLEANUP=True` Kind still continues to run, allowing you subsequent test runs.
+
+To run an individual test, run the following command in the `e2e` directory
+
+```bash
+NOCLEANUP=True ./run.sh main tests.test_e2e.EndToEndTestCase.test_lazy_spilo_upgrade
+```
+
+## Inspecting Kind
+
+If you want to inspect Kind/Kubernetes cluster, use the following script to exec into the K8s setup and then use `kubectl`
+
+```bash
+./exec_into_env.sh
+
+# use kube ctl
+kubectl get pods
+
+# watch relevant objects
+./scripts/watch_objects.sh
+
+# get operator logs
+./scripts/get_logs.sh
+```
+
+## Cleaning up Kind
+
+To cleanup kind and start fresh
+
+```bash
+e2e/run.sh cleanup
+```
+
 ## Covered use cases
 
 The current tests are all bundled in [`test_e2e.py`](tests/test_e2e.py):

e2e/exec_into_env.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+export cluster_name="postgres-operator-e2e-tests"
+export kubeconfig_path="/tmp/kind-config-${cluster_name}"
+export operator_image="registry.opensource.zalan.do/acid/postgres-operator:latest"
+export e2e_test_runner_image="registry.opensource.zalan.do/acid/postgres-operator-e2e-tests-runner:0.3"
+
+docker run -it --entrypoint /bin/bash --network=host -e "TERM=xterm-256color" \
+    --mount type=bind,source="$(readlink -f ${kubeconfig_path})",target=/root/.kube/config \
+    --mount type=bind,source="$(readlink -f manifests)",target=/manifests \
+    --mount type=bind,source="$(readlink -f tests)",target=/tests \
+    --mount type=bind,source="$(readlink -f exec.sh)",target=/exec.sh \
+    --mount type=bind,source="$(readlink -f scripts)",target=/scripts \
+    -e OPERATOR_IMAGE="${operator_image}" "${e2e_test_runner_image}"

e2e/run.sh

@@ -9,6 +9,10 @@ IFS=$'\n\t'
 readonly cluster_name="postgres-operator-e2e-tests"
 readonly kubeconfig_path="/tmp/kind-config-${cluster_name}"
 readonly spilo_image="registry.opensource.zalan.do/acid/spilo-12:1.6-p5"
+readonly e2e_test_runner_image="registry.opensource.zalan.do/acid/postgres-operator-e2e-tests-runner:0.3"
+
+export GOPATH=${GOPATH-~/go}
+export PATH=${GOPATH}/bin:$PATH
 
 echo "Clustername: ${cluster_name}"
 echo "Kubeconfig path: ${kubeconfig_path}"
@@ -19,12 +23,7 @@ function pull_images(){
   then
     docker pull registry.opensource.zalan.do/acid/postgres-operator:latest
   fi
-
   operator_image=$(docker images --filter=reference="registry.opensource.zalan.do/acid/postgres-operator" --format "{{.Repository}}:{{.Tag}}" | head -1)
-
-  # this image does not contain the tests; a container mounts them from a local "./tests" dir at start time
-  e2e_test_runner_image="registry.opensource.zalan.do/acid/postgres-operator-e2e-tests-runner:latest"
-  docker pull ${e2e_test_runner_image}
 }
 
 function start_kind(){
@@ -37,11 +36,16 @@ function start_kind(){
 
   export KUBECONFIG="${kubeconfig_path}"
   kind create cluster --name ${cluster_name} --config kind-cluster-postgres-operator-e2e-tests.yaml  
-  kind load docker-image "${operator_image}" --name ${cluster_name}
   docker pull "${spilo_image}"
   kind load docker-image "${spilo_image}" --name ${cluster_name}
 }
 
+function load_operator_image() {
+  echo "Loading operator image"
+  export KUBECONFIG="${kubeconfig_path}"
+  kind load docker-image "${operator_image}" --name ${cluster_name}
+}
+
 function set_kind_api_server_ip(){
   echo "Setting up kind API server ip"
   # use the actual kubeconfig to connect to the 'kind' API server
@@ -52,8 +56,7 @@ function set_kind_api_server_ip(){
 }
 
 function run_tests(){
-  echo "Running tests..."
+  echo "Running tests... image: ${e2e_test_runner_image}"
-
   # tests modify files in ./manifests, so we mount a copy of this directory done by the e2e Makefile
 
   docker run --rm --network=host -e "TERM=xterm-256color" \
@@ -61,11 +64,11 @@ function run_tests(){
   --mount type=bind,source="$(readlink -f manifests)",target=/manifests \
   --mount type=bind,source="$(readlink -f tests)",target=/tests \
   --mount type=bind,source="$(readlink -f exec.sh)",target=/exec.sh \
-  -e OPERATOR_IMAGE="${operator_image}" "${e2e_test_runner_image}"
+  --mount type=bind,source="$(readlink -f scripts)",target=/scripts \
-  
+  -e OPERATOR_IMAGE="${operator_image}" "${e2e_test_runner_image}" ${E2E_TEST_CASE-} $@
 }
 
-function clean_up(){
+function cleanup(){
   echo "Executing cleanup"
   unset KUBECONFIG
   kind delete cluster --name ${cluster_name}
@@ -73,14 +76,16 @@ function clean_up(){
 }
 
 function main(){
+  echo "Entering main function..."
+  [[ -z ${NOCLEANUP-} ]] && trap "cleanup" QUIT TERM EXIT
+  pull_images
+  [[ ! -f ${kubeconfig_path} ]] && start_kind
+  load_operator_image
+  set_kind_api_server_ip
 
-  trap "clean_up" QUIT TERM EXIT
+  shift
-
+  run_tests $@
-  time pull_images
-  time start_kind
-  time set_kind_api_server_ip
-  run_tests
   exit 0
 }
 
-"$@"
+"$1" $@

e2e/scripts/cleanup.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+kubectl delete postgresql acid-minimal-cluster
+kubectl delete deployments -l application=db-connection-pooler,cluster-name=acid-minimal-cluster
+kubectl delete statefulsets -l application=spilo,cluster-name=acid-minimal-cluster
+kubectl delete services -l application=spilo,cluster-name=acid-minimal-cluster
+kubectl delete configmap postgres-operator
+kubectl delete deployment postgres-operator

e2e/scripts/get_logs.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+kubectl logs $(kubectl get pods -l name=postgres-operator --field-selector status.phase=Running -o jsonpath='{.items..metadata.name}')

e2e/scripts/watch_objects.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+watch -c "
+kubectl get postgresql
+echo
+echo -n 'Rolling upgrade pending: '
+kubectl get statefulset -o jsonpath='{.items..metadata.annotations.zalando-postgres-operator-rolling-update-required}'
+echo
+echo
+kubectl get pods -o wide
+echo
+kubectl get statefulsets
+echo
+kubectl get deployments
+echo
+kubectl get pods -l name=postgres-operator -o jsonpath='{.items..metadata.annotations.step}'
+echo
+kubectl get pods -l application=spilo -o jsonpath='{.items..spec.containers..image}'
+"

e2e/tests/k8s_api.py

@@ -0,0 +1,522 @@
+import json
+import unittest
+import time
+import timeout_decorator
+import subprocess
+import warnings
+import os
+import yaml
+
+from datetime import datetime
+from kubernetes import client, config
+from kubernetes.client.rest import ApiException
+
+def to_selector(labels):
+    return ",".join(["=".join(l) for l in labels.items()])
+
+class K8sApi:
+
+    def __init__(self):
+
+        # https://github.com/kubernetes-client/python/issues/309
+        warnings.simplefilter("ignore", ResourceWarning)
+
+        self.config = config.load_kube_config()
+        self.k8s_client = client.ApiClient()
+
+        self.core_v1 = client.CoreV1Api()
+        self.apps_v1 = client.AppsV1Api()
+        self.batch_v1_beta1 = client.BatchV1beta1Api()
+        self.custom_objects_api = client.CustomObjectsApi()
+        self.policy_v1_beta1 = client.PolicyV1beta1Api()
+        self.storage_v1_api = client.StorageV1Api()
+
+
+class K8s:
+    '''
+    Wraps around K8s api client and helper methods.
+    '''
+
+    RETRY_TIMEOUT_SEC = 1
+
+    def __init__(self, labels='x=y', namespace='default'):
+        self.api = K8sApi()
+        self.labels=labels
+        self.namespace=namespace
+
+    def get_pg_nodes(self, pg_cluster_name, namespace='default'):
+        master_pod_node = ''
+        replica_pod_nodes = []
+        podsList = self.api.core_v1.list_namespaced_pod(namespace, label_selector=pg_cluster_name)
+        for pod in podsList.items:
+            if pod.metadata.labels.get('spilo-role') == 'master':
+                master_pod_node = pod.spec.node_name
+            elif pod.metadata.labels.get('spilo-role') == 'replica':
+                replica_pod_nodes.append(pod.spec.node_name)
+
+        return master_pod_node, replica_pod_nodes
+
+    def get_cluster_nodes(self, cluster_labels='cluster-name=acid-minimal-cluster', namespace='default'):
+        m = []
+        r = []
+        podsList = self.api.core_v1.list_namespaced_pod(namespace, label_selector=cluster_labels)
+        for pod in podsList.items:
+            if pod.metadata.labels.get('spilo-role') == 'master' and pod.status.phase == 'Running':
+                m.append(pod.spec.node_name)
+            elif pod.metadata.labels.get('spilo-role') == 'replica' and pod.status.phase == 'Running':
+                r.append(pod.spec.node_name)
+
+        return m, r
+
+    def wait_for_operator_pod_start(self):
+        self.wait_for_pod_start("name=postgres-operator")
+        # give operator time to subscribe to objects
+        time.sleep(1)
+        return True
+
+    def get_operator_pod(self):
+        pods = self.api.core_v1.list_namespaced_pod(
+            'default', label_selector='name=postgres-operator'
+        ).items
+
+        pods = list(filter(lambda x: x.status.phase=='Running', pods))
+
+        if len(pods):
+            return pods[0]
+
+        return None
+
+    def get_operator_log(self):
+        operator_pod = self.get_operator_pod()
+        pod_name = operator_pod.metadata.name
+        return self.api.core_v1.read_namespaced_pod_log(
+            name=pod_name,
+            namespace='default'
+        )
+
+    def pg_get_status(self, name="acid-minimal-cluster", namespace="default"):
+        pg = self.api.custom_objects_api.get_namespaced_custom_object(
+            "acid.zalan.do", "v1", namespace, "postgresqls", name)
+        return pg.get("status", {}).get("PostgresClusterStatus", None)
+
+    def wait_for_pod_start(self, pod_labels, namespace='default'):
+        pod_phase = 'No pod running'
+        while pod_phase != 'Running':
+            pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=pod_labels).items
+            if pods:
+                pod_phase = pods[0].status.phase
+
+            time.sleep(self.RETRY_TIMEOUT_SEC)
+
+
+    def get_service_type(self, svc_labels, namespace='default'):
+        svc_type = ''
+        svcs = self.api.core_v1.list_namespaced_service(namespace, label_selector=svc_labels, limit=1).items
+        for svc in svcs:
+            svc_type = svc.spec.type
+        return svc_type
+
+    def check_service_annotations(self, svc_labels, annotations, namespace='default'):
+        svcs = self.api.core_v1.list_namespaced_service(namespace, label_selector=svc_labels, limit=1).items
+        for svc in svcs:
+            for key, value in annotations.items():
+                if not svc.metadata.annotations or key not in svc.metadata.annotations or svc.metadata.annotations[key] != value:
+                    print("Expected key {} not found in annotations {}".format(key, svc.metadata.annotations))
+                    return False
+        return True
+
+    def check_statefulset_annotations(self, sset_labels, annotations, namespace='default'):
+        ssets = self.api.apps_v1.list_namespaced_stateful_set(namespace, label_selector=sset_labels, limit=1).items
+        for sset in ssets:
+            for key, value in annotations.items():
+                if key not in sset.metadata.annotations or sset.metadata.annotations[key] != value:
+                    print("Expected key {} not found in annotations {}".format(key, sset.metadata.annotations))
+                    return False
+        return True
+
+    def scale_cluster(self, number_of_instances, name="acid-minimal-cluster", namespace="default"):
+        body = {
+            "spec": {
+                "numberOfInstances": number_of_instances
+            }
+        }
+        self.api.custom_objects_api.patch_namespaced_custom_object(
+            "acid.zalan.do", "v1", namespace, "postgresqls", name, body)
+
+    def wait_for_running_pods(self, labels, number, namespace=''):
+        while self.count_pods_with_label(labels) != number:
+            time.sleep(self.RETRY_TIMEOUT_SEC)
+
+    def wait_for_pods_to_stop(self, labels, namespace=''):
+        while self.count_pods_with_label(labels) != 0:
+            time.sleep(self.RETRY_TIMEOUT_SEC)
+
+    def wait_for_service(self, labels, namespace='default'):
+        def get_services():
+            return self.api.core_v1.list_namespaced_service(
+                namespace, label_selector=labels
+            ).items
+
+        while not get_services():
+            time.sleep(self.RETRY_TIMEOUT_SEC)
+
+    def count_pods_with_label(self, labels, namespace='default'):
+        return len(self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items)
+
+    def count_services_with_label(self, labels, namespace='default'):
+        return len(self.api.core_v1.list_namespaced_service(namespace, label_selector=labels).items)
+
+    def count_endpoints_with_label(self, labels, namespace='default'):
+        return len(self.api.core_v1.list_namespaced_endpoints(namespace, label_selector=labels).items)
+
+    def count_secrets_with_label(self, labels, namespace='default'):
+        return len(self.api.core_v1.list_namespaced_secret(namespace, label_selector=labels).items)
+
+    def count_statefulsets_with_label(self, labels, namespace='default'):
+        return len(self.api.apps_v1.list_namespaced_stateful_set(namespace, label_selector=labels).items)
+
+    def count_deployments_with_label(self, labels, namespace='default'):
+        return len(self.api.apps_v1.list_namespaced_deployment(namespace, label_selector=labels).items)
+
+    def count_pdbs_with_label(self, labels, namespace='default'):
+        return len(self.api.policy_v1_beta1.list_namespaced_pod_disruption_budget(
+            namespace, label_selector=labels).items)
+  
+    def count_running_pods(self, labels='application=spilo,cluster-name=acid-minimal-cluster', namespace='default'):
+        pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items
+        return len(list(filter(lambda x: x.status.phase=='Running', pods)))
+
+    def wait_for_pod_failover(self, failover_targets, labels, namespace='default'):
+        pod_phase = 'Failing over'
+        new_pod_node = ''
+
+        while (pod_phase != 'Running') or (new_pod_node not in failover_targets):
+            pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items
+            if pods:
+                new_pod_node = pods[0].spec.node_name
+                pod_phase = pods[0].status.phase
+            time.sleep(self.RETRY_TIMEOUT_SEC)
+
+    def get_logical_backup_job(self, namespace='default'):
+        return self.api.batch_v1_beta1.list_namespaced_cron_job(namespace, label_selector="application=spilo")
+
+    def wait_for_logical_backup_job(self, expected_num_of_jobs):
+        while (len(self.get_logical_backup_job().items) != expected_num_of_jobs):
+            time.sleep(self.RETRY_TIMEOUT_SEC)
+
+    def wait_for_logical_backup_job_deletion(self):
+        self.wait_for_logical_backup_job(expected_num_of_jobs=0)
+
+    def wait_for_logical_backup_job_creation(self):
+        self.wait_for_logical_backup_job(expected_num_of_jobs=1)
+
+    def delete_operator_pod(self, step="Delete operator deplyment"):
+        operator_pod = self.api.core_v1.list_namespaced_pod('default', label_selector="name=postgres-operator").items[0].metadata.name
+        self.api.apps_v1.patch_namespaced_deployment("postgres-operator","default", {"spec":{"template":{"metadata":{"annotations":{"step":"{}-{}".format(step, time.time())}}}}})
+        self.wait_for_operator_pod_start()
+
+    def update_config(self, config_map_patch, step="Updating operator deployment"):
+        self.api.core_v1.patch_namespaced_config_map("postgres-operator", "default", config_map_patch)
+        self.delete_operator_pod(step=step)
+
+    def patch_statefulset(self, data, name="acid-minimal-cluster", namespace="default"):
+        self.api.apps_v1.patch_namespaced_stateful_set(name, namespace, data)
+
+    def create_with_kubectl(self, path):
+        return subprocess.run(
+            ["kubectl", "apply", "-f", path],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE)
+
+    def exec_with_kubectl(self, pod, cmd):
+        return subprocess.run(["./exec.sh", pod, cmd],
+                              stdout=subprocess.PIPE,
+                              stderr=subprocess.PIPE)
+
+    def get_patroni_state(self, pod):
+        r = self.exec_with_kubectl(pod, "patronictl list -f json")
+        if not r.returncode == 0 or not r.stdout.decode()[0:1]=="[":
+            return []
+        return json.loads(r.stdout.decode())
+
+    def get_patroni_running_members(self, pod="acid-minimal-cluster-0"):
+        result = self.get_patroni_state(pod)
+        return list(filter(lambda x: "State" in x and x["State"] == "running", result))
+
+    def get_deployment_replica_count(self, name="acid-minimal-cluster-pooler", namespace="default"):
+        try:
+            deployment = self.api.apps_v1.read_namespaced_deployment(name, namespace)
+            return deployment.spec.replicas
+        except ApiException as e:
+            return None
+    
+    def get_statefulset_image(self, label_selector="application=spilo,cluster-name=acid-minimal-cluster", namespace='default'):
+        ssets = self.api.apps_v1.list_namespaced_stateful_set(namespace, label_selector=label_selector, limit=1)
+        if len(ssets.items) == 0:
+            return None
+        return ssets.items[0].spec.template.spec.containers[0].image
+
+    def get_effective_pod_image(self, pod_name, namespace='default'):
+        '''
+        Get the Spilo image pod currently uses. In case of lazy rolling updates
+        it may differ from the one specified in the stateful set.
+        '''
+        pod = self.api.core_v1.list_namespaced_pod(
+            namespace, label_selector="statefulset.kubernetes.io/pod-name=" + pod_name)
+        
+        if len(pod.items) == 0:
+            return None
+        return pod.items[0].spec.containers[0].image
+
+    def get_cluster_leader_pod(self, pg_cluster_name, namespace='default'):
+        labels = {
+            'application': 'spilo',
+            'cluster-name': pg_cluster_name,
+            'spilo-role': 'master',
+        }
+
+        pods = self.api.core_v1.list_namespaced_pod(
+                namespace, label_selector=to_selector(labels)).items
+
+        if pods:
+            return pods[0]
+
+
+class K8sBase:
+    '''
+    K8s basic API wrapper class supposed to be inherited by other more specific classes for e2e tests
+    '''
+
+    RETRY_TIMEOUT_SEC = 1
+
+    def __init__(self, labels='x=y', namespace='default'):
+        self.api = K8sApi()
+        self.labels=labels
+        self.namespace=namespace
+
+    def get_pg_nodes(self, pg_cluster_labels='cluster-name=acid-minimal-cluster', namespace='default'):
+        master_pod_node = ''
+        replica_pod_nodes = []
+        podsList = self.api.core_v1.list_namespaced_pod(namespace, label_selector=pg_cluster_labels)
+        for pod in podsList.items:
+            if pod.metadata.labels.get('spilo-role') == 'master':
+                master_pod_node = pod.spec.node_name
+            elif pod.metadata.labels.get('spilo-role') == 'replica':
+                replica_pod_nodes.append(pod.spec.node_name)
+
+        return master_pod_node, replica_pod_nodes
+
+    def get_cluster_nodes(self, cluster_labels='cluster-name=acid-minimal-cluster', namespace='default'):
+        m = []
+        r = []
+        podsList = self.api.core_v1.list_namespaced_pod(namespace, label_selector=cluster_labels)
+        for pod in podsList.items:
+            if pod.metadata.labels.get('spilo-role') == 'master' and pod.status.phase == 'Running':
+                m.append(pod.spec.node_name)
+            elif pod.metadata.labels.get('spilo-role') == 'replica' and pod.status.phase == 'Running':
+                r.append(pod.spec.node_name)
+
+        return m, r
+
+    def wait_for_operator_pod_start(self):
+        self.wait_for_pod_start("name=postgres-operator")
+
+    def get_operator_pod(self):
+        pods = self.api.core_v1.list_namespaced_pod(
+            'default', label_selector='name=postgres-operator'
+        ).items
+
+        if pods:
+            return pods[0]
+
+        return None
+
+    def get_operator_log(self):
+        operator_pod = self.get_operator_pod()
+        pod_name = operator_pod.metadata.name
+        return self.api.core_v1.read_namespaced_pod_log(
+            name=pod_name,
+            namespace='default'
+        )
+
+    def wait_for_pod_start(self, pod_labels, namespace='default'):
+        pod_phase = 'No pod running'
+        while pod_phase != 'Running':
+            pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=pod_labels).items
+            if pods:
+                pod_phase = pods[0].status.phase
+
+            time.sleep(self.RETRY_TIMEOUT_SEC)
+
+    def get_service_type(self, svc_labels, namespace='default'):
+        svc_type = ''
+        svcs = self.api.core_v1.list_namespaced_service(namespace, label_selector=svc_labels, limit=1).items
+        for svc in svcs:
+            svc_type = svc.spec.type
+        return svc_type
+
+    def check_service_annotations(self, svc_labels, annotations, namespace='default'):
+        svcs = self.api.core_v1.list_namespaced_service(namespace, label_selector=svc_labels, limit=1).items
+        for svc in svcs:
+            for key, value in annotations.items():
+                if key not in svc.metadata.annotations or svc.metadata.annotations[key] != value:
+                    print("Expected key {} not found in annotations {}".format(key, svc.metadata.annotation))
+                    return False
+        return True
+
+    def check_statefulset_annotations(self, sset_labels, annotations, namespace='default'):
+        ssets = self.api.apps_v1.list_namespaced_stateful_set(namespace, label_selector=sset_labels, limit=1).items
+        for sset in ssets:
+            for key, value in annotations.items():
+                if key not in sset.metadata.annotations or sset.metadata.annotations[key] != value:
+                    print("Expected key {} not found in annotations {}".format(key, sset.metadata.annotation))
+                    return False
+        return True
+
+    def scale_cluster(self, number_of_instances, name="acid-minimal-cluster", namespace="default"):
+        body = {
+            "spec": {
+                "numberOfInstances": number_of_instances
+            }
+        }
+        self.api.custom_objects_api.patch_namespaced_custom_object(
+            "acid.zalan.do", "v1", namespace, "postgresqls", name, body)
+
+    def wait_for_running_pods(self, labels, number, namespace=''):
+        while self.count_pods_with_label(labels) != number:
+            time.sleep(self.RETRY_TIMEOUT_SEC)
+
+    def wait_for_pods_to_stop(self, labels, namespace=''):
+        while self.count_pods_with_label(labels) != 0:
+            time.sleep(self.RETRY_TIMEOUT_SEC)
+
+    def wait_for_service(self, labels, namespace='default'):
+        def get_services():
+            return self.api.core_v1.list_namespaced_service(
+                namespace, label_selector=labels
+            ).items
+
+        while not get_services():
+            time.sleep(self.RETRY_TIMEOUT_SEC)
+
+    def count_pods_with_label(self, labels, namespace='default'):
+        return len(self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items)
+
+    def count_services_with_label(self, labels, namespace='default'):
+        return len(self.api.core_v1.list_namespaced_service(namespace, label_selector=labels).items)
+
+    def count_endpoints_with_label(self, labels, namespace='default'):
+        return len(self.api.core_v1.list_namespaced_endpoints(namespace, label_selector=labels).items)
+
+    def count_secrets_with_label(self, labels, namespace='default'):
+        return len(self.api.core_v1.list_namespaced_secret(namespace, label_selector=labels).items)
+
+    def count_statefulsets_with_label(self, labels, namespace='default'):
+        return len(self.api.apps_v1.list_namespaced_stateful_set(namespace, label_selector=labels).items)
+
+    def count_deployments_with_label(self, labels, namespace='default'):
+        return len(self.api.apps_v1.list_namespaced_deployment(namespace, label_selector=labels).items)
+
+    def count_pdbs_with_label(self, labels, namespace='default'):
+        return len(self.api.policy_v1_beta1.list_namespaced_pod_disruption_budget(
+            namespace, label_selector=labels).items)
+  
+    def count_running_pods(self, labels='application=spilo,cluster-name=acid-minimal-cluster', namespace='default'):
+        pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items
+        return len(list(filter(lambda x: x.status.phase=='Running', pods)))
+
+    def wait_for_pod_failover(self, failover_targets, labels, namespace='default'):
+        pod_phase = 'Failing over'
+        new_pod_node = ''
+
+        while (pod_phase != 'Running') or (new_pod_node not in failover_targets):
+            pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items
+            if pods:
+                new_pod_node = pods[0].spec.node_name
+                pod_phase = pods[0].status.phase
+            time.sleep(self.RETRY_TIMEOUT_SEC)
+
+    def get_logical_backup_job(self, namespace='default'):
+        return self.api.batch_v1_beta1.list_namespaced_cron_job(namespace, label_selector="application=spilo")
+
+    def wait_for_logical_backup_job(self, expected_num_of_jobs):
+        while (len(self.get_logical_backup_job().items) != expected_num_of_jobs):
+            time.sleep(self.RETRY_TIMEOUT_SEC)
+
+    def wait_for_logical_backup_job_deletion(self):
+        self.wait_for_logical_backup_job(expected_num_of_jobs=0)
+
+    def wait_for_logical_backup_job_creation(self):
+        self.wait_for_logical_backup_job(expected_num_of_jobs=1)
+
+    def delete_operator_pod(self, step="Delete operator deplyment"):
+        operator_pod = self.api.core_v1.list_namespaced_pod('default', label_selector="name=postgres-operator").items[0].metadata.name
+        self.api.apps_v1.patch_namespaced_deployment("postgres-operator","default", {"spec":{"template":{"metadata":{"annotations":{"step":"{}-{}".format(step, time.time())}}}}})
+        self.wait_for_operator_pod_start()
+
+    def update_config(self, config_map_patch, step="Updating operator deployment"):
+        self.api.core_v1.patch_namespaced_config_map("postgres-operator", "default", config_map_patch)
+        self.delete_operator_pod(step=step)
+
+    def create_with_kubectl(self, path):
+        return subprocess.run(
+            ["kubectl", "apply", "-f", path],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE)
+
+    def exec_with_kubectl(self, pod, cmd):
+        return subprocess.run(["./exec.sh", pod, cmd],
+                              stdout=subprocess.PIPE,
+                              stderr=subprocess.PIPE)
+
+    def get_patroni_state(self, pod):
+        r = self.exec_with_kubectl(pod, "patronictl list -f json")
+        if not r.returncode == 0 or not r.stdout.decode()[0:1]=="[":
+            return []
+        return json.loads(r.stdout.decode())
+
+    def get_patroni_running_members(self, pod):
+        result = self.get_patroni_state(pod)
+        return list(filter(lambda x: x["State"]=="running", result))
+    
+    def get_statefulset_image(self, label_selector="application=spilo,cluster-name=acid-minimal-cluster", namespace='default'):
+        ssets = self.api.apps_v1.list_namespaced_stateful_set(namespace, label_selector=label_selector, limit=1)
+        if len(ssets.items) == 0:
+            return None
+        return ssets.items[0].spec.template.spec.containers[0].image
+
+    def get_effective_pod_image(self, pod_name, namespace='default'):
+        '''
+        Get the Spilo image pod currently uses. In case of lazy rolling updates
+        it may differ from the one specified in the stateful set.
+        '''
+        pod = self.api.core_v1.list_namespaced_pod(
+            namespace, label_selector="statefulset.kubernetes.io/pod-name=" + pod_name)
+        
+        if len(pod.items) == 0:
+            return None
+        return pod.items[0].spec.containers[0].image
+
+
+"""
+  Inspiriational classes towards easier writing of end to end tests with one cluster per test case
+"""
+class K8sOperator(K8sBase):
+    def __init__(self, labels="name=postgres-operator", namespace="default"):
+        super().__init__(labels, namespace)
+
+class K8sPostgres(K8sBase):
+    def __init__(self, labels="cluster-name=acid-minimal-cluster", namespace="default"):
+        super().__init__(labels, namespace)
+    
+    def get_pg_nodes(self):
+        master_pod_node = ''
+        replica_pod_nodes = []
+        podsList = self.api.core_v1.list_namespaced_pod(self.namespace, label_selector=self.labels)
+        for pod in podsList.items:
+            if pod.metadata.labels.get('spilo-role') == 'master':
+                master_pod_node = pod.spec.node_name
+            elif pod.metadata.labels.get('spilo-role') == 'replica':
+                replica_pod_nodes.append(pod.spec.node_name)
+
+        return master_pod_node, replica_pod_nodes

manifests/postgres-operator.yaml

@@ -4,6 +4,8 @@ metadata:
   name: postgres-operator
 spec:
   replicas: 1
+  strategy:
+    type: "Recreate"
   selector:
     matchLabels:
       name: postgres-operator

pkg/cluster/cluster.go

@@ -371,11 +371,11 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa
 	//TODO: improve me
 	if *c.Statefulset.Spec.Replicas != *statefulSet.Spec.Replicas {
 		match = false
-		reasons = append(reasons, "new statefulset's number of replicas doesn't match the current one")
+		reasons = append(reasons, "new statefulset's number of replicas does not match the current one")
 	}
 	if !reflect.DeepEqual(c.Statefulset.Annotations, statefulSet.Annotations) {
 		match = false
-		reasons = append(reasons, "new statefulset's annotations doesn't match the current one")
+		reasons = append(reasons, "new statefulset's annotations does not match the current one")
 	}
 
 	needsRollUpdate, reasons = c.compareContainers("initContainers", c.Statefulset.Spec.Template.Spec.InitContainers, statefulSet.Spec.Template.Spec.InitContainers, needsRollUpdate, reasons)
@@ -392,24 +392,24 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa
 	if c.Statefulset.Spec.Template.Spec.ServiceAccountName != statefulSet.Spec.Template.Spec.ServiceAccountName {
 		needsReplace = true
 		needsRollUpdate = true
-		reasons = append(reasons, "new statefulset's serviceAccountName service account name doesn't match the current one")
+		reasons = append(reasons, "new statefulset's serviceAccountName service account name does not match the current one")
 	}
 	if *c.Statefulset.Spec.Template.Spec.TerminationGracePeriodSeconds != *statefulSet.Spec.Template.Spec.TerminationGracePeriodSeconds {
 		needsReplace = true
 		needsRollUpdate = true
-		reasons = append(reasons, "new statefulset's terminationGracePeriodSeconds doesn't match the current one")
+		reasons = append(reasons, "new statefulset's terminationGracePeriodSeconds does not match the current one")
 	}
 	if !reflect.DeepEqual(c.Statefulset.Spec.Template.Spec.Affinity, statefulSet.Spec.Template.Spec.Affinity) {
 		needsReplace = true
 		needsRollUpdate = true
-		reasons = append(reasons, "new statefulset's pod affinity doesn't match the current one")
+		reasons = append(reasons, "new statefulset's pod affinity does not match the current one")
 	}
 
 	// Some generated fields like creationTimestamp make it not possible to use DeepCompare on Spec.Template.ObjectMeta
 	if !reflect.DeepEqual(c.Statefulset.Spec.Template.Labels, statefulSet.Spec.Template.Labels) {
 		needsReplace = true
 		needsRollUpdate = true
-		reasons = append(reasons, "new statefulset's metadata labels doesn't match the current one")
+		reasons = append(reasons, "new statefulset's metadata labels does not match the current one")
 	}
 	if (c.Statefulset.Spec.Selector != nil) && (statefulSet.Spec.Selector != nil) {
 		if !reflect.DeepEqual(c.Statefulset.Spec.Selector.MatchLabels, statefulSet.Spec.Selector.MatchLabels) {
@@ -420,7 +420,7 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa
 				return &compareStatefulsetResult{}
 			}
 			needsReplace = true
-			reasons = append(reasons, "new statefulset's selector doesn't match the current one")
+			reasons = append(reasons, "new statefulset's selector does not match the current one")
 		}
 	}
 
@@ -434,7 +434,7 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa
 		match = false
 		needsReplace = true
 		needsRollUpdate = true
-		reasons = append(reasons, "new statefulset's pod template security context in spec doesn't match the current one")
+		reasons = append(reasons, "new statefulset's pod template security context in spec does not match the current one")
 	}
 	if len(c.Statefulset.Spec.VolumeClaimTemplates) != len(statefulSet.Spec.VolumeClaimTemplates) {
 		needsReplace = true
@@ -445,17 +445,17 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa
 		// Some generated fields like creationTimestamp make it not possible to use DeepCompare on ObjectMeta
 		if name != statefulSet.Spec.VolumeClaimTemplates[i].Name {
 			needsReplace = true
-			reasons = append(reasons, fmt.Sprintf("new statefulset's name for volume %d doesn't match the current one", i))
+			reasons = append(reasons, fmt.Sprintf("new statefulset's name for volume %d does not match the current one", i))
 			continue
 		}
 		if !reflect.DeepEqual(c.Statefulset.Spec.VolumeClaimTemplates[i].Annotations, statefulSet.Spec.VolumeClaimTemplates[i].Annotations) {
 			needsReplace = true
-			reasons = append(reasons, fmt.Sprintf("new statefulset's annotations for volume %q doesn't match the current one", name))
+			reasons = append(reasons, fmt.Sprintf("new statefulset's annotations for volume %q does not match the current one", name))
 		}
 		if !reflect.DeepEqual(c.Statefulset.Spec.VolumeClaimTemplates[i].Spec, statefulSet.Spec.VolumeClaimTemplates[i].Spec) {
 			name := c.Statefulset.Spec.VolumeClaimTemplates[i].Name
 			needsReplace = true
-			reasons = append(reasons, fmt.Sprintf("new statefulset's volumeClaimTemplates specification for volume %q doesn't match the current one", name))
+			reasons = append(reasons, fmt.Sprintf("new statefulset's volumeClaimTemplates specification for volume %q does not match the current one", name))
 		}
 	}
 
@@ -465,14 +465,14 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa
 		match = false
 		needsReplace = true
 		needsRollUpdate = true
-		reasons = append(reasons, "new statefulset's pod priority class in spec doesn't match the current one")
+		reasons = append(reasons, "new statefulset's pod priority class in spec does not match the current one")
 	}
 
 	// lazy Spilo update: modify the image in the statefulset itself but let its pods run with the old image
 	// until they are re-created for other reasons, for example node rotation
 	if c.OpConfig.EnableLazySpiloUpgrade && !reflect.DeepEqual(c.Statefulset.Spec.Template.Spec.Containers[0].Image, statefulSet.Spec.Template.Spec.Containers[0].Image) {
 		needsReplace = true
-		reasons = append(reasons, "lazy Spilo update: new statefulset's pod image doesn't match the current one")
+		reasons = append(reasons, "lazy Spilo update: new statefulset's pod image does not match the current one")
 	}
 
 	if needsRollUpdate || needsReplace {
@@ -582,7 +582,7 @@ func (c *Cluster) enforceMinResourceLimits(spec *acidv1.PostgresSpec) error {
 			return fmt.Errorf("could not compare defined CPU limit %s with configured minimum value %s: %v", cpuLimit, minCPULimit, err)
 		}
 		if isSmaller {
-			c.logger.Warningf("defined CPU limit %s is below required minimum %s and will be set to it", cpuLimit, minCPULimit)
+			c.logger.Warningf("defined CPU limit %s is below required minimum %s and will be increased", cpuLimit, minCPULimit)
 			c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "ResourceLimits", "defined CPU limit %s is below required minimum %s and will be set to it", cpuLimit, minCPULimit)
 			spec.Resources.ResourceLimits.CPU = minCPULimit
 		}
@@ -595,7 +595,7 @@ func (c *Cluster) enforceMinResourceLimits(spec *acidv1.PostgresSpec) error {
 			return fmt.Errorf("could not compare defined memory limit %s with configured minimum value %s: %v", memoryLimit, minMemoryLimit, err)
 		}
 		if isSmaller {
-			c.logger.Warningf("defined memory limit %s is below required minimum %s and will be set to it", memoryLimit, minMemoryLimit)
+			c.logger.Warningf("defined memory limit %s is below required minimum %s and will be increased", memoryLimit, minMemoryLimit)
 			c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "ResourceLimits", "defined memory limit %s is below required minimum %s and will be set to it", memoryLimit, minMemoryLimit)
 			spec.Resources.ResourceLimits.Memory = minMemoryLimit
 		}

pkg/cluster/database.go

@@ -527,7 +527,7 @@ func (c *Cluster) installLookupFunction(poolerSchema, poolerUser string) error {
 			continue
 		}
 
-		c.logger.Infof("Install pooler lookup function into %s", dbname)
+		c.logger.Infof("install pooler lookup function into database '%s'", dbname)
 
 		// golang sql will do retries couple of times if pq driver reports
 		// connections issues (driver.ErrBadConn), but since our query is

pkg/cluster/k8sres.go

@@ -1157,7 +1157,9 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
 	}
 
 	// generate the spilo container
-	c.logger.Debugf("Generating Spilo container, environment variables: %v", spiloEnvVars)
+	c.logger.Debugf("Generating Spilo container, environment variables")
+	c.logger.Debugf("%v", spiloEnvVars)
+
 	spiloContainer := generateContainer(c.containerName(),
 		&effectiveDockerImage,
 		resourceRequirements,
@@ -2055,7 +2057,8 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar {
 		envVars = append(envVars, v1.EnvVar{Name: "AWS_SECRET_ACCESS_KEY", Value: c.OpConfig.LogicalBackup.LogicalBackupS3SecretAccessKey})
 	}
 
-	c.logger.Debugf("Generated logical backup env vars %v", envVars)
+	c.logger.Debugf("Generated logical backup env vars")
+	c.logger.Debugf("%v", envVars)
 	return envVars
 }
 

pkg/cluster/pod.go

@@ -304,9 +304,16 @@ func (c *Cluster) isSafeToRecreatePods(pods *v1.PodList) bool {
 	 after this check succeeds but before a pod is re-created
 	*/
 
+	for _, pod := range pods.Items {
+		c.logger.Debugf("name=%s phase=%s ip=%s", pod.Name, pod.Status.Phase, pod.Status.PodIP)
+	}
+
 	for _, pod := range pods.Items {
 		state, err := c.patroni.GetPatroniMemberState(&pod)
-		if err != nil || state == "creating replica" {
+		if err != nil {
+			c.logger.Errorf("failed to get Patroni state for pod: %s", err)
+			return false
+		} else if state == "creating replica" {
 			c.logger.Warningf("cannot re-create replica %s: it is currently being initialized", pod.Name)
 			return false
 		}

pkg/cluster/resources.go

@@ -293,7 +293,7 @@ func (c *Cluster) preScaleDown(newStatefulSet *appsv1.StatefulSet) error {
 
 // setRollingUpdateFlagForStatefulSet sets the indicator or the rolling update requirement
 // in the StatefulSet annotation.
-func (c *Cluster) setRollingUpdateFlagForStatefulSet(sset *appsv1.StatefulSet, val bool) {
+func (c *Cluster) setRollingUpdateFlagForStatefulSet(sset *appsv1.StatefulSet, val bool, msg string) {
 	anno := sset.GetAnnotations()
 	if anno == nil {
 		anno = make(map[string]string)
@@ -301,13 +301,13 @@ func (c *Cluster) setRollingUpdateFlagForStatefulSet(sset *appsv1.StatefulSet, v
 
 	anno[rollingUpdateStatefulsetAnnotationKey] = strconv.FormatBool(val)
 	sset.SetAnnotations(anno)
-	c.logger.Debugf("statefulset's rolling update annotation has been set to %t", val)
+	c.logger.Debugf("set statefulset's rolling update annotation to %t: caller/reason %s", val, msg)
 }
 
 // applyRollingUpdateFlagforStatefulSet sets the rolling update flag for the cluster's StatefulSet
 // and applies that setting to the actual running cluster.
 func (c *Cluster) applyRollingUpdateFlagforStatefulSet(val bool) error {
-	c.setRollingUpdateFlagForStatefulSet(c.Statefulset, val)
+	c.setRollingUpdateFlagForStatefulSet(c.Statefulset, val, "applyRollingUpdateFlag")
 	sset, err := c.updateStatefulSetAnnotations(c.Statefulset.GetAnnotations())
 	if err != nil {
 		return err
@@ -359,14 +359,13 @@ func (c *Cluster) mergeRollingUpdateFlagUsingCache(runningStatefulSet *appsv1.St
 			podsRollingUpdateRequired = false
 		} else {
 			c.logger.Infof("found a statefulset with an unfinished rolling update of the pods")
-
 		}
 	}
 	return podsRollingUpdateRequired
 }
 
 func (c *Cluster) updateStatefulSetAnnotations(annotations map[string]string) (*appsv1.StatefulSet, error) {
-	c.logger.Debugf("updating statefulset annotations")
+	c.logger.Debugf("patching statefulset annotations")
 	patchData, err := metaAnnotationsPatch(annotations)
 	if err != nil {
 		return nil, fmt.Errorf("could not form patch for the statefulset metadata: %v", err)

pkg/cluster/sync.go

@@ -348,13 +348,13 @@ func (c *Cluster) syncStatefulSet() error {
 		if err != nil {
 			return fmt.Errorf("could not generate statefulset: %v", err)
 		}
-		c.setRollingUpdateFlagForStatefulSet(desiredSS, podsRollingUpdateRequired)
+		c.setRollingUpdateFlagForStatefulSet(desiredSS, podsRollingUpdateRequired, "from cache")
 
 		cmp := c.compareStatefulSetWith(desiredSS)
 		if !cmp.match {
 			if cmp.rollingUpdate && !podsRollingUpdateRequired {
 				podsRollingUpdateRequired = true
-				c.setRollingUpdateFlagForStatefulSet(desiredSS, podsRollingUpdateRequired)
+				c.setRollingUpdateFlagForStatefulSet(desiredSS, podsRollingUpdateRequired, "statefulset changes")
 			}
 
 			c.logStatefulSetChanges(c.Statefulset, desiredSS, false, cmp.reasons)
@@ -497,11 +497,11 @@ func (c *Cluster) syncSecrets() error {
 				return fmt.Errorf("could not get current secret: %v", err)
 			}
 			if secretUsername != string(secret.Data["username"]) {
-				c.logger.Warningf("secret %q does not contain the role %q", secretSpec.Name, secretUsername)
+				c.logger.Warningf("secret %s does not contain the role %q", secretSpec.Name, secretUsername)
 				continue
 			}
 			c.Secrets[secret.UID] = secret
-			c.logger.Debugf("secret %q already exists, fetching its password", util.NameFromMeta(secret.ObjectMeta))
+			c.logger.Debugf("secret %s already exists, fetching its password", util.NameFromMeta(secret.ObjectMeta))
 			if secretUsername == c.systemUsers[constants.SuperuserKeyName].Name {
 				secretUsername = constants.SuperuserKeyName
 				userMap = c.systemUsers
@@ -804,7 +804,7 @@ func (c *Cluster) syncLogicalBackupJob() error {
 			return fmt.Errorf("could not generate the desired logical backup job state: %v", err)
 		}
 		if match, reason := k8sutil.SameLogicalBackupJob(job, desiredJob); !match {
-			c.logger.Infof("logical job %q is not in the desired state and needs to be updated",
+			c.logger.Infof("logical job %s is not in the desired state and needs to be updated",
 				c.getLogicalBackupJobName(),
 			)
 			if reason != "" {
@@ -825,12 +825,12 @@ func (c *Cluster) syncLogicalBackupJob() error {
 	c.logger.Info("could not find the cluster's logical backup job")
 
 	if err = c.createLogicalBackupJob(); err == nil {
-		c.logger.Infof("created missing logical backup job %q", jobName)
+		c.logger.Infof("created missing logical backup job %s", jobName)
 	} else {
 		if !k8sutil.ResourceAlreadyExists(err) {
 			return fmt.Errorf("could not create missing logical backup job: %v", err)
 		}
-		c.logger.Infof("logical backup job %q already exists", jobName)
+		c.logger.Infof("logical backup job %s already exists", jobName)
 		if _, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Get(context.TODO(), jobName, metav1.GetOptions{}); err != nil {
 			return fmt.Errorf("could not fetch existing logical backup job: %v", err)
 		}
@@ -975,7 +975,7 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql
 			newConnectionPooler = &acidv1.ConnectionPooler{}
 		}
 
-		c.logger.Infof("Old: %+v, New %+v", oldConnectionPooler, newConnectionPooler)
+		logNiceDiff(c.logger, oldConnectionPooler, newConnectionPooler)
 
 		specSync, specReason := c.needSyncConnectionPoolerSpecs(oldConnectionPooler, newConnectionPooler)
 		defaultsSync, defaultsReason := c.needSyncConnectionPoolerDefaults(newConnectionPooler, deployment)

pkg/cluster/util.go

@@ -18,12 +18,14 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 
+	"github.com/sirupsen/logrus"
 	acidzalando "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do"
 	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
 	"github.com/zalando/postgres-operator/pkg/spec"
 	"github.com/zalando/postgres-operator/pkg/util"
 	"github.com/zalando/postgres-operator/pkg/util/constants"
 	"github.com/zalando/postgres-operator/pkg/util/k8sutil"
+	"github.com/zalando/postgres-operator/pkg/util/nicediff"
 	"github.com/zalando/postgres-operator/pkg/util/retryutil"
 )
 
@@ -166,40 +168,59 @@ func (c *Cluster) logPDBChanges(old, new *policybeta1.PodDisruptionBudget, isUpd
 		)
 	}
 
-	c.logger.Debugf("diff\n%s\n", util.PrettyDiff(old.Spec, new.Spec))
+	logNiceDiff(c.logger, old.Spec, new.Spec)
+}
+
+func logNiceDiff(log *logrus.Entry, old, new interface{}) {
+	o, erro := json.MarshalIndent(old, "", "  ")
+	n, errn := json.MarshalIndent(new, "", "  ")
+
+	if erro != nil || errn != nil {
+		panic("could not marshal API objects, should not happen")
+	}
+
+	nice := nicediff.Diff(string(o), string(n), true)
+	for _, s := range strings.Split(nice, "\n") {
+		// " is not needed in the value to understand
+		log.Debugf(strings.ReplaceAll(s, "\"", ""))
+	}
 }
 
 func (c *Cluster) logStatefulSetChanges(old, new *appsv1.StatefulSet, isUpdate bool, reasons []string) {
 	if isUpdate {
-		c.logger.Infof("statefulset %q has been changed", util.NameFromMeta(old.ObjectMeta))
+		c.logger.Infof("statefulset %s has been changed", util.NameFromMeta(old.ObjectMeta))
 	} else {
-		c.logger.Infof("statefulset %q is not in the desired state and needs to be updated",
+		c.logger.Infof("statefulset %s is not in the desired state and needs to be updated",
 			util.NameFromMeta(old.ObjectMeta),
 		)
 	}
+
+	logNiceDiff(c.logger, old.Spec, new.Spec)
+
 	if !reflect.DeepEqual(old.Annotations, new.Annotations) {
-		c.logger.Debugf("metadata.annotation diff\n%s\n", util.PrettyDiff(old.Annotations, new.Annotations))
+		c.logger.Debugf("metadata.annotation are different")
+		logNiceDiff(c.logger, old.Annotations, new.Annotations)
 	}
-	c.logger.Debugf("spec diff between old and new statefulsets: \n%s\n", util.PrettyDiff(old.Spec, new.Spec))
 
 	if len(reasons) > 0 {
 		for _, reason := range reasons {
-			c.logger.Infof("reason: %q", reason)
+			c.logger.Infof("reason: %s", reason)
 		}
 	}
 }
 
 func (c *Cluster) logServiceChanges(role PostgresRole, old, new *v1.Service, isUpdate bool, reason string) {
 	if isUpdate {
-		c.logger.Infof("%s service %q has been changed",
+		c.logger.Infof("%s service %s has been changed",
 			role, util.NameFromMeta(old.ObjectMeta),
 		)
 	} else {
-		c.logger.Infof("%s service %q is not in the desired state and needs to be updated",
+		c.logger.Infof("%s service %s is not in the desired state and needs to be updated",
 			role, util.NameFromMeta(old.ObjectMeta),
 		)
 	}
-	c.logger.Debugf("diff\n%s\n", util.PrettyDiff(old.Spec, new.Spec))
+
+	logNiceDiff(c.logger, old.Spec, new.Spec)
 
 	if reason != "" {
 		c.logger.Infof("reason: %s", reason)
@@ -208,7 +229,7 @@ func (c *Cluster) logServiceChanges(role PostgresRole, old, new *v1.Service, isU
 
 func (c *Cluster) logVolumeChanges(old, new acidv1.Volume) {
 	c.logger.Infof("volume specification has been changed")
-	c.logger.Debugf("diff\n%s\n", util.PrettyDiff(old, new))
+	logNiceDiff(c.logger, old, new)
 }
 
 func (c *Cluster) getTeamMembers(teamID string) ([]string, error) {

pkg/controller/controller.go

@@ -1,9 +1,12 @@
 package controller
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"os"
+	"strings"
 	"sync"
 	"time"
 
@@ -73,6 +76,10 @@ func NewController(controllerConfig *spec.ControllerConfig, controllerId string)
 	logger := logrus.New()
 	if controllerConfig.EnableJsonLogging {
 		logger.SetFormatter(&logrus.JSONFormatter{})
+	} else {
+		if os.Getenv("LOG_NOQUOTE") != "" {
+			logger.SetFormatter(&logrus.TextFormatter{PadLevelText: true, DisableQuote: true})
+		}
 	}
 
 	var myComponentName = "postgres-operator"
@@ -81,7 +88,10 @@ func NewController(controllerConfig *spec.ControllerConfig, controllerId string)
 	}
 
 	eventBroadcaster := record.NewBroadcaster()
-	eventBroadcaster.StartLogging(logger.Infof)
+
+	// disabling the sending of events also to the logoutput
+	// the operator currently duplicates a lot of log entries with this setup
+	// eventBroadcaster.StartLogging(logger.Infof)
 	recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: myComponentName})
 
 	c := &Controller{
@@ -190,10 +200,18 @@ func (c *Controller) warnOnDeprecatedOperatorParameters() {
 	}
 }
 
+func compactValue(v string) string {
+	var compact bytes.Buffer
+	if err := json.Compact(&compact, []byte(v)); err != nil {
+		panic("Hard coded json strings broken!")
+	}
+	return compact.String()
+}
+
 func (c *Controller) initPodServiceAccount() {
 
 	if c.opConfig.PodServiceAccountDefinition == "" {
-		c.opConfig.PodServiceAccountDefinition = `
+		stringValue := `
 		{
 			"apiVersion": "v1",
 			"kind": "ServiceAccount",
@@ -201,6 +219,9 @@ func (c *Controller) initPodServiceAccount() {
 				"name": "postgres-pod"
 			}
 		}`
+
+		c.opConfig.PodServiceAccountDefinition = compactValue(stringValue)
+
 	}
 
 	// re-uses k8s internal parsing. See k8s client-go issue #193 for explanation
@@ -230,7 +251,7 @@ func (c *Controller) initRoleBinding() {
 	// operator binds it to the cluster role with sufficient privileges
 	// we assume the role is created by the k8s administrator
 	if c.opConfig.PodServiceAccountRoleBindingDefinition == "" {
-		c.opConfig.PodServiceAccountRoleBindingDefinition = fmt.Sprintf(`
+		stringValue := fmt.Sprintf(`
 		{
 			"apiVersion": "rbac.authorization.k8s.io/v1",
 			"kind": "RoleBinding",
@@ -249,6 +270,7 @@ func (c *Controller) initRoleBinding() {
 				}
 			]
 		}`, c.PodServiceAccount.Name, c.PodServiceAccount.Name, c.PodServiceAccount.Name)
+		c.opConfig.PodServiceAccountRoleBindingDefinition = compactValue(stringValue)
 	}
 	c.logger.Info("Parse role bindings")
 	// re-uses k8s internal parsing. See k8s client-go issue #193 for explanation
@@ -267,7 +289,14 @@ func (c *Controller) initRoleBinding() {
 
 	}
 
-	// actual roles bindings are deployed at the time of Postgres/Spilo cluster creation
+	// actual roles bindings ar*logrus.Entrye deployed at the time of Postgres/Spilo cluster creation
+}
+
+func logMultiLineConfig(log *logrus.Entry, config string) {
+	lines := strings.Split(config, "\n")
+	for _, l := range lines {
+		log.Infof("%s", l)
+	}
 }
 
 func (c *Controller) initController() {
@@ -301,7 +330,7 @@ func (c *Controller) initController() {
 		c.logger.Logger.Level = logrus.DebugLevel
 	}
 
-	c.logger.Infof("config: %s", c.opConfig.MustMarshal())
+	logMultiLineConfig(c.logger, c.opConfig.MustMarshal())
 
 	roleDefs := c.getInfrastructureRoleDefinitions()
 	if infraRoles, err := c.getInfrastructureRoles(roleDefs); err != nil {

pkg/controller/node.go

@@ -42,7 +42,7 @@ func (c *Controller) nodeAdd(obj interface{}) {
 		return
 	}
 
-	c.logger.Debugf("new node has been added: %q (%s)", util.NameFromMeta(node.ObjectMeta), node.Spec.ProviderID)
+	c.logger.Debugf("new node has been added: %s (%s)", util.NameFromMeta(node.ObjectMeta), node.Spec.ProviderID)
 
 	// check if the node became not ready while the operator was down (otherwise we would have caught it in nodeUpdate)
 	if !c.nodeIsReady(node) {

pkg/controller/postgresql.go

@@ -225,7 +225,7 @@ func (c *Controller) processEvent(event ClusterEvent) {
 	switch event.EventType {
 	case EventAdd:
 		if clusterFound {
-			lg.Debugf("cluster already exists")
+			lg.Debugf("Recieved add event for existing cluster")
 			return
 		}
 

pkg/util/config/config.go

@@ -199,7 +199,7 @@ type Config struct {
 
 // MustMarshal marshals the config or panics
 func (c Config) MustMarshal() string {
-	b, err := json.MarshalIndent(c, "", "\t")
+	b, err := json.MarshalIndent(c, "", "   ")
 	if err != nil {
 		panic(err)
 	}

pkg/util/nicediff/diff.go

@@ -0,0 +1,191 @@
+// Copyright 2013 Google Inc.  All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package diff implements a linewise diff algorithm.
+package nicediff
+
+import (
+	"fmt"
+	"strings"
+)
+
+// Chunk represents a piece of the diff.  A chunk will not have both added and
+// deleted lines.  Equal lines are always after any added or deleted lines.
+// A Chunk may or may not have any lines in it, especially for the first or last
+// chunk in a computation.
+type Chunk struct {
+	Added   []string
+	Deleted []string
+	Equal   []string
+}
+
+func (c *Chunk) empty() bool {
+	return len(c.Added) == 0 && len(c.Deleted) == 0 && len(c.Equal) == 0
+}
+
+// Diff returns a string containing a line-by-line unified diff of the linewise
+// changes required to make A into B.  Each line is prefixed with '+', '-', or
+// ' ' to indicate if it should be added, removed, or is correct respectively.
+func Diff(A, B string, skipEqual bool) string {
+	aLines := strings.Split(A, "\n")
+	bLines := strings.Split(B, "\n")
+	return Render(DiffChunks(aLines, bLines), skipEqual)
+}
+
+// Render renders the slice of chunks into a representation that prefixes
+// the lines with '+', '-', or ' ' depending on whether the line was added,
+// removed, or equal (respectively).
+func Render(chunks []Chunk, skipEqual bool) string {
+	buf := new(strings.Builder)
+	for _, c := range chunks {
+		for _, line := range c.Added {
+			fmt.Fprintf(buf, "+%s\n", line)
+		}
+		for _, line := range c.Deleted {
+			fmt.Fprintf(buf, "-%s\n", line)
+		}
+		if !skipEqual {
+			for _, line := range c.Equal {
+				fmt.Fprintf(buf, " %s\n", line)
+			}
+		}
+	}
+	return strings.TrimRight(buf.String(), "\n")
+}
+
+// DiffChunks uses an O(D(N+M)) shortest-edit-script algorithm
+// to compute the edits required from A to B and returns the
+// edit chunks.
+func DiffChunks(a, b []string) []Chunk {
+	// algorithm: http://www.xmailserver.org/diff2.pdf
+
+	// We'll need these quantities a lot.
+	alen, blen := len(a), len(b) // M, N
+
+	// At most, it will require len(a) deletions and len(b) additions
+	// to transform a into b.
+	maxPath := alen + blen // MAX
+	if maxPath == 0 {
+		// degenerate case: two empty lists are the same
+		return nil
+	}
+
+	// Store the endpoint of the path for diagonals.
+	// We store only the a index, because the b index on any diagonal
+	// (which we know during the loop below) is aidx-diag.
+	// endpoint[maxPath] represents the 0 diagonal.
+	//
+	// Stated differently:
+	// endpoint[d] contains the aidx of a furthest reaching path in diagonal d
+	endpoint := make([]int, 2*maxPath+1) // V
+
+	saved := make([][]int, 0, 8) // Vs
+	save := func() {
+		dup := make([]int, len(endpoint))
+		copy(dup, endpoint)
+		saved = append(saved, dup)
+	}
+
+	var editDistance int // D
+dLoop:
+	for editDistance = 0; editDistance <= maxPath; editDistance++ {
+		// The 0 diag(onal) represents equality of a and b.  Each diagonal to
+		// the left is numbered one lower, to the right is one higher, from
+		// -alen to +blen.  Negative diagonals favor differences from a,
+		// positive diagonals favor differences from b.  The edit distance to a
+		// diagonal d cannot be shorter than d itself.
+		//
+		// The iterations of this loop cover either odds or evens, but not both,
+		// If odd indices are inputs, even indices are outputs and vice versa.
+		for diag := -editDistance; diag <= editDistance; diag += 2 { // k
+			var aidx int // x
+			switch {
+			case diag == -editDistance:
+				// This is a new diagonal; copy from previous iter
+				aidx = endpoint[maxPath-editDistance+1] + 0
+			case diag == editDistance:
+				// This is a new diagonal; copy from previous iter
+				aidx = endpoint[maxPath+editDistance-1] + 1
+			case endpoint[maxPath+diag+1] > endpoint[maxPath+diag-1]:
+				// diagonal d+1 was farther along, so use that
+				aidx = endpoint[maxPath+diag+1] + 0
+			default:
+				// diagonal d-1 was farther (or the same), so use that
+				aidx = endpoint[maxPath+diag-1] + 1
+			}
+			// On diagonal d, we can compute bidx from aidx.
+			bidx := aidx - diag // y
+			// See how far we can go on this diagonal before we find a difference.
+			for aidx < alen && bidx < blen && a[aidx] == b[bidx] {
+				aidx++
+				bidx++
+			}
+			// Store the end of the current edit chain.
+			endpoint[maxPath+diag] = aidx
+			// If we've found the end of both inputs, we're done!
+			if aidx >= alen && bidx >= blen {
+				save() // save the final path
+				break dLoop
+			}
+		}
+		save() // save the current path
+	}
+	if editDistance == 0 {
+		return nil
+	}
+	chunks := make([]Chunk, editDistance+1)
+
+	x, y := alen, blen
+	for d := editDistance; d > 0; d-- {
+		endpoint := saved[d]
+		diag := x - y
+		insert := diag == -d || (diag != d && endpoint[maxPath+diag-1] < endpoint[maxPath+diag+1])
+
+		x1 := endpoint[maxPath+diag]
+		var x0, xM, kk int
+		if insert {
+			kk = diag + 1
+			x0 = endpoint[maxPath+kk]
+			xM = x0
+		} else {
+			kk = diag - 1
+			x0 = endpoint[maxPath+kk]
+			xM = x0 + 1
+		}
+		y0 := x0 - kk
+
+		var c Chunk
+		if insert {
+			c.Added = b[y0:][:1]
+		} else {
+			c.Deleted = a[x0:][:1]
+		}
+		if xM < x1 {
+			c.Equal = a[xM:][:x1-xM]
+		}
+
+		x, y = x0, y0
+		chunks[d] = c
+	}
+	if x > 0 {
+		chunks[0].Equal = a[:x]
+	}
+	if chunks[0].empty() {
+		chunks = chunks[1:]
+	}
+	if len(chunks) == 0 {
+		return nil
+	}
+	return chunks
+}

pkg/util/util_test.go

@@ -180,3 +180,13 @@ func TestIsSmallerQuantity(t *testing.T) {
 		}
 	}
 }
+
+/*
+func TestNiceDiff(t *testing.T) {
+	o := "a\nb\nc\n"
+	n := "b\nd\n"
+	d := nicediff.Diff(o, n, true)
+	t.Log(d)
+	// t.Errorf("Lets see output")
+}
+*/