public_git
/
postgres-operator
mirror of https://github.com/zalando/postgres-operator.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Update e2e pipeline (#1202) 

* clean up after test_multi_namespace test

* see the PR description for complete list of changes

Co-authored-by: Sergey Dudoladov <sergey.dudoladov@zalando.de>
This commit is contained in:
Sergey Dudoladov 2020-11-11 10:21:46 +01:00 committed by GitHub
parent b379db20ed
commit e779eab22f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 70 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
e2e/README.md
View File

@ -56,12 +56,24 @@ NOCLEANUP=True ./run.sh main tests.test_e2e.EndToEndTestCase.test_lazy_spilo_upg
## Inspecting Kind ## Inspecting Kind
If you want to inspect Kind/Kubernetes cluster, use the following script to exec into the K8s setup and then use `kubectl` If you want to inspect Kind/Kubernetes cluster, switch `kubeconfig` file and context
```bash
# save the old config in case you have it
export KUBECONFIG_SAVED=$KUBECONFIG
# use the one created by e2e tests
export KUBECONFIG=/tmp/kind-config-postgres-operator-e2e-tests
# this kubeconfig defines a single context
kubectl config use-context kind-postgres-operator-e2e-tests
```
or use the following script to exec into the K8s setup and then use `kubectl`
```bash ```bash
./exec_into_env.sh ./exec_into_env.sh
# use kube ctl # use kubectl
kubectl get pods kubectl get pods
# watch relevant objects # watch relevant objects
@ -71,6 +83,14 @@ kubectl get pods
./scripts/get_logs.sh ./scripts/get_logs.sh
``` ```
If you want to inspect the state of the `kind` cluster manually with a single command, add a `context` flag
```bash
kubectl get pods --context kind-kind
```
or set the context for a few commands at once
## Cleaning up Kind ## Cleaning up Kind
To cleanup kind and start fresh To cleanup kind and start fresh
@ -79,6 +99,12 @@ To cleanup kind and start fresh
e2e/run.sh cleanup e2e/run.sh cleanup
``` ```
That also helps in case you see the
```
ERROR: no nodes found for cluster "postgres-operator-e2e-tests"
```
that happens when the `kind` cluster was deleted manually but its configuraiton file was not.
## Covered use cases ## Covered use cases
The current tests are all bundled in [`test_e2e.py`](tests/test_e2e.py): The current tests are all bundled in [`test_e2e.py`](tests/test_e2e.py):

24
e2e/tests/k8s_api.py
View File

@ -11,9 +11,11 @@ from datetime import datetime
from kubernetes import client, config from kubernetes import client, config
from kubernetes.client.rest import ApiException from kubernetes.client.rest import ApiException
def to_selector(labels): def to_selector(labels):
return ",".join(["=".join(l) for l in labels.items()]) return ",".join(["=".join(l) for l in labels.items()])
class K8sApi: class K8sApi:
def __init__(self): def __init__(self):
@ -181,10 +183,10 @@ class K8s:
def count_pdbs_with_label(self, labels, namespace='default'): def count_pdbs_with_label(self, labels, namespace='default'):
return len(self.api.policy_v1_beta1.list_namespaced_pod_disruption_budget( return len(self.api.policy_v1_beta1.list_namespaced_pod_disruption_budget(
namespace, label_selector=labels).items) namespace, label_selector=labels).items)
def count_running_pods(self, labels='application=spilo,cluster-name=acid-minimal-cluster', namespace='default'): def count_running_pods(self, labels='application=spilo,cluster-name=acid-minimal-cluster', namespace='default'):
pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items
return len(list(filter(lambda x: x.status.phase=='Running', pods))) return len(list(filter(lambda x: x.status.phase == 'Running', pods)))
def wait_for_pod_failover(self, failover_targets, labels, namespace='default'): def wait_for_pod_failover(self, failover_targets, labels, namespace='default'):
pod_phase = 'Failing over' pod_phase = 'Failing over'
@ -210,9 +212,9 @@ class K8s:
def wait_for_logical_backup_job_creation(self): def wait_for_logical_backup_job_creation(self):
self.wait_for_logical_backup_job(expected_num_of_jobs=1) self.wait_for_logical_backup_job(expected_num_of_jobs=1)
def delete_operator_pod(self, step="Delete operator deplyment"): def delete_operator_pod(self, step="Delete operator pod"):
operator_pod = self.api.core_v1.list_namespaced_pod('default', label_selector="name=postgres-operator").items[0].metadata.name # patching the pod template in the deployment restarts the operator pod
self.api.apps_v1.patch_namespaced_deployment("postgres-operator","default", {"spec":{"template":{"metadata":{"annotations":{"step":"{}-{}".format(step, time.time())}}}}}) self.api.apps_v1.patch_namespaced_deployment("postgres-operator","default", {"spec":{"template":{"metadata":{"annotations":{"step":"{}-{}".format(step, datetime.fromtimestamp(time.time()))}}}}})
self.wait_for_operator_pod_start() self.wait_for_operator_pod_start()
def update_config(self, config_map_patch, step="Updating operator deployment"): def update_config(self, config_map_patch, step="Updating operator deployment"):
@ -241,7 +243,7 @@ class K8s:
def get_operator_state(self): def get_operator_state(self):
pod = self.get_operator_pod() pod = self.get_operator_pod()
if pod == None: if pod is None:
return None return None
pod = pod.metadata.name pod = pod.metadata.name
@ -251,7 +253,6 @@ class K8s:
return json.loads(r.stdout.decode()) return json.loads(r.stdout.decode())
def get_patroni_running_members(self, pod="acid-minimal-cluster-0"): def get_patroni_running_members(self, pod="acid-minimal-cluster-0"):
result = self.get_patroni_state(pod) result = self.get_patroni_state(pod)
return list(filter(lambda x: "State" in x and x["State"] == "running", result)) return list(filter(lambda x: "State" in x and x["State"] == "running", result))
@ -260,9 +261,9 @@ class K8s:
try: try:
deployment = self.api.apps_v1.read_namespaced_deployment(name, namespace) deployment = self.api.apps_v1.read_namespaced_deployment(name, namespace)
return deployment.spec.replicas return deployment.spec.replicas
except ApiException as e: except ApiException:
return None return None
def get_statefulset_image(self, label_selector="application=spilo,cluster-name=acid-minimal-cluster", namespace='default'): def get_statefulset_image(self, label_selector="application=spilo,cluster-name=acid-minimal-cluster", namespace='default'):
ssets = self.api.apps_v1.list_namespaced_stateful_set(namespace, label_selector=label_selector, limit=1) ssets = self.api.apps_v1.list_namespaced_stateful_set(namespace, label_selector=label_selector, limit=1)
if len(ssets.items) == 0: if len(ssets.items) == 0:
@ -463,7 +464,6 @@ class K8sBase:
self.wait_for_logical_backup_job(expected_num_of_jobs=1) self.wait_for_logical_backup_job(expected_num_of_jobs=1)
def delete_operator_pod(self, step="Delete operator deplyment"): def delete_operator_pod(self, step="Delete operator deplyment"):
operator_pod = self.api.core_v1.list_namespaced_pod('default', label_selector="name=postgres-operator").items[0].metadata.name
self.api.apps_v1.patch_namespaced_deployment("postgres-operator","default", {"spec":{"template":{"metadata":{"annotations":{"step":"{}-{}".format(step, time.time())}}}}}) self.api.apps_v1.patch_namespaced_deployment("postgres-operator","default", {"spec":{"template":{"metadata":{"annotations":{"step":"{}-{}".format(step, time.time())}}}}})
self.wait_for_operator_pod_start() self.wait_for_operator_pod_start()
@ -521,7 +521,7 @@ class K8sOperator(K8sBase):
class K8sPostgres(K8sBase): class K8sPostgres(K8sBase):
def __init__(self, labels="cluster-name=acid-minimal-cluster", namespace="default"): def __init__(self, labels="cluster-name=acid-minimal-cluster", namespace="default"):
super().__init__(labels, namespace) super().__init__(labels, namespace)
def get_pg_nodes(self): def get_pg_nodes(self):
master_pod_node = '' master_pod_node = ''
replica_pod_nodes = [] replica_pod_nodes = []
@ -532,4 +532,4 @@ class K8sPostgres(K8sBase):
elif pod.metadata.labels.get('spilo-role') == 'replica': elif pod.metadata.labels.get('spilo-role') == 'replica':
replica_pod_nodes.append(pod.spec.node_name) replica_pod_nodes.append(pod.spec.node_name)
return master_pod_node, replica_pod_nodes return master_pod_node, replica_pod_nodes

54
e2e/tests/test_e2e.py
View File

@ -2,15 +2,14 @@ import json
import unittest import unittest
import time import time
import timeout_decorator import timeout_decorator
import subprocess
import warnings
import os import os
import yaml import yaml
from datetime import datetime from datetime import datetime
from kubernetes import client, config from kubernetes import client
from tests.k8s_api import K8s from tests.k8s_api import K8s
from kubernetes.client.rest import ApiException
SPILO_CURRENT = "registry.opensource.zalan.do/acid/spilo-12:1.6-p5" SPILO_CURRENT = "registry.opensource.zalan.do/acid/spilo-12:1.6-p5"
SPILO_LAZY = "registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p114" SPILO_LAZY = "registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p114"
@ -89,17 +88,17 @@ class EndToEndTestCase(unittest.TestCase):
# remove existing local storage class and create hostpath class # remove existing local storage class and create hostpath class
try: try:
k8s.api.storage_v1_api.delete_storage_class("standard") k8s.api.storage_v1_api.delete_storage_class("standard")
except: except ApiException as e:
print("Storage class has already been remove") print("Failed to delete the 'standard' storage class: {0}".format(e))
# operator deploys pod service account there on start up # operator deploys pod service account there on start up
# needed for test_multi_namespace_support() # needed for test_multi_namespace_support()
cls.namespace = "test" cls.test_namespace = "test"
try: try:
v1_namespace = client.V1Namespace(metadata=client.V1ObjectMeta(name=cls.namespace)) v1_namespace = client.V1Namespace(metadata=client.V1ObjectMeta(name=cls.test_namespace))
k8s.api.core_v1.create_namespace(v1_namespace) k8s.api.core_v1.create_namespace(v1_namespace)
except: except ApiException as e:
print("Namespace already present") print("Failed to create the '{0}' namespace: {1}".format(cls.test_namespace, e))
# submit the most recent operator image built on the Docker host # submit the most recent operator image built on the Docker host
with open("manifests/postgres-operator.yaml", 'r+') as f: with open("manifests/postgres-operator.yaml", 'r+') as f:
@ -135,10 +134,8 @@ class EndToEndTestCase(unittest.TestCase):
# make sure we start a new operator on every new run, # make sure we start a new operator on every new run,
# this tackles the problem when kind is reused # this tackles the problem when kind is reused
# and the Docker image is infact changed (dirty one) # and the Docker image is in fact changed (dirty one)
# patch resync period, this can catch some problems with hanging e2e tests
# k8s.update_config({"data": {"resync_period":"30s"}},step="TestSuite setup")
k8s.update_config({}, step="TestSuite Startup") k8s.update_config({}, step="TestSuite Startup")
actual_operator_image = k8s.api.core_v1.list_namespaced_pod( actual_operator_image = k8s.api.core_v1.list_namespaced_pod(
@ -170,9 +167,6 @@ class EndToEndTestCase(unittest.TestCase):
'connection-pooler': 'acid-minimal-cluster-pooler', 'connection-pooler': 'acid-minimal-cluster-pooler',
}) })
pod_selector = to_selector(pod_labels)
service_selector = to_selector(service_labels)
# enable connection pooler # enable connection pooler
k8s.api.custom_objects_api.patch_namespaced_custom_object( k8s.api.custom_objects_api.patch_namespaced_custom_object(
'acid.zalan.do', 'v1', 'default', 'acid.zalan.do', 'v1', 'default',
@ -347,7 +341,7 @@ class EndToEndTestCase(unittest.TestCase):
}, },
} }
k8s.update_config(patch_infrastructure_roles) k8s.update_config(patch_infrastructure_roles)
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0":"idle"}, "Operator does not get in sync") self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
try: try:
# check that new roles are represented in the config by requesting the # check that new roles are represented in the config by requesting the
@ -604,17 +598,25 @@ class EndToEndTestCase(unittest.TestCase):
with open("manifests/complete-postgres-manifest.yaml", 'r+') as f: with open("manifests/complete-postgres-manifest.yaml", 'r+') as f:
pg_manifest = yaml.safe_load(f) pg_manifest = yaml.safe_load(f)
pg_manifest["metadata"]["namespace"] = self.namespace pg_manifest["metadata"]["namespace"] = self.test_namespace
yaml.dump(pg_manifest, f, Dumper=yaml.Dumper) yaml.dump(pg_manifest, f, Dumper=yaml.Dumper)
try: try:
k8s.create_with_kubectl("manifests/complete-postgres-manifest.yaml") k8s.create_with_kubectl("manifests/complete-postgres-manifest.yaml")
k8s.wait_for_pod_start("spilo-role=master", self.namespace) k8s.wait_for_pod_start("spilo-role=master", self.test_namespace)
self.assert_master_is_unique(self.namespace, "acid-test-cluster") self.assert_master_is_unique(self.test_namespace, "acid-test-cluster")
except timeout_decorator.TimeoutError: except timeout_decorator.TimeoutError:
print('Operator log: {}'.format(k8s.get_operator_log())) print('Operator log: {}'.format(k8s.get_operator_log()))
raise raise
finally:
# delete the new cluster so that the k8s_api.get_operator_state works correctly in subsequent tests
# ideally we should delete the 'test' namespace here but
# the pods inside the namespace stuck in the Terminating state making the test time out
k8s.api.custom_objects_api.delete_namespaced_custom_object(
"acid.zalan.do", "v1", self.test_namespace, "postgresqls", "acid-test-cluster")
time.sleep(5)
@timeout_decorator.timeout(TEST_TIMEOUT_SEC) @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
def test_zz_node_readiness_label(self): def test_zz_node_readiness_label(self):
@ -746,12 +748,12 @@ class EndToEndTestCase(unittest.TestCase):
} }
k8s.api.custom_objects_api.patch_namespaced_custom_object( k8s.api.custom_objects_api.patch_namespaced_custom_object(
"acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_crd_annotations) "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_crd_annotations)
annotations = { annotations = {
"deployment-time": "2020-04-30 12:00:00", "deployment-time": "2020-04-30 12:00:00",
"downscaler/downtime_replicas": "0", "downscaler/downtime_replicas": "0",
} }
self.eventuallyTrue(lambda: k8s.check_statefulset_annotations(cluster_label, annotations), "Annotations missing") self.eventuallyTrue(lambda: k8s.check_statefulset_annotations(cluster_label, annotations), "Annotations missing")
@ -823,14 +825,14 @@ class EndToEndTestCase(unittest.TestCase):
} }
} }
k8s.update_config(patch_delete_annotations) k8s.update_config(patch_delete_annotations)
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0":"idle"}, "Operator does not get in sync") self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
try: try:
# this delete attempt should be omitted because of missing annotations # this delete attempt should be omitted because of missing annotations
k8s.api.custom_objects_api.delete_namespaced_custom_object( k8s.api.custom_objects_api.delete_namespaced_custom_object(
"acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster") "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster")
time.sleep(5) time.sleep(5)
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0":"idle"}, "Operator does not get in sync") self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
# check that pods and services are still there # check that pods and services are still there
k8s.wait_for_running_pods(cluster_label, 2) k8s.wait_for_running_pods(cluster_label, 2)
@ -841,7 +843,7 @@ class EndToEndTestCase(unittest.TestCase):
# wait a little before proceeding # wait a little before proceeding
time.sleep(10) time.sleep(10)
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0":"idle"}, "Operator does not get in sync") self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
# add annotations to manifest # add annotations to manifest
delete_date = datetime.today().strftime('%Y-%m-%d') delete_date = datetime.today().strftime('%Y-%m-%d')
@ -855,7 +857,7 @@ class EndToEndTestCase(unittest.TestCase):
} }
k8s.api.custom_objects_api.patch_namespaced_custom_object( k8s.api.custom_objects_api.patch_namespaced_custom_object(
"acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_delete_annotations) "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_delete_annotations)
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0":"idle"}, "Operator does not get in sync") self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
# wait a little before proceeding # wait a little before proceeding
time.sleep(20) time.sleep(20)
@ -882,7 +884,7 @@ class EndToEndTestCase(unittest.TestCase):
print('Operator log: {}'.format(k8s.get_operator_log())) print('Operator log: {}'.format(k8s.get_operator_log()))
raise raise
#reset configmap # reset configmap
patch_delete_annotations = { patch_delete_annotations = {
"data": { "data": {
"delete_annotation_date_key": "", "delete_annotation_date_key": "",

4
pkg/cluster/cluster.go
View File

@ -626,14 +626,14 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
} }
}() }()
if oldSpec.Spec.PostgresqlParam.PgVersion >= newSpec.Spec.PostgresqlParam.PgVersion { if oldSpec.Spec.PostgresqlParam.PgVersion > newSpec.Spec.PostgresqlParam.PgVersion {
c.logger.Warningf("postgresql version change(%q -> %q) has no effect", c.logger.Warningf("postgresql version change(%q -> %q) has no effect",
oldSpec.Spec.PostgresqlParam.PgVersion, newSpec.Spec.PostgresqlParam.PgVersion) oldSpec.Spec.PostgresqlParam.PgVersion, newSpec.Spec.PostgresqlParam.PgVersion)
c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "PostgreSQL", "postgresql version change(%q -> %q) has no effect", c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "PostgreSQL", "postgresql version change(%q -> %q) has no effect",
oldSpec.Spec.PostgresqlParam.PgVersion, newSpec.Spec.PostgresqlParam.PgVersion) oldSpec.Spec.PostgresqlParam.PgVersion, newSpec.Spec.PostgresqlParam.PgVersion)
// we need that hack to generate statefulset with the old version // we need that hack to generate statefulset with the old version
newSpec.Spec.PostgresqlParam.PgVersion = oldSpec.Spec.PostgresqlParam.PgVersion newSpec.Spec.PostgresqlParam.PgVersion = oldSpec.Spec.PostgresqlParam.PgVersion
} else { } else if oldSpec.Spec.PostgresqlParam.PgVersion < newSpec.Spec.PostgresqlParam.PgVersion {
c.logger.Infof("postgresql version increased (%q -> %q), major version upgrade can be done manually after StatefulSet Sync", c.logger.Infof("postgresql version increased (%q -> %q), major version upgrade can be done manually after StatefulSet Sync",
oldSpec.Spec.PostgresqlParam.PgVersion, newSpec.Spec.PostgresqlParam.PgVersion) oldSpec.Spec.PostgresqlParam.PgVersion, newSpec.Spec.PostgresqlParam.PgVersion)
} }