allow delete only if annotations meet configured criteria (#1069)

* define annotations for delete protection

* change log level and reduce log lines for e2e tests

* reduce wait_for_pod_start even further

e2e/requirements.txt

@@ -1,3 +1,3 @@
-kubernetes==9.0.0
+kubernetes==11.0.0
 timeout_decorator==0.4.1
-pyyaml==5.1
+pyyaml==5.3.1

e2e/tests/test_e2e.py

@@ -7,6 +7,7 @@ import warnings
 import os
 import yaml
 
+from datetime import datetime
 from kubernetes import client, config
 
 
@@ -614,6 +615,71 @@ class EndToEndTestCase(unittest.TestCase):
             "Origin": 2,
         })
 
+    @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
+    def test_x_cluster_deletion(self):
+        '''
+           Test deletion with configured protection
+        '''
+        k8s = self.k8s
+        cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster'
+
+        # configure delete protection
+        patch_delete_annotations = {
+            "data": {
+                "delete_annotation_date_key": "delete-date",
+                "delete_annotation_name_key": "delete-clustername"
+            }
+        }
+        k8s.update_config(patch_delete_annotations)
+
+        # this delete attempt should be omitted because of missing annotations
+        k8s.api.custom_objects_api.delete_namespaced_custom_object(
+            "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster")
+
+        # check that pods and services are still there
+        k8s.wait_for_running_pods(cluster_label, 2)
+        k8s.wait_for_service(cluster_label)
+
+        # recreate Postgres cluster resource
+        k8s.create_with_kubectl("manifests/minimal-postgres-manifest.yaml")
+
+        # wait a little before proceeding
+        time.sleep(10)
+
+        # add annotations to manifest
+        deleteDate = datetime.today().strftime('%Y-%m-%d')
+        pg_patch_delete_annotations = {
+            "metadata": {
+                "annotations": {
+                    "delete-date": deleteDate,
+                    "delete-clustername": "acid-minimal-cluster",
+                }
+            }
+        }
+        k8s.api.custom_objects_api.patch_namespaced_custom_object(
+            "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_delete_annotations)
+
+        # wait a little before proceeding
+        time.sleep(10)
+        k8s.wait_for_running_pods(cluster_label, 2)
+        k8s.wait_for_service(cluster_label)
+
+        # now delete process should be triggered
+        k8s.api.custom_objects_api.delete_namespaced_custom_object(
+            "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster")
+
+        # wait until cluster is deleted
+        time.sleep(120)
+
+        # check if everything has been deleted
+        self.assertEqual(0, k8s.count_pods_with_label(cluster_label))
+        self.assertEqual(0, k8s.count_services_with_label(cluster_label))
+        self.assertEqual(0, k8s.count_endpoints_with_label(cluster_label))
+        self.assertEqual(0, k8s.count_statefulsets_with_label(cluster_label))
+        self.assertEqual(0, k8s.count_deployments_with_label(cluster_label))
+        self.assertEqual(0, k8s.count_pdbs_with_label(cluster_label))
+        self.assertEqual(0, k8s.count_secrets_with_label(cluster_label))
+
     def get_failover_targets(self, master_node, replica_nodes):
         '''
            If all pods live on the same node, failover will happen to other worker(s)
@@ -700,11 +766,12 @@ class K8sApi:
         self.apps_v1 = client.AppsV1Api()
         self.batch_v1_beta1 = client.BatchV1beta1Api()
         self.custom_objects_api = client.CustomObjectsApi()
+        self.policy_v1_beta1 = client.PolicyV1beta1Api()
 
 
 class K8s:
     '''
-    Wraps around K8 api client and helper methods.
+    Wraps around K8s api client and helper methods.
     '''
 
     RETRY_TIMEOUT_SEC = 10
@@ -755,14 +822,6 @@ class K8s:
             if pods:
                 pod_phase = pods[0].status.phase
 
-            if pods and pod_phase != 'Running':
-                pod_name = pods[0].metadata.name
-                response = self.api.core_v1.read_namespaced_pod(
-                    name=pod_name,
-                    namespace=namespace
-                )
-                print("Pod description {}".format(response))
-
             time.sleep(self.RETRY_TIMEOUT_SEC)
 
     def get_service_type(self, svc_labels, namespace='default'):
@@ -824,6 +883,25 @@ class K8s:
     def count_pods_with_label(self, labels, namespace='default'):
         return len(self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items)
 
+    def count_services_with_label(self, labels, namespace='default'):
+        return len(self.api.core_v1.list_namespaced_service(namespace, label_selector=labels).items)
+
+    def count_endpoints_with_label(self, labels, namespace='default'):
+        return len(self.api.core_v1.list_namespaced_endpoints(namespace, label_selector=labels).items)
+
+    def count_secrets_with_label(self, labels, namespace='default'):
+        return len(self.api.core_v1.list_namespaced_secret(namespace, label_selector=labels).items)
+
+    def count_statefulsets_with_label(self, labels, namespace='default'):
+        return len(self.api.apps_v1.list_namespaced_stateful_set(namespace, label_selector=labels).items)
+
+    def count_deployments_with_label(self, labels, namespace='default'):
+        return len(self.api.apps_v1.list_namespaced_deployment(namespace, label_selector=labels).items)
+
+    def count_pdbs_with_label(self, labels, namespace='default'):
+        return len(self.api.policy_v1_beta1.list_namespaced_pod_disruption_budget(
+            namespace, label_selector=labels).items)
+
     def wait_for_pod_failover(self, failover_targets, labels, namespace='default'):
         pod_phase = 'Failing over'
         new_pod_node = ''