merge with master and resolve conflicts

e2e/README.md

@@ -44,3 +44,4 @@ The current tests are all bundled in [`test_e2e.py`](tests/test_e2e.py):
 * taint-based eviction of Postgres pods
 * invoking logical backup cron job
 * uniqueness of master pod
+* custom service annotations

e2e/tests/test_e2e.py

@@ -10,9 +10,9 @@ from kubernetes import client, config
 
 
 class EndToEndTestCase(unittest.TestCase):
-    """
+    '''
     Test interaction of the operator with multiple K8s components.
-    """
+    '''
 
     # `kind` pods may stuck in the `Terminating` phase for a few minutes; hence high test timeout
     TEST_TIMEOUT_SEC = 600
@@ -20,14 +20,14 @@ class EndToEndTestCase(unittest.TestCase):
     @classmethod
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
     def setUpClass(cls):
-        """
+        '''
         Deploy operator to a "kind" cluster created by run.sh using examples from /manifests.
         This operator deployment is to be shared among all tests.
 
         run.sh deletes the 'kind' cluster after successful run along with all operator-related entities.
         In the case of test failure the cluster will stay to enable manual examination;
         next invocation of "make test" will re-create it.
-        """
+        '''
 
         # set a single K8s wrapper for all tests
         k8s = cls.k8s = K8s()
@@ -55,11 +55,113 @@ class EndToEndTestCase(unittest.TestCase):
         k8s.create_with_kubectl("manifests/minimal-postgres-manifest.yaml")
         k8s.wait_for_pod_start('spilo-role=master')
 
+    @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
+    def test_enable_load_balancer(self):
+        '''
+        Test if services are updated when enabling/disabling load balancers
+        '''
+
+        k8s = self.k8s
+        cluster_label = 'cluster-name=acid-minimal-cluster'
+
+        # enable load balancer services
+        pg_patch_enable_lbs = {
+            "spec": {
+                "enableMasterLoadBalancer": True,
+                "enableReplicaLoadBalancer": True
+            }
+        }
+        k8s.api.custom_objects_api.patch_namespaced_custom_object(
+            "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_lbs)
+        # wait for service recreation
+        time.sleep(60)
+
+        master_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=master')
+        self.assertEqual(master_svc_type, 'LoadBalancer',
+                         "Expected LoadBalancer service type for master, found {}".format(master_svc_type))
+
+        repl_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=replica')
+        self.assertEqual(repl_svc_type, 'LoadBalancer',
+                         "Expected LoadBalancer service type for replica, found {}".format(repl_svc_type))
+
+        # disable load balancer services again
+        pg_patch_disable_lbs = {
+            "spec": {
+                "enableMasterLoadBalancer": False,
+                "enableReplicaLoadBalancer": False
+            }
+        }
+        k8s.api.custom_objects_api.patch_namespaced_custom_object(
+            "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_lbs)
+        # wait for service recreation
+        time.sleep(60)
+
+        master_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=master')
+        self.assertEqual(master_svc_type, 'ClusterIP',
+                         "Expected ClusterIP service type for master, found {}".format(master_svc_type))
+
+        repl_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=replica')
+        self.assertEqual(repl_svc_type, 'ClusterIP',
+                         "Expected ClusterIP service type for replica, found {}".format(repl_svc_type))
+
+    @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
+    def test_min_resource_limits(self):
+        '''
+        Lower resource limits below configured minimum and let operator fix it
+        '''
+        k8s = self.k8s
+        cluster_label = 'cluster-name=acid-minimal-cluster'
+        _, failover_targets = k8s.get_pg_nodes(cluster_label)
+
+        # configure minimum boundaries for CPU and memory limits
+        minCPULimit = '500m'
+        minMemoryLimit = '500Mi'
+        patch_min_resource_limits = {
+            "configuration": {
+                "postgres_pod_resources": {
+                    "min_cpu_limit": minCPULimit,
+                    "min_memory_limit": minMemoryLimit
+                }
+            }
+        }
+        k8s.update_config(patch_min_resource_limits)
+
+        # lower resource limits below minimum
+        pg_patch_resources = {
+            "spec": {
+                "resources": {
+                    "requests": {
+                        "cpu": "10m",
+                        "memory": "50Mi"
+                    },
+                    "limits": {
+                        "cpu": "200m",
+                        "memory": "200Mi"
+                    }
+                }
+            }
+        }
+        k8s.api.custom_objects_api.patch_namespaced_custom_object(
+            "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_resources)
+        k8s.wait_for_master_failover(failover_targets)
+
+        pods = k8s.api.core_v1.list_namespaced_pod(
+            'default', label_selector='spilo-role=master,' + cluster_label).items
+        self.assert_master_is_unique()
+        masterPod = pods[0]
+
+        self.assertEqual(masterPod.spec.containers[0].resources.limits['cpu'], minCPULimit,
+                         "Expected CPU limit {}, found {}"
+                         .format(minCPULimit, masterPod.spec.containers[0].resources.limits['cpu']))
+        self.assertEqual(masterPod.spec.containers[0].resources.limits['memory'], minMemoryLimit,
+                         "Expected memory limit {}, found {}"
+                         .format(minMemoryLimit, masterPod.spec.containers[0].resources.limits['memory']))
+
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
     def test_multi_namespace_support(self):
-        """
+        '''
         Create a customized Postgres cluster in a non-default namespace.
-        """
+        '''
         k8s = self.k8s
 
         with open("manifests/complete-postgres-manifest.yaml", 'r+') as f:
@@ -69,14 +171,13 @@ class EndToEndTestCase(unittest.TestCase):
 
         k8s.create_with_kubectl("manifests/complete-postgres-manifest.yaml")
         k8s.wait_for_pod_start("spilo-role=master", self.namespace)
-        self.assert_master_is_unique(self.namespace, clusterName="acid-test-cluster")
+        self.assert_master_is_unique(self.namespace, "acid-test-cluster")
 
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
     def test_scaling(self):
-        """
-        Scale up from 2 to 3 and back to 2 pods by updating the Postgres manifest at runtime.
-        """
-
+        '''
+           Scale up from 2 to 3 and back to 2 pods by updating the Postgres manifest at runtime.
+        '''
         k8s = self.k8s
         labels = "cluster-name=acid-minimal-cluster"
 
@@ -90,9 +191,9 @@ class EndToEndTestCase(unittest.TestCase):
 
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
     def test_taint_based_eviction(self):
-        """
-        Add taint "postgres=:NoExecute" to node with master. This must cause a failover.
-        """
+        '''
+           Add taint "postgres=:NoExecute" to node with master. This must cause a failover.
+        '''
         k8s = self.k8s
         cluster_label = 'cluster-name=acid-minimal-cluster'
 
@@ -123,7 +224,7 @@ class EndToEndTestCase(unittest.TestCase):
         # patch node and test if master is failing over to one of the expected nodes
         k8s.api.core_v1.patch_node(current_master_node, body)
         k8s.wait_for_master_failover(failover_targets)
-        k8s.wait_for_pod_start('spilo-role=replica')
+        k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label)
 
         new_master_node, new_replica_nodes = k8s.get_pg_nodes(cluster_label)
         self.assertNotEqual(current_master_node, new_master_node,
@@ -142,7 +243,7 @@ class EndToEndTestCase(unittest.TestCase):
 
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
     def test_logical_backup_cron_job(self):
-        """
+        '''
         Ensure we can (a) create the cron job at user request for a specific PG cluster
                       (b) update the cluster-wide image for the logical backup pod
                       (c) delete the job at user request
@@ -150,7 +251,7 @@ class EndToEndTestCase(unittest.TestCase):
         Limitations:
         (a) Does not run the actual batch job because there is no S3 mock to upload backups to
         (b) Assumes 'acid-minimal-cluster' exists as defined in setUp
-        """
+        '''
 
         k8s = self.k8s
 
@@ -158,8 +259,8 @@ class EndToEndTestCase(unittest.TestCase):
         schedule = "7 7 7 7 *"
         pg_patch_enable_backup = {
             "spec": {
-               "enableLogicalBackup": True,
-               "logicalBackupSchedule": schedule
+                "enableLogicalBackup": True,
+                "logicalBackupSchedule": schedule
             }
         }
         k8s.api.custom_objects_api.patch_namespaced_custom_object(
@@ -196,7 +297,7 @@ class EndToEndTestCase(unittest.TestCase):
         # delete the logical backup cron job
         pg_patch_disable_backup = {
             "spec": {
-               "enableLogicalBackup": False,
+                "enableLogicalBackup": False,
             }
         }
         k8s.api.custom_objects_api.patch_namespaced_custom_object(
@@ -206,11 +307,55 @@ class EndToEndTestCase(unittest.TestCase):
         self.assertEqual(0, len(jobs),
                          "Expected 0 logical backup jobs, found {}".format(len(jobs)))
 
+    @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
+    def test_service_annotations(self):
+        '''
+        Create a Postgres cluster with service annotations and check them.
+        '''
+        k8s = self.k8s
+        patch_custom_service_annotations = {
+            "configuration": {
+                "load_balancer": {
+                    "custom_service_annotations": "foo:bar",
+                }
+            }
+        }
+        k8s.update_config(patch_custom_service_annotations)
+
+        pg_patch_custom_annotations = {
+            "spec": {
+                "serviceAnnotations": {
+                    "annotation.key": "value"
+                }
+            }
+        }
+        k8s.api.custom_objects_api.patch_namespaced_custom_object(
+            "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_custom_annotations)
+
+        annotations = {
+            "annotation.key": "value",
+            "foo": "bar",
+        }
+        self.assertTrue(k8s.check_service_annotations(
+            "cluster-name=acid-service-annotations,spilo-role=master", annotations))
+        self.assertTrue(k8s.check_service_annotations(
+            "cluster-name=acid-service-annotations,spilo-role=replica", annotations))
+
+        # clean up
+        unpatch_custom_service_annotations = {
+            "configuration": {
+                "load_balancer": {
+                    "custom_service_annotations": "",
+                }
+            }
+        }
+        k8s.update_config(unpatch_custom_service_annotations)
+
     def assert_master_is_unique(self, namespace='default', clusterName="acid-minimal-cluster"):
-        """
-        Check that there is a single pod in the k8s cluster with the label "spilo-role=master"
-        To be called manually after operations that affect pods
-        """
+        '''
+           Check that there is a single pod in the k8s cluster with the label "spilo-role=master"
+           To be called manually after operations that affect pods
+        '''
 
         k8s = self.k8s
         labels = 'spilo-role=master,cluster-name=' + clusterName
@@ -236,9 +381,9 @@ class K8sApi:
 
 
 class K8s:
-    """
+    '''
     Wraps around K8 api client and helper methods.
-    """
+    '''
 
     RETRY_TIMEOUT_SEC = 5
 
@@ -271,6 +416,23 @@ class K8s:
                 pod_phase = pods[0].status.phase
             time.sleep(self.RETRY_TIMEOUT_SEC)
 
+    def get_service_type(self, svc_labels, namespace='default'):
+        svc_type = ''
+        svcs = self.api.core_v1.list_namespaced_service(namespace, label_selector=svc_labels, limit=1).items
+        for svc in svcs:
+            svc_type = svc.spec.type
+        return svc_type
+
+    def check_service_annotations(self, svc_labels, annotations, namespace='default'):
+        svcs = self.api.core_v1.list_namespaced_service(namespace, label_selector=svc_labels, limit=1).items
+        for svc in svcs:
+            if len(svc.metadata.annotations) != len(annotations):
+                return False
+            for key in svc.metadata.annotations:
+                if svc.metadata.annotations[key] != annotations[key]:
+                    return False
+        return True
+
     def wait_for_pg_to_scale(self, number_of_instances, namespace='default'):
 
         body = {
@@ -279,7 +441,7 @@ class K8s:
             }
         }
         _ = self.api.custom_objects_api.patch_namespaced_custom_object(
-                    "acid.zalan.do", "v1", namespace, "postgresqls", "acid-minimal-cluster", body)
+            "acid.zalan.do", "v1", namespace, "postgresqls", "acid-minimal-cluster", body)
 
         labels = 'cluster-name=acid-minimal-cluster'
         while self.count_pods_with_label(labels) != number_of_instances: