Add ServiceAnnotations cluster config (#803)

The [operator parameters][1] already support the
`custom_service_annotations` config.With this parameter is possible to
define custom annotations that will be used on the services created by the
operator. The `custom_service_annotations` as all the other
[operator parameters][1] are defined on the operator level and do not allow
customization on the cluster level. A cluster may require different service
annotations, as for example, set up different cloud load balancers
timeouts, different ingress annotations, and/or enable more customizable
environments.

This commit introduces a new parameter on the cluster level, called
`serviceAnnotations`, responsible for defining custom annotations just for
the services created by the operator to the specifically defined cluster.
It allows a mix of configuration between `custom_service_annotations` and
`serviceAnnotations` where the latest one will have priority. In order to
allow custom service annotations to be used on services without
LoadBalancers (as for example, service mesh services annotations) both
`custom_service_annotations` and `serviceAnnotations` are applied
independently of load-balancing configuration. For retro-compatibility
purposes, `custom_service_annotations` is still under
[Load balancer related options][2]. The two default annotations when using
LoadBalancer services, `external-dns.alpha.kubernetes.io/hostname` and
`service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout` are
still defined by the operator.
`service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout` can
be overridden by `custom_service_annotations` or `serviceAnnotations`,
allowing a more customizable environment.
`external-dns.alpha.kubernetes.io/hostname` can not be overridden once
there is no differentiation between custom service annotations for
replicas and masters.

It updates the documentation and creates the necessary unit and e2e
tests to the above-described feature too.

[1]: https://github.com/zalando/postgres-operator/blob/master/docs/reference/operator_parameters.md
[2]: https://github.com/zalando/postgres-operator/blob/master/docs/reference/operator_parameters.md#load-balancer-related-options

e2e/README.md

@@ -44,3 +44,4 @@ The current tests are all bundled in [`test_e2e.py`](tests/test_e2e.py):
 * taint-based eviction of Postgres pods
 * invoking logical backup cron job
 * uniqueness of master pod
+* custom service annotations

e2e/tests/test_e2e.py

@@ -211,8 +211,8 @@ class EndToEndTestCase(unittest.TestCase):
         schedule = "7 7 7 7 *"
         pg_patch_enable_backup = {
             "spec": {
-               "enableLogicalBackup": True,
-               "logicalBackupSchedule": schedule
+                "enableLogicalBackup": True,
+                "logicalBackupSchedule": schedule
             }
         }
         k8s.api.custom_objects_api.patch_namespaced_custom_object(
@@ -234,7 +234,7 @@ class EndToEndTestCase(unittest.TestCase):
         image = "test-image-name"
         patch_logical_backup_image = {
             "data": {
-               "logical_backup_docker_image": image,
+                "logical_backup_docker_image": image,
             }
         }
         k8s.update_config(patch_logical_backup_image)
@@ -247,7 +247,7 @@ class EndToEndTestCase(unittest.TestCase):
         # delete the logical backup cron job
         pg_patch_disable_backup = {
             "spec": {
-               "enableLogicalBackup": False,
+                "enableLogicalBackup": False,
             }
         }
         k8s.api.custom_objects_api.patch_namespaced_custom_object(
@@ -257,6 +257,37 @@ class EndToEndTestCase(unittest.TestCase):
         self.assertEqual(0, len(jobs),
                          "Expected 0 logical backup jobs, found {}".format(len(jobs)))
 
+    @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
+    def test_service_annotations(self):
+        '''
+        Create a Postgres cluster with service annotations and check them.
+        '''
+        k8s = self.k8s
+        patch_custom_service_annotations = {
+            "data": {
+                "custom_service_annotations": "foo:bar",
+            }
+        }
+        k8s.update_config(patch_custom_service_annotations)
+
+        k8s.create_with_kubectl("manifests/postgres-manifest-with-service-annotations.yaml")
+        annotations = {
+            "annotation.key": "value",
+            "foo": "bar",
+        }
+        self.assertTrue(k8s.check_service_annotations(
+            "version=acid-service-annotations,spilo-role=master", annotations))
+        self.assertTrue(k8s.check_service_annotations(
+            "version=acid-service-annotations,spilo-role=replica", annotations))
+
+        # clean up
+        unpatch_custom_service_annotations = {
+            "data": {
+                "custom_service_annotations": "",
+            }
+        }
+        k8s.update_config(unpatch_custom_service_annotations)
+
     def assert_master_is_unique(self, namespace='default', version="acid-minimal-cluster"):
         '''
            Check that there is a single pod in the k8s cluster with the label "spilo-role=master"
@@ -322,6 +353,16 @@ class K8s:
                 pod_phase = pods[0].status.phase
             time.sleep(self.RETRY_TIMEOUT_SEC)
 
+    def check_service_annotations(self, svc_labels, annotations, namespace='default'):
+        svcs = self.api.core_v1.list_namespaced_service(namespace, label_selector=svc_labels, limit=1).items
+        for svc in svcs:
+            if len(svc.metadata.annotations) != len(annotations):
+                return False
+            for key in svc.metadata.annotations:
+                if svc.metadata.annotations[key] != annotations[key]:
+                    return False
+        return True
+
     def wait_for_pg_to_scale(self, number_of_instances, namespace='default'):
 
         body = {
@@ -330,7 +371,7 @@ class K8s:
             }
         }
         _ = self.api.custom_objects_api.patch_namespaced_custom_object(
-                    "acid.zalan.do", "v1", namespace, "postgresqls", "acid-minimal-cluster", body)
+            "acid.zalan.do", "v1", namespace, "postgresqls", "acid-minimal-cluster", body)
 
         labels = 'version=acid-minimal-cluster'
         while self.count_pods_with_label(labels) != number_of_instances: