Add ServiceAnnotations cluster config (#803)

The [operator parameters][1] already support the
`custom_service_annotations` config.With this parameter is possible to
define custom annotations that will be used on the services created by the
operator. The `custom_service_annotations` as all the other
[operator parameters][1] are defined on the operator level and do not allow
customization on the cluster level. A cluster may require different service
annotations, as for example, set up different cloud load balancers
timeouts, different ingress annotations, and/or enable more customizable
environments.

This commit introduces a new parameter on the cluster level, called
`serviceAnnotations`, responsible for defining custom annotations just for
the services created by the operator to the specifically defined cluster.
It allows a mix of configuration between `custom_service_annotations` and
`serviceAnnotations` where the latest one will have priority. In order to
allow custom service annotations to be used on services without
LoadBalancers (as for example, service mesh services annotations) both
`custom_service_annotations` and `serviceAnnotations` are applied
independently of load-balancing configuration. For retro-compatibility
purposes, `custom_service_annotations` is still under
[Load balancer related options][2]. The two default annotations when using
LoadBalancer services, `external-dns.alpha.kubernetes.io/hostname` and
`service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout` are
still defined by the operator.
`service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout` can
be overridden by `custom_service_annotations` or `serviceAnnotations`,
allowing a more customizable environment.
`external-dns.alpha.kubernetes.io/hostname` can not be overridden once
there is no differentiation between custom service annotations for
replicas and masters.

It updates the documentation and creates the necessary unit and e2e
tests to the above-described feature too.

[1]: https://github.com/zalando/postgres-operator/blob/master/docs/reference/operator_parameters.md
[2]: https://github.com/zalando/postgres-operator/blob/master/docs/reference/operator_parameters.md#load-balancer-related-options

pkg/apis/acid.zalan.do/v1/crds.go

@@ -383,6 +383,14 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{
 							},
 						},
 					},
+					"serviceAnnotations": {
+						Type: "object",
+						AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{
+							Schema: &apiextv1beta1.JSONSchemaProps{
+								Type: "string",
+							},
+						},
+					},
 					"sidecars": {
 						Type: "array",
 						Items: &apiextv1beta1.JSONSchemaPropsOrArray{

pkg/apis/acid.zalan.do/v1/postgresql_type.go

@@ -60,6 +60,7 @@ type PostgresSpec struct {
 	LogicalBackupSchedule string               `json:"logicalBackupSchedule,omitempty"`
 	StandbyCluster        *StandbyDescription  `json:"standby"`
 	PodAnnotations        map[string]string    `json:"podAnnotations"`
+	ServiceAnnotations    map[string]string    `json:"serviceAnnotations"`
 
 	// deprecated json tags
 	InitContainersOld       []v1.Container `json:"init_containers,omitempty"`

pkg/apis/acid.zalan.do/v1/util_test.go

@@ -456,18 +456,84 @@ var postgresqlList = []struct {
 		PostgresqlList{},
 		errors.New("unexpected end of JSON input")}}
 
-var annotations = []struct {
+var podAnnotations = []struct {
 	about       string
 	in          []byte
 	annotations map[string]string
 	err         error
 }{{
-	about:       "common annotations",
-	in:          []byte(`{"kind": "Postgresql","apiVersion": "acid.zalan.do/v1","metadata": {"name": "acid-testcluster1"}, "spec": {"podAnnotations": {"foo": "bar"},"teamId": "acid", "clone": {"cluster": "team-batman"}}}`),
+	about: "common annotations",
+	in: []byte(`{
+		"kind": "Postgresql",
+		"apiVersion": "acid.zalan.do/v1",
+		"metadata": {
+			"name": "acid-testcluster1"
+		},
+		"spec": {
+			"podAnnotations": {
+				"foo": "bar"
+			},
+			"teamId": "acid",
+			"clone": {
+				"cluster": "team-batman"
+			}
+		}
+	}`),
 	annotations: map[string]string{"foo": "bar"},
 	err:         nil},
 }
 
+var serviceAnnotations = []struct {
+	about       string
+	in          []byte
+	annotations map[string]string
+	err         error
+}{
+	{
+		about: "common single annotation",
+		in: []byte(`{
+			"kind": "Postgresql",
+			"apiVersion": "acid.zalan.do/v1",
+			"metadata": {
+				"name": "acid-testcluster1"
+			},
+			"spec": {
+				"serviceAnnotations": {
+					"foo": "bar"
+				},
+				"teamId": "acid",
+				"clone": {
+					"cluster": "team-batman"
+				}
+			}
+		}`),
+		annotations: map[string]string{"foo": "bar"},
+		err:         nil,
+	},
+	{
+		about: "common two annotations",
+		in: []byte(`{
+			"kind": "Postgresql",
+			"apiVersion": "acid.zalan.do/v1",
+			"metadata": {
+				"name": "acid-testcluster1"
+			},
+			"spec": {
+				"serviceAnnotations": {
+					"foo": "bar",
+					"post": "gres"
+				},
+				"teamId": "acid",
+				"clone": {
+					"cluster": "team-batman"
+				}
+			}
+		}`),
+		annotations: map[string]string{"foo": "bar", "post": "gres"},
+		err:         nil,
+	},
+}
+
 func mustParseTime(s string) metav1.Time {
 	v, err := time.Parse("15:04", s)
 	if err != nil {
@@ -517,21 +583,42 @@ func TestWeekdayTime(t *testing.T) {
 	}
 }
 
-func TestClusterAnnotations(t *testing.T) {
-	for _, tt := range annotations {
+func TestPodAnnotations(t *testing.T) {
+	for _, tt := range podAnnotations {
 		t.Run(tt.about, func(t *testing.T) {
 			var cluster Postgresql
 			err := cluster.UnmarshalJSON(tt.in)
 			if err != nil {
 				if tt.err == nil || err.Error() != tt.err.Error() {
-					t.Errorf("Unable to marshal cluster with annotations: expected %v got %v", tt.err, err)
+					t.Errorf("Unable to marshal cluster with podAnnotations: expected %v got %v", tt.err, err)
 				}
 				return
 			}
 			for k, v := range cluster.Spec.PodAnnotations {
 				found, expected := v, tt.annotations[k]
 				if found != expected {
-					t.Errorf("Didn't find correct value for key %v in for podAnnotations:  Expected %v found %v", k, expected, found)
+					t.Errorf("Didn't find correct value for key %v in for podAnnotations: Expected %v found %v", k, expected, found)
+				}
+			}
+		})
+	}
+}
+
+func TestServiceAnnotations(t *testing.T) {
+	for _, tt := range serviceAnnotations {
+		t.Run(tt.about, func(t *testing.T) {
+			var cluster Postgresql
+			err := cluster.UnmarshalJSON(tt.in)
+			if err != nil {
+				if tt.err == nil || err.Error() != tt.err.Error() {
+					t.Errorf("Unable to marshal cluster with serviceAnnotations: expected %v got %v", tt.err, err)
+				}
+				return
+			}
+			for k, v := range cluster.Spec.ServiceAnnotations {
+				found, expected := v, tt.annotations[k]
+				if found != expected {
+					t.Errorf("Didn't find correct value for key %v in for serviceAnnotations: Expected %v found %v", k, expected, found)
 				}
 			}
 		})

pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go

@@ -514,6 +514,13 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) {
 			(*out)[key] = val
 		}
 	}
+	if in.ServiceAnnotations != nil {
+		in, out := &in.ServiceAnnotations, &out.ServiceAnnotations
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
 	if in.InitContainersOld != nil {
 		in, out := &in.InitContainersOld, &out.InitContainersOld
 		*out = make([]corev1.Container, len(*in))