finish ignored_annotations feature

charts/postgres-operator/crds/operatorconfigurations.yaml

@@ -212,6 +212,10 @@ spec:
                   enable_sidecars:
                     type: boolean
                     default: true
+                  ignored_annotations:
+                    type: array
+                    items:
+                      type: string
                   infrastructure_roles_secret_name:
                     type: string
                   infrastructure_roles_secrets:

charts/postgres-operator/values.yaml

@@ -124,6 +124,13 @@ configKubernetes:
   enable_pod_disruption_budget: true
   # enables sidecar containers to run alongside Spilo in the same pod
   enable_sidecars: true
+
+  # annotations to be ignored when comparing statefulsets, services etc.
+  # ignored_annotations:
+  # - "deployment-time"
+  # - "k8s.v1.cni.cncf.io/network-status"
+
+
   # namespaced name of the secret containing infrastructure roles names and passwords
   # infrastructure_roles_secret_name: postgresql-infrastructure-roles
 

cmd/main.go

@@ -2,13 +2,14 @@ package main
 
 import (
 	"flag"
-	log "github.com/sirupsen/logrus"
 	"os"
 	"os/signal"
 	"sync"
 	"syscall"
 	"time"
 
+	log "github.com/sirupsen/logrus"
+
 	"github.com/zalando/postgres-operator/pkg/controller"
 	"github.com/zalando/postgres-operator/pkg/spec"
 	"github.com/zalando/postgres-operator/pkg/util/k8sutil"

docs/reference/operator_parameters.md

@@ -287,6 +287,12 @@ configuration they are grouped under the `kubernetes` key.
   Regular expressions like `downscaler/*` etc. are also accepted. Can be used
   with [kube-downscaler](https://github.com/hjacobs/kube-downscaler).
 
+* **ignored_annotations**
+  Some K8s tools inject and update annotations out of the Postgres Operator
+  control. This can cause rolling updates on each sync cycle of clusters.
+  With this option you can sepecify an array of annotations keys that should
+  be ignored when comparing K8s resources. The default is empty.
+
 * **watched_namespace**
   The operator watches for Postgres objects in the given namespace. If not
   specified, the value is taken from the operator namespace. A special `*`

e2e/tests/test_e2e.py

@@ -204,7 +204,7 @@ class EndToEndTestCase(unittest.TestCase):
         }
 
         # get node and replica (expected target of new master)
-        _, replica_nodes = k8s.get_pg_nodes(cluster_label)
+        master_nodes, replica_nodes = k8s.get_pg_nodes(cluster_label)
 
         try:
             k8s.update_config(patch_capabilities)
@@ -704,6 +704,43 @@ class EndToEndTestCase(unittest.TestCase):
             print('Operator log: {}'.format(k8s.get_operator_log()))
             raise
 
+    @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
+    def test_ignored_annotations(self):
+        '''
+           Test if injected annotation does not cause failover when listed under ignored_annotations
+        '''
+        k8s = self.k8s
+        cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster'
+
+        # get node of current master
+        master_node, _ = k8s.get_pg_nodes(cluster_label)
+
+        patch_config_ignored_annotations = {
+            "data": {
+                "ignored_annotations": "deployment-time",
+            }
+        }
+        k8s.update_config(patch_config_ignored_annotations)
+
+        pg_crd_annotation = {
+            "metadata": {
+                "annotations": {
+                    "deployment-time": "2022-04-01 12:00:00"
+                },
+            }
+        }
+        k8s.api.custom_objects_api.patch_namespaced_custom_object(
+            "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_crd_annotation)
+
+        annotations = {
+            "deployment-time": "2022-04-01 12:00:00",
+        }
+        self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
+        self.eventuallyTrue(lambda: k8s.check_statefulset_annotations(cluster_label, annotations), "Annotations missing")
+
+        current_master_node, _ = k8s.get_pg_nodes(cluster_label)
+        self.eventuallyEqual(lambda: master_node, current_master_node, "unexpected rolling update happened")
+
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
     def test_infrastructure_roles(self):
         '''

manifests/configmap.yaml

@@ -64,6 +64,7 @@ data:
   external_traffic_policy: "Cluster"
   # gcp_credentials: ""
   # kubernetes_use_configmaps: "false"
+  # ignored_annotations: ""
   # infrastructure_roles_secret_name: "postgresql-infrastructure-roles"
   # infrastructure_roles_secrets: "secretname:monitoring-roles,userkey:user,passwordkey:password,rolekey:inrole"
   # inherited_annotations: owned-by

manifests/operatorconfiguration.crd.yaml

@@ -10,9 +10,9 @@ spec:
     plural: operatorconfigurations
     singular: operatorconfiguration
     shortNames:
-    - opconfig
+      - opconfig
     categories:
-    - all
+      - all
   scope: Namespaced
   versions:
   - name: v1
@@ -115,10 +115,46 @@ spec:
                 type: object
                 additionalProperties:
                   type: string
-              sidecars:
+                  default: "registry.opensource.zalan.do/acid/spilo-13:2.0-p6"
-                type: array
+                enable_crd_validation:
-                nullable: true
+                  type: boolean
-                items:
+                  default: true
+                enable_lazy_spilo_upgrade:
+                  type: boolean
+                  default: false
+                enable_pgversion_env_var:
+                  type: boolean
+                  default: true
+                enable_shm_volume:
+                  type: boolean
+                  default: true
+                enable_spilo_wal_path_compat:
+                  type: boolean
+                  default: false
+                etcd_host:
+                  type: string
+                  default: ""
+                kubernetes_use_configmaps:
+                  type: boolean
+                  default: false
+                max_instances:
+                  type: integer
+                  minimum: -1 # -1 = disabled
+                  default: -1
+                min_instances:
+                  type: integer
+                  minimum: -1 # -1 = disabled
+                  default: -1
+                resync_period:
+                  type: string
+                  default: "30m"
+                repair_period:
+                  type: string
+                  default: "5m"
+                set_memory_request_to_limit:
+                  type: boolean
+                  default: false
+                sidecar_docker_images:
                   type: object
                   x-kubernetes-preserve-unknown-fields: true
               workers:
@@ -173,19 +209,23 @@ spec:
                       type: string
                   cluster_domain:
                     type: string
-                    default: "cluster.local"
+                sidecars:
-                  cluster_labels:
+                  type: array
+                  nullable: true
+                  items:
                     type: object
-                    additionalProperties:
+                    x-kubernetes-preserve-unknown-fields: true
+                workers:
+                  type: integer
+                  minimum: 1
+                  default: 8
+                users:
+                  type: object
+                  properties:
+                    replication_username:
                       type: string
-                    default:
+                      default: standby
-                      application: spilo
+                    super_username:
-                  cluster_name_label:
-                    type: string
-                    default: "cluster-name"
-                  custom_pod_annotations:
-                    type: object
-                    additionalProperties:
                       type: string
                   delete_annotation_date_key:
                     type: string
@@ -210,6 +250,10 @@ spec:
                   enable_sidecars:
                     type: boolean
                     default: true
+                  ignored_annotations:
+                    type: array
+                    items:
+                      type: string
                   infrastructure_roles_secret_name:
                     type: string
                   infrastructure_roles_secrets:
@@ -217,41 +261,11 @@ spec:
                     nullable: true
                     items:
                       type: object
-                      required:
+                      additionalProperties:
-                        - secretname
+                        type: string
-                        - userkey
+                      default:
-                        - passwordkey
+                        application: spilo
-                      properties:
+                    cluster_name_label:
-                        secretname:
-                          type: string
-                        userkey:
-                          type: string
-                        passwordkey:
-                          type: string
-                        rolekey:
-                          type: string
-                        defaultuservalue:
-                          type: string
-                        defaultrolevalue:
-                          type: string
-                        details:
-                          type: string
-                        template:
-                          type: boolean
-                  inherited_annotations:
-                    type: array
-                    items:
-                      type: string
-                  inherited_labels:
-                    type: array
-                    items:
-                      type: string
-                  master_pod_move_timeout:
-                    type: string
-                    default: "20m"
-                  node_readiness_label:
-                    type: object
-                    additionalProperties:
                       type: string
                   node_readiness_label_merge:
                     type: string

manifests/postgresql-operator-default-configuration.yaml

@@ -59,6 +59,9 @@ configuration:
     enable_pod_antiaffinity: false
     enable_pod_disruption_budget: true
     enable_sidecars: true
+    # ignored_annotations:
+    # - "deployment-time"
+    # - "k8s.v1.cni.cncf.io/network-status"
     # infrastructure_roles_secret_name: "postgresql-infrastructure-roles"
     # infrastructure_roles_secrets:
     # - secretname: "monitoring-roles"

pkg/apis/acid.zalan.do/v1/crds.go

@@ -1258,6 +1258,14 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{
 							"enable_sidecars": {
 								Type: "boolean",
 							},
+							"ignored_annotations": {
+								Type: "array",
+								Items: &apiextv1.JSONSchemaPropsOrArray{
+									Schema: &apiextv1.JSONSchemaProps{
+										Type: "string",
+									},
+								},
+							},
 							"infrastructure_roles_secret_name": {
 								Type: "string",
 							},

pkg/apis/acid.zalan.do/v1/operator_configuration_type.go

@@ -82,6 +82,7 @@ type KubernetesMetaConfiguration struct {
 	InheritedLabels                        []string                     `json:"inherited_labels,omitempty"`
 	InheritedAnnotations                   []string                     `json:"inherited_annotations,omitempty"`
 	DownscalerAnnotations                  []string                     `json:"downscaler_annotations,omitempty"`
+	IgnoredAnnotations                     []string                     `json:"ignored_annotations,omitempty"`
 	ClusterNameLabel                       string                       `json:"cluster_name_label,omitempty"`
 	DeleteAnnotationDateKey                string                       `json:"delete_annotation_date_key,omitempty"`
 	DeleteAnnotationNameKey                string                       `json:"delete_annotation_name_key,omitempty"`

pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go

@@ -106,7 +106,11 @@ func (in *ConnectionPooler) DeepCopyInto(out *ConnectionPooler) {
 		*out = new(int32)
 		**out = **in
 	}
-	out.Resources = in.Resources
+	if in.Resources != nil {
+		in, out := &in.Resources, &out.Resources
+		*out = new(Resources)
+		**out = **in
+	}
 	return
 }
 
@@ -209,6 +213,11 @@ func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfigura
 			(*out)[key] = val
 		}
 	}
+	if in.IgnoredAnnotations != nil {
+		in, out := &in.IgnoredAnnotations, &out.IgnoredAnnotations
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	if in.InheritedLabels != nil {
 		in, out := &in.InheritedLabels, &out.InheritedLabels
 		*out = make([]string, len(*in))
@@ -575,7 +584,11 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) {
 	in.PostgresqlParam.DeepCopyInto(&out.PostgresqlParam)
 	in.Volume.DeepCopyInto(&out.Volume)
 	in.Patroni.DeepCopyInto(&out.Patroni)
-	out.Resources = in.Resources
+	if in.Resources != nil {
+		in, out := &in.Resources, &out.Resources
+		*out = new(Resources)
+		**out = **in
+	}
 	if in.EnableConnectionPooler != nil {
 		in, out := &in.EnableConnectionPooler, &out.EnableConnectionPooler
 		*out = new(bool)
@@ -1132,7 +1145,11 @@ func (in *ScalyrConfiguration) DeepCopy() *ScalyrConfiguration {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Sidecar) DeepCopyInto(out *Sidecar) {
 	*out = *in
-	out.Resources = in.Resources
+	if in.Resources != nil {
+		in, out := &in.Resources, &out.Resources
+		*out = new(Resources)
+		**out = **in
+	}
 	if in.Ports != nil {
 		in, out := &in.Ports, &out.Ports
 		*out = make([]corev1.ContainerPort, len(*in))

pkg/cluster/cluster.go

@@ -382,9 +382,10 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa
 		match = false
 		reasons = append(reasons, "new statefulset's number of replicas does not match the current one")
 	}
-	if !reflect.DeepEqual(c.Statefulset.Annotations, statefulSet.Annotations) {
+	if changed, reason := c.compareAnnotations(c.Statefulset.Annotations, statefulSet.Annotations); changed {
+		match = false
 		needsReplace = true
-		reasons = append(reasons, "new statefulset's annotations do not match the current one")
+		reasons = append(reasons, "new statefulset's annotations do not match "+reason)
 	}
 
 	needsRollUpdate, reasons = c.compareContainers("initContainers", c.Statefulset.Spec.Template.Spec.InitContainers, statefulSet.Spec.Template.Spec.InitContainers, needsRollUpdate, reasons)
@@ -438,10 +439,11 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa
 		}
 	}
 
-	if !reflect.DeepEqual(c.Statefulset.Spec.Template.Annotations, statefulSet.Spec.Template.Annotations) {
+	if changed, reason := c.compareAnnotations(c.Statefulset.Spec.Template.Annotations, statefulSet.Spec.Template.Annotations); changed {
+		match = false
 		needsReplace = true
 		needsRollUpdate = true
-		reasons = append(reasons, "new statefulset's pod template metadata annotations does not match the current one")
+		reasons = append(reasons, "new statefulset's pod template metadata annotations does not match "+reason)
 	}
 	if !reflect.DeepEqual(c.Statefulset.Spec.Template.Spec.SecurityContext, statefulSet.Spec.Template.Spec.SecurityContext) {
 		needsReplace = true
@@ -720,6 +722,70 @@ func (c *Cluster) enforceMinResourceLimits(spec *acidv1.PostgresSpec) error {
 	return nil
 }
 
+func (c *Cluster) compareAnnotations(old, new map[string]string) (bool, string) {
+	ignored := make(map[string]bool)
+	for _, ignore := range c.OpConfig.IgnoredAnnotations {
+		ignored[ignore] = true
+	}
+
+	// changed := false
+	reason := ""
+
+	for key := range old {
+		if _, ok := ignored[key]; ok {
+			continue
+		}
+		if _, ok := new[key]; !ok {
+			reason += fmt.Sprintf(" Removed '%s'.", key)
+		}
+	}
+
+	for key := range new {
+		if _, ok := ignored[key]; ok {
+			continue
+		}
+
+		v, ok := old[key]
+		if !ok {
+			reason += fmt.Sprintf(" Added '%s' with value '%s'.", key, new[key])
+		} else if v != new[key] {
+			reason += fmt.Sprintf(" '%s' changed from '%s' to '%s'.", key, v, new[key])
+		}
+	}
+
+	if reason != "" {
+		return true, reason
+	}
+	return false, ""
+
+}
+
+// SameService compares the Services
+func (c *Cluster) compareServices(old, new *v1.Service) (bool, string) {
+	//TODO: improve comparison
+	if old.Spec.Type != new.Spec.Type {
+		return false, fmt.Sprintf("new service's type %q does not match the current one %q",
+			new.Spec.Type, old.Spec.Type)
+	}
+
+	oldSourceRanges := old.Spec.LoadBalancerSourceRanges
+	newSourceRanges := new.Spec.LoadBalancerSourceRanges
+
+	/* work around Kubernetes 1.6 serializing [] as nil. See https://github.com/kubernetes/kubernetes/issues/43203 */
+	if (len(oldSourceRanges) != 0) || (len(newSourceRanges) != 0) {
+		if !util.IsEqualIgnoreOrder(oldSourceRanges, newSourceRanges) {
+			return false, "new service's LoadBalancerSourceRange does not match the current one"
+		}
+	}
+
+	if changed, reason := c.compareAnnotations(old.Annotations, new.Annotations); changed {
+		return !changed, "new service's annotations does not match the current one:" + reason
+	}
+
+	return true, ""
+
+}
+
 // Update changes Kubernetes objects according to the new specification. Unlike the sync case, the missing object
 // (i.e. service) is treated as an error
 // logical backup cron jobs are an exception: a user-initiated Update can enable a logical backup job
@@ -821,7 +887,7 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
 			updateFailed = true
 			return
 		}
-		if syncStatefulSet || !reflect.DeepEqual(oldSs, newSs) || !reflect.DeepEqual(oldSpec.Annotations, newSpec.Annotations) {
+		if syncStatefulSet || !reflect.DeepEqual(oldSs, newSs) {
 			c.logger.Debugf("syncing statefulsets")
 			syncStatefulSet = false
 			// TODO: avoid generating the StatefulSet object twice by passing it to syncStatefulSet

pkg/cluster/cluster_test.go

@@ -3,6 +3,7 @@ package cluster
 import (
 	"fmt"
 	"reflect"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -1054,6 +1055,359 @@ func TestCompareEnv(t *testing.T) {
 	}
 }
 
+func newFakeK8sServiceClient() (k8sutil.KubernetesClient, *fake.Clientset) {
+	clientSet := fake.NewSimpleClientset()
+
+	return k8sutil.KubernetesClient{
+		ServicesGetter: clientSet.CoreV1(),
+	}, clientSet
+}
+
+func newService(ann map[string]string, svcT v1.ServiceType, lbSr []string) *v1.Service {
+	svc := &v1.Service{
+		Spec: v1.ServiceSpec{
+			Type:                     svcT,
+			LoadBalancerSourceRanges: lbSr,
+		},
+	}
+	svc.Annotations = ann
+	return svc
+}
+
+func TestSameService(t *testing.T) {
+	testName := "test comparing services"
+	client, _ := newFakeK8sServiceClient()
+	namespace := "default"
+
+	pg := acidv1.Postgresql{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "acid-fake-cluster",
+			Namespace: namespace,
+			Annotations: map[string]string{
+				"deployment-time": "2022-02-02 12:00:00",
+			},
+		},
+		Spec: acidv1.PostgresSpec{
+			Volume: acidv1.Volume{
+				Size: "1Gi",
+			},
+		},
+	}
+	var cluster = New(
+		Config{
+			OpConfig: config.Config{
+				Resources: config.Resources{
+					IgnoredAnnotations: []string{
+						"k8s.v1.cni.cncf.io/network-status",
+					},
+				},
+			},
+		}, client, pg, logger, eventRecorder)
+
+	tests := []struct {
+		about   string
+		current *v1.Service
+		new     *v1.Service
+		reason  string
+		match   bool
+	}{
+		{
+			about: "two equal services",
+			current: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+				},
+				v1.ServiceTypeClusterIP,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			new: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+				},
+				v1.ServiceTypeClusterIP,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			match: true,
+		},
+		{
+			about: "services differ on service type",
+			current: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+				},
+				v1.ServiceTypeClusterIP,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			new: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			match:  false,
+			reason: `new service's type "LoadBalancer" does not match the current one "ClusterIP"`,
+		},
+		{
+			about: "services differ on lb source ranges",
+			current: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			new: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"185.249.56.0/22"}),
+			match:  false,
+			reason: `new service's LoadBalancerSourceRange does not match the current one`,
+		},
+		{
+			about: "new service doesn't have lb source ranges",
+			current: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			new: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{}),
+			match:  false,
+			reason: `new service's LoadBalancerSourceRange does not match the current one`,
+		},
+		{
+			about: "services differ on DNS annotation",
+			current: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			new: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "new_clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			match:  false,
+			reason: `new service's annotations does not match the current one: 'external-dns.alpha.kubernetes.io/hostname' changed from 'clstr.acid.zalan.do' to 'new_clstr.acid.zalan.do'.`,
+		},
+		{
+			about: "services differ on AWS ELB annotation",
+			current: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			new: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: "1800",
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			match:  false,
+			reason: `new service's annotations does not match the current one: 'service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout' changed from '3600' to '1800'.`,
+		},
+		{
+			about: "service changes existing annotation",
+			current: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+					"foo":                              "bar",
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			new: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+					"foo":                              "baz",
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			match:  false,
+			reason: `new service's annotations does not match the current one: 'foo' changed from 'bar' to 'baz'.`,
+		},
+		{
+			about: "service changes multiple existing annotations",
+			current: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+					"foo":                              "bar",
+					"bar":                              "foo",
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			new: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+					"foo":                              "baz",
+					"bar":                              "fooz",
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			match: false,
+			// Test just the prefix to avoid flakiness and map sorting
+			reason: `new service's annotations does not match the current one:`,
+		},
+		{
+			about: "service adds a new custom annotation",
+			current: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			new: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+					"foo":                              "bar",
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			match:  false,
+			reason: `new service's annotations does not match the current one: Added 'foo' with value 'bar'.`,
+		},
+		{
+			about: "service removes a custom annotation",
+			current: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+					"foo":                              "bar",
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			new: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			match:  false,
+			reason: `new service's annotations does not match the current one: Removed 'foo'.`,
+		},
+		{
+			about: "service removes a custom annotation and adds a new one",
+			current: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+					"foo":                              "bar",
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			new: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+					"bar":                              "foo",
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			match:  false,
+			reason: `new service's annotations does not match the current one: Removed 'foo'. Added 'bar' with value 'foo'.`,
+		},
+		{
+			about: "service removes a custom annotation, adds a new one and change another",
+			current: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+					"foo":                              "bar",
+					"zalan":                            "do",
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			new: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+					"bar":                              "foo",
+					"zalan":                            "do.com",
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			match: false,
+			// Test just the prefix to avoid flakiness and map sorting
+			reason: `new service's annotations does not match the current one: Removed 'foo'.`,
+		},
+		{
+			about: "service add annotations",
+			current: newService(
+				map[string]string{},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			new: newService(
+				map[string]string{
+					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
+					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			match: false,
+			// Test just the prefix to avoid flakiness and map sorting
+			reason: `new service's annotations does not match the current one: Added `,
+		},
+		{
+			about: "ignored annotations",
+			current: newService(
+				map[string]string{},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			new: newService(
+				map[string]string{
+					"k8s.v1.cni.cncf.io/network-status": "up",
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			match: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.about, func(t *testing.T) {
+			match, reason := cluster.compareServices(tt.current, tt.new)
+			if match && !tt.match {
+				t.Logf("match=%v current=%v, old=%v reason=%s", match, tt.current.Annotations, tt.new.Annotations, reason)
+				t.Errorf("%s - expected services to do not match: '%q' and '%q'", testName, tt.current, tt.new)
+				return
+			}
+			if !match && tt.match {
+				t.Errorf("%s - expected services to be the same: '%q' and '%q'", testName, tt.current, tt.new)
+				return
+			}
+			if !match && !tt.match {
+				if !strings.HasPrefix(reason, tt.reason) {
+					t.Errorf("%s - expected reason prefix '%s', found '%s'", testName, tt.reason, reason)
+					return
+				}
+			}
+		})
+	}
+}
+
 func TestCrossNamespacedSecrets(t *testing.T) {
 	testName := "test secrets in different namespace"
 	clientSet := fake.NewSimpleClientset()

pkg/cluster/connection_pooler.go

@@ -910,7 +910,7 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql
 	if service, err = c.KubeClient.Services(c.Namespace).Get(context.TODO(), c.connectionPoolerName(role), metav1.GetOptions{}); err == nil {
 		c.ConnectionPooler[role].Service = service
 		desiredSvc := c.generateConnectionPoolerService(c.ConnectionPooler[role])
-		if match, reason := k8sutil.SameService(service, desiredSvc); !match {
+		if match, reason := c.compareServices(service, desiredSvc); !match {
 			syncReason = append(syncReason, reason)
 			c.logServiceChanges(role, service, desiredSvc, false, reason)
 			newService, err = c.updateService(role, service, desiredSvc)

pkg/cluster/sync.go

@@ -172,7 +172,7 @@ func (c *Cluster) syncService(role PostgresRole) error {
 	if svc, err = c.KubeClient.Services(c.Namespace).Get(context.TODO(), c.serviceName(role), metav1.GetOptions{}); err == nil {
 		c.Services[role] = svc
 		desiredSvc := c.generateService(role, &c.Spec)
-		if match, reason := k8sutil.SameService(svc, desiredSvc); !match {
+		if match, reason := c.compareServices(svc, desiredSvc); !match {
 			c.logServiceChanges(role, svc, desiredSvc, false, reason)
 			updatedSvc, err := c.updateService(role, svc, desiredSvc)
 			if err != nil {

pkg/controller/controller.go

@@ -186,6 +186,7 @@ func (c *Controller) modifyConfigFromEnvironment() {
 	if c.config.NoTeamsAPI {
 		c.opConfig.EnableTeamsAPI = false
 	}
+
 	scalyrAPIKey := os.Getenv("SCALYR_API_KEY")
 	if scalyrAPIKey != "" {
 		c.opConfig.ScalyrAPIKey = scalyrAPIKey

pkg/controller/operator_config.go

@@ -108,6 +108,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	result.InheritedLabels = fromCRD.Kubernetes.InheritedLabels
 	result.InheritedAnnotations = fromCRD.Kubernetes.InheritedAnnotations
 	result.DownscalerAnnotations = fromCRD.Kubernetes.DownscalerAnnotations
+	result.IgnoredAnnotations = fromCRD.Kubernetes.IgnoredAnnotations
 	result.ClusterNameLabel = util.Coalesce(fromCRD.Kubernetes.ClusterNameLabel, "cluster-name")
 	result.DeleteAnnotationDateKey = fromCRD.Kubernetes.DeleteAnnotationDateKey
 	result.DeleteAnnotationNameKey = fromCRD.Kubernetes.DeleteAnnotationNameKey

pkg/spec/types.go

@@ -119,6 +119,7 @@ type ControllerConfig struct {
 	CRDReadyWaitTimeout  time.Duration
 	ConfigMapName        NamespacedName
 	Namespace            string
+	IgnoredAnnotations   []string
 
 	EnableJsonLogging bool
 }

pkg/util/config/config.go

@@ -42,6 +42,7 @@ type Resources struct {
 	InheritedLabels               []string            `name:"inherited_labels" default:""`
 	InheritedAnnotations          []string            `name:"inherited_annotations" default:""`
 	DownscalerAnnotations         []string            `name:"downscaler_annotations"`
+	IgnoredAnnotations            []string            `name:"ignored_annotations"`
 	ClusterNameLabel              string              `name:"cluster_name_label" default:"cluster-name"`
 	DeleteAnnotationDateKey       string              `name:"delete_annotation_date_key"`
 	DeleteAnnotationNameKey       string              `name:"delete_annotation_name_key"`

pkg/util/k8sutil/k8sutil.go

@@ -213,57 +213,6 @@ func (client *KubernetesClient) SetPostgresCRDStatus(clusterName spec.Namespaced
 	return pg, nil
 }
 
-// SameService compares the Services
-func SameService(cur, new *v1.Service) (match bool, reason string) {
-	//TODO: improve comparison
-	if cur.Spec.Type != new.Spec.Type {
-		return false, fmt.Sprintf("new service's type %q does not match the current one %q",
-			new.Spec.Type, cur.Spec.Type)
-	}
-
-	oldSourceRanges := cur.Spec.LoadBalancerSourceRanges
-	newSourceRanges := new.Spec.LoadBalancerSourceRanges
-
-	/* work around Kubernetes 1.6 serializing [] as nil. See https://github.com/kubernetes/kubernetes/issues/43203 */
-	if (len(oldSourceRanges) != 0) || (len(newSourceRanges) != 0) {
-		if !reflect.DeepEqual(oldSourceRanges, newSourceRanges) {
-			return false, "new service's LoadBalancerSourceRange does not match the current one"
-		}
-	}
-
-	match = true
-
-	reasonPrefix := "new service's annotations does not match the current one:"
-	for ann := range cur.Annotations {
-		if _, ok := new.Annotations[ann]; !ok {
-			match = false
-			if len(reason) == 0 {
-				reason = reasonPrefix
-			}
-			reason += fmt.Sprintf(" Removed '%s'.", ann)
-		}
-	}
-
-	for ann := range new.Annotations {
-		v, ok := cur.Annotations[ann]
-		if !ok {
-			if len(reason) == 0 {
-				reason = reasonPrefix
-			}
-			reason += fmt.Sprintf(" Added '%s' with value '%s'.", ann, new.Annotations[ann])
-			match = false
-		} else if v != new.Annotations[ann] {
-			if len(reason) == 0 {
-				reason = reasonPrefix
-			}
-			reason += fmt.Sprintf(" '%s' changed from '%s' to '%s'.", ann, v, new.Annotations[ann])
-			match = false
-		}
-	}
-
-	return match, reason
-}
-
 // SamePDB compares the PodDisruptionBudgets
 func SamePDB(cur, new *policybeta1.PodDisruptionBudget) (match bool, reason string) {
 	//TODO: improve comparison

pkg/util/k8sutil/k8sutil_test.go

@@ -1,311 +0,0 @@
-package k8sutil
-
-import (
-	"strings"
-	"testing"
-
-	"github.com/zalando/postgres-operator/pkg/util/constants"
-
-	v1 "k8s.io/api/core/v1"
-)
-
-func newsService(ann map[string]string, svcT v1.ServiceType, lbSr []string) *v1.Service {
-	svc := &v1.Service{
-		Spec: v1.ServiceSpec{
-			Type:                     svcT,
-			LoadBalancerSourceRanges: lbSr,
-		},
-	}
-	svc.Annotations = ann
-	return svc
-}
-
-func TestSameService(t *testing.T) {
-	tests := []struct {
-		about   string
-		current *v1.Service
-		new     *v1.Service
-		reason  string
-		match   bool
-	}{
-		{
-			about: "two equal services",
-			current: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-				},
-				v1.ServiceTypeClusterIP,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			new: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-				},
-				v1.ServiceTypeClusterIP,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			match: true,
-		},
-		{
-			about: "services differ on service type",
-			current: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-				},
-				v1.ServiceTypeClusterIP,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			new: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			match:  false,
-			reason: `new service's type "LoadBalancer" does not match the current one "ClusterIP"`,
-		},
-		{
-			about: "services differ on lb source ranges",
-			current: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			new: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"185.249.56.0/22"}),
-			match:  false,
-			reason: `new service's LoadBalancerSourceRange does not match the current one`,
-		},
-		{
-			about: "new service doesn't have lb source ranges",
-			current: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			new: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{}),
-			match:  false,
-			reason: `new service's LoadBalancerSourceRange does not match the current one`,
-		},
-		{
-			about: "services differ on DNS annotation",
-			current: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			new: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "new_clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			match:  false,
-			reason: `new service's annotations does not match the current one: 'external-dns.alpha.kubernetes.io/hostname' changed from 'clstr.acid.zalan.do' to 'new_clstr.acid.zalan.do'.`,
-		},
-		{
-			about: "services differ on AWS ELB annotation",
-			current: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			new: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: "1800",
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			match:  false,
-			reason: `new service's annotations does not match the current one: 'service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout' changed from '3600' to '1800'.`,
-		},
-		{
-			about: "service changes existing annotation",
-			current: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-					"foo":                              "bar",
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			new: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-					"foo":                              "baz",
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			match:  false,
-			reason: `new service's annotations does not match the current one: 'foo' changed from 'bar' to 'baz'.`,
-		},
-		{
-			about: "service changes multiple existing annotations",
-			current: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-					"foo":                              "bar",
-					"bar":                              "foo",
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			new: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-					"foo":                              "baz",
-					"bar":                              "fooz",
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			match: false,
-			// Test just the prefix to avoid flakiness and map sorting
-			reason: `new service's annotations does not match the current one:`,
-		},
-		{
-			about: "service adds a new custom annotation",
-			current: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			new: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-					"foo":                              "bar",
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			match:  false,
-			reason: `new service's annotations does not match the current one: Added 'foo' with value 'bar'.`,
-		},
-		{
-			about: "service removes a custom annotation",
-			current: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-					"foo":                              "bar",
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			new: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			match:  false,
-			reason: `new service's annotations does not match the current one: Removed 'foo'.`,
-		},
-		{
-			about: "service removes a custom annotation and adds a new one",
-			current: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-					"foo":                              "bar",
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			new: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-					"bar":                              "foo",
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			match:  false,
-			reason: `new service's annotations does not match the current one: Removed 'foo'. Added 'bar' with value 'foo'.`,
-		},
-		{
-			about: "service removes a custom annotation, adds a new one and change another",
-			current: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-					"foo":                              "bar",
-					"zalan":                            "do",
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			new: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-					"bar":                              "foo",
-					"zalan":                            "do.com",
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			match: false,
-			// Test just the prefix to avoid flakiness and map sorting
-			reason: `new service's annotations does not match the current one: Removed 'foo'.`,
-		},
-		{
-			about: "service add annotations",
-			current: newsService(
-				map[string]string{},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			new: newsService(
-				map[string]string{
-					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
-					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
-				},
-				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
-			match: false,
-			// Test just the prefix to avoid flakiness and map sorting
-			reason: `new service's annotations does not match the current one: Added `,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.about, func(t *testing.T) {
-			match, reason := SameService(tt.current, tt.new)
-			if match && !tt.match {
-				t.Errorf("expected services to do not match: '%q' and '%q'", tt.current, tt.new)
-				return
-			}
-			if !match && tt.match {
-				t.Errorf("expected services to be the same: '%q' and '%q'", tt.current, tt.new)
-				return
-			}
-			if !match && !tt.match {
-				if !strings.HasPrefix(reason, tt.reason) {
-					t.Errorf("expected reason prefix '%s', found '%s'", tt.reason, reason)
-					return
-				}
-			}
-		})
-	}
-}