diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 5d76b9540..283acee2e 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -212,6 +212,10 @@ spec: enable_sidecars: type: boolean default: true + ignored_annotations: + type: array + items: + type: string infrastructure_roles_secret_name: type: string infrastructure_roles_secrets: diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 4c373ac8d..7ad804114 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -124,6 +124,13 @@ configKubernetes: enable_pod_disruption_budget: true # enables sidecar containers to run alongside Spilo in the same pod enable_sidecars: true + + # annotations to be ignored when comparing statefulsets, services etc. + # ignored_annotations: + # - "deployment-time" + # - "k8s.v1.cni.cncf.io/network-status" + + # namespaced name of the secret containing infrastructure roles names and passwords # infrastructure_roles_secret_name: postgresql-infrastructure-roles diff --git a/cmd/main.go b/cmd/main.go index 376df0bad..0b48ac863 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -2,13 +2,14 @@ package main import ( "flag" - log "github.com/sirupsen/logrus" "os" "os/signal" "sync" "syscall" "time" + log "github.com/sirupsen/logrus" + "github.com/zalando/postgres-operator/pkg/controller" "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util/k8sutil" diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 7692a4369..96c940949 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -287,6 +287,12 @@ configuration they are grouped under the `kubernetes` key. Regular expressions like `downscaler/*` etc. are also accepted. Can be used with [kube-downscaler](https://github.com/hjacobs/kube-downscaler). +* **ignored_annotations** + Some K8s tools inject and update annotations out of the Postgres Operator + control. This can cause rolling updates on each sync cycle of clusters. + With this option you can sepecify an array of annotations keys that should + be ignored when comparing K8s resources. The default is empty. + * **watched_namespace** The operator watches for Postgres objects in the given namespace. If not specified, the value is taken from the operator namespace. A special `*` diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 44c72cb44..9bae875df 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -204,7 +204,7 @@ class EndToEndTestCase(unittest.TestCase): } # get node and replica (expected target of new master) - _, replica_nodes = k8s.get_pg_nodes(cluster_label) + master_nodes, replica_nodes = k8s.get_pg_nodes(cluster_label) try: k8s.update_config(patch_capabilities) @@ -704,6 +704,43 @@ class EndToEndTestCase(unittest.TestCase): print('Operator log: {}'.format(k8s.get_operator_log())) raise + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_ignored_annotations(self): + ''' + Test if injected annotation does not cause failover when listed under ignored_annotations + ''' + k8s = self.k8s + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + + # get node of current master + master_node, _ = k8s.get_pg_nodes(cluster_label) + + patch_config_ignored_annotations = { + "data": { + "ignored_annotations": "deployment-time", + } + } + k8s.update_config(patch_config_ignored_annotations) + + pg_crd_annotation = { + "metadata": { + "annotations": { + "deployment-time": "2022-04-01 12:00:00" + }, + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_crd_annotation) + + annotations = { + "deployment-time": "2022-04-01 12:00:00", + } + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + self.eventuallyTrue(lambda: k8s.check_statefulset_annotations(cluster_label, annotations), "Annotations missing") + + current_master_node, _ = k8s.get_pg_nodes(cluster_label) + self.eventuallyEqual(lambda: master_node, current_master_node, "unexpected rolling update happened") + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_infrastructure_roles(self): ''' diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index bc1fe6ffc..130a35176 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -64,6 +64,7 @@ data: external_traffic_policy: "Cluster" # gcp_credentials: "" # kubernetes_use_configmaps: "false" + # ignored_annotations: "" # infrastructure_roles_secret_name: "postgresql-infrastructure-roles" # infrastructure_roles_secrets: "secretname:monitoring-roles,userkey:user,passwordkey:password,rolekey:inrole" # inherited_annotations: owned-by diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index c67d4f19b..2d22a5daa 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -10,9 +10,9 @@ spec: plural: operatorconfigurations singular: operatorconfiguration shortNames: - - opconfig + - opconfig categories: - - all + - all scope: Namespaced versions: - name: v1 @@ -115,10 +115,46 @@ spec: type: object additionalProperties: type: string - sidecars: - type: array - nullable: true - items: + default: "registry.opensource.zalan.do/acid/spilo-13:2.0-p6" + enable_crd_validation: + type: boolean + default: true + enable_lazy_spilo_upgrade: + type: boolean + default: false + enable_pgversion_env_var: + type: boolean + default: true + enable_shm_volume: + type: boolean + default: true + enable_spilo_wal_path_compat: + type: boolean + default: false + etcd_host: + type: string + default: "" + kubernetes_use_configmaps: + type: boolean + default: false + max_instances: + type: integer + minimum: -1 # -1 = disabled + default: -1 + min_instances: + type: integer + minimum: -1 # -1 = disabled + default: -1 + resync_period: + type: string + default: "30m" + repair_period: + type: string + default: "5m" + set_memory_request_to_limit: + type: boolean + default: false + sidecar_docker_images: type: object x-kubernetes-preserve-unknown-fields: true workers: @@ -173,19 +209,23 @@ spec: type: string cluster_domain: type: string - default: "cluster.local" - cluster_labels: + sidecars: + type: array + nullable: true + items: type: object - additionalProperties: + x-kubernetes-preserve-unknown-fields: true + workers: + type: integer + minimum: 1 + default: 8 + users: + type: object + properties: + replication_username: type: string - default: - application: spilo - cluster_name_label: - type: string - default: "cluster-name" - custom_pod_annotations: - type: object - additionalProperties: + default: standby + super_username: type: string delete_annotation_date_key: type: string @@ -210,6 +250,10 @@ spec: enable_sidecars: type: boolean default: true + ignored_annotations: + type: array + items: + type: string infrastructure_roles_secret_name: type: string infrastructure_roles_secrets: @@ -217,41 +261,11 @@ spec: nullable: true items: type: object - required: - - secretname - - userkey - - passwordkey - properties: - secretname: - type: string - userkey: - type: string - passwordkey: - type: string - rolekey: - type: string - defaultuservalue: - type: string - defaultrolevalue: - type: string - details: - type: string - template: - type: boolean - inherited_annotations: - type: array - items: - type: string - inherited_labels: - type: array - items: - type: string - master_pod_move_timeout: - type: string - default: "20m" - node_readiness_label: - type: object - additionalProperties: + additionalProperties: + type: string + default: + application: spilo + cluster_name_label: type: string node_readiness_label_merge: type: string diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index ea5032e3c..1d16e51fd 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -59,6 +59,9 @@ configuration: enable_pod_antiaffinity: false enable_pod_disruption_budget: true enable_sidecars: true + # ignored_annotations: + # - "deployment-time" + # - "k8s.v1.cni.cncf.io/network-status" # infrastructure_roles_secret_name: "postgresql-infrastructure-roles" # infrastructure_roles_secrets: # - secretname: "monitoring-roles" diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 7f8178bab..e6a2ee9b3 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -1258,6 +1258,14 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{ "enable_sidecars": { Type: "boolean", }, + "ignored_annotations": { + Type: "array", + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ + Type: "string", + }, + }, + }, "infrastructure_roles_secret_name": { Type: "string", }, diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 3d94ae038..0be275553 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -82,6 +82,7 @@ type KubernetesMetaConfiguration struct { InheritedLabels []string `json:"inherited_labels,omitempty"` InheritedAnnotations []string `json:"inherited_annotations,omitempty"` DownscalerAnnotations []string `json:"downscaler_annotations,omitempty"` + IgnoredAnnotations []string `json:"ignored_annotations,omitempty"` ClusterNameLabel string `json:"cluster_name_label,omitempty"` DeleteAnnotationDateKey string `json:"delete_annotation_date_key,omitempty"` DeleteAnnotationNameKey string `json:"delete_annotation_name_key,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index d7c3b1a86..8ac0adaef 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -106,7 +106,11 @@ func (in *ConnectionPooler) DeepCopyInto(out *ConnectionPooler) { *out = new(int32) **out = **in } - out.Resources = in.Resources + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(Resources) + **out = **in + } return } @@ -209,6 +213,11 @@ func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfigura (*out)[key] = val } } + if in.IgnoredAnnotations != nil { + in, out := &in.IgnoredAnnotations, &out.IgnoredAnnotations + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.InheritedLabels != nil { in, out := &in.InheritedLabels, &out.InheritedLabels *out = make([]string, len(*in)) @@ -575,7 +584,11 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { in.PostgresqlParam.DeepCopyInto(&out.PostgresqlParam) in.Volume.DeepCopyInto(&out.Volume) in.Patroni.DeepCopyInto(&out.Patroni) - out.Resources = in.Resources + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(Resources) + **out = **in + } if in.EnableConnectionPooler != nil { in, out := &in.EnableConnectionPooler, &out.EnableConnectionPooler *out = new(bool) @@ -1132,7 +1145,11 @@ func (in *ScalyrConfiguration) DeepCopy() *ScalyrConfiguration { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Sidecar) DeepCopyInto(out *Sidecar) { *out = *in - out.Resources = in.Resources + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(Resources) + **out = **in + } if in.Ports != nil { in, out := &in.Ports, &out.Ports *out = make([]corev1.ContainerPort, len(*in)) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 9c1aada79..3b94c9230 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -382,9 +382,10 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa match = false reasons = append(reasons, "new statefulset's number of replicas does not match the current one") } - if !reflect.DeepEqual(c.Statefulset.Annotations, statefulSet.Annotations) { + if changed, reason := c.compareAnnotations(c.Statefulset.Annotations, statefulSet.Annotations); changed { + match = false needsReplace = true - reasons = append(reasons, "new statefulset's annotations do not match the current one") + reasons = append(reasons, "new statefulset's annotations do not match "+reason) } needsRollUpdate, reasons = c.compareContainers("initContainers", c.Statefulset.Spec.Template.Spec.InitContainers, statefulSet.Spec.Template.Spec.InitContainers, needsRollUpdate, reasons) @@ -438,10 +439,11 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa } } - if !reflect.DeepEqual(c.Statefulset.Spec.Template.Annotations, statefulSet.Spec.Template.Annotations) { + if changed, reason := c.compareAnnotations(c.Statefulset.Spec.Template.Annotations, statefulSet.Spec.Template.Annotations); changed { + match = false needsReplace = true needsRollUpdate = true - reasons = append(reasons, "new statefulset's pod template metadata annotations does not match the current one") + reasons = append(reasons, "new statefulset's pod template metadata annotations does not match "+reason) } if !reflect.DeepEqual(c.Statefulset.Spec.Template.Spec.SecurityContext, statefulSet.Spec.Template.Spec.SecurityContext) { needsReplace = true @@ -720,6 +722,70 @@ func (c *Cluster) enforceMinResourceLimits(spec *acidv1.PostgresSpec) error { return nil } +func (c *Cluster) compareAnnotations(old, new map[string]string) (bool, string) { + ignored := make(map[string]bool) + for _, ignore := range c.OpConfig.IgnoredAnnotations { + ignored[ignore] = true + } + + // changed := false + reason := "" + + for key := range old { + if _, ok := ignored[key]; ok { + continue + } + if _, ok := new[key]; !ok { + reason += fmt.Sprintf(" Removed '%s'.", key) + } + } + + for key := range new { + if _, ok := ignored[key]; ok { + continue + } + + v, ok := old[key] + if !ok { + reason += fmt.Sprintf(" Added '%s' with value '%s'.", key, new[key]) + } else if v != new[key] { + reason += fmt.Sprintf(" '%s' changed from '%s' to '%s'.", key, v, new[key]) + } + } + + if reason != "" { + return true, reason + } + return false, "" + +} + +// SameService compares the Services +func (c *Cluster) compareServices(old, new *v1.Service) (bool, string) { + //TODO: improve comparison + if old.Spec.Type != new.Spec.Type { + return false, fmt.Sprintf("new service's type %q does not match the current one %q", + new.Spec.Type, old.Spec.Type) + } + + oldSourceRanges := old.Spec.LoadBalancerSourceRanges + newSourceRanges := new.Spec.LoadBalancerSourceRanges + + /* work around Kubernetes 1.6 serializing [] as nil. See https://github.com/kubernetes/kubernetes/issues/43203 */ + if (len(oldSourceRanges) != 0) || (len(newSourceRanges) != 0) { + if !util.IsEqualIgnoreOrder(oldSourceRanges, newSourceRanges) { + return false, "new service's LoadBalancerSourceRange does not match the current one" + } + } + + if changed, reason := c.compareAnnotations(old.Annotations, new.Annotations); changed { + return !changed, "new service's annotations does not match the current one:" + reason + } + + return true, "" + +} + // Update changes Kubernetes objects according to the new specification. Unlike the sync case, the missing object // (i.e. service) is treated as an error // logical backup cron jobs are an exception: a user-initiated Update can enable a logical backup job @@ -821,7 +887,7 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { updateFailed = true return } - if syncStatefulSet || !reflect.DeepEqual(oldSs, newSs) || !reflect.DeepEqual(oldSpec.Annotations, newSpec.Annotations) { + if syncStatefulSet || !reflect.DeepEqual(oldSs, newSs) { c.logger.Debugf("syncing statefulsets") syncStatefulSet = false // TODO: avoid generating the StatefulSet object twice by passing it to syncStatefulSet diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index 401b1bc94..c2b3118e5 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -3,6 +3,7 @@ package cluster import ( "fmt" "reflect" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -1054,6 +1055,359 @@ func TestCompareEnv(t *testing.T) { } } +func newFakeK8sServiceClient() (k8sutil.KubernetesClient, *fake.Clientset) { + clientSet := fake.NewSimpleClientset() + + return k8sutil.KubernetesClient{ + ServicesGetter: clientSet.CoreV1(), + }, clientSet +} + +func newService(ann map[string]string, svcT v1.ServiceType, lbSr []string) *v1.Service { + svc := &v1.Service{ + Spec: v1.ServiceSpec{ + Type: svcT, + LoadBalancerSourceRanges: lbSr, + }, + } + svc.Annotations = ann + return svc +} + +func TestSameService(t *testing.T) { + testName := "test comparing services" + client, _ := newFakeK8sServiceClient() + namespace := "default" + + pg := acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: "acid-fake-cluster", + Namespace: namespace, + Annotations: map[string]string{ + "deployment-time": "2022-02-02 12:00:00", + }, + }, + Spec: acidv1.PostgresSpec{ + Volume: acidv1.Volume{ + Size: "1Gi", + }, + }, + } + var cluster = New( + Config{ + OpConfig: config.Config{ + Resources: config.Resources{ + IgnoredAnnotations: []string{ + "k8s.v1.cni.cncf.io/network-status", + }, + }, + }, + }, client, pg, logger, eventRecorder) + + tests := []struct { + about string + current *v1.Service + new *v1.Service + reason string + match bool + }{ + { + about: "two equal services", + current: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeClusterIP, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeClusterIP, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: true, + }, + { + about: "services differ on service type", + current: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeClusterIP, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: false, + reason: `new service's type "LoadBalancer" does not match the current one "ClusterIP"`, + }, + { + about: "services differ on lb source ranges", + current: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{"185.249.56.0/22"}), + match: false, + reason: `new service's LoadBalancerSourceRange does not match the current one`, + }, + { + about: "new service doesn't have lb source ranges", + current: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{}), + match: false, + reason: `new service's LoadBalancerSourceRange does not match the current one`, + }, + { + about: "services differ on DNS annotation", + current: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "new_clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: false, + reason: `new service's annotations does not match the current one: 'external-dns.alpha.kubernetes.io/hostname' changed from 'clstr.acid.zalan.do' to 'new_clstr.acid.zalan.do'.`, + }, + { + about: "services differ on AWS ELB annotation", + current: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: "1800", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: false, + reason: `new service's annotations does not match the current one: 'service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout' changed from '3600' to '1800'.`, + }, + { + about: "service changes existing annotation", + current: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + "foo": "bar", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + "foo": "baz", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: false, + reason: `new service's annotations does not match the current one: 'foo' changed from 'bar' to 'baz'.`, + }, + { + about: "service changes multiple existing annotations", + current: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + "foo": "bar", + "bar": "foo", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + "foo": "baz", + "bar": "fooz", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: false, + // Test just the prefix to avoid flakiness and map sorting + reason: `new service's annotations does not match the current one:`, + }, + { + about: "service adds a new custom annotation", + current: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + "foo": "bar", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: false, + reason: `new service's annotations does not match the current one: Added 'foo' with value 'bar'.`, + }, + { + about: "service removes a custom annotation", + current: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + "foo": "bar", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: false, + reason: `new service's annotations does not match the current one: Removed 'foo'.`, + }, + { + about: "service removes a custom annotation and adds a new one", + current: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + "foo": "bar", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + "bar": "foo", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: false, + reason: `new service's annotations does not match the current one: Removed 'foo'. Added 'bar' with value 'foo'.`, + }, + { + about: "service removes a custom annotation, adds a new one and change another", + current: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + "foo": "bar", + "zalan": "do", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + "bar": "foo", + "zalan": "do.com", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: false, + // Test just the prefix to avoid flakiness and map sorting + reason: `new service's annotations does not match the current one: Removed 'foo'.`, + }, + { + about: "service add annotations", + current: newService( + map[string]string{}, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: false, + // Test just the prefix to avoid flakiness and map sorting + reason: `new service's annotations does not match the current one: Added `, + }, + { + about: "ignored annotations", + current: newService( + map[string]string{}, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newService( + map[string]string{ + "k8s.v1.cni.cncf.io/network-status": "up", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: true, + }, + } + for _, tt := range tests { + t.Run(tt.about, func(t *testing.T) { + match, reason := cluster.compareServices(tt.current, tt.new) + if match && !tt.match { + t.Logf("match=%v current=%v, old=%v reason=%s", match, tt.current.Annotations, tt.new.Annotations, reason) + t.Errorf("%s - expected services to do not match: '%q' and '%q'", testName, tt.current, tt.new) + return + } + if !match && tt.match { + t.Errorf("%s - expected services to be the same: '%q' and '%q'", testName, tt.current, tt.new) + return + } + if !match && !tt.match { + if !strings.HasPrefix(reason, tt.reason) { + t.Errorf("%s - expected reason prefix '%s', found '%s'", testName, tt.reason, reason) + return + } + } + }) + } +} + func TestCrossNamespacedSecrets(t *testing.T) { testName := "test secrets in different namespace" clientSet := fake.NewSimpleClientset() diff --git a/pkg/cluster/connection_pooler.go b/pkg/cluster/connection_pooler.go index 5639b0283..193b97f6b 100644 --- a/pkg/cluster/connection_pooler.go +++ b/pkg/cluster/connection_pooler.go @@ -910,7 +910,7 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql if service, err = c.KubeClient.Services(c.Namespace).Get(context.TODO(), c.connectionPoolerName(role), metav1.GetOptions{}); err == nil { c.ConnectionPooler[role].Service = service desiredSvc := c.generateConnectionPoolerService(c.ConnectionPooler[role]) - if match, reason := k8sutil.SameService(service, desiredSvc); !match { + if match, reason := c.compareServices(service, desiredSvc); !match { syncReason = append(syncReason, reason) c.logServiceChanges(role, service, desiredSvc, false, reason) newService, err = c.updateService(role, service, desiredSvc) diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 4548a5b14..8fd4fd5d4 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -172,7 +172,7 @@ func (c *Cluster) syncService(role PostgresRole) error { if svc, err = c.KubeClient.Services(c.Namespace).Get(context.TODO(), c.serviceName(role), metav1.GetOptions{}); err == nil { c.Services[role] = svc desiredSvc := c.generateService(role, &c.Spec) - if match, reason := k8sutil.SameService(svc, desiredSvc); !match { + if match, reason := c.compareServices(svc, desiredSvc); !match { c.logServiceChanges(role, svc, desiredSvc, false, reason) updatedSvc, err := c.updateService(role, svc, desiredSvc) if err != nil { diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index de0dec69f..d62036c67 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -186,6 +186,7 @@ func (c *Controller) modifyConfigFromEnvironment() { if c.config.NoTeamsAPI { c.opConfig.EnableTeamsAPI = false } + scalyrAPIKey := os.Getenv("SCALYR_API_KEY") if scalyrAPIKey != "" { c.opConfig.ScalyrAPIKey = scalyrAPIKey diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index be7118d9e..8e4ad1f69 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -108,6 +108,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.InheritedLabels = fromCRD.Kubernetes.InheritedLabels result.InheritedAnnotations = fromCRD.Kubernetes.InheritedAnnotations result.DownscalerAnnotations = fromCRD.Kubernetes.DownscalerAnnotations + result.IgnoredAnnotations = fromCRD.Kubernetes.IgnoredAnnotations result.ClusterNameLabel = util.Coalesce(fromCRD.Kubernetes.ClusterNameLabel, "cluster-name") result.DeleteAnnotationDateKey = fromCRD.Kubernetes.DeleteAnnotationDateKey result.DeleteAnnotationNameKey = fromCRD.Kubernetes.DeleteAnnotationNameKey diff --git a/pkg/spec/types.go b/pkg/spec/types.go index 428bcbdc6..e594a7978 100644 --- a/pkg/spec/types.go +++ b/pkg/spec/types.go @@ -119,6 +119,7 @@ type ControllerConfig struct { CRDReadyWaitTimeout time.Duration ConfigMapName NamespacedName Namespace string + IgnoredAnnotations []string EnableJsonLogging bool } diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index e6b2c03c0..eb2eb07cf 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -42,6 +42,7 @@ type Resources struct { InheritedLabels []string `name:"inherited_labels" default:""` InheritedAnnotations []string `name:"inherited_annotations" default:""` DownscalerAnnotations []string `name:"downscaler_annotations"` + IgnoredAnnotations []string `name:"ignored_annotations"` ClusterNameLabel string `name:"cluster_name_label" default:"cluster-name"` DeleteAnnotationDateKey string `name:"delete_annotation_date_key"` DeleteAnnotationNameKey string `name:"delete_annotation_name_key"` diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index fd5de8195..f4ea014f2 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -213,57 +213,6 @@ func (client *KubernetesClient) SetPostgresCRDStatus(clusterName spec.Namespaced return pg, nil } -// SameService compares the Services -func SameService(cur, new *v1.Service) (match bool, reason string) { - //TODO: improve comparison - if cur.Spec.Type != new.Spec.Type { - return false, fmt.Sprintf("new service's type %q does not match the current one %q", - new.Spec.Type, cur.Spec.Type) - } - - oldSourceRanges := cur.Spec.LoadBalancerSourceRanges - newSourceRanges := new.Spec.LoadBalancerSourceRanges - - /* work around Kubernetes 1.6 serializing [] as nil. See https://github.com/kubernetes/kubernetes/issues/43203 */ - if (len(oldSourceRanges) != 0) || (len(newSourceRanges) != 0) { - if !reflect.DeepEqual(oldSourceRanges, newSourceRanges) { - return false, "new service's LoadBalancerSourceRange does not match the current one" - } - } - - match = true - - reasonPrefix := "new service's annotations does not match the current one:" - for ann := range cur.Annotations { - if _, ok := new.Annotations[ann]; !ok { - match = false - if len(reason) == 0 { - reason = reasonPrefix - } - reason += fmt.Sprintf(" Removed '%s'.", ann) - } - } - - for ann := range new.Annotations { - v, ok := cur.Annotations[ann] - if !ok { - if len(reason) == 0 { - reason = reasonPrefix - } - reason += fmt.Sprintf(" Added '%s' with value '%s'.", ann, new.Annotations[ann]) - match = false - } else if v != new.Annotations[ann] { - if len(reason) == 0 { - reason = reasonPrefix - } - reason += fmt.Sprintf(" '%s' changed from '%s' to '%s'.", ann, v, new.Annotations[ann]) - match = false - } - } - - return match, reason -} - // SamePDB compares the PodDisruptionBudgets func SamePDB(cur, new *policybeta1.PodDisruptionBudget) (match bool, reason string) { //TODO: improve comparison diff --git a/pkg/util/k8sutil/k8sutil_test.go b/pkg/util/k8sutil/k8sutil_test.go deleted file mode 100644 index b3e768501..000000000 --- a/pkg/util/k8sutil/k8sutil_test.go +++ /dev/null @@ -1,311 +0,0 @@ -package k8sutil - -import ( - "strings" - "testing" - - "github.com/zalando/postgres-operator/pkg/util/constants" - - v1 "k8s.io/api/core/v1" -) - -func newsService(ann map[string]string, svcT v1.ServiceType, lbSr []string) *v1.Service { - svc := &v1.Service{ - Spec: v1.ServiceSpec{ - Type: svcT, - LoadBalancerSourceRanges: lbSr, - }, - } - svc.Annotations = ann - return svc -} - -func TestSameService(t *testing.T) { - tests := []struct { - about string - current *v1.Service - new *v1.Service - reason string - match bool - }{ - { - about: "two equal services", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeClusterIP, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeClusterIP, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: true, - }, - { - about: "services differ on service type", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeClusterIP, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - reason: `new service's type "LoadBalancer" does not match the current one "ClusterIP"`, - }, - { - about: "services differ on lb source ranges", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{"185.249.56.0/22"}), - match: false, - reason: `new service's LoadBalancerSourceRange does not match the current one`, - }, - { - about: "new service doesn't have lb source ranges", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{}), - match: false, - reason: `new service's LoadBalancerSourceRange does not match the current one`, - }, - { - about: "services differ on DNS annotation", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "new_clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - reason: `new service's annotations does not match the current one: 'external-dns.alpha.kubernetes.io/hostname' changed from 'clstr.acid.zalan.do' to 'new_clstr.acid.zalan.do'.`, - }, - { - about: "services differ on AWS ELB annotation", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: "1800", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - reason: `new service's annotations does not match the current one: 'service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout' changed from '3600' to '1800'.`, - }, - { - about: "service changes existing annotation", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - "foo": "bar", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - "foo": "baz", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - reason: `new service's annotations does not match the current one: 'foo' changed from 'bar' to 'baz'.`, - }, - { - about: "service changes multiple existing annotations", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - "foo": "bar", - "bar": "foo", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - "foo": "baz", - "bar": "fooz", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - // Test just the prefix to avoid flakiness and map sorting - reason: `new service's annotations does not match the current one:`, - }, - { - about: "service adds a new custom annotation", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - "foo": "bar", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - reason: `new service's annotations does not match the current one: Added 'foo' with value 'bar'.`, - }, - { - about: "service removes a custom annotation", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - "foo": "bar", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - reason: `new service's annotations does not match the current one: Removed 'foo'.`, - }, - { - about: "service removes a custom annotation and adds a new one", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - "foo": "bar", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - "bar": "foo", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - reason: `new service's annotations does not match the current one: Removed 'foo'. Added 'bar' with value 'foo'.`, - }, - { - about: "service removes a custom annotation, adds a new one and change another", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - "foo": "bar", - "zalan": "do", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - "bar": "foo", - "zalan": "do.com", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - // Test just the prefix to avoid flakiness and map sorting - reason: `new service's annotations does not match the current one: Removed 'foo'.`, - }, - { - about: "service add annotations", - current: newsService( - map[string]string{}, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - // Test just the prefix to avoid flakiness and map sorting - reason: `new service's annotations does not match the current one: Added `, - }, - } - for _, tt := range tests { - t.Run(tt.about, func(t *testing.T) { - match, reason := SameService(tt.current, tt.new) - if match && !tt.match { - t.Errorf("expected services to do not match: '%q' and '%q'", tt.current, tt.new) - return - } - if !match && tt.match { - t.Errorf("expected services to be the same: '%q' and '%q'", tt.current, tt.new) - return - } - if !match && !tt.match { - if !strings.HasPrefix(reason, tt.reason) { - t.Errorf("expected reason prefix '%s', found '%s'", tt.reason, reason) - return - } - } - }) - } -}