feat: add ignored annotations

cmd/main.go

@@ -2,13 +2,15 @@ package main
 
 import (
 	"flag"
-	log "github.com/sirupsen/logrus"
 	"os"
 	"os/signal"
+	"strings"
 	"sync"
 	"syscall"
 	"time"
 
+	log "github.com/sirupsen/logrus"
+
 	"github.com/zalando/postgres-operator/pkg/controller"
 	"github.com/zalando/postgres-operator/pkg/spec"
 	"github.com/zalando/postgres-operator/pkg/util/k8sutil"
@@ -36,6 +38,7 @@ func init() {
 	flag.BoolVar(&config.NoTeamsAPI, "noteamsapi", false, "Disable all access to the teams API")
 	flag.Parse()
 
+	config.IgnoredAnnotations = strings.Split(os.Getenv("IGNORED_ANNOTATIONS"), ",")
 	config.EnableJsonLogging = os.Getenv("ENABLE_JSON_LOGGING") == "true"
 
 	configMapRawName := os.Getenv("CONFIG_MAP_NAME")

pkg/apis/acid.zalan.do/v1/operator_configuration_type.go

@@ -74,6 +74,7 @@ type KubernetesMetaConfiguration struct {
 	InfrastructureRolesDefs                []*config.InfrastructureRole `json:"infrastructure_roles_secrets,omitempty"`
 	PodRoleLabel                           string                       `json:"pod_role_label,omitempty"`
 	ClusterLabels                          map[string]string            `json:"cluster_labels,omitempty"`
+	IgnoredAnnotations                     []string                     `json:"ignored_annotations,omitempty"`
 	InheritedLabels                        []string                     `json:"inherited_labels,omitempty"`
 	InheritedAnnotations                   []string                     `json:"inherited_annotations,omitempty"`
 	DownscalerAnnotations                  []string                     `json:"downscaler_annotations,omitempty"`

pkg/cluster/cluster.go

@@ -359,10 +359,10 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa
 		match = false
 		reasons = append(reasons, "new statefulset's number of replicas does not match the current one")
 	}
-	if !reflect.DeepEqual(c.Statefulset.Annotations, statefulSet.Annotations) {
+	if changed, reason := c.compareAnnotations(c.Statefulset.Annotations, statefulSet.Annotations); changed {
 		match = false
 		needsReplace = true
-		reasons = append(reasons, "new statefulset's annotations do not match the current one")
+		reasons = append(reasons, "new statefulset's annotations do not match "+reason)
 	}
 
 	needsRollUpdate, reasons = c.compareContainers("initContainers", c.Statefulset.Spec.Template.Spec.InitContainers, statefulSet.Spec.Template.Spec.InitContainers, needsRollUpdate, reasons)
@@ -411,11 +411,11 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa
 		}
 	}
 
-	if !reflect.DeepEqual(c.Statefulset.Spec.Template.Annotations, statefulSet.Spec.Template.Annotations) {
+	if changed, reason := c.compareAnnotations(c.Statefulset.Spec.Template.Annotations, statefulSet.Spec.Template.Annotations); changed {
 		match = false
 		needsReplace = true
 		needsRollUpdate = true
-		reasons = append(reasons, "new statefulset's pod template metadata annotations does not match the current one")
+		reasons = append(reasons, "new statefulset's pod template metadata annotations does not match "+reason)
 	}
 	if !reflect.DeepEqual(c.Statefulset.Spec.Template.Spec.SecurityContext, statefulSet.Spec.Template.Spec.SecurityContext) {
 		match = false
@@ -689,7 +689,8 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
 			updateFailed = true
 			return
 		}
-		if syncStatefulSet || !reflect.DeepEqual(oldSs, newSs) || !reflect.DeepEqual(oldSpec.Annotations, newSpec.Annotations) {
+		annotationsChanged, _ := c.compareAnnotations(oldSpec.Annotations, newSpec.Annotations)
+		if syncStatefulSet || !reflect.DeepEqual(oldSs, newSs) || annotationsChanged {
 			c.logger.Debugf("syncing statefulsets")
 			syncStatefulSet = false
 			// TODO: avoid generating the StatefulSet object twice by passing it to syncStatefulSet

pkg/cluster/comparison_test.go

@@ -1,9 +1,10 @@
-package k8sutil
+package cluster
 
 import (
 	"strings"
 	"testing"
 
+	"github.com/zalando/postgres-operator/pkg/util/config"
 	"github.com/zalando/postgres-operator/pkg/util/constants"
 
 	v1 "k8s.io/api/core/v1"
@@ -21,6 +22,15 @@ func newsService(ann map[string]string, svcT v1.ServiceType, lbSr []string) *v1.
 }
 
 func TestSameService(t *testing.T) {
+	cluster := Cluster{
+		Config: Config{
+			OpConfig: config.Config{
+				IgnoredAnnotations: []string{
+					"k8s.v1.cni.cncf.io/network-status",
+				},
+			},
+		},
+	}
 	tests := []struct {
 		about   string
 		current *v1.Service
@@ -288,11 +298,26 @@ func TestSameService(t *testing.T) {
 			// Test just the prefix to avoid flakiness and map sorting
 			reason: `new service's annotations does not match the current one: Added `,
 		},
+		{
+			about: "ignored annotations",
+			current: newsService(
+				map[string]string{},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			new: newsService(
+				map[string]string{
+					"k8s.v1.cni.cncf.io/network-status": "up",
+				},
+				v1.ServiceTypeLoadBalancer,
+				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+			match: true,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.about, func(t *testing.T) {
-			match, reason := SameService(tt.current, tt.new)
+			match, reason := cluster.SameService(tt.current, tt.new)
 			if match && !tt.match {
+				t.Logf("match=%v current=%v, old=%v reason=%s", match, tt.current.Annotations, tt.new.Annotations, reason)
 				t.Errorf("expected services to do not match: '%q' and '%q'", tt.current, tt.new)
 				return
 			}

pkg/cluster/sync.go

@@ -153,7 +153,7 @@ func (c *Cluster) syncService(role PostgresRole) error {
 	if svc, err = c.KubeClient.Services(c.Namespace).Get(context.TODO(), c.serviceName(role), metav1.GetOptions{}); err == nil {
 		c.Services[role] = svc
 		desiredSvc := c.generateService(role, &c.Spec)
-		if match, reason := k8sutil.SameService(svc, desiredSvc); !match {
+		if match, reason := c.SameService(svc, desiredSvc); !match {
 			c.logServiceChanges(role, svc, desiredSvc, false, reason)
 			if err = c.updateService(role, desiredSvc); err != nil {
 				return fmt.Errorf("could not update %s service to match desired state: %v", role, err)

pkg/cluster/util.go

@@ -186,6 +186,70 @@ func logNiceDiff(log *logrus.Entry, old, new interface{}) {
 	}
 }
 
+func (c *Cluster) compareAnnotations(old, new map[string]string) (bool, string) {
+	ignored := make(map[string]bool)
+	for _, ignore := range c.OpConfig.IgnoredAnnotations {
+		ignored[ignore] = true
+	}
+
+	// changed := false
+	reason := ""
+
+	for key := range old {
+		if _, ok := ignored[key]; ok {
+			continue
+		}
+		if _, ok := new[key]; !ok {
+			reason += fmt.Sprintf(" Removed '%s'.", key)
+		}
+	}
+
+	for key := range new {
+		if _, ok := ignored[key]; ok {
+			continue
+		}
+
+		v, ok := old[key]
+		if !ok {
+			reason += fmt.Sprintf(" Added '%s' with value '%s'.", key, new[key])
+		} else if v != new[key] {
+			reason += fmt.Sprintf(" '%s' changed from '%s' to '%s'.", key, v, new[key])
+		}
+	}
+
+	if reason != "" {
+		return true, reason
+	}
+	return false, ""
+
+}
+
+// SameService compares the Services
+func (c *Cluster) SameService(old, new *v1.Service) (bool, string) {
+	//TODO: improve comparison
+	if old.Spec.Type != new.Spec.Type {
+		return false, fmt.Sprintf("new service's type %q does not match the current one %q",
+			new.Spec.Type, old.Spec.Type)
+	}
+
+	oldSourceRanges := old.Spec.LoadBalancerSourceRanges
+	newSourceRanges := new.Spec.LoadBalancerSourceRanges
+
+	/* work around Kubernetes 1.6 serializing [] as nil. See https://github.com/kubernetes/kubernetes/issues/43203 */
+	if (len(oldSourceRanges) != 0) || (len(newSourceRanges) != 0) {
+		if !reflect.DeepEqual(oldSourceRanges, newSourceRanges) {
+			return false, "new service's LoadBalancerSourceRange does not match the current one"
+		}
+	}
+
+	if changed, reason := c.compareAnnotations(old.Annotations, new.Annotations); changed {
+		return !changed, "new service's annotations does not match the current one:" + reason
+	}
+
+	return true, ""
+
+}
+
 func (c *Cluster) logStatefulSetChanges(old, new *appsv1.StatefulSet, isUpdate bool, reasons []string) {
 	if isUpdate {
 		c.logger.Infof("statefulset %s has been changed", util.NameFromMeta(old.ObjectMeta))

pkg/controller/controller.go

@@ -184,6 +184,9 @@ func (c *Controller) modifyConfigFromEnvironment() {
 	if c.config.NoTeamsAPI {
 		c.opConfig.EnableTeamsAPI = false
 	}
+
+	c.opConfig.IgnoredAnnotations = append(c.opConfig.IgnoredAnnotations, c.config.IgnoredAnnotations...)
+
 	scalyrAPIKey := os.Getenv("SCALYR_API_KEY")
 	if scalyrAPIKey != "" {
 		c.opConfig.ScalyrAPIKey = scalyrAPIKey

pkg/controller/operator_config.go

@@ -60,6 +60,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	result.TargetMajorVersion = util.Coalesce(fromCRD.MajorVersionUpgrade.TargetMajorVersion, "13")
 
 	// kubernetes config
+	result.IgnoredAnnotations = fromCRD.Kubernetes.IgnoredAnnotations
 	result.CustomPodAnnotations = fromCRD.Kubernetes.CustomPodAnnotations
 	result.PodServiceAccountName = util.Coalesce(fromCRD.Kubernetes.PodServiceAccountName, "postgres-pod")
 	result.PodServiceAccountDefinition = fromCRD.Kubernetes.PodServiceAccountDefinition

pkg/spec/types.go

@@ -114,6 +114,7 @@ type ControllerConfig struct {
 	CRDReadyWaitTimeout  time.Duration
 	ConfigMapName        NamespacedName
 	Namespace            string
+	IgnoredAnnotations   []string
 
 	EnableJsonLogging bool
 }

pkg/util/config/config.go

@@ -181,6 +181,7 @@ type Config struct {
 	EnablePostgresTeamCRDSuperusers        bool              `name:"enable_postgres_team_crd_superusers" default:"false"`
 	EnableMasterLoadBalancer               bool              `name:"enable_master_load_balancer" default:"true"`
 	EnableReplicaLoadBalancer              bool              `name:"enable_replica_load_balancer" default:"false"`
+	IgnoredAnnotations                     []string          `json:"ignored_annotations"`
 	CustomServiceAnnotations               map[string]string `name:"custom_service_annotations"`
 	CustomPodAnnotations                   map[string]string `name:"custom_pod_annotations"`
 	EnablePodAntiAffinity                  bool              `name:"enable_pod_antiaffinity" default:"false"`

pkg/util/k8sutil/k8sutil.go

@@ -197,57 +197,6 @@ func (client *KubernetesClient) SetPostgresCRDStatus(clusterName spec.Namespaced
 	return pg, nil
 }
 
-// SameService compares the Services
-func SameService(cur, new *v1.Service) (match bool, reason string) {
-	//TODO: improve comparison
-	if cur.Spec.Type != new.Spec.Type {
-		return false, fmt.Sprintf("new service's type %q does not match the current one %q",
-			new.Spec.Type, cur.Spec.Type)
-	}
-
-	oldSourceRanges := cur.Spec.LoadBalancerSourceRanges
-	newSourceRanges := new.Spec.LoadBalancerSourceRanges
-
-	/* work around Kubernetes 1.6 serializing [] as nil. See https://github.com/kubernetes/kubernetes/issues/43203 */
-	if (len(oldSourceRanges) != 0) || (len(newSourceRanges) != 0) {
-		if !reflect.DeepEqual(oldSourceRanges, newSourceRanges) {
-			return false, "new service's LoadBalancerSourceRange does not match the current one"
-		}
-	}
-
-	match = true
-
-	reasonPrefix := "new service's annotations does not match the current one:"
-	for ann := range cur.Annotations {
-		if _, ok := new.Annotations[ann]; !ok {
-			match = false
-			if len(reason) == 0 {
-				reason = reasonPrefix
-			}
-			reason += fmt.Sprintf(" Removed '%s'.", ann)
-		}
-	}
-
-	for ann := range new.Annotations {
-		v, ok := cur.Annotations[ann]
-		if !ok {
-			if len(reason) == 0 {
-				reason = reasonPrefix
-			}
-			reason += fmt.Sprintf(" Added '%s' with value '%s'.", ann, new.Annotations[ann])
-			match = false
-		} else if v != new.Annotations[ann] {
-			if len(reason) == 0 {
-				reason = reasonPrefix
-			}
-			reason += fmt.Sprintf(" '%s' changed from '%s' to '%s'.", ann, v, new.Annotations[ann])
-			match = false
-		}
-	}
-
-	return match, reason
-}
-
 // SamePDB compares the PodDisruptionBudgets
 func SamePDB(cur, new *policybeta1.PodDisruptionBudget) (match bool, reason string) {
 	//TODO: improve comparison