Custom annotations 329 (#657)

* Add ability for custom annotations to database pods

charts/postgres-operator/values-crd.yaml

@@ -53,9 +53,11 @@ configKubernetes:
   cluster_domain: cluster.local
   # additional labels assigned to the cluster objects
   cluster_labels:
-      application: spilo
+    application: spilo
   # label assigned to Kubernetes objects created by the operator
   cluster_name_label: cluster-name
+  # additional annotations to add to every database pod
+  custom_pod_annotations:
   # toggles pod anti affinity on the Postgres pods
   enable_pod_antiaffinity: false
   # toggles PDB to set to MinAvailabe 0 or 1

charts/postgres-operator/values.yaml

@@ -54,6 +54,8 @@ configKubernetes:
   cluster_labels: application:spilo
   # label assigned to Kubernetes objects created by the operator
   cluster_name_label: version
+  # annotations attached to each database pod
+  # custom_pod_annotations: keya:valuea
   # toggles pod anti affinity on the Postgres pods
   enable_pod_antiaffinity: "false"
   # toggles PDB to set to MinAvailabe 0 or 1
@@ -127,8 +129,7 @@ configLoadBalancer:
   # DNS zone for cluster DNS name when load balancer is configured for cluster
   db_hosted_zone: db.example.com
   # annotations to apply to service when load balancing is enabled
-  # custom_service_annotations:
-  #   "keyx:valuez,keya:valuea"
+  # custom_service_annotations: "keyx:valuez,keya:valuea"
 
   # toggles service type load balancer pointing to the master pod of the cluster
   enable_master_load_balancer: "true"

docs/reference/cluster_manifest.md

@@ -118,6 +118,11 @@ These parameters are grouped directly under  the `spec` key in the manifest.
    then the default priority class is taken. The priority class itself must be
    defined in advance. Optional.
 
+* **podAnnotations**
+  A map of key value pairs that gets attached as [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/)
+  to each pod created for the database.
+
+
 * **enableShmVolume**
   Start a database pod without limitations on shm memory. By default docker
   limit `/dev/shm` to `64M` (see e.g. the [docker

docs/reference/operator_parameters.md

@@ -168,6 +168,11 @@ configuration they are grouped under the `kubernetes` key.
   Postgres pods are [terminated forcefully](https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods)
   after this timeout. The default is `5m`.
 
+* **custom_pod_annotations**
+  This key/value map provides a list of annotations that get attached to each pod
+  of a database created by the operator. If the annotation key is also provided
+  by the database definition, the database definition value is used.
+
 * **watched_namespace**
   The operator watches for Postgres objects in the given namespace. If not
   specified, the value is taken from the operator namespace. A special `*`

manifests/complete-postgres-manifest.yaml

@@ -25,7 +25,8 @@ spec:
   - 127.0.0.1/32
   databases:
     foo: zalando
-
+# podAnnotations:
+#   annotation.key: value
 # Expert section
 
   enableShmVolume: true

manifests/configmap.yaml

@@ -11,8 +11,8 @@ data:
   cluster_history_entries: "1000"
   cluster_labels: application:spilo
   cluster_name_label: version
-  # custom_service_annotations:
-  #   "keyx:valuez,keya:valuea"
+  # custom_service_annotations: "keyx:valuez,keya:valuea"
+  # custom_pod_annotations: "keya:valuea"
   db_hosted_zone: db.example.com
   debug_logging: "true"
   # default_cpu_limit: "3"
@@ -37,7 +37,7 @@ data:
   # logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup"
   # logical_backup_s3_bucket: "my-bucket-url"
   # logical_backup_schedule: "30 00 * * *"
-  master_dns_name_format: '{cluster}.{team}.staging.{hostedzone}'
+  master_dns_name_format: "{cluster}.{team}.staging.{hostedzone}"
   # master_pod_move_timeout: 10m
   # max_instances: "-1"
   # min_instances: "-1"
@@ -60,13 +60,13 @@ data:
   ready_wait_interval: 3s
   ready_wait_timeout: 30s
   repair_period: 5m
-  replica_dns_name_format: '{cluster}-repl.{team}.staging.{hostedzone}'
+  replica_dns_name_format: "{cluster}-repl.{team}.staging.{hostedzone}"
   replication_username: standby
   resource_check_interval: 3s
   resource_check_timeout: 10m
   resync_period: 5m
   ring_log_lines: "100"
-  secret_name_template: '{username}.{cluster}.credentials'
+  secret_name_template: "{username}.{cluster}.credentials"
   # sidecar_docker_images: ""
   # set_memory_request_to_limit: "false"
   spilo_privileged: "false"

manifests/postgresql-operator-default-configuration.yaml

@@ -22,6 +22,9 @@ configuration:
     cluster_labels:
         application: spilo
     cluster_name_label: cluster-name
+    # custom_pod_annotations:
+    #   keya: valuea
+    #   keyb: valueb
     enable_pod_antiaffinity: false
     enable_pod_disruption_budget: true
     # infrastructure_roles_secret_name: ""

pkg/apis/acid.zalan.do/v1/operator_configuration_type.go

@@ -59,6 +59,7 @@ type KubernetesMetaConfiguration struct {
 	InheritedLabels                        []string              `json:"inherited_labels,omitempty"`
 	ClusterNameLabel                       string                `json:"cluster_name_label,omitempty"`
 	NodeReadinessLabel                     map[string]string     `json:"node_readiness_label,omitempty"`
+	CustomPodAnnotations                   map[string]string     `json:"custom_pod_annotations,omitempty"`
 	// TODO: use a proper toleration structure?
 	PodToleration map[string]string `json:"toleration,omitempty"`
 	// TODO: use namespacedname

pkg/apis/acid.zalan.do/v1/postgresql_type.go

@@ -59,6 +59,7 @@ type PostgresSpec struct {
 	EnableLogicalBackup   bool                 `json:"enableLogicalBackup,omitempty"`
 	LogicalBackupSchedule string               `json:"logicalBackupSchedule,omitempty"`
 	StandbyCluster        *StandbyDescription  `json:"standby"`
+	PodAnnotations        map[string]string    `json:"podAnnotations"`
 
 	// deprecated json tags
 	InitContainersOld       []v1.Container `json:"init_containers,omitempty"`

pkg/apis/acid.zalan.do/v1/util_test.go

@@ -437,6 +437,16 @@ var postgresqlList = []struct {
 		PostgresqlList{},
 		errors.New("unexpected end of JSON input")}}
 
+var annotations = []struct {
+	in          []byte
+	annotations map[string]string
+	err         error
+}{{
+	in:          []byte(`{"kind": "Postgresql","apiVersion": "acid.zalan.do/v1","metadata": {"name": "acid-testcluster1"}, "spec": {"podAnnotations": {"foo": "bar"},"teamId": "acid", "clone": {"cluster": "team-batman"}}}`),
+	annotations: map[string]string{"foo": "bar"},
+	err:         nil},
+}
+
 func mustParseTime(s string) metav1.Time {
 	v, err := time.Parse("15:04", s)
 	if err != nil {
@@ -482,6 +492,25 @@ func TestWeekdayTime(t *testing.T) {
 	}
 }
 
+func TestClusterAnnotations(t *testing.T) {
+	for _, tt := range annotations {
+		var cluster Postgresql
+		err := cluster.UnmarshalJSON(tt.in)
+		if err != nil {
+			if tt.err == nil || err.Error() != tt.err.Error() {
+				t.Errorf("Unable to marshal cluster with annotations: expected %v got %v", tt.err, err)
+			}
+			continue
+		}
+		for k, v := range cluster.Spec.PodAnnotations {
+			found, expected := v, tt.annotations[k]
+			if found != expected {
+				t.Errorf("Didn't find correct value for key %v in for podAnnotations:  Expected %v found %v", k, expected, found)
+			}
+		}
+	}
+}
+
 func TestClusterName(t *testing.T) {
 	for _, tt := range clusterNames {
 		name, err := extractClusterName(tt.in, tt.inTeam)

pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go

@@ -102,6 +102,13 @@ func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfigura
 			(*out)[key] = val
 		}
 	}
+	if in.CustomPodAnnotations != nil {
+		in, out := &in.CustomPodAnnotations, &out.CustomPodAnnotations
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
 	if in.PodToleration != nil {
 		in, out := &in.PodToleration, &out.PodToleration
 		*out = make(map[string]string, len(*in))
@@ -513,6 +520,13 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) {
 		*out = new(StandbyDescription)
 		**out = **in
 	}
+	if in.PodAnnotations != nil {
+		in, out := &in.PodAnnotations, &out.PodAnnotations
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
 	if in.InitContainersOld != nil {
 		in, out := &in.InitContainersOld, &out.InitContainersOld
 		*out = make([]corev1.Container, len(*in))

pkg/cluster/cluster_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/zalando/postgres-operator/pkg/util/config"
 	"github.com/zalando/postgres-operator/pkg/util/k8sutil"
 	"github.com/zalando/postgres-operator/pkg/util/teams"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 )
 
 const (
@@ -328,3 +328,57 @@ func TestShouldDeleteSecret(t *testing.T) {
 		}
 	}
 }
+
+func TestPodAnnotations(t *testing.T) {
+	testName := "TestPodAnnotations"
+	tests := []struct {
+		subTest  string
+		operator map[string]string
+		database map[string]string
+		merged   map[string]string
+	}{
+		{
+			subTest:  "No Annotations",
+			operator: make(map[string]string),
+			database: make(map[string]string),
+			merged:   make(map[string]string),
+		},
+		{
+			subTest:  "Operator Config Annotations",
+			operator: map[string]string{"foo": "bar"},
+			database: make(map[string]string),
+			merged:   map[string]string{"foo": "bar"},
+		},
+		{
+			subTest:  "Database Config Annotations",
+			operator: make(map[string]string),
+			database: map[string]string{"foo": "bar"},
+			merged:   map[string]string{"foo": "bar"},
+		},
+		{
+			subTest:  "Database Config overrides Operator Config Annotations",
+			operator: map[string]string{"foo": "bar", "global": "foo"},
+			database: map[string]string{"foo": "baz", "local": "foo"},
+			merged:   map[string]string{"foo": "baz", "global": "foo", "local": "foo"},
+		},
+	}
+
+	for _, tt := range tests {
+		cl.OpConfig.CustomPodAnnotations = tt.operator
+		cl.Postgresql.Spec.PodAnnotations = tt.database
+
+		annotations := cl.generatePodAnnotations(&cl.Postgresql.Spec)
+		for k, v := range annotations {
+			if observed, expected := v, tt.merged[k]; observed != expected {
+				t.Errorf("%v expects annotation value %v for key %v, but found %v",
+					testName+"/"+tt.subTest, expected, observed, k)
+			}
+		}
+		for k, v := range tt.merged {
+			if observed, expected := annotations[k], v; observed != expected {
+				t.Errorf("%v expects annotation value %v for key %v, but found %v",
+					testName+"/"+tt.subTest, expected, observed, k)
+			}
+		}
+	}
+}

pkg/cluster/k8sres.go

@@ -430,6 +430,7 @@ func mountShmVolumeNeeded(opConfig config.Config, pgSpec *acidv1.PostgresSpec) *
 func generatePodTemplate(
 	namespace string,
 	labels labels.Set,
+	annotations map[string]string,
 	spiloContainer *v1.Container,
 	initContainers []v1.Container,
 	sidecarContainers []v1.Container,
@@ -485,13 +486,17 @@ func generatePodTemplate(
 
 	template := v1.PodTemplateSpec{
 		ObjectMeta: metav1.ObjectMeta{
-			Labels:    labels,
-			Namespace: namespace,
+			Labels:      labels,
+			Namespace:   namespace,
+			Annotations: annotations,
 		},
 		Spec: podSpec,
 	}
 	if kubeIAMRole != "" {
-		template.Annotations = map[string]string{constants.KubeIAmAnnotation: kubeIAMRole}
+		if template.Annotations == nil{
+			template.Annotations = make(map[string]string)
+		}
+		template.Annotations[constants.KubeIAmAnnotation] = kubeIAMRole
 	}
 
 	return &template, nil
@@ -881,10 +886,13 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
 		effectiveFSGroup = spec.SpiloFSGroup
 	}
 
+	annotations := c.generatePodAnnotations(spec)
+
 	// generate pod template for the statefulset, based on the spilo container and sidecars
 	if podTemplate, err = generatePodTemplate(
 		c.Namespace,
 		c.labelsSet(true),
+		annotations,
 		spiloContainer,
 		spec.InitContainers,
 		sidecarContainers,
@@ -949,6 +957,24 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
 	return statefulSet, nil
 }
 
+func (c *Cluster) generatePodAnnotations(spec *acidv1.PostgresSpec) map[string]string {
+	annotations := make(map[string]string)
+	for k, v := range c.OpConfig.CustomPodAnnotations {
+		annotations[k] = v
+	}
+	if spec != nil || spec.PodAnnotations != nil {
+		for k, v := range spec.PodAnnotations {
+			annotations[k] = v
+		}
+	}
+	
+	if len(annotations) == 0 {
+		return nil
+	}
+
+	return annotations
+}
+
 func generateScalyrSidecarSpec(clusterName, APIKey, serverURL, dockerImage string,
 	containerResources *acidv1.Resources, logger *logrus.Entry) *acidv1.Sidecar {
 	if APIKey == "" || dockerImage == "" {
@@ -1462,10 +1488,13 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) {
 			},
 		}}
 
+	annotations := c.generatePodAnnotations(&c.Spec)
+
 	// re-use the method that generates DB pod templates
 	if podTemplate, err = generatePodTemplate(
 		c.Namespace,
 		labels,
+		annotations,
 		logicalBackupContainer,
 		[]v1.Container{},
 		[]v1.Container{},

pkg/controller/operator_config.go

@@ -41,6 +41,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	result.ReplicationUsername = fromCRD.PostgresUsersConfiguration.ReplicationUsername
 
 	// kubernetes config
+	result.CustomPodAnnotations = fromCRD.Kubernetes.CustomPodAnnotations
 	result.PodServiceAccountName = fromCRD.Kubernetes.PodServiceAccountName
 	result.PodServiceAccountDefinition = fromCRD.Kubernetes.PodServiceAccountDefinition
 	result.PodServiceAccountRoleBindingDefinition = fromCRD.Kubernetes.PodServiceAccountRoleBindingDefinition

pkg/util/config/config.go

@@ -109,6 +109,7 @@ type Config struct {
 	EnableMasterLoadBalancer               bool              `name:"enable_master_load_balancer" default:"true"`
 	EnableReplicaLoadBalancer              bool              `name:"enable_replica_load_balancer" default:"false"`
 	CustomServiceAnnotations               map[string]string `name:"custom_service_annotations"`
+	CustomPodAnnotations                   map[string]string `name:"custom_pod_annotations"`
 	EnablePodAntiAffinity                  bool              `name:"enable_pod_antiaffinity" default:"false"`
 	PodAntiAffinityTopologyKey             string            `name:"pod_antiaffinity_topology_key" default:"kubernetes.io/hostname"`
 	// deprecated and kept for backward compatibility