diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 2728b245c..e1ed381cc 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -53,9 +53,11 @@ configKubernetes: cluster_domain: cluster.local # additional labels assigned to the cluster objects cluster_labels: - application: spilo + application: spilo # label assigned to Kubernetes objects created by the operator cluster_name_label: cluster-name + # additional annotations to add to every database pod + custom_pod_annotations: # toggles pod anti affinity on the Postgres pods enable_pod_antiaffinity: false # toggles PDB to set to MinAvailabe 0 or 1 diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index a14c8ab92..2af14984b 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -54,6 +54,8 @@ configKubernetes: cluster_labels: application:spilo # label assigned to Kubernetes objects created by the operator cluster_name_label: version + # annotations attached to each database pod + # custom_pod_annotations: keya:valuea # toggles pod anti affinity on the Postgres pods enable_pod_antiaffinity: "false" # toggles PDB to set to MinAvailabe 0 or 1 @@ -127,8 +129,7 @@ configLoadBalancer: # DNS zone for cluster DNS name when load balancer is configured for cluster db_hosted_zone: db.example.com # annotations to apply to service when load balancing is enabled - # custom_service_annotations: - # "keyx:valuez,keya:valuea" + # custom_service_annotations: "keyx:valuez,keya:valuea" # toggles service type load balancer pointing to the master pod of the cluster enable_master_load_balancer: "true" diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index edaada9b4..cf522d73d 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -118,6 +118,11 @@ These parameters are grouped directly under the `spec` key in the manifest. then the default priority class is taken. The priority class itself must be defined in advance. Optional. +* **podAnnotations** + A map of key value pairs that gets attached as [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) + to each pod created for the database. + + * **enableShmVolume** Start a database pod without limitations on shm memory. By default docker limit `/dev/shm` to `64M` (see e.g. the [docker diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 41e62cc7f..9e43919fd 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -168,6 +168,11 @@ configuration they are grouped under the `kubernetes` key. Postgres pods are [terminated forcefully](https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods) after this timeout. The default is `5m`. +* **custom_pod_annotations** + This key/value map provides a list of annotations that get attached to each pod + of a database created by the operator. If the annotation key is also provided + by the database definition, the database definition value is used. + * **watched_namespace** The operator watches for Postgres objects in the given namespace. If not specified, the value is taken from the operator namespace. A special `*` diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index d1bd471db..402d007e6 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -25,7 +25,8 @@ spec: - 127.0.0.1/32 databases: foo: zalando - +# podAnnotations: +# annotation.key: value # Expert section enableShmVolume: true diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index f9223f3fe..879223a57 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -11,8 +11,8 @@ data: cluster_history_entries: "1000" cluster_labels: application:spilo cluster_name_label: version - # custom_service_annotations: - # "keyx:valuez,keya:valuea" + # custom_service_annotations: "keyx:valuez,keya:valuea" + # custom_pod_annotations: "keya:valuea" db_hosted_zone: db.example.com debug_logging: "true" # default_cpu_limit: "3" @@ -37,7 +37,7 @@ data: # logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup" # logical_backup_s3_bucket: "my-bucket-url" # logical_backup_schedule: "30 00 * * *" - master_dns_name_format: '{cluster}.{team}.staging.{hostedzone}' + master_dns_name_format: "{cluster}.{team}.staging.{hostedzone}" # master_pod_move_timeout: 10m # max_instances: "-1" # min_instances: "-1" @@ -60,13 +60,13 @@ data: ready_wait_interval: 3s ready_wait_timeout: 30s repair_period: 5m - replica_dns_name_format: '{cluster}-repl.{team}.staging.{hostedzone}' + replica_dns_name_format: "{cluster}-repl.{team}.staging.{hostedzone}" replication_username: standby resource_check_interval: 3s resource_check_timeout: 10m resync_period: 5m ring_log_lines: "100" - secret_name_template: '{username}.{cluster}.credentials' + secret_name_template: "{username}.{cluster}.credentials" # sidecar_docker_images: "" # set_memory_request_to_limit: "false" spilo_privileged: "false" diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index ad4d028c3..e1ecd1038 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -22,6 +22,9 @@ configuration: cluster_labels: application: spilo cluster_name_label: cluster-name + # custom_pod_annotations: + # keya: valuea + # keyb: valueb enable_pod_antiaffinity: false enable_pod_disruption_budget: true # infrastructure_roles_secret_name: "" diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 02648b83b..6adfb778e 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -59,6 +59,7 @@ type KubernetesMetaConfiguration struct { InheritedLabels []string `json:"inherited_labels,omitempty"` ClusterNameLabel string `json:"cluster_name_label,omitempty"` NodeReadinessLabel map[string]string `json:"node_readiness_label,omitempty"` + CustomPodAnnotations map[string]string `json:"custom_pod_annotations,omitempty"` // TODO: use a proper toleration structure? PodToleration map[string]string `json:"toleration,omitempty"` // TODO: use namespacedname diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 72e40e122..515a73ff0 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -59,6 +59,7 @@ type PostgresSpec struct { EnableLogicalBackup bool `json:"enableLogicalBackup,omitempty"` LogicalBackupSchedule string `json:"logicalBackupSchedule,omitempty"` StandbyCluster *StandbyDescription `json:"standby"` + PodAnnotations map[string]string `json:"podAnnotations"` // deprecated json tags InitContainersOld []v1.Container `json:"init_containers,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/util_test.go b/pkg/apis/acid.zalan.do/v1/util_test.go index 6c1b63ece..cf3b080a5 100644 --- a/pkg/apis/acid.zalan.do/v1/util_test.go +++ b/pkg/apis/acid.zalan.do/v1/util_test.go @@ -437,6 +437,16 @@ var postgresqlList = []struct { PostgresqlList{}, errors.New("unexpected end of JSON input")}} +var annotations = []struct { + in []byte + annotations map[string]string + err error +}{{ + in: []byte(`{"kind": "Postgresql","apiVersion": "acid.zalan.do/v1","metadata": {"name": "acid-testcluster1"}, "spec": {"podAnnotations": {"foo": "bar"},"teamId": "acid", "clone": {"cluster": "team-batman"}}}`), + annotations: map[string]string{"foo": "bar"}, + err: nil}, +} + func mustParseTime(s string) metav1.Time { v, err := time.Parse("15:04", s) if err != nil { @@ -482,6 +492,25 @@ func TestWeekdayTime(t *testing.T) { } } +func TestClusterAnnotations(t *testing.T) { + for _, tt := range annotations { + var cluster Postgresql + err := cluster.UnmarshalJSON(tt.in) + if err != nil { + if tt.err == nil || err.Error() != tt.err.Error() { + t.Errorf("Unable to marshal cluster with annotations: expected %v got %v", tt.err, err) + } + continue + } + for k, v := range cluster.Spec.PodAnnotations { + found, expected := v, tt.annotations[k] + if found != expected { + t.Errorf("Didn't find correct value for key %v in for podAnnotations: Expected %v found %v", k, expected, found) + } + } + } +} + func TestClusterName(t *testing.T) { for _, tt := range clusterNames { name, err := extractClusterName(tt.in, tt.inTeam) diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 8e3411219..793f236a5 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -102,6 +102,13 @@ func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfigura (*out)[key] = val } } + if in.CustomPodAnnotations != nil { + in, out := &in.CustomPodAnnotations, &out.CustomPodAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.PodToleration != nil { in, out := &in.PodToleration, &out.PodToleration *out = make(map[string]string, len(*in)) @@ -513,6 +520,13 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { *out = new(StandbyDescription) **out = **in } + if in.PodAnnotations != nil { + in, out := &in.PodAnnotations, &out.PodAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.InitContainersOld != nil { in, out := &in.InitContainersOld, &out.InitContainersOld *out = make([]corev1.Container, len(*in)) diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index 6f10aae22..85d014d8a 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -11,7 +11,7 @@ import ( "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/k8sutil" "github.com/zalando/postgres-operator/pkg/util/teams" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" ) const ( @@ -328,3 +328,57 @@ func TestShouldDeleteSecret(t *testing.T) { } } } + +func TestPodAnnotations(t *testing.T) { + testName := "TestPodAnnotations" + tests := []struct { + subTest string + operator map[string]string + database map[string]string + merged map[string]string + }{ + { + subTest: "No Annotations", + operator: make(map[string]string), + database: make(map[string]string), + merged: make(map[string]string), + }, + { + subTest: "Operator Config Annotations", + operator: map[string]string{"foo": "bar"}, + database: make(map[string]string), + merged: map[string]string{"foo": "bar"}, + }, + { + subTest: "Database Config Annotations", + operator: make(map[string]string), + database: map[string]string{"foo": "bar"}, + merged: map[string]string{"foo": "bar"}, + }, + { + subTest: "Database Config overrides Operator Config Annotations", + operator: map[string]string{"foo": "bar", "global": "foo"}, + database: map[string]string{"foo": "baz", "local": "foo"}, + merged: map[string]string{"foo": "baz", "global": "foo", "local": "foo"}, + }, + } + + for _, tt := range tests { + cl.OpConfig.CustomPodAnnotations = tt.operator + cl.Postgresql.Spec.PodAnnotations = tt.database + + annotations := cl.generatePodAnnotations(&cl.Postgresql.Spec) + for k, v := range annotations { + if observed, expected := v, tt.merged[k]; observed != expected { + t.Errorf("%v expects annotation value %v for key %v, but found %v", + testName+"/"+tt.subTest, expected, observed, k) + } + } + for k, v := range tt.merged { + if observed, expected := annotations[k], v; observed != expected { + t.Errorf("%v expects annotation value %v for key %v, but found %v", + testName+"/"+tt.subTest, expected, observed, k) + } + } + } +} diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 78d128387..ddadf8d52 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -430,6 +430,7 @@ func mountShmVolumeNeeded(opConfig config.Config, pgSpec *acidv1.PostgresSpec) * func generatePodTemplate( namespace string, labels labels.Set, + annotations map[string]string, spiloContainer *v1.Container, initContainers []v1.Container, sidecarContainers []v1.Container, @@ -485,13 +486,17 @@ func generatePodTemplate( template := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - Namespace: namespace, + Labels: labels, + Namespace: namespace, + Annotations: annotations, }, Spec: podSpec, } if kubeIAMRole != "" { - template.Annotations = map[string]string{constants.KubeIAmAnnotation: kubeIAMRole} + if template.Annotations == nil{ + template.Annotations = make(map[string]string) + } + template.Annotations[constants.KubeIAmAnnotation] = kubeIAMRole } return &template, nil @@ -881,10 +886,13 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef effectiveFSGroup = spec.SpiloFSGroup } + annotations := c.generatePodAnnotations(spec) + // generate pod template for the statefulset, based on the spilo container and sidecars if podTemplate, err = generatePodTemplate( c.Namespace, c.labelsSet(true), + annotations, spiloContainer, spec.InitContainers, sidecarContainers, @@ -949,6 +957,24 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef return statefulSet, nil } +func (c *Cluster) generatePodAnnotations(spec *acidv1.PostgresSpec) map[string]string { + annotations := make(map[string]string) + for k, v := range c.OpConfig.CustomPodAnnotations { + annotations[k] = v + } + if spec != nil || spec.PodAnnotations != nil { + for k, v := range spec.PodAnnotations { + annotations[k] = v + } + } + + if len(annotations) == 0 { + return nil + } + + return annotations +} + func generateScalyrSidecarSpec(clusterName, APIKey, serverURL, dockerImage string, containerResources *acidv1.Resources, logger *logrus.Entry) *acidv1.Sidecar { if APIKey == "" || dockerImage == "" { @@ -1462,10 +1488,13 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) { }, }} + annotations := c.generatePodAnnotations(&c.Spec) + // re-use the method that generates DB pod templates if podTemplate, err = generatePodTemplate( c.Namespace, labels, + annotations, logicalBackupContainer, []v1.Container{}, []v1.Container{}, diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 9da4bbaf8..b91fd511f 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -41,6 +41,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.ReplicationUsername = fromCRD.PostgresUsersConfiguration.ReplicationUsername // kubernetes config + result.CustomPodAnnotations = fromCRD.Kubernetes.CustomPodAnnotations result.PodServiceAccountName = fromCRD.Kubernetes.PodServiceAccountName result.PodServiceAccountDefinition = fromCRD.Kubernetes.PodServiceAccountDefinition result.PodServiceAccountRoleBindingDefinition = fromCRD.Kubernetes.PodServiceAccountRoleBindingDefinition diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 5dd25d401..bdbbf0fdc 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -109,6 +109,7 @@ type Config struct { EnableMasterLoadBalancer bool `name:"enable_master_load_balancer" default:"true"` EnableReplicaLoadBalancer bool `name:"enable_replica_load_balancer" default:"false"` CustomServiceAnnotations map[string]string `name:"custom_service_annotations"` + CustomPodAnnotations map[string]string `name:"custom_pod_annotations"` EnablePodAntiAffinity bool `name:"enable_pod_antiaffinity" default:"false"` PodAntiAffinityTopologyKey string `name:"pod_antiaffinity_topology_key" default:"kubernetes.io/hostname"` // deprecated and kept for backward compatibility