From d52296c3235956ea4cb34f78f6e4cf0aa374d09d Mon Sep 17 00:00:00 2001 From: Rafia Sabih Date: Mon, 4 May 2020 14:46:56 +0200 Subject: [PATCH 1/2] Propagate annotations to the StatefulSet (#932) * Initial commit * Corrections - set the type of the new configuration parameter to be array of strings - propagate the annotations to statefulset at sync * Enable regular expression matching * Improvements -handle rollingUpdate flag -modularize code -rename config parameter name * fix merge error * Pass annotations to connection pooler deployment * update code-gen * Add documentation and update manifests * add e2e test and introduce option in configmap * fix service annotations test * Add unit test * fix e2e tests * better key lookup of annotations tests * add debug message for annotation tests * Fix typos * minor fix for looping * Handle update path and renaming - handle the update path to update sts and connection pooler deployment. This way no need to wait for sync - rename the parameter to downscaler_annotations - handle other review comments * another try to fix python loops * Avoid unneccessary update events * Update manifests * some final polishing * fix cluster_test after polishing Co-authored-by: Rafia Sabih Co-authored-by: Felix Kunde --- .../crds/operatorconfigurations.yaml | 4 ++ charts/postgres-operator/values-crd.yaml | 9 ++- charts/postgres-operator/values.yaml | 3 + docs/reference/cluster_manifest.md | 2 +- docs/reference/operator_parameters.md | 6 ++ e2e/tests/test_e2e.py | 63 ++++++++++++++++--- manifests/configmap.yaml | 1 + manifests/operatorconfiguration.crd.yaml | 4 ++ ...gresql-operator-default-configuration.yaml | 3 + pkg/apis/acid.zalan.do/v1/crds.go | 34 ++++++---- .../v1/operator_configuration_type.go | 1 + .../acid.zalan.do/v1/zz_generated.deepcopy.go | 5 ++ pkg/cluster/cluster.go | 3 +- pkg/cluster/cluster_test.go | 33 +++++++++- pkg/cluster/k8sres.go | 6 +- pkg/cluster/resources.go | 21 +++++++ pkg/cluster/sync.go | 32 ++++++++++ pkg/controller/operator_config.go | 1 + pkg/controller/postgresql.go | 4 +- pkg/util/config/config.go | 1 + 20 files changed, 205 insertions(+), 31 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 71260bdb6..ffcef7b4a 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -117,6 +117,10 @@ spec: type: object additionalProperties: type: string + downscaler_annotations: + type: array + items: + type: string enable_init_containers: type: boolean enable_pod_antiaffinity: diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index e6428c478..98a399c8b 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -67,6 +67,11 @@ configKubernetes: # keya: valuea # keyb: valueb + # list of annotations propagated from cluster manifest to statefulset and deployment + # downscaler_annotations: + # - deployment-time + # - downscaler/* + # enables initContainers to run actions before Spilo is started enable_init_containers: true # toggles pod anti affinity on the Postgres pods @@ -262,11 +267,11 @@ configConnectionPooler: # docker image connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer" # max db connections the pooler should hold - connection_pooler_max_db_connections: "60" + connection_pooler_max_db_connections: 60 # default pooling mode connection_pooler_mode: "transaction" # number of pooler instances - connection_pooler_number_of_instances: "2" + connection_pooler_number_of_instances: 2 # default resources connection_pooler_default_cpu_request: 500m connection_pooler_default_memory_request: 100Mi diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index e5cdcee47..5578a5ed1 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -63,6 +63,9 @@ configKubernetes: # annotations attached to each database pod # custom_pod_annotations: "keya:valuea,keyb:valueb" + # list of annotations propagated from cluster manifest to statefulset and deployment + # downscaler_annotations: "deployment-time,downscaler/*" + # enables initContainers to run actions before Spilo is started enable_init_containers: "true" # toggles pod anti affinity on the Postgres pods diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index c87728812..576031543 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -165,7 +165,7 @@ These parameters are grouped directly under the `spec` key in the manifest. If `targetContainers` is empty, additional volumes will be mounted only in the `postgres` container. If you set the `all` special item, it will be mounted in all containers (postgres + sidecars). Else you can set the list of target containers in which the additional volumes will be mounted (eg : postgres, telegraf) - + ## Postgres parameters Those parameters are grouped under the `postgresql` top-level key, which is diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index af6e93d25..a81cabfc4 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -200,6 +200,12 @@ configuration they are grouped under the `kubernetes` key. of a database created by the operator. If the annotation key is also provided by the database definition, the database definition value is used. +* **downscaler_annotations** + An array of annotations that should be passed from Postgres CRD on to the + statefulset and, if exists, to the connection pooler deployment as well. + Regular expressions like `downscaler/*` etc. are also accepted. Can be used + with [kube-downscaler](https://github.com/hjacobs/kube-downscaler). + * **watched_namespace** The operator watches for Postgres objects in the given namespace. If not specified, the value is taken from the operator namespace. A special `*` diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index aa6f1205d..18b9852c4 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -428,7 +428,7 @@ class EndToEndTestCase(unittest.TestCase): k8s.api.core_v1.patch_node(current_master_node, patch_readiness_label) # wait a little before proceeding with the pod distribution test - time.sleep(k8s.RETRY_TIMEOUT_SEC) + time.sleep(30) # toggle pod anti affinity to move replica away from master node self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) @@ -465,21 +465,24 @@ class EndToEndTestCase(unittest.TestCase): pg_patch_custom_annotations = { "spec": { "serviceAnnotations": { - "annotation.key": "value" + "annotation.key": "value", + "foo": "bar", } } } k8s.api.custom_objects_api.patch_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_custom_annotations) + # wait a little before proceeding + time.sleep(30) annotations = { "annotation.key": "value", "foo": "bar", } self.assertTrue(k8s.check_service_annotations( - "cluster-name=acid-service-annotations,spilo-role=master", annotations)) + "cluster-name=acid-minimal-cluster,spilo-role=master", annotations)) self.assertTrue(k8s.check_service_annotations( - "cluster-name=acid-service-annotations,spilo-role=replica", annotations)) + "cluster-name=acid-minimal-cluster,spilo-role=replica", annotations)) # clean up unpatch_custom_service_annotations = { @@ -489,6 +492,40 @@ class EndToEndTestCase(unittest.TestCase): } k8s.update_config(unpatch_custom_service_annotations) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_statefulset_annotation_propagation(self): + ''' + Inject annotation to Postgresql CRD and check it's propagation to stateful set + ''' + k8s = self.k8s + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + + patch_sset_propagate_annotations = { + "data": { + "downscaler_annotations": "deployment-time,downscaler/*", + } + } + k8s.update_config(patch_sset_propagate_annotations) + + pg_crd_annotations = { + "metadata": { + "annotations": { + "deployment-time": "2020-04-30 12:00:00", + "downscaler/downtime_replicas": "0", + }, + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_crd_annotations) + + # wait a little before proceeding + time.sleep(60) + annotations = { + "deployment-time": "2020-04-30 12:00:00", + "downscaler/downtime_replicas": "0", + } + self.assertTrue(k8s.check_statefulset_annotations(cluster_label, annotations)) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_taint_based_eviction(self): ''' @@ -528,7 +565,7 @@ class EndToEndTestCase(unittest.TestCase): k8s.update_config(patch_toleration_config) # wait a little before proceeding with the pod distribution test - time.sleep(k8s.RETRY_TIMEOUT_SEC) + time.sleep(30) # toggle pod anti affinity to move replica away from master node self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) @@ -694,10 +731,18 @@ class K8s: def check_service_annotations(self, svc_labels, annotations, namespace='default'): svcs = self.api.core_v1.list_namespaced_service(namespace, label_selector=svc_labels, limit=1).items for svc in svcs: - if len(svc.metadata.annotations) != len(annotations): - return False - for key in svc.metadata.annotations: - if svc.metadata.annotations[key] != annotations[key]: + for key, value in annotations.items(): + if key not in svc.metadata.annotations or svc.metadata.annotations[key] != value: + print("Expected key {} not found in annotations {}".format(key, svc.metadata.annotation)) + return False + return True + + def check_statefulset_annotations(self, sset_labels, annotations, namespace='default'): + ssets = self.api.apps_v1.list_namespaced_stateful_set(namespace, label_selector=sset_labels, limit=1).items + for sset in ssets: + for key, value in annotations.items(): + if key not in sset.metadata.annotations or sset.metadata.annotations[key] != value: + print("Expected key {} not found in annotations {}".format(key, sset.metadata.annotation)) return False return True diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 8719e76a1..537055b18 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -30,6 +30,7 @@ data: # default_memory_limit: 500Mi # default_memory_request: 100Mi docker_image: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115 + # downscaler_annotations: "deployment-time,downscaler/*" # enable_admin_role_for_users: "true" # enable_crd_validation: "true" # enable_database_access: "true" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 2d3e614b9..23b5ff0fc 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -93,6 +93,10 @@ spec: type: object additionalProperties: type: string + downscaler_annotations: + type: array + items: + type: string enable_init_containers: type: boolean enable_pod_antiaffinity: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 78312b684..9ae1b3b26 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -31,6 +31,9 @@ configuration: # custom_pod_annotations: # keya: valuea # keyb: valueb + # downscaler_annotations: + # - deployment-time + # - downscaler/* enable_init_containers: true enable_pod_antiaffinity: false enable_pod_disruption_budget: true diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 35037ec3c..ad1b79a45 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -21,48 +21,48 @@ const ( // PostgresCRDResourceColumns definition of AdditionalPrinterColumns for postgresql CRD var PostgresCRDResourceColumns = []apiextv1beta1.CustomResourceColumnDefinition{ - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Team", Type: "string", Description: "Team responsible for Postgres cluster", JSONPath: ".spec.teamId", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Version", Type: "string", Description: "PostgreSQL version", JSONPath: ".spec.postgresql.version", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Pods", Type: "integer", Description: "Number of Pods per Postgres cluster", JSONPath: ".spec.numberOfInstances", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Volume", Type: "string", Description: "Size of the bound volume", JSONPath: ".spec.volume.size", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "CPU-Request", Type: "string", Description: "Requested CPU for Postgres containers", JSONPath: ".spec.resources.requests.cpu", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Memory-Request", Type: "string", Description: "Requested memory for Postgres containers", JSONPath: ".spec.resources.requests.memory", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Age", Type: "date", JSONPath: ".metadata.creationTimestamp", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Status", Type: "string", Description: "Current sync status of postgresql resource", @@ -72,31 +72,31 @@ var PostgresCRDResourceColumns = []apiextv1beta1.CustomResourceColumnDefinition{ // OperatorConfigCRDResourceColumns definition of AdditionalPrinterColumns for OperatorConfiguration CRD var OperatorConfigCRDResourceColumns = []apiextv1beta1.CustomResourceColumnDefinition{ - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Image", Type: "string", Description: "Spilo image to be used for Pods", JSONPath: ".configuration.docker_image", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Cluster-Label", Type: "string", Description: "Label for K8s resources created by operator", JSONPath: ".configuration.kubernetes.cluster_name_label", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Service-Account", Type: "string", Description: "Name of service account to be used", JSONPath: ".configuration.kubernetes.pod_service_account_name", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Min-Instances", Type: "integer", Description: "Minimum number of instances per Postgres cluster", JSONPath: ".configuration.min_instances", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Age", Type: "date", JSONPath: ".metadata.creationTimestamp", @@ -888,6 +888,14 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, }, }, + "downscaler_annotations": { + Type: "array", + Items: &apiextv1beta1.JSONSchemaPropsOrArray{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "string", + }, + }, + }, "enable_init_containers": { Type: "boolean", }, diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 4a0abf3ca..d3a9f6ec2 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -62,6 +62,7 @@ type KubernetesMetaConfiguration struct { PodRoleLabel string `json:"pod_role_label,omitempty"` ClusterLabels map[string]string `json:"cluster_labels,omitempty"` InheritedLabels []string `json:"inherited_labels,omitempty"` + DownscalerAnnotations []string `json:"downscaler_annotations,omitempty"` ClusterNameLabel string `json:"cluster_name_label,omitempty"` NodeReadinessLabel map[string]string `json:"node_readiness_label,omitempty"` CustomPodAnnotations map[string]string `json:"custom_pod_annotations,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 5b4d6cdcd..5879c9b73 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -180,6 +180,11 @@ func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfigura *out = make([]string, len(*in)) copy(*out, *in) } + if in.DownscalerAnnotations != nil { + in, out := &in.DownscalerAnnotations, &out.DownscalerAnnotations + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.NodeReadinessLabel != nil { in, out := &in.NodeReadinessLabel, &out.NodeReadinessLabel *out = make(map[string]string, len(*in)) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 387107540..31b8fa155 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -711,8 +711,7 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { updateFailed = true return } - - if !reflect.DeepEqual(oldSs, newSs) { + if !reflect.DeepEqual(oldSs, newSs) || !reflect.DeepEqual(oldSpec.Annotations, newSpec.Annotations) { c.logger.Debugf("syncing statefulsets") // TODO: avoid generating the StatefulSet object twice by passing it to syncStatefulSet if err := c.syncStatefulSet(); err != nil { diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index 539038bff..4562d525e 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -28,19 +28,48 @@ var eventRecorder = record.NewFakeRecorder(1) var cl = New( Config{ OpConfig: config.Config{ - ProtectedRoles: []string{"admin"}, + PodManagementPolicy: "ordered_ready", + ProtectedRoles: []string{"admin"}, Auth: config.Auth{ SuperUsername: superUserName, ReplicationUsername: replicationUserName, }, + Resources: config.Resources{ + DownscalerAnnotations: []string{"downscaler/*"}, + }, }, }, k8sutil.NewMockKubernetesClient(), - acidv1.Postgresql{ObjectMeta: metav1.ObjectMeta{Name: "acid-test", Namespace: "test"}}, + acidv1.Postgresql{ObjectMeta: metav1.ObjectMeta{Name: "acid-test", Namespace: "test", Annotations: map[string]string{"downscaler/downtime_replicas": "0"}}}, logger, eventRecorder, ) +func TestStatefulSetAnnotations(t *testing.T) { + testName := "CheckStatefulsetAnnotations" + spec := acidv1.PostgresSpec{ + TeamID: "myapp", NumberOfInstances: 1, + Resources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + }, + Volume: acidv1.Volume{ + Size: "1G", + }, + } + ss, err := cl.generateStatefulSet(&spec) + if err != nil { + t.Errorf("in %s no statefulset created %v", testName, err) + } + if ss != nil { + annotation := ss.ObjectMeta.GetAnnotations() + if _, ok := annotation["downscaler/downtime_replicas"]; !ok { + t.Errorf("in %s respective annotation not found on sts", testName) + } + } + +} + func TestInitRobotUsers(t *testing.T) { testName := "TestInitRobotUsers" tests := []struct { diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 9b92f2fb5..534ae7b8e 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -6,6 +6,7 @@ import ( "fmt" "path" "sort" + "strconv" "github.com/sirupsen/logrus" @@ -1182,12 +1183,15 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef return nil, fmt.Errorf("could not set the pod management policy to the unknown value: %v", c.OpConfig.PodManagementPolicy) } + annotations = make(map[string]string) + annotations[rollingUpdateStatefulsetAnnotationKey] = strconv.FormatBool(false) + statefulSet := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: c.statefulSetName(), Namespace: c.Namespace, Labels: c.labelsSet(true), - Annotations: map[string]string{rollingUpdateStatefulsetAnnotationKey: "false"}, + Annotations: c.AnnotationsToPropagate(annotations), }, Spec: appsv1.StatefulSetSpec{ Replicas: &numberOfInstances, diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index b38458af8..3528c46f4 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -853,3 +853,24 @@ func (c *Cluster) updateConnectionPoolerDeployment(oldDeploymentSpec, newDeploym return deployment, nil } + +//updateConnectionPoolerAnnotations updates the annotations of connection pooler deployment +func (c *Cluster) updateConnectionPoolerAnnotations(annotations map[string]string) (*appsv1.Deployment, error) { + c.logger.Debugf("updating connection pooler annotations") + patchData, err := metaAnnotationsPatch(annotations) + if err != nil { + return nil, fmt.Errorf("could not form patch for the deployment metadata: %v", err) + } + result, err := c.KubeClient.Deployments(c.ConnectionPooler.Deployment.Namespace).Patch( + context.TODO(), + c.ConnectionPooler.Deployment.Name, + types.MergePatchType, + []byte(patchData), + metav1.PatchOptions{}, + "") + if err != nil { + return nil, fmt.Errorf("could not patch connection pooler annotations %q: %v", patchData, err) + } + return result, nil + +} diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 0c4c662d4..697fc2d05 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -3,6 +3,7 @@ package cluster import ( "context" "fmt" + "regexp" "strings" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" @@ -358,6 +359,8 @@ func (c *Cluster) syncStatefulSet() error { } } } + annotations := c.AnnotationsToPropagate(c.Statefulset.Annotations) + c.updateStatefulSetAnnotations(annotations) if !podsRollingUpdateRequired && !c.OpConfig.EnableLazySpiloUpgrade { // even if desired and actual statefulsets match @@ -397,6 +400,30 @@ func (c *Cluster) syncStatefulSet() error { return nil } +// AnnotationsToPropagate get the annotations to update if required +// based on the annotations in postgres CRD +func (c *Cluster) AnnotationsToPropagate(annotations map[string]string) map[string]string { + toPropagateAnnotations := c.OpConfig.DownscalerAnnotations + pgCRDAnnotations := c.Postgresql.ObjectMeta.GetAnnotations() + + if toPropagateAnnotations != nil && pgCRDAnnotations != nil { + for _, anno := range toPropagateAnnotations { + for k, v := range pgCRDAnnotations { + matched, err := regexp.MatchString(anno, k) + if err != nil { + c.logger.Errorf("annotations matching issue: %v", err) + return nil + } + if matched { + annotations[k] = v + } + } + } + } + + return annotations +} + // checkAndSetGlobalPostgreSQLConfiguration checks whether cluster-wide API parameters // (like max_connections) has changed and if necessary sets it via the Patroni API func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration() error { @@ -939,6 +966,11 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql } } + newAnnotations := c.AnnotationsToPropagate(c.ConnectionPooler.Deployment.Annotations) + if newAnnotations != nil { + c.updateConnectionPoolerAnnotations(newAnnotations) + } + service, err := c.KubeClient. Services(c.Namespace). Get(context.TODO(), c.connectionPoolerName(), metav1.GetOptions{}) diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index f66eafb1e..4eed44924 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -73,6 +73,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.PodRoleLabel = fromCRD.Kubernetes.PodRoleLabel result.ClusterLabels = fromCRD.Kubernetes.ClusterLabels result.InheritedLabels = fromCRD.Kubernetes.InheritedLabels + result.DownscalerAnnotations = fromCRD.Kubernetes.DownscalerAnnotations result.ClusterNameLabel = fromCRD.Kubernetes.ClusterNameLabel result.NodeReadinessLabel = fromCRD.Kubernetes.NodeReadinessLabel result.PodPriorityClassName = fromCRD.Kubernetes.PodPriorityClassName diff --git a/pkg/controller/postgresql.go b/pkg/controller/postgresql.go index 2a9e1b650..c243f330f 100644 --- a/pkg/controller/postgresql.go +++ b/pkg/controller/postgresql.go @@ -487,7 +487,9 @@ func (c *Controller) postgresqlUpdate(prev, cur interface{}) { if pgOld != nil && pgNew != nil { // Avoid the inifinite recursion for status updates if reflect.DeepEqual(pgOld.Spec, pgNew.Spec) { - return + if reflect.DeepEqual(pgNew.Annotations, pgOld.Annotations) { + return + } } c.queueClusterEvent(pgOld, pgNew, EventUpdate) } diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 37ba947d6..d5c4b6671 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -34,6 +34,7 @@ type Resources struct { SpiloPrivileged bool `name:"spilo_privileged" default:"false"` ClusterLabels map[string]string `name:"cluster_labels" default:"application:spilo"` InheritedLabels []string `name:"inherited_labels" default:""` + DownscalerAnnotations []string `name:"downscaler_annotations"` ClusterNameLabel string `name:"cluster_name_label" default:"cluster-name"` PodRoleLabel string `name:"pod_role_label" default:"spilo-role"` PodToleration map[string]string `name:"toleration" default:""` From 76d43525f7463fb7598ea83736052e9d4ee91325 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 4 May 2020 16:23:21 +0200 Subject: [PATCH 2/2] define more default values for opConfig CRD (#955) --- charts/postgres-operator/values.yaml | 2 +- manifests/configmap.yaml | 10 +++---- pkg/controller/operator_config.go | 42 ++++++++++++++-------------- pkg/util/config/config.go | 2 +- pkg/util/util.go | 14 ++++++++-- 5 files changed, 39 insertions(+), 31 deletions(-) diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 5578a5ed1..cb8e29081 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -109,7 +109,7 @@ configKubernetes: # Postgres pods are terminated forcefully after this timeout pod_terminate_grace_period: 5m # template for database user secrets generated by the operator - secret_name_template: '{username}.{cluster}.credentials' + secret_name_template: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}" # group ID with write-access to volumes (required to run Spilo as non-root process) # spilo_fsgroup: "103" diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 537055b18..0a740e198 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -50,16 +50,16 @@ data: # inherited_labels: application,environment # kube_iam_role: "" # log_s3_bucket: "" - # logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup" + logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup" # logical_backup_s3_access_key_id: "" - # logical_backup_s3_bucket: "my-bucket-url" + logical_backup_s3_bucket: "my-bucket-url" # logical_backup_s3_region: "" # logical_backup_s3_endpoint: "" # logical_backup_s3_secret_access_key: "" - # logical_backup_s3_sse: "AES256" - # logical_backup_schedule: "30 00 * * *" + logical_backup_s3_sse: "AES256" + logical_backup_schedule: "30 00 * * *" master_dns_name_format: "{cluster}.{team}.{hostedzone}" - # master_pod_move_timeout: 10m + # master_pod_move_timeout: 20m # max_instances: "-1" # min_instances: "-1" # min_cpu_limit: 250m diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 4eed44924..389240c09 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -33,28 +33,28 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result := &config.Config{} // general config - result.EnableCRDValidation = fromCRD.EnableCRDValidation + result.EnableCRDValidation = util.CoalesceBool(fromCRD.EnableCRDValidation, util.True()) result.EnableLazySpiloUpgrade = fromCRD.EnableLazySpiloUpgrade result.EtcdHost = fromCRD.EtcdHost result.KubernetesUseConfigMaps = fromCRD.KubernetesUseConfigMaps - result.DockerImage = fromCRD.DockerImage + result.DockerImage = util.Coalesce(fromCRD.DockerImage, "registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115") result.Workers = fromCRD.Workers result.MinInstances = fromCRD.MinInstances result.MaxInstances = fromCRD.MaxInstances result.ResyncPeriod = time.Duration(fromCRD.ResyncPeriod) result.RepairPeriod = time.Duration(fromCRD.RepairPeriod) result.SetMemoryRequestToLimit = fromCRD.SetMemoryRequestToLimit - result.ShmVolume = fromCRD.ShmVolume + result.ShmVolume = util.CoalesceBool(fromCRD.ShmVolume, util.True()) result.SidecarImages = fromCRD.SidecarImages result.SidecarContainers = fromCRD.SidecarContainers // user config - result.SuperUsername = fromCRD.PostgresUsersConfiguration.SuperUsername - result.ReplicationUsername = fromCRD.PostgresUsersConfiguration.ReplicationUsername + result.SuperUsername = util.Coalesce(fromCRD.PostgresUsersConfiguration.SuperUsername, "postgres") + result.ReplicationUsername = util.Coalesce(fromCRD.PostgresUsersConfiguration.ReplicationUsername, "standby") // kubernetes config result.CustomPodAnnotations = fromCRD.Kubernetes.CustomPodAnnotations - result.PodServiceAccountName = fromCRD.Kubernetes.PodServiceAccountName + result.PodServiceAccountName = util.Coalesce(fromCRD.Kubernetes.PodServiceAccountName, "postgres-pod") result.PodServiceAccountDefinition = fromCRD.Kubernetes.PodServiceAccountDefinition result.PodServiceAccountRoleBindingDefinition = fromCRD.Kubernetes.PodServiceAccountRoleBindingDefinition result.PodEnvironmentConfigMap = fromCRD.Kubernetes.PodEnvironmentConfigMap @@ -64,31 +64,31 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.ClusterDomain = util.Coalesce(fromCRD.Kubernetes.ClusterDomain, "cluster.local") result.WatchedNamespace = fromCRD.Kubernetes.WatchedNamespace result.PDBNameFormat = fromCRD.Kubernetes.PDBNameFormat - result.EnablePodDisruptionBudget = fromCRD.Kubernetes.EnablePodDisruptionBudget - result.EnableInitContainers = fromCRD.Kubernetes.EnableInitContainers - result.EnableSidecars = fromCRD.Kubernetes.EnableSidecars + result.EnablePodDisruptionBudget = util.CoalesceBool(fromCRD.Kubernetes.EnablePodDisruptionBudget, util.True()) + result.EnableInitContainers = util.CoalesceBool(fromCRD.Kubernetes.EnableInitContainers, util.True()) + result.EnableSidecars = util.CoalesceBool(fromCRD.Kubernetes.EnableSidecars, util.True()) result.SecretNameTemplate = fromCRD.Kubernetes.SecretNameTemplate result.OAuthTokenSecretName = fromCRD.Kubernetes.OAuthTokenSecretName result.InfrastructureRolesSecretName = fromCRD.Kubernetes.InfrastructureRolesSecretName - result.PodRoleLabel = fromCRD.Kubernetes.PodRoleLabel + result.PodRoleLabel = util.Coalesce(fromCRD.Kubernetes.PodRoleLabel, "spilo-role") result.ClusterLabels = fromCRD.Kubernetes.ClusterLabels result.InheritedLabels = fromCRD.Kubernetes.InheritedLabels result.DownscalerAnnotations = fromCRD.Kubernetes.DownscalerAnnotations - result.ClusterNameLabel = fromCRD.Kubernetes.ClusterNameLabel + result.ClusterNameLabel = util.Coalesce(fromCRD.Kubernetes.ClusterNameLabel, "cluster-name") result.NodeReadinessLabel = fromCRD.Kubernetes.NodeReadinessLabel result.PodPriorityClassName = fromCRD.Kubernetes.PodPriorityClassName - result.PodManagementPolicy = fromCRD.Kubernetes.PodManagementPolicy + result.PodManagementPolicy = util.Coalesce(fromCRD.Kubernetes.PodManagementPolicy, "ordered_ready") result.MasterPodMoveTimeout = time.Duration(fromCRD.Kubernetes.MasterPodMoveTimeout) result.EnablePodAntiAffinity = fromCRD.Kubernetes.EnablePodAntiAffinity - result.PodAntiAffinityTopologyKey = fromCRD.Kubernetes.PodAntiAffinityTopologyKey + result.PodAntiAffinityTopologyKey = util.Coalesce(fromCRD.Kubernetes.PodAntiAffinityTopologyKey, "kubernetes.io/hostname") // Postgres Pod resources - result.DefaultCPURequest = fromCRD.PostgresPodResources.DefaultCPURequest - result.DefaultMemoryRequest = fromCRD.PostgresPodResources.DefaultMemoryRequest - result.DefaultCPULimit = fromCRD.PostgresPodResources.DefaultCPULimit - result.DefaultMemoryLimit = fromCRD.PostgresPodResources.DefaultMemoryLimit - result.MinCPULimit = fromCRD.PostgresPodResources.MinCPULimit - result.MinMemoryLimit = fromCRD.PostgresPodResources.MinMemoryLimit + result.DefaultCPURequest = util.Coalesce(fromCRD.PostgresPodResources.DefaultCPURequest, "100m") + result.DefaultMemoryRequest = util.Coalesce(fromCRD.PostgresPodResources.DefaultMemoryRequest, "100Mi") + result.DefaultCPULimit = util.Coalesce(fromCRD.PostgresPodResources.DefaultCPULimit, "1") + result.DefaultMemoryLimit = util.Coalesce(fromCRD.PostgresPodResources.DefaultMemoryLimit, "500Mi") + result.MinCPULimit = util.Coalesce(fromCRD.PostgresPodResources.MinCPULimit, "250m") + result.MinMemoryLimit = util.Coalesce(fromCRD.PostgresPodResources.MinMemoryLimit, "250Mi") // timeout config result.ResourceCheckInterval = time.Duration(fromCRD.Timeouts.ResourceCheckInterval) @@ -115,8 +115,8 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.AdditionalSecretMountPath = fromCRD.AWSGCP.AdditionalSecretMountPath // logical backup config - result.LogicalBackupSchedule = fromCRD.LogicalBackup.Schedule - result.LogicalBackupDockerImage = fromCRD.LogicalBackup.DockerImage + result.LogicalBackupSchedule = util.Coalesce(fromCRD.LogicalBackup.Schedule, "30 00 * * *") + result.LogicalBackupDockerImage = util.Coalesce(fromCRD.LogicalBackup.DockerImage, "registry.opensource.zalan.do/acid/logical-backup") result.LogicalBackupS3Bucket = fromCRD.LogicalBackup.S3Bucket result.LogicalBackupS3Region = fromCRD.LogicalBackup.S3Region result.LogicalBackupS3Endpoint = fromCRD.LogicalBackup.S3Endpoint diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index d5c4b6671..d8c92ba3e 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -83,7 +83,7 @@ type LogicalBackup struct { LogicalBackupS3Endpoint string `name:"logical_backup_s3_endpoint" default:""` LogicalBackupS3AccessKeyID string `name:"logical_backup_s3_access_key_id" default:""` LogicalBackupS3SecretAccessKey string `name:"logical_backup_s3_secret_access_key" default:""` - LogicalBackupS3SSE string `name:"logical_backup_s3_sse" default:"AES256"` + LogicalBackupS3SSE string `name:"logical_backup_s3_sse" default:""` } // Operator options for connection pooler diff --git a/pkg/util/util.go b/pkg/util/util.go index 46df5d345..5701429aa 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -147,7 +147,7 @@ func Coalesce(val, defaultVal string) string { return val } -// Yeah, golang +// CoalesceInt32 works like coalesce but for *int32 func CoalesceInt32(val, defaultVal *int32) *int32 { if val == nil { return defaultVal @@ -155,6 +155,14 @@ func CoalesceInt32(val, defaultVal *int32) *int32 { return val } +// CoalesceBool works like coalesce but for *bool +func CoalesceBool(val, defaultVal *bool) *bool { + if val == nil { + return defaultVal + } + return val +} + // Test if any of the values is nil func testNil(values ...*int32) bool { for _, v := range values { @@ -166,8 +174,8 @@ func testNil(values ...*int32) bool { return false } -// Return maximum of two integers provided via pointers. If one value is not -// defined, return the other one. If both are not defined, result is also +// MaxInt32 : Return maximum of two integers provided via pointers. If one value +// is not defined, return the other one. If both are not defined, result is also // undefined, caller needs to check for that. func MaxInt32(a, b *int32) *int32 { if testNil(a, b) {