From a3d17c12d27ba63b1497fc993ef566759b828cf9 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 2 Jul 2026 14:04:57 +0200 Subject: [PATCH] document changed configmap default (#3116) * document changed configmap default * update config reference * add warning message in sync * address review comments --- charts/postgres-operator/values.yaml | 2 +- docs/migrate.md | 17 +++++++ docs/reference/operator_parameters.md | 6 +-- ...gresql-operator-default-configuration.yaml | 2 +- .../v1/operator_configuration_type.go | 2 +- .../acid.zalan.do/v1/zz_generated.deepcopy.go | 5 ++ pkg/cluster/cluster_test.go | 1 + pkg/cluster/k8sres_test.go | 49 ++++++++++--------- pkg/cluster/sync.go | 4 ++ pkg/cluster/util.go | 5 +- pkg/cluster/util_test.go | 6 +-- pkg/controller/operator_config.go | 2 +- pkg/util/config/config.go | 2 +- 13 files changed, 66 insertions(+), 37 deletions(-) create mode 100644 docs/migrate.md diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index b880916f5..3a2194ef0 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -50,7 +50,7 @@ configGeneral: # ignore_resources_limits_annotation_key: "" # Select if setup uses endpoints (default), or configmaps to manage leader (DCS=k8s) - # kubernetes_use_configmaps: false + kubernetes_use_configmaps: true # maintenance windows applied to all Postgres clusters unless overridden in the manifest # maintenance_windows: diff --git a/docs/migrate.md b/docs/migrate.md new file mode 100644 index 000000000..7ec5befb8 --- /dev/null +++ b/docs/migrate.md @@ -0,0 +1,17 @@ +

Migrate from v1 to v2

+ +Version 2.0 changes some default settings and removes deprecated fields. Please read the following sections before upgrading the Postgres Operator deployment. + +## K8s Endpoints are deprecated + +If your current operator v1.x deployment is relying on K8s endpoints (the default setup) for Patroni to manage the HA state you have to start planning to switch to configmaps, because endpoints are deprecated from K8s 1.33 onwards. The default of the corresponding parameter `kubernetes_use_configmaps` is changing to `true` with v2.0 of the operator. This means you have to explicity set it to `false` in your configuration before you start the upgrade. + +We explicitly warn you to go straight to configmap-based HA management with database clusters that use replicas, because there's is a danger to run into split-brain scenarios during the rolling update of pods. To play it safe, here is what you should do - before or after the Postgres Operator upgrade: + +1. Scale-in all your database clusters to only one primary instance. This can be done by changing the global config options `max_instances` and `min_instances` to `1`. If you have allowed users to ignore globally defined instance limits by configuring an `ignore_instance_limits_annotation_key`, remove it for now. + +2. Wait for all clusters to be healthy and change the `kubernetes_use_configmaps` setting to `true`. This will trigger the replacement of the primary pod of all clusters and cause downtime for as long as the pods are rescheduled and start up. + +3. Check again that all clusters are healthy with configmaps created. There should be three for each cluster called like cluster name with suffixes `-config`, `-failover` and `-leader`. Now, revert the changes from step 1 and scale-out the to number of instances set in the manifests. + +4. The orphaned endpoints, which use the same names like the new configmaps, have to be deleted by you or your K8s garbage collection. diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 13d29c950..dd7487352 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -100,15 +100,13 @@ Those are top-level keys, containing both leaf keys and groups. Kubernetes-native DCS). * **kubernetes_use_configmaps** - Select if setup uses endpoints (default), or configmaps to manage leader when + Select if setup uses endpoints or configmaps (default) to manage leader when DCS is kubernetes (not etcd or similar). In OpenShift it is not possible to use endpoints option, and configmaps is required. Starting with K8s 1.33, endpoints are marked as deprecated. It's recommended to switch to config maps instead. But, to do so make sure you scale the Postgres cluster down to just one primary pod (e.g. using `max_instances` option). Otherwise, you risk - running into a split-brain scenario. - By default, `kubernetes_use_configmaps: false`, meaning endpoints will be used. - Starting from v1.16.0 the default will be changed to `true`. + running into a split-brain scenario. Default is `true`. * **docker_image** Spilo Docker image for Postgres instances. For production, don't rely on the diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 88af48b66..daa7f7b13 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -16,7 +16,7 @@ configuration: etcd_host: "" # ignore_instance_limits_annotation_key: "" # ignore_resources_limits_annotation_key: "" - # kubernetes_use_configmaps: false + kubernetes_use_configmaps: true # maintenance_windows: # - "Sat:22:00-23:59" # - "Sun:00:00-01:00" diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 27063fd23..b84bce03a 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -410,7 +410,7 @@ type OperatorConfigurationData struct { // +kubebuilder:default="" EtcdHost string `json:"etcd_host,omitempty"` // +kubebuilder:default=true - KubernetesUseConfigMaps bool `json:"kubernetes_use_configmaps,omitempty"` + KubernetesUseConfigMaps *bool `json:"kubernetes_use_configmaps,omitempty"` // +kubebuilder:default="ghcr.io/zalando/spilo-18:4.1-p1" DockerImage string `json:"docker_image,omitempty"` // +kubebuilder:validation:Minimum=1 diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index ddccccf7f..83c4763ac 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -443,6 +443,11 @@ func (in *OperatorConfigurationData) DeepCopyInto(out *OperatorConfigurationData *out = make([]string, len(*in)) copy(*out, *in) } + if in.KubernetesUseConfigMaps != nil { + in, out := &in.KubernetesUseConfigMaps, &out.KubernetesUseConfigMaps + *out = new(bool) + **out = **in + } if in.ResyncPeriod != nil { in, out := &in.ResyncPeriod, &out.ResyncPeriod *out = new(metav1.Duration) diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index 00e69d4d7..b2c6bee68 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -95,6 +95,7 @@ func TestCreate(t *testing.T) { client := k8sutil.KubernetesClient{ DeploymentsGetter: clientSet.AppsV1(), + ConfigMapsGetter: clientSet.CoreV1(), CronJobsGetter: clientSet.BatchV1(), EndpointsGetter: clientSet.CoreV1(), PersistentVolumeClaimsGetter: clientSet.CoreV1(), diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 3f385b456..d51efb085 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -582,60 +582,60 @@ func TestGenerateSpiloPodEnvVars(t *testing.T) { } expectedValuesS3Bucket := []ExpectedValue{ { - envIndex: 15, + envIndex: 16, envVarConstant: "WAL_S3_BUCKET", envVarValue: "global-s3-bucket", }, { - envIndex: 16, + envIndex: 17, envVarConstant: "WAL_BUCKET_SCOPE_SUFFIX", envVarValue: fmt.Sprintf("/%s", dummyUUID), }, { - envIndex: 17, + envIndex: 18, envVarConstant: "WAL_BUCKET_SCOPE_PREFIX", envVarValue: "", }, } expectedValuesGCPCreds := []ExpectedValue{ { - envIndex: 15, + envIndex: 16, envVarConstant: "WAL_GS_BUCKET", envVarValue: "global-gs-bucket", }, { - envIndex: 16, + envIndex: 17, envVarConstant: "WAL_BUCKET_SCOPE_SUFFIX", envVarValue: fmt.Sprintf("/%s", dummyUUID), }, { - envIndex: 17, + envIndex: 18, envVarConstant: "WAL_BUCKET_SCOPE_PREFIX", envVarValue: "", }, { - envIndex: 18, + envIndex: 19, envVarConstant: "GOOGLE_APPLICATION_CREDENTIALS", envVarValue: "some-path-to-credentials", }, } expectedS3BucketConfigMap := []ExpectedValue{ { - envIndex: 17, + envIndex: 18, envVarConstant: "wal_s3_bucket", envVarValue: "global-s3-bucket-configmap", }, } expectedCustomS3BucketSpec := []ExpectedValue{ { - envIndex: 15, + envIndex: 16, envVarConstant: "WAL_S3_BUCKET", envVarValue: "custom-s3-bucket", }, } expectedCustomVariableSecret := []ExpectedValue{ { - envIndex: 16, + envIndex: 17, envVarConstant: "custom_variable", envVarValueRef: &v1.EnvVarSource{ SecretKeyRef: &v1.SecretKeySelector{ @@ -649,72 +649,72 @@ func TestGenerateSpiloPodEnvVars(t *testing.T) { } expectedCustomVariableConfigMap := []ExpectedValue{ { - envIndex: 16, + envIndex: 17, envVarConstant: "custom_variable", envVarValue: "configmap-test", }, } expectedCustomVariableSpec := []ExpectedValue{ { - envIndex: 15, + envIndex: 16, envVarConstant: "CUSTOM_VARIABLE", envVarValue: "spec-env-test", }, } expectedCloneEnvSpec := []ExpectedValue{ { - envIndex: 16, + envIndex: 17, envVarConstant: "CLONE_WALE_S3_PREFIX", envVarValue: "s3://another-bucket", }, { - envIndex: 19, + envIndex: 20, envVarConstant: "CLONE_WAL_BUCKET_SCOPE_PREFIX", envVarValue: "", }, { - envIndex: 20, + envIndex: 21, envVarConstant: "CLONE_AWS_ENDPOINT", envVarValue: "s3.eu-central-1.amazonaws.com", }, } expectedCloneEnvSpecEnv := []ExpectedValue{ { - envIndex: 15, + envIndex: 16, envVarConstant: "CLONE_WAL_BUCKET_SCOPE_PREFIX", envVarValue: "test-cluster", }, { - envIndex: 17, + envIndex: 18, envVarConstant: "CLONE_WALE_S3_PREFIX", envVarValue: "s3://another-bucket", }, { - envIndex: 21, + envIndex: 22, envVarConstant: "CLONE_AWS_ENDPOINT", envVarValue: "s3.eu-central-1.amazonaws.com", }, } expectedCloneEnvConfigMap := []ExpectedValue{ { - envIndex: 16, + envIndex: 17, envVarConstant: "CLONE_WAL_S3_BUCKET", envVarValue: "global-s3-bucket", }, { - envIndex: 17, + envIndex: 18, envVarConstant: "CLONE_WAL_BUCKET_SCOPE_SUFFIX", envVarValue: fmt.Sprintf("/%s", dummyUUID), }, { - envIndex: 21, + envIndex: 22, envVarConstant: "clone_aws_endpoint", envVarValue: "s3.eu-west-1.amazonaws.com", }, } expectedCloneEnvSecret := []ExpectedValue{ { - envIndex: 21, + envIndex: 22, envVarConstant: "clone_aws_access_key_id", envVarValueRef: &v1.EnvVarSource{ SecretKeyRef: &v1.SecretKeySelector{ @@ -728,12 +728,12 @@ func TestGenerateSpiloPodEnvVars(t *testing.T) { } expectedStandbyEnvSecret := []ExpectedValue{ { - envIndex: 15, + envIndex: 16, envVarConstant: "STANDBY_WALE_GS_PREFIX", envVarValue: "gs://some/path/", }, { - envIndex: 20, + envIndex: 21, envVarConstant: "standby_google_application_credentials", envVarValueRef: &v1.EnvVarSource{ SecretKeyRef: &v1.SecretKeySelector{ @@ -2976,6 +2976,7 @@ func getServices(serviceType v1.ServiceType, sourceRanges []string, extTrafficPo ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyType(extTrafficPolicy), LoadBalancerSourceRanges: sourceRanges, Ports: []v1.ServicePort{{Name: "postgresql", Port: 5432, TargetPort: intstr.IntOrString{IntVal: 5432}, NodePort: nodePort}}, + Selector: map[string]string{"spilo-role": "master", "application": "spilo", "cluster-name": clusterName}, Type: serviceType, }, { diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index e15b5fedc..af3f786df 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -67,6 +67,10 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { } }() + if !c.patroniKubernetesUseConfigMaps() { + c.logger.Warning("K8s endpoints are deprecated. Please, enable kubernetes_use_configmaps. Requires scale-in to a single primary, see v1 -> v2 migration docs!") + } + if err = c.syncFinalizer(); err != nil { c.logger.Debugf("could not sync finalizers: %v", err) } diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go index c3a9dda31..c2a847ba4 100644 --- a/pkg/cluster/util.go +++ b/pkg/cluster/util.go @@ -627,9 +627,12 @@ func (c *Cluster) patroniKubernetesUseConfigMaps() bool { if !c.patroniUsesKubernetes() { return false } + if c.OpConfig.KubernetesUseConfigMaps == nil { + return true + } // otherwise, follow the operator configuration - return c.OpConfig.KubernetesUseConfigMaps + return *c.OpConfig.KubernetesUseConfigMaps } // Earlier arguments take priority diff --git a/pkg/cluster/util_test.go b/pkg/cluster/util_test.go index 0239abf45..d97668ee8 100644 --- a/pkg/cluster/util_test.go +++ b/pkg/cluster/util_test.go @@ -299,7 +299,7 @@ func newInheritedAnnotationsCluster(client k8sutil.KubernetesClient) (*Cluster, OpConfig: config.Config{ PatroniAPICheckInterval: &metav1.Duration{Duration: 1 * time.Second}, PatroniAPICheckTimeout: &metav1.Duration{Duration: 5 * time.Second}, - KubernetesUseConfigMaps: true, + KubernetesUseConfigMaps: util.True(), ConnectionPooler: config.ConnectionPooler{ ConnectionPoolerDefaultCPURequest: "100m", ConnectionPoolerDefaultCPULimit: "100m", @@ -388,7 +388,7 @@ func createPatroniResources(cluster *Cluster) error { Labels: cluster.labelsSet(false), } - if cluster.OpConfig.KubernetesUseConfigMaps { + if cluster.OpConfig.KubernetesUseConfigMaps != nil && *cluster.OpConfig.KubernetesUseConfigMaps { configMap := v1.ConfigMap{ ObjectMeta: metadata, } @@ -598,7 +598,7 @@ func TestInheritedAnnotations(t *testing.T) { // 3. Change from ConfigMaps to Endpoints err = cluster.deletePatroniResources() assert.NoError(t, err) - cluster.OpConfig.KubernetesUseConfigMaps = false + cluster.OpConfig.KubernetesUseConfigMaps = util.False() err = createPatroniResources(cluster) assert.NoError(t, err) err = cluster.Sync(newSpec.DeepCopy()) diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 55f1f00d8..b3765e387 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -35,7 +35,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.EnableSpiloWalPathCompat = fromCRD.EnableSpiloWalPathCompat result.EnableTeamIdClusternamePrefix = fromCRD.EnableTeamIdClusternamePrefix result.EtcdHost = fromCRD.EtcdHost - result.KubernetesUseConfigMaps = fromCRD.KubernetesUseConfigMaps + result.KubernetesUseConfigMaps = util.CoalesceBool(fromCRD.KubernetesUseConfigMaps, util.True()) result.DockerImage = util.Coalesce(fromCRD.DockerImage, "ghcr.io/zalando/spilo-18:4.1-p1") result.Workers = util.CoalesceUInt32(fromCRD.Workers, 8) result.MinInstances = fromCRD.MinInstances diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 85119b18b..14ae537cc 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -177,7 +177,7 @@ type Config struct { ConnectionPooler WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to' - KubernetesUseConfigMaps bool `name:"kubernetes_use_configmaps" default:"false"` + KubernetesUseConfigMaps *bool `name:"kubernetes_use_configmaps" default:"true"` EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS EnableMaintenanceWindows *bool `name:"enable_maintenance_windows" default:"true"` MaintenanceWindows []string `name:"maintenance_windows"`