From 4c1bb1c0ea6378a745dd6ca7d0396d3e51e61525 Mon Sep 17 00:00:00 2001 From: Juhani Pelli Date: Mon, 27 Jul 2026 18:36:42 +0300 Subject: [PATCH] Use maxUnavailable for the critical-op PDB to stop idle alert noise (#3141) * Use maxUnavailable for the critical-op PDB to stop idle alert noise The critical-op PDB is created with minAvailable equal to numberOfInstances while its selector (critical-operation=true) matches no pods during normal operation. This leaves status.desiredHealthy at N and currentHealthy at 0 permanently, so monitoring stacks fire alerts like kube-prometheus-stack's KubePdbNotEnoughHealthyPods for every idle cluster (#3020). maxUnavailable: 0 provides the same protection while a critical operation is running - no voluntary evictions of labeled pods - but keeps the budget satisfied (desiredHealthy 0) when nothing matches. When PDBs are disabled or there are no instances, the budget relaxes to maxUnavailable 100% instead of minAvailable 0. Fixes #3020 Co-Authored-By: Claude Fable 5 * Update PDB docs for critical-op maxUnavailable semantics Co-Authored-By: Claude Fable 5 * Clarify why the two PDBs use different budget fields Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 Co-authored-by: Felix Kunde --- docs/administrator.md | 17 +++++++++++++---- pkg/cluster/k8sres.go | 14 ++++++++++---- pkg/cluster/k8sres_test.go | 19 +++++++++++++++---- 3 files changed, 38 insertions(+), 12 deletions(-) diff --git a/docs/administrator.md b/docs/administrator.md index adfee6065..f7f0cb982 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -639,9 +639,11 @@ masters in single-node clusters and/or the last remaining running instance in a cluster. ## PDB for critical operations -The `MinAvailable` parameter of this PDB is equal to the `numberOfInstances` set in the -cluster manifest, while label selector includes `critical-operation=true` condition. This -allows to protect all pods of a cluster, given they are labeled accordingly. +The `MaxUnavailable` parameter of this PDB is set to `0`, while label selector includes +`critical-operation=true` condition. This blocks voluntary disruptions for all pods of a +cluster that are labeled accordingly, without leaving an unsatisfiable budget behind when +no pods carry the label (which previously kept monitoring alerts like +`KubePdbNotEnoughHealthyPods` firing permanently). For example, Operator labels all Spilo pods with `critical-operation=true` during the major version upgrade run. You may want to protect cluster pods during other critical operations by assigning the label to pods yourself or using other means of automation. @@ -651,7 +653,14 @@ The PDB is only relaxed in two scenarios: * If a cluster is scaled down to `0` instances (e.g. for draining nodes) * If the PDB is disabled in the configuration (`enable_pod_disruption_budget`) -The PDBs are still in place having `MinAvailable` set to `0`. Disabling PDBs +The PDBs are still in place but fully relaxed: the primary PDB with `MinAvailable` +set to `0` and the critical operations PDB with `MaxUnavailable` set to `100%`. +The two PDBs intentionally use different budget fields matching their purposes: +the primary PDB guarantees a minimum count of always-present pods +(`MinAvailable`), while the critical operations PDB freezes disruptions for +whatever pods currently carry the `critical-operation=true` label - a +usually-empty set, which `MaxUnavailable: 0` expresses without producing an +unsatisfiable budget while idle. Disabling PDBs helps avoiding blocking Kubernetes upgrades in managed K8s environments at the cost of prolonged DB downtime. See PR [#384](https://github.com/zalando/postgres-operator/pull/384) for the use case. diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 4814a5546..5af6ead45 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -2315,12 +2315,18 @@ func (c *Cluster) generatePrimaryPodDisruptionBudget() *policyv1.PodDisruptionBu } func (c *Cluster) generateCriticalOpPodDisruptionBudget() *policyv1.PodDisruptionBudget { - minAvailable := intstr.FromInt32(c.Spec.NumberOfInstances) + // MaxUnavailable: 0 blocks voluntary disruption of any pod carrying the + // critical-operation label, while keeping the budget satisfied when no + // pod matches (status.desiredHealthy stays 0 outside critical + // operations). The previous MinAvailable: N spec left desiredHealthy at + // N with zero matching pods during normal operation, permanently firing + // alerts like kube-prometheus-stack's KubePdbNotEnoughHealthyPods (#3020). + maxUnavailable := intstr.FromInt32(0) pdbEnabled := c.OpConfig.EnablePodDisruptionBudget - // if PodDisruptionBudget is disabled or if there are no DB pods, set the budget to 0. + // if PodDisruptionBudget is disabled or if there are no DB pods, allow all disruptions. if (pdbEnabled != nil && !(*pdbEnabled)) || c.Spec.NumberOfInstances <= 0 { - minAvailable = intstr.FromInt(0) + maxUnavailable = intstr.FromString("100%") } labels := c.labelsSet(false) @@ -2335,7 +2341,7 @@ func (c *Cluster) generateCriticalOpPodDisruptionBudget() *policyv1.PodDisruptio OwnerReferences: c.ownerReferences(), }, Spec: policyv1.PodDisruptionBudgetSpec{ - MinAvailable: &minAvailable, + MaxUnavailable: &maxUnavailable, Selector: &metav1.LabelSelector{ MatchLabels: labels, }, diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index c9ebc020f..d7afd80cc 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -2556,6 +2556,17 @@ func TestGeneratePodDisruptionBudget(t *testing.T) { } } + hasMaxUnavailable := func(expected string) func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error { + return func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error { + actual := podDisruptionBudget.Spec.MaxUnavailable.String() + if actual != expected { + return fmt.Errorf("PodDisruptionBudget MaxUnavailable is incorrect, got %s, expected %s", + actual, expected) + } + return nil + } + } + hasMinAvailable := func(expectedMinAvailable int) func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error { return func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error { actual := podDisruptionBudget.Spec.MinAvailable.IntVal @@ -2749,7 +2760,7 @@ func TestGeneratePodDisruptionBudget(t *testing.T) { check: []func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error{ testPodDisruptionBudgetOwnerReference, hasName("postgres-myapp-database-critical-op-pdb"), - hasMinAvailable(3), + hasMaxUnavailable("0"), testLabelsAndSelectors(false), }, }, @@ -2766,7 +2777,7 @@ func TestGeneratePodDisruptionBudget(t *testing.T) { check: []func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error{ testPodDisruptionBudgetOwnerReference, hasName("postgres-myapp-database-critical-op-pdb"), - hasMinAvailable(0), + hasMaxUnavailable("100%"), testLabelsAndSelectors(false), }, }, @@ -2783,7 +2794,7 @@ func TestGeneratePodDisruptionBudget(t *testing.T) { check: []func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error{ testPodDisruptionBudgetOwnerReference, hasName("postgres-myapp-database-critical-op-pdb"), - hasMinAvailable(0), + hasMaxUnavailable("100%"), testLabelsAndSelectors(false), }, }, @@ -2800,7 +2811,7 @@ func TestGeneratePodDisruptionBudget(t *testing.T) { check: []func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error{ testPodDisruptionBudgetOwnerReference, hasName("postgres-myapp-database-critical-op-pdb"), - hasMinAvailable(3), + hasMaxUnavailable("0"), testLabelsAndSelectors(false), }, },