Use maxUnavailable for the critical-op PDB to stop idle alert noise
The critical-op PDB is created with minAvailable equal to numberOfInstances while its selector (critical-operation=true) matches no pods during normal operation. This leaves status.desiredHealthy at N and currentHealthy at 0 permanently, so monitoring stacks fire alerts like kube-prometheus-stack's KubePdbNotEnoughHealthyPods for every idle cluster (#3020). maxUnavailable: 0 provides the same protection while a critical operation is running - no voluntary evictions of labeled pods - but keeps the budget satisfied (desiredHealthy 0) when nothing matches. When PDBs are disabled or there are no instances, the budget relaxes to maxUnavailable 100% instead of minAvailable 0. Fixes #3020 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
bffde0f5e7
commit
61e506dea0
|
|
@ -2315,12 +2315,18 @@ func (c *Cluster) generatePrimaryPodDisruptionBudget() *policyv1.PodDisruptionBu
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Cluster) generateCriticalOpPodDisruptionBudget() *policyv1.PodDisruptionBudget {
|
func (c *Cluster) generateCriticalOpPodDisruptionBudget() *policyv1.PodDisruptionBudget {
|
||||||
minAvailable := intstr.FromInt32(c.Spec.NumberOfInstances)
|
// MaxUnavailable: 0 blocks voluntary disruption of any pod carrying the
|
||||||
|
// critical-operation label, while keeping the budget satisfied when no
|
||||||
|
// pod matches (status.desiredHealthy stays 0 outside critical
|
||||||
|
// operations). The previous MinAvailable: N spec left desiredHealthy at
|
||||||
|
// N with zero matching pods during normal operation, permanently firing
|
||||||
|
// alerts like kube-prometheus-stack's KubePdbNotEnoughHealthyPods (#3020).
|
||||||
|
maxUnavailable := intstr.FromInt32(0)
|
||||||
pdbEnabled := c.OpConfig.EnablePodDisruptionBudget
|
pdbEnabled := c.OpConfig.EnablePodDisruptionBudget
|
||||||
|
|
||||||
// if PodDisruptionBudget is disabled or if there are no DB pods, set the budget to 0.
|
// if PodDisruptionBudget is disabled or if there are no DB pods, allow all disruptions.
|
||||||
if (pdbEnabled != nil && !(*pdbEnabled)) || c.Spec.NumberOfInstances <= 0 {
|
if (pdbEnabled != nil && !(*pdbEnabled)) || c.Spec.NumberOfInstances <= 0 {
|
||||||
minAvailable = intstr.FromInt(0)
|
maxUnavailable = intstr.FromString("100%")
|
||||||
}
|
}
|
||||||
|
|
||||||
labels := c.labelsSet(false)
|
labels := c.labelsSet(false)
|
||||||
|
|
@ -2335,7 +2341,7 @@ func (c *Cluster) generateCriticalOpPodDisruptionBudget() *policyv1.PodDisruptio
|
||||||
OwnerReferences: c.ownerReferences(),
|
OwnerReferences: c.ownerReferences(),
|
||||||
},
|
},
|
||||||
Spec: policyv1.PodDisruptionBudgetSpec{
|
Spec: policyv1.PodDisruptionBudgetSpec{
|
||||||
MinAvailable: &minAvailable,
|
MaxUnavailable: &maxUnavailable,
|
||||||
Selector: &metav1.LabelSelector{
|
Selector: &metav1.LabelSelector{
|
||||||
MatchLabels: labels,
|
MatchLabels: labels,
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -2556,6 +2556,17 @@ func TestGeneratePodDisruptionBudget(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
hasMaxUnavailable := func(expected string) func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error {
|
||||||
|
return func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error {
|
||||||
|
actual := podDisruptionBudget.Spec.MaxUnavailable.String()
|
||||||
|
if actual != expected {
|
||||||
|
return fmt.Errorf("PodDisruptionBudget MaxUnavailable is incorrect, got %s, expected %s",
|
||||||
|
actual, expected)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
hasMinAvailable := func(expectedMinAvailable int) func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error {
|
hasMinAvailable := func(expectedMinAvailable int) func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error {
|
||||||
return func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error {
|
return func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error {
|
||||||
actual := podDisruptionBudget.Spec.MinAvailable.IntVal
|
actual := podDisruptionBudget.Spec.MinAvailable.IntVal
|
||||||
|
|
@ -2749,7 +2760,7 @@ func TestGeneratePodDisruptionBudget(t *testing.T) {
|
||||||
check: []func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error{
|
check: []func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error{
|
||||||
testPodDisruptionBudgetOwnerReference,
|
testPodDisruptionBudgetOwnerReference,
|
||||||
hasName("postgres-myapp-database-critical-op-pdb"),
|
hasName("postgres-myapp-database-critical-op-pdb"),
|
||||||
hasMinAvailable(3),
|
hasMaxUnavailable("0"),
|
||||||
testLabelsAndSelectors(false),
|
testLabelsAndSelectors(false),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|
@ -2766,7 +2777,7 @@ func TestGeneratePodDisruptionBudget(t *testing.T) {
|
||||||
check: []func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error{
|
check: []func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error{
|
||||||
testPodDisruptionBudgetOwnerReference,
|
testPodDisruptionBudgetOwnerReference,
|
||||||
hasName("postgres-myapp-database-critical-op-pdb"),
|
hasName("postgres-myapp-database-critical-op-pdb"),
|
||||||
hasMinAvailable(0),
|
hasMaxUnavailable("100%"),
|
||||||
testLabelsAndSelectors(false),
|
testLabelsAndSelectors(false),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|
@ -2783,7 +2794,7 @@ func TestGeneratePodDisruptionBudget(t *testing.T) {
|
||||||
check: []func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error{
|
check: []func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error{
|
||||||
testPodDisruptionBudgetOwnerReference,
|
testPodDisruptionBudgetOwnerReference,
|
||||||
hasName("postgres-myapp-database-critical-op-pdb"),
|
hasName("postgres-myapp-database-critical-op-pdb"),
|
||||||
hasMinAvailable(0),
|
hasMaxUnavailable("100%"),
|
||||||
testLabelsAndSelectors(false),
|
testLabelsAndSelectors(false),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|
@ -2800,7 +2811,7 @@ func TestGeneratePodDisruptionBudget(t *testing.T) {
|
||||||
check: []func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error{
|
check: []func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error{
|
||||||
testPodDisruptionBudgetOwnerReference,
|
testPodDisruptionBudgetOwnerReference,
|
||||||
hasName("postgres-myapp-database-critical-op-pdb"),
|
hasName("postgres-myapp-database-critical-op-pdb"),
|
||||||
hasMinAvailable(3),
|
hasMaxUnavailable("0"),
|
||||||
testLabelsAndSelectors(false),
|
testLabelsAndSelectors(false),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue