public_git/postgres-operator
1
0
Fork 0
mirror of https://github.com/zalando/postgres-operator.git synced 2026-10-17 21:55:46 +02:00
Code Issues Projects Releases Wiki Activity

Fix rolling update deadlock when pods are stuck in non-running state (#3051)

Browse Source 
* add fix to recreate non running pods in syncStatefulsets

* remove TestSyncStatefulSetNonRunningPodsDoNotBlockRecreatio

* revert pod_test

* pod without status

---------

Co-authored-by: Felix Kunde <felix-kunde@gmx.de>
Co-authored-by: Ida Novindasari <idanovinda@gmail.com>
This commit is contained in:
annielzy
2026-04-28 12:08:34 +02:00
committed by GitHub
co-authored by Felix Kunde Ida Novindasari
parent 688bbf1b9e
commit 97f4de7cc0
3 changed files with 349 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/cluster/sync.go
+14 -2
View File
@@ -719,14 +719,26 @@ func (c *Cluster) syncStatefulSet() error {
if configPatched, restartPrimaryFirst, restartWait, err = c.syncPatroniConfig(pods, c.Spec.Patroni, requiredPgParameters); err != nil {
c.logger.Warningf("Patroni config updated? %v - errors during config sync: %v", configPatched, err)
postponeReasons = append(postponeReasons, "errors during Patroni config sync")
isSafeToRecreatePods = false
// Only mark unsafe if all pods are running. If some pods are not running,
// Patroni API errors are expected and should not block pod recreation,
// which is the only way to fix non-running pods.
if c.allPodsRunning(pods) {
isSafeToRecreatePods = false
} else {
c.logger.Warningf("ignoring Patroni config sync errors because some pods are not running")
}
}
// restart Postgres where it is still pending
if err = c.restartInstances(pods, restartWait, restartPrimaryFirst); err != nil {
c.logger.Errorf("errors while restarting Postgres in pods via Patroni API: %v", err)
postponeReasons = append(postponeReasons, "errors while restarting Postgres via Patroni API")
isSafeToRecreatePods = false
// Same logic: don't let unreachable non-running pods block recreation.
if c.allPodsRunning(pods) {
isSafeToRecreatePods = false
} else {
c.logger.Warningf("ignoring Patroni restart errors because some pods are not running")
}
}
// if we get here we also need to re-create the pods (either leftovers from the old