Retry moveMasterPodsOffNode on failure instead of aborting (#3179)

attemptToMoveMasterPodsOffNode's transient errors (e.g. no synced
standby available yet) were treated as terminal, so the retry loop
gave up after a single ~4-5s attempt instead of retrying over
master_pod_move_timeout. This left masters stuck on draining nodes
indefinitely once a PodDisruptionBudget started rejecting evictions.

Log the failure and return (false, nil) so the retry loop keeps
polling every minute until the timeout expires.

Fixes #3176

pkg/controller/node_test.go

@@ -1,11 +1,18 @@
 package controller
 
 import (
+	"fmt"
+	"strings"
 	"testing"
+	"time"
 
+	logrustest "github.com/sirupsen/logrus/hooks/test"
 	"github.com/zalando/postgres-operator/v2/pkg/spec"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes/fake"
+	k8stesting "k8s.io/client-go/testing"
 )
 
 const (
@@ -93,3 +100,32 @@ func TestNodeIsReady(t *testing.T) {
 		}
 	}
 }
+
+// TestMoveMasterPodsOffNodeRetriesOnError ensures a failed attempt to move
+// master pods off a node is retried rather than aborting the whole retry
+// loop on the first error.
+func TestMoveMasterPodsOffNodeRetriesOnError(t *testing.T) {
+	clientSet := fake.NewSimpleClientset()
+	clientSet.PrependReactor("list", "pods", func(action k8stesting.Action) (bool, runtime.Object, error) {
+		return true, nil, fmt.Errorf("could not list pods")
+	})
+
+	controller := newNodeTestController()
+	controller.KubeClient.PodsGetter = clientSet.CoreV1()
+	// timeout == the retry interval hardcoded in moveMasterPodsOffNode, so
+	// the single retry attempt resolves synchronously without a real sleep.
+	controller.opConfig.MasterPodMoveTimeout = &metav1.Duration{Duration: 1 * time.Minute}
+
+	logger, hook := logrustest.NewNullLogger()
+	controller.logger = logger.WithField("pkg", "controller")
+
+	controller.moveMasterPodsOffNode(makeNode(map[string]string{}, false))
+
+	lastEntry := hook.LastEntry()
+	if lastEntry == nil {
+		t.Fatal("expected moveMasterPodsOffNode to log a warning")
+	}
+	if !strings.Contains(lastEntry.Message, "still failing after") {
+		t.Errorf("expected the retry loop to run out of attempts instead of aborting on the first error, got log message: %q", lastEntry.Message)
+	}
+}