diff --git a/pkg/controller/node_test.go b/pkg/controller/node_test.go index b9326d9ef..1e2252a05 100644 --- a/pkg/controller/node_test.go +++ b/pkg/controller/node_test.go @@ -1,11 +1,19 @@ package controller import ( + "bytes" + "fmt" + "strings" "testing" + "time" + "github.com/sirupsen/logrus" "github.com/zalando/postgres-operator/v2/pkg/spec" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" ) const ( @@ -93,3 +101,33 @@ func TestNodeIsReady(t *testing.T) { } } } + +// TestMoveMasterPodsOffNodeRetriesOnError ensures a failed attempt to move +// master pods off a node is retried rather than aborting the whole retry +// loop on the first error. +func TestMoveMasterPodsOffNodeRetriesOnError(t *testing.T) { + clientSet := fake.NewSimpleClientset() + clientSet.PrependReactor("list", "pods", func(action k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, fmt.Errorf("could not list pods") + }) + + controller := newNodeTestController() + controller.KubeClient.PodsGetter = clientSet.CoreV1() + // timeout == the retry interval hardcoded in moveMasterPodsOffNode, so + // the single retry attempt resolves synchronously without a real sleep. + controller.opConfig.MasterPodMoveTimeout = &metav1.Duration{Duration: 1 * time.Minute} + + var logOutput bytes.Buffer + logger := logrus.New() + logger.SetOutput(&logOutput) + controller.logger = logger.WithField("pkg", "controller") + + controller.moveMasterPodsOffNode(makeNode(map[string]string{}, false)) + + if logOutput.Len() == 0 { + t.Fatal("expected moveMasterPodsOffNode to log a warning") + } + if !strings.Contains(logOutput.String(), "still failing after") { + t.Errorf("expected the retry loop to run out of attempts instead of aborting on the first error, got log output: %q", logOutput.String()) + } +}