From b3b9e52f8b9d84e49981452dacc8922d9d71c4cb Mon Sep 17 00:00:00 2001
From: Sergey Dudoladov <sergey.dudoladov@gmail.com>
Date: Mon, 31 May 2021 16:27:34 +0200
Subject: [PATCH] add basic retry around switchover

---
 pkg/cluster/pod.go | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/pkg/cluster/pod.go b/pkg/cluster/pod.go
index c8dfe2a13..a1cc0ffd4 100644
--- a/pkg/cluster/pod.go
+++ b/pkg/cluster/pod.go
@@ -304,9 +304,16 @@ func (c *Cluster) MigrateMasterPod(podName spec.NamespacedName) error {
 	}
 
 	masterCandidateName := util.NameFromMeta(masterCandidatePod.ObjectMeta)
-	if err := c.Switchover(oldMaster, masterCandidateName); err != nil {
-		return fmt.Errorf("could not failover to pod %q: %v", masterCandidateName, err)
-	}
+	_ = retryutil.Retry(1*time.Minute, 5*time.Minute,
+		func() (bool, error) {
+			err := c.Switchover(oldMaster, masterCandidateName)
+			if err != nil {
+				c.logger.Errorf("could not failover to pod %q: %v", masterCandidateName, err)
+				return false, nil
+			}
+			return true, nil
+		},
+	)
 
 	return nil
 }