add basic retry around switchover

pkg/cluster/pod.go

@@ -304,9 +304,16 @@ func (c *Cluster) MigrateMasterPod(podName spec.NamespacedName) error {
 	}
 
 	masterCandidateName := util.NameFromMeta(masterCandidatePod.ObjectMeta)
-	if err := c.Switchover(oldMaster, masterCandidateName); err != nil {
-		return fmt.Errorf("could not failover to pod %q: %v", masterCandidateName, err)
-	}
+	_ = retryutil.Retry(1*time.Minute, 5*time.Minute,
+		func() (bool, error) {
+			err := c.Switchover(oldMaster, masterCandidateName)
+			if err != nil {
+				c.logger.Errorf("could not failover to pod %q: %v", masterCandidateName, err)
+				return false, nil
+			}
+			return true, nil
+		},
+	)
 
 	return nil
 }