From 53fb540c3577c2d7c273e2b4fed23e6c874eadfe Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Thu, 17 Jun 2021 08:48:26 +0200 Subject: [PATCH] Add basic retry around switchover (#1510) * add basic retry around switchover Co-authored-by: Sergey Dudoladov --- pkg/cluster/pod.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/pkg/cluster/pod.go b/pkg/cluster/pod.go index c8dfe2a13..229648dd1 100644 --- a/pkg/cluster/pod.go +++ b/pkg/cluster/pod.go @@ -304,8 +304,19 @@ func (c *Cluster) MigrateMasterPod(podName spec.NamespacedName) error { } masterCandidateName := util.NameFromMeta(masterCandidatePod.ObjectMeta) - if err := c.Switchover(oldMaster, masterCandidateName); err != nil { - return fmt.Errorf("could not failover to pod %q: %v", masterCandidateName, err) + err = retryutil.Retry(1*time.Minute, 5*time.Minute, + func() (bool, error) { + err := c.Switchover(oldMaster, masterCandidateName) + if err != nil { + c.logger.Errorf("could not failover to pod %q: %v", masterCandidateName, err) + return false, nil + } + return true, nil + }, + ) + + if err != nil { + return fmt.Errorf("could not migrate master pod: %v", err) } return nil