Add basic retry around switchover (#1510)

* add basic retry around switchover


Co-authored-by: Sergey Dudoladov <sergey.dudoladov@gmail.com>
This commit is contained in:
Sergey Dudoladov 2021-06-17 08:48:26 +02:00 committed by GitHub
parent ebb3204cdd
commit 53fb540c35
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 13 additions and 2 deletions

View File

@ -304,8 +304,19 @@ func (c *Cluster) MigrateMasterPod(podName spec.NamespacedName) error {
}
masterCandidateName := util.NameFromMeta(masterCandidatePod.ObjectMeta)
if err := c.Switchover(oldMaster, masterCandidateName); err != nil {
return fmt.Errorf("could not failover to pod %q: %v", masterCandidateName, err)
err = retryutil.Retry(1*time.Minute, 5*time.Minute,
func() (bool, error) {
err := c.Switchover(oldMaster, masterCandidateName)
if err != nil {
c.logger.Errorf("could not failover to pod %q: %v", masterCandidateName, err)
return false, nil
}
return true, nil
},
)
if err != nil {
return fmt.Errorf("could not migrate master pod: %v", err)
}
return nil