Add basic retry around switchover (zalando#1510)

* add basic retry around switchover


Co-authored-by: Sergey Dudoladov <[email protected]>

Author: sdudoladov

pkg/cluster/pod.go

@@ -304,8 +304,19 @@ func (c *Cluster) MigrateMasterPod(podName spec.NamespacedName) error {
 	}
 
 	masterCandidateName := util.NameFromMeta(masterCandidatePod.ObjectMeta)
-	if err := c.Switchover(oldMaster, masterCandidateName); err != nil {
-		return fmt.Errorf("could not failover to pod %q: %v", masterCandidateName, err)
+	err = retryutil.Retry(1*time.Minute, 5*time.Minute,
+		func() (bool, error) {
+			err := c.Switchover(oldMaster, masterCandidateName)
+			if err != nil {
+				c.logger.Errorf("could not failover to pod %q: %v", masterCandidateName, err)
+				return false, nil
+			}
+			return true, nil
+		},
+	)
+
+	if err != nil {
+		return fmt.Errorf("could not migrate master pod: %v", err)
 	}
 
 	return nil