wrap getting Patroni state into retry (zalando#1293)

Retry calls to Patorni API to get cluster state

Co-authored-by: Sergey Dudoladov <[email protected]>

Author: sdudoladov

pkg/cluster/pod.go

@@ -4,13 +4,15 @@ import (
 	"context"
 	"fmt"
 	"math/rand"
+	"time"
 
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/zalando/postgres-operator/pkg/spec"
 	"github.com/zalando/postgres-operator/pkg/util"
+	"github.com/zalando/postgres-operator/pkg/util/retryutil"
 )
 
 func (c *Cluster) listPods() ([]v1.Pod, error) {
@@ -309,7 +311,23 @@ func (c *Cluster) isSafeToRecreatePods(pods *v1.PodList) bool {
 	}
 
 	for _, pod := range pods.Items {
-		state, err := c.patroni.GetPatroniMemberState(&pod)
+
+		var state string
+
+		err := retryutil.Retry(1*time.Second, 5*time.Second,
+			func() (bool, error) {
+
+				var err error
+
+				state, err = c.patroni.GetPatroniMemberState(&pod)
+
+				if err != nil {
+					return false, err
+				}
+				return true, nil
+			},
+		)
+
 		if err != nil {
 			c.logger.Errorf("failed to get Patroni state for pod: %s", err)
 			return false