diff --git a/pkg/cluster/pod.go b/pkg/cluster/pod.go index 80e3e8235..41705968f 100644 --- a/pkg/cluster/pod.go +++ b/pkg/cluster/pod.go @@ -296,10 +296,18 @@ func (c *Cluster) recreatePod(podName spec.NamespacedName) (*v1.Pod, error) { func (c *Cluster) isSafeToRecreatePods(pods *v1.PodList) bool { + /* + Operator should not re-create pods if there is at least one replica being bootstrapped + because Patroni might use other replicas to take basebackup from (see Patroni's "clonefrom" tag). + + XXX operator cannot forbid replica re-init, so we might still fail if re-init is started + after this check succeeds but before a pod is re-created + */ + for _, pod := range pods.Items { state, err := c.patroni.GetNodeState(&pod) if err != nil || state != "running" { - c.logger.Warningf("cannot re-create pod %s: patroni not in 'running' state", pod.Name) + c.logger.Warningf("cannot re-create pod %s: patroni in %s state", pod.Name, state) return false } @@ -323,7 +331,7 @@ func (c *Cluster) recreatePods() error { c.logger.Infof("there are %d pods in the cluster to recreate", len(pods.Items)) if !c.isSafeToRecreatePods(pods) { - return fmt.Errorf("postpone pod recreation until next Sync: some pods are being initilalized and recreation is unsafe") + return fmt.Errorf("postpone pod recreation until next Sync: recreation is unsafe because pods are being initilalized") } var ( diff --git a/pkg/util/patroni/patroni.go b/pkg/util/patroni/patroni.go index 76f0570c1..1e2158f01 100644 --- a/pkg/util/patroni/patroni.go +++ b/pkg/util/patroni/patroni.go @@ -38,18 +38,18 @@ type HttpGetResponse struct { Timeline int `json:"type,omitempty"` Xlog Xlog `json:"type,omitempty"` DatabaseSystemIdentifier string `json:"type,omitempty"` - patroniInfo Info `json:"type,omitempty"` + PatroniInfo Info `json:"type,omitempty"` } // Xlog contains wal locaiton type Xlog struct { - location int `json:"type,omitempty"` + Location int `json:"type,omitempty"` } // Info cotains Patroni version and cluser scope type Info struct { - version string `json:"type,omitempty"` - scope string `json:"type,omitempty"` + Version string `json:"type,omitempty"` + Scope string `json:"type,omitempty"` } // Patroni API client @@ -170,6 +170,7 @@ func (p *Patroni) GetNodeState(server *v1.Pod) (string, error) { if err != nil { return "", fmt.Errorf("could not unmarshal response: %v", err) } + p.logger.Infof("http get response: %s", pResponse) return pResponse.State, nil