choose switchover candidate based on lag and role (#1700)

* choose switchover candidate based on lowest lag in MB and role (in synchronous mode)

pkg/util/patroni/patroni.go

@@ -21,6 +21,7 @@ import (
 const (
 	failoverPath = "/failover"
 	configPath   = "/config"
+	clusterPath  = "/cluster"
 	statusPath   = "/patroni"
 	restartPath  = "/restart"
 	apiPort      = 8008
@@ -29,6 +30,7 @@ const (
 
 // Interface describe patroni methods
 type Interface interface {
+	GetClusterMembers(master *v1.Pod) ([]ClusterMember, error)
 	Switchover(master *v1.Pod, candidate string) error
 	SetPostgresParameters(server *v1.Pod, options map[string]string) error
 	GetMemberData(server *v1.Pod) (MemberData, error)
@@ -175,6 +177,20 @@ func (p *Patroni) SetConfig(server *v1.Pod, config map[string]interface{}) error
 	return p.httpPostOrPatch(http.MethodPatch, apiURLString+configPath, buf)
 }
 
+// ClusterMembers array of cluster members from Patroni API
+type ClusterMembers struct {
+	Members []ClusterMember `json:"members"`
+}
+
+// ClusterMember cluster member data from Patroni API
+type ClusterMember struct {
+	Name     string `json:"name"`
+	Role     string `json:"role"`
+	State    string `json:"state"`
+	Timeline int    `json:"timeline"`
+	LagInMb  int    `json:"lag"`
+}
+
 // MemberDataPatroni child element
 type MemberDataPatroni struct {
 	Version string `json:"version"`
@@ -246,6 +262,27 @@ func (p *Patroni) Restart(server *v1.Pod) error {
 	return nil
 }
 
+// GetClusterMembers read cluster data from patroni API
+func (p *Patroni) GetClusterMembers(server *v1.Pod) ([]ClusterMember, error) {
+
+	apiURLString, err := apiURL(server)
+	if err != nil {
+		return []ClusterMember{}, err
+	}
+	body, err := p.httpGet(apiURLString + clusterPath)
+	if err != nil {
+		return []ClusterMember{}, err
+	}
+
+	data := ClusterMembers{}
+	err = json.Unmarshal([]byte(body), &data)
+	if err != nil {
+		return []ClusterMember{}, err
+	}
+
+	return data.Members, nil
+}
+
 // GetMemberData read member data from patroni API
 func (p *Patroni) GetMemberData(server *v1.Pod) (MemberData, error) {
 

pkg/util/patroni/patroni_test.go

@@ -85,6 +85,55 @@ func TestApiURL(t *testing.T) {
 	}
 }
 
+func TestGetClusterMembers(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	expectedClusterMemberData := []ClusterMember{
+		{
+			Name:     "acid-test-cluster-0",
+			Role:     "leader",
+			State:    "running",
+			Timeline: 1,
+			LagInMb:  0,
+		}, {
+			Name:     "acid-test-cluster-1",
+			Role:     "sync_standby",
+			State:    "running",
+			Timeline: 1,
+			LagInMb:  0,
+		}, {
+			Name:     "acid-test-cluster-2",
+			Role:     "replica",
+			State:    "running",
+			Timeline: 1,
+			LagInMb:  0,
+		}}
+
+	json := `{"members": [{"name": "acid-test-cluster-0", "role": "leader", "state": "running", "api_url": "http://192.168.100.1:8008/patroni", "host": "192.168.100.1", "port": 5432, "timeline": 1}, {"name": "acid-test-cluster-1", "role": "sync_standby", "state": "running", "api_url": "http://192.168.100.2:8008/patroni", "host": "192.168.100.2", "port": 5432, "timeline": 1, "lag": 0}, {"name": "acid-test-cluster-2", "role": "replica", "state": "running", "api_url": "http://192.168.100.3:8008/patroni", "host": "192.168.100.3", "port": 5432, "timeline": 1, "lag": 0}]}`
+	r := ioutil.NopCloser(bytes.NewReader([]byte(json)))
+
+	response := http.Response{
+		StatusCode: 200,
+		Body:       r,
+	}
+
+	mockClient := mocks.NewMockHTTPClient(ctrl)
+	mockClient.EXPECT().Get(gomock.Any()).Return(&response, nil)
+
+	p := New(logger, mockClient)
+
+	clusterMemberData, err := p.GetClusterMembers(newMockPod("192.168.100.1"))
+
+	if !reflect.DeepEqual(expectedClusterMemberData, clusterMemberData) {
+		t.Errorf("Patroni cluster members differ: expected: %#v, got: %#v", expectedClusterMemberData, clusterMemberData)
+	}
+
+	if err != nil {
+		t.Errorf("Could not read Patroni data: %v", err)
+	}
+}
+
 func TestGetMemberData(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()