restart instances via rest api instead of recreating pods, fixes bug with being unable to decrease some values, like max_connections (#1103)

* restart instances via rest api instead of recreating pods
* Ignore differences in bootstrap.dcs when compare SPILO_CONFIGURATION
* isBootstrapOnlyParameter is rewritten, instead of whitelist it uses blacklist
* added e2e test for max_connections decreasing
* documentation updated
* pending_restart flag added to restart api call, wait fot ttl seconds after restart
* refactoring, /restart returns error if pending_restart is set to true and patroni is not pending restart
* restart postgresql instances within pods only if pod's restart is not required
* patroni might need to restart postgresql after pods were recreated if values like max_connections decreased
* instancesRestart is not critical, try to restart pods if not successful
* cleanup

Co-authored-by: Felix Kunde <felix-kunde@gmx.de>

pkg/util/patroni/patroni.go

@@ -19,6 +19,8 @@ import (
 const (
 	failoverPath = "/failover"
 	configPath   = "/config"
+	statusPath   = "/patroni"
+	restartPath  = "/restart"
 	apiPort      = 8008
 	timeout      = 30 * time.Second
 )
@@ -28,6 +30,8 @@ type Interface interface {
 	Switchover(master *v1.Pod, candidate string) error
 	SetPostgresParameters(server *v1.Pod, options map[string]string) error
 	GetMemberData(server *v1.Pod) (MemberData, error)
+	Restart(server *v1.Pod) error
+	GetConfig(server *v1.Pod) (map[string]interface{}, error)
 }
 
 // Patroni API client
@@ -103,6 +107,32 @@ func (p *Patroni) httpPostOrPatch(method string, url string, body *bytes.Buffer)
 	return nil
 }
 
+func (p *Patroni) httpGet(url string) (string, error) {
+	request, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return "", fmt.Errorf("could not create request: %v", err)
+	}
+
+	p.logger.Debugf("making GET http request: %s", request.URL.String())
+
+	resp, err := p.httpClient.Do(request)
+	if err != nil {
+		return "", fmt.Errorf("could not make request: %v", err)
+	}
+	bodyBytes, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("could not read response: %v", err)
+	}
+	if err := resp.Body.Close(); err != nil {
+		return "", fmt.Errorf("could not close request: %v", err)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return string(bodyBytes), fmt.Errorf("patroni returned '%d'", resp.StatusCode)
+	}
+	return string(bodyBytes), nil
+}
+
 // Switchover by calling Patroni REST API
 func (p *Patroni) Switchover(master *v1.Pod, candidate string) error {
 	buf := &bytes.Buffer{}
@@ -149,6 +179,48 @@ type MemberData struct {
 	Patroni         MemberDataPatroni `json:"patroni"`
 }
 
+func (p *Patroni) GetConfigOrStatus(server *v1.Pod, path string) (map[string]interface{}, error) {
+	result := make(map[string]interface{})
+	apiURLString, err := apiURL(server)
+	if err != nil {
+		return result, err
+	}
+	body, err := p.httpGet(apiURLString + path)
+	err = json.Unmarshal([]byte(body), &result)
+	if err != nil {
+		return result, err
+	}
+
+	return result, err
+}
+
+func (p *Patroni) GetStatus(server *v1.Pod) (map[string]interface{}, error) {
+	return p.GetConfigOrStatus(server, statusPath)
+}
+
+func (p *Patroni) GetConfig(server *v1.Pod) (map[string]interface{}, error) {
+	return p.GetConfigOrStatus(server, configPath)
+}
+
+//Restart method restarts instance via Patroni POST API call.
+func (p *Patroni) Restart(server *v1.Pod) error {
+	buf := &bytes.Buffer{}
+	err := json.NewEncoder(buf).Encode(map[string]interface{}{"restart_pending": true})
+	if err != nil {
+		return fmt.Errorf("could not encode json: %v", err)
+	}
+	apiURLString, err := apiURL(server)
+	if err != nil {
+		return err
+	}
+	status, err := p.GetStatus(server)
+	pending_restart, ok := status["pending_restart"]
+	if !ok || !pending_restart.(bool) {
+		return nil
+	}
+	return p.httpPostOrPatch(http.MethodPost, apiURLString+restartPath, buf)
+}
+
 // GetMemberData read member data from patroni API
 func (p *Patroni) GetMemberData(server *v1.Pod) (MemberData, error) {