reflect latest change in restart logic

This commit is contained in:
Felix Kunde 2021-10-26 17:06:06 +02:00
commit a5acfd3b42
9 changed files with 303 additions and 69 deletions

View File

@ -23,6 +23,7 @@ RUN apt-get update \
&& curl --silent https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ && curl --silent https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
&& apt-get update \ && apt-get update \
&& apt-get install --no-install-recommends -y \ && apt-get install --no-install-recommends -y \
postgresql-client-14 \
postgresql-client-13 \ postgresql-client-13 \
postgresql-client-12 \ postgresql-client-12 \
postgresql-client-11 \ postgresql-client-11 \

View File

@ -110,7 +110,7 @@ These parameters are grouped directly under the `spec` key in the manifest.
`CREATEDB`, `BYPASSURL`. A login user is created by default unless NOLOGIN is `CREATEDB`, `BYPASSURL`. A login user is created by default unless NOLOGIN is
specified, in which case the operator creates a role. One can specify empty specified, in which case the operator creates a role. One can specify empty
flags by providing a JSON empty array '*[]*'. If the config option flags by providing a JSON empty array '*[]*'. If the config option
`enable_cross_namespace_secrets` is enabled you can specify the namespace in `enable_cross_namespace_secret` is enabled you can specify the namespace in
the user name in the form `{namespace}.{username}` and the operator will the user name in the form `{namespace}.{username}` and the operator will
create the K8s secret in that namespace. The part after the first `.` is create the K8s secret in that namespace. The part after the first `.` is
considered to be the user name. Optional. considered to be the user name. Optional.
@ -215,7 +215,7 @@ information, see [user docs](../user.md#prepared-databases-with-roles-and-defaul
* **secretNamespace** * **secretNamespace**
for each default LOGIN role the operator will create a secret. You can for each default LOGIN role the operator will create a secret. You can
specify the namespace in which these secrets will get created, if specify the namespace in which these secrets will get created, if
`enable_cross_namespace_secrets` is set to `true` in the config. Otherwise, `enable_cross_namespace_secret` is set to `true` in the config. Otherwise,
the cluster namespace is used. the cluster namespace is used.
## Postgres parameters ## Postgres parameters

View File

@ -264,7 +264,7 @@ configuration they are grouped under the `kubernetes` key.
[admin docs](../administrator.md#pod-disruption-budget) for more information. [admin docs](../administrator.md#pod-disruption-budget) for more information.
Default is true. Default is true.
* **enable_cross_namespace_secrets** * **enable_cross_namespace_secret**
To allow secrets in a different namespace other than the Postgres cluster To allow secrets in a different namespace other than the Postgres cluster
namespace. Once enabled, specify the namespace in the user name under the namespace. Once enabled, specify the namespace in the user name under the
`users` section in the form `{namespace}.{username}`. The default is `false`. `users` section in the form `{namespace}.{username}`. The default is `false`.

View File

@ -16,7 +16,7 @@ RUN apt-get update \
curl \ curl \
vim \ vim \
&& pip3 install --no-cache-dir -r requirements.txt \ && pip3 install --no-cache-dir -r requirements.txt \
&& curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl \ && curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.22.0/bin/linux/amd64/kubectl \
&& chmod +x ./kubectl \ && chmod +x ./kubectl \
&& mv ./kubectl /usr/local/bin/kubectl \ && mv ./kubectl /usr/local/bin/kubectl \
&& apt-get clean \ && apt-get clean \

View File

@ -8,7 +8,7 @@ IFS=$'\n\t'
readonly cluster_name="postgres-operator-e2e-tests" readonly cluster_name="postgres-operator-e2e-tests"
readonly kubeconfig_path="/tmp/kind-config-${cluster_name}" readonly kubeconfig_path="/tmp/kind-config-${cluster_name}"
readonly spilo_image="registry.opensource.zalan.do/acid/spilo-13-e2e:0.3" readonly spilo_image="registry.opensource.zalan.do/acid/spilo-14-e2e:0.1"
readonly e2e_test_runner_image="registry.opensource.zalan.do/acid/postgres-operator-e2e-tests-runner:0.3" readonly e2e_test_runner_image="registry.opensource.zalan.do/acid/postgres-operator-e2e-tests-runner:0.3"
export GOPATH=${GOPATH-~/go} export GOPATH=${GOPATH-~/go}

View File

@ -11,8 +11,8 @@ from kubernetes import client
from tests.k8s_api import K8s from tests.k8s_api import K8s
from kubernetes.client.rest import ApiException from kubernetes.client.rest import ApiException
SPILO_CURRENT = "registry.opensource.zalan.do/acid/spilo-13-e2e:0.3" SPILO_CURRENT = "registry.opensource.zalan.do/acid/spilo-14-e2e:0.1"
SPILO_LAZY = "registry.opensource.zalan.do/acid/spilo-13-e2e:0.4" SPILO_LAZY = "registry.opensource.zalan.do/acid/spilo-14-e2e:0.2"
def to_selector(labels): def to_selector(labels):
@ -85,6 +85,7 @@ class EndToEndTestCase(unittest.TestCase):
# set a single K8s wrapper for all tests # set a single K8s wrapper for all tests
k8s = cls.k8s = K8s() k8s = cls.k8s = K8s()
cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster'
# remove existing local storage class and create hostpath class # remove existing local storage class and create hostpath class
try: try:
@ -150,8 +151,8 @@ class EndToEndTestCase(unittest.TestCase):
result = k8s.create_with_kubectl("manifests/minimal-postgres-manifest.yaml") result = k8s.create_with_kubectl("manifests/minimal-postgres-manifest.yaml")
print('stdout: {}, stderr: {}'.format(result.stdout, result.stderr)) print('stdout: {}, stderr: {}'.format(result.stdout, result.stderr))
try: try:
k8s.wait_for_pod_start('spilo-role=master') k8s.wait_for_pod_start('spilo-role=master,' + cluster_label)
k8s.wait_for_pod_start('spilo-role=replica') k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label)
except timeout_decorator.TimeoutError: except timeout_decorator.TimeoutError:
print('Operator log: {}'.format(k8s.get_operator_log())) print('Operator log: {}'.format(k8s.get_operator_log()))
raise raise
@ -1064,12 +1065,13 @@ class EndToEndTestCase(unittest.TestCase):
via restarting cluster through Patroni's rest api via restarting cluster through Patroni's rest api
''' '''
k8s = self.k8s k8s = self.k8s
masterPod = k8s.get_cluster_leader_pod() leader = k8s.get_cluster_leader_pod()
labels = 'application=spilo,cluster-name=acid-minimal-cluster,spilo-role=master' replica = k8s.get_cluster_replica_pod()
creationTimestamp = masterPod.metadata.creation_timestamp masterCreationTimestamp = leader.metadata.creation_timestamp
replicaCreationTimestamp = replica.metadata.creation_timestamp
new_max_connections_value = "50" new_max_connections_value = "50"
# adjust max_connection # adjust Postgres config
pg_patch_config = { pg_patch_config = {
"spec": { "spec": {
"postgresql": { "postgresql": {
@ -1098,7 +1100,7 @@ class EndToEndTestCase(unittest.TestCase):
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
def compare_config(): def compare_config():
effective_config = k8s.patroni_rest(masterPod.metadata.name, "config") effective_config = k8s.patroni_rest(leader.metadata.name, "config")
desired_config = pg_patch_config["spec"]["patroni"] desired_config = pg_patch_config["spec"]["patroni"]
desired_parameters = pg_patch_config["spec"]["postgresql"]["parameters"] desired_parameters = pg_patch_config["spec"]["postgresql"]["parameters"]
effective_parameters = effective_config["postgresql"]["parameters"] effective_parameters = effective_config["postgresql"]["parameters"]
@ -1115,19 +1117,63 @@ class EndToEndTestCase(unittest.TestCase):
"synchronous_mode not updated") "synchronous_mode not updated")
return True return True
# check if Patroni config has been updated
self.eventuallyTrue(compare_config, "Postgres config not applied") self.eventuallyTrue(compare_config, "Postgres config not applied")
# make sure that pods were not recreated
leader = k8s.get_cluster_leader_pod()
replica = k8s.get_cluster_replica_pod()
self.assertEqual(masterCreationTimestamp, leader.metadata.creation_timestamp,
"Master pod creation timestamp is updated")
self.assertEqual(replicaCreationTimestamp, replica.metadata.creation_timestamp,
"Master pod creation timestamp is updated")
# query max_connections setting
setting_query = """ setting_query = """
SELECT setting SELECT setting
FROM pg_settings FROM pg_settings
WHERE name = 'max_connections'; WHERE name = 'max_connections';
""" """
self.eventuallyEqual(lambda: self.query_database(masterPod.metadata.name, "postgres", setting_query)[0], new_max_connections_value, self.eventuallyEqual(lambda: self.query_database(leader.metadata.name, "postgres", setting_query)[0], new_max_connections_value,
"New max_connections setting not applied", 10, 5) "New max_connections setting not applied on master", 10, 5)
self.eventuallyNotEqual(lambda: self.query_database(replica.metadata.name, "postgres", setting_query)[0], new_max_connections_value,
"Expected max_connections not to be updated on replica since Postgres was restarted there first", 10, 5)
# make sure that pod wasn't recreated # the next sync should restart the replica because it has pending_restart flag set
self.assertEqual(creationTimestamp, masterPod.metadata.creation_timestamp, # force next sync by deleting the operator pod
"Master pod creation timestamp is updated") k8s.delete_operator_pod()
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
self.eventuallyEqual(lambda: self.query_database(replica.metadata.name, "postgres", setting_query)[0], new_max_connections_value,
"New max_connections setting not applied on replica", 10, 5)
# decrease max_connections again
# this time restart will be correct and new value should appear on both instances
lower_max_connections_value = "30"
pg_patch_max_connections = {
"spec": {
"postgresql": {
"parameters": {
"max_connections": lower_max_connections_value
}
}
}
}
k8s.api.custom_objects_api.patch_namespaced_custom_object(
"acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_max_connections)
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
# check Patroni config again
pg_patch_config["spec"]["postgresql"]["parameters"]["max_connections"] = lower_max_connections_value
self.eventuallyTrue(compare_config, "Postgres config not applied")
# and query max_connections setting again
self.eventuallyEqual(lambda: self.query_database(leader.metadata.name, "postgres", setting_query)[0], lower_max_connections_value,
"Previous max_connections setting not applied on master", 10, 5)
self.eventuallyEqual(lambda: self.query_database(replica.metadata.name, "postgres", setting_query)[0], lower_max_connections_value,
"Previous max_connections setting not applied on replica", 10, 5)
except timeout_decorator.TimeoutError: except timeout_decorator.TimeoutError:
print('Operator log: {}'.format(k8s.get_operator_log())) print('Operator log: {}'.format(k8s.get_operator_log()))
@ -1554,6 +1600,7 @@ class EndToEndTestCase(unittest.TestCase):
Toggle pod anti affinty to distribute pods accross nodes (replica in particular). Toggle pod anti affinty to distribute pods accross nodes (replica in particular).
''' '''
k8s = self.k8s k8s = self.k8s
cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster'
failover_targets = self.get_failover_targets(master_node, replica_nodes) failover_targets = self.get_failover_targets(master_node, replica_nodes)
# enable pod anti affintiy in config map which should trigger movement of replica # enable pod anti affintiy in config map which should trigger movement of replica
@ -1572,8 +1619,8 @@ class EndToEndTestCase(unittest.TestCase):
} }
} }
k8s.update_config(patch_disable_antiaffinity, "disable antiaffinity") k8s.update_config(patch_disable_antiaffinity, "disable antiaffinity")
k8s.wait_for_pod_start('spilo-role=master') k8s.wait_for_pod_start('spilo-role=master,' + cluster_label)
k8s.wait_for_pod_start('spilo-role=replica') k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label)
return True return True
def list_databases(self, pod_name): def list_databases(self, pod_name):

View File

@ -6,6 +6,7 @@ import (
"fmt" "fmt"
"reflect" "reflect"
"regexp" "regexp"
"strconv"
"strings" "strings"
"time" "time"
@ -20,6 +21,14 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
) )
var requireMasterRestartWhenDecreased = []string{
"max_connections",
"max_prepared_transactions",
"max_locks_per_transaction",
"max_worker_processes",
"max_wal_senders",
}
// Sync syncs the cluster, making sure the actual Kubernetes objects correspond to what is defined in the manifest. // Sync syncs the cluster, making sure the actual Kubernetes objects correspond to what is defined in the manifest.
// Unlike the update, sync does not error out if some objects do not exist and takes care of creating them. // Unlike the update, sync does not error out if some objects do not exist and takes care of creating them.
func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
@ -270,11 +279,9 @@ func (c *Cluster) syncPodDisruptionBudget(isUpdate bool) error {
func (c *Cluster) syncStatefulSet() error { func (c *Cluster) syncStatefulSet() error {
var ( var (
masterPod *v1.Pod restartWait uint32
restartTTL uint32 restartMasterFirst bool
instanceRestartRequired bool
) )
podsToRecreate := make([]v1.Pod, 0) podsToRecreate := make([]v1.Pod, 0)
switchoverCandidates := make([]spec.NamespacedName, 0) switchoverCandidates := make([]spec.NamespacedName, 0)
@ -408,35 +415,43 @@ func (c *Cluster) syncStatefulSet() error {
c.logger.Warningf("could not get Postgres config from pod %s: %v", podName, err) c.logger.Warningf("could not get Postgres config from pod %s: %v", podName, err)
continue continue
} }
restartWait = patroniConfig.LoopWait
// empty config probably means cluster is not fully initialized yet, e.g. restoring from backup // empty config probably means cluster is not fully initialized yet, e.g. restoring from backup
// do not attempt a restart // do not attempt a restart
if !reflect.DeepEqual(patroniConfig, emptyPatroniConfig) || len(pgParameters) > 0 { if !reflect.DeepEqual(patroniConfig, emptyPatroniConfig) || len(pgParameters) > 0 {
instanceRestartRequired, err = c.checkAndSetGlobalPostgreSQLConfiguration(&pod, patroniConfig, c.Spec.Patroni, pgParameters, c.Spec.Parameters) restartMasterFirst, err = c.checkAndSetGlobalPostgreSQLConfiguration(&pod, patroniConfig, c.Spec.Patroni, pgParameters, c.Spec.Parameters)
if err != nil { if err != nil {
c.logger.Warningf("could not set PostgreSQL configuration options for pod %s: %v", podName, err) c.logger.Warningf("could not set PostgreSQL configuration options for pod %s: %v", podName, err)
continue continue
} }
restartTTL = patroniConfig.TTL
// it could take up to LoopWait to apply the config
time.Sleep(time.Duration(restartWait)*time.Second + time.Second*2)
break break
} }
} }
// if the config update requires a restart, call Patroni restart for replicas first, then master // restart instances if required
if instanceRestartRequired { remainingPods := make([]*v1.Pod, 0)
c.logger.Debug("restarting Postgres server within pods") skipRole := Master
for i, pod := range pods { if restartMasterFirst {
role := PostgresRole(pod.Labels[c.OpConfig.PodRoleLabel]) skipRole = Replica
if role == Master { }
masterPod = &pods[i] for i, pod := range pods {
continue role := PostgresRole(pod.Labels[c.OpConfig.PodRoleLabel])
} if role == skipRole {
c.restartInstance(&pod) remainingPods = append(remainingPods, &pods[i])
time.Sleep(time.Duration(restartTTL) * time.Second) continue
} }
c.restartInstance(&pod, restartWait)
}
if masterPod != nil { // in most cases only the master should be left to restart
c.restartInstance(masterPod) if len(remainingPods) > 0 {
for _, remainingPod := range remainingPods {
c.restartInstance(remainingPod, restartWait)
} }
} }
@ -453,19 +468,27 @@ func (c *Cluster) syncStatefulSet() error {
return nil return nil
} }
func (c *Cluster) restartInstance(pod *v1.Pod) { func (c *Cluster) restartInstance(pod *v1.Pod, restartWait uint32) {
podName := util.NameFromMeta(pod.ObjectMeta) podName := util.NameFromMeta(pod.ObjectMeta)
role := PostgresRole(pod.Labels[c.OpConfig.PodRoleLabel]) role := PostgresRole(pod.Labels[c.OpConfig.PodRoleLabel])
c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", fmt.Sprintf("restarting Postgres server within %s pod %s", role, pod.Name)) // if the config update requires a restart, call Patroni restart
memberData, err := c.patroni.GetMemberData(pod)
if err := c.patroni.Restart(pod); err != nil { if err != nil {
c.logger.Warningf("could not restart Postgres server within %s pod %s: %v", role, podName, err) c.logger.Debugf("could not get member data of %s pod %s - skipping possible restart attempt: %v", role, podName, err)
return return
} }
c.logger.Debugf("Postgres server successfuly restarted in %s pod %s", role, podName) // do restart only when it is pending
c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", fmt.Sprintf("Postgres server restart done for %s pod %s", role, pod.Name)) if memberData.PendingRestart {
c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", fmt.Sprintf("restarting Postgres server within %s pod %s", role, pod.Name))
if err := c.patroni.Restart(pod); err != nil {
c.logger.Warningf("could not restart Postgres server within %s pod %s: %v", role, podName, err)
return
}
time.Sleep(time.Duration(restartWait) * time.Second)
c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", fmt.Sprintf("Postgres server restart done for %s pod %s", role, pod.Name))
}
} }
// AnnotationsToPropagate get the annotations to update if required // AnnotationsToPropagate get the annotations to update if required
@ -505,20 +528,10 @@ func (c *Cluster) AnnotationsToPropagate(annotations map[string]string) map[stri
func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration(pod *v1.Pod, effectivePatroniConfig, desiredPatroniConfig acidv1.Patroni, effectivePgParameters, desiredPgParameters map[string]string) (bool, error) { func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration(pod *v1.Pod, effectivePatroniConfig, desiredPatroniConfig acidv1.Patroni, effectivePgParameters, desiredPgParameters map[string]string) (bool, error) {
configToSet := make(map[string]interface{}) configToSet := make(map[string]interface{})
parametersToSet := make(map[string]string) parametersToSet := make(map[string]string)
restartMaster := make([]bool, 0)
requiresMasterRestart := false
// compare parameters under postgresql section with c.Spec.Postgresql.Parameters from manifest // compare effective and desired Patroni config options
for desiredOption, desiredValue := range desiredPgParameters {
effectiveValue := effectivePgParameters[desiredOption]
if isBootstrapOnlyParameter(desiredOption) && (effectiveValue != desiredValue) {
parametersToSet[desiredOption] = desiredValue
}
}
if len(parametersToSet) > 0 {
configToSet["postgresql"] = map[string]interface{}{constants.PatroniPGParametersParameterName: parametersToSet}
}
// compare other options from config with c.Spec.Patroni from manifest
if desiredPatroniConfig.LoopWait > 0 && desiredPatroniConfig.LoopWait != effectivePatroniConfig.LoopWait { if desiredPatroniConfig.LoopWait > 0 && desiredPatroniConfig.LoopWait != effectivePatroniConfig.LoopWait {
configToSet["loop_wait"] = desiredPatroniConfig.LoopWait configToSet["loop_wait"] = desiredPatroniConfig.LoopWait
} }
@ -555,6 +568,34 @@ func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration(pod *v1.Pod, effectiv
configToSet["slots"] = slotsToSet configToSet["slots"] = slotsToSet
} }
// compare parameters under postgresql section with c.Spec.Postgresql.Parameters from manifest
for desiredOption, desiredValue := range desiredPgParameters {
effectiveValue := effectivePgParameters[desiredOption]
if isBootstrapOnlyParameter(desiredOption) && (effectiveValue != desiredValue) {
parametersToSet[desiredOption] = desiredValue
if util.SliceContains(requireMasterRestartWhenDecreased, desiredOption) {
effectiveValueNum, errConv := strconv.Atoi(effectiveValue)
desiredValueNum, errConv2 := strconv.Atoi(desiredValue)
if errConv != nil || errConv2 != nil {
continue
}
if effectiveValueNum > desiredValueNum {
restartMaster = append(restartMaster, true)
continue
}
}
restartMaster = append(restartMaster, false)
}
}
if !util.SliceContains(restartMaster, false) && len(configToSet) == 0 {
requiresMasterRestart = true
}
if len(parametersToSet) > 0 {
configToSet["postgresql"] = map[string]interface{}{constants.PatroniPGParametersParameterName: parametersToSet}
}
if len(configToSet) == 0 { if len(configToSet) == 0 {
return false, nil return false, nil
} }
@ -570,10 +611,10 @@ func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration(pod *v1.Pod, effectiv
c.logger.Debugf("patching Postgres config via Patroni API on pod %s with following options: %s", c.logger.Debugf("patching Postgres config via Patroni API on pod %s with following options: %s",
podName, configToSetJson) podName, configToSetJson)
if err = c.patroni.SetConfig(pod, configToSet); err != nil { if err = c.patroni.SetConfig(pod, configToSet); err != nil {
return true, fmt.Errorf("could not patch postgres parameters within pod %s: %v", podName, err) return requiresMasterRestart, fmt.Errorf("could not patch postgres parameters with a pod %s: %v", podName, err)
} }
return true, nil return requiresMasterRestart, nil
} }
func (c *Cluster) syncSecrets() error { func (c *Cluster) syncSecrets() error {

View File

@ -1,22 +1,40 @@
package cluster package cluster
import ( import (
"bytes"
"io/ioutil"
"net/http"
"testing" "testing"
"time" "time"
"context" "context"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/types"
"github.com/golang/mock/gomock"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/zalando/postgres-operator/mocks"
acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
fakeacidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/fake" fakeacidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/fake"
"github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/config"
"github.com/zalando/postgres-operator/pkg/util/k8sutil" "github.com/zalando/postgres-operator/pkg/util/k8sutil"
"github.com/zalando/postgres-operator/pkg/util/patroni"
"k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/kubernetes/fake"
) )
var patroniLogger = logrus.New().WithField("test", "patroni")
func newMockPod(ip string) *v1.Pod {
return &v1.Pod{
Status: v1.PodStatus{
PodIP: ip,
},
}
}
func newFakeK8sSyncClient() (k8sutil.KubernetesClient, *fake.Clientset) { func newFakeK8sSyncClient() (k8sutil.KubernetesClient, *fake.Clientset) {
acidClientSet := fakeacidv1.NewSimpleClientset() acidClientSet := fakeacidv1.NewSimpleClientset()
clientSet := fake.NewSimpleClientset() clientSet := fake.NewSimpleClientset()
@ -113,3 +131,134 @@ func TestSyncStatefulSetsAnnotations(t *testing.T) {
t.Errorf("%s: inherited annotation not found in desired statefulset: %#v", testName, desiredSts.Annotations) t.Errorf("%s: inherited annotation not found in desired statefulset: %#v", testName, desiredSts.Annotations)
} }
} }
func TestCheckAndSetGlobalPostgreSQLConfiguration(t *testing.T) {
testName := "test config comparison"
client, _ := newFakeK8sSyncClient()
clusterName := "acid-test-cluster"
namespace := "default"
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pg := acidv1.Postgresql{
ObjectMeta: metav1.ObjectMeta{
Name: clusterName,
Namespace: namespace,
},
Spec: acidv1.PostgresSpec{
Patroni: acidv1.Patroni{
TTL: 20,
},
PostgresqlParam: acidv1.PostgresqlParam{
Parameters: map[string]string{
"log_min_duration_statement": "200",
"max_connections": "50",
},
},
Volume: acidv1.Volume{
Size: "1Gi",
},
},
}
var cluster = New(
Config{
OpConfig: config.Config{
PodManagementPolicy: "ordered_ready",
Resources: config.Resources{
ClusterLabels: map[string]string{"application": "spilo"},
ClusterNameLabel: "cluster-name",
DefaultCPURequest: "300m",
DefaultCPULimit: "300m",
DefaultMemoryRequest: "300Mi",
DefaultMemoryLimit: "300Mi",
PodRoleLabel: "spilo-role",
ResourceCheckInterval: time.Duration(3),
ResourceCheckTimeout: time.Duration(10),
},
},
}, client, pg, logger, eventRecorder)
// mocking a config after setConfig is called
configJson := `{"postgresql": {"parameters": {"log_min_duration_statement": 200, "max_connections": 50}}}, "ttl": 20}`
r := ioutil.NopCloser(bytes.NewReader([]byte(configJson)))
response := http.Response{
StatusCode: 200,
Body: r,
}
mockClient := mocks.NewMockHTTPClient(ctrl)
mockClient.EXPECT().Do(gomock.Any()).Return(&response, nil).AnyTimes()
p := patroni.New(patroniLogger, mockClient)
cluster.patroni = p
mockPod := newMockPod("192.168.100.1")
// simulate existing config that differs with cluster.Spec
tests := []struct {
subtest string
pod *v1.Pod
patroni acidv1.Patroni
pgParams map[string]string
restartMaster bool
}{
{
subtest: "Patroni and Postgresql.Parameters differ - restart replica first",
pod: mockPod,
patroni: acidv1.Patroni{
TTL: 30, // desired 20
},
pgParams: map[string]string{
"log_min_duration_statement": "500", // desired 200
"max_connections": "100", // desired 50
},
restartMaster: false,
},
{
subtest: "multiple Postgresql.Parameters differ - restart replica first",
pod: mockPod,
patroni: acidv1.Patroni{
TTL: 20,
},
pgParams: map[string]string{
"log_min_duration_statement": "500", // desired 200
"max_connections": "100", // desired 50
},
restartMaster: false,
},
{
subtest: "desired max_connections bigger - restart replica first",
pod: mockPod,
patroni: acidv1.Patroni{
TTL: 20,
},
pgParams: map[string]string{
"log_min_duration_statement": "200",
"max_connections": "30", // desired 50
},
restartMaster: false,
},
{
subtest: "desired max_connections smaller - restart master first",
pod: mockPod,
patroni: acidv1.Patroni{
TTL: 20,
},
pgParams: map[string]string{
"log_min_duration_statement": "200",
"max_connections": "100", // desired 50
},
restartMaster: true,
},
}
for _, tt := range tests {
requireMasterRestart, err := cluster.checkAndSetGlobalPostgreSQLConfiguration(tt.pod, tt.patroni, cluster.Spec.Patroni, tt.pgParams, cluster.Spec.Parameters)
assert.NoError(t, err)
if requireMasterRestart != tt.restartMaster {
t.Errorf("%s - %s: unexpect master restart strategy, got %v, expected %v", testName, tt.subtest, requireMasterRestart, tt.restartMaster)
}
}
}

View File

@ -238,16 +238,12 @@ func (p *Patroni) Restart(server *v1.Pod) error {
if err != nil { if err != nil {
return err return err
} }
memberData, err := p.GetMemberData(server) if err := p.httpPostOrPatch(http.MethodPost, apiURLString+restartPath, buf); err != nil {
if err != nil {
return err return err
} }
p.logger.Infof("Postgres server successfuly restarted in pod %s", server.Name)
// do restart only when it is pending return nil
if !memberData.PendingRestart {
return nil
}
return p.httpPostOrPatch(http.MethodPost, apiURLString+restartPath, buf)
} }
// GetMemberData read member data from patroni API // GetMemberData read member data from patroni API