remove final failover check in node affinity test
This commit is contained in:
Felix Kunde
parent
87499cc068
commit
61c30beb2d
|
|
@ -205,7 +205,7 @@ class EndToEndTestCase(unittest.TestCase):
|
||||||
"enable_team_member_deprecation": "true",
|
"enable_team_member_deprecation": "true",
|
||||||
"role_deletion_suffix": "_delete_me",
|
"role_deletion_suffix": "_delete_me",
|
||||||
"resync_period": "15s",
|
"resync_period": "15s",
|
||||||
"repair_period": "10s",
|
"repair_period": "15s",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
k8s.update_config(enable_postgres_team_crd)
|
k8s.update_config(enable_postgres_team_crd)
|
||||||
|
|
@ -296,6 +296,133 @@ class EndToEndTestCase(unittest.TestCase):
|
||||||
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"},
|
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"},
|
||||||
"Operator does not get in sync")
|
"Operator does not get in sync")
|
||||||
|
|
||||||
|
@timeout_decorator.timeout(TEST_TIMEOUT_SEC)
|
||||||
|
def test_config_update(self):
|
||||||
|
'''
|
||||||
|
Change Postgres config under Spec.Postgresql.Parameters and Spec.Patroni
|
||||||
|
and query Patroni config endpoint to check if manifest changes got applied
|
||||||
|
via restarting cluster through Patroni's rest api
|
||||||
|
'''
|
||||||
|
k8s = self.k8s
|
||||||
|
leader = k8s.get_cluster_leader_pod()
|
||||||
|
replica = k8s.get_cluster_replica_pod()
|
||||||
|
masterCreationTimestamp = leader.metadata.creation_timestamp
|
||||||
|
replicaCreationTimestamp = replica.metadata.creation_timestamp
|
||||||
|
new_max_connections_value = "50"
|
||||||
|
|
||||||
|
# adjust Postgres config
|
||||||
|
pg_patch_config = {
|
||||||
|
"spec": {
|
||||||
|
"postgresql": {
|
||||||
|
"parameters": {
|
||||||
|
"max_connections": new_max_connections_value
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"patroni": {
|
||||||
|
"slots": {
|
||||||
|
"test_slot": {
|
||||||
|
"type": "physical"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"ttl": 29,
|
||||||
|
"loop_wait": 9,
|
||||||
|
"retry_timeout": 9,
|
||||||
|
"synchronous_mode": True
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
k8s.api.custom_objects_api.patch_namespaced_custom_object(
|
||||||
|
"acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_config)
|
||||||
|
|
||||||
|
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
|
||||||
|
|
||||||
|
def compare_config():
|
||||||
|
effective_config = k8s.patroni_rest(leader.metadata.name, "config")
|
||||||
|
desired_config = pg_patch_config["spec"]["patroni"]
|
||||||
|
desired_parameters = pg_patch_config["spec"]["postgresql"]["parameters"]
|
||||||
|
effective_parameters = effective_config["postgresql"]["parameters"]
|
||||||
|
self.assertEqual(desired_parameters["max_connections"], effective_parameters["max_connections"],
|
||||||
|
"max_connections not updated")
|
||||||
|
self.assertTrue(effective_config["slots"] is not None, "physical replication slot not added")
|
||||||
|
self.assertEqual(desired_config["ttl"], effective_config["ttl"],
|
||||||
|
"ttl not updated")
|
||||||
|
self.assertEqual(desired_config["loop_wait"], effective_config["loop_wait"],
|
||||||
|
"loop_wait not updated")
|
||||||
|
self.assertEqual(desired_config["retry_timeout"], effective_config["retry_timeout"],
|
||||||
|
"retry_timeout not updated")
|
||||||
|
self.assertEqual(desired_config["synchronous_mode"], effective_config["synchronous_mode"],
|
||||||
|
"synchronous_mode not updated")
|
||||||
|
return True
|
||||||
|
|
||||||
|
# check if Patroni config has been updated
|
||||||
|
self.eventuallyTrue(compare_config, "Postgres config not applied")
|
||||||
|
|
||||||
|
# make sure that pods were not recreated
|
||||||
|
leader = k8s.get_cluster_leader_pod()
|
||||||
|
replica = k8s.get_cluster_replica_pod()
|
||||||
|
self.assertEqual(masterCreationTimestamp, leader.metadata.creation_timestamp,
|
||||||
|
"Master pod creation timestamp is updated")
|
||||||
|
self.assertEqual(replicaCreationTimestamp, replica.metadata.creation_timestamp,
|
||||||
|
"Master pod creation timestamp is updated")
|
||||||
|
|
||||||
|
# query max_connections setting
|
||||||
|
setting_query = """
|
||||||
|
SELECT setting
|
||||||
|
FROM pg_settings
|
||||||
|
WHERE name = 'max_connections';
|
||||||
|
"""
|
||||||
|
self.eventuallyEqual(lambda: self.query_database(leader.metadata.name, "postgres", setting_query)[0], new_max_connections_value,
|
||||||
|
"New max_connections setting not applied on master", 10, 5)
|
||||||
|
self.eventuallyNotEqual(lambda: self.query_database(replica.metadata.name, "postgres", setting_query)[0], new_max_connections_value,
|
||||||
|
"Expected max_connections not to be updated on replica since Postgres was restarted there first", 10, 5)
|
||||||
|
|
||||||
|
# the next sync should restart the replica because it has pending_restart flag set
|
||||||
|
# force next sync by deleting the operator pod
|
||||||
|
k8s.delete_operator_pod()
|
||||||
|
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
|
||||||
|
|
||||||
|
self.eventuallyEqual(lambda: self.query_database(replica.metadata.name, "postgres", setting_query)[0], new_max_connections_value,
|
||||||
|
"New max_connections setting not applied on replica", 10, 5)
|
||||||
|
|
||||||
|
# decrease max_connections again
|
||||||
|
# this time restart will be correct and new value should appear on both instances
|
||||||
|
lower_max_connections_value = "30"
|
||||||
|
pg_patch_max_connections = {
|
||||||
|
"spec": {
|
||||||
|
"postgresql": {
|
||||||
|
"parameters": {
|
||||||
|
"max_connections": lower_max_connections_value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
k8s.api.custom_objects_api.patch_namespaced_custom_object(
|
||||||
|
"acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_max_connections)
|
||||||
|
|
||||||
|
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
|
||||||
|
|
||||||
|
# check Patroni config again
|
||||||
|
pg_patch_config["spec"]["postgresql"]["parameters"]["max_connections"] = lower_max_connections_value
|
||||||
|
self.eventuallyTrue(compare_config, "Postgres config not applied")
|
||||||
|
|
||||||
|
# and query max_connections setting again
|
||||||
|
self.eventuallyEqual(lambda: self.query_database(leader.metadata.name, "postgres", setting_query)[0], lower_max_connections_value,
|
||||||
|
"Previous max_connections setting not applied on master", 10, 5)
|
||||||
|
self.eventuallyEqual(lambda: self.query_database(replica.metadata.name, "postgres", setting_query)[0], lower_max_connections_value,
|
||||||
|
"Previous max_connections setting not applied on replica", 10, 5)
|
||||||
|
|
||||||
|
except timeout_decorator.TimeoutError:
|
||||||
|
print('Operator log: {}'.format(k8s.get_operator_log()))
|
||||||
|
raise
|
||||||
|
|
||||||
|
# make sure cluster is in a good state for further tests
|
||||||
|
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
|
||||||
|
self.eventuallyEqual(lambda: k8s.count_running_pods(), 2,
|
||||||
|
"No 2 pods running")
|
||||||
|
|
||||||
@timeout_decorator.timeout(TEST_TIMEOUT_SEC)
|
@timeout_decorator.timeout(TEST_TIMEOUT_SEC)
|
||||||
def test_cross_namespace_secrets(self):
|
def test_cross_namespace_secrets(self):
|
||||||
'''
|
'''
|
||||||
|
|
@ -968,15 +1095,15 @@ class EndToEndTestCase(unittest.TestCase):
|
||||||
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
|
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
|
||||||
|
|
||||||
# node affinity change should cause another rolling update and relocation of replica
|
# node affinity change should cause another rolling update and relocation of replica
|
||||||
k8s.wait_for_pod_failover(replica_nodes, 'spilo-role=replica,' + cluster_label)
|
|
||||||
k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label)
|
k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label)
|
||||||
|
k8s.wait_for_pod_start('spilo-role=master,' + cluster_label)
|
||||||
|
|
||||||
except timeout_decorator.TimeoutError:
|
except timeout_decorator.TimeoutError:
|
||||||
print('Operator log: {}'.format(k8s.get_operator_log()))
|
print('Operator log: {}'.format(k8s.get_operator_log()))
|
||||||
raise
|
raise
|
||||||
|
|
||||||
# toggle pod anti affinity to make sure replica and master run on separate nodes
|
# toggle pod anti affinity to make sure replica and master run on separate nodes
|
||||||
self.assert_distributed_pods(replica_nodes)
|
self.assert_distributed_pods(master_nodes)
|
||||||
|
|
||||||
@timeout_decorator.timeout(TEST_TIMEOUT_SEC)
|
@timeout_decorator.timeout(TEST_TIMEOUT_SEC)
|
||||||
def test_node_readiness_label(self):
|
def test_node_readiness_label(self):
|
||||||
|
|
@ -1192,133 +1319,6 @@ class EndToEndTestCase(unittest.TestCase):
|
||||||
self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "postgres", user_query)), 2,
|
self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "postgres", user_query)), 2,
|
||||||
"Found incorrect number of rotation users", 10, 5)
|
"Found incorrect number of rotation users", 10, 5)
|
||||||
|
|
||||||
@timeout_decorator.timeout(TEST_TIMEOUT_SEC)
|
|
||||||
def test_patroni_config_update(self):
|
|
||||||
'''
|
|
||||||
Change Postgres config under Spec.Postgresql.Parameters and Spec.Patroni
|
|
||||||
and query Patroni config endpoint to check if manifest changes got applied
|
|
||||||
via restarting cluster through Patroni's rest api
|
|
||||||
'''
|
|
||||||
k8s = self.k8s
|
|
||||||
leader = k8s.get_cluster_leader_pod()
|
|
||||||
replica = k8s.get_cluster_replica_pod()
|
|
||||||
masterCreationTimestamp = leader.metadata.creation_timestamp
|
|
||||||
replicaCreationTimestamp = replica.metadata.creation_timestamp
|
|
||||||
new_max_connections_value = "50"
|
|
||||||
|
|
||||||
# adjust Postgres config
|
|
||||||
pg_patch_config = {
|
|
||||||
"spec": {
|
|
||||||
"postgresql": {
|
|
||||||
"parameters": {
|
|
||||||
"max_connections": new_max_connections_value
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"patroni": {
|
|
||||||
"slots": {
|
|
||||||
"test_slot": {
|
|
||||||
"type": "physical"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"ttl": 29,
|
|
||||||
"loop_wait": 9,
|
|
||||||
"retry_timeout": 9,
|
|
||||||
"synchronous_mode": True
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
try:
|
|
||||||
k8s.api.custom_objects_api.patch_namespaced_custom_object(
|
|
||||||
"acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_config)
|
|
||||||
|
|
||||||
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
|
|
||||||
|
|
||||||
def compare_config():
|
|
||||||
effective_config = k8s.patroni_rest(leader.metadata.name, "config")
|
|
||||||
desired_config = pg_patch_config["spec"]["patroni"]
|
|
||||||
desired_parameters = pg_patch_config["spec"]["postgresql"]["parameters"]
|
|
||||||
effective_parameters = effective_config["postgresql"]["parameters"]
|
|
||||||
self.assertEqual(desired_parameters["max_connections"], effective_parameters["max_connections"],
|
|
||||||
"max_connections not updated")
|
|
||||||
self.assertTrue(effective_config["slots"] is not None, "physical replication slot not added")
|
|
||||||
self.assertEqual(desired_config["ttl"], effective_config["ttl"],
|
|
||||||
"ttl not updated")
|
|
||||||
self.assertEqual(desired_config["loop_wait"], effective_config["loop_wait"],
|
|
||||||
"loop_wait not updated")
|
|
||||||
self.assertEqual(desired_config["retry_timeout"], effective_config["retry_timeout"],
|
|
||||||
"retry_timeout not updated")
|
|
||||||
self.assertEqual(desired_config["synchronous_mode"], effective_config["synchronous_mode"],
|
|
||||||
"synchronous_mode not updated")
|
|
||||||
return True
|
|
||||||
|
|
||||||
# check if Patroni config has been updated
|
|
||||||
self.eventuallyTrue(compare_config, "Postgres config not applied")
|
|
||||||
|
|
||||||
# make sure that pods were not recreated
|
|
||||||
leader = k8s.get_cluster_leader_pod()
|
|
||||||
replica = k8s.get_cluster_replica_pod()
|
|
||||||
self.assertEqual(masterCreationTimestamp, leader.metadata.creation_timestamp,
|
|
||||||
"Master pod creation timestamp is updated")
|
|
||||||
self.assertEqual(replicaCreationTimestamp, replica.metadata.creation_timestamp,
|
|
||||||
"Master pod creation timestamp is updated")
|
|
||||||
|
|
||||||
# query max_connections setting
|
|
||||||
setting_query = """
|
|
||||||
SELECT setting
|
|
||||||
FROM pg_settings
|
|
||||||
WHERE name = 'max_connections';
|
|
||||||
"""
|
|
||||||
self.eventuallyEqual(lambda: self.query_database(leader.metadata.name, "postgres", setting_query)[0], new_max_connections_value,
|
|
||||||
"New max_connections setting not applied on master", 10, 5)
|
|
||||||
self.eventuallyNotEqual(lambda: self.query_database(replica.metadata.name, "postgres", setting_query)[0], new_max_connections_value,
|
|
||||||
"Expected max_connections not to be updated on replica since Postgres was restarted there first", 10, 5)
|
|
||||||
|
|
||||||
# the next sync should restart the replica because it has pending_restart flag set
|
|
||||||
# force next sync by deleting the operator pod
|
|
||||||
k8s.delete_operator_pod()
|
|
||||||
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
|
|
||||||
|
|
||||||
self.eventuallyEqual(lambda: self.query_database(replica.metadata.name, "postgres", setting_query)[0], new_max_connections_value,
|
|
||||||
"New max_connections setting not applied on replica", 10, 5)
|
|
||||||
|
|
||||||
# decrease max_connections again
|
|
||||||
# this time restart will be correct and new value should appear on both instances
|
|
||||||
lower_max_connections_value = "30"
|
|
||||||
pg_patch_max_connections = {
|
|
||||||
"spec": {
|
|
||||||
"postgresql": {
|
|
||||||
"parameters": {
|
|
||||||
"max_connections": lower_max_connections_value
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
k8s.api.custom_objects_api.patch_namespaced_custom_object(
|
|
||||||
"acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_max_connections)
|
|
||||||
|
|
||||||
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
|
|
||||||
|
|
||||||
# check Patroni config again
|
|
||||||
pg_patch_config["spec"]["postgresql"]["parameters"]["max_connections"] = lower_max_connections_value
|
|
||||||
self.eventuallyTrue(compare_config, "Postgres config not applied")
|
|
||||||
|
|
||||||
# and query max_connections setting again
|
|
||||||
self.eventuallyEqual(lambda: self.query_database(leader.metadata.name, "postgres", setting_query)[0], lower_max_connections_value,
|
|
||||||
"Previous max_connections setting not applied on master", 10, 5)
|
|
||||||
self.eventuallyEqual(lambda: self.query_database(replica.metadata.name, "postgres", setting_query)[0], lower_max_connections_value,
|
|
||||||
"Previous max_connections setting not applied on replica", 10, 5)
|
|
||||||
|
|
||||||
except timeout_decorator.TimeoutError:
|
|
||||||
print('Operator log: {}'.format(k8s.get_operator_log()))
|
|
||||||
raise
|
|
||||||
|
|
||||||
# make sure cluster is in a good state for further tests
|
|
||||||
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
|
|
||||||
self.eventuallyEqual(lambda: k8s.count_running_pods(), 2,
|
|
||||||
"No 2 pods running")
|
|
||||||
|
|
||||||
@timeout_decorator.timeout(TEST_TIMEOUT_SEC)
|
@timeout_decorator.timeout(TEST_TIMEOUT_SEC)
|
||||||
def test_rolling_update_flag(self):
|
def test_rolling_update_flag(self):
|
||||||
'''
|
'''
|
||||||
|
|
@ -1405,7 +1405,7 @@ class EndToEndTestCase(unittest.TestCase):
|
||||||
"data": {
|
"data": {
|
||||||
"pod_label_wait_timeout": "2s",
|
"pod_label_wait_timeout": "2s",
|
||||||
"resync_period": "30s",
|
"resync_period": "30s",
|
||||||
"repair_period": "10s",
|
"repair_period": "30s",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue