bring back all e2e tests
This commit is contained in:
Felix Kunde
parent
1d2b3f5291
commit
e1f5173daf
|
|
@ -158,6 +158,806 @@ class EndToEndTestCase(unittest.TestCase):
|
|||
print('Operator log: {}'.format(k8s.get_operator_log()))
|
||||
raise
|
||||
|
||||
@timeout_decorator.timeout(TEST_TIMEOUT_SEC)
|
||||
def test_additional_owner_roles(self):
|
||||
'''
|
||||
Test adding additional member roles to existing database owner roles
|
||||
'''
|
||||
k8s = self.k8s
|
||||
|
||||
# enable PostgresTeam CRD and lower resync
|
||||
owner_roles = {
|
||||
"data": {
|
||||
"additional_owner_roles": "cron_admin",
|
||||
},
|
||||
}
|
||||
k8s.update_config(owner_roles)
|
||||
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"},
|
||||
"Operator does not get in sync")
|
||||
|
||||
leader = k8s.get_cluster_leader_pod()
|
||||
owner_query = """
|
||||
SELECT a2.rolname
|
||||
FROM pg_catalog.pg_authid a
|
||||
JOIN pg_catalog.pg_auth_members am
|
||||
ON a.oid = am.member
|
||||
AND a.rolname = 'cron_admin'
|
||||
JOIN pg_catalog.pg_authid a2
|
||||
ON a2.oid = am.roleid
|
||||
WHERE a2.rolname IN ('zalando', 'bar_owner', 'bar_data_owner');
|
||||
"""
|
||||
self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "postgres", owner_query)), 3,
|
||||
"Not all additional users found in database", 10, 5)
|
||||
|
||||
@timeout_decorator.timeout(TEST_TIMEOUT_SEC)
|
||||
def test_additional_pod_capabilities(self):
|
||||
'''
|
||||
Extend postgres container capabilities
|
||||
'''
|
||||
k8s = self.k8s
|
||||
cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster'
|
||||
capabilities = ["SYS_NICE","CHOWN"]
|
||||
patch_capabilities = {
|
||||
"data": {
|
||||
"additional_pod_capabilities": ','.join(capabilities),
|
||||
},
|
||||
}
|
||||
|
||||
# get node and replica (expected target of new master)
|
||||
_, replica_nodes = k8s.get_pg_nodes(cluster_label)
|
||||
|
||||
try:
|
||||
k8s.update_config(patch_capabilities)
|
||||
|
||||
# changed security context of postgres container should trigger a rolling update
|
||||
k8s.wait_for_pod_failover(replica_nodes, 'spilo-role=master,' + cluster_label)
|
||||
k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label)
|
||||
|
||||
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
|
||||
self.eventuallyEqual(lambda: k8s.count_pods_with_container_capabilities(capabilities, cluster_label),
|
||||
2, "Container capabilities not updated")
|
||||
|
||||
except timeout_decorator.TimeoutError:
|
||||
print('Operator log: {}'.format(k8s.get_operator_log()))
|
||||
raise
|
||||
|
||||
@timeout_decorator.timeout(TEST_TIMEOUT_SEC)
|
||||
def test_additional_teams_and_members(self):
|
||||
'''
|
||||
Test PostgresTeam CRD with extra teams and members
|
||||
'''
|
||||
k8s = self.k8s
|
||||
|
||||
# enable PostgresTeam CRD and lower resync
|
||||
enable_postgres_team_crd = {
|
||||
"data": {
|
||||
"enable_postgres_team_crd": "true",
|
||||
"enable_team_member_deprecation": "true",
|
||||
"role_deletion_suffix": "_delete_me",
|
||||
"resync_period": "15s",
|
||||
"repair_period": "15s",
|
||||
},
|
||||
}
|
||||
k8s.update_config(enable_postgres_team_crd)
|
||||
|
||||
k8s.api.custom_objects_api.patch_namespaced_custom_object(
|
||||
'acid.zalan.do', 'v1', 'default',
|
||||
'postgresteams', 'custom-team-membership',
|
||||
{
|
||||
'spec': {
|
||||
'additionalTeams': {
|
||||
'acid': [
|
||||
'e2e'
|
||||
]
|
||||
},
|
||||
'additionalMembers': {
|
||||
'e2e': [
|
||||
'kind'
|
||||
]
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
leader = k8s.get_cluster_leader_pod()
|
||||
user_query = """
|
||||
SELECT rolname
|
||||
FROM pg_catalog.pg_roles
|
||||
WHERE rolname IN ('elephant', 'kind');
|
||||
"""
|
||||
self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "postgres", user_query)), 2,
|
||||
"Not all additional users found in database", 10, 5)
|
||||
|
||||
# replace additional member and check if the removed member's role is renamed
|
||||
k8s.api.custom_objects_api.patch_namespaced_custom_object(
|
||||
'acid.zalan.do', 'v1', 'default',
|
||||
'postgresteams', 'custom-team-membership',
|
||||
{
|
||||
'spec': {
|
||||
'additionalMembers': {
|
||||
'e2e': [
|
||||
'tester'
|
||||
]
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
user_query = """
|
||||
SELECT rolname
|
||||
FROM pg_catalog.pg_roles
|
||||
WHERE (rolname = 'tester' AND rolcanlogin)
|
||||
OR (rolname = 'kind_delete_me' AND NOT rolcanlogin);
|
||||
"""
|
||||
self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "postgres", user_query)), 2,
|
||||
"Database role of replaced member in PostgresTeam not renamed", 10, 5)
|
||||
|
||||
# re-add additional member and check if the role is renamed back
|
||||
k8s.api.custom_objects_api.patch_namespaced_custom_object(
|
||||
'acid.zalan.do', 'v1', 'default',
|
||||
'postgresteams', 'custom-team-membership',
|
||||
{
|
||||
'spec': {
|
||||
'additionalMembers': {
|
||||
'e2e': [
|
||||
'kind'
|
||||
]
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
user_query = """
|
||||
SELECT rolname
|
||||
FROM pg_catalog.pg_roles
|
||||
WHERE (rolname = 'kind' AND rolcanlogin)
|
||||
OR (rolname = 'tester_delete_me' AND NOT rolcanlogin);
|
||||
"""
|
||||
self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "postgres", user_query)), 2,
|
||||
"Database role of recreated member in PostgresTeam not renamed back to original name", 10, 5)
|
||||
|
||||
# revert config change
|
||||
revert_resync = {
|
||||
"data": {
|
||||
"resync_period": "4m",
|
||||
"repair_period": "1m",
|
||||
},
|
||||
}
|
||||
k8s.update_config(revert_resync)
|
||||
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"},
|
||||
"Operator does not get in sync")
|
||||
|
||||
@timeout_decorator.timeout(TEST_TIMEOUT_SEC)
|
||||
def test_config_update(self):
|
||||
'''
|
||||
Change Postgres config under Spec.Postgresql.Parameters and Spec.Patroni
|
||||
and query Patroni config endpoint to check if manifest changes got applied
|
||||
via restarting cluster through Patroni's rest api
|
||||
'''
|
||||
k8s = self.k8s
|
||||
leader = k8s.get_cluster_leader_pod()
|
||||
replica = k8s.get_cluster_replica_pod()
|
||||
masterCreationTimestamp = leader.metadata.creation_timestamp
|
||||
replicaCreationTimestamp = replica.metadata.creation_timestamp
|
||||
new_max_connections_value = "50"
|
||||
|
||||
# adjust Postgres config
|
||||
pg_patch_config = {
|
||||
"spec": {
|
||||
"postgresql": {
|
||||
"parameters": {
|
||||
"max_connections": new_max_connections_value
|
||||
}
|
||||
},
|
||||
"patroni": {
|
||||
"slots": {
|
||||
"test_slot": {
|
||||
"type": "physical"
|
||||
}
|
||||
},
|
||||
"ttl": 29,
|
||||
"loop_wait": 9,
|
||||
"retry_timeout": 9,
|
||||
"synchronous_mode": True
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try:
|
||||
k8s.api.custom_objects_api.patch_namespaced_custom_object(
|
||||
"acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_config)
|
||||
|
||||
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
|
||||
|
||||
def compare_config():
|
||||
effective_config = k8s.patroni_rest(leader.metadata.name, "config")
|
||||
desired_config = pg_patch_config["spec"]["patroni"]
|
||||
desired_parameters = pg_patch_config["spec"]["postgresql"]["parameters"]
|
||||
effective_parameters = effective_config["postgresql"]["parameters"]
|
||||
self.assertEqual(desired_parameters["max_connections"], effective_parameters["max_connections"],
|
||||
"max_connections not updated")
|
||||
self.assertTrue(effective_config["slots"] is not None, "physical replication slot not added")
|
||||
self.assertEqual(desired_config["ttl"], effective_config["ttl"],
|
||||
"ttl not updated")
|
||||
self.assertEqual(desired_config["loop_wait"], effective_config["loop_wait"],
|
||||
"loop_wait not updated")
|
||||
self.assertEqual(desired_config["retry_timeout"], effective_config["retry_timeout"],
|
||||
"retry_timeout not updated")
|
||||
self.assertEqual(desired_config["synchronous_mode"], effective_config["synchronous_mode"],
|
||||
"synchronous_mode not updated")
|
||||
return True
|
||||
|
||||
# check if Patroni config has been updated
|
||||
self.eventuallyTrue(compare_config, "Postgres config not applied")
|
||||
|
||||
# make sure that pods were not recreated
|
||||
leader = k8s.get_cluster_leader_pod()
|
||||
replica = k8s.get_cluster_replica_pod()
|
||||
self.assertEqual(masterCreationTimestamp, leader.metadata.creation_timestamp,
|
||||
"Master pod creation timestamp is updated")
|
||||
self.assertEqual(replicaCreationTimestamp, replica.metadata.creation_timestamp,
|
||||
"Master pod creation timestamp is updated")
|
||||
|
||||
# query max_connections setting
|
||||
setting_query = """
|
||||
SELECT setting
|
||||
FROM pg_settings
|
||||
WHERE name = 'max_connections';
|
||||
"""
|
||||
self.eventuallyEqual(lambda: self.query_database(leader.metadata.name, "postgres", setting_query)[0], new_max_connections_value,
|
||||
"New max_connections setting not applied on master", 10, 5)
|
||||
self.eventuallyNotEqual(lambda: self.query_database(replica.metadata.name, "postgres", setting_query)[0], new_max_connections_value,
|
||||
"Expected max_connections not to be updated on replica since Postgres was restarted there first", 10, 5)
|
||||
|
||||
# the next sync should restart the replica because it has pending_restart flag set
|
||||
# force next sync by deleting the operator pod
|
||||
k8s.delete_operator_pod()
|
||||
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
|
||||
|
||||
self.eventuallyEqual(lambda: self.query_database(replica.metadata.name, "postgres", setting_query)[0], new_max_connections_value,
|
||||
"New max_connections setting not applied on replica", 10, 5)
|
||||
|
||||
# decrease max_connections again
|
||||
# this time restart will be correct and new value should appear on both instances
|
||||
lower_max_connections_value = "30"
|
||||
pg_patch_max_connections = {
|
||||
"spec": {
|
||||
"postgresql": {
|
||||
"parameters": {
|
||||
"max_connections": lower_max_connections_value
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
k8s.api.custom_objects_api.patch_namespaced_custom_object(
|
||||
"acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_max_connections)
|
||||
|
||||
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
|
||||
|
||||
# check Patroni config again
|
||||
pg_patch_config["spec"]["postgresql"]["parameters"]["max_connections"] = lower_max_connections_value
|
||||
self.eventuallyTrue(compare_config, "Postgres config not applied")
|
||||
|
||||
# and query max_connections setting again
|
||||
self.eventuallyEqual(lambda: self.query_database(leader.metadata.name, "postgres", setting_query)[0], lower_max_connections_value,
|
||||
"Previous max_connections setting not applied on master", 10, 5)
|
||||
self.eventuallyEqual(lambda: self.query_database(replica.metadata.name, "postgres", setting_query)[0], lower_max_connections_value,
|
||||
"Previous max_connections setting not applied on replica", 10, 5)
|
||||
|
||||
except timeout_decorator.TimeoutError:
|
||||
print('Operator log: {}'.format(k8s.get_operator_log()))
|
||||
raise
|
||||
|
||||
# make sure cluster is in a good state for further tests
|
||||
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
|
||||
self.eventuallyEqual(lambda: k8s.count_running_pods(), 2,
|
||||
"No 2 pods running")
|
||||
|
||||
@timeout_decorator.timeout(TEST_TIMEOUT_SEC)
|
||||
def test_cross_namespace_secrets(self):
|
||||
'''
|
||||
Test secrets in different namespace
|
||||
'''
|
||||
k8s = self.k8s
|
||||
|
||||
# enable secret creation in separate namespace
|
||||
patch_cross_namespace_secret = {
|
||||
"data": {
|
||||
"enable_cross_namespace_secret": "true"
|
||||
}
|
||||
}
|
||||
k8s.update_config(patch_cross_namespace_secret,
|
||||
step="cross namespace secrets enabled")
|
||||
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"},
|
||||
"Operator does not get in sync")
|
||||
|
||||
# create secret in test namespace
|
||||
k8s.api.custom_objects_api.patch_namespaced_custom_object(
|
||||
'acid.zalan.do', 'v1', 'default',
|
||||
'postgresqls', 'acid-minimal-cluster',
|
||||
{
|
||||
'spec': {
|
||||
'users':{
|
||||
'test.db_user': [],
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"},
|
||||
"Operator does not get in sync")
|
||||
self.eventuallyEqual(lambda: k8s.count_secrets_with_label("cluster-name=acid-minimal-cluster,application=spilo", self.test_namespace),
|
||||
1, "Secret not created for user in namespace")
|
||||
|
||||
@timeout_decorator.timeout(TEST_TIMEOUT_SEC)
|
||||
def test_enable_disable_connection_pooler(self):
|
||||
'''
|
||||
For a database without connection pooler, then turns it on, scale up,
|
||||
turn off and on again. Test with different ways of doing this (via
|
||||
enableConnectionPooler or connectionPooler configuration section). At
|
||||
the end turn connection pooler off to not interfere with other tests.
|
||||
'''
|
||||
k8s = self.k8s
|
||||
pooler_label = 'application=db-connection-pooler,cluster-name=acid-minimal-cluster'
|
||||
master_pooler_label = 'connection-pooler=acid-minimal-cluster-pooler'
|
||||
replica_pooler_label = master_pooler_label + '-repl'
|
||||
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
|
||||
|
||||
k8s.api.custom_objects_api.patch_namespaced_custom_object(
|
||||
'acid.zalan.do', 'v1', 'default',
|
||||
'postgresqls', 'acid-minimal-cluster',
|
||||
{
|
||||
'spec': {
|
||||
'enableConnectionPooler': True,
|
||||
'enableReplicaConnectionPooler': True,
|
||||
}
|
||||
})
|
||||
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
|
||||
|
||||
self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(), 2, "Deployment replicas is 2 default")
|
||||
self.eventuallyEqual(lambda: k8s.count_running_pods(master_pooler_label), 2, "No pooler pods found")
|
||||
self.eventuallyEqual(lambda: k8s.count_running_pods(replica_pooler_label), 2, "No pooler replica pods found")
|
||||
self.eventuallyEqual(lambda: k8s.count_services_with_label(pooler_label), 2, "No pooler service found")
|
||||
self.eventuallyEqual(lambda: k8s.count_secrets_with_label(pooler_label), 1, "Pooler secret not created")
|
||||
|
||||
k8s.api.custom_objects_api.patch_namespaced_custom_object(
|
||||
'acid.zalan.do', 'v1', 'default',
|
||||
'postgresqls', 'acid-minimal-cluster',
|
||||
{
|
||||
'spec': {
|
||||
'enableMasterPoolerLoadBalancer': True,
|
||||
'enableReplicaPoolerLoadBalancer': True,
|
||||
}
|
||||
})
|
||||
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
|
||||
self.eventuallyEqual(lambda: k8s.get_service_type(master_pooler_label+","+pooler_label),
|
||||
'LoadBalancer',
|
||||
"Expected LoadBalancer service type for master pooler pod, found {}")
|
||||
self.eventuallyEqual(lambda: k8s.get_service_type(replica_pooler_label+","+pooler_label),
|
||||
'LoadBalancer',
|
||||
"Expected LoadBalancer service type for replica pooler pod, found {}")
|
||||
|
||||
# Turn off only master connection pooler
|
||||
k8s.api.custom_objects_api.patch_namespaced_custom_object(
|
||||
'acid.zalan.do', 'v1', 'default',
|
||||
'postgresqls', 'acid-minimal-cluster',
|
||||
{
|
||||
'spec': {
|
||||
'enableConnectionPooler': False,
|
||||
'enableReplicaConnectionPooler': True,
|
||||
}
|
||||
})
|
||||
|
||||
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
|
||||
|
||||
self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(name="acid-minimal-cluster-pooler-repl"), 2,
|
||||
"Deployment replicas is 2 default")
|
||||
self.eventuallyEqual(lambda: k8s.count_running_pods(master_pooler_label),
|
||||
0, "Master pooler pods not deleted")
|
||||
self.eventuallyEqual(lambda: k8s.count_running_pods(replica_pooler_label),
|
||||
2, "Pooler replica pods not found")
|
||||
self.eventuallyEqual(lambda: k8s.count_services_with_label(pooler_label),
|
||||
1, "No pooler service found")
|
||||
self.eventuallyEqual(lambda: k8s.count_secrets_with_label(pooler_label),
|
||||
1, "Secret not created")
|
||||
|
||||
# Turn off only replica connection pooler
|
||||
k8s.api.custom_objects_api.patch_namespaced_custom_object(
|
||||
'acid.zalan.do', 'v1', 'default',
|
||||
'postgresqls', 'acid-minimal-cluster',
|
||||
{
|
||||
'spec': {
|
||||
'enableConnectionPooler': True,
|
||||
'enableReplicaConnectionPooler': False,
|
||||
'enableMasterPoolerLoadBalancer': False,
|
||||
}
|
||||
})
|
||||
|
||||
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
|
||||
|
||||
self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(), 2,
|
||||
"Deployment replicas is 2 default")
|
||||
self.eventuallyEqual(lambda: k8s.count_running_pods(master_pooler_label),
|
||||
2, "Master pooler pods not found")
|
||||
self.eventuallyEqual(lambda: k8s.count_running_pods(replica_pooler_label),
|
||||
0, "Pooler replica pods not deleted")
|
||||
self.eventuallyEqual(lambda: k8s.count_services_with_label(pooler_label),
|
||||
1, "No pooler service found")
|
||||
self.eventuallyEqual(lambda: k8s.get_service_type(master_pooler_label+","+pooler_label),
|
||||
'ClusterIP',
|
||||
"Expected LoadBalancer service type for master, found {}")
|
||||
self.eventuallyEqual(lambda: k8s.count_secrets_with_label(pooler_label),
|
||||
1, "Secret not created")
|
||||
|
||||
# scale up connection pooler deployment
|
||||
k8s.api.custom_objects_api.patch_namespaced_custom_object(
|
||||
'acid.zalan.do', 'v1', 'default',
|
||||
'postgresqls', 'acid-minimal-cluster',
|
||||
{
|
||||
'spec': {
|
||||
'connectionPooler': {
|
||||
'numberOfInstances': 3,
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(), 3,
|
||||
"Deployment replicas is scaled to 3")
|
||||
self.eventuallyEqual(lambda: k8s.count_running_pods(master_pooler_label),
|
||||
3, "Scale up of pooler pods does not work")
|
||||
|
||||
# turn it off, keeping config should be overwritten by false
|
||||
k8s.api.custom_objects_api.patch_namespaced_custom_object(
|
||||
'acid.zalan.do', 'v1', 'default',
|
||||
'postgresqls', 'acid-minimal-cluster',
|
||||
{
|
||||
'spec': {
|
||||
'enableConnectionPooler': False,
|
||||
'enableReplicaConnectionPooler': False,
|
||||
'enableReplicaPoolerLoadBalancer': False,
|
||||
}
|
||||
})
|
||||
|
||||
self.eventuallyEqual(lambda: k8s.count_running_pods(master_pooler_label),
|
||||
0, "Pooler pods not scaled down")
|
||||
self.eventuallyEqual(lambda: k8s.count_services_with_label(pooler_label),
|
||||
0, "Pooler service not removed")
|
||||
self.eventuallyEqual(lambda: k8s.count_secrets_with_label('application=spilo,cluster-name=acid-minimal-cluster'),
|
||||
4, "Secrets not deleted")
|
||||
|
||||
# Verify that all the databases have pooler schema installed.
|
||||
# Do this via psql, since otherwise we need to deal with
|
||||
# credentials.
|
||||
db_list = []
|
||||
|
||||
leader = k8s.get_cluster_leader_pod()
|
||||
schemas_query = """
|
||||
SELECT schema_name
|
||||
FROM information_schema.schemata
|
||||
WHERE schema_name = 'pooler'
|
||||
"""
|
||||
|
||||
db_list = self.list_databases(leader.metadata.name)
|
||||
for db in db_list:
|
||||
self.eventuallyNotEqual(lambda: len(self.query_database(leader.metadata.name, db, schemas_query)), 0,
|
||||
"Pooler schema not found in database {}".format(db))
|
||||
|
||||
# remove config section to make test work next time
|
||||
k8s.api.custom_objects_api.patch_namespaced_custom_object(
|
||||
'acid.zalan.do', 'v1', 'default',
|
||||
'postgresqls', 'acid-minimal-cluster',
|
||||
{
|
||||
'spec': {
|
||||
'connectionPooler': None,
|
||||
'EnableReplicaConnectionPooler': False,
|
||||
}
|
||||
})
|
||||
|
||||
@timeout_decorator.timeout(TEST_TIMEOUT_SEC)
|
||||
def test_enable_load_balancer(self):
|
||||
'''
|
||||
Test if services are updated when enabling/disabling load balancers in Postgres manifest
|
||||
'''
|
||||
|
||||
k8s = self.k8s
|
||||
cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster,spilo-role={}'
|
||||
|
||||
self.eventuallyEqual(lambda: k8s.get_service_type(cluster_label.format("master")),
|
||||
'ClusterIP',
|
||||
"Expected ClusterIP type initially, found {}")
|
||||
|
||||
try:
|
||||
# enable load balancer services
|
||||
pg_patch_enable_lbs = {
|
||||
"spec": {
|
||||
"enableMasterLoadBalancer": True,
|
||||
"enableReplicaLoadBalancer": True
|
||||
}
|
||||
}
|
||||
k8s.api.custom_objects_api.patch_namespaced_custom_object(
|
||||
"acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_lbs)
|
||||
|
||||
self.eventuallyEqual(lambda: k8s.get_service_type(cluster_label.format("master")),
|
||||
'LoadBalancer',
|
||||
"Expected LoadBalancer service type for master, found {}")
|
||||
|
||||
self.eventuallyEqual(lambda: k8s.get_service_type(cluster_label.format("replica")),
|
||||
'LoadBalancer',
|
||||
"Expected LoadBalancer service type for master, found {}")
|
||||
|
||||
# disable load balancer services again
|
||||
pg_patch_disable_lbs = {
|
||||
"spec": {
|
||||
"enableMasterLoadBalancer": False,
|
||||
"enableReplicaLoadBalancer": False
|
||||
}
|
||||
}
|
||||
k8s.api.custom_objects_api.patch_namespaced_custom_object(
|
||||
"acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_lbs)
|
||||
|
||||
self.eventuallyEqual(lambda: k8s.get_service_type(cluster_label.format("master")),
|
||||
'ClusterIP',
|
||||
"Expected LoadBalancer service type for master, found {}")
|
||||
|
||||
self.eventuallyEqual(lambda: k8s.get_service_type(cluster_label.format("replica")),
|
||||
'ClusterIP',
|
||||
"Expected LoadBalancer service type for master, found {}")
|
||||
|
||||
except timeout_decorator.TimeoutError:
|
||||
print('Operator log: {}'.format(k8s.get_operator_log()))
|
||||
raise
|
||||
|
||||
@timeout_decorator.timeout(TEST_TIMEOUT_SEC)
|
||||
def test_infrastructure_roles(self):
|
||||
'''
|
||||
Test using external secrets for infrastructure roles
|
||||
'''
|
||||
k8s = self.k8s
|
||||
# update infrastructure roles description
|
||||
secret_name = "postgresql-infrastructure-roles"
|
||||
roles = "secretname: postgresql-infrastructure-roles-new, userkey: user,"\
|
||||
"rolekey: memberof, passwordkey: password, defaultrolevalue: robot_zmon"
|
||||
patch_infrastructure_roles = {
|
||||
"data": {
|
||||
"infrastructure_roles_secret_name": secret_name,
|
||||
"infrastructure_roles_secrets": roles,
|
||||
},
|
||||
}
|
||||
k8s.update_config(patch_infrastructure_roles)
|
||||
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"},
|
||||
"Operator does not get in sync")
|
||||
|
||||
try:
|
||||
# check that new roles are represented in the config by requesting the
|
||||
# operator configuration via API
|
||||
|
||||
def verify_role():
|
||||
try:
|
||||
operator_pod = k8s.get_operator_pod()
|
||||
get_config_cmd = "wget --quiet -O - localhost:8080/config"
|
||||
result = k8s.exec_with_kubectl(operator_pod.metadata.name,
|
||||
get_config_cmd)
|
||||
try:
|
||||
roles_dict = (json.loads(result.stdout)
|
||||
.get("controller", {})
|
||||
.get("InfrastructureRoles"))
|
||||
except:
|
||||
return False
|
||||
|
||||
if "robot_zmon_acid_monitoring_new" in roles_dict:
|
||||
role = roles_dict["robot_zmon_acid_monitoring_new"]
|
||||
role.pop("Password", None)
|
||||
self.assertDictEqual(role, {
|
||||
"Name": "robot_zmon_acid_monitoring_new",
|
||||
"Namespace":"",
|
||||
"Flags": None,
|
||||
"MemberOf": ["robot_zmon"],
|
||||
"Parameters": None,
|
||||
"AdminRole": "",
|
||||
"Origin": 2,
|
||||
"IsDbOwner": False,
|
||||
"Deleted": False
|
||||
})
|
||||
return True
|
||||
except:
|
||||
pass
|
||||
|
||||
return False
|
||||
|
||||
self.eventuallyTrue(verify_role, "infrastructure role setup is not loaded")
|
||||
|
||||
except timeout_decorator.TimeoutError:
|
||||
print('Operator log: {}'.format(k8s.get_operator_log()))
|
||||
raise
|
||||
|
||||
@timeout_decorator.timeout(TEST_TIMEOUT_SEC)
|
||||
def test_lazy_spilo_upgrade(self):
|
||||
'''
|
||||
Test lazy upgrade for the Spilo image: operator changes a stateful set
|
||||
but lets pods run with the old image until they are recreated for
|
||||
reasons other than operator's activity. That works because the operator
|
||||
configures stateful sets to use "onDelete" pod update policy.
|
||||
The test covers:
|
||||
1) enabling lazy upgrade in existing operator deployment
|
||||
2) forcing the normal rolling upgrade by changing the operator
|
||||
configmap and restarting its pod
|
||||
'''
|
||||
|
||||
k8s = self.k8s
|
||||
|
||||
pod0 = 'acid-minimal-cluster-0'
|
||||
pod1 = 'acid-minimal-cluster-1'
|
||||
|
||||
self.eventuallyEqual(lambda: k8s.count_running_pods(), 2,
|
||||
"No 2 pods running")
|
||||
self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members(pod0)),
|
||||
2, "Postgres status did not enter running")
|
||||
|
||||
patch_lazy_spilo_upgrade = {
|
||||
"data": {
|
||||
"docker_image": SPILO_CURRENT,
|
||||
"enable_lazy_spilo_upgrade": "false"
|
||||
}
|
||||
}
|
||||
k8s.update_config(patch_lazy_spilo_upgrade,
|
||||
step="Init baseline image version")
|
||||
|
||||
self.eventuallyEqual(lambda: k8s.get_statefulset_image(), SPILO_CURRENT,
|
||||
"Statefulset not updated initially")
|
||||
self.eventuallyEqual(lambda: k8s.count_running_pods(), 2,
|
||||
"No 2 pods running")
|
||||
self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members(pod0)),
|
||||
2, "Postgres status did not enter running")
|
||||
|
||||
self.eventuallyEqual(lambda: k8s.get_effective_pod_image(pod0),
|
||||
SPILO_CURRENT, "Rolling upgrade was not executed")
|
||||
self.eventuallyEqual(lambda: k8s.get_effective_pod_image(pod1),
|
||||
SPILO_CURRENT, "Rolling upgrade was not executed")
|
||||
|
||||
# update docker image in config and enable the lazy upgrade
|
||||
conf_image = SPILO_LAZY
|
||||
patch_lazy_spilo_upgrade = {
|
||||
"data": {
|
||||
"docker_image": conf_image,
|
||||
"enable_lazy_spilo_upgrade": "true"
|
||||
}
|
||||
}
|
||||
k8s.update_config(patch_lazy_spilo_upgrade,
|
||||
step="patch image and lazy upgrade")
|
||||
self.eventuallyEqual(lambda: k8s.get_statefulset_image(), conf_image,
|
||||
"Statefulset not updated to next Docker image")
|
||||
|
||||
try:
|
||||
# restart the pod to get a container with the new image
|
||||
k8s.api.core_v1.delete_namespaced_pod(pod0, 'default')
|
||||
|
||||
# verify only pod-0 which was deleted got new image from statefulset
|
||||
self.eventuallyEqual(lambda: k8s.get_effective_pod_image(pod0),
|
||||
conf_image, "Delete pod-0 did not get new spilo image")
|
||||
self.eventuallyEqual(lambda: k8s.count_running_pods(), 2,
|
||||
"No two pods running after lazy rolling upgrade")
|
||||
self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members(pod0)),
|
||||
2, "Postgres status did not enter running")
|
||||
self.assertNotEqual(lambda: k8s.get_effective_pod_image(pod1),
|
||||
SPILO_CURRENT,
|
||||
"pod-1 should not have change Docker image to {}".format(SPILO_CURRENT))
|
||||
|
||||
# clean up
|
||||
unpatch_lazy_spilo_upgrade = {
|
||||
"data": {
|
||||
"enable_lazy_spilo_upgrade": "false",
|
||||
}
|
||||
}
|
||||
k8s.update_config(unpatch_lazy_spilo_upgrade, step="patch lazy upgrade")
|
||||
|
||||
# at this point operator will complete the normal rolling upgrade
|
||||
# so we additionally test if disabling the lazy upgrade - forcing the normal rolling upgrade - works
|
||||
self.eventuallyEqual(lambda: k8s.get_effective_pod_image(pod0),
|
||||
conf_image, "Rolling upgrade was not executed",
|
||||
50, 3)
|
||||
self.eventuallyEqual(lambda: k8s.get_effective_pod_image(pod1),
|
||||
conf_image, "Rolling upgrade was not executed",
|
||||
50, 3)
|
||||
self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members(pod0)),
|
||||
2, "Postgres status did not enter running")
|
||||
|
||||
except timeout_decorator.TimeoutError:
|
||||
print('Operator log: {}'.format(k8s.get_operator_log()))
|
||||
raise
|
||||
|
||||
@timeout_decorator.timeout(TEST_TIMEOUT_SEC)
|
||||
def test_logical_backup_cron_job(self):
|
||||
'''
|
||||
Ensure we can (a) create the cron job at user request for a specific PG cluster
|
||||
(b) update the cluster-wide image for the logical backup pod
|
||||
(c) delete the job at user request
|
||||
Limitations:
|
||||
(a) Does not run the actual batch job because there is no S3 mock to upload backups to
|
||||
(b) Assumes 'acid-minimal-cluster' exists as defined in setUp
|
||||
'''
|
||||
|
||||
k8s = self.k8s
|
||||
|
||||
# create the cron job
|
||||
schedule = "7 7 7 7 *"
|
||||
pg_patch_enable_backup = {
|
||||
"spec": {
|
||||
"enableLogicalBackup": True,
|
||||
"logicalBackupSchedule": schedule
|
||||
}
|
||||
}
|
||||
k8s.api.custom_objects_api.patch_namespaced_custom_object(
|
||||
"acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_backup)
|
||||
|
||||
try:
|
||||
self.eventuallyEqual(lambda: len(k8s.get_logical_backup_job().items), 1, "failed to create logical backup job")
|
||||
|
||||
job = k8s.get_logical_backup_job().items[0]
|
||||
self.assertEqual(job.metadata.name, "logical-backup-acid-minimal-cluster",
|
||||
"Expected job name {}, found {}"
|
||||
.format("logical-backup-acid-minimal-cluster", job.metadata.name))
|
||||
self.assertEqual(job.spec.schedule, schedule,
|
||||
"Expected {} schedule, found {}"
|
||||
.format(schedule, job.spec.schedule))
|
||||
|
||||
# update the cluster-wide image of the logical backup pod
|
||||
image = "test-image-name"
|
||||
patch_logical_backup_image = {
|
||||
"data": {
|
||||
"logical_backup_docker_image": image,
|
||||
}
|
||||
}
|
||||
k8s.update_config(patch_logical_backup_image, step="patch logical backup image")
|
||||
|
||||
def get_docker_image():
|
||||
jobs = k8s.get_logical_backup_job().items
|
||||
return jobs[0].spec.job_template.spec.template.spec.containers[0].image
|
||||
|
||||
self.eventuallyEqual(get_docker_image, image,
|
||||
"Expected job image {}, found {}".format(image, "{}"))
|
||||
|
||||
# delete the logical backup cron job
|
||||
pg_patch_disable_backup = {
|
||||
"spec": {
|
||||
"enableLogicalBackup": False,
|
||||
}
|
||||
}
|
||||
k8s.api.custom_objects_api.patch_namespaced_custom_object(
|
||||
"acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_backup)
|
||||
|
||||
self.eventuallyEqual(lambda: len(k8s.get_logical_backup_job().items), 0, "failed to create logical backup job")
|
||||
|
||||
except timeout_decorator.TimeoutError:
|
||||
print('Operator log: {}'.format(k8s.get_operator_log()))
|
||||
raise
|
||||
|
||||
# ensure cluster is healthy after tests
|
||||
self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members("acid-minimal-cluster-0")), 2, "Postgres status did not enter running")
|
||||
|
||||
@timeout_decorator.timeout(TEST_TIMEOUT_SEC)
|
||||
@unittest.skip("Skipping this test until fixed")
|
||||
def test_major_version_upgrade(self):
|
||||
k8s = self.k8s
|
||||
result = k8s.create_with_kubectl("manifests/minimal-postgres-manifest-12.yaml")
|
||||
self.eventuallyEqual(lambda: k8s.count_running_pods(labels="application=spilo,cluster-name=acid-upgrade-test"), 2, "No 2 pods running")
|
||||
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
|
||||
|
||||
pg_patch_version = {
|
||||
"spec": {
|
||||
"postgres": {
|
||||
"version": "14"
|
||||
}
|
||||
}
|
||||
}
|
||||
k8s.api.custom_objects_api.patch_namespaced_custom_object(
|
||||
"acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version)
|
||||
|
||||
self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
|
||||
|
||||
def check_version_14():
|
||||
p = k8s.get_patroni_state("acid-upgrade-test-0")
|
||||
version = p["server_version"][0:2]
|
||||
return version
|
||||
|
||||
self.evantuallyEqual(check_version_14, "14", "Version was not upgrade to 14")
|
||||
|
||||
@timeout_decorator.timeout(TEST_TIMEOUT_SEC)
|
||||
def test_min_resource_limits(self):
|
||||
'''
|
||||
|
|
|
|||
Loading…
Reference in New Issue