From 1d2b3f52917f450d614fa5af1052c71a7a6e539f Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Sat, 19 Mar 2022 17:30:58 +0100 Subject: [PATCH] revert change for node affinity test --- e2e/tests/test_e2e.py | 802 +----------------------------------------- 1 file changed, 2 insertions(+), 800 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 88cb6792d..a346e09ad 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -158,806 +158,6 @@ class EndToEndTestCase(unittest.TestCase): print('Operator log: {}'.format(k8s.get_operator_log())) raise - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_additional_owner_roles(self): - ''' - Test adding additional member roles to existing database owner roles - ''' - k8s = self.k8s - - # enable PostgresTeam CRD and lower resync - owner_roles = { - "data": { - "additional_owner_roles": "cron_admin", - }, - } - k8s.update_config(owner_roles) - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, - "Operator does not get in sync") - - leader = k8s.get_cluster_leader_pod() - owner_query = """ - SELECT a2.rolname - FROM pg_catalog.pg_authid a - JOIN pg_catalog.pg_auth_members am - ON a.oid = am.member - AND a.rolname = 'cron_admin' - JOIN pg_catalog.pg_authid a2 - ON a2.oid = am.roleid - WHERE a2.rolname IN ('zalando', 'bar_owner', 'bar_data_owner'); - """ - self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "postgres", owner_query)), 3, - "Not all additional users found in database", 10, 5) - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_additional_pod_capabilities(self): - ''' - Extend postgres container capabilities - ''' - k8s = self.k8s - cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' - capabilities = ["SYS_NICE","CHOWN"] - patch_capabilities = { - "data": { - "additional_pod_capabilities": ','.join(capabilities), - }, - } - - # get node and replica (expected target of new master) - _, replica_nodes = k8s.get_pg_nodes(cluster_label) - - try: - k8s.update_config(patch_capabilities) - - # changed security context of postgres container should trigger a rolling update - k8s.wait_for_pod_failover(replica_nodes, 'spilo-role=master,' + cluster_label) - k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) - - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - self.eventuallyEqual(lambda: k8s.count_pods_with_container_capabilities(capabilities, cluster_label), - 2, "Container capabilities not updated") - - except timeout_decorator.TimeoutError: - print('Operator log: {}'.format(k8s.get_operator_log())) - raise - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_additional_teams_and_members(self): - ''' - Test PostgresTeam CRD with extra teams and members - ''' - k8s = self.k8s - - # enable PostgresTeam CRD and lower resync - enable_postgres_team_crd = { - "data": { - "enable_postgres_team_crd": "true", - "enable_team_member_deprecation": "true", - "role_deletion_suffix": "_delete_me", - "resync_period": "15s", - "repair_period": "15s", - }, - } - k8s.update_config(enable_postgres_team_crd) - - k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresteams', 'custom-team-membership', - { - 'spec': { - 'additionalTeams': { - 'acid': [ - 'e2e' - ] - }, - 'additionalMembers': { - 'e2e': [ - 'kind' - ] - } - } - }) - - leader = k8s.get_cluster_leader_pod() - user_query = """ - SELECT rolname - FROM pg_catalog.pg_roles - WHERE rolname IN ('elephant', 'kind'); - """ - self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "postgres", user_query)), 2, - "Not all additional users found in database", 10, 5) - - # replace additional member and check if the removed member's role is renamed - k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresteams', 'custom-team-membership', - { - 'spec': { - 'additionalMembers': { - 'e2e': [ - 'tester' - ] - }, - } - }) - - user_query = """ - SELECT rolname - FROM pg_catalog.pg_roles - WHERE (rolname = 'tester' AND rolcanlogin) - OR (rolname = 'kind_delete_me' AND NOT rolcanlogin); - """ - self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "postgres", user_query)), 2, - "Database role of replaced member in PostgresTeam not renamed", 10, 5) - - # re-add additional member and check if the role is renamed back - k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresteams', 'custom-team-membership', - { - 'spec': { - 'additionalMembers': { - 'e2e': [ - 'kind' - ] - }, - } - }) - - user_query = """ - SELECT rolname - FROM pg_catalog.pg_roles - WHERE (rolname = 'kind' AND rolcanlogin) - OR (rolname = 'tester_delete_me' AND NOT rolcanlogin); - """ - self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "postgres", user_query)), 2, - "Database role of recreated member in PostgresTeam not renamed back to original name", 10, 5) - - # revert config change - revert_resync = { - "data": { - "resync_period": "4m", - "repair_period": "1m", - }, - } - k8s.update_config(revert_resync) - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, - "Operator does not get in sync") - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_config_update(self): - ''' - Change Postgres config under Spec.Postgresql.Parameters and Spec.Patroni - and query Patroni config endpoint to check if manifest changes got applied - via restarting cluster through Patroni's rest api - ''' - k8s = self.k8s - leader = k8s.get_cluster_leader_pod() - replica = k8s.get_cluster_replica_pod() - masterCreationTimestamp = leader.metadata.creation_timestamp - replicaCreationTimestamp = replica.metadata.creation_timestamp - new_max_connections_value = "50" - - # adjust Postgres config - pg_patch_config = { - "spec": { - "postgresql": { - "parameters": { - "max_connections": new_max_connections_value - } - }, - "patroni": { - "slots": { - "test_slot": { - "type": "physical" - } - }, - "ttl": 29, - "loop_wait": 9, - "retry_timeout": 9, - "synchronous_mode": True - } - } - } - - try: - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_config) - - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - - def compare_config(): - effective_config = k8s.patroni_rest(leader.metadata.name, "config") - desired_config = pg_patch_config["spec"]["patroni"] - desired_parameters = pg_patch_config["spec"]["postgresql"]["parameters"] - effective_parameters = effective_config["postgresql"]["parameters"] - self.assertEqual(desired_parameters["max_connections"], effective_parameters["max_connections"], - "max_connections not updated") - self.assertTrue(effective_config["slots"] is not None, "physical replication slot not added") - self.assertEqual(desired_config["ttl"], effective_config["ttl"], - "ttl not updated") - self.assertEqual(desired_config["loop_wait"], effective_config["loop_wait"], - "loop_wait not updated") - self.assertEqual(desired_config["retry_timeout"], effective_config["retry_timeout"], - "retry_timeout not updated") - self.assertEqual(desired_config["synchronous_mode"], effective_config["synchronous_mode"], - "synchronous_mode not updated") - return True - - # check if Patroni config has been updated - self.eventuallyTrue(compare_config, "Postgres config not applied") - - # make sure that pods were not recreated - leader = k8s.get_cluster_leader_pod() - replica = k8s.get_cluster_replica_pod() - self.assertEqual(masterCreationTimestamp, leader.metadata.creation_timestamp, - "Master pod creation timestamp is updated") - self.assertEqual(replicaCreationTimestamp, replica.metadata.creation_timestamp, - "Master pod creation timestamp is updated") - - # query max_connections setting - setting_query = """ - SELECT setting - FROM pg_settings - WHERE name = 'max_connections'; - """ - self.eventuallyEqual(lambda: self.query_database(leader.metadata.name, "postgres", setting_query)[0], new_max_connections_value, - "New max_connections setting not applied on master", 10, 5) - self.eventuallyNotEqual(lambda: self.query_database(replica.metadata.name, "postgres", setting_query)[0], new_max_connections_value, - "Expected max_connections not to be updated on replica since Postgres was restarted there first", 10, 5) - - # the next sync should restart the replica because it has pending_restart flag set - # force next sync by deleting the operator pod - k8s.delete_operator_pod() - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - - self.eventuallyEqual(lambda: self.query_database(replica.metadata.name, "postgres", setting_query)[0], new_max_connections_value, - "New max_connections setting not applied on replica", 10, 5) - - # decrease max_connections again - # this time restart will be correct and new value should appear on both instances - lower_max_connections_value = "30" - pg_patch_max_connections = { - "spec": { - "postgresql": { - "parameters": { - "max_connections": lower_max_connections_value - } - } - } - } - - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_max_connections) - - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - - # check Patroni config again - pg_patch_config["spec"]["postgresql"]["parameters"]["max_connections"] = lower_max_connections_value - self.eventuallyTrue(compare_config, "Postgres config not applied") - - # and query max_connections setting again - self.eventuallyEqual(lambda: self.query_database(leader.metadata.name, "postgres", setting_query)[0], lower_max_connections_value, - "Previous max_connections setting not applied on master", 10, 5) - self.eventuallyEqual(lambda: self.query_database(replica.metadata.name, "postgres", setting_query)[0], lower_max_connections_value, - "Previous max_connections setting not applied on replica", 10, 5) - - except timeout_decorator.TimeoutError: - print('Operator log: {}'.format(k8s.get_operator_log())) - raise - - # make sure cluster is in a good state for further tests - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, - "No 2 pods running") - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_cross_namespace_secrets(self): - ''' - Test secrets in different namespace - ''' - k8s = self.k8s - - # enable secret creation in separate namespace - patch_cross_namespace_secret = { - "data": { - "enable_cross_namespace_secret": "true" - } - } - k8s.update_config(patch_cross_namespace_secret, - step="cross namespace secrets enabled") - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, - "Operator does not get in sync") - - # create secret in test namespace - k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresqls', 'acid-minimal-cluster', - { - 'spec': { - 'users':{ - 'test.db_user': [], - } - } - }) - - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, - "Operator does not get in sync") - self.eventuallyEqual(lambda: k8s.count_secrets_with_label("cluster-name=acid-minimal-cluster,application=spilo", self.test_namespace), - 1, "Secret not created for user in namespace") - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_enable_disable_connection_pooler(self): - ''' - For a database without connection pooler, then turns it on, scale up, - turn off and on again. Test with different ways of doing this (via - enableConnectionPooler or connectionPooler configuration section). At - the end turn connection pooler off to not interfere with other tests. - ''' - k8s = self.k8s - pooler_label = 'application=db-connection-pooler,cluster-name=acid-minimal-cluster' - master_pooler_label = 'connection-pooler=acid-minimal-cluster-pooler' - replica_pooler_label = master_pooler_label + '-repl' - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - - k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresqls', 'acid-minimal-cluster', - { - 'spec': { - 'enableConnectionPooler': True, - 'enableReplicaConnectionPooler': True, - } - }) - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - - self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(), 2, "Deployment replicas is 2 default") - self.eventuallyEqual(lambda: k8s.count_running_pods(master_pooler_label), 2, "No pooler pods found") - self.eventuallyEqual(lambda: k8s.count_running_pods(replica_pooler_label), 2, "No pooler replica pods found") - self.eventuallyEqual(lambda: k8s.count_services_with_label(pooler_label), 2, "No pooler service found") - self.eventuallyEqual(lambda: k8s.count_secrets_with_label(pooler_label), 1, "Pooler secret not created") - - k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresqls', 'acid-minimal-cluster', - { - 'spec': { - 'enableMasterPoolerLoadBalancer': True, - 'enableReplicaPoolerLoadBalancer': True, - } - }) - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - self.eventuallyEqual(lambda: k8s.get_service_type(master_pooler_label+","+pooler_label), - 'LoadBalancer', - "Expected LoadBalancer service type for master pooler pod, found {}") - self.eventuallyEqual(lambda: k8s.get_service_type(replica_pooler_label+","+pooler_label), - 'LoadBalancer', - "Expected LoadBalancer service type for replica pooler pod, found {}") - - # Turn off only master connection pooler - k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresqls', 'acid-minimal-cluster', - { - 'spec': { - 'enableConnectionPooler': False, - 'enableReplicaConnectionPooler': True, - } - }) - - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - - self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(name="acid-minimal-cluster-pooler-repl"), 2, - "Deployment replicas is 2 default") - self.eventuallyEqual(lambda: k8s.count_running_pods(master_pooler_label), - 0, "Master pooler pods not deleted") - self.eventuallyEqual(lambda: k8s.count_running_pods(replica_pooler_label), - 2, "Pooler replica pods not found") - self.eventuallyEqual(lambda: k8s.count_services_with_label(pooler_label), - 1, "No pooler service found") - self.eventuallyEqual(lambda: k8s.count_secrets_with_label(pooler_label), - 1, "Secret not created") - - # Turn off only replica connection pooler - k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresqls', 'acid-minimal-cluster', - { - 'spec': { - 'enableConnectionPooler': True, - 'enableReplicaConnectionPooler': False, - 'enableMasterPoolerLoadBalancer': False, - } - }) - - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - - self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(), 2, - "Deployment replicas is 2 default") - self.eventuallyEqual(lambda: k8s.count_running_pods(master_pooler_label), - 2, "Master pooler pods not found") - self.eventuallyEqual(lambda: k8s.count_running_pods(replica_pooler_label), - 0, "Pooler replica pods not deleted") - self.eventuallyEqual(lambda: k8s.count_services_with_label(pooler_label), - 1, "No pooler service found") - self.eventuallyEqual(lambda: k8s.get_service_type(master_pooler_label+","+pooler_label), - 'ClusterIP', - "Expected LoadBalancer service type for master, found {}") - self.eventuallyEqual(lambda: k8s.count_secrets_with_label(pooler_label), - 1, "Secret not created") - - # scale up connection pooler deployment - k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresqls', 'acid-minimal-cluster', - { - 'spec': { - 'connectionPooler': { - 'numberOfInstances': 3, - }, - } - }) - - self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(), 3, - "Deployment replicas is scaled to 3") - self.eventuallyEqual(lambda: k8s.count_running_pods(master_pooler_label), - 3, "Scale up of pooler pods does not work") - - # turn it off, keeping config should be overwritten by false - k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresqls', 'acid-minimal-cluster', - { - 'spec': { - 'enableConnectionPooler': False, - 'enableReplicaConnectionPooler': False, - 'enableReplicaPoolerLoadBalancer': False, - } - }) - - self.eventuallyEqual(lambda: k8s.count_running_pods(master_pooler_label), - 0, "Pooler pods not scaled down") - self.eventuallyEqual(lambda: k8s.count_services_with_label(pooler_label), - 0, "Pooler service not removed") - self.eventuallyEqual(lambda: k8s.count_secrets_with_label('application=spilo,cluster-name=acid-minimal-cluster'), - 4, "Secrets not deleted") - - # Verify that all the databases have pooler schema installed. - # Do this via psql, since otherwise we need to deal with - # credentials. - db_list = [] - - leader = k8s.get_cluster_leader_pod() - schemas_query = """ - SELECT schema_name - FROM information_schema.schemata - WHERE schema_name = 'pooler' - """ - - db_list = self.list_databases(leader.metadata.name) - for db in db_list: - self.eventuallyNotEqual(lambda: len(self.query_database(leader.metadata.name, db, schemas_query)), 0, - "Pooler schema not found in database {}".format(db)) - - # remove config section to make test work next time - k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresqls', 'acid-minimal-cluster', - { - 'spec': { - 'connectionPooler': None, - 'EnableReplicaConnectionPooler': False, - } - }) - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_enable_load_balancer(self): - ''' - Test if services are updated when enabling/disabling load balancers in Postgres manifest - ''' - - k8s = self.k8s - cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster,spilo-role={}' - - self.eventuallyEqual(lambda: k8s.get_service_type(cluster_label.format("master")), - 'ClusterIP', - "Expected ClusterIP type initially, found {}") - - try: - # enable load balancer services - pg_patch_enable_lbs = { - "spec": { - "enableMasterLoadBalancer": True, - "enableReplicaLoadBalancer": True - } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_lbs) - - self.eventuallyEqual(lambda: k8s.get_service_type(cluster_label.format("master")), - 'LoadBalancer', - "Expected LoadBalancer service type for master, found {}") - - self.eventuallyEqual(lambda: k8s.get_service_type(cluster_label.format("replica")), - 'LoadBalancer', - "Expected LoadBalancer service type for master, found {}") - - # disable load balancer services again - pg_patch_disable_lbs = { - "spec": { - "enableMasterLoadBalancer": False, - "enableReplicaLoadBalancer": False - } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_lbs) - - self.eventuallyEqual(lambda: k8s.get_service_type(cluster_label.format("master")), - 'ClusterIP', - "Expected LoadBalancer service type for master, found {}") - - self.eventuallyEqual(lambda: k8s.get_service_type(cluster_label.format("replica")), - 'ClusterIP', - "Expected LoadBalancer service type for master, found {}") - - except timeout_decorator.TimeoutError: - print('Operator log: {}'.format(k8s.get_operator_log())) - raise - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_infrastructure_roles(self): - ''' - Test using external secrets for infrastructure roles - ''' - k8s = self.k8s - # update infrastructure roles description - secret_name = "postgresql-infrastructure-roles" - roles = "secretname: postgresql-infrastructure-roles-new, userkey: user,"\ - "rolekey: memberof, passwordkey: password, defaultrolevalue: robot_zmon" - patch_infrastructure_roles = { - "data": { - "infrastructure_roles_secret_name": secret_name, - "infrastructure_roles_secrets": roles, - }, - } - k8s.update_config(patch_infrastructure_roles) - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, - "Operator does not get in sync") - - try: - # check that new roles are represented in the config by requesting the - # operator configuration via API - - def verify_role(): - try: - operator_pod = k8s.get_operator_pod() - get_config_cmd = "wget --quiet -O - localhost:8080/config" - result = k8s.exec_with_kubectl(operator_pod.metadata.name, - get_config_cmd) - try: - roles_dict = (json.loads(result.stdout) - .get("controller", {}) - .get("InfrastructureRoles")) - except: - return False - - if "robot_zmon_acid_monitoring_new" in roles_dict: - role = roles_dict["robot_zmon_acid_monitoring_new"] - role.pop("Password", None) - self.assertDictEqual(role, { - "Name": "robot_zmon_acid_monitoring_new", - "Namespace":"", - "Flags": None, - "MemberOf": ["robot_zmon"], - "Parameters": None, - "AdminRole": "", - "Origin": 2, - "IsDbOwner": False, - "Deleted": False - }) - return True - except: - pass - - return False - - self.eventuallyTrue(verify_role, "infrastructure role setup is not loaded") - - except timeout_decorator.TimeoutError: - print('Operator log: {}'.format(k8s.get_operator_log())) - raise - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_lazy_spilo_upgrade(self): - ''' - Test lazy upgrade for the Spilo image: operator changes a stateful set - but lets pods run with the old image until they are recreated for - reasons other than operator's activity. That works because the operator - configures stateful sets to use "onDelete" pod update policy. - The test covers: - 1) enabling lazy upgrade in existing operator deployment - 2) forcing the normal rolling upgrade by changing the operator - configmap and restarting its pod - ''' - - k8s = self.k8s - - pod0 = 'acid-minimal-cluster-0' - pod1 = 'acid-minimal-cluster-1' - - self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, - "No 2 pods running") - self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members(pod0)), - 2, "Postgres status did not enter running") - - patch_lazy_spilo_upgrade = { - "data": { - "docker_image": SPILO_CURRENT, - "enable_lazy_spilo_upgrade": "false" - } - } - k8s.update_config(patch_lazy_spilo_upgrade, - step="Init baseline image version") - - self.eventuallyEqual(lambda: k8s.get_statefulset_image(), SPILO_CURRENT, - "Statefulset not updated initially") - self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, - "No 2 pods running") - self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members(pod0)), - 2, "Postgres status did not enter running") - - self.eventuallyEqual(lambda: k8s.get_effective_pod_image(pod0), - SPILO_CURRENT, "Rolling upgrade was not executed") - self.eventuallyEqual(lambda: k8s.get_effective_pod_image(pod1), - SPILO_CURRENT, "Rolling upgrade was not executed") - - # update docker image in config and enable the lazy upgrade - conf_image = SPILO_LAZY - patch_lazy_spilo_upgrade = { - "data": { - "docker_image": conf_image, - "enable_lazy_spilo_upgrade": "true" - } - } - k8s.update_config(patch_lazy_spilo_upgrade, - step="patch image and lazy upgrade") - self.eventuallyEqual(lambda: k8s.get_statefulset_image(), conf_image, - "Statefulset not updated to next Docker image") - - try: - # restart the pod to get a container with the new image - k8s.api.core_v1.delete_namespaced_pod(pod0, 'default') - - # verify only pod-0 which was deleted got new image from statefulset - self.eventuallyEqual(lambda: k8s.get_effective_pod_image(pod0), - conf_image, "Delete pod-0 did not get new spilo image") - self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, - "No two pods running after lazy rolling upgrade") - self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members(pod0)), - 2, "Postgres status did not enter running") - self.assertNotEqual(lambda: k8s.get_effective_pod_image(pod1), - SPILO_CURRENT, - "pod-1 should not have change Docker image to {}".format(SPILO_CURRENT)) - - # clean up - unpatch_lazy_spilo_upgrade = { - "data": { - "enable_lazy_spilo_upgrade": "false", - } - } - k8s.update_config(unpatch_lazy_spilo_upgrade, step="patch lazy upgrade") - - # at this point operator will complete the normal rolling upgrade - # so we additionally test if disabling the lazy upgrade - forcing the normal rolling upgrade - works - self.eventuallyEqual(lambda: k8s.get_effective_pod_image(pod0), - conf_image, "Rolling upgrade was not executed", - 50, 3) - self.eventuallyEqual(lambda: k8s.get_effective_pod_image(pod1), - conf_image, "Rolling upgrade was not executed", - 50, 3) - self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members(pod0)), - 2, "Postgres status did not enter running") - - except timeout_decorator.TimeoutError: - print('Operator log: {}'.format(k8s.get_operator_log())) - raise - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_logical_backup_cron_job(self): - ''' - Ensure we can (a) create the cron job at user request for a specific PG cluster - (b) update the cluster-wide image for the logical backup pod - (c) delete the job at user request - Limitations: - (a) Does not run the actual batch job because there is no S3 mock to upload backups to - (b) Assumes 'acid-minimal-cluster' exists as defined in setUp - ''' - - k8s = self.k8s - - # create the cron job - schedule = "7 7 7 7 *" - pg_patch_enable_backup = { - "spec": { - "enableLogicalBackup": True, - "logicalBackupSchedule": schedule - } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_backup) - - try: - self.eventuallyEqual(lambda: len(k8s.get_logical_backup_job().items), 1, "failed to create logical backup job") - - job = k8s.get_logical_backup_job().items[0] - self.assertEqual(job.metadata.name, "logical-backup-acid-minimal-cluster", - "Expected job name {}, found {}" - .format("logical-backup-acid-minimal-cluster", job.metadata.name)) - self.assertEqual(job.spec.schedule, schedule, - "Expected {} schedule, found {}" - .format(schedule, job.spec.schedule)) - - # update the cluster-wide image of the logical backup pod - image = "test-image-name" - patch_logical_backup_image = { - "data": { - "logical_backup_docker_image": image, - } - } - k8s.update_config(patch_logical_backup_image, step="patch logical backup image") - - def get_docker_image(): - jobs = k8s.get_logical_backup_job().items - return jobs[0].spec.job_template.spec.template.spec.containers[0].image - - self.eventuallyEqual(get_docker_image, image, - "Expected job image {}, found {}".format(image, "{}")) - - # delete the logical backup cron job - pg_patch_disable_backup = { - "spec": { - "enableLogicalBackup": False, - } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_backup) - - self.eventuallyEqual(lambda: len(k8s.get_logical_backup_job().items), 0, "failed to create logical backup job") - - except timeout_decorator.TimeoutError: - print('Operator log: {}'.format(k8s.get_operator_log())) - raise - - # ensure cluster is healthy after tests - self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members("acid-minimal-cluster-0")), 2, "Postgres status did not enter running") - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - @unittest.skip("Skipping this test until fixed") - def test_major_version_upgrade(self): - k8s = self.k8s - result = k8s.create_with_kubectl("manifests/minimal-postgres-manifest-12.yaml") - self.eventuallyEqual(lambda: k8s.count_running_pods(labels="application=spilo,cluster-name=acid-upgrade-test"), 2, "No 2 pods running") - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - - pg_patch_version = { - "spec": { - "postgres": { - "version": "14" - } - } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version) - - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - - def check_version_14(): - p = k8s.get_patroni_state("acid-upgrade-test-0") - version = p["server_version"][0:2] - return version - - self.evantuallyEqual(check_version_14, "14", "Version was not upgrade to 14") - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_min_resource_limits(self): ''' @@ -1106,6 +306,8 @@ class EndToEndTestCase(unittest.TestCase): plural="postgresqls", name="acid-minimal-cluster", body=patch_node_affinity_config) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, + "Operator does not get in sync") # node affinity change should cause replica to relocate from replica node to master node due to node affinity requirement k8s.wait_for_pod_failover(master_nodes, 'spilo-role=replica,' + cluster_label)