From 51909204fd7085f5b890cc4689221fece54ccecc Mon Sep 17 00:00:00 2001 From: Hengchu Zhang Date: Fri, 28 Feb 2020 08:13:58 -0500 Subject: [PATCH 001/168] Change `logging_rest_api.api_port` to `8080` instead of `8008` (#848) The documentation states that the default operator REST service is at port `8080`, but the current default CRD based configuration is `8008`. Changing the default config to match documentation. --- manifests/postgresql-operator-default-configuration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index bdb131fc5..33838b2a9 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -110,7 +110,7 @@ configuration: log_statement: all # teams_api_url: "" logging_rest_api: - api_port: 8008 + api_port: 8080 cluster_history_entries: 1000 ring_log_lines: 100 scalyr: From ae2a38d62a7202f9bda9070a8f9dd0406557fb0d Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 6 Mar 2020 12:55:34 +0100 Subject: [PATCH 002/168] add e2e test for node readiness label (#846) * add e2e test for node readiness label * refactoring and order tests alphabetically * always wait for replica after failover --- e2e/tests/test_e2e.py | 372 +++++++++++++++++++++++++---------------- pkg/controller/node.go | 8 +- 2 files changed, 235 insertions(+), 145 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 12106601e..6760e815d 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -57,6 +57,7 @@ class EndToEndTestCase(unittest.TestCase): k8s.create_with_kubectl("manifests/minimal-postgres-manifest.yaml") k8s.wait_for_pod_start('spilo-role=master') + k8s.wait_for_pod_start('spilo-role=replica') @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_enable_load_balancer(self): @@ -107,141 +108,6 @@ class EndToEndTestCase(unittest.TestCase): self.assertEqual(repl_svc_type, 'ClusterIP', "Expected ClusterIP service type for replica, found {}".format(repl_svc_type)) - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_min_resource_limits(self): - ''' - Lower resource limits below configured minimum and let operator fix it - ''' - k8s = self.k8s - cluster_label = 'cluster-name=acid-minimal-cluster' - _, failover_targets = k8s.get_pg_nodes(cluster_label) - - # configure minimum boundaries for CPU and memory limits - minCPULimit = '500m' - minMemoryLimit = '500Mi' - patch_min_resource_limits = { - "data": { - "min_cpu_limit": minCPULimit, - "min_memory_limit": minMemoryLimit - } - } - k8s.update_config(patch_min_resource_limits) - - # lower resource limits below minimum - pg_patch_resources = { - "spec": { - "resources": { - "requests": { - "cpu": "10m", - "memory": "50Mi" - }, - "limits": { - "cpu": "200m", - "memory": "200Mi" - } - } - } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_resources) - k8s.wait_for_master_failover(failover_targets) - - pods = k8s.api.core_v1.list_namespaced_pod( - 'default', label_selector='spilo-role=master,' + cluster_label).items - self.assert_master_is_unique() - masterPod = pods[0] - - self.assertEqual(masterPod.spec.containers[0].resources.limits['cpu'], minCPULimit, - "Expected CPU limit {}, found {}" - .format(minCPULimit, masterPod.spec.containers[0].resources.limits['cpu'])) - self.assertEqual(masterPod.spec.containers[0].resources.limits['memory'], minMemoryLimit, - "Expected memory limit {}, found {}" - .format(minMemoryLimit, masterPod.spec.containers[0].resources.limits['memory'])) - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_multi_namespace_support(self): - ''' - Create a customized Postgres cluster in a non-default namespace. - ''' - k8s = self.k8s - - with open("manifests/complete-postgres-manifest.yaml", 'r+') as f: - pg_manifest = yaml.safe_load(f) - pg_manifest["metadata"]["namespace"] = self.namespace - yaml.dump(pg_manifest, f, Dumper=yaml.Dumper) - - k8s.create_with_kubectl("manifests/complete-postgres-manifest.yaml") - k8s.wait_for_pod_start("spilo-role=master", self.namespace) - self.assert_master_is_unique(self.namespace, "acid-test-cluster") - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_scaling(self): - ''' - Scale up from 2 to 3 and back to 2 pods by updating the Postgres manifest at runtime. - ''' - k8s = self.k8s - labels = "cluster-name=acid-minimal-cluster" - - k8s.wait_for_pg_to_scale(3) - self.assertEqual(3, k8s.count_pods_with_label(labels)) - self.assert_master_is_unique() - - k8s.wait_for_pg_to_scale(2) - self.assertEqual(2, k8s.count_pods_with_label(labels)) - self.assert_master_is_unique() - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_taint_based_eviction(self): - ''' - Add taint "postgres=:NoExecute" to node with master. This must cause a failover. - ''' - k8s = self.k8s - cluster_label = 'cluster-name=acid-minimal-cluster' - - # get nodes of master and replica(s) (expected target of new master) - current_master_node, failover_targets = k8s.get_pg_nodes(cluster_label) - num_replicas = len(failover_targets) - - # if all pods live on the same node, failover will happen to other worker(s) - failover_targets = [x for x in failover_targets if x != current_master_node] - if len(failover_targets) == 0: - nodes = k8s.api.core_v1.list_node() - for n in nodes.items: - if "node-role.kubernetes.io/master" not in n.metadata.labels and n.metadata.name != current_master_node: - failover_targets.append(n.metadata.name) - - # taint node with postgres=:NoExecute to force failover - body = { - "spec": { - "taints": [ - { - "effect": "NoExecute", - "key": "postgres" - } - ] - } - } - - # patch node and test if master is failing over to one of the expected nodes - k8s.api.core_v1.patch_node(current_master_node, body) - k8s.wait_for_master_failover(failover_targets) - k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) - - new_master_node, new_replica_nodes = k8s.get_pg_nodes(cluster_label) - self.assertNotEqual(current_master_node, new_master_node, - "Master on {} did not fail over to one of {}".format(current_master_node, failover_targets)) - self.assertEqual(num_replicas, len(new_replica_nodes), - "Expected {} replicas, found {}".format(num_replicas, len(new_replica_nodes))) - self.assert_master_is_unique() - - # undo the tainting - body = { - "spec": { - "taints": [] - } - } - k8s.api.core_v1.patch_node(new_master_node, body) - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_logical_backup_cron_job(self): ''' @@ -306,6 +172,133 @@ class EndToEndTestCase(unittest.TestCase): self.assertEqual(0, len(jobs), "Expected 0 logical backup jobs, found {}".format(len(jobs))) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_min_resource_limits(self): + ''' + Lower resource limits below configured minimum and let operator fix it + ''' + k8s = self.k8s + cluster_label = 'cluster-name=acid-minimal-cluster' + labels = 'spilo-role=master,' + cluster_label + _, failover_targets = k8s.get_pg_nodes(cluster_label) + + # configure minimum boundaries for CPU and memory limits + minCPULimit = '500m' + minMemoryLimit = '500Mi' + patch_min_resource_limits = { + "data": { + "min_cpu_limit": minCPULimit, + "min_memory_limit": minMemoryLimit + } + } + k8s.update_config(patch_min_resource_limits) + + # lower resource limits below minimum + pg_patch_resources = { + "spec": { + "resources": { + "requests": { + "cpu": "10m", + "memory": "50Mi" + }, + "limits": { + "cpu": "200m", + "memory": "200Mi" + } + } + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_resources) + k8s.wait_for_pod_failover(failover_targets, labels) + k8s.wait_for_pod_start('spilo-role=replica') + + pods = k8s.api.core_v1.list_namespaced_pod( + 'default', label_selector=labels).items + self.assert_master_is_unique() + masterPod = pods[0] + + self.assertEqual(masterPod.spec.containers[0].resources.limits['cpu'], minCPULimit, + "Expected CPU limit {}, found {}" + .format(minCPULimit, masterPod.spec.containers[0].resources.limits['cpu'])) + self.assertEqual(masterPod.spec.containers[0].resources.limits['memory'], minMemoryLimit, + "Expected memory limit {}, found {}" + .format(minMemoryLimit, masterPod.spec.containers[0].resources.limits['memory'])) + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_multi_namespace_support(self): + ''' + Create a customized Postgres cluster in a non-default namespace. + ''' + k8s = self.k8s + + with open("manifests/complete-postgres-manifest.yaml", 'r+') as f: + pg_manifest = yaml.safe_load(f) + pg_manifest["metadata"]["namespace"] = self.namespace + yaml.dump(pg_manifest, f, Dumper=yaml.Dumper) + + k8s.create_with_kubectl("manifests/complete-postgres-manifest.yaml") + k8s.wait_for_pod_start("spilo-role=master", self.namespace) + self.assert_master_is_unique(self.namespace, "acid-test-cluster") + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_node_readiness_label(self): + ''' + Remove node readiness label from master node. This must cause a failover. + ''' + k8s = self.k8s + cluster_label = 'cluster-name=acid-minimal-cluster' + labels = 'spilo-role=master,' + cluster_label + readiness_label = 'lifecycle-status' + readiness_value = 'ready' + + # get nodes of master and replica(s) (expected target of new master) + current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label) + num_replicas = len(current_replica_nodes) + failover_targets = self.get_failover_targets(current_master_node, current_replica_nodes) + + # add node_readiness_label to potential failover nodes + patch_readiness_label = { + "metadata": { + "labels": { + readiness_label: readiness_value + } + } + } + for failover_target in failover_targets: + k8s.api.core_v1.patch_node(failover_target, patch_readiness_label) + + # define node_readiness_label in config map which should trigger a failover of the master + patch_readiness_label_config = { + "data": { + "node_readiness_label": readiness_label + ':' + readiness_value, + } + } + k8s.update_config(patch_readiness_label_config) + new_master_node, new_replica_nodes = self.assert_failover( + current_master_node, num_replicas, failover_targets, cluster_label) + + # patch also node where master ran before + k8s.api.core_v1.patch_node(current_master_node, patch_readiness_label) + # toggle pod anti affinity to move replica away from master node + self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_scaling(self): + ''' + Scale up from 2 to 3 and back to 2 pods by updating the Postgres manifest at runtime. + ''' + k8s = self.k8s + labels = "cluster-name=acid-minimal-cluster" + + k8s.wait_for_pg_to_scale(3) + self.assertEqual(3, k8s.count_pods_with_label(labels)) + self.assert_master_is_unique() + + k8s.wait_for_pg_to_scale(2) + self.assertEqual(2, k8s.count_pods_with_label(labels)) + self.assert_master_is_unique() + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_service_annotations(self): ''' @@ -346,18 +339,116 @@ class EndToEndTestCase(unittest.TestCase): } k8s.update_config(unpatch_custom_service_annotations) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_taint_based_eviction(self): + ''' + Add taint "postgres=:NoExecute" to node with master. This must cause a failover. + ''' + k8s = self.k8s + cluster_label = 'cluster-name=acid-minimal-cluster' + + # get nodes of master and replica(s) (expected target of new master) + current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label) + num_replicas = len(current_replica_nodes) + failover_targets = self.get_failover_targets(current_master_node, current_replica_nodes) + + # taint node with postgres=:NoExecute to force failover + body = { + "spec": { + "taints": [ + { + "effect": "NoExecute", + "key": "postgres" + } + ] + } + } + + # patch node and test if master is failing over to one of the expected nodes + k8s.api.core_v1.patch_node(current_master_node, body) + new_master_node, new_replica_nodes = self.assert_failover( + current_master_node, num_replicas, failover_targets, cluster_label) + + # add toleration to pods + patch_toleration_config = { + "data": { + "toleration": "key:postgres,operator:Exists,effect:NoExecute" + } + } + k8s.update_config(patch_toleration_config) + + # toggle pod anti affinity to move replica away from master node + self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) + + def get_failover_targets(self, master_node, replica_nodes): + ''' + If all pods live on the same node, failover will happen to other worker(s) + ''' + k8s = self.k8s + + failover_targets = [x for x in replica_nodes if x != master_node] + if len(failover_targets) == 0: + nodes = k8s.api.core_v1.list_node() + for n in nodes.items: + if "node-role.kubernetes.io/master" not in n.metadata.labels and n.metadata.name != master_node: + failover_targets.append(n.metadata.name) + + return failover_targets + + def assert_failover(self, current_master_node, num_replicas, failover_targets, cluster_label): + ''' + Check if master is failing over. The replica should move first to be the switchover target + ''' + k8s = self.k8s + k8s.wait_for_pod_failover(failover_targets, 'spilo-role=master,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) + + new_master_node, new_replica_nodes = k8s.get_pg_nodes(cluster_label) + self.assertNotEqual(current_master_node, new_master_node, + "Master on {} did not fail over to one of {}".format(current_master_node, failover_targets)) + self.assertEqual(num_replicas, len(new_replica_nodes), + "Expected {} replicas, found {}".format(num_replicas, len(new_replica_nodes))) + self.assert_master_is_unique() + + return new_master_node, new_replica_nodes + def assert_master_is_unique(self, namespace='default', clusterName="acid-minimal-cluster"): ''' Check that there is a single pod in the k8s cluster with the label "spilo-role=master" To be called manually after operations that affect pods ''' - k8s = self.k8s labels = 'spilo-role=master,cluster-name=' + clusterName num_of_master_pods = k8s.count_pods_with_label(labels, namespace) self.assertEqual(num_of_master_pods, 1, "Expected 1 master pod, found {}".format(num_of_master_pods)) + def assert_distributed_pods(self, master_node, replica_nodes, cluster_label): + ''' + Other tests can lead to the situation that master and replica are on the same node. + Toggle pod anti affinty to distribute pods accross nodes (replica in particular). + ''' + k8s = self.k8s + failover_targets = self.get_failover_targets(master_node, replica_nodes) + + # enable pod anti affintiy in config map which should trigger movement of replica + patch_enable_antiaffinity = { + "data": { + "enable_pod_antiaffinity": "true" + } + } + k8s.update_config(patch_enable_antiaffinity) + self.assert_failover( + master_node, len(replica_nodes), failover_targets, cluster_label) + + # disable pod anti affintiy again + patch_disable_antiaffinity = { + "data": { + "enable_pod_antiaffinity": "false" + } + } + k8s.update_config(patch_disable_antiaffinity) + class K8sApi: @@ -445,15 +536,14 @@ class K8s: def count_pods_with_label(self, labels, namespace='default'): return len(self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items) - def wait_for_master_failover(self, expected_master_nodes, namespace='default'): + def wait_for_pod_failover(self, failover_targets, labels, namespace='default'): pod_phase = 'Failing over' - new_master_node = '' - labels = 'spilo-role=master,cluster-name=acid-minimal-cluster' + new_pod_node = '' - while (pod_phase != 'Running') or (new_master_node not in expected_master_nodes): + while (pod_phase != 'Running') or (new_pod_node not in failover_targets): pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items if pods: - new_master_node = pods[0].spec.node_name + new_pod_node = pods[0].spec.node_name pod_phase = pods[0].status.phase time.sleep(self.RETRY_TIMEOUT_SEC) diff --git a/pkg/controller/node.go b/pkg/controller/node.go index 6f7befa27..8052458c3 100644 --- a/pkg/controller/node.go +++ b/pkg/controller/node.go @@ -5,7 +5,7 @@ import ( "time" "github.com/zalando/postgres-operator/pkg/util/retryutil" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -172,19 +172,19 @@ func (c *Controller) nodeDelete(obj interface{}) { } func (c *Controller) moveMasterPodsOffNode(node *v1.Node) { - + // retry to move master until configured timeout is reached err := retryutil.Retry(1*time.Minute, c.opConfig.MasterPodMoveTimeout, func() (bool, error) { err := c.attemptToMoveMasterPodsOffNode(node) if err != nil { - return false, fmt.Errorf("unable to move master pods off the unschedulable node; will retry after delay of 1 minute") + return false, err } return true, nil }, ) if err != nil { - c.logger.Warningf("failed to move master pods from the node %q: timeout of %v minutes expired", node.Name, c.opConfig.MasterPodMoveTimeout) + c.logger.Warningf("failed to move master pods from the node %q: %v", node.Name, err) } } From 35b2213e058aebfde274844844ea66ee9556fd3f Mon Sep 17 00:00:00 2001 From: Jonathan Herlin Date: Wed, 11 Mar 2020 11:32:13 +0100 Subject: [PATCH 003/168] Fix typo in values file (#861) * Fix typo Co-authored-by: Jonathan Herlin --- charts/postgres-operator/values-crd.yaml | 2 +- charts/postgres-operator/values.yaml | 2 +- docs/reference/operator_parameters.md | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index b5d561807..d170e0b77 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -218,7 +218,7 @@ configLogicalBackup: logical_backup_s3_endpoint: "" # S3 Secret Access Key logical_backup_s3_secret_access_key: "" - # S3 server side encription + # S3 server side encryption logical_backup_s3_sse: "AES256" # backup schedule in the cron format logical_backup_schedule: "30 00 * * *" diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 07ba76285..b6f18f305 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -209,7 +209,7 @@ configLogicalBackup: logical_backup_s3_endpoint: "" # S3 Secret Access Key logical_backup_s3_secret_access_key: "" - # S3 server side encription + # S3 server side encryption logical_backup_s3_sse: "AES256" # backup schedule in the cron format logical_backup_schedule: "30 00 * * *" diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index ad519b657..ba8e73cf8 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -284,7 +284,7 @@ configuration they are grouped under the `kubernetes` key. used for AWS volume resizing and not required if you don't need that capability. The default is `false`. - * **master_pod_move_timeout** +* **master_pod_move_timeout** The period of time to wait for the success of migration of master pods from an unschedulable node. The migration includes Patroni switchovers to respective replicas on healthy nodes. The situation where master pods still @@ -472,7 +472,7 @@ grouped under the `logical_backup` key. When using non-AWS S3 storage, endpoint can be set as a ENV variable. The default is empty. * **logical_backup_s3_sse** - Specify server side encription that S3 storage is using. If empty string + Specify server side encryption that S3 storage is using. If empty string is specified, no argument will be passed to `aws s3` command. Default: "AES256". * **logical_backup_s3_access_key_id** From cde61f3f0b9d91863ac16cab89463c647d297517 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 11 Mar 2020 14:08:54 +0100 Subject: [PATCH 004/168] e2e: wait for pods after disabling anti affinity (#862) --- e2e/tests/test_e2e.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 6760e815d..f6be8a600 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -441,13 +441,15 @@ class EndToEndTestCase(unittest.TestCase): self.assert_failover( master_node, len(replica_nodes), failover_targets, cluster_label) - # disable pod anti affintiy again + # now disable pod anti affintiy again which will cause yet another failover patch_disable_antiaffinity = { "data": { "enable_pod_antiaffinity": "false" } } k8s.update_config(patch_disable_antiaffinity) + k8s.wait_for_pod_start('spilo-role=master') + k8s.wait_for_pod_start('spilo-role=replica') class K8sApi: From 650b8daf77f35d03b0fbd989908c3dea4a78108f Mon Sep 17 00:00:00 2001 From: grantlanglois Date: Thu, 12 Mar 2020 04:12:53 -0700 Subject: [PATCH 005/168] add json:omitempty option to ClusterDomain (#851) Co-authored-by: mlu42 --- pkg/apis/acid.zalan.do/v1/operator_configuration_type.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index ded5261fb..8463be6bd 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -53,7 +53,7 @@ type KubernetesMetaConfiguration struct { EnableInitContainers *bool `json:"enable_init_containers,omitempty"` EnableSidecars *bool `json:"enable_sidecars,omitempty"` SecretNameTemplate config.StringTemplate `json:"secret_name_template,omitempty"` - ClusterDomain string `json:"cluster_domain"` + ClusterDomain string `json:"cluster_domain,omitempty"` OAuthTokenSecretName spec.NamespacedName `json:"oauth_token_secret_name,omitempty"` InfrastructureRolesSecretName spec.NamespacedName `json:"infrastructure_roles_secret_name,omitempty"` PodRoleLabel string `json:"pod_role_label,omitempty"` From 65fb2ce1a622109d4ec0f9cc40bf8590d8647659 Mon Sep 17 00:00:00 2001 From: zimbatm Date: Fri, 13 Mar 2020 11:44:38 +0100 Subject: [PATCH 006/168] add support for custom TLS certificates (#798) * add support for custom TLS certificates --- docs/reference/cluster_manifest.md | 21 ++++ docs/user.md | 47 ++++++++ go.mod | 1 + go.sum | 1 + manifests/complete-postgres-manifest.yaml | 7 ++ manifests/postgresql.crd.yaml | 13 +++ pkg/apis/acid.zalan.do/v1/crds.go | 18 ++++ pkg/apis/acid.zalan.do/v1/postgresql_type.go | 8 ++ .../acid.zalan.do/v1/zz_generated.deepcopy.go | 21 ++++ pkg/cluster/k8sres.go | 100 +++++++++++++++--- pkg/cluster/k8sres_test.go | 67 +++++++++++- 11 files changed, 285 insertions(+), 19 deletions(-) diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 7b049b6fa..92e457d7e 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -359,3 +359,24 @@ CPU and memory limits for the sidecar container. * **memory** memory limits for the sidecar container. Optional, overrides the `default_memory_limits` operator configuration parameter. Optional. + +## Custom TLS certificates + +Those parameters are grouped under the `tls` top-level key. + +* **secretName** + By setting the `secretName` value, the cluster will switch to load the given + Kubernetes Secret into the container as a volume and uses that as the + certificate instead. It is up to the user to create and manage the + Kubernetes Secret either by hand or using a tool like the CertManager + operator. + +* **certificateFile** + Filename of the certificate. Defaults to "tls.crt". + +* **privateKeyFile** + Filename of the private key. Defaults to "tls.key". + +* **caFile** + Optional filename to the CA certificate. Useful when the client connects + with `sslmode=verify-ca` or `sslmode=verify-full`. diff --git a/docs/user.md b/docs/user.md index 295c149bd..6e71d0404 100644 --- a/docs/user.md +++ b/docs/user.md @@ -511,3 +511,50 @@ monitoring is outside the scope of operator responsibilities. See [configuration reference](reference/cluster_manifest.md) and [administrator documentation](administrator.md) for details on how backups are executed. + +## Custom TLS certificates + +By default, the spilo image generates its own TLS certificate during startup. +This certificate is not secure since it cannot be verified and thus doesn't +protect from active MITM attacks. In this section we show how a Kubernete +Secret resources can be loaded with a custom TLS certificate. + +Before applying these changes, the operator must also be configured with the +`spilo_fsgroup` set to the GID matching the postgres user group. If the value +is not provided, the cluster will default to `103` which is the GID from the +default spilo image. + +Upload the cert as a kubernetes secret: +```sh +kubectl create secret tls pg-tls \ + --key pg-tls.key \ + --cert pg-tls.crt +``` + +Or with a CA: +```sh +kubectl create secret generic pg-tls \ + --from-file=tls.crt=server.crt \ + --from-file=tls.key=server.key \ + --from-file=ca.crt=ca.crt +``` + +Alternatively it is also possible to use +[cert-manager](https://cert-manager.io/docs/) to generate these secrets. + +Then configure the postgres resource with the TLS secret: + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: postgresql + +metadata: + name: acid-test-cluster +spec: + tls: + secretName: "pg-tls" + caFile: "ca.crt" # add this if the secret is configured with a CA +``` + +Certificate rotation is handled in the spilo image which checks every 5 +minutes if the certificates have changed and reloads postgres accordingly. diff --git a/go.mod b/go.mod index 36686dcf6..be1ec32d4 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/lib/pq v1.2.0 github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d github.com/sirupsen/logrus v1.4.2 + github.com/stretchr/testify v1.4.0 golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 // indirect golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 // indirect diff --git a/go.sum b/go.sum index f85dd060f..0737d3a5d 100644 --- a/go.sum +++ b/go.sum @@ -275,6 +275,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 5ae817ca3..5ba1fd1bc 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -100,3 +100,10 @@ spec: # env: # - name: "USEFUL_VAR" # value: "perhaps-true" + +# Custom TLS certificate. Disabled unless tls.secretName has a value. + tls: + secretName: "" # should correspond to a Kubernetes Secret resource to load + certificateFile: "tls.crt" + privateKeyFile: "tls.key" + caFile: "" # optionally configure Postgres with a CA certificate diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 453916b26..04c789fb9 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -251,6 +251,19 @@ spec: type: string teamId: type: string + tls: + type: object + required: + - secretName + properties: + secretName: + type: string + certificateFile: + type: string + privateKeyFile: + type: string + caFile: + type: string tolerations: type: array items: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 28dfa1566..eff47c255 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -417,6 +417,24 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ "teamId": { Type: "string", }, + "tls": { + Type: "object", + Required: []string{"secretName"}, + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "secretName": { + Type: "string", + }, + "certificateFile": { + Type: "string", + }, + "privateKeyFile": { + Type: "string", + }, + "caFile": { + Type: "string", + }, + }, + }, "tolerations": { Type: "array", Items: &apiextv1beta1.JSONSchemaPropsOrArray{ diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 07b42d4d4..862db6a4e 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -61,6 +61,7 @@ type PostgresSpec struct { StandbyCluster *StandbyDescription `json:"standby"` PodAnnotations map[string]string `json:"podAnnotations"` ServiceAnnotations map[string]string `json:"serviceAnnotations"` + TLS *TLSDescription `json:"tls"` // deprecated json tags InitContainersOld []v1.Container `json:"init_containers,omitempty"` @@ -126,6 +127,13 @@ type StandbyDescription struct { S3WalPath string `json:"s3_wal_path,omitempty"` } +type TLSDescription struct { + SecretName string `json:"secretName,omitempty"` + CertificateFile string `json:"certificateFile,omitempty"` + PrivateKeyFile string `json:"privateKeyFile,omitempty"` + CAFile string `json:"caFile,omitempty"` +} + // CloneDescription describes which cluster the new should clone and up to which point in time type CloneDescription struct { ClusterName string `json:"cluster,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index aaae1f04b..753b0490c 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -521,6 +521,11 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { (*out)[key] = val } } + if in.TLS != nil { + in, out := &in.TLS, &out.TLS + *out = new(TLSDescription) + **out = **in + } if in.InitContainersOld != nil { in, out := &in.InitContainersOld, &out.InitContainersOld *out = make([]corev1.Container, len(*in)) @@ -752,6 +757,22 @@ func (in *StandbyDescription) DeepCopy() *StandbyDescription { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSDescription) DeepCopyInto(out *TLSDescription) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSDescription. +func (in *TLSDescription) DeepCopy() *TLSDescription { + if in == nil { + return nil + } + out := new(TLSDescription) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TeamsAPIConfiguration) DeepCopyInto(out *TeamsAPIConfiguration) { *out = *in diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index e2251a67c..84c407700 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -3,6 +3,7 @@ package cluster import ( "encoding/json" "fmt" + "path" "sort" "github.com/sirupsen/logrus" @@ -30,7 +31,10 @@ const ( patroniPGBinariesParameterName = "bin_dir" patroniPGParametersParameterName = "parameters" patroniPGHBAConfParameterName = "pg_hba" - localHost = "127.0.0.1/32" + + // the gid of the postgres user in the default spilo image + spiloPostgresGID = 103 + localHost = "127.0.0.1/32" ) type pgUser struct { @@ -446,6 +450,7 @@ func generatePodTemplate( podAntiAffinityTopologyKey string, additionalSecretMount string, additionalSecretMountPath string, + volumes []v1.Volume, ) (*v1.PodTemplateSpec, error) { terminateGracePeriodSeconds := terminateGracePeriod @@ -464,6 +469,7 @@ func generatePodTemplate( InitContainers: initContainers, Tolerations: *tolerationsSpec, SecurityContext: &securityContext, + Volumes: volumes, } if shmVolume != nil && *shmVolume { @@ -724,6 +730,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef sidecarContainers []v1.Container podTemplate *v1.PodTemplateSpec volumeClaimTemplate *v1.PersistentVolumeClaim + volumes []v1.Volume ) // Improve me. Please. @@ -840,21 +847,76 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef } // generate environment variables for the spilo container - spiloEnvVars := deduplicateEnvVars( - c.generateSpiloPodEnvVars(c.Postgresql.GetUID(), spiloConfiguration, &spec.Clone, - spec.StandbyCluster, customPodEnvVarsList), c.containerName(), c.logger) + spiloEnvVars := c.generateSpiloPodEnvVars( + c.Postgresql.GetUID(), + spiloConfiguration, + &spec.Clone, + spec.StandbyCluster, + customPodEnvVarsList, + ) // pickup the docker image for the spilo container effectiveDockerImage := util.Coalesce(spec.DockerImage, c.OpConfig.DockerImage) + // determine the FSGroup for the spilo pod + effectiveFSGroup := c.OpConfig.Resources.SpiloFSGroup + if spec.SpiloFSGroup != nil { + effectiveFSGroup = spec.SpiloFSGroup + } + volumeMounts := generateVolumeMounts(spec.Volume) + // configure TLS with a custom secret volume + if spec.TLS != nil && spec.TLS.SecretName != "" { + if effectiveFSGroup == nil { + c.logger.Warnf("Setting the default FSGroup to satisfy the TLS configuration") + fsGroup := int64(spiloPostgresGID) + effectiveFSGroup = &fsGroup + } + // this is combined with the FSGroup above to give read access to the + // postgres user + defaultMode := int32(0640) + volumes = append(volumes, v1.Volume{ + Name: "tls-secret", + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: spec.TLS.SecretName, + DefaultMode: &defaultMode, + }, + }, + }) + + mountPath := "/tls" + volumeMounts = append(volumeMounts, v1.VolumeMount{ + MountPath: mountPath, + Name: "tls-secret", + ReadOnly: true, + }) + + // use the same filenames as Secret resources by default + certFile := ensurePath(spec.TLS.CertificateFile, mountPath, "tls.crt") + privateKeyFile := ensurePath(spec.TLS.PrivateKeyFile, mountPath, "tls.key") + spiloEnvVars = append( + spiloEnvVars, + v1.EnvVar{Name: "SSL_CERTIFICATE_FILE", Value: certFile}, + v1.EnvVar{Name: "SSL_PRIVATE_KEY_FILE", Value: privateKeyFile}, + ) + + if spec.TLS.CAFile != "" { + caFile := ensurePath(spec.TLS.CAFile, mountPath, "") + spiloEnvVars = append( + spiloEnvVars, + v1.EnvVar{Name: "SSL_CA_FILE", Value: caFile}, + ) + } + } + // generate the spilo container c.logger.Debugf("Generating Spilo container, environment variables: %v", spiloEnvVars) spiloContainer := generateContainer(c.containerName(), &effectiveDockerImage, resourceRequirements, - spiloEnvVars, + deduplicateEnvVars(spiloEnvVars, c.containerName(), c.logger), volumeMounts, c.OpConfig.Resources.SpiloPrivileged, ) @@ -893,16 +955,10 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef tolerationSpec := tolerations(&spec.Tolerations, c.OpConfig.PodToleration) effectivePodPriorityClassName := util.Coalesce(spec.PodPriorityClassName, c.OpConfig.PodPriorityClassName) - // determine the FSGroup for the spilo pod - effectiveFSGroup := c.OpConfig.Resources.SpiloFSGroup - if spec.SpiloFSGroup != nil { - effectiveFSGroup = spec.SpiloFSGroup - } - annotations := c.generatePodAnnotations(spec) // generate pod template for the statefulset, based on the spilo container and sidecars - if podTemplate, err = generatePodTemplate( + podTemplate, err = generatePodTemplate( c.Namespace, c.labelsSet(true), annotations, @@ -920,10 +976,9 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef c.OpConfig.EnablePodAntiAffinity, c.OpConfig.PodAntiAffinityTopologyKey, c.OpConfig.AdditionalSecretMount, - c.OpConfig.AdditionalSecretMountPath); err != nil { - return nil, fmt.Errorf("could not generate pod template: %v", err) - } - + c.OpConfig.AdditionalSecretMountPath, + volumes, + ) if err != nil { return nil, fmt.Errorf("could not generate pod template: %v", err) } @@ -1539,7 +1594,8 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) { false, "", c.OpConfig.AdditionalSecretMount, - c.OpConfig.AdditionalSecretMountPath); err != nil { + c.OpConfig.AdditionalSecretMountPath, + nil); err != nil { return nil, fmt.Errorf("could not generate pod template for logical backup pod: %v", err) } @@ -1671,3 +1727,13 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar { func (c *Cluster) getLogicalBackupJobName() (jobName string) { return "logical-backup-" + c.clusterName().Name } + +func ensurePath(file string, defaultDir string, defaultFile string) string { + if file == "" { + return path.Join(defaultDir, defaultFile) + } + if !path.IsAbs(file) { + return path.Join(defaultDir, file) + } + return file +} diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index e8fe05456..7fd4cd3e6 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -3,16 +3,17 @@ package cluster import ( "reflect" - v1 "k8s.io/api/core/v1" - "testing" + "github.com/stretchr/testify/assert" + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" + v1 "k8s.io/api/core/v1" policyv1beta1 "k8s.io/api/policy/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -451,3 +452,65 @@ func TestSecretVolume(t *testing.T) { } } } + +func TestTLS(t *testing.T) { + var err error + var spec acidv1.PostgresSpec + var cluster *Cluster + + makeSpec := func(tls acidv1.TLSDescription) acidv1.PostgresSpec { + return acidv1.PostgresSpec{ + TeamID: "myapp", NumberOfInstances: 1, + Resources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + }, + Volume: acidv1.Volume{ + Size: "1G", + }, + TLS: &tls, + } + } + + cluster = New( + Config{ + OpConfig: config.Config{ + PodManagementPolicy: "ordered_ready", + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + spec = makeSpec(acidv1.TLSDescription{SecretName: "my-secret", CAFile: "ca.crt"}) + s, err := cluster.generateStatefulSet(&spec) + if err != nil { + assert.NoError(t, err) + } + + fsGroup := int64(103) + assert.Equal(t, &fsGroup, s.Spec.Template.Spec.SecurityContext.FSGroup, "has a default FSGroup assigned") + + defaultMode := int32(0640) + volume := v1.Volume{ + Name: "tls-secret", + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: "my-secret", + DefaultMode: &defaultMode, + }, + }, + } + assert.Contains(t, s.Spec.Template.Spec.Volumes, volume, "the pod gets a secret volume") + + assert.Contains(t, s.Spec.Template.Spec.Containers[0].VolumeMounts, v1.VolumeMount{ + MountPath: "/tls", + Name: "tls-secret", + ReadOnly: true, + }, "the volume gets mounted in /tls") + + assert.Contains(t, s.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "SSL_CERTIFICATE_FILE", Value: "/tls/tls.crt"}) + assert.Contains(t, s.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "SSL_PRIVATE_KEY_FILE", Value: "/tls/tls.key"}) + assert.Contains(t, s.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "SSL_CA_FILE", Value: "/tls/ca.crt"}) +} From b66734a0a9a94147bc99bffa6844e146b036a7f7 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 13 Mar 2020 11:48:19 +0100 Subject: [PATCH 007/168] omit PgVersion diff on sync (#860) * use PostgresParam.PgVersion everywhere * on sync compare pgVersion with SpiloConfiguration * update getNewPgVersion and added tests --- kubectl-pg/cmd/list.go | 25 ++++--- pkg/cluster/cluster.go | 7 +- pkg/cluster/k8sres.go | 48 +++++++++++++- pkg/cluster/k8sres_test.go | 129 +++++++++++++++++++++++++++++++++++++ pkg/cluster/sync.go | 12 ++++ 5 files changed, 208 insertions(+), 13 deletions(-) diff --git a/kubectl-pg/cmd/list.go b/kubectl-pg/cmd/list.go index df827ffaf..f4dea882d 100644 --- a/kubectl-pg/cmd/list.go +++ b/kubectl-pg/cmd/list.go @@ -24,13 +24,14 @@ package cmd import ( "fmt" - "github.com/spf13/cobra" - "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" - PostgresqlLister "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "log" "strconv" "time" + + "github.com/spf13/cobra" + v1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + PostgresqlLister "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( @@ -95,8 +96,12 @@ func listAll(listPostgres *v1.PostgresqlList) { template := "%-32s%-16s%-12s%-12s%-12s%-12s%-12s\n" fmt.Printf(template, "NAME", "STATUS", "INSTANCES", "VERSION", "AGE", "VOLUME", "NAMESPACE") for _, pgObjs := range listPostgres.Items { - fmt.Printf(template, pgObjs.Name, pgObjs.Status.PostgresClusterStatus, strconv.Itoa(int(pgObjs.Spec.NumberOfInstances)), - pgObjs.Spec.PgVersion, time.Since(pgObjs.CreationTimestamp.Time).Truncate(TrimCreateTimestamp), pgObjs.Spec.Size, pgObjs.Namespace) + fmt.Printf(template, pgObjs.Name, + pgObjs.Status.PostgresClusterStatus, + strconv.Itoa(int(pgObjs.Spec.NumberOfInstances)), + pgObjs.Spec.PostgresqlParam.PgVersion, + time.Since(pgObjs.CreationTimestamp.Time).Truncate(TrimCreateTimestamp), + pgObjs.Spec.Size, pgObjs.Namespace) } } @@ -104,8 +109,12 @@ func listWithNamespace(listPostgres *v1.PostgresqlList) { template := "%-32s%-16s%-12s%-12s%-12s%-12s\n" fmt.Printf(template, "NAME", "STATUS", "INSTANCES", "VERSION", "AGE", "VOLUME") for _, pgObjs := range listPostgres.Items { - fmt.Printf(template, pgObjs.Name, pgObjs.Status.PostgresClusterStatus, strconv.Itoa(int(pgObjs.Spec.NumberOfInstances)), - pgObjs.Spec.PgVersion, time.Since(pgObjs.CreationTimestamp.Time).Truncate(TrimCreateTimestamp), pgObjs.Spec.Size) + fmt.Printf(template, pgObjs.Name, + pgObjs.Status.PostgresClusterStatus, + strconv.Itoa(int(pgObjs.Spec.NumberOfInstances)), + pgObjs.Spec.PostgresqlParam.PgVersion, + time.Since(pgObjs.CreationTimestamp.Time).Truncate(TrimCreateTimestamp), + pgObjs.Spec.Size) } } diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 91e7a5195..d740260d2 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -554,10 +554,11 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { } }() - if oldSpec.Spec.PgVersion != newSpec.Spec.PgVersion { // PG versions comparison - c.logger.Warningf("postgresql version change(%q -> %q) has no effect", oldSpec.Spec.PgVersion, newSpec.Spec.PgVersion) + if oldSpec.Spec.PostgresqlParam.PgVersion != newSpec.Spec.PostgresqlParam.PgVersion { // PG versions comparison + c.logger.Warningf("postgresql version change(%q -> %q) has no effect", + oldSpec.Spec.PostgresqlParam.PgVersion, newSpec.Spec.PostgresqlParam.PgVersion) //we need that hack to generate statefulset with the old version - newSpec.Spec.PgVersion = oldSpec.Spec.PgVersion + newSpec.Spec.PostgresqlParam.PgVersion = oldSpec.Spec.PostgresqlParam.PgVersion } // Service diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 84c407700..aaa27384a 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -27,7 +27,7 @@ import ( ) const ( - pgBinariesLocationTemplate = "/usr/lib/postgresql/%s/bin" + pgBinariesLocationTemplate = "/usr/lib/postgresql/%v/bin" patroniPGBinariesParameterName = "bin_dir" patroniPGParametersParameterName = "parameters" patroniPGHBAConfParameterName = "pg_hba" @@ -722,6 +722,50 @@ func makeResources(cpuRequest, memoryRequest, cpuLimit, memoryLimit string) acid } } +func extractPgVersionFromBinPath(binPath string, template string) (string, error) { + var pgVersion float32 + _, err := fmt.Sscanf(binPath, template, &pgVersion) + if err != nil { + return "", err + } + return fmt.Sprintf("%v", pgVersion), nil +} + +func (c *Cluster) getNewPgVersion(container v1.Container, newPgVersion string) (string, error) { + var ( + spiloConfiguration spiloConfiguration + runningPgVersion string + err error + ) + + for _, env := range container.Env { + if env.Name != "SPILO_CONFIGURATION" { + continue + } + err = json.Unmarshal([]byte(env.Value), &spiloConfiguration) + if err != nil { + return newPgVersion, err + } + } + + if len(spiloConfiguration.PgLocalConfiguration) > 0 { + currentBinPath := fmt.Sprintf("%v", spiloConfiguration.PgLocalConfiguration[patroniPGBinariesParameterName]) + runningPgVersion, err = extractPgVersionFromBinPath(currentBinPath, pgBinariesLocationTemplate) + if err != nil { + return "", fmt.Errorf("could not extract Postgres version from %v in SPILO_CONFIGURATION", currentBinPath) + } + } else { + return "", fmt.Errorf("could not find %q setting in SPILO_CONFIGURATION", patroniPGBinariesParameterName) + } + + if runningPgVersion != newPgVersion { + c.logger.Warningf("postgresql version change(%q -> %q) has no effect", runningPgVersion, newPgVersion) + newPgVersion = runningPgVersion + } + + return newPgVersion, nil +} + func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.StatefulSet, error) { var ( @@ -1680,7 +1724,7 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar { // Postgres env vars { Name: "PG_VERSION", - Value: c.Spec.PgVersion, + Value: c.Spec.PostgresqlParam.PgVersion, }, { Name: "PGPORT", diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 7fd4cd3e6..25e0f7af4 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -382,6 +382,135 @@ func TestCloneEnv(t *testing.T) { } } +func TestExtractPgVersionFromBinPath(t *testing.T) { + testName := "TestExtractPgVersionFromBinPath" + tests := []struct { + subTest string + binPath string + template string + expected string + }{ + { + subTest: "test current bin path with decimal against hard coded template", + binPath: "/usr/lib/postgresql/9.6/bin", + template: pgBinariesLocationTemplate, + expected: "9.6", + }, + { + subTest: "test current bin path against hard coded template", + binPath: "/usr/lib/postgresql/12/bin", + template: pgBinariesLocationTemplate, + expected: "12", + }, + { + subTest: "test alternative bin path against a matching template", + binPath: "/usr/pgsql-12/bin", + template: "/usr/pgsql-%v/bin", + expected: "12", + }, + } + + for _, tt := range tests { + pgVersion, err := extractPgVersionFromBinPath(tt.binPath, tt.template) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + if pgVersion != tt.expected { + t.Errorf("%s %s: Expected version %s, have %s instead", + testName, tt.subTest, tt.expected, pgVersion) + } + } +} + +func TestGetPgVersion(t *testing.T) { + testName := "TestGetPgVersion" + tests := []struct { + subTest string + pgContainer v1.Container + currentPgVersion string + newPgVersion string + }{ + { + subTest: "new version with decimal point differs from current SPILO_CONFIGURATION", + pgContainer: v1.Container{ + Name: "postgres", + Env: []v1.EnvVar{ + { + Name: "SPILO_CONFIGURATION", + Value: "{\"postgresql\": {\"bin_dir\": \"/usr/lib/postgresql/9.6/bin\"}}", + }, + }, + }, + currentPgVersion: "9.6", + newPgVersion: "12", + }, + { + subTest: "new version differs from current SPILO_CONFIGURATION", + pgContainer: v1.Container{ + Name: "postgres", + Env: []v1.EnvVar{ + { + Name: "SPILO_CONFIGURATION", + Value: "{\"postgresql\": {\"bin_dir\": \"/usr/lib/postgresql/11/bin\"}}", + }, + }, + }, + currentPgVersion: "11", + newPgVersion: "12", + }, + { + subTest: "new version is lower than the one found in current SPILO_CONFIGURATION", + pgContainer: v1.Container{ + Name: "postgres", + Env: []v1.EnvVar{ + { + Name: "SPILO_CONFIGURATION", + Value: "{\"postgresql\": {\"bin_dir\": \"/usr/lib/postgresql/12/bin\"}}", + }, + }, + }, + currentPgVersion: "12", + newPgVersion: "11", + }, + { + subTest: "new version is the same like in the current SPILO_CONFIGURATION", + pgContainer: v1.Container{ + Name: "postgres", + Env: []v1.EnvVar{ + { + Name: "SPILO_CONFIGURATION", + Value: "{\"postgresql\": {\"bin_dir\": \"/usr/lib/postgresql/12/bin\"}}", + }, + }, + }, + currentPgVersion: "12", + newPgVersion: "12", + }, + } + + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + + for _, tt := range tests { + pgVersion, err := cluster.getNewPgVersion(tt.pgContainer, tt.newPgVersion) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + if pgVersion != tt.currentPgVersion { + t.Errorf("%s %s: Expected version %s, have %s instead", + testName, tt.subTest, tt.currentPgVersion, pgVersion) + } + } +} + func TestSecretVolume(t *testing.T) { testName := "TestSecretVolume" tests := []struct { diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 053db9ff7..b04ff863b 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -285,6 +285,18 @@ func (c *Cluster) syncStatefulSet() error { // statefulset is already there, make sure we use its definition in order to compare with the spec. c.Statefulset = sset + // check if there is no Postgres version mismatch + for _, container := range c.Statefulset.Spec.Template.Spec.Containers { + if container.Name != "postgres" { + continue + } + pgVersion, err := c.getNewPgVersion(container, c.Spec.PostgresqlParam.PgVersion) + if err != nil { + return fmt.Errorf("could not parse current Postgres version: %v", err) + } + c.Spec.PostgresqlParam.PgVersion = pgVersion + } + desiredSS, err := c.generateStatefulSet(&c.Spec) if err != nil { return fmt.Errorf("could not generate statefulset: %v", err) From d666c521726eaf3ceaba8ac24171f290d4277742 Mon Sep 17 00:00:00 2001 From: Dmitry Dolgov <9erthalion6@gmail.com> Date: Fri, 13 Mar 2020 11:51:39 +0100 Subject: [PATCH 008/168] ClusterDomain default (#863) * add json:omitempty option to ClusterDomain * Add default value for ClusterDomain Unfortunately, omitempty in operator configuration CRD doesn't mean that defauls from operator config object will be picked up automatically. Make sure that ClusterDomain default is specified, so that even when someone will set cluster_domain = "", it will be overwritted with a default value. Co-authored-by: mlu42 --- pkg/controller/operator_config.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index c6f10faa0..03602c3bd 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -6,6 +6,7 @@ import ( "time" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -50,7 +51,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.PodTerminateGracePeriod = time.Duration(fromCRD.Kubernetes.PodTerminateGracePeriod) result.SpiloPrivileged = fromCRD.Kubernetes.SpiloPrivileged result.SpiloFSGroup = fromCRD.Kubernetes.SpiloFSGroup - result.ClusterDomain = fromCRD.Kubernetes.ClusterDomain + result.ClusterDomain = util.Coalesce(fromCRD.Kubernetes.ClusterDomain, "cluster.local") result.WatchedNamespace = fromCRD.Kubernetes.WatchedNamespace result.PDBNameFormat = fromCRD.Kubernetes.PDBNameFormat result.EnablePodDisruptionBudget = fromCRD.Kubernetes.EnablePodDisruptionBudget From cf829df1a49ebe9a72e2bbc9c9289b1077ff4a48 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 17 Mar 2020 16:34:31 +0100 Subject: [PATCH 009/168] define ownership between operator and clusters via annotation (#802) * define ownership between operator and postgres clusters * add documentation * add unit test --- cmd/main.go | 2 +- docs/administrator.md | 28 ++++++++++ manifests/complete-postgres-manifest.yaml | 2 + manifests/postgres-operator.yaml | 3 ++ pkg/controller/controller.go | 19 ++++++- pkg/controller/node_test.go | 14 ++--- pkg/controller/postgresql.go | 62 +++++++++++++++-------- pkg/controller/postgresql_test.go | 58 +++++++++++++++++++-- pkg/controller/util_test.go | 14 ++--- pkg/util/constants/annotations.go | 1 + 10 files changed, 160 insertions(+), 43 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 7fadd611a..a178c187e 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -77,7 +77,7 @@ func main() { log.Fatalf("couldn't get REST config: %v", err) } - c := controller.NewController(&config) + c := controller.NewController(&config, "") c.Run(stop, wg) diff --git a/docs/administrator.md b/docs/administrator.md index 9d877c783..a3a0f70cc 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -95,6 +95,34 @@ lacks access rights to any of them (except K8s system namespaces like 'list pods' execute at the cluster scope and fail at the first violation of access rights. +## Operators with defined ownership of certain Postgres clusters + +By default, multiple operators can only run together in one K8s cluster when +isolated into their [own namespaces](administrator.md#specify-the-namespace-to-watch). +But, it is also possible to define ownership between operator instances and +Postgres clusters running all in the same namespace or K8s cluster without +interfering. + +First, define the [`CONTROLLER_ID`](../../manifests/postgres-operator.yaml#L38) +environment variable in the operator deployment manifest. Then specify the ID +in every Postgres cluster manifest you want this operator to watch using the +`"acid.zalan.do/controller"` annotation: + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: postgresql +metadata: + name: demo-cluster + annotations: + "acid.zalan.do/controller": "second-operator" +spec: + ... +``` + +Every other Postgres cluster which lacks the annotation will be ignored by this +operator. Conversely, operators without a defined `CONTROLLER_ID` will ignore +clusters with defined ownership of another operator. + ## Role-based access control for the operator The manifest [`operator-service-account-rbac.yaml`](../manifests/operator-service-account-rbac.yaml) diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 5ba1fd1bc..ceb27a5c3 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -4,6 +4,8 @@ metadata: name: acid-test-cluster # labels: # environment: demo +# annotations: +# "acid.zalan.do/controller": "second-operator" spec: dockerImage: registry.opensource.zalan.do/acid/spilo-12:1.6-p2 teamId: "acid" diff --git a/manifests/postgres-operator.yaml b/manifests/postgres-operator.yaml index 63f17d9fa..4b254822c 100644 --- a/manifests/postgres-operator.yaml +++ b/manifests/postgres-operator.yaml @@ -35,3 +35,6 @@ spec: # In order to use the CRD OperatorConfiguration instead, uncomment these lines and comment out the two lines above # - name: POSTGRES_OPERATOR_CONFIGURATION_OBJECT # value: postgresql-operator-default-configuration + # Define an ID to isolate controllers from each other + # - name: CONTROLLER_ID + # value: "second-operator" diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 140d2bc4e..0ce0d026e 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -13,6 +13,7 @@ import ( "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/cache" + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/apiserver" "github.com/zalando/postgres-operator/pkg/cluster" "github.com/zalando/postgres-operator/pkg/spec" @@ -36,6 +37,7 @@ type Controller struct { stopCh chan struct{} + controllerID string curWorkerID uint32 //initialized with 0 curWorkerCluster sync.Map clusterWorkers map[spec.NamespacedName]uint32 @@ -61,13 +63,14 @@ type Controller struct { } // NewController creates a new controller -func NewController(controllerConfig *spec.ControllerConfig) *Controller { +func NewController(controllerConfig *spec.ControllerConfig, controllerId string) *Controller { logger := logrus.New() c := &Controller{ config: *controllerConfig, opConfig: &config.Config{}, logger: logger.WithField("pkg", "controller"), + controllerID: controllerId, curWorkerCluster: sync.Map{}, clusterWorkers: make(map[spec.NamespacedName]uint32), clusters: make(map[spec.NamespacedName]*cluster.Cluster), @@ -239,6 +242,7 @@ func (c *Controller) initRoleBinding() { func (c *Controller) initController() { c.initClients() + c.controllerID = os.Getenv("CONTROLLER_ID") if configObjectName := os.Getenv("POSTGRES_OPERATOR_CONFIGURATION_OBJECT"); configObjectName != "" { if err := c.createConfigurationCRD(c.opConfig.EnableCRDValidation); err != nil { @@ -412,3 +416,16 @@ func (c *Controller) getEffectiveNamespace(namespaceFromEnvironment, namespaceFr return namespace } + +// hasOwnership returns true if the controller is the "owner" of the postgresql. +// Whether it's owner is determined by the value of 'acid.zalan.do/controller' +// annotation. If the value matches the controllerID then it owns it, or if the +// controllerID is "" and there's no annotation set. +func (c *Controller) hasOwnership(postgresql *acidv1.Postgresql) bool { + if postgresql.Annotations != nil { + if owner, ok := postgresql.Annotations[constants.PostgresqlControllerAnnotationKey]; ok { + return owner == c.controllerID + } + } + return c.controllerID == "" +} diff --git a/pkg/controller/node_test.go b/pkg/controller/node_test.go index c0ec78aa8..28e178bfb 100644 --- a/pkg/controller/node_test.go +++ b/pkg/controller/node_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/zalando/postgres-operator/pkg/spec" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -13,10 +13,10 @@ const ( readyValue = "ready" ) -func initializeController() *Controller { - var c = NewController(&spec.ControllerConfig{}) - c.opConfig.NodeReadinessLabel = map[string]string{readyLabel: readyValue} - return c +func newNodeTestController() *Controller { + var controller = NewController(&spec.ControllerConfig{}, "node-test") + controller.opConfig.NodeReadinessLabel = map[string]string{readyLabel: readyValue} + return controller } func makeNode(labels map[string]string, isSchedulable bool) *v1.Node { @@ -31,7 +31,7 @@ func makeNode(labels map[string]string, isSchedulable bool) *v1.Node { } } -var c = initializeController() +var nodeTestController = newNodeTestController() func TestNodeIsReady(t *testing.T) { testName := "TestNodeIsReady" @@ -57,7 +57,7 @@ func TestNodeIsReady(t *testing.T) { }, } for _, tt := range testTable { - if isReady := c.nodeIsReady(tt.in); isReady != tt.out { + if isReady := nodeTestController.nodeIsReady(tt.in); isReady != tt.out { t.Errorf("%s: expected response %t doesn't match the actual %t for the node %#v", testName, tt.out, isReady, tt.in) } diff --git a/pkg/controller/postgresql.go b/pkg/controller/postgresql.go index 96d12bb9f..5d48bac39 100644 --- a/pkg/controller/postgresql.go +++ b/pkg/controller/postgresql.go @@ -40,12 +40,23 @@ func (c *Controller) clusterResync(stopCh <-chan struct{}, wg *sync.WaitGroup) { // clusterListFunc obtains a list of all PostgreSQL clusters func (c *Controller) listClusters(options metav1.ListOptions) (*acidv1.PostgresqlList, error) { + var pgList acidv1.PostgresqlList + // TODO: use the SharedInformer cache instead of quering Kubernetes API directly. list, err := c.KubeClient.AcidV1ClientSet.AcidV1().Postgresqls(c.opConfig.WatchedNamespace).List(options) if err != nil { c.logger.Errorf("could not list postgresql objects: %v", err) } - return list, err + if c.controllerID != "" { + c.logger.Debugf("watch only clusters with controllerID %q", c.controllerID) + } + for _, pg := range list.Items { + if pg.Error == "" && c.hasOwnership(&pg) { + pgList.Items = append(pgList.Items, pg) + } + } + + return &pgList, err } // clusterListAndSync lists all manifests and decides whether to run the sync or repair. @@ -455,41 +466,48 @@ func (c *Controller) queueClusterEvent(informerOldSpec, informerNewSpec *acidv1. } func (c *Controller) postgresqlAdd(obj interface{}) { - pg, ok := obj.(*acidv1.Postgresql) - if !ok { - c.logger.Errorf("could not cast to postgresql spec") - return + pg := c.postgresqlCheck(obj) + if pg != nil { + // We will not get multiple Add events for the same cluster + c.queueClusterEvent(nil, pg, EventAdd) } - // We will not get multiple Add events for the same cluster - c.queueClusterEvent(nil, pg, EventAdd) + return } func (c *Controller) postgresqlUpdate(prev, cur interface{}) { - pgOld, ok := prev.(*acidv1.Postgresql) - if !ok { - c.logger.Errorf("could not cast to postgresql spec") - } - pgNew, ok := cur.(*acidv1.Postgresql) - if !ok { - c.logger.Errorf("could not cast to postgresql spec") - } - // Avoid the inifinite recursion for status updates - if reflect.DeepEqual(pgOld.Spec, pgNew.Spec) { - return + pgOld := c.postgresqlCheck(prev) + pgNew := c.postgresqlCheck(cur) + if pgOld != nil && pgNew != nil { + // Avoid the inifinite recursion for status updates + if reflect.DeepEqual(pgOld.Spec, pgNew.Spec) { + return + } + c.queueClusterEvent(pgOld, pgNew, EventUpdate) } - c.queueClusterEvent(pgOld, pgNew, EventUpdate) + return } func (c *Controller) postgresqlDelete(obj interface{}) { + pg := c.postgresqlCheck(obj) + if pg != nil { + c.queueClusterEvent(pg, nil, EventDelete) + } + + return +} + +func (c *Controller) postgresqlCheck(obj interface{}) *acidv1.Postgresql { pg, ok := obj.(*acidv1.Postgresql) if !ok { c.logger.Errorf("could not cast to postgresql spec") - return + return nil } - - c.queueClusterEvent(pg, nil, EventDelete) + if !c.hasOwnership(pg) { + return nil + } + return pg } /* diff --git a/pkg/controller/postgresql_test.go b/pkg/controller/postgresql_test.go index 3d7785f92..b36519c5a 100644 --- a/pkg/controller/postgresql_test.go +++ b/pkg/controller/postgresql_test.go @@ -1,10 +1,12 @@ package controller import ( - acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" - "github.com/zalando/postgres-operator/pkg/spec" "reflect" "testing" + + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + "github.com/zalando/postgres-operator/pkg/spec" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) var ( @@ -12,9 +14,55 @@ var ( False = false ) -func TestMergeDeprecatedPostgreSQLSpecParameters(t *testing.T) { - c := NewController(&spec.ControllerConfig{}) +func newPostgresqlTestController() *Controller { + controller := NewController(&spec.ControllerConfig{}, "postgresql-test") + return controller +} +var postgresqlTestController = newPostgresqlTestController() + +func TestControllerOwnershipOnPostgresql(t *testing.T) { + tests := []struct { + name string + pg *acidv1.Postgresql + owned bool + error string + }{ + { + "Postgres cluster with defined ownership of mocked controller", + &acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{"acid.zalan.do/controller": "postgresql-test"}, + }, + }, + True, + "Postgres cluster should be owned by operator, but controller says no", + }, + { + "Postgres cluster with defined ownership of another controller", + &acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{"acid.zalan.do/controller": "stups-test"}, + }, + }, + False, + "Postgres cluster should be owned by another operator, but controller say yes", + }, + { + "Test Postgres cluster without defined ownership", + &acidv1.Postgresql{}, + False, + "Postgres cluster should be owned by operator with empty controller ID, but controller says yes", + }, + } + for _, tt := range tests { + if postgresqlTestController.hasOwnership(tt.pg) != tt.owned { + t.Errorf("%s: %v", tt.name, tt.error) + } + } +} + +func TestMergeDeprecatedPostgreSQLSpecParameters(t *testing.T) { tests := []struct { name string in *acidv1.PostgresSpec @@ -36,7 +84,7 @@ func TestMergeDeprecatedPostgreSQLSpecParameters(t *testing.T) { }, } for _, tt := range tests { - result := c.mergeDeprecatedPostgreSQLSpecParameters(tt.in) + result := postgresqlTestController.mergeDeprecatedPostgreSQLSpecParameters(tt.in) if !reflect.DeepEqual(result, tt.out) { t.Errorf("%s: %v", tt.name, tt.error) } diff --git a/pkg/controller/util_test.go b/pkg/controller/util_test.go index a5d3c7ac5..ef182248e 100644 --- a/pkg/controller/util_test.go +++ b/pkg/controller/util_test.go @@ -17,8 +17,8 @@ const ( testInfrastructureRolesSecretName = "infrastructureroles-test" ) -func newMockController() *Controller { - controller := NewController(&spec.ControllerConfig{}) +func newUtilTestController() *Controller { + controller := NewController(&spec.ControllerConfig{}, "util-test") controller.opConfig.ClusterNameLabel = "cluster-name" controller.opConfig.InfrastructureRolesSecretName = spec.NamespacedName{Namespace: v1.NamespaceDefault, Name: testInfrastructureRolesSecretName} @@ -27,7 +27,7 @@ func newMockController() *Controller { return controller } -var mockController = newMockController() +var utilTestController = newUtilTestController() func TestPodClusterName(t *testing.T) { var testTable = []struct { @@ -43,7 +43,7 @@ func TestPodClusterName(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Namespace: v1.NamespaceDefault, Labels: map[string]string{ - mockController.opConfig.ClusterNameLabel: "testcluster", + utilTestController.opConfig.ClusterNameLabel: "testcluster", }, }, }, @@ -51,7 +51,7 @@ func TestPodClusterName(t *testing.T) { }, } for _, test := range testTable { - resp := mockController.podClusterName(test.in) + resp := utilTestController.podClusterName(test.in) if resp != test.expected { t.Errorf("expected response %v does not match the actual %v", test.expected, resp) } @@ -73,7 +73,7 @@ func TestClusterWorkerID(t *testing.T) { }, } for _, test := range testTable { - resp := mockController.clusterWorkerID(test.in) + resp := utilTestController.clusterWorkerID(test.in) if resp != test.expected { t.Errorf("expected response %v does not match the actual %v", test.expected, resp) } @@ -116,7 +116,7 @@ func TestGetInfrastructureRoles(t *testing.T) { }, } for _, test := range testTable { - roles, err := mockController.getInfrastructureRoles(&test.secretName) + roles, err := utilTestController.getInfrastructureRoles(&test.secretName) if err != test.expectedError { if err != nil && test.expectedError != nil && err.Error() == test.expectedError.Error() { continue diff --git a/pkg/util/constants/annotations.go b/pkg/util/constants/annotations.go index 0b93fc2e1..fc5a84fa5 100644 --- a/pkg/util/constants/annotations.go +++ b/pkg/util/constants/annotations.go @@ -7,4 +7,5 @@ const ( ElbTimeoutAnnotationValue = "3600" KubeIAmAnnotation = "iam.amazonaws.com/role" VolumeStorateProvisionerAnnotation = "pv.kubernetes.io/provisioned-by" + PostgresqlControllerAnnotationKey = "acid.zalan.do/controller" ) From 9ddee8f3029ed086d2123d678bcb0147745edafd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20=C3=98strem?= Date: Wed, 18 Mar 2020 10:28:39 +0100 Subject: [PATCH 010/168] Use cryptographically secure password generation (#854) The current password generation algorithm is extremely deterministic, due to being based on the standard random number generator with a deterministic seed based on the current Unix timestamp (in seconds). This can lead to a number of security issues, including: The same passwords being used in different Kubernetes clusters if the operator is deployed in parallel. (This issue was discovered because of four deployments having the same generated passwords due to automatically being deployed in parallel.) The passwords being easily guessable based on the time the operator pod started when the database was created. (This would typically be present in logs, metrics, etc., that may typically be accessible to more people than should have database access.) Fix this issue by replacing the current randomness source with crypto/rand, which should produce cryptographically secure random data that is virtually unguessable. This will avoid both of the above problems as each deployment will be guaranteed to have unique, indeterministic passwords. --- pkg/util/util.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/util/util.go b/pkg/util/util.go index ad6de14a2..d9803ab48 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -2,8 +2,10 @@ package util import ( "crypto/md5" // #nosec we need it to for PostgreSQL md5 passwords + cryptoRand "crypto/rand" "encoding/hex" "fmt" + "math/big" "math/rand" "regexp" "strings" @@ -37,13 +39,17 @@ func False() *bool { return &b } -// RandomPassword generates random alphanumeric password of a given length. +// RandomPassword generates a secure, random alphanumeric password of a given length. func RandomPassword(n int) string { b := make([]byte, n) for i := range b { - b[i] = passwordChars[rand.Intn(len(passwordChars))] + maxN := big.NewInt(int64(len(passwordChars))) + if n, err := cryptoRand.Int(cryptoRand.Reader, maxN); err != nil { + panic(fmt.Errorf("Unable to generate secure, random password: %v", err)) + } else { + b[i] = passwordChars[n.Int64()] + } } - return string(b) } From 07c5da35e3572b8302f33e3dd8fbf5e8ca846229 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 18 Mar 2020 15:02:13 +0100 Subject: [PATCH 011/168] fix minor issues in docs and manifests (#866) * fix minor issues in docs and manifests * double retry_timeout_sec --- docs/reference/cluster_manifest.md | 20 ++++++++++---------- docs/reference/operator_parameters.md | 10 +++++----- docs/user.md | 8 ++++---- e2e/tests/test_e2e.py | 2 +- manifests/complete-postgres-manifest.yaml | 2 +- manifests/minimal-postgres-manifest.yaml | 2 +- manifests/standby-manifest.yaml | 2 +- ui/manifests/deployment.yaml | 2 +- 8 files changed, 24 insertions(+), 24 deletions(-) diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 92e457d7e..955622843 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -111,12 +111,12 @@ These parameters are grouped directly under the `spec` key in the manifest. value overrides the `pod_toleration` setting from the operator. Optional. * **podPriorityClassName** - a name of the [priority - class](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass) - that should be assigned to the cluster pods. When not specified, the value - is taken from the `pod_priority_class_name` operator parameter, if not set - then the default priority class is taken. The priority class itself must be - defined in advance. Optional. + a name of the [priority + class](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass) + that should be assigned to the cluster pods. When not specified, the value + is taken from the `pod_priority_class_name` operator parameter, if not set + then the default priority class is taken. The priority class itself must be + defined in advance. Optional. * **podAnnotations** A map of key value pairs that gets attached as [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) @@ -184,9 +184,9 @@ explanation of `ttl` and `loop_wait` parameters. ``` hostssl all +pamrole all pam ``` - , where pamrole is the name of the role for the pam authentication; any - custom `pg_hba` should include the pam line to avoid breaking pam - authentication. Optional. + where pamrole is the name of the role for the pam authentication; any + custom `pg_hba` should include the pam line to avoid breaking pam + authentication. Optional. * **ttl** Patroni `ttl` parameter value, optional. The default is set by the Spilo @@ -379,4 +379,4 @@ Those parameters are grouped under the `tls` top-level key. * **caFile** Optional filename to the CA certificate. Useful when the client connects - with `sslmode=verify-ca` or `sslmode=verify-full`. + with `sslmode=verify-ca` or `sslmode=verify-full`. Default is empty. diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index ba8e73cf8..86eedd33c 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -285,11 +285,11 @@ configuration they are grouped under the `kubernetes` key. capability. The default is `false`. * **master_pod_move_timeout** - The period of time to wait for the success of migration of master pods from - an unschedulable node. The migration includes Patroni switchovers to - respective replicas on healthy nodes. The situation where master pods still - exist on the old node after this timeout expires has to be fixed manually. - The default is 20 minutes. + The period of time to wait for the success of migration of master pods from + an unschedulable node. The migration includes Patroni switchovers to + respective replicas on healthy nodes. The situation where master pods still + exist on the old node after this timeout expires has to be fixed manually. + The default is 20 minutes. * **enable_pod_antiaffinity** toggles [pod anti affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) diff --git a/docs/user.md b/docs/user.md index 6e71d0404..9a6752185 100644 --- a/docs/user.md +++ b/docs/user.md @@ -30,7 +30,7 @@ spec: databases: foo: zalando postgresql: - version: "11" + version: "12" ``` Once you cloned the Postgres Operator [repository](https://github.com/zalando/postgres-operator) @@ -515,9 +515,9 @@ executed. ## Custom TLS certificates By default, the spilo image generates its own TLS certificate during startup. -This certificate is not secure since it cannot be verified and thus doesn't -protect from active MITM attacks. In this section we show how a Kubernete -Secret resources can be loaded with a custom TLS certificate. +However, this certificate cannot be verified and thus doesn't protect from +active MITM attacks. In this section we show how to specify a custom TLS +certificate which is mounted in the database pods via a K8s Secret. Before applying these changes, the operator must also be configured with the `spilo_fsgroup` set to the GID matching the postgres user group. If the value diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index f6be8a600..f0d8a0b23 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -473,7 +473,7 @@ class K8s: Wraps around K8 api client and helper methods. ''' - RETRY_TIMEOUT_SEC = 5 + RETRY_TIMEOUT_SEC = 10 def __init__(self): self.api = K8sApi() diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index ceb27a5c3..c82f1eac5 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -24,7 +24,7 @@ spec: databases: foo: zalando postgresql: - version: "11" + version: "12" parameters: # Expert section shared_buffers: "32MB" max_connections: "10" diff --git a/manifests/minimal-postgres-manifest.yaml b/manifests/minimal-postgres-manifest.yaml index 75dfdf07f..af0add8e6 100644 --- a/manifests/minimal-postgres-manifest.yaml +++ b/manifests/minimal-postgres-manifest.yaml @@ -16,4 +16,4 @@ spec: databases: foo: zalando # dbname: owner postgresql: - version: "11" + version: "12" diff --git a/manifests/standby-manifest.yaml b/manifests/standby-manifest.yaml index 2b621bd10..4c8d09650 100644 --- a/manifests/standby-manifest.yaml +++ b/manifests/standby-manifest.yaml @@ -9,7 +9,7 @@ spec: size: 1Gi numberOfInstances: 1 postgresql: - version: "11" + version: "12" # Make this a standby cluster and provide the s3 bucket path of source cluster for continuous streaming. standby: s3_wal_path: "s3://path/to/bucket/containing/wal/of/source/cluster/" diff --git a/ui/manifests/deployment.yaml b/ui/manifests/deployment.yaml index 477e4d655..6138ca1a8 100644 --- a/ui/manifests/deployment.yaml +++ b/ui/manifests/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: postgres-operator-ui containers: - name: "service" - image: registry.opensource.zalan.do/acid/postgres-operator-ui:v1.3.0 + image: registry.opensource.zalan.do/acid/postgres-operator-ui:v1.4.0 ports: - containerPort: 8081 protocol: "TCP" From cc1ffdc7b6ed83ef2d1be575db8e4aa76f8c6e57 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 25 Mar 2020 09:31:30 +0100 Subject: [PATCH 012/168] enable controllerID for chart and allow configurable pod cluster role (#876) --- charts/postgres-operator/templates/_helpers.tpl | 14 ++++++++++++++ .../templates/clusterrole-postgres-pod.yaml | 2 +- .../postgres-operator/templates/configmap.yaml | 1 + .../postgres-operator/templates/deployment.yaml | 4 ++++ .../templates/operatorconfiguration.yaml | 1 + charts/postgres-operator/values-crd.yaml | 16 ++++++++++++++-- charts/postgres-operator/values.yaml | 16 ++++++++++++++-- 7 files changed, 49 insertions(+), 5 deletions(-) diff --git a/charts/postgres-operator/templates/_helpers.tpl b/charts/postgres-operator/templates/_helpers.tpl index 306613ac3..e49670763 100644 --- a/charts/postgres-operator/templates/_helpers.tpl +++ b/charts/postgres-operator/templates/_helpers.tpl @@ -31,6 +31,20 @@ Create a service account name. {{ default (include "postgres-operator.fullname" .) .Values.serviceAccount.name }} {{- end -}} +{{/* +Create a pod service account name. +*/}} +{{- define "postgres-pod.serviceAccountName" -}} +{{ default (printf "%s-%v" (include "postgres-operator.fullname" .) "pod") .Values.podServiceAccount.name }} +{{- end -}} + +{{/* +Create a controller ID. +*/}} +{{- define "postgres-operator.controllerID" -}} +{{ default (include "postgres-operator.fullname" .) .Values.controllerID.name }} +{{- end -}} + {{/* Create chart name and version as used by the chart label. */}} diff --git a/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml b/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml index c327d9101..ef607ae3c 100644 --- a/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml +++ b/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml @@ -2,7 +2,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: postgres-pod + name: {{ include "postgres-pod.serviceAccountName" . }} labels: app.kubernetes.io/name: {{ template "postgres-operator.name" . }} helm.sh/chart: {{ template "postgres-operator.chart" . }} diff --git a/charts/postgres-operator/templates/configmap.yaml b/charts/postgres-operator/templates/configmap.yaml index 0b976294e..00ebc6676 100644 --- a/charts/postgres-operator/templates/configmap.yaml +++ b/charts/postgres-operator/templates/configmap.yaml @@ -9,6 +9,7 @@ metadata: app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/instance: {{ .Release.Name }} data: + pod_service_account_name: {{ include "postgres-pod.serviceAccountName" . }} {{ toYaml .Values.configGeneral | indent 2 }} {{ toYaml .Values.configUsers | indent 2 }} {{ toYaml .Values.configKubernetes | indent 2 }} diff --git a/charts/postgres-operator/templates/deployment.yaml b/charts/postgres-operator/templates/deployment.yaml index 1f7e39bbc..2d8eebcb3 100644 --- a/charts/postgres-operator/templates/deployment.yaml +++ b/charts/postgres-operator/templates/deployment.yaml @@ -43,6 +43,10 @@ spec: {{- else }} - name: POSTGRES_OPERATOR_CONFIGURATION_OBJECT value: {{ template "postgres-operator.fullname" . }} + {{- end }} + {{- if .Values.controllerID.create }} + - name: CONTROLLER_ID + value: {{ template "postgres-operator.controllerID" . }} {{- end }} resources: {{ toYaml .Values.resources | indent 10 }} diff --git a/charts/postgres-operator/templates/operatorconfiguration.yaml b/charts/postgres-operator/templates/operatorconfiguration.yaml index 06e9c7605..ccbbf59c6 100644 --- a/charts/postgres-operator/templates/operatorconfiguration.yaml +++ b/charts/postgres-operator/templates/operatorconfiguration.yaml @@ -13,6 +13,7 @@ configuration: users: {{ toYaml .Values.configUsers | indent 4 }} kubernetes: + pod_service_account_name: {{ include "postgres-pod.serviceAccountName" . }} oauth_token_secret_name: {{ template "postgres-operator.fullname" . }} {{ toYaml .Values.configKubernetes | indent 4 }} postgres_pod_resources: diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index d170e0b77..f9c20d8d6 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -103,8 +103,6 @@ configKubernetes: # service account definition as JSON/YAML string to be used by postgres cluster pods # pod_service_account_definition: "" - # name of service account to be used by postgres cluster pods - pod_service_account_name: "postgres-pod" # role binding definition as JSON/YAML string to be used by pod service account # pod_service_account_role_binding_definition: "" @@ -284,6 +282,11 @@ serviceAccount: # If not set and create is true, a name is generated using the fullname template name: +podServiceAccount: + # The name of the ServiceAccount to be used by postgres cluster pods + # If not set a name is generated using the fullname template and "-pod" suffix + name: "postgres-pod" + priorityClassName: "" resources: @@ -305,3 +308,12 @@ tolerations: [] # Node labels for pod assignment # Ref: https://kubernetes.io/docs/user-guide/node-selection/ nodeSelector: {} + +controllerID: + # Specifies whether a controller ID should be defined for the operator + # Note, all postgres manifest must then contain the following annotation to be found by this operator + # "acid.zalan.do/controller": + create: false + # The name of the controller ID to use. + # If not set and create is true, a name is generated using the fullname template + name: diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index b6f18f305..ea1028f23 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -96,8 +96,6 @@ configKubernetes: # service account definition as JSON/YAML string to be used by postgres cluster pods # pod_service_account_definition: "" - # name of service account to be used by postgres cluster pods - pod_service_account_name: "postgres-pod" # role binding definition as JSON/YAML string to be used by pod service account # pod_service_account_role_binding_definition: "" @@ -260,6 +258,11 @@ serviceAccount: # If not set and create is true, a name is generated using the fullname template name: +podServiceAccount: + # The name of the ServiceAccount to be used by postgres cluster pods + # If not set a name is generated using the fullname template and "-pod" suffix + name: "postgres-pod" + priorityClassName: "" resources: @@ -281,3 +284,12 @@ tolerations: [] # Node labels for pod assignment # Ref: https://kubernetes.io/docs/user-guide/node-selection/ nodeSelector: {} + +controllerID: + # Specifies whether a controller ID should be defined for the operator + # Note, all postgres manifest must then contain the following annotation to be found by this operator + # "acid.zalan.do/controller": + create: false + # The name of the controller ID to use. + # If not set and create is true, a name is generated using the fullname template + name: From 579f78864bbe67ad4068e02967c272d6c68c19a9 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 25 Mar 2020 09:59:54 +0100 Subject: [PATCH 013/168] pass cluster labels as JSON to Spilo (#877) --- pkg/cluster/k8sres.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index aaa27384a..39b5c16a0 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -549,10 +549,6 @@ func (c *Cluster) generateSpiloPodEnvVars(uid types.UID, spiloConfiguration stri Name: "KUBERNETES_ROLE_LABEL", Value: c.OpConfig.PodRoleLabel, }, - { - Name: "KUBERNETES_LABELS", - Value: labels.Set(c.OpConfig.ClusterLabels).String(), - }, { Name: "PGPASSWORD_SUPERUSER", ValueFrom: &v1.EnvVarSource{ @@ -588,6 +584,12 @@ func (c *Cluster) generateSpiloPodEnvVars(uid types.UID, spiloConfiguration stri Value: c.OpConfig.PamRoleName, }, } + // Spilo expects cluster labels as JSON + if clusterLabels, err := json.Marshal(labels.Set(c.OpConfig.ClusterLabels)); err != nil { + envVars = append(envVars, v1.EnvVar{Name: "KUBERNETES_LABELS", Value: labels.Set(c.OpConfig.ClusterLabels).String()}) + } else { + envVars = append(envVars, v1.EnvVar{Name: "KUBERNETES_LABELS", Value: string(clusterLabels)}) + } if spiloConfiguration != "" { envVars = append(envVars, v1.EnvVar{Name: "SPILO_CONFIGURATION", Value: spiloConfiguration}) } From 9dfa433363cbe997baf69a99c7046166fa966fa3 Mon Sep 17 00:00:00 2001 From: Dmitry Dolgov <9erthalion6@gmail.com> Date: Wed, 25 Mar 2020 12:57:26 +0100 Subject: [PATCH 014/168] Connection pooler (#799) Connection pooler support Add support for a connection pooler. The idea is to make it generic enough to be able to switch between different implementations (e.g. pgbouncer or odyssey). Operator needs to create a deployment with pooler and a service for it to access. For connection pool to work properly, a database needs to be prepared by operator, namely a separate user have to be created with an access to an installed lookup function (to fetch credential for other users). This setups is supposed to be used only by robot/application users. Usually a connection pool implementation is more CPU bounded, so it makes sense to create several pods for connection pool with more emphasize on cpu resources. At the moment there are no special affinity or tolerations assigned to bring those pods closer to the database. For availability purposes minimal number of connection pool pods is 2, ideally they have to be distributed between different nodes/AZ, but it's not enforced in the operator itself. Available configuration supposed to be ergonomic and in the normal case require minimum changes to a manifest to enable connection pool. To have more control over the configuration and functionality on the pool side one can customize the corresponding docker image. Co-authored-by: Felix Kunde --- .../crds/operatorconfigurations.yaml | 41 ++ .../postgres-operator/crds/postgresqls.yaml | 51 +++ .../templates/clusterrole.yaml | 1 + .../templates/configmap.yaml | 1 + .../templates/operatorconfiguration.yaml | 2 + charts/postgres-operator/values-crd.yaml | 19 + charts/postgres-operator/values.yaml | 20 + docker/DebugDockerfile | 13 +- docs/reference/cluster_manifest.md | 34 ++ docs/reference/operator_parameters.md | 37 ++ docs/user.md | 53 +++ e2e/tests/test_e2e.py | 165 +++++++- manifests/configmap.yaml | 10 + manifests/operator-service-account-rbac.yaml | 1 + manifests/operatorconfiguration.crd.yaml | 41 ++ ...gresql-operator-default-configuration.yaml | 11 + manifests/postgresql.crd.yaml | 51 +++ pkg/apis/acid.zalan.do/v1/crds.go | 124 +++++- .../v1/operator_configuration_type.go | 29 +- pkg/apis/acid.zalan.do/v1/postgresql_type.go | 26 ++ .../acid.zalan.do/v1/zz_generated.deepcopy.go | 64 +++ pkg/cluster/cluster.go | 205 +++++++++- pkg/cluster/cluster_test.go | 18 + pkg/cluster/database.go | 125 +++++- pkg/cluster/k8sres.go | 380 +++++++++++++++++- pkg/cluster/k8sres_test.go | 372 +++++++++++++++++ pkg/cluster/resources.go | 157 ++++++++ pkg/cluster/resources_test.go | 127 ++++++ pkg/cluster/sync.go | 182 ++++++++- pkg/cluster/sync_test.go | 212 ++++++++++ pkg/cluster/types.go | 4 + pkg/cluster/util.go | 39 +- pkg/controller/operator_config.go | 51 +++ pkg/spec/types.go | 6 +- pkg/util/config/config.go | 21 + pkg/util/constants/pooler.go | 18 + pkg/util/constants/roles.go | 23 +- pkg/util/k8sutil/k8sutil.go | 170 +++++++- pkg/util/util.go | 34 ++ 39 files changed, 2885 insertions(+), 53 deletions(-) create mode 100644 pkg/cluster/resources_test.go create mode 100644 pkg/cluster/sync_test.go create mode 100644 pkg/util/constants/pooler.go diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 9725c2708..7e3b607c0 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -318,6 +318,47 @@ spec: pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' scalyr_server_url: type: string + connection_pool: + type: object + properties: + connection_pool_schema: + type: string + #default: "pooler" + connection_pool_user: + type: string + #default: "pooler" + connection_pool_image: + type: string + #default: "registry.opensource.zalan.do/acid/pgbouncer" + connection_pool_max_db_connections: + type: integer + #default: 60 + connection_pool_mode: + type: string + enum: + - "session" + - "transaction" + #default: "transaction" + connection_pool_number_of_instances: + type: integer + minimum: 2 + #default: 2 + connection_pool_default_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + #default: "1" + connection_pool_default_cpu_request: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + #default: "500m" + connection_pool_default_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + #default: "100Mi" + connection_pool_default_memory_request: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + #default: "100Mi" status: type: object additionalProperties: diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index af535e2c8..a4c0e4f3a 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -106,6 +106,55 @@ spec: uid: format: uuid type: string + connectionPool: + type: object + properties: + dockerImage: + type: string + maxDBConnections: + type: integer + mode: + type: string + enum: + - "session" + - "transaction" + numberOfInstances: + type: integer + minimum: 2 + resources: + type: object + required: + - requests + - limits + properties: + limits: + type: object + required: + - cpu + - memory + properties: + cpu: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + memory: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + requests: + type: object + required: + - cpu + - memory + properties: + cpu: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + memory: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + schema: + type: string + user: + type: string databases: type: object additionalProperties: @@ -113,6 +162,8 @@ spec: # Note: usernames specified here as database owners must be declared in the users key of the spec key. dockerImage: type: string + enableConnectionPool: + type: boolean enableLogicalBackup: type: boolean enableMasterLoadBalancer: diff --git a/charts/postgres-operator/templates/clusterrole.yaml b/charts/postgres-operator/templates/clusterrole.yaml index 7b3dd462d..38ce85e7a 100644 --- a/charts/postgres-operator/templates/clusterrole.yaml +++ b/charts/postgres-operator/templates/clusterrole.yaml @@ -128,6 +128,7 @@ rules: - apps resources: - statefulsets + - deployments verbs: - create - delete diff --git a/charts/postgres-operator/templates/configmap.yaml b/charts/postgres-operator/templates/configmap.yaml index 00ebc6676..e8a805db7 100644 --- a/charts/postgres-operator/templates/configmap.yaml +++ b/charts/postgres-operator/templates/configmap.yaml @@ -20,4 +20,5 @@ data: {{ toYaml .Values.configDebug | indent 2 }} {{ toYaml .Values.configLoggingRestApi | indent 2 }} {{ toYaml .Values.configTeamsApi | indent 2 }} +{{ toYaml .Values.configConnectionPool | indent 2 }} {{- end }} diff --git a/charts/postgres-operator/templates/operatorconfiguration.yaml b/charts/postgres-operator/templates/operatorconfiguration.yaml index ccbbf59c6..b52b3d664 100644 --- a/charts/postgres-operator/templates/operatorconfiguration.yaml +++ b/charts/postgres-operator/templates/operatorconfiguration.yaml @@ -34,4 +34,6 @@ configuration: {{ toYaml .Values.configLoggingRestApi | indent 4 }} scalyr: {{ toYaml .Values.configScalyr | indent 4 }} + connection_pool: +{{ toYaml .Values.configConnectionPool | indent 4 }} {{- end }} diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index f9c20d8d6..cf9fbab15 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -267,6 +267,25 @@ configScalyr: # Memory request value for the Scalyr sidecar scalyr_memory_request: 50Mi +configConnectionPool: + # db schema to install lookup function into + connection_pool_schema: "pooler" + # db user for pooler to use + connection_pool_user: "pooler" + # docker image + connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer" + # max db connections the pooler should hold + connection_pool_max_db_connections: 60 + # default pooling mode + connection_pool_mode: "transaction" + # number of pooler instances + connection_pool_number_of_instances: 2 + # default resources + connection_pool_default_cpu_request: 500m + connection_pool_default_memory_request: 100Mi + connection_pool_default_cpu_limit: "1" + connection_pool_default_memory_limit: 100Mi + rbac: # Specifies whether RBAC resources should be created create: true diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index ea1028f23..503bf4562 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -243,6 +243,26 @@ configTeamsApi: # URL of the Teams API service # teams_api_url: http://fake-teams-api.default.svc.cluster.local +# configure connection pooler deployment created by the operator +configConnectionPool: + # db schema to install lookup function into + connection_pool_schema: "pooler" + # db user for pooler to use + connection_pool_user: "pooler" + # docker image + connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer" + # max db connections the pooler should hold + connection_pool_max_db_connections: 60 + # default pooling mode + connection_pool_mode: "transaction" + # number of pooler instances + connection_pool_number_of_instances: 2 + # default resources + connection_pool_default_cpu_request: 500m + connection_pool_default_memory_request: 100Mi + connection_pool_default_cpu_limit: "1" + connection_pool_default_memory_limit: 100Mi + rbac: # Specifies whether RBAC resources should be created create: true diff --git a/docker/DebugDockerfile b/docker/DebugDockerfile index 76dadf6df..0c11fe3b4 100644 --- a/docker/DebugDockerfile +++ b/docker/DebugDockerfile @@ -3,8 +3,17 @@ MAINTAINER Team ACID @ Zalando # We need root certificates to deal with teams api over https RUN apk --no-cache add ca-certificates go git musl-dev -RUN go get github.com/derekparker/delve/cmd/dlv COPY build/* / -CMD ["/root/go/bin/dlv", "--listen=:7777", "--headless=true", "--api-version=2", "exec", "/postgres-operator"] +RUN addgroup -g 1000 pgo +RUN adduser -D -u 1000 -G pgo -g 'Postgres Operator' pgo + +RUN go get github.com/derekparker/delve/cmd/dlv +RUN cp /root/go/bin/dlv /dlv +RUN chown -R pgo:pgo /dlv + +USER pgo:pgo +RUN ls -l / + +CMD ["/dlv", "--listen=:7777", "--headless=true", "--api-version=2", "exec", "/postgres-operator"] diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 955622843..4400cb666 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -140,6 +140,11 @@ These parameters are grouped directly under the `spec` key in the manifest. is `false`, then no volume will be mounted no matter how operator was configured (so you can override the operator configuration). Optional. +* **enableConnectionPool** + Tells the operator to create a connection pool with a database. If this + field is true, a connection pool deployment will be created even if + `connectionPool` section is empty. Optional, not set by default. + * **enableLogicalBackup** Determines if the logical backup of this cluster should be taken and uploaded to S3. Default: false. Optional. @@ -360,6 +365,35 @@ CPU and memory limits for the sidecar container. memory limits for the sidecar container. Optional, overrides the `default_memory_limits` operator configuration parameter. Optional. +## Connection pool + +Parameters are grouped under the `connectionPool` top-level key and specify +configuration for connection pool. If this section is not empty, a connection +pool will be created for a database even if `enableConnectionPool` is not +present. + +* **numberOfInstances** + How many instances of connection pool to create. + +* **schema** + Schema to create for credentials lookup function. + +* **user** + User to create for connection pool to be able to connect to a database. + +* **dockerImage** + Which docker image to use for connection pool deployment. + +* **maxDBConnections** + How many connections the pooler can max hold. This value is divided among the + pooler pods. + +* **mode** + In which mode to run connection pool, transaction or session. + +* **resources** + Resource configuration for connection pool deployment. + ## Custom TLS certificates Those parameters are grouped under the `tls` top-level key. diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 86eedd33c..848fa1cf2 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -595,3 +595,40 @@ scalyr sidecar. In the CRD-based configuration they are grouped under the * **scalyr_memory_limit** Memory limit value for the Scalyr sidecar. The default is `500Mi`. + +## Connection pool configuration + +Parameters are grouped under the `connection_pool` top-level key and specify +default configuration for connection pool, if a postgres manifest requests it +but do not specify some of the parameters. All of them are optional with the +operator being able to provide some reasonable defaults. + +* **connection_pool_number_of_instances** + How many instances of connection pool to create. Default is 2 which is also + the required minimum. + +* **connection_pool_schema** + Schema to create for credentials lookup function. Default is `pooler`. + +* **connection_pool_user** + User to create for connection pool to be able to connect to a database. + Default is `pooler`. + +* **connection_pool_image** + Docker image to use for connection pool deployment. + Default: "registry.opensource.zalan.do/acid/pgbouncer" + +* **connection_pool_max_db_connections** + How many connections the pooler can max hold. This value is divided among the + pooler pods. Default is 60 which will make up 30 connections per pod for the + default setup with two instances. + +* **connection_pool_mode** + Default pool mode, `session` or `transaction`. Default is `transaction`. + +* **connection_pool_default_cpu_request** + **connection_pool_default_memory_reques** + **connection_pool_default_cpu_limit** + **connection_pool_default_memory_limit** + Default resource configuration for connection pool deployment. The internal + default for memory request and limit is `100Mi`, for CPU it is `500m` and `1`. diff --git a/docs/user.md b/docs/user.md index 9a6752185..8c79bb485 100644 --- a/docs/user.md +++ b/docs/user.md @@ -512,6 +512,59 @@ monitoring is outside the scope of operator responsibilities. See [administrator documentation](administrator.md) for details on how backups are executed. +## Connection pool + +The operator can create a database side connection pool for those applications, +where an application side pool is not feasible, but a number of connections is +high. To create a connection pool together with a database, modify the +manifest: + +```yaml +spec: + enableConnectionPool: true +``` + +This will tell the operator to create a connection pool with default +configuration, through which one can access the master via a separate service +`{cluster-name}-pooler`. In most of the cases provided default configuration +should be good enough. + +To configure a new connection pool, specify: + +``` +spec: + connectionPool: + # how many instances of connection pool to create + number_of_instances: 2 + + # in which mode to run, session or transaction + mode: "transaction" + + # schema, which operator will create to install credentials lookup + # function + schema: "pooler" + + # user, which operator will create for connection pool + user: "pooler" + + # resources for each instance + resources: + requests: + cpu: 500m + memory: 100Mi + limits: + cpu: "1" + memory: 100Mi +``` + +By default `pgbouncer` is used to create a connection pool. To find out about +pool modes see [docs](https://www.pgbouncer.org/config.html#pool_mode) (but it +should be general approach between different implementation). + +Note, that using `pgbouncer` means meaningful resource CPU limit should be less +than 1 core (there is a way to utilize more than one, but in K8S it's easier +just to spin up more instances). + ## Custom TLS certificates By default, the spilo image generates its own TLS certificate during startup. diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index f0d8a0b23..cc90aa5e2 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -9,6 +9,10 @@ import yaml from kubernetes import client, config +def to_selector(labels): + return ",".join(["=".join(l) for l in labels.items()]) + + class EndToEndTestCase(unittest.TestCase): ''' Test interaction of the operator with multiple K8s components. @@ -47,7 +51,8 @@ class EndToEndTestCase(unittest.TestCase): for filename in ["operator-service-account-rbac.yaml", "configmap.yaml", "postgres-operator.yaml"]: - k8s.create_with_kubectl("manifests/" + filename) + result = k8s.create_with_kubectl("manifests/" + filename) + print("stdout: {}, stderr: {}".format(result.stdout, result.stderr)) k8s.wait_for_operator_pod_start() @@ -55,9 +60,14 @@ class EndToEndTestCase(unittest.TestCase): 'default', label_selector='name=postgres-operator').items[0].spec.containers[0].image print("Tested operator image: {}".format(actual_operator_image)) # shows up after tests finish - k8s.create_with_kubectl("manifests/minimal-postgres-manifest.yaml") - k8s.wait_for_pod_start('spilo-role=master') - k8s.wait_for_pod_start('spilo-role=replica') + result = k8s.create_with_kubectl("manifests/minimal-postgres-manifest.yaml") + print('stdout: {}, stderr: {}'.format(result.stdout, result.stderr)) + try: + k8s.wait_for_pod_start('spilo-role=master') + k8s.wait_for_pod_start('spilo-role=replica') + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_enable_load_balancer(self): @@ -66,7 +76,7 @@ class EndToEndTestCase(unittest.TestCase): ''' k8s = self.k8s - cluster_label = 'cluster-name=acid-minimal-cluster' + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' # enable load balancer services pg_patch_enable_lbs = { @@ -178,7 +188,7 @@ class EndToEndTestCase(unittest.TestCase): Lower resource limits below configured minimum and let operator fix it ''' k8s = self.k8s - cluster_label = 'cluster-name=acid-minimal-cluster' + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' labels = 'spilo-role=master,' + cluster_label _, failover_targets = k8s.get_pg_nodes(cluster_label) @@ -247,7 +257,7 @@ class EndToEndTestCase(unittest.TestCase): Remove node readiness label from master node. This must cause a failover. ''' k8s = self.k8s - cluster_label = 'cluster-name=acid-minimal-cluster' + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' labels = 'spilo-role=master,' + cluster_label readiness_label = 'lifecycle-status' readiness_value = 'ready' @@ -289,7 +299,7 @@ class EndToEndTestCase(unittest.TestCase): Scale up from 2 to 3 and back to 2 pods by updating the Postgres manifest at runtime. ''' k8s = self.k8s - labels = "cluster-name=acid-minimal-cluster" + labels = "application=spilo,cluster-name=acid-minimal-cluster" k8s.wait_for_pg_to_scale(3) self.assertEqual(3, k8s.count_pods_with_label(labels)) @@ -339,13 +349,99 @@ class EndToEndTestCase(unittest.TestCase): } k8s.update_config(unpatch_custom_service_annotations) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_enable_disable_connection_pool(self): + ''' + For a database without connection pool, then turns it on, scale up, + turn off and on again. Test with different ways of doing this (via + enableConnectionPool or connectionPool configuration section). At the + end turn the connection pool off to not interfere with other tests. + ''' + k8s = self.k8s + service_labels = { + 'cluster-name': 'acid-minimal-cluster', + } + pod_labels = dict({ + 'connection-pool': 'acid-minimal-cluster-pooler', + }) + + pod_selector = to_selector(pod_labels) + service_selector = to_selector(service_labels) + + try: + # enable connection pool + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'enableConnectionPool': True, + } + }) + k8s.wait_for_pod_start(pod_selector) + + pods = k8s.api.core_v1.list_namespaced_pod( + 'default', label_selector=pod_selector + ).items + + self.assertTrue(pods, 'No connection pool pods') + + k8s.wait_for_service(service_selector) + services = k8s.api.core_v1.list_namespaced_service( + 'default', label_selector=service_selector + ).items + services = [ + s for s in services + if s.metadata.name.endswith('pooler') + ] + + self.assertTrue(services, 'No connection pool service') + + # scale up connection pool deployment + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'connectionPool': { + 'numberOfInstances': 2, + }, + } + }) + + k8s.wait_for_running_pods(pod_selector, 2) + + # turn it off, keeping configuration section + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'enableConnectionPool': False, + } + }) + k8s.wait_for_pods_to_stop(pod_selector) + + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'enableConnectionPool': True, + } + }) + k8s.wait_for_pod_start(pod_selector) + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_taint_based_eviction(self): ''' Add taint "postgres=:NoExecute" to node with master. This must cause a failover. ''' k8s = self.k8s - cluster_label = 'cluster-name=acid-minimal-cluster' + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' # get nodes of master and replica(s) (expected target of new master) current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label) @@ -496,12 +592,39 @@ class K8s: # for local execution ~ 10 seconds suffices time.sleep(60) + def get_operator_pod(self): + pods = self.api.core_v1.list_namespaced_pod( + 'default', label_selector='name=postgres-operator' + ).items + + if pods: + return pods[0] + + return None + + def get_operator_log(self): + operator_pod = self.get_operator_pod() + pod_name = operator_pod.metadata.name + return self.api.core_v1.read_namespaced_pod_log( + name=pod_name, + namespace='default' + ) + def wait_for_pod_start(self, pod_labels, namespace='default'): pod_phase = 'No pod running' while pod_phase != 'Running': pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=pod_labels).items if pods: pod_phase = pods[0].status.phase + + if pods and pod_phase != 'Running': + pod_name = pods[0].metadata.name + response = self.api.core_v1.read_namespaced_pod( + name=pod_name, + namespace=namespace + ) + print("Pod description {}".format(response)) + time.sleep(self.RETRY_TIMEOUT_SEC) def get_service_type(self, svc_labels, namespace='default'): @@ -531,10 +654,27 @@ class K8s: _ = self.api.custom_objects_api.patch_namespaced_custom_object( "acid.zalan.do", "v1", namespace, "postgresqls", "acid-minimal-cluster", body) - labels = 'cluster-name=acid-minimal-cluster' + labels = 'application=spilo,cluster-name=acid-minimal-cluster' while self.count_pods_with_label(labels) != number_of_instances: time.sleep(self.RETRY_TIMEOUT_SEC) + def wait_for_running_pods(self, labels, number, namespace=''): + while self.count_pods_with_label(labels) != number: + time.sleep(self.RETRY_TIMEOUT_SEC) + + def wait_for_pods_to_stop(self, labels, namespace=''): + while self.count_pods_with_label(labels) != 0: + time.sleep(self.RETRY_TIMEOUT_SEC) + + def wait_for_service(self, labels, namespace='default'): + def get_services(): + return self.api.core_v1.list_namespaced_service( + namespace, label_selector=labels + ).items + + while not get_services(): + time.sleep(self.RETRY_TIMEOUT_SEC) + def count_pods_with_label(self, labels, namespace='default'): return len(self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items) @@ -571,7 +711,10 @@ class K8s: self.wait_for_operator_pod_start() def create_with_kubectl(self, path): - subprocess.run(["kubectl", "create", "-f", path]) + return subprocess.run( + ["kubectl", "create", "-f", path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) if __name__ == '__main__': diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 0300b5495..fdc2d5d56 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -11,6 +11,16 @@ data: cluster_history_entries: "1000" cluster_labels: application:spilo cluster_name_label: cluster-name + # connection_pool_default_cpu_limit: "1" + # connection_pool_default_cpu_request: "500m" + # connection_pool_default_memory_limit: 100Mi + # connection_pool_default_memory_request: 100Mi + connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer:master-5" + # connection_pool_max_db_connections: 60 + # connection_pool_mode: "transaction" + # connection_pool_number_of_instances: 2 + # connection_pool_schema: "pooler" + # connection_pool_user: "pooler" # custom_service_annotations: "keyx:valuez,keya:valuea" # custom_pod_annotations: "keya:valuea,keyb:valueb" db_hosted_zone: db.example.com diff --git a/manifests/operator-service-account-rbac.yaml b/manifests/operator-service-account-rbac.yaml index e5bc49f83..83cd721e7 100644 --- a/manifests/operator-service-account-rbac.yaml +++ b/manifests/operator-service-account-rbac.yaml @@ -129,6 +129,7 @@ rules: - apps resources: - statefulsets + - deployments verbs: - create - delete diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 7bd5c529c..4e6858af8 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -294,6 +294,47 @@ spec: pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' scalyr_server_url: type: string + connection_pool: + type: object + properties: + connection_pool_schema: + type: string + #default: "pooler" + connection_pool_user: + type: string + #default: "pooler" + connection_pool_image: + type: string + #default: "registry.opensource.zalan.do/acid/pgbouncer" + connection_pool_max_db_connections: + type: integer + #default: 60 + connection_pool_mode: + type: string + enum: + - "session" + - "transaction" + #default: "transaction" + connection_pool_number_of_instances: + type: integer + minimum: 2 + #default: 2 + connection_pool_default_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + #default: "1" + connection_pool_default_cpu_request: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + #default: "500m" + connection_pool_default_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + #default: "100Mi" + connection_pool_default_memory_request: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + #default: "100Mi" status: type: object additionalProperties: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 33838b2a9..d4c9b518f 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -121,3 +121,14 @@ configuration: scalyr_memory_limit: 500Mi scalyr_memory_request: 50Mi # scalyr_server_url: "" + connection_pool: + connection_pool_default_cpu_limit: "1" + connection_pool_default_cpu_request: "500m" + connection_pool_default_memory_limit: 100Mi + connection_pool_default_memory_request: 100Mi + connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer:master-5" + # connection_pool_max_db_connections: 60 + connection_pool_mode: "transaction" + connection_pool_number_of_instances: 2 + # connection_pool_schema: "pooler" + # connection_pool_user: "pooler" diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 04c789fb9..06434da14 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -70,6 +70,55 @@ spec: uid: format: uuid type: string + connectionPool: + type: object + properties: + dockerImage: + type: string + maxDBConnections: + type: integer + mode: + type: string + enum: + - "session" + - "transaction" + numberOfInstances: + type: integer + minimum: 2 + resources: + type: object + required: + - requests + - limits + properties: + limits: + type: object + required: + - cpu + - memory + properties: + cpu: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + memory: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + requests: + type: object + required: + - cpu + - memory + properties: + cpu: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + memory: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + schema: + type: string + user: + type: string databases: type: object additionalProperties: @@ -77,6 +126,8 @@ spec: # Note: usernames specified here as database owners must be declared in the users key of the spec key. dockerImage: type: string + enableConnectionPool: + type: boolean enableLogicalBackup: type: boolean enableMasterLoadBalancer: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index eff47c255..dc552d3f4 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -105,6 +105,7 @@ var OperatorConfigCRDResourceColumns = []apiextv1beta1.CustomResourceColumnDefin var min0 = 0.0 var min1 = 1.0 +var min2 = 2.0 var minDisable = -1.0 // PostgresCRDResourceValidation to check applied manifest parameters @@ -176,6 +177,76 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, }, }, + "connectionPool": { + Type: "object", + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "dockerImage": { + Type: "string", + }, + "maxDBConnections": { + Type: "integer", + }, + "mode": { + Type: "string", + Enum: []apiextv1beta1.JSON{ + { + Raw: []byte(`"session"`), + }, + { + Raw: []byte(`"transaction"`), + }, + }, + }, + "numberOfInstances": { + Type: "integer", + Minimum: &min2, + }, + "resources": { + Type: "object", + Required: []string{"requests", "limits"}, + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "limits": { + Type: "object", + Required: []string{"cpu", "memory"}, + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "cpu": { + Type: "string", + Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0", + Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", + }, + "memory": { + Type: "string", + Description: "Plain integer or fixed-point integer using one of these suffixes: E, P, T, G, M, k (with or without a tailing i). Must be greater than 0", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + }, + }, + }, + "requests": { + Type: "object", + Required: []string{"cpu", "memory"}, + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "cpu": { + Type: "string", + Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0", + Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", + }, + "memory": { + Type: "string", + Description: "Plain integer or fixed-point integer using one of these suffixes: E, P, T, G, M, k (with or without a tailing i). Must be greater than 0", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + }, + }, + }, + }, + }, + "schema": { + Type: "string", + }, + "user": { + Type: "string", + }, + }, + }, "databases": { Type: "object", AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ @@ -188,6 +259,9 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ "dockerImage": { Type: "string", }, + "enableConnectionPool": { + Type: "boolean", + }, "enableLogicalBackup": { Type: "boolean", }, @@ -418,7 +492,7 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ Type: "string", }, "tls": { - Type: "object", + Type: "object", Required: []string{"secretName"}, Properties: map[string]apiextv1beta1.JSONSchemaProps{ "secretName": { @@ -1055,6 +1129,54 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, }, }, + "connection_pool": { + Type: "object", + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "connection_pool_default_cpu_limit": { + Type: "string", + Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", + }, + "connection_pool_default_cpu_request": { + Type: "string", + Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", + }, + "connection_pool_default_memory_limit": { + Type: "string", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + }, + "connection_pool_default_memory_request": { + Type: "string", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + }, + "connection_pool_image": { + Type: "string", + }, + "connection_pool_max_db_connections": { + Type: "integer", + }, + "connection_pool_mode": { + Type: "string", + Enum: []apiextv1beta1.JSON{ + { + Raw: []byte(`"session"`), + }, + { + Raw: []byte(`"transaction"`), + }, + }, + }, + "connection_pool_number_of_instances": { + Type: "integer", + Minimum: &min2, + }, + "connection_pool_schema": { + Type: "string", + }, + "connection_pool_user": { + Type: "string", + }, + }, + }, }, }, "status": { diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 8463be6bd..7c1d2b7e8 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -1,5 +1,7 @@ package v1 +// Operator configuration CRD definition, please use snake_case for field names. + import ( "github.com/zalando/postgres-operator/pkg/util/config" @@ -65,12 +67,12 @@ type KubernetesMetaConfiguration struct { // TODO: use a proper toleration structure? PodToleration map[string]string `json:"toleration,omitempty"` // TODO: use namespacedname - PodEnvironmentConfigMap string `json:"pod_environment_configmap,omitempty"` - PodPriorityClassName string `json:"pod_priority_class_name,omitempty"` - MasterPodMoveTimeout Duration `json:"master_pod_move_timeout,omitempty"` - EnablePodAntiAffinity bool `json:"enable_pod_antiaffinity,omitempty"` - PodAntiAffinityTopologyKey string `json:"pod_antiaffinity_topology_key,omitempty"` - PodManagementPolicy string `json:"pod_management_policy,omitempty"` + PodEnvironmentConfigMap string `json:"pod_environment_configmap,omitempty"` + PodPriorityClassName string `json:"pod_priority_class_name,omitempty"` + MasterPodMoveTimeout Duration `json:"master_pod_move_timeout,omitempty"` + EnablePodAntiAffinity bool `json:"enable_pod_antiaffinity,omitempty"` + PodAntiAffinityTopologyKey string `json:"pod_antiaffinity_topology_key,omitempty"` + PodManagementPolicy string `json:"pod_management_policy,omitempty"` } // PostgresPodResourcesDefaults defines the spec of default resources @@ -152,6 +154,20 @@ type ScalyrConfiguration struct { ScalyrMemoryLimit string `json:"scalyr_memory_limit,omitempty"` } +// Defines default configuration for connection pool +type ConnectionPoolConfiguration struct { + NumberOfInstances *int32 `json:"connection_pool_number_of_instances,omitempty"` + Schema string `json:"connection_pool_schema,omitempty"` + User string `json:"connection_pool_user,omitempty"` + Image string `json:"connection_pool_image,omitempty"` + Mode string `json:"connection_pool_mode,omitempty"` + MaxDBConnections *int32 `json:"connection_pool_max_db_connections,omitempty"` + DefaultCPURequest string `json:"connection_pool_default_cpu_request,omitempty"` + DefaultMemoryRequest string `json:"connection_pool_default_memory_request,omitempty"` + DefaultCPULimit string `json:"connection_pool_default_cpu_limit,omitempty"` + DefaultMemoryLimit string `json:"connection_pool_default_memory_limit,omitempty"` +} + // OperatorLogicalBackupConfiguration defines configuration for logical backup type OperatorLogicalBackupConfiguration struct { Schedule string `json:"logical_backup_schedule,omitempty"` @@ -188,6 +204,7 @@ type OperatorConfigurationData struct { LoggingRESTAPI LoggingRESTAPIConfiguration `json:"logging_rest_api"` Scalyr ScalyrConfiguration `json:"scalyr"` LogicalBackup OperatorLogicalBackupConfiguration `json:"logical_backup"` + ConnectionPool ConnectionPoolConfiguration `json:"connection_pool"` } //Duration shortens this frequently used name diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 862db6a4e..1784f8235 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -1,5 +1,7 @@ package v1 +// Postgres CRD definition, please use CamelCase for field names. + import ( "time" @@ -27,6 +29,9 @@ type PostgresSpec struct { Patroni `json:"patroni,omitempty"` Resources `json:"resources,omitempty"` + EnableConnectionPool *bool `json:"enableConnectionPool,omitempty"` + ConnectionPool *ConnectionPool `json:"connectionPool,omitempty"` + TeamID string `json:"teamId"` DockerImage string `json:"dockerImage,omitempty"` @@ -162,3 +167,24 @@ type UserFlags []string type PostgresStatus struct { PostgresClusterStatus string `json:"PostgresClusterStatus"` } + +// Options for connection pooler +// +// TODO: prepared snippets of configuration, one can choose via type, e.g. +// pgbouncer-large (with higher resources) or odyssey-small (with smaller +// resources) +// Type string `json:"type,omitempty"` +// +// TODO: figure out what other important parameters of the connection pool it +// makes sense to expose. E.g. pool size (min/max boundaries), max client +// connections etc. +type ConnectionPool struct { + NumberOfInstances *int32 `json:"numberOfInstances,omitempty"` + Schema string `json:"schema,omitempty"` + User string `json:"user,omitempty"` + Mode string `json:"mode,omitempty"` + DockerImage string `json:"dockerImage,omitempty"` + MaxDBConnections *int32 `json:"maxDBConnections,omitempty"` + + Resources `json:"resources,omitempty"` +} diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 753b0490c..fcab394ca 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -68,6 +68,59 @@ func (in *CloneDescription) DeepCopy() *CloneDescription { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConnectionPool) DeepCopyInto(out *ConnectionPool) { + *out = *in + if in.NumberOfInstances != nil { + in, out := &in.NumberOfInstances, &out.NumberOfInstances + *out = new(int32) + **out = **in + } + if in.MaxDBConnections != nil { + in, out := &in.MaxDBConnections, &out.MaxDBConnections + *out = new(int32) + **out = **in + } + out.Resources = in.Resources + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectionPool. +func (in *ConnectionPool) DeepCopy() *ConnectionPool { + if in == nil { + return nil + } + out := new(ConnectionPool) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConnectionPoolConfiguration) DeepCopyInto(out *ConnectionPoolConfiguration) { + *out = *in + if in.NumberOfInstances != nil { + in, out := &in.NumberOfInstances, &out.NumberOfInstances + *out = new(int32) + **out = **in + } + if in.MaxDBConnections != nil { + in, out := &in.MaxDBConnections, &out.MaxDBConnections + *out = new(int32) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectionPoolConfiguration. +func (in *ConnectionPoolConfiguration) DeepCopy() *ConnectionPoolConfiguration { + if in == nil { + return nil + } + out := new(ConnectionPoolConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfiguration) { *out = *in @@ -254,6 +307,7 @@ func (in *OperatorConfigurationData) DeepCopyInto(out *OperatorConfigurationData out.LoggingRESTAPI = in.LoggingRESTAPI out.Scalyr = in.Scalyr out.LogicalBackup = in.LogicalBackup + in.ConnectionPool.DeepCopyInto(&out.ConnectionPool) return } @@ -416,6 +470,16 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { out.Volume = in.Volume in.Patroni.DeepCopyInto(&out.Patroni) out.Resources = in.Resources + if in.EnableConnectionPool != nil { + in, out := &in.EnableConnectionPool, &out.EnableConnectionPool + *out = new(bool) + **out = **in + } + if in.ConnectionPool != nil { + in, out := &in.ConnectionPool, &out.ConnectionPool + *out = new(ConnectionPool) + (*in).DeepCopyInto(*out) + } if in.SpiloFSGroup != nil { in, out := &in.SpiloFSGroup, &out.SpiloFSGroup *out = new(int64) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index d740260d2..dba67c142 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -11,6 +11,7 @@ import ( "sync" "time" + "github.com/r3labs/diff" "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -48,11 +49,26 @@ type Config struct { PodServiceAccountRoleBinding *rbacv1.RoleBinding } +// K8S objects that are belongs to a connection pool +type ConnectionPoolObjects struct { + Deployment *appsv1.Deployment + Service *v1.Service + + // It could happen that a connection pool was enabled, but the operator was + // not able to properly process a corresponding event or was restarted. In + // this case we will miss missing/require situation and a lookup function + // will not be installed. To avoid synchronizing it all the time to prevent + // this, we can remember the result in memory at least until the next + // restart. + LookupFunction bool +} + type kubeResources struct { Services map[PostgresRole]*v1.Service Endpoints map[PostgresRole]*v1.Endpoints Secrets map[types.UID]*v1.Secret Statefulset *appsv1.StatefulSet + ConnectionPool *ConnectionPoolObjects PodDisruptionBudget *policybeta1.PodDisruptionBudget //Pods are treated separately //PVCs are treated separately @@ -184,7 +200,8 @@ func (c *Cluster) isNewCluster() bool { func (c *Cluster) initUsers() error { c.setProcessName("initializing users") - // clear our the previous state of the cluster users (in case we are running a sync). + // clear our the previous state of the cluster users (in case we are + // running a sync). c.systemUsers = map[string]spec.PgUser{} c.pgUsers = map[string]spec.PgUser{} @@ -292,8 +309,10 @@ func (c *Cluster) Create() error { } c.logger.Infof("pods are ready") - // create database objects unless we are running without pods or disabled that feature explicitly + // create database objects unless we are running without pods or disabled + // that feature explicitly if !(c.databaseAccessDisabled() || c.getNumberOfInstances(&c.Spec) <= 0 || c.Spec.StandbyCluster != nil) { + c.logger.Infof("Create roles") if err = c.createRoles(); err != nil { return fmt.Errorf("could not create users: %v", err) } @@ -316,6 +335,26 @@ func (c *Cluster) Create() error { c.logger.Errorf("could not list resources: %v", err) } + // Create connection pool deployment and services if necessary. Since we + // need to peform some operations with the database itself (e.g. install + // lookup function), do it as the last step, when everything is available. + // + // Do not consider connection pool as a strict requirement, and if + // something fails, report warning + if c.needConnectionPool() { + if c.ConnectionPool != nil { + c.logger.Warning("Connection pool already exists in the cluster") + return nil + } + connPool, err := c.createConnectionPool(c.installLookupFunction) + if err != nil { + c.logger.Warningf("could not create connection pool: %v", err) + return nil + } + c.logger.Infof("connection pool %q has been successfully created", + util.NameFromMeta(connPool.Deployment.ObjectMeta)) + } + return nil } @@ -571,7 +610,11 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { } } - if !reflect.DeepEqual(oldSpec.Spec.Users, newSpec.Spec.Users) { + // connection pool needs one system user created, which is done in + // initUsers. Check if it needs to be called. + sameUsers := reflect.DeepEqual(oldSpec.Spec.Users, newSpec.Spec.Users) + needConnPool := c.needConnectionPoolWorker(&newSpec.Spec) + if !sameUsers || needConnPool { c.logger.Debugf("syncing secrets") if err := c.initUsers(); err != nil { c.logger.Errorf("could not init users: %v", err) @@ -695,6 +738,11 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { } } + // sync connection pool + if err := c.syncConnectionPool(oldSpec, newSpec, c.installLookupFunction); err != nil { + return fmt.Errorf("could not sync connection pool: %v", err) + } + return nil } @@ -746,6 +794,12 @@ func (c *Cluster) Delete() { c.logger.Warningf("could not remove leftover patroni objects; %v", err) } + // Delete connection pool objects anyway, even if it's not mentioned in the + // manifest, just to not keep orphaned components in case if something went + // wrong + if err := c.deleteConnectionPool(); err != nil { + c.logger.Warningf("could not remove connection pool: %v", err) + } } //NeedsRepair returns true if the cluster should be included in the repair scan (based on its in-memory status). @@ -812,6 +866,35 @@ func (c *Cluster) initSystemUsers() { Name: c.OpConfig.ReplicationUsername, Password: util.RandomPassword(constants.PasswordLength), } + + // Connection pool user is an exception, if requested it's going to be + // created by operator as a normal pgUser + if c.needConnectionPool() { + // initialize empty connection pool if not done yet + if c.Spec.ConnectionPool == nil { + c.Spec.ConnectionPool = &acidv1.ConnectionPool{} + } + + username := util.Coalesce( + c.Spec.ConnectionPool.User, + c.OpConfig.ConnectionPool.User) + + // connection pooler application should be able to login with this role + connPoolUser := spec.PgUser{ + Origin: spec.RoleConnectionPool, + Name: username, + Flags: []string{constants.RoleFlagLogin}, + Password: util.RandomPassword(constants.PasswordLength), + } + + if _, exists := c.pgUsers[username]; !exists { + c.pgUsers[username] = connPoolUser + } + + if _, exists := c.systemUsers[constants.ConnectionPoolUserKeyName]; !exists { + c.systemUsers[constants.ConnectionPoolUserKeyName] = connPoolUser + } + } } func (c *Cluster) initRobotUsers() error { @@ -1138,3 +1221,119 @@ func (c *Cluster) deletePatroniClusterConfigMaps() error { return c.deleteClusterObject(get, deleteConfigMapFn, "configmap") } + +// Test if two connection pool configuration needs to be synced. For simplicity +// compare not the actual K8S objects, but the configuration itself and request +// sync if there is any difference. +func (c *Cluster) needSyncConnPoolSpecs(oldSpec, newSpec *acidv1.ConnectionPool) (sync bool, reasons []string) { + reasons = []string{} + sync = false + + changelog, err := diff.Diff(oldSpec, newSpec) + if err != nil { + c.logger.Infof("Cannot get diff, do not do anything, %+v", err) + return false, reasons + } + + if len(changelog) > 0 { + sync = true + } + + for _, change := range changelog { + msg := fmt.Sprintf("%s %+v from '%+v' to '%+v'", + change.Type, change.Path, change.From, change.To) + reasons = append(reasons, msg) + } + + return sync, reasons +} + +func syncResources(a, b *v1.ResourceRequirements) bool { + for _, res := range []v1.ResourceName{ + v1.ResourceCPU, + v1.ResourceMemory, + } { + if !a.Limits[res].Equal(b.Limits[res]) || + !a.Requests[res].Equal(b.Requests[res]) { + return true + } + } + + return false +} + +// Check if we need to synchronize connection pool deployment due to new +// defaults, that are different from what we see in the DeploymentSpec +func (c *Cluster) needSyncConnPoolDefaults( + spec *acidv1.ConnectionPool, + deployment *appsv1.Deployment) (sync bool, reasons []string) { + + reasons = []string{} + sync = false + + config := c.OpConfig.ConnectionPool + podTemplate := deployment.Spec.Template + poolContainer := podTemplate.Spec.Containers[constants.ConnPoolContainer] + + if spec == nil { + spec = &acidv1.ConnectionPool{} + } + + if spec.NumberOfInstances == nil && + *deployment.Spec.Replicas != *config.NumberOfInstances { + + sync = true + msg := fmt.Sprintf("NumberOfInstances is different (having %d, required %d)", + *deployment.Spec.Replicas, *config.NumberOfInstances) + reasons = append(reasons, msg) + } + + if spec.DockerImage == "" && + poolContainer.Image != config.Image { + + sync = true + msg := fmt.Sprintf("DockerImage is different (having %s, required %s)", + poolContainer.Image, config.Image) + reasons = append(reasons, msg) + } + + expectedResources, err := generateResourceRequirements(spec.Resources, + c.makeDefaultConnPoolResources()) + + // An error to generate expected resources means something is not quite + // right, but for the purpose of robustness do not panic here, just report + // and ignore resources comparison (in the worst case there will be no + // updates for new resource values). + if err == nil && syncResources(&poolContainer.Resources, expectedResources) { + sync = true + msg := fmt.Sprintf("Resources are different (having %+v, required %+v)", + poolContainer.Resources, expectedResources) + reasons = append(reasons, msg) + } + + if err != nil { + c.logger.Warningf("Cannot generate expected resources, %v", err) + } + + for _, env := range poolContainer.Env { + if spec.User == "" && env.Name == "PGUSER" { + ref := env.ValueFrom.SecretKeyRef.LocalObjectReference + + if ref.Name != c.credentialSecretName(config.User) { + sync = true + msg := fmt.Sprintf("Pool user is different (having %s, required %s)", + ref.Name, config.User) + reasons = append(reasons, msg) + } + } + + if spec.Schema == "" && env.Name == "PGSCHEMA" && env.Value != config.Schema { + sync = true + msg := fmt.Sprintf("Pool schema is different (having %s, required %s)", + env.Value, config.Schema) + reasons = append(reasons, msg) + } + } + + return sync, reasons +} diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index 9efbc51c6..a1b361642 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -9,6 +9,7 @@ import ( acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util/config" + "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" "github.com/zalando/postgres-operator/pkg/util/teams" v1 "k8s.io/api/core/v1" @@ -704,3 +705,20 @@ func TestServiceAnnotations(t *testing.T) { }) } } + +func TestInitSystemUsers(t *testing.T) { + testName := "Test system users initialization" + + // default cluster without connection pool + cl.initSystemUsers() + if _, exist := cl.systemUsers[constants.ConnectionPoolUserKeyName]; exist { + t.Errorf("%s, connection pool user is present", testName) + } + + // cluster with connection pool + cl.Spec.EnableConnectionPool = boolToPointer(true) + cl.initSystemUsers() + if _, exist := cl.systemUsers[constants.ConnectionPoolUserKeyName]; !exist { + t.Errorf("%s, connection pool user is not present", testName) + } +} diff --git a/pkg/cluster/database.go b/pkg/cluster/database.go index 07ea011a6..bca68c188 100644 --- a/pkg/cluster/database.go +++ b/pkg/cluster/database.go @@ -1,10 +1,12 @@ package cluster import ( + "bytes" "database/sql" "fmt" "net" "strings" + "text/template" "time" "github.com/lib/pq" @@ -28,13 +30,37 @@ const ( getDatabasesSQL = `SELECT datname, pg_get_userbyid(datdba) AS owner FROM pg_database;` createDatabaseSQL = `CREATE DATABASE "%s" OWNER "%s";` alterDatabaseOwnerSQL = `ALTER DATABASE "%s" OWNER TO "%s";` + connectionPoolLookup = ` + CREATE SCHEMA IF NOT EXISTS {{.pool_schema}}; + + CREATE OR REPLACE FUNCTION {{.pool_schema}}.user_lookup( + in i_username text, out uname text, out phash text) + RETURNS record AS $$ + BEGIN + SELECT usename, passwd FROM pg_catalog.pg_shadow + WHERE usename = i_username INTO uname, phash; + RETURN; + END; + $$ LANGUAGE plpgsql SECURITY DEFINER; + + REVOKE ALL ON FUNCTION {{.pool_schema}}.user_lookup(text) + FROM public, {{.pool_user}}; + GRANT EXECUTE ON FUNCTION {{.pool_schema}}.user_lookup(text) + TO {{.pool_user}}; + GRANT USAGE ON SCHEMA {{.pool_schema}} TO {{.pool_user}}; + ` ) -func (c *Cluster) pgConnectionString() string { +func (c *Cluster) pgConnectionString(dbname string) string { password := c.systemUsers[constants.SuperuserKeyName].Password - return fmt.Sprintf("host='%s' dbname=postgres sslmode=require user='%s' password='%s' connect_timeout='%d'", + if dbname == "" { + dbname = "postgres" + } + + return fmt.Sprintf("host='%s' dbname='%s' sslmode=require user='%s' password='%s' connect_timeout='%d'", fmt.Sprintf("%s.%s.svc.%s", c.Name, c.Namespace, c.OpConfig.ClusterDomain), + dbname, c.systemUsers[constants.SuperuserKeyName].Name, strings.Replace(password, "$", "\\$", -1), constants.PostgresConnectTimeout/time.Second) @@ -49,13 +75,17 @@ func (c *Cluster) databaseAccessDisabled() bool { } func (c *Cluster) initDbConn() error { + return c.initDbConnWithName("") +} + +func (c *Cluster) initDbConnWithName(dbname string) error { c.setProcessName("initializing db connection") if c.pgDb != nil { return nil } var conn *sql.DB - connstring := c.pgConnectionString() + connstring := c.pgConnectionString(dbname) finalerr := retryutil.Retry(constants.PostgresConnectTimeout, constants.PostgresConnectRetryTimeout, func() (bool, error) { @@ -94,6 +124,10 @@ func (c *Cluster) initDbConn() error { return nil } +func (c *Cluster) connectionIsClosed() bool { + return c.pgDb == nil +} + func (c *Cluster) closeDbConn() (err error) { c.setProcessName("closing db connection") if c.pgDb != nil { @@ -243,3 +277,88 @@ func makeUserFlags(rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin return result } + +// Creates a connection pool credentials lookup function in every database to +// perform remote authentification. +func (c *Cluster) installLookupFunction(poolSchema, poolUser string) error { + var stmtBytes bytes.Buffer + c.logger.Info("Installing lookup function") + + if err := c.initDbConn(); err != nil { + return fmt.Errorf("could not init database connection") + } + defer func() { + if c.connectionIsClosed() { + return + } + + if err := c.closeDbConn(); err != nil { + c.logger.Errorf("could not close database connection: %v", err) + } + }() + + currentDatabases, err := c.getDatabases() + if err != nil { + msg := "could not get databases to install pool lookup function: %v" + return fmt.Errorf(msg, err) + } + + templater := template.Must(template.New("sql").Parse(connectionPoolLookup)) + + for dbname, _ := range currentDatabases { + if dbname == "template0" || dbname == "template1" { + continue + } + + if err := c.initDbConnWithName(dbname); err != nil { + return fmt.Errorf("could not init database connection to %s", dbname) + } + + c.logger.Infof("Install pool lookup function into %s", dbname) + + params := TemplateParams{ + "pool_schema": poolSchema, + "pool_user": poolUser, + } + + if err := templater.Execute(&stmtBytes, params); err != nil { + c.logger.Errorf("could not prepare sql statement %+v: %v", + params, err) + // process other databases + continue + } + + // golang sql will do retries couple of times if pq driver reports + // connections issues (driver.ErrBadConn), but since our query is + // idempotent, we can retry in a view of other errors (e.g. due to + // failover a db is temporary in a read-only mode or so) to make sure + // it was applied. + execErr := retryutil.Retry( + constants.PostgresConnectTimeout, + constants.PostgresConnectRetryTimeout, + func() (bool, error) { + if _, err := c.pgDb.Exec(stmtBytes.String()); err != nil { + msg := fmt.Errorf("could not execute sql statement %s: %v", + stmtBytes.String(), err) + return false, msg + } + + return true, nil + }) + + if execErr != nil { + c.logger.Errorf("could not execute after retries %s: %v", + stmtBytes.String(), err) + // process other databases + continue + } + + c.logger.Infof("Pool lookup function installed into %s", dbname) + if err := c.closeDbConn(); err != nil { + c.logger.Errorf("could not close database connection: %v", err) + } + } + + c.ConnectionPool.LookupFunction = true + return nil +} diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 39b5c16a0..34b409d4e 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -21,6 +21,7 @@ import ( "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/constants" + "github.com/zalando/postgres-operator/pkg/util/k8sutil" batchv1 "k8s.io/api/batch/v1" batchv1beta1 "k8s.io/api/batch/v1beta1" "k8s.io/apimachinery/pkg/labels" @@ -31,10 +32,12 @@ const ( patroniPGBinariesParameterName = "bin_dir" patroniPGParametersParameterName = "parameters" patroniPGHBAConfParameterName = "pg_hba" + localHost = "127.0.0.1/32" + connectionPoolContainer = "connection-pool" + pgPort = 5432 // the gid of the postgres user in the default spilo image spiloPostgresGID = 103 - localHost = "127.0.0.1/32" ) type pgUser struct { @@ -70,6 +73,10 @@ func (c *Cluster) statefulSetName() string { return c.Name } +func (c *Cluster) connPoolName() string { + return c.Name + "-pooler" +} + func (c *Cluster) endpointName(role PostgresRole) string { name := c.Name if role == Replica { @@ -88,6 +95,28 @@ func (c *Cluster) serviceName(role PostgresRole) string { return name } +func (c *Cluster) serviceAddress(role PostgresRole) string { + service, exist := c.Services[role] + + if exist { + return service.ObjectMeta.Name + } + + c.logger.Warningf("No service for role %s", role) + return "" +} + +func (c *Cluster) servicePort(role PostgresRole) string { + service, exist := c.Services[role] + + if exist { + return fmt.Sprint(service.Spec.Ports[0].Port) + } + + c.logger.Warningf("No service for role %s", role) + return "" +} + func (c *Cluster) podDisruptionBudgetName() string { return c.OpConfig.PDBNameFormat.Format("cluster", c.Name) } @@ -96,10 +125,39 @@ func (c *Cluster) makeDefaultResources() acidv1.Resources { config := c.OpConfig - defaultRequests := acidv1.ResourceDescription{CPU: config.DefaultCPURequest, Memory: config.DefaultMemoryRequest} - defaultLimits := acidv1.ResourceDescription{CPU: config.DefaultCPULimit, Memory: config.DefaultMemoryLimit} + defaultRequests := acidv1.ResourceDescription{ + CPU: config.Resources.DefaultCPURequest, + Memory: config.Resources.DefaultMemoryRequest, + } + defaultLimits := acidv1.ResourceDescription{ + CPU: config.Resources.DefaultCPULimit, + Memory: config.Resources.DefaultMemoryLimit, + } - return acidv1.Resources{ResourceRequests: defaultRequests, ResourceLimits: defaultLimits} + return acidv1.Resources{ + ResourceRequests: defaultRequests, + ResourceLimits: defaultLimits, + } +} + +// Generate default resource section for connection pool deployment, to be used +// if nothing custom is specified in the manifest +func (c *Cluster) makeDefaultConnPoolResources() acidv1.Resources { + config := c.OpConfig + + defaultRequests := acidv1.ResourceDescription{ + CPU: config.ConnectionPool.ConnPoolDefaultCPURequest, + Memory: config.ConnectionPool.ConnPoolDefaultMemoryRequest, + } + defaultLimits := acidv1.ResourceDescription{ + CPU: config.ConnectionPool.ConnPoolDefaultCPULimit, + Memory: config.ConnectionPool.ConnPoolDefaultMemoryLimit, + } + + return acidv1.Resources{ + ResourceRequests: defaultRequests, + ResourceLimits: defaultLimits, + } } func generateResourceRequirements(resources acidv1.Resources, defaultResources acidv1.Resources) (*v1.ResourceRequirements, error) { @@ -319,7 +377,11 @@ func tolerations(tolerationsSpec *[]v1.Toleration, podToleration map[string]stri return *tolerationsSpec } - if len(podToleration["key"]) > 0 || len(podToleration["operator"]) > 0 || len(podToleration["value"]) > 0 || len(podToleration["effect"]) > 0 { + if len(podToleration["key"]) > 0 || + len(podToleration["operator"]) > 0 || + len(podToleration["value"]) > 0 || + len(podToleration["effect"]) > 0 { + return []v1.Toleration{ { Key: podToleration["key"], @@ -786,12 +848,12 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef request := spec.Resources.ResourceRequests.Memory if request == "" { - request = c.OpConfig.DefaultMemoryRequest + request = c.OpConfig.Resources.DefaultMemoryRequest } limit := spec.Resources.ResourceLimits.Memory if limit == "" { - limit = c.OpConfig.DefaultMemoryLimit + limit = c.OpConfig.Resources.DefaultMemoryLimit } isSmaller, err := util.IsSmallerQuantity(request, limit) @@ -813,12 +875,12 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef // TODO #413 sidecarRequest := sidecar.Resources.ResourceRequests.Memory if request == "" { - request = c.OpConfig.DefaultMemoryRequest + request = c.OpConfig.Resources.DefaultMemoryRequest } sidecarLimit := sidecar.Resources.ResourceLimits.Memory if limit == "" { - limit = c.OpConfig.DefaultMemoryLimit + limit = c.OpConfig.Resources.DefaultMemoryLimit } isSmaller, err := util.IsSmallerQuantity(sidecarRequest, sidecarLimit) @@ -1774,6 +1836,306 @@ func (c *Cluster) getLogicalBackupJobName() (jobName string) { return "logical-backup-" + c.clusterName().Name } +// Generate pool size related environment variables. +// +// MAX_DB_CONN would specify the global maximum for connections to a target +// database. +// +// MAX_CLIENT_CONN is not configurable at the moment, just set it high enough. +// +// DEFAULT_SIZE is a pool size per db/user (having in mind the use case when +// most of the queries coming through a connection pooler are from the same +// user to the same db). In case if we want to spin up more connection pool +// instances, take this into account and maintain the same number of +// connections. +// +// MIN_SIZE is a pool minimal size, to prevent situation when sudden workload +// have to wait for spinning up a new connections. +// +// RESERVE_SIZE is how many additional connections to allow for a pool. +func (c *Cluster) getConnPoolEnvVars(spec *acidv1.PostgresSpec) []v1.EnvVar { + effectiveMode := util.Coalesce( + spec.ConnectionPool.Mode, + c.OpConfig.ConnectionPool.Mode) + + numberOfInstances := spec.ConnectionPool.NumberOfInstances + if numberOfInstances == nil { + numberOfInstances = util.CoalesceInt32( + c.OpConfig.ConnectionPool.NumberOfInstances, + k8sutil.Int32ToPointer(1)) + } + + effectiveMaxDBConn := util.CoalesceInt32( + spec.ConnectionPool.MaxDBConnections, + c.OpConfig.ConnectionPool.MaxDBConnections) + + if effectiveMaxDBConn == nil { + effectiveMaxDBConn = k8sutil.Int32ToPointer( + constants.ConnPoolMaxDBConnections) + } + + maxDBConn := *effectiveMaxDBConn / *numberOfInstances + + defaultSize := maxDBConn / 2 + minSize := defaultSize / 2 + reserveSize := minSize + + return []v1.EnvVar{ + { + Name: "CONNECTION_POOL_PORT", + Value: fmt.Sprint(pgPort), + }, + { + Name: "CONNECTION_POOL_MODE", + Value: effectiveMode, + }, + { + Name: "CONNECTION_POOL_DEFAULT_SIZE", + Value: fmt.Sprint(defaultSize), + }, + { + Name: "CONNECTION_POOL_MIN_SIZE", + Value: fmt.Sprint(minSize), + }, + { + Name: "CONNECTION_POOL_RESERVE_SIZE", + Value: fmt.Sprint(reserveSize), + }, + { + Name: "CONNECTION_POOL_MAX_CLIENT_CONN", + Value: fmt.Sprint(constants.ConnPoolMaxClientConnections), + }, + { + Name: "CONNECTION_POOL_MAX_DB_CONN", + Value: fmt.Sprint(maxDBConn), + }, + } +} + +func (c *Cluster) generateConnPoolPodTemplate(spec *acidv1.PostgresSpec) ( + *v1.PodTemplateSpec, error) { + + gracePeriod := int64(c.OpConfig.PodTerminateGracePeriod.Seconds()) + resources, err := generateResourceRequirements( + spec.ConnectionPool.Resources, + c.makeDefaultConnPoolResources()) + + effectiveDockerImage := util.Coalesce( + spec.ConnectionPool.DockerImage, + c.OpConfig.ConnectionPool.Image) + + effectiveSchema := util.Coalesce( + spec.ConnectionPool.Schema, + c.OpConfig.ConnectionPool.Schema) + + if err != nil { + return nil, fmt.Errorf("could not generate resource requirements: %v", err) + } + + secretSelector := func(key string) *v1.SecretKeySelector { + effectiveUser := util.Coalesce( + spec.ConnectionPool.User, + c.OpConfig.ConnectionPool.User) + + return &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: c.credentialSecretName(effectiveUser), + }, + Key: key, + } + } + + envVars := []v1.EnvVar{ + { + Name: "PGHOST", + Value: c.serviceAddress(Master), + }, + { + Name: "PGPORT", + Value: c.servicePort(Master), + }, + { + Name: "PGUSER", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: secretSelector("username"), + }, + }, + // the convention is to use the same schema name as + // connection pool username + { + Name: "PGSCHEMA", + Value: effectiveSchema, + }, + { + Name: "PGPASSWORD", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: secretSelector("password"), + }, + }, + } + + envVars = append(envVars, c.getConnPoolEnvVars(spec)...) + + poolerContainer := v1.Container{ + Name: connectionPoolContainer, + Image: effectiveDockerImage, + ImagePullPolicy: v1.PullIfNotPresent, + Resources: *resources, + Ports: []v1.ContainerPort{ + { + ContainerPort: pgPort, + Protocol: v1.ProtocolTCP, + }, + }, + Env: envVars, + } + + podTemplate := &v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: c.connPoolLabelsSelector().MatchLabels, + Namespace: c.Namespace, + Annotations: c.generatePodAnnotations(spec), + }, + Spec: v1.PodSpec{ + ServiceAccountName: c.OpConfig.PodServiceAccountName, + TerminationGracePeriodSeconds: &gracePeriod, + Containers: []v1.Container{poolerContainer}, + // TODO: add tolerations to scheduler pooler on the same node + // as database + //Tolerations: *tolerationsSpec, + }, + } + + return podTemplate, nil +} + +// Return an array of ownerReferences to make an arbitraty object dependent on +// the StatefulSet. Dependency is made on StatefulSet instead of PostgreSQL CRD +// while the former is represent the actual state, and only it's deletion means +// we delete the cluster (e.g. if CRD was deleted, StatefulSet somehow +// survived, we can't delete an object because it will affect the functioning +// cluster). +func (c *Cluster) ownerReferences() []metav1.OwnerReference { + controller := true + + if c.Statefulset == nil { + c.logger.Warning("Cannot get owner reference, no statefulset") + return []metav1.OwnerReference{} + } + + return []metav1.OwnerReference{ + { + UID: c.Statefulset.ObjectMeta.UID, + APIVersion: "apps/v1", + Kind: "StatefulSet", + Name: c.Statefulset.ObjectMeta.Name, + Controller: &controller, + }, + } +} + +func (c *Cluster) generateConnPoolDeployment(spec *acidv1.PostgresSpec) ( + *appsv1.Deployment, error) { + + // there are two ways to enable connection pooler, either to specify a + // connectionPool section or enableConnectionPool. In the second case + // spec.connectionPool will be nil, so to make it easier to calculate + // default values, initialize it to an empty structure. It could be done + // anywhere, but here is the earliest common entry point between sync and + // create code, so init here. + if spec.ConnectionPool == nil { + spec.ConnectionPool = &acidv1.ConnectionPool{} + } + + podTemplate, err := c.generateConnPoolPodTemplate(spec) + numberOfInstances := spec.ConnectionPool.NumberOfInstances + if numberOfInstances == nil { + numberOfInstances = util.CoalesceInt32( + c.OpConfig.ConnectionPool.NumberOfInstances, + k8sutil.Int32ToPointer(1)) + } + + if *numberOfInstances < constants.ConnPoolMinInstances { + msg := "Adjusted number of connection pool instances from %d to %d" + c.logger.Warningf(msg, numberOfInstances, constants.ConnPoolMinInstances) + + *numberOfInstances = constants.ConnPoolMinInstances + } + + if err != nil { + return nil, err + } + + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: c.connPoolName(), + Namespace: c.Namespace, + Labels: c.connPoolLabelsSelector().MatchLabels, + Annotations: map[string]string{}, + // make StatefulSet object its owner to represent the dependency. + // By itself StatefulSet is being deleted with "Orphaned" + // propagation policy, which means that it's deletion will not + // clean up this deployment, but there is a hope that this object + // will be garbage collected if something went wrong and operator + // didn't deleted it. + OwnerReferences: c.ownerReferences(), + }, + Spec: appsv1.DeploymentSpec{ + Replicas: numberOfInstances, + Selector: c.connPoolLabelsSelector(), + Template: *podTemplate, + }, + } + + return deployment, nil +} + +func (c *Cluster) generateConnPoolService(spec *acidv1.PostgresSpec) *v1.Service { + + // there are two ways to enable connection pooler, either to specify a + // connectionPool section or enableConnectionPool. In the second case + // spec.connectionPool will be nil, so to make it easier to calculate + // default values, initialize it to an empty structure. It could be done + // anywhere, but here is the earliest common entry point between sync and + // create code, so init here. + if spec.ConnectionPool == nil { + spec.ConnectionPool = &acidv1.ConnectionPool{} + } + + serviceSpec := v1.ServiceSpec{ + Ports: []v1.ServicePort{ + { + Name: c.connPoolName(), + Port: pgPort, + TargetPort: intstr.IntOrString{StrVal: c.servicePort(Master)}, + }, + }, + Type: v1.ServiceTypeClusterIP, + Selector: map[string]string{ + "connection-pool": c.connPoolName(), + }, + } + + service := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: c.connPoolName(), + Namespace: c.Namespace, + Labels: c.connPoolLabelsSelector().MatchLabels, + Annotations: map[string]string{}, + // make StatefulSet object its owner to represent the dependency. + // By itself StatefulSet is being deleted with "Orphaned" + // propagation policy, which means that it's deletion will not + // clean up this service, but there is a hope that this object will + // be garbage collected if something went wrong and operator didn't + // deleted it. + OwnerReferences: c.ownerReferences(), + }, + Spec: serviceSpec, + } + + return service +} + func ensurePath(file string, defaultDir string, defaultFile string) string { if file == "" { return path.Join(defaultDir, defaultFile) diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 25e0f7af4..e04b281ba 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -1,6 +1,8 @@ package cluster import ( + "errors" + "fmt" "reflect" "testing" @@ -13,6 +15,7 @@ import ( "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" + appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" policyv1beta1 "k8s.io/api/policy/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -582,6 +585,375 @@ func TestSecretVolume(t *testing.T) { } } +func testResources(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { + cpuReq := podSpec.Spec.Containers[0].Resources.Requests["cpu"] + if cpuReq.String() != cluster.OpConfig.ConnectionPool.ConnPoolDefaultCPURequest { + return fmt.Errorf("CPU request doesn't match, got %s, expected %s", + cpuReq.String(), cluster.OpConfig.ConnectionPool.ConnPoolDefaultCPURequest) + } + + memReq := podSpec.Spec.Containers[0].Resources.Requests["memory"] + if memReq.String() != cluster.OpConfig.ConnectionPool.ConnPoolDefaultMemoryRequest { + return fmt.Errorf("Memory request doesn't match, got %s, expected %s", + memReq.String(), cluster.OpConfig.ConnectionPool.ConnPoolDefaultMemoryRequest) + } + + cpuLim := podSpec.Spec.Containers[0].Resources.Limits["cpu"] + if cpuLim.String() != cluster.OpConfig.ConnectionPool.ConnPoolDefaultCPULimit { + return fmt.Errorf("CPU limit doesn't match, got %s, expected %s", + cpuLim.String(), cluster.OpConfig.ConnectionPool.ConnPoolDefaultCPULimit) + } + + memLim := podSpec.Spec.Containers[0].Resources.Limits["memory"] + if memLim.String() != cluster.OpConfig.ConnectionPool.ConnPoolDefaultMemoryLimit { + return fmt.Errorf("Memory limit doesn't match, got %s, expected %s", + memLim.String(), cluster.OpConfig.ConnectionPool.ConnPoolDefaultMemoryLimit) + } + + return nil +} + +func testLabels(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { + poolLabels := podSpec.ObjectMeta.Labels["connection-pool"] + + if poolLabels != cluster.connPoolLabelsSelector().MatchLabels["connection-pool"] { + return fmt.Errorf("Pod labels do not match, got %+v, expected %+v", + podSpec.ObjectMeta.Labels, cluster.connPoolLabelsSelector().MatchLabels) + } + + return nil +} + +func testEnvs(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { + required := map[string]bool{ + "PGHOST": false, + "PGPORT": false, + "PGUSER": false, + "PGSCHEMA": false, + "PGPASSWORD": false, + "CONNECTION_POOL_MODE": false, + "CONNECTION_POOL_PORT": false, + } + + envs := podSpec.Spec.Containers[0].Env + for _, env := range envs { + required[env.Name] = true + } + + for env, value := range required { + if !value { + return fmt.Errorf("Environment variable %s is not present", env) + } + } + + return nil +} + +func testCustomPodTemplate(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { + if podSpec.ObjectMeta.Name != "test-pod-template" { + return fmt.Errorf("Custom pod template is not used, current spec %+v", + podSpec) + } + + return nil +} + +func TestConnPoolPodSpec(t *testing.T) { + testName := "Test connection pool pod template generation" + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPool: config.ConnectionPool{ + MaxDBConnections: int32ToPointer(60), + ConnPoolDefaultCPURequest: "100m", + ConnPoolDefaultCPULimit: "100m", + ConnPoolDefaultMemoryRequest: "100Mi", + ConnPoolDefaultMemoryLimit: "100Mi", + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + + var clusterNoDefaultRes = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPool: config.ConnectionPool{}, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + + noCheck := func(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { return nil } + + tests := []struct { + subTest string + spec *acidv1.PostgresSpec + expected error + cluster *Cluster + check func(cluster *Cluster, podSpec *v1.PodTemplateSpec) error + }{ + { + subTest: "default configuration", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: nil, + cluster: cluster, + check: noCheck, + }, + { + subTest: "no default resources", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: errors.New(`could not generate resource requirements: could not fill resource requests: could not parse default CPU quantity: quantities must match the regular expression '^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$'`), + cluster: clusterNoDefaultRes, + check: noCheck, + }, + { + subTest: "default resources are set", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: nil, + cluster: cluster, + check: testResources, + }, + { + subTest: "labels for service", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: nil, + cluster: cluster, + check: testLabels, + }, + { + subTest: "required envs", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: nil, + cluster: cluster, + check: testEnvs, + }, + } + for _, tt := range tests { + podSpec, err := tt.cluster.generateConnPoolPodTemplate(tt.spec) + + if err != tt.expected && err.Error() != tt.expected.Error() { + t.Errorf("%s [%s]: Could not generate pod template,\n %+v, expected\n %+v", + testName, tt.subTest, err, tt.expected) + } + + err = tt.check(cluster, podSpec) + if err != nil { + t.Errorf("%s [%s]: Pod spec is incorrect, %+v", + testName, tt.subTest, err) + } + } +} + +func testDeploymentOwnwerReference(cluster *Cluster, deployment *appsv1.Deployment) error { + owner := deployment.ObjectMeta.OwnerReferences[0] + + if owner.Name != cluster.Statefulset.ObjectMeta.Name { + return fmt.Errorf("Ownere reference is incorrect, got %s, expected %s", + owner.Name, cluster.Statefulset.ObjectMeta.Name) + } + + return nil +} + +func testSelector(cluster *Cluster, deployment *appsv1.Deployment) error { + labels := deployment.Spec.Selector.MatchLabels + expected := cluster.connPoolLabelsSelector().MatchLabels + + if labels["connection-pool"] != expected["connection-pool"] { + return fmt.Errorf("Labels are incorrect, got %+v, expected %+v", + labels, expected) + } + + return nil +} + +func TestConnPoolDeploymentSpec(t *testing.T) { + testName := "Test connection pool deployment spec generation" + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPool: config.ConnectionPool{ + ConnPoolDefaultCPURequest: "100m", + ConnPoolDefaultCPULimit: "100m", + ConnPoolDefaultMemoryRequest: "100Mi", + ConnPoolDefaultMemoryLimit: "100Mi", + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + cluster.Statefulset = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sts", + }, + } + + noCheck := func(cluster *Cluster, deployment *appsv1.Deployment) error { + return nil + } + + tests := []struct { + subTest string + spec *acidv1.PostgresSpec + expected error + cluster *Cluster + check func(cluster *Cluster, deployment *appsv1.Deployment) error + }{ + { + subTest: "default configuration", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: nil, + cluster: cluster, + check: noCheck, + }, + { + subTest: "owner reference", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: nil, + cluster: cluster, + check: testDeploymentOwnwerReference, + }, + { + subTest: "selector", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: nil, + cluster: cluster, + check: testSelector, + }, + } + for _, tt := range tests { + deployment, err := tt.cluster.generateConnPoolDeployment(tt.spec) + + if err != tt.expected && err.Error() != tt.expected.Error() { + t.Errorf("%s [%s]: Could not generate deployment spec,\n %+v, expected\n %+v", + testName, tt.subTest, err, tt.expected) + } + + err = tt.check(cluster, deployment) + if err != nil { + t.Errorf("%s [%s]: Deployment spec is incorrect, %+v", + testName, tt.subTest, err) + } + } +} + +func testServiceOwnwerReference(cluster *Cluster, service *v1.Service) error { + owner := service.ObjectMeta.OwnerReferences[0] + + if owner.Name != cluster.Statefulset.ObjectMeta.Name { + return fmt.Errorf("Ownere reference is incorrect, got %s, expected %s", + owner.Name, cluster.Statefulset.ObjectMeta.Name) + } + + return nil +} + +func testServiceSelector(cluster *Cluster, service *v1.Service) error { + selector := service.Spec.Selector + + if selector["connection-pool"] != cluster.connPoolName() { + return fmt.Errorf("Selector is incorrect, got %s, expected %s", + selector["connection-pool"], cluster.connPoolName()) + } + + return nil +} + +func TestConnPoolServiceSpec(t *testing.T) { + testName := "Test connection pool service spec generation" + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPool: config.ConnectionPool{ + ConnPoolDefaultCPURequest: "100m", + ConnPoolDefaultCPULimit: "100m", + ConnPoolDefaultMemoryRequest: "100Mi", + ConnPoolDefaultMemoryLimit: "100Mi", + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + cluster.Statefulset = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sts", + }, + } + + noCheck := func(cluster *Cluster, deployment *v1.Service) error { + return nil + } + + tests := []struct { + subTest string + spec *acidv1.PostgresSpec + cluster *Cluster + check func(cluster *Cluster, deployment *v1.Service) error + }{ + { + subTest: "default configuration", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + cluster: cluster, + check: noCheck, + }, + { + subTest: "owner reference", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + cluster: cluster, + check: testServiceOwnwerReference, + }, + { + subTest: "selector", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + cluster: cluster, + check: testServiceSelector, + }, + } + for _, tt := range tests { + service := tt.cluster.generateConnPoolService(tt.spec) + + if err := tt.check(cluster, service); err != nil { + t.Errorf("%s [%s]: Service spec is incorrect, %+v", + testName, tt.subTest, err) + } + } +} + func TestTLS(t *testing.T) { var err error var spec acidv1.PostgresSpec diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index d6c2149bf..2e02a9a83 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -90,6 +90,132 @@ func (c *Cluster) createStatefulSet() (*appsv1.StatefulSet, error) { return statefulSet, nil } +// Prepare the database for connection pool to be used, i.e. install lookup +// function (do it first, because it should be fast and if it didn't succeed, +// it doesn't makes sense to create more K8S objects. At this moment we assume +// that necessary connection pool user exists. +// +// After that create all the objects for connection pool, namely a deployment +// with a chosen pooler and a service to expose it. +func (c *Cluster) createConnectionPool(lookup InstallFunction) (*ConnectionPoolObjects, error) { + var msg string + c.setProcessName("creating connection pool") + + schema := c.Spec.ConnectionPool.Schema + if schema == "" { + schema = c.OpConfig.ConnectionPool.Schema + } + + user := c.Spec.ConnectionPool.User + if user == "" { + user = c.OpConfig.ConnectionPool.User + } + + err := lookup(schema, user) + + if err != nil { + msg = "could not prepare database for connection pool: %v" + return nil, fmt.Errorf(msg, err) + } + + deploymentSpec, err := c.generateConnPoolDeployment(&c.Spec) + if err != nil { + msg = "could not generate deployment for connection pool: %v" + return nil, fmt.Errorf(msg, err) + } + + // client-go does retry 10 times (with NoBackoff by default) when the API + // believe a request can be retried and returns Retry-After header. This + // should be good enough to not think about it here. + deployment, err := c.KubeClient. + Deployments(deploymentSpec.Namespace). + Create(deploymentSpec) + + if err != nil { + return nil, err + } + + serviceSpec := c.generateConnPoolService(&c.Spec) + service, err := c.KubeClient. + Services(serviceSpec.Namespace). + Create(serviceSpec) + + if err != nil { + return nil, err + } + + c.ConnectionPool = &ConnectionPoolObjects{ + Deployment: deployment, + Service: service, + } + c.logger.Debugf("created new connection pool %q, uid: %q", + util.NameFromMeta(deployment.ObjectMeta), deployment.UID) + + return c.ConnectionPool, nil +} + +func (c *Cluster) deleteConnectionPool() (err error) { + c.setProcessName("deleting connection pool") + c.logger.Debugln("deleting connection pool") + + // Lack of connection pooler objects is not a fatal error, just log it if + // it was present before in the manifest + if c.ConnectionPool == nil { + c.logger.Infof("No connection pool to delete") + return nil + } + + // Clean up the deployment object. If deployment resource we've remembered + // is somehow empty, try to delete based on what would we generate + deploymentName := c.connPoolName() + deployment := c.ConnectionPool.Deployment + + if deployment != nil { + deploymentName = deployment.Name + } + + // set delete propagation policy to foreground, so that replica set will be + // also deleted. + policy := metav1.DeletePropagationForeground + options := metav1.DeleteOptions{PropagationPolicy: &policy} + err = c.KubeClient. + Deployments(c.Namespace). + Delete(deploymentName, &options) + + if !k8sutil.ResourceNotFound(err) { + c.logger.Debugf("Connection pool deployment was already deleted") + } else if err != nil { + return fmt.Errorf("could not delete deployment: %v", err) + } + + c.logger.Infof("Connection pool deployment %q has been deleted", deploymentName) + + // Repeat the same for the service object + service := c.ConnectionPool.Service + serviceName := c.connPoolName() + + if service != nil { + serviceName = service.Name + } + + // set delete propagation policy to foreground, so that all the dependant + // will be deleted. + err = c.KubeClient. + Services(c.Namespace). + Delete(serviceName, &options) + + if !k8sutil.ResourceNotFound(err) { + c.logger.Debugf("Connection pool service was already deleted") + } else if err != nil { + return fmt.Errorf("could not delete service: %v", err) + } + + c.logger.Infof("Connection pool service %q has been deleted", serviceName) + + c.ConnectionPool = nil + return nil +} + func getPodIndex(podName string) (int32, error) { parts := strings.Split(podName, "-") if len(parts) == 0 { @@ -674,3 +800,34 @@ func (c *Cluster) GetStatefulSet() *appsv1.StatefulSet { func (c *Cluster) GetPodDisruptionBudget() *policybeta1.PodDisruptionBudget { return c.PodDisruptionBudget } + +// Perform actual patching of a connection pool deployment, assuming that all +// the check were already done before. +func (c *Cluster) updateConnPoolDeployment(oldDeploymentSpec, newDeployment *appsv1.Deployment) (*appsv1.Deployment, error) { + c.setProcessName("updating connection pool") + if c.ConnectionPool == nil || c.ConnectionPool.Deployment == nil { + return nil, fmt.Errorf("there is no connection pool in the cluster") + } + + patchData, err := specPatch(newDeployment.Spec) + if err != nil { + return nil, fmt.Errorf("could not form patch for the deployment: %v", err) + } + + // An update probably requires RetryOnConflict, but since only one operator + // worker at one time will try to update it chances of conflicts are + // minimal. + deployment, err := c.KubeClient. + Deployments(c.ConnectionPool.Deployment.Namespace). + Patch( + c.ConnectionPool.Deployment.Name, + types.MergePatchType, + patchData, "") + if err != nil { + return nil, fmt.Errorf("could not patch deployment: %v", err) + } + + c.ConnectionPool.Deployment = deployment + + return deployment, nil +} diff --git a/pkg/cluster/resources_test.go b/pkg/cluster/resources_test.go new file mode 100644 index 000000000..f06e96e65 --- /dev/null +++ b/pkg/cluster/resources_test.go @@ -0,0 +1,127 @@ +package cluster + +import ( + "testing" + + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + "github.com/zalando/postgres-operator/pkg/util/config" + "github.com/zalando/postgres-operator/pkg/util/k8sutil" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func mockInstallLookupFunction(schema string, user string) error { + return nil +} + +func boolToPointer(value bool) *bool { + return &value +} + +func TestConnPoolCreationAndDeletion(t *testing.T) { + testName := "Test connection pool creation" + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPool: config.ConnectionPool{ + ConnPoolDefaultCPURequest: "100m", + ConnPoolDefaultCPULimit: "100m", + ConnPoolDefaultMemoryRequest: "100Mi", + ConnPoolDefaultMemoryLimit: "100Mi", + }, + }, + }, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger) + + cluster.Statefulset = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sts", + }, + } + + cluster.Spec = acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + } + poolResources, err := cluster.createConnectionPool(mockInstallLookupFunction) + + if err != nil { + t.Errorf("%s: Cannot create connection pool, %s, %+v", + testName, err, poolResources) + } + + if poolResources.Deployment == nil { + t.Errorf("%s: Connection pool deployment is empty", testName) + } + + if poolResources.Service == nil { + t.Errorf("%s: Connection pool service is empty", testName) + } + + err = cluster.deleteConnectionPool() + if err != nil { + t.Errorf("%s: Cannot delete connection pool, %s", testName, err) + } +} + +func TestNeedConnPool(t *testing.T) { + testName := "Test how connection pool can be enabled" + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPool: config.ConnectionPool{ + ConnPoolDefaultCPURequest: "100m", + ConnPoolDefaultCPULimit: "100m", + ConnPoolDefaultMemoryRequest: "100Mi", + ConnPoolDefaultMemoryLimit: "100Mi", + }, + }, + }, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger) + + cluster.Spec = acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + } + + if !cluster.needConnectionPool() { + t.Errorf("%s: Connection pool is not enabled with full definition", + testName) + } + + cluster.Spec = acidv1.PostgresSpec{ + EnableConnectionPool: boolToPointer(true), + } + + if !cluster.needConnectionPool() { + t.Errorf("%s: Connection pool is not enabled with flag", + testName) + } + + cluster.Spec = acidv1.PostgresSpec{ + EnableConnectionPool: boolToPointer(false), + ConnectionPool: &acidv1.ConnectionPool{}, + } + + if cluster.needConnectionPool() { + t.Errorf("%s: Connection pool is still enabled with flag being false", + testName) + } + + cluster.Spec = acidv1.PostgresSpec{ + EnableConnectionPool: boolToPointer(true), + ConnectionPool: &acidv1.ConnectionPool{}, + } + + if !cluster.needConnectionPool() { + t.Errorf("%s: Connection pool is not enabled with flag and full", + testName) + } +} diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index b04ff863b..a7c933ae7 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -23,6 +23,7 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { c.mu.Lock() defer c.mu.Unlock() + oldSpec := c.Postgresql c.setSpec(newSpec) defer func() { @@ -108,6 +109,11 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { } } + // sync connection pool + if err = c.syncConnectionPool(&oldSpec, newSpec, c.installLookupFunction); err != nil { + return fmt.Errorf("could not sync connection pool: %v", err) + } + return err } @@ -424,7 +430,9 @@ func (c *Cluster) syncSecrets() error { } pwdUser := userMap[secretUsername] // if this secret belongs to the infrastructure role and the password has changed - replace it in the secret - if pwdUser.Password != string(secret.Data["password"]) && pwdUser.Origin == spec.RoleOriginInfrastructure { + if pwdUser.Password != string(secret.Data["password"]) && + pwdUser.Origin == spec.RoleOriginInfrastructure { + c.logger.Debugf("updating the secret %q from the infrastructure roles", secretSpec.Name) if _, err = c.KubeClient.Secrets(secretSpec.Namespace).Update(secretSpec); err != nil { return fmt.Errorf("could not update infrastructure role secret for role %q: %v", secretUsername, err) @@ -468,6 +476,16 @@ func (c *Cluster) syncRoles() (err error) { for _, u := range c.pgUsers { userNames = append(userNames, u.Name) } + + if c.needConnectionPool() { + connPoolUser := c.systemUsers[constants.ConnectionPoolUserKeyName] + userNames = append(userNames, connPoolUser.Name) + + if _, exists := c.pgUsers[connPoolUser.Name]; !exists { + c.pgUsers[connPoolUser.Name] = connPoolUser + } + } + dbUsers, err = c.readPgUsersFromDatabase(userNames) if err != nil { return fmt.Errorf("error getting users from the database: %v", err) @@ -600,3 +618,165 @@ func (c *Cluster) syncLogicalBackupJob() error { return nil } + +func (c *Cluster) syncConnectionPool(oldSpec, newSpec *acidv1.Postgresql, lookup InstallFunction) error { + if c.ConnectionPool == nil { + c.ConnectionPool = &ConnectionPoolObjects{} + } + + newNeedConnPool := c.needConnectionPoolWorker(&newSpec.Spec) + oldNeedConnPool := c.needConnectionPoolWorker(&oldSpec.Spec) + + if newNeedConnPool { + // Try to sync in any case. If we didn't needed connection pool before, + // it means we want to create it. If it was already present, still sync + // since it could happen that there is no difference in specs, and all + // the resources are remembered, but the deployment was manualy deleted + // in between + c.logger.Debug("syncing connection pool") + + // in this case also do not forget to install lookup function as for + // creating cluster + if !oldNeedConnPool || !c.ConnectionPool.LookupFunction { + newConnPool := newSpec.Spec.ConnectionPool + + specSchema := "" + specUser := "" + + if newConnPool != nil { + specSchema = newConnPool.Schema + specUser = newConnPool.User + } + + schema := util.Coalesce( + specSchema, + c.OpConfig.ConnectionPool.Schema) + + user := util.Coalesce( + specUser, + c.OpConfig.ConnectionPool.User) + + if err := lookup(schema, user); err != nil { + return err + } + } + + if err := c.syncConnectionPoolWorker(oldSpec, newSpec); err != nil { + c.logger.Errorf("could not sync connection pool: %v", err) + return err + } + } + + if oldNeedConnPool && !newNeedConnPool { + // delete and cleanup resources + if err := c.deleteConnectionPool(); err != nil { + c.logger.Warningf("could not remove connection pool: %v", err) + } + } + + if !oldNeedConnPool && !newNeedConnPool { + // delete and cleanup resources if not empty + if c.ConnectionPool != nil && + (c.ConnectionPool.Deployment != nil || + c.ConnectionPool.Service != nil) { + + if err := c.deleteConnectionPool(); err != nil { + c.logger.Warningf("could not remove connection pool: %v", err) + } + } + } + + return nil +} + +// Synchronize connection pool resources. Effectively we're interested only in +// synchronizing the corresponding deployment, but in case of deployment or +// service is missing, create it. After checking, also remember an object for +// the future references. +func (c *Cluster) syncConnectionPoolWorker(oldSpec, newSpec *acidv1.Postgresql) error { + deployment, err := c.KubeClient. + Deployments(c.Namespace). + Get(c.connPoolName(), metav1.GetOptions{}) + + if err != nil && k8sutil.ResourceNotFound(err) { + msg := "Deployment %s for connection pool synchronization is not found, create it" + c.logger.Warningf(msg, c.connPoolName()) + + deploymentSpec, err := c.generateConnPoolDeployment(&newSpec.Spec) + if err != nil { + msg = "could not generate deployment for connection pool: %v" + return fmt.Errorf(msg, err) + } + + deployment, err := c.KubeClient. + Deployments(deploymentSpec.Namespace). + Create(deploymentSpec) + + if err != nil { + return err + } + + c.ConnectionPool.Deployment = deployment + } else if err != nil { + return fmt.Errorf("could not get connection pool deployment to sync: %v", err) + } else { + c.ConnectionPool.Deployment = deployment + + // actual synchronization + oldConnPool := oldSpec.Spec.ConnectionPool + newConnPool := newSpec.Spec.ConnectionPool + specSync, specReason := c.needSyncConnPoolSpecs(oldConnPool, newConnPool) + defaultsSync, defaultsReason := c.needSyncConnPoolDefaults(newConnPool, deployment) + reason := append(specReason, defaultsReason...) + if specSync || defaultsSync { + c.logger.Infof("Update connection pool deployment %s, reason: %+v", + c.connPoolName(), reason) + + newDeploymentSpec, err := c.generateConnPoolDeployment(&newSpec.Spec) + if err != nil { + msg := "could not generate deployment for connection pool: %v" + return fmt.Errorf(msg, err) + } + + oldDeploymentSpec := c.ConnectionPool.Deployment + + deployment, err := c.updateConnPoolDeployment( + oldDeploymentSpec, + newDeploymentSpec) + + if err != nil { + return err + } + + c.ConnectionPool.Deployment = deployment + return nil + } + } + + service, err := c.KubeClient. + Services(c.Namespace). + Get(c.connPoolName(), metav1.GetOptions{}) + + if err != nil && k8sutil.ResourceNotFound(err) { + msg := "Service %s for connection pool synchronization is not found, create it" + c.logger.Warningf(msg, c.connPoolName()) + + serviceSpec := c.generateConnPoolService(&newSpec.Spec) + service, err := c.KubeClient. + Services(serviceSpec.Namespace). + Create(serviceSpec) + + if err != nil { + return err + } + + c.ConnectionPool.Service = service + } else if err != nil { + return fmt.Errorf("could not get connection pool service to sync: %v", err) + } else { + // Service updates are not supported and probably not that useful anyway + c.ConnectionPool.Service = service + } + + return nil +} diff --git a/pkg/cluster/sync_test.go b/pkg/cluster/sync_test.go new file mode 100644 index 000000000..483d4ba58 --- /dev/null +++ b/pkg/cluster/sync_test.go @@ -0,0 +1,212 @@ +package cluster + +import ( + "fmt" + "testing" + + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + "github.com/zalando/postgres-operator/pkg/util/config" + "github.com/zalando/postgres-operator/pkg/util/k8sutil" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func int32ToPointer(value int32) *int32 { + return &value +} + +func deploymentUpdated(cluster *Cluster, err error) error { + if cluster.ConnectionPool.Deployment.Spec.Replicas == nil || + *cluster.ConnectionPool.Deployment.Spec.Replicas != 2 { + return fmt.Errorf("Wrong nubmer of instances") + } + + return nil +} + +func objectsAreSaved(cluster *Cluster, err error) error { + if cluster.ConnectionPool == nil { + return fmt.Errorf("Connection pool resources are empty") + } + + if cluster.ConnectionPool.Deployment == nil { + return fmt.Errorf("Deployment was not saved") + } + + if cluster.ConnectionPool.Service == nil { + return fmt.Errorf("Service was not saved") + } + + return nil +} + +func objectsAreDeleted(cluster *Cluster, err error) error { + if cluster.ConnectionPool != nil { + return fmt.Errorf("Connection pool was not deleted") + } + + return nil +} + +func TestConnPoolSynchronization(t *testing.T) { + testName := "Test connection pool synchronization" + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPool: config.ConnectionPool{ + ConnPoolDefaultCPURequest: "100m", + ConnPoolDefaultCPULimit: "100m", + ConnPoolDefaultMemoryRequest: "100Mi", + ConnPoolDefaultMemoryLimit: "100Mi", + NumberOfInstances: int32ToPointer(1), + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + + cluster.Statefulset = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sts", + }, + } + + clusterMissingObjects := *cluster + clusterMissingObjects.KubeClient = k8sutil.ClientMissingObjects() + + clusterMock := *cluster + clusterMock.KubeClient = k8sutil.NewMockKubernetesClient() + + clusterDirtyMock := *cluster + clusterDirtyMock.KubeClient = k8sutil.NewMockKubernetesClient() + clusterDirtyMock.ConnectionPool = &ConnectionPoolObjects{ + Deployment: &appsv1.Deployment{}, + Service: &v1.Service{}, + } + + clusterNewDefaultsMock := *cluster + clusterNewDefaultsMock.KubeClient = k8sutil.NewMockKubernetesClient() + cluster.OpConfig.ConnectionPool.Image = "pooler:2.0" + cluster.OpConfig.ConnectionPool.NumberOfInstances = int32ToPointer(2) + + tests := []struct { + subTest string + oldSpec *acidv1.Postgresql + newSpec *acidv1.Postgresql + cluster *Cluster + check func(cluster *Cluster, err error) error + }{ + { + subTest: "create if doesn't exist", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + }, + cluster: &clusterMissingObjects, + check: objectsAreSaved, + }, + { + subTest: "create if doesn't exist with a flag", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{}, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + EnableConnectionPool: boolToPointer(true), + }, + }, + cluster: &clusterMissingObjects, + check: objectsAreSaved, + }, + { + subTest: "create from scratch", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{}, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + }, + cluster: &clusterMissingObjects, + check: objectsAreSaved, + }, + { + subTest: "delete if not needed", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{}, + }, + cluster: &clusterMock, + check: objectsAreDeleted, + }, + { + subTest: "cleanup if still there", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{}, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{}, + }, + cluster: &clusterDirtyMock, + check: objectsAreDeleted, + }, + { + subTest: "update deployment", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{ + NumberOfInstances: int32ToPointer(1), + }, + }, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{ + NumberOfInstances: int32ToPointer(2), + }, + }, + }, + cluster: &clusterMock, + check: deploymentUpdated, + }, + { + subTest: "update image from changed defaults", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + }, + cluster: &clusterNewDefaultsMock, + check: deploymentUpdated, + }, + } + for _, tt := range tests { + err := tt.cluster.syncConnectionPool(tt.oldSpec, tt.newSpec, mockInstallLookupFunction) + + if err := tt.check(tt.cluster, err); err != nil { + t.Errorf("%s [%s]: Could not synchronize, %+v", + testName, tt.subTest, err) + } + } +} diff --git a/pkg/cluster/types.go b/pkg/cluster/types.go index 138b7015c..04d00cb58 100644 --- a/pkg/cluster/types.go +++ b/pkg/cluster/types.go @@ -69,3 +69,7 @@ type ClusterStatus struct { Spec acidv1.PostgresSpec Error error } + +type TemplateParams map[string]interface{} + +type InstallFunction func(schema string, user string) error diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go index 8c02fed2e..dc1e93954 100644 --- a/pkg/cluster/util.go +++ b/pkg/cluster/util.go @@ -408,7 +408,32 @@ func (c *Cluster) labelsSet(shouldAddExtraLabels bool) labels.Set { } func (c *Cluster) labelsSelector() *metav1.LabelSelector { - return &metav1.LabelSelector{MatchLabels: c.labelsSet(false), MatchExpressions: nil} + return &metav1.LabelSelector{ + MatchLabels: c.labelsSet(false), + MatchExpressions: nil, + } +} + +// Return connection pool labels selector, which should from one point of view +// inherit most of the labels from the cluster itself, but at the same time +// have e.g. different `application` label, so that recreatePod operation will +// not interfere with it (it lists all the pods via labels, and if there would +// be no difference, it will recreate also pooler pods). +func (c *Cluster) connPoolLabelsSelector() *metav1.LabelSelector { + connPoolLabels := labels.Set(map[string]string{}) + + extraLabels := labels.Set(map[string]string{ + "connection-pool": c.connPoolName(), + "application": "db-connection-pool", + }) + + connPoolLabels = labels.Merge(connPoolLabels, c.labelsSet(false)) + connPoolLabels = labels.Merge(connPoolLabels, extraLabels) + + return &metav1.LabelSelector{ + MatchLabels: connPoolLabels, + MatchExpressions: nil, + } } func (c *Cluster) roleLabelsSet(shouldAddExtraLabels bool, role PostgresRole) labels.Set { @@ -483,3 +508,15 @@ func (c *Cluster) GetSpec() (*acidv1.Postgresql, error) { func (c *Cluster) patroniUsesKubernetes() bool { return c.OpConfig.EtcdHost == "" } + +func (c *Cluster) needConnectionPoolWorker(spec *acidv1.PostgresSpec) bool { + if spec.EnableConnectionPool == nil { + return spec.ConnectionPool != nil + } else { + return *spec.EnableConnectionPool + } +} + +func (c *Cluster) needConnectionPool() bool { + return c.needConnectionPoolWorker(&c.Spec) +} diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 03602c3bd..970eef701 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -8,6 +8,7 @@ import ( acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" + "github.com/zalando/postgres-operator/pkg/util/constants" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -21,6 +22,10 @@ func (c *Controller) readOperatorConfigurationFromCRD(configObjectNamespace, con return config, nil } +func int32ToPointer(value int32) *int32 { + return &value +} + // importConfigurationFromCRD is a transitional function that converts CRD configuration to the one based on the configmap func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigurationData) *config.Config { result := &config.Config{} @@ -143,5 +148,51 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.ScalyrCPULimit = fromCRD.Scalyr.ScalyrCPULimit result.ScalyrMemoryLimit = fromCRD.Scalyr.ScalyrMemoryLimit + // Connection pool. Looks like we can't use defaulting in CRD before 1.17, + // so ensure default values here. + result.ConnectionPool.NumberOfInstances = util.CoalesceInt32( + fromCRD.ConnectionPool.NumberOfInstances, + int32ToPointer(2)) + + result.ConnectionPool.NumberOfInstances = util.MaxInt32( + result.ConnectionPool.NumberOfInstances, + int32ToPointer(2)) + + result.ConnectionPool.Schema = util.Coalesce( + fromCRD.ConnectionPool.Schema, + constants.ConnectionPoolSchemaName) + + result.ConnectionPool.User = util.Coalesce( + fromCRD.ConnectionPool.User, + constants.ConnectionPoolUserName) + + result.ConnectionPool.Image = util.Coalesce( + fromCRD.ConnectionPool.Image, + "registry.opensource.zalan.do/acid/pgbouncer") + + result.ConnectionPool.Mode = util.Coalesce( + fromCRD.ConnectionPool.Mode, + constants.ConnectionPoolDefaultMode) + + result.ConnectionPool.ConnPoolDefaultCPURequest = util.Coalesce( + fromCRD.ConnectionPool.DefaultCPURequest, + constants.ConnectionPoolDefaultCpuRequest) + + result.ConnectionPool.ConnPoolDefaultMemoryRequest = util.Coalesce( + fromCRD.ConnectionPool.DefaultMemoryRequest, + constants.ConnectionPoolDefaultMemoryRequest) + + result.ConnectionPool.ConnPoolDefaultCPULimit = util.Coalesce( + fromCRD.ConnectionPool.DefaultCPULimit, + constants.ConnectionPoolDefaultCpuLimit) + + result.ConnectionPool.ConnPoolDefaultMemoryLimit = util.Coalesce( + fromCRD.ConnectionPool.DefaultMemoryLimit, + constants.ConnectionPoolDefaultMemoryLimit) + + result.ConnectionPool.MaxDBConnections = util.CoalesceInt32( + fromCRD.ConnectionPool.MaxDBConnections, + int32ToPointer(constants.ConnPoolMaxDBConnections)) + return result } diff --git a/pkg/spec/types.go b/pkg/spec/types.go index 3e6bec8db..36783204d 100644 --- a/pkg/spec/types.go +++ b/pkg/spec/types.go @@ -23,13 +23,15 @@ const fileWithNamespace = "/var/run/secrets/kubernetes.io/serviceaccount/namespa // RoleOrigin contains the code of the origin of a role type RoleOrigin int -// The rolesOrigin constant values must be sorted by the role priority for resolveNameConflict(...) to work. +// The rolesOrigin constant values must be sorted by the role priority for +// resolveNameConflict(...) to work. const ( RoleOriginUnknown RoleOrigin = iota RoleOriginManifest RoleOriginInfrastructure RoleOriginTeamsAPI RoleOriginSystem + RoleConnectionPool ) type syncUserOperation int @@ -178,6 +180,8 @@ func (r RoleOrigin) String() string { return "teams API role" case RoleOriginSystem: return "system role" + case RoleConnectionPool: + return "connection pool role" default: panic(fmt.Sprintf("bogus role origin value %d", r)) } diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index fee65be81..e0e095617 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/zalando/postgres-operator/pkg/spec" + "github.com/zalando/postgres-operator/pkg/util/constants" ) // CRD describes CustomResourceDefinition specific configuration parameters @@ -83,6 +84,20 @@ type LogicalBackup struct { LogicalBackupS3SSE string `name:"logical_backup_s3_sse" default:"AES256"` } +// Operator options for connection pooler +type ConnectionPool struct { + NumberOfInstances *int32 `name:"connection_pool_number_of_instances" default:"2"` + Schema string `name:"connection_pool_schema" default:"pooler"` + User string `name:"connection_pool_user" default:"pooler"` + Image string `name:"connection_pool_image" default:"registry.opensource.zalan.do/acid/pgbouncer"` + Mode string `name:"connection_pool_mode" default:"transaction"` + MaxDBConnections *int32 `name:"connection_pool_max_db_connections" default:"60"` + ConnPoolDefaultCPURequest string `name:"connection_pool_default_cpu_request" default:"500m"` + ConnPoolDefaultMemoryRequest string `name:"connection_pool_default_memory_request" default:"100Mi"` + ConnPoolDefaultCPULimit string `name:"connection_pool_default_cpu_limit" default:"1"` + ConnPoolDefaultMemoryLimit string `name:"connection_pool_default_memory_limit" default:"100Mi"` +} + // Config describes operator config type Config struct { CRD @@ -90,6 +105,7 @@ type Config struct { Auth Scalyr LogicalBackup + ConnectionPool WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to' EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS @@ -196,5 +212,10 @@ func validate(cfg *Config) (err error) { if cfg.Workers == 0 { err = fmt.Errorf("number of workers should be higher than 0") } + + if *cfg.ConnectionPool.NumberOfInstances < constants.ConnPoolMinInstances { + msg := "number of connection pool instances should be higher than %d" + err = fmt.Errorf(msg, constants.ConnPoolMinInstances) + } return } diff --git a/pkg/util/constants/pooler.go b/pkg/util/constants/pooler.go new file mode 100644 index 000000000..540d64e2c --- /dev/null +++ b/pkg/util/constants/pooler.go @@ -0,0 +1,18 @@ +package constants + +// Connection pool specific constants +const ( + ConnectionPoolUserName = "pooler" + ConnectionPoolSchemaName = "pooler" + ConnectionPoolDefaultType = "pgbouncer" + ConnectionPoolDefaultMode = "transaction" + ConnectionPoolDefaultCpuRequest = "500m" + ConnectionPoolDefaultCpuLimit = "1" + ConnectionPoolDefaultMemoryRequest = "100Mi" + ConnectionPoolDefaultMemoryLimit = "100Mi" + + ConnPoolContainer = 0 + ConnPoolMaxDBConnections = 60 + ConnPoolMaxClientConnections = 10000 + ConnPoolMinInstances = 2 +) diff --git a/pkg/util/constants/roles.go b/pkg/util/constants/roles.go index 2c20d69db..3d201142c 100644 --- a/pkg/util/constants/roles.go +++ b/pkg/util/constants/roles.go @@ -2,15 +2,16 @@ package constants // Roles specific constants const ( - PasswordLength = 64 - SuperuserKeyName = "superuser" - ReplicationUserKeyName = "replication" - RoleFlagSuperuser = "SUPERUSER" - RoleFlagInherit = "INHERIT" - RoleFlagLogin = "LOGIN" - RoleFlagNoLogin = "NOLOGIN" - RoleFlagCreateRole = "CREATEROLE" - RoleFlagCreateDB = "CREATEDB" - RoleFlagReplication = "REPLICATION" - RoleFlagByPassRLS = "BYPASSRLS" + PasswordLength = 64 + SuperuserKeyName = "superuser" + ConnectionPoolUserKeyName = "pooler" + ReplicationUserKeyName = "replication" + RoleFlagSuperuser = "SUPERUSER" + RoleFlagInherit = "INHERIT" + RoleFlagLogin = "LOGIN" + RoleFlagNoLogin = "NOLOGIN" + RoleFlagCreateRole = "CREATEROLE" + RoleFlagCreateDB = "CREATEDB" + RoleFlagReplication = "REPLICATION" + RoleFlagByPassRLS = "BYPASSRLS" ) diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index 509b12c19..75b99ec7c 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -9,11 +9,13 @@ import ( batchv1beta1 "k8s.io/api/batch/v1beta1" clientbatchv1beta1 "k8s.io/client-go/kubernetes/typed/batch/v1beta1" + apiappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" policybeta1 "k8s.io/api/policy/v1beta1" apiextclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apiextbeta1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" appsv1 "k8s.io/client-go/kubernetes/typed/apps/v1" corev1 "k8s.io/client-go/kubernetes/typed/core/v1" @@ -26,6 +28,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +func Int32ToPointer(value int32) *int32 { + return &value +} + // KubernetesClient describes getters for Kubernetes objects type KubernetesClient struct { corev1.SecretsGetter @@ -39,6 +45,7 @@ type KubernetesClient struct { corev1.NamespacesGetter corev1.ServiceAccountsGetter appsv1.StatefulSetsGetter + appsv1.DeploymentsGetter rbacv1.RoleBindingsGetter policyv1beta1.PodDisruptionBudgetsGetter apiextbeta1.CustomResourceDefinitionsGetter @@ -55,6 +62,34 @@ type mockSecret struct { type MockSecretGetter struct { } +type mockDeployment struct { + appsv1.DeploymentInterface +} + +type mockDeploymentNotExist struct { + appsv1.DeploymentInterface +} + +type MockDeploymentGetter struct { +} + +type MockDeploymentNotExistGetter struct { +} + +type mockService struct { + corev1.ServiceInterface +} + +type mockServiceNotExist struct { + corev1.ServiceInterface +} + +type MockServiceGetter struct { +} + +type MockServiceNotExistGetter struct { +} + type mockConfigMap struct { corev1.ConfigMapInterface } @@ -101,6 +136,7 @@ func NewFromConfig(cfg *rest.Config) (KubernetesClient, error) { kubeClient.NodesGetter = client.CoreV1() kubeClient.NamespacesGetter = client.CoreV1() kubeClient.StatefulSetsGetter = client.AppsV1() + kubeClient.DeploymentsGetter = client.AppsV1() kubeClient.PodDisruptionBudgetsGetter = client.PolicyV1beta1() kubeClient.RESTClient = client.CoreV1().RESTClient() kubeClient.RoleBindingsGetter = client.RbacV1() @@ -230,19 +266,145 @@ func (c *mockConfigMap) Get(name string, options metav1.GetOptions) (*v1.ConfigM } // Secrets to be mocked -func (c *MockSecretGetter) Secrets(namespace string) corev1.SecretInterface { +func (mock *MockSecretGetter) Secrets(namespace string) corev1.SecretInterface { return &mockSecret{} } // ConfigMaps to be mocked -func (c *MockConfigMapsGetter) ConfigMaps(namespace string) corev1.ConfigMapInterface { +func (mock *MockConfigMapsGetter) ConfigMaps(namespace string) corev1.ConfigMapInterface { return &mockConfigMap{} } +func (mock *MockDeploymentGetter) Deployments(namespace string) appsv1.DeploymentInterface { + return &mockDeployment{} +} + +func (mock *MockDeploymentNotExistGetter) Deployments(namespace string) appsv1.DeploymentInterface { + return &mockDeploymentNotExist{} +} + +func (mock *mockDeployment) Create(*apiappsv1.Deployment) (*apiappsv1.Deployment, error) { + return &apiappsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment", + }, + Spec: apiappsv1.DeploymentSpec{ + Replicas: Int32ToPointer(1), + }, + }, nil +} + +func (mock *mockDeployment) Delete(name string, opts *metav1.DeleteOptions) error { + return nil +} + +func (mock *mockDeployment) Get(name string, opts metav1.GetOptions) (*apiappsv1.Deployment, error) { + return &apiappsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment", + }, + Spec: apiappsv1.DeploymentSpec{ + Replicas: Int32ToPointer(1), + Template: v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + v1.Container{ + Image: "pooler:1.0", + }, + }, + }, + }, + }, + }, nil +} + +func (mock *mockDeployment) Patch(name string, t types.PatchType, data []byte, subres ...string) (*apiappsv1.Deployment, error) { + return &apiappsv1.Deployment{ + Spec: apiappsv1.DeploymentSpec{ + Replicas: Int32ToPointer(2), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment", + }, + }, nil +} + +func (mock *mockDeploymentNotExist) Get(name string, opts metav1.GetOptions) (*apiappsv1.Deployment, error) { + return nil, &apierrors.StatusError{ + ErrStatus: metav1.Status{ + Reason: metav1.StatusReasonNotFound, + }, + } +} + +func (mock *mockDeploymentNotExist) Create(*apiappsv1.Deployment) (*apiappsv1.Deployment, error) { + return &apiappsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment", + }, + Spec: apiappsv1.DeploymentSpec{ + Replicas: Int32ToPointer(1), + }, + }, nil +} + +func (mock *MockServiceGetter) Services(namespace string) corev1.ServiceInterface { + return &mockService{} +} + +func (mock *MockServiceNotExistGetter) Services(namespace string) corev1.ServiceInterface { + return &mockServiceNotExist{} +} + +func (mock *mockService) Create(*v1.Service) (*v1.Service, error) { + return &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-service", + }, + }, nil +} + +func (mock *mockService) Delete(name string, opts *metav1.DeleteOptions) error { + return nil +} + +func (mock *mockService) Get(name string, opts metav1.GetOptions) (*v1.Service, error) { + return &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-service", + }, + }, nil +} + +func (mock *mockServiceNotExist) Create(*v1.Service) (*v1.Service, error) { + return &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-service", + }, + }, nil +} + +func (mock *mockServiceNotExist) Get(name string, opts metav1.GetOptions) (*v1.Service, error) { + return nil, &apierrors.StatusError{ + ErrStatus: metav1.Status{ + Reason: metav1.StatusReasonNotFound, + }, + } +} + // NewMockKubernetesClient for other tests func NewMockKubernetesClient() KubernetesClient { return KubernetesClient{ - SecretsGetter: &MockSecretGetter{}, - ConfigMapsGetter: &MockConfigMapsGetter{}, + SecretsGetter: &MockSecretGetter{}, + ConfigMapsGetter: &MockConfigMapsGetter{}, + DeploymentsGetter: &MockDeploymentGetter{}, + ServicesGetter: &MockServiceGetter{}, + } +} + +func ClientMissingObjects() KubernetesClient { + return KubernetesClient{ + DeploymentsGetter: &MockDeploymentNotExistGetter{}, + ServicesGetter: &MockServiceNotExistGetter{}, } } diff --git a/pkg/util/util.go b/pkg/util/util.go index d9803ab48..46df5d345 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -147,6 +147,40 @@ func Coalesce(val, defaultVal string) string { return val } +// Yeah, golang +func CoalesceInt32(val, defaultVal *int32) *int32 { + if val == nil { + return defaultVal + } + return val +} + +// Test if any of the values is nil +func testNil(values ...*int32) bool { + for _, v := range values { + if v == nil { + return true + } + } + + return false +} + +// Return maximum of two integers provided via pointers. If one value is not +// defined, return the other one. If both are not defined, result is also +// undefined, caller needs to check for that. +func MaxInt32(a, b *int32) *int32 { + if testNil(a, b) { + return nil + } + + if *a > *b { + return a + } + + return b +} + // IsSmallerQuantity : checks if first resource is of a smaller quantity than the second func IsSmallerQuantity(requestStr, limitStr string) (bool, error) { From ba9cf686502077babbda1803e2b54b83c6a153b9 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 25 Mar 2020 15:59:31 +0100 Subject: [PATCH 015/168] Change type of pod environment config map to NamespacedName (#870) * allow PodEnvironmentConfigMap in other namespaces * update codegen * update docs and comments --- charts/postgres-operator/values-crd.yaml | 8 +-- charts/postgres-operator/values.yaml | 8 +-- docs/administrator.md | 15 +++--- docs/reference/operator_parameters.md | 17 ++++--- manifests/configmap.yaml | 2 +- ...gresql-operator-default-configuration.yaml | 2 +- .../v1/operator_configuration_type.go | 15 +++--- .../acid.zalan.do/v1/zz_generated.deepcopy.go | 1 + pkg/cluster/k8sres.go | 19 ++++--- pkg/util/config/config.go | 50 +++++++++---------- 10 files changed, 73 insertions(+), 64 deletions(-) diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index cf9fbab15..79940b236 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -71,7 +71,7 @@ configKubernetes: enable_pod_disruption_budget: true # enables sidecar containers to run alongside Spilo in the same pod enable_sidecars: true - # name of the secret containing infrastructure roles names and passwords + # namespaced name of the secret containing infrastructure roles names and passwords # infrastructure_roles_secret_name: postgresql-infrastructure-roles # list of labels that can be inherited from the cluster manifest @@ -86,15 +86,15 @@ configKubernetes: # node_readiness_label: # status: ready - # name of the secret containing the OAuth2 token to pass to the teams API + # namespaced name of the secret containing the OAuth2 token to pass to the teams API # oauth_token_secret_name: postgresql-operator # defines the template for PDB (Pod Disruption Budget) names pdb_name_format: "postgres-{cluster}-pdb" # override topology key for pod anti affinity pod_antiaffinity_topology_key: "kubernetes.io/hostname" - # name of the ConfigMap with environment variables to populate on every pod - # pod_environment_configmap: "" + # namespaced name of the ConfigMap with environment variables to populate on every pod + # pod_environment_configmap: "default/my-custom-config" # specify the pod management policy of stateful sets of Postgres clusters pod_management_policy: "ordered_ready" diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 503bf4562..29f85339d 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -67,7 +67,7 @@ configKubernetes: enable_pod_disruption_budget: "true" # enables sidecar containers to run alongside Spilo in the same pod enable_sidecars: "true" - # name of the secret containing infrastructure roles names and passwords + # namespaced name of the secret containing infrastructure roles names and passwords # infrastructure_roles_secret_name: postgresql-infrastructure-roles # list of labels that can be inherited from the cluster manifest @@ -79,15 +79,15 @@ configKubernetes: # set of labels that a running and active node should possess to be considered ready # node_readiness_label: "" - # name of the secret containing the OAuth2 token to pass to the teams API + # namespaced name of the secret containing the OAuth2 token to pass to the teams API # oauth_token_secret_name: postgresql-operator # defines the template for PDB (Pod Disruption Budget) names pdb_name_format: "postgres-{cluster}-pdb" # override topology key for pod anti affinity pod_antiaffinity_topology_key: "kubernetes.io/hostname" - # name of the ConfigMap with environment variables to populate on every pod - # pod_environment_configmap: "" + # namespaced name of the ConfigMap with environment variables to populate on every pod + # pod_environment_configmap: "default/my-custom-config" # specify the pod management policy of stateful sets of Postgres clusters pod_management_policy: "ordered_ready" diff --git a/docs/administrator.md b/docs/administrator.md index a3a0f70cc..93adf2eb1 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -321,11 +321,12 @@ spec: ## Custom Pod Environment Variables It is possible to configure a ConfigMap which is used by the Postgres pods as -an additional provider for environment variables. - -One use case is to customize the Spilo image and configure it with environment -variables. The ConfigMap with the additional settings is configured in the -operator's main ConfigMap: +an additional provider for environment variables. One use case is to customize +the Spilo image and configure it with environment variables. The ConfigMap with +the additional settings is referenced in the operator's main configuration. +A namespace can be specified along with the name. If left out, the configured +default namespace of your K8s client will be used and if the ConfigMap is not +found there, the Postgres cluster's namespace is taken when different: **postgres-operator ConfigMap** @@ -336,7 +337,7 @@ metadata: name: postgres-operator data: # referencing config map with custom settings - pod_environment_configmap: postgres-pod-config + pod_environment_configmap: default/postgres-pod-config ``` **OperatorConfiguration** @@ -349,7 +350,7 @@ metadata: configuration: kubernetes: # referencing config map with custom settings - pod_environment_configmap: postgres-pod-config + pod_environment_configmap: default/postgres-pod-config ``` **referenced ConfigMap `postgres-pod-config`** diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 848fa1cf2..1ab92a287 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -221,11 +221,12 @@ configuration they are grouped under the `kubernetes` key. to the Postgres clusters after creation. * **oauth_token_secret_name** - a name of the secret containing the `OAuth2` token to pass to the teams API. - The default is `postgresql-operator`. + namespaced name of the secret containing the `OAuth2` token to pass to the + teams API. The default is `postgresql-operator`. * **infrastructure_roles_secret_name** - name of the secret containing infrastructure roles names and passwords. + namespaced name of the secret containing infrastructure roles names and + passwords. * **pod_role_label** name of the label assigned to the Postgres pods (and services/endpoints) by @@ -262,11 +263,11 @@ configuration they are grouped under the `kubernetes` key. for details on taints and tolerations. The default is empty. * **pod_environment_configmap** - a name of the ConfigMap with environment variables to populate on every pod. - Right now this ConfigMap is searched in the namespace of the Postgres cluster. - All variables from that ConfigMap are injected to the pod's environment, on - conflicts they are overridden by the environment variables generated by the - operator. The default is empty. + namespaced name of the ConfigMap with environment variables to populate on + every pod. Right now this ConfigMap is searched in the namespace of the + Postgres cluster. All variables from that ConfigMap are injected to the pod's + environment, on conflicts they are overridden by the environment variables + generated by the operator. The default is empty. * **pod_priority_class_name** a name of the [priority class](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass) diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index fdc2d5d56..67c3368f3 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -69,7 +69,7 @@ data: pdb_name_format: "postgres-{cluster}-pdb" # pod_antiaffinity_topology_key: "kubernetes.io/hostname" pod_deletion_wait_timeout: 10m - # pod_environment_configmap: "" + # pod_environment_configmap: "default/my-custom-config" pod_label_wait_timeout: 10m pod_management_policy: "ordered_ready" pod_role_label: spilo-role diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index d4c9b518f..9d609713c 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -40,7 +40,7 @@ configuration: oauth_token_secret_name: postgresql-operator pdb_name_format: "postgres-{cluster}-pdb" pod_antiaffinity_topology_key: "kubernetes.io/hostname" - # pod_environment_configmap: "" + # pod_environment_configmap: "default/my-custom-config" pod_management_policy: "ordered_ready" # pod_priority_class_name: "" pod_role_label: spilo-role diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 7c1d2b7e8..3dbe96b7f 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -65,14 +65,13 @@ type KubernetesMetaConfiguration struct { NodeReadinessLabel map[string]string `json:"node_readiness_label,omitempty"` CustomPodAnnotations map[string]string `json:"custom_pod_annotations,omitempty"` // TODO: use a proper toleration structure? - PodToleration map[string]string `json:"toleration,omitempty"` - // TODO: use namespacedname - PodEnvironmentConfigMap string `json:"pod_environment_configmap,omitempty"` - PodPriorityClassName string `json:"pod_priority_class_name,omitempty"` - MasterPodMoveTimeout Duration `json:"master_pod_move_timeout,omitempty"` - EnablePodAntiAffinity bool `json:"enable_pod_antiaffinity,omitempty"` - PodAntiAffinityTopologyKey string `json:"pod_antiaffinity_topology_key,omitempty"` - PodManagementPolicy string `json:"pod_management_policy,omitempty"` + PodToleration map[string]string `json:"toleration,omitempty"` + PodEnvironmentConfigMap spec.NamespacedName `json:"pod_environment_configmap,omitempty"` + PodPriorityClassName string `json:"pod_priority_class_name,omitempty"` + MasterPodMoveTimeout Duration `json:"master_pod_move_timeout,omitempty"` + EnablePodAntiAffinity bool `json:"enable_pod_antiaffinity,omitempty"` + PodAntiAffinityTopologyKey string `json:"pod_antiaffinity_topology_key,omitempty"` + PodManagementPolicy string `json:"pod_management_policy,omitempty"` } // PostgresPodResourcesDefaults defines the spec of default resources diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index fcab394ca..65a19600a 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -179,6 +179,7 @@ func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfigura (*out)[key] = val } } + out.PodEnvironmentConfigMap = in.PodEnvironmentConfigMap return } diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 34b409d4e..2c40bb0ba 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -18,6 +18,7 @@ import ( acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/spec" + pkgspec "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/constants" @@ -485,9 +486,9 @@ func generateSidecarContainers(sidecars []acidv1.Sidecar, // Check whether or not we're requested to mount an shm volume, // taking into account that PostgreSQL manifest has precedence. -func mountShmVolumeNeeded(opConfig config.Config, pgSpec *acidv1.PostgresSpec) *bool { - if pgSpec.ShmVolume != nil && *pgSpec.ShmVolume { - return pgSpec.ShmVolume +func mountShmVolumeNeeded(opConfig config.Config, spec *acidv1.PostgresSpec) *bool { + if spec.ShmVolume != nil && *spec.ShmVolume { + return spec.ShmVolume } return opConfig.ShmVolume @@ -911,11 +912,17 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef customPodEnvVarsList := make([]v1.EnvVar, 0) - if c.OpConfig.PodEnvironmentConfigMap != "" { + if c.OpConfig.PodEnvironmentConfigMap != (pkgspec.NamespacedName{}) { var cm *v1.ConfigMap - cm, err = c.KubeClient.ConfigMaps(c.Namespace).Get(c.OpConfig.PodEnvironmentConfigMap, metav1.GetOptions{}) + cm, err = c.KubeClient.ConfigMaps(c.OpConfig.PodEnvironmentConfigMap.Namespace).Get(c.OpConfig.PodEnvironmentConfigMap.Name, metav1.GetOptions{}) if err != nil { - return nil, fmt.Errorf("could not read PodEnvironmentConfigMap: %v", err) + // if not found, try again using the cluster's namespace if it's different (old behavior) + if k8sutil.ResourceNotFound(err) && c.Namespace != c.OpConfig.PodEnvironmentConfigMap.Namespace { + cm, err = c.KubeClient.ConfigMaps(c.Namespace).Get(c.OpConfig.PodEnvironmentConfigMap.Name, metav1.GetOptions{}) + } + if err != nil { + return nil, fmt.Errorf("could not read PodEnvironmentConfigMap: %v", err) + } } for k, v := range cm.Data { customPodEnvVarsList = append(customPodEnvVarsList, v1.EnvVar{Name: k, Value: v}) diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index e0e095617..403615f06 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -22,31 +22,31 @@ type CRD struct { // Resources describes kubernetes resource specific configuration parameters type Resources struct { - ResourceCheckInterval time.Duration `name:"resource_check_interval" default:"3s"` - ResourceCheckTimeout time.Duration `name:"resource_check_timeout" default:"10m"` - PodLabelWaitTimeout time.Duration `name:"pod_label_wait_timeout" default:"10m"` - PodDeletionWaitTimeout time.Duration `name:"pod_deletion_wait_timeout" default:"10m"` - PodTerminateGracePeriod time.Duration `name:"pod_terminate_grace_period" default:"5m"` - SpiloFSGroup *int64 `name:"spilo_fsgroup"` - PodPriorityClassName string `name:"pod_priority_class_name"` - ClusterDomain string `name:"cluster_domain" default:"cluster.local"` - SpiloPrivileged bool `name:"spilo_privileged" default:"false"` - ClusterLabels map[string]string `name:"cluster_labels" default:"application:spilo"` - InheritedLabels []string `name:"inherited_labels" default:""` - ClusterNameLabel string `name:"cluster_name_label" default:"cluster-name"` - PodRoleLabel string `name:"pod_role_label" default:"spilo-role"` - PodToleration map[string]string `name:"toleration" default:""` - DefaultCPURequest string `name:"default_cpu_request" default:"100m"` - DefaultMemoryRequest string `name:"default_memory_request" default:"100Mi"` - DefaultCPULimit string `name:"default_cpu_limit" default:"1"` - DefaultMemoryLimit string `name:"default_memory_limit" default:"500Mi"` - MinCPULimit string `name:"min_cpu_limit" default:"250m"` - MinMemoryLimit string `name:"min_memory_limit" default:"250Mi"` - PodEnvironmentConfigMap string `name:"pod_environment_configmap" default:""` - NodeReadinessLabel map[string]string `name:"node_readiness_label" default:""` - MaxInstances int32 `name:"max_instances" default:"-1"` - MinInstances int32 `name:"min_instances" default:"-1"` - ShmVolume *bool `name:"enable_shm_volume" default:"true"` + ResourceCheckInterval time.Duration `name:"resource_check_interval" default:"3s"` + ResourceCheckTimeout time.Duration `name:"resource_check_timeout" default:"10m"` + PodLabelWaitTimeout time.Duration `name:"pod_label_wait_timeout" default:"10m"` + PodDeletionWaitTimeout time.Duration `name:"pod_deletion_wait_timeout" default:"10m"` + PodTerminateGracePeriod time.Duration `name:"pod_terminate_grace_period" default:"5m"` + SpiloFSGroup *int64 `name:"spilo_fsgroup"` + PodPriorityClassName string `name:"pod_priority_class_name"` + ClusterDomain string `name:"cluster_domain" default:"cluster.local"` + SpiloPrivileged bool `name:"spilo_privileged" default:"false"` + ClusterLabels map[string]string `name:"cluster_labels" default:"application:spilo"` + InheritedLabels []string `name:"inherited_labels" default:""` + ClusterNameLabel string `name:"cluster_name_label" default:"cluster-name"` + PodRoleLabel string `name:"pod_role_label" default:"spilo-role"` + PodToleration map[string]string `name:"toleration" default:""` + DefaultCPURequest string `name:"default_cpu_request" default:"100m"` + DefaultMemoryRequest string `name:"default_memory_request" default:"100Mi"` + DefaultCPULimit string `name:"default_cpu_limit" default:"1"` + DefaultMemoryLimit string `name:"default_memory_limit" default:"500Mi"` + MinCPULimit string `name:"min_cpu_limit" default:"250m"` + MinMemoryLimit string `name:"min_memory_limit" default:"250Mi"` + PodEnvironmentConfigMap spec.NamespacedName `name:"pod_environment_configmap"` + NodeReadinessLabel map[string]string `name:"node_readiness_label" default:""` + MaxInstances int32 `name:"max_instances" default:"-1"` + MinInstances int32 `name:"min_instances" default:"-1"` + ShmVolume *bool `name:"enable_shm_volume" default:"true"` } // Auth describes authentication specific configuration parameters From 66f2cda87fb041a6cc0151d2b442aad2c9755075 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 30 Mar 2020 15:50:17 +0200 Subject: [PATCH 016/168] Move operator to go 1.14 (#882) * update go modules march 2020 * update to GO 1.14 * reflect k8s client API changes --- .travis.yml | 2 +- delivery.yaml | 2 +- go.mod | 31 ++-- go.sum | 168 +++++++++--------- pkg/cluster/cluster.go | 20 ++- pkg/cluster/exec.go | 5 +- pkg/cluster/k8sres.go | 11 +- pkg/cluster/pod.go | 21 ++- pkg/cluster/resources.go | 85 +++++---- pkg/cluster/sync.go | 33 ++-- pkg/cluster/util.go | 11 +- pkg/cluster/volumes.go | 11 +- pkg/controller/controller.go | 5 +- pkg/controller/node.go | 7 +- pkg/controller/operator_config.go | 4 +- pkg/controller/pod.go | 8 +- pkg/controller/postgresql.go | 11 +- pkg/controller/util.go | 13 +- .../clientset/versioned/clientset.go | 2 +- .../v1/fake/fake_operatorconfiguration.go | 4 +- .../acid.zalan.do/v1/fake/fake_postgresql.go | 22 +-- .../acid.zalan.do/v1/operatorconfiguration.go | 8 +- .../typed/acid.zalan.do/v1/postgresql.go | 72 ++++---- .../acid.zalan.do/v1/postgresql.go | 5 +- pkg/util/k8sutil/k8sutil.go | 27 +-- pkg/util/teams/teams_test.go | 4 +- 26 files changed, 319 insertions(+), 273 deletions(-) diff --git a/.travis.yml b/.travis.yml index 589eb03a4..091c875ba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ branches: language: go go: - - "1.12.x" + - "1.14.x" before_install: - go get github.com/mattn/goveralls diff --git a/delivery.yaml b/delivery.yaml index 144448ea9..d0884f982 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -12,7 +12,7 @@ pipeline: - desc: 'Install go' cmd: | cd /tmp - wget -q https://storage.googleapis.com/golang/go1.12.linux-amd64.tar.gz -O go.tar.gz + wget -q https://storage.googleapis.com/golang/go1.14.linux-amd64.tar.gz -O go.tar.gz tar -xf go.tar.gz mv go /usr/local ln -s /usr/local/go/bin/go /usr/bin/go diff --git a/go.mod b/go.mod index be1ec32d4..8b9f55509 100644 --- a/go.mod +++ b/go.mod @@ -1,26 +1,19 @@ module github.com/zalando/postgres-operator -go 1.12 +go 1.14 require ( - github.com/aws/aws-sdk-go v1.25.44 - github.com/emicklei/go-restful v2.9.6+incompatible // indirect - github.com/evanphx/json-patch v4.5.0+incompatible // indirect - github.com/googleapis/gnostic v0.3.0 // indirect - github.com/imdario/mergo v0.3.8 // indirect - github.com/lib/pq v1.2.0 + github.com/aws/aws-sdk-go v1.29.33 + github.com/lib/pq v1.3.0 github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d - github.com/sirupsen/logrus v1.4.2 + github.com/r3labs/diff v0.0.0-20191120142937-b4ed99a31f5a + github.com/sirupsen/logrus v1.5.0 github.com/stretchr/testify v1.4.0 - golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 // indirect - golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect - golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 // indirect - golang.org/x/tools v0.0.0-20191209225234-22774f7dae43 // indirect - gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect - gopkg.in/yaml.v2 v2.2.4 - k8s.io/api v0.0.0-20191121015604-11707872ac1c - k8s.io/apiextensions-apiserver v0.0.0-20191204090421-cd61debedab5 - k8s.io/apimachinery v0.0.0-20191203211716-adc6f4cd9e7d - k8s.io/client-go v0.0.0-20191204082520-bc9b51d240b2 - k8s.io/code-generator v0.0.0-20191121015212-c4c8f8345c7e + golang.org/x/tools v0.0.0-20200326210457-5d86d385bf88 // indirect + gopkg.in/yaml.v2 v2.2.8 + k8s.io/api v0.18.0 + k8s.io/apiextensions-apiserver v0.18.0 + k8s.io/apimachinery v0.18.0 + k8s.io/client-go v0.18.0 + k8s.io/code-generator v0.18.0 ) diff --git a/go.sum b/go.sum index 0737d3a5d..e8607922b 100644 --- a/go.sum +++ b/go.sum @@ -11,7 +11,6 @@ github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6L github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -27,12 +26,13 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.25.44 h1:n9ahFoiyn66smjF34hYr3tb6/ZdBcLuFz7BCDhHyJ7I= -github.com/aws/aws-sdk-go v1.25.44/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.29.33 h1:WP85+WHalTFQR2wYp5xR2sjiVAZXew2bBQXGU1QJBXI= +github.com/aws/aws-sdk-go v1.29.33/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -46,7 +46,6 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -58,15 +57,15 @@ github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QL github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e h1:p1yVGRW3nmb85p1Sh1ZJSDm4A4iKLS5QNbvUHMgGu/M= -github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.6+incompatible h1:tfrHha8zJ01ywiOEC1miGY8st1/igzWB8OmvPgoYX7w= -github.com/emicklei/go-restful v2.9.6+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= -github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -124,11 +123,12 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I= -github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -136,6 +136,7 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -144,18 +145,20 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0= -github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.1.0 h1:rVsPeBmXbYv4If/cumu1AzZPwV58q433hvONV1UEZoI= +github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -169,15 +172,14 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ= -github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -195,8 +197,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= +github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -214,7 +216,6 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -227,8 +228,8 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= -github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -236,9 +237,8 @@ github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= @@ -246,26 +246,30 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= +github.com/r3labs/diff v0.0.0-20191120142937-b4ed99a31f5a h1:2v4Ipjxa3sh+xn6GvtgrMub2ci4ZLQMvTaYIba2lfdc= +github.com/r3labs/diff v0.0.0-20191120142937-b4ed99a31f5a/go.mod h1:ozniNEFS3j1qCwHKdvraMn1WJOsUxHd7lYfukEIS4cs= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q= +github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -273,7 +277,6 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -286,6 +289,7 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -302,19 +306,17 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g= -golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 h1:/Tl7pH94bvbAAHBdZJT947M/+gp0+CqQXDtMRC0fseo= +golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495 h1:I6A9Ag9FpEKOjcKrRNjQkPHawoXIhKyTGfvvjFAiiAk= -golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -332,8 +334,9 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= @@ -343,6 +346,7 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -352,20 +356,21 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f h1:25KHgbfyiSm6vwQLbM3zZIe1v9p/3ea4Rz+nnM5K/i4= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 h1:gSbV7h1NRL2G1xTg/owz62CST1oJBmxy4QpMMregXVQ= -golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 h1:HmbHVPwrPEKPGLAcHSrMe6+hqSUlvZU0rab6x5EXfGU= +golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -375,23 +380,20 @@ golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191209225234-22774f7dae43 h1:NfPq5mgc5ArFgVLCpeS4z07IoxSAqVfV/gQ5vxdgaxI= -golang.org/x/tools v0.0.0-20191209225234-22774f7dae43/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200326210457-5d86d385bf88 h1:F7fM2kxXfuWw820fa+MMCCLH6hmYe+jtLnZpwoiLK4Q= +golang.org/x/tools v0.0.0-20200326210457-5d86d385bf88/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485 h1:OB/uP/Puiu5vS5QMRPrXCDWUPb+kt8f1KW8oQzFejQw= -gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= -gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= -gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e h1:jRyg0XfpwWlhEV8mDfdNGBeSJM2fuyh9Yjrnd8kF2Ts= -gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -400,14 +402,15 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -423,45 +426,40 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.0.0-20191121015604-11707872ac1c h1:Z87my3sF4WhG0OMxzARkWY/IKBtOr+MhXZAb4ts6qFc= -k8s.io/api v0.0.0-20191121015604-11707872ac1c/go.mod h1:R/s4gKT0V/cWEnbQa9taNRJNbWUK57/Dx6cPj6MD3A0= -k8s.io/apiextensions-apiserver v0.0.0-20191204090421-cd61debedab5 h1:g+GvnbGqLU1Jxb/9iFm/BFcmkqG9HdsGh52+wHirpsM= -k8s.io/apiextensions-apiserver v0.0.0-20191204090421-cd61debedab5/go.mod h1:CPw0IHz1YrWGy0+8mG/76oTHXvChlgCb3EAezKQKB2I= -k8s.io/apimachinery v0.0.0-20191121015412-41065c7a8c2a/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= -k8s.io/apimachinery v0.0.0-20191123233150-4c4803ed55e3/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= -k8s.io/apimachinery v0.0.0-20191128180518-03184f823e28/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= -k8s.io/apimachinery v0.0.0-20191203211716-adc6f4cd9e7d h1:q+OZmYewHJeMCzwpHkXlNTtk5bvaUMPCikKvf77RBlo= -k8s.io/apimachinery v0.0.0-20191203211716-adc6f4cd9e7d/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= -k8s.io/apiserver v0.0.0-20191204084332-137a9d3b886b/go.mod h1:itgfam5HJbT/4b2BGfpUkkxfheMmDH+Ix+tEAP3uqZk= -k8s.io/client-go v0.0.0-20191204082517-8c19b9f4a642/go.mod h1:HMVIZ0dPop3WCrPEaJ+v5/94cjt56avdDFshpX0Fjvo= -k8s.io/client-go v0.0.0-20191204082519-e9644b2e3edc/go.mod h1:5lSG1yeDZVwDYAHe9VK48SCe5zmcnkAcf2Mx59TuhmM= -k8s.io/client-go v0.0.0-20191204082520-bc9b51d240b2 h1:T2HGghBOPAOEjWuIyFSeCsWEwsxa6unkBvy3PHfqonM= -k8s.io/client-go v0.0.0-20191204082520-bc9b51d240b2/go.mod h1:5lSG1yeDZVwDYAHe9VK48SCe5zmcnkAcf2Mx59TuhmM= -k8s.io/code-generator v0.0.0-20191121015212-c4c8f8345c7e h1:HB9Zu5ZUvJfNpLiTPhz+CebVKV8C39qTBMQkAgAZLNw= -k8s.io/code-generator v0.0.0-20191121015212-c4c8f8345c7e/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s= -k8s.io/component-base v0.0.0-20191204083903-0d4d24e738e4/go.mod h1:8VIh1jErItC4bg9hLBkPneyS77Tin8KwSzbYepHJnQI= -k8s.io/component-base v0.0.0-20191204083906-3ac1376c73aa/go.mod h1:mECWvHCPhJudDVDMtBl+AIf/YnTMp5r1F947OYFUwP0= +k8s.io/api v0.18.0 h1:lwYk8Vt7rsVTwjRU6pzEsa9YNhThbmbocQlKvNBB4EQ= +k8s.io/api v0.18.0/go.mod h1:q2HRQkfDzHMBZL9l/y9rH63PkQl4vae0xRT+8prbrK8= +k8s.io/apiextensions-apiserver v0.18.0 h1:HN4/P8vpGZFvB5SOMuPPH2Wt9Y/ryX+KRvIyAkchu1Q= +k8s.io/apiextensions-apiserver v0.18.0/go.mod h1:18Cwn1Xws4xnWQNC00FLq1E350b9lUF+aOdIWDOZxgo= +k8s.io/apimachinery v0.18.0 h1:fuPfYpk3cs1Okp/515pAf0dNhL66+8zk8RLbSX+EgAE= +k8s.io/apimachinery v0.18.0/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= +k8s.io/apiserver v0.18.0/go.mod h1:3S2O6FeBBd6XTo0njUrLxiqk8GNy6wWOftjhJcXYnjw= +k8s.io/client-go v0.18.0 h1:yqKw4cTUQraZK3fcVCMeSa+lqKwcjZ5wtcOIPnxQno4= +k8s.io/client-go v0.18.0/go.mod h1:uQSYDYs4WhVZ9i6AIoEZuwUggLVEF64HOD37boKAtF8= +k8s.io/code-generator v0.18.0 h1:0xIRWzym+qMgVpGmLESDeMfz/orwgxwxFFAo1xfGNtQ= +k8s.io/code-generator v0.18.0/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= +k8s.io/component-base v0.18.0/go.mod h1:u3BCg0z1uskkzrnAKFzulmYaEpZF7XC9Pf/uFyb1v2c= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20190822140433-26a664648505 h1:ZY6yclUKVbZ+SdWnkfY+Je5vrMpKOxmGeKRbsXVmqYM= -k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20200114144118-36b2048a9120 h1:RPscN6KhmG54S33L+lr3GS+oD1jmchIU0ll519K6FA4= +k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= -k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU= -k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= -k8s.io/utils v0.0.0-20191114184206-e782cd3c129f h1:GiPwtSzdP43eI1hpPCbROQCCIgCuiMMNF8YUVLF3vJo= -k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= -modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= -modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= -modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= -modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= -sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= -sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06/go.mod h1:/ULNhyfzRopfcjskuui0cTITekDduZ7ycKN3oUT9R18= +k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c h1:/KUFqjjqAcY4Us6luF5RDNZ16KJtb49HfR3ZHB9qYXM= +k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= +k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89 h1:d4vVOjXm687F1iLSP2q3lyPPuyvTUt3aVoBpi2DqRsU= +k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index dba67c142..45ee55300 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -3,6 +3,7 @@ package cluster // Postgres CustomResourceDefinition object i.e. Spilo import ( + "context" "database/sql" "encoding/json" "fmt" @@ -88,7 +89,7 @@ type Cluster struct { pgDb *sql.DB mu sync.Mutex userSyncStrategy spec.UserSyncer - deleteOptions *metav1.DeleteOptions + deleteOptions metav1.DeleteOptions podEventsQueue *cache.FIFO teamsAPIClient teams.Interface @@ -131,7 +132,7 @@ func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgres Services: make(map[PostgresRole]*v1.Service), Endpoints: make(map[PostgresRole]*v1.Endpoints)}, userSyncStrategy: users.DefaultUserSyncStrategy{}, - deleteOptions: &metav1.DeleteOptions{PropagationPolicy: &deletePropagationPolicy}, + deleteOptions: metav1.DeleteOptions{PropagationPolicy: &deletePropagationPolicy}, podEventsQueue: podEventsQueue, KubeClient: kubeClient, } @@ -182,7 +183,8 @@ func (c *Cluster) setStatus(status string) { // we cannot do a full scale update here without fetching the previous manifest (as the resourceVersion may differ), // however, we could do patch without it. In the future, once /status subresource is there (starting Kubernetes 1.11) // we should take advantage of it. - newspec, err := c.KubeClient.AcidV1ClientSet.AcidV1().Postgresqls(c.clusterNamespace()).Patch(c.Name, types.MergePatchType, patch, "status") + newspec, err := c.KubeClient.AcidV1ClientSet.AcidV1().Postgresqls(c.clusterNamespace()).Patch( + context.TODO(), c.Name, types.MergePatchType, patch, metav1.PatchOptions{}, "status") if err != nil { c.logger.Errorf("could not update status: %v", err) // return as newspec is empty, see PR654 @@ -1185,12 +1187,12 @@ func (c *Cluster) deleteClusterObject( func (c *Cluster) deletePatroniClusterServices() error { get := func(name string) (spec.NamespacedName, error) { - svc, err := c.KubeClient.Services(c.Namespace).Get(name, metav1.GetOptions{}) + svc, err := c.KubeClient.Services(c.Namespace).Get(context.TODO(), name, metav1.GetOptions{}) return util.NameFromMeta(svc.ObjectMeta), err } deleteServiceFn := func(name string) error { - return c.KubeClient.Services(c.Namespace).Delete(name, c.deleteOptions) + return c.KubeClient.Services(c.Namespace).Delete(context.TODO(), name, c.deleteOptions) } return c.deleteClusterObject(get, deleteServiceFn, "service") @@ -1198,12 +1200,12 @@ func (c *Cluster) deletePatroniClusterServices() error { func (c *Cluster) deletePatroniClusterEndpoints() error { get := func(name string) (spec.NamespacedName, error) { - ep, err := c.KubeClient.Endpoints(c.Namespace).Get(name, metav1.GetOptions{}) + ep, err := c.KubeClient.Endpoints(c.Namespace).Get(context.TODO(), name, metav1.GetOptions{}) return util.NameFromMeta(ep.ObjectMeta), err } deleteEndpointFn := func(name string) error { - return c.KubeClient.Endpoints(c.Namespace).Delete(name, c.deleteOptions) + return c.KubeClient.Endpoints(c.Namespace).Delete(context.TODO(), name, c.deleteOptions) } return c.deleteClusterObject(get, deleteEndpointFn, "endpoint") @@ -1211,12 +1213,12 @@ func (c *Cluster) deletePatroniClusterEndpoints() error { func (c *Cluster) deletePatroniClusterConfigMaps() error { get := func(name string) (spec.NamespacedName, error) { - cm, err := c.KubeClient.ConfigMaps(c.Namespace).Get(name, metav1.GetOptions{}) + cm, err := c.KubeClient.ConfigMaps(c.Namespace).Get(context.TODO(), name, metav1.GetOptions{}) return util.NameFromMeta(cm.ObjectMeta), err } deleteConfigMapFn := func(name string) error { - return c.KubeClient.ConfigMaps(c.Namespace).Delete(name, c.deleteOptions) + return c.KubeClient.ConfigMaps(c.Namespace).Delete(context.TODO(), name, c.deleteOptions) } return c.deleteClusterObject(get, deleteConfigMapFn, "configmap") diff --git a/pkg/cluster/exec.go b/pkg/cluster/exec.go index 8dd6bd91d..8b5089b4e 100644 --- a/pkg/cluster/exec.go +++ b/pkg/cluster/exec.go @@ -2,10 +2,11 @@ package cluster import ( "bytes" + "context" "fmt" "strings" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/remotecommand" @@ -23,7 +24,7 @@ func (c *Cluster) ExecCommand(podName *spec.NamespacedName, command ...string) ( execErr bytes.Buffer ) - pod, err := c.KubeClient.Pods(podName.Namespace).Get(podName.Name, metav1.GetOptions{}) + pod, err := c.KubeClient.Pods(podName.Namespace).Get(context.TODO(), podName.Name, metav1.GetOptions{}) if err != nil { return "", fmt.Errorf("could not get pod info: %v", err) } diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 2c40bb0ba..c4919c62d 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1,6 +1,7 @@ package cluster import ( + "context" "encoding/json" "fmt" "path" @@ -914,11 +915,17 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef if c.OpConfig.PodEnvironmentConfigMap != (pkgspec.NamespacedName{}) { var cm *v1.ConfigMap - cm, err = c.KubeClient.ConfigMaps(c.OpConfig.PodEnvironmentConfigMap.Namespace).Get(c.OpConfig.PodEnvironmentConfigMap.Name, metav1.GetOptions{}) + cm, err = c.KubeClient.ConfigMaps(c.OpConfig.PodEnvironmentConfigMap.Namespace).Get( + context.TODO(), + c.OpConfig.PodEnvironmentConfigMap.Name, + metav1.GetOptions{}) if err != nil { // if not found, try again using the cluster's namespace if it's different (old behavior) if k8sutil.ResourceNotFound(err) && c.Namespace != c.OpConfig.PodEnvironmentConfigMap.Namespace { - cm, err = c.KubeClient.ConfigMaps(c.Namespace).Get(c.OpConfig.PodEnvironmentConfigMap.Name, metav1.GetOptions{}) + cm, err = c.KubeClient.ConfigMaps(c.Namespace).Get( + context.TODO(), + c.OpConfig.PodEnvironmentConfigMap.Name, + metav1.GetOptions{}) } if err != nil { return nil, fmt.Errorf("could not read PodEnvironmentConfigMap: %v", err) diff --git a/pkg/cluster/pod.go b/pkg/cluster/pod.go index 095f859f0..9991621cc 100644 --- a/pkg/cluster/pod.go +++ b/pkg/cluster/pod.go @@ -1,6 +1,7 @@ package cluster import ( + "context" "fmt" "math/rand" @@ -17,7 +18,7 @@ func (c *Cluster) listPods() ([]v1.Pod, error) { LabelSelector: c.labelsSet(false).String(), } - pods, err := c.KubeClient.Pods(c.Namespace).List(listOptions) + pods, err := c.KubeClient.Pods(c.Namespace).List(context.TODO(), listOptions) if err != nil { return nil, fmt.Errorf("could not get list of pods: %v", err) } @@ -30,7 +31,7 @@ func (c *Cluster) getRolePods(role PostgresRole) ([]v1.Pod, error) { LabelSelector: c.roleLabelsSet(false, role).String(), } - pods, err := c.KubeClient.Pods(c.Namespace).List(listOptions) + pods, err := c.KubeClient.Pods(c.Namespace).List(context.TODO(), listOptions) if err != nil { return nil, fmt.Errorf("could not get list of pods: %v", err) } @@ -73,7 +74,7 @@ func (c *Cluster) deletePod(podName spec.NamespacedName) error { ch := c.registerPodSubscriber(podName) defer c.unregisterPodSubscriber(podName) - if err := c.KubeClient.Pods(podName.Namespace).Delete(podName.Name, c.deleteOptions); err != nil { + if err := c.KubeClient.Pods(podName.Namespace).Delete(context.TODO(), podName.Name, c.deleteOptions); err != nil { return err } @@ -183,7 +184,7 @@ func (c *Cluster) MigrateMasterPod(podName spec.NamespacedName) error { eol bool ) - oldMaster, err := c.KubeClient.Pods(podName.Namespace).Get(podName.Name, metav1.GetOptions{}) + oldMaster, err := c.KubeClient.Pods(podName.Namespace).Get(context.TODO(), podName.Name, metav1.GetOptions{}) if err != nil { return fmt.Errorf("could not get pod: %v", err) @@ -206,7 +207,9 @@ func (c *Cluster) MigrateMasterPod(podName spec.NamespacedName) error { // we must have a statefulset in the cluster for the migration to work if c.Statefulset == nil { var sset *appsv1.StatefulSet - if sset, err = c.KubeClient.StatefulSets(c.Namespace).Get(c.statefulSetName(), + if sset, err = c.KubeClient.StatefulSets(c.Namespace).Get( + context.TODO(), + c.statefulSetName(), metav1.GetOptions{}); err != nil { return fmt.Errorf("could not retrieve cluster statefulset: %v", err) } @@ -247,7 +250,7 @@ func (c *Cluster) MigrateMasterPod(podName spec.NamespacedName) error { // MigrateReplicaPod recreates pod on a new node func (c *Cluster) MigrateReplicaPod(podName spec.NamespacedName, fromNodeName string) error { - replicaPod, err := c.KubeClient.Pods(podName.Namespace).Get(podName.Name, metav1.GetOptions{}) + replicaPod, err := c.KubeClient.Pods(podName.Namespace).Get(context.TODO(), podName.Name, metav1.GetOptions{}) if err != nil { return fmt.Errorf("could not get pod: %v", err) } @@ -276,7 +279,7 @@ func (c *Cluster) recreatePod(podName spec.NamespacedName) (*v1.Pod, error) { defer c.unregisterPodSubscriber(podName) stopChan := make(chan struct{}) - if err := c.KubeClient.Pods(podName.Namespace).Delete(podName.Name, c.deleteOptions); err != nil { + if err := c.KubeClient.Pods(podName.Namespace).Delete(context.TODO(), podName.Name, c.deleteOptions); err != nil { return nil, fmt.Errorf("could not delete pod: %v", err) } @@ -300,7 +303,7 @@ func (c *Cluster) recreatePods() error { LabelSelector: ls.String(), } - pods, err := c.KubeClient.Pods(namespace).List(listOptions) + pods, err := c.KubeClient.Pods(namespace).List(context.TODO(), listOptions) if err != nil { return fmt.Errorf("could not get the list of pods: %v", err) } @@ -349,7 +352,7 @@ func (c *Cluster) recreatePods() error { } func (c *Cluster) podIsEndOfLife(pod *v1.Pod) (bool, error) { - node, err := c.KubeClient.Nodes().Get(pod.Spec.NodeName, metav1.GetOptions{}) + node, err := c.KubeClient.Nodes().Get(context.TODO(), pod.Spec.NodeName, metav1.GetOptions{}) if err != nil { return false, err } diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index 2e02a9a83..c0c731ed8 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -1,6 +1,7 @@ package cluster import ( + "context" "fmt" "strconv" "strings" @@ -80,7 +81,10 @@ func (c *Cluster) createStatefulSet() (*appsv1.StatefulSet, error) { if err != nil { return nil, fmt.Errorf("could not generate statefulset: %v", err) } - statefulSet, err := c.KubeClient.StatefulSets(statefulSetSpec.Namespace).Create(statefulSetSpec) + statefulSet, err := c.KubeClient.StatefulSets(statefulSetSpec.Namespace).Create( + context.TODO(), + statefulSetSpec, + metav1.CreateOptions{}) if err != nil { return nil, err } @@ -129,7 +133,7 @@ func (c *Cluster) createConnectionPool(lookup InstallFunction) (*ConnectionPoolO // should be good enough to not think about it here. deployment, err := c.KubeClient. Deployments(deploymentSpec.Namespace). - Create(deploymentSpec) + Create(context.TODO(), deploymentSpec, metav1.CreateOptions{}) if err != nil { return nil, err @@ -138,7 +142,7 @@ func (c *Cluster) createConnectionPool(lookup InstallFunction) (*ConnectionPoolO serviceSpec := c.generateConnPoolService(&c.Spec) service, err := c.KubeClient. Services(serviceSpec.Namespace). - Create(serviceSpec) + Create(context.TODO(), serviceSpec, metav1.CreateOptions{}) if err != nil { return nil, err @@ -180,7 +184,7 @@ func (c *Cluster) deleteConnectionPool() (err error) { options := metav1.DeleteOptions{PropagationPolicy: &policy} err = c.KubeClient. Deployments(c.Namespace). - Delete(deploymentName, &options) + Delete(context.TODO(), deploymentName, options) if !k8sutil.ResourceNotFound(err) { c.logger.Debugf("Connection pool deployment was already deleted") @@ -202,7 +206,7 @@ func (c *Cluster) deleteConnectionPool() (err error) { // will be deleted. err = c.KubeClient. Services(c.Namespace). - Delete(serviceName, &options) + Delete(context.TODO(), serviceName, options) if !k8sutil.ResourceNotFound(err) { c.logger.Debugf("Connection pool service was already deleted") @@ -251,7 +255,7 @@ func (c *Cluster) preScaleDown(newStatefulSet *appsv1.StatefulSet) error { } podName := fmt.Sprintf("%s-0", c.Statefulset.Name) - masterCandidatePod, err := c.KubeClient.Pods(c.clusterNamespace()).Get(podName, metav1.GetOptions{}) + masterCandidatePod, err := c.KubeClient.Pods(c.clusterNamespace()).Get(context.TODO(), podName, metav1.GetOptions{}) if err != nil { return fmt.Errorf("could not get master candidate pod: %v", err) } @@ -350,9 +354,12 @@ func (c *Cluster) updateStatefulSetAnnotations(annotations map[string]string) (* return nil, fmt.Errorf("could not form patch for the statefulset metadata: %v", err) } result, err := c.KubeClient.StatefulSets(c.Statefulset.Namespace).Patch( + context.TODO(), c.Statefulset.Name, types.MergePatchType, - []byte(patchData), "") + []byte(patchData), + metav1.PatchOptions{}, + "") if err != nil { return nil, fmt.Errorf("could not patch statefulset annotations %q: %v", patchData, err) } @@ -380,9 +387,12 @@ func (c *Cluster) updateStatefulSet(newStatefulSet *appsv1.StatefulSet) error { } statefulSet, err := c.KubeClient.StatefulSets(c.Statefulset.Namespace).Patch( + context.TODO(), c.Statefulset.Name, types.MergePatchType, - patchData, "") + patchData, + metav1.PatchOptions{}, + "") if err != nil { return fmt.Errorf("could not patch statefulset spec %q: %v", statefulSetName, err) } @@ -414,7 +424,7 @@ func (c *Cluster) replaceStatefulSet(newStatefulSet *appsv1.StatefulSet) error { oldStatefulset := c.Statefulset options := metav1.DeleteOptions{PropagationPolicy: &deletePropagationPolicy} - err := c.KubeClient.StatefulSets(oldStatefulset.Namespace).Delete(oldStatefulset.Name, &options) + err := c.KubeClient.StatefulSets(oldStatefulset.Namespace).Delete(context.TODO(), oldStatefulset.Name, options) if err != nil { return fmt.Errorf("could not delete statefulset %q: %v", statefulSetName, err) } @@ -425,7 +435,7 @@ func (c *Cluster) replaceStatefulSet(newStatefulSet *appsv1.StatefulSet) error { err = retryutil.Retry(c.OpConfig.ResourceCheckInterval, c.OpConfig.ResourceCheckTimeout, func() (bool, error) { - _, err2 := c.KubeClient.StatefulSets(oldStatefulset.Namespace).Get(oldStatefulset.Name, metav1.GetOptions{}) + _, err2 := c.KubeClient.StatefulSets(oldStatefulset.Namespace).Get(context.TODO(), oldStatefulset.Name, metav1.GetOptions{}) if err2 == nil { return false, nil } @@ -439,7 +449,7 @@ func (c *Cluster) replaceStatefulSet(newStatefulSet *appsv1.StatefulSet) error { } // create the new statefulset with the desired spec. It would take over the remaining pods. - createdStatefulset, err := c.KubeClient.StatefulSets(newStatefulSet.Namespace).Create(newStatefulSet) + createdStatefulset, err := c.KubeClient.StatefulSets(newStatefulSet.Namespace).Create(context.TODO(), newStatefulSet, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("could not create statefulset %q: %v", statefulSetName, err) } @@ -460,7 +470,7 @@ func (c *Cluster) deleteStatefulSet() error { return fmt.Errorf("there is no statefulset in the cluster") } - err := c.KubeClient.StatefulSets(c.Statefulset.Namespace).Delete(c.Statefulset.Name, c.deleteOptions) + err := c.KubeClient.StatefulSets(c.Statefulset.Namespace).Delete(context.TODO(), c.Statefulset.Name, c.deleteOptions) if err != nil { return err } @@ -482,7 +492,7 @@ func (c *Cluster) createService(role PostgresRole) (*v1.Service, error) { c.setProcessName("creating %v service", role) serviceSpec := c.generateService(role, &c.Spec) - service, err := c.KubeClient.Services(serviceSpec.Namespace).Create(serviceSpec) + service, err := c.KubeClient.Services(serviceSpec.Namespace).Create(context.TODO(), serviceSpec, metav1.CreateOptions{}) if err != nil { return nil, err } @@ -509,9 +519,12 @@ func (c *Cluster) updateService(role PostgresRole, newService *v1.Service) error if len(newService.ObjectMeta.Annotations) > 0 { if annotationsPatchData, err := metaAnnotationsPatch(newService.ObjectMeta.Annotations); err == nil { _, err = c.KubeClient.Services(serviceName.Namespace).Patch( + context.TODO(), serviceName.Name, types.MergePatchType, - []byte(annotationsPatchData), "") + []byte(annotationsPatchData), + metav1.PatchOptions{}, + "") if err != nil { return fmt.Errorf("could not replace annotations for the service %q: %v", serviceName, err) @@ -528,7 +541,7 @@ func (c *Cluster) updateService(role PostgresRole, newService *v1.Service) error if newServiceType == "ClusterIP" && newServiceType != oldServiceType { newService.ResourceVersion = c.Services[role].ResourceVersion newService.Spec.ClusterIP = c.Services[role].Spec.ClusterIP - svc, err = c.KubeClient.Services(serviceName.Namespace).Update(newService) + svc, err = c.KubeClient.Services(serviceName.Namespace).Update(context.TODO(), newService, metav1.UpdateOptions{}) if err != nil { return fmt.Errorf("could not update service %q: %v", serviceName, err) } @@ -539,9 +552,7 @@ func (c *Cluster) updateService(role PostgresRole, newService *v1.Service) error } svc, err = c.KubeClient.Services(serviceName.Namespace).Patch( - serviceName.Name, - types.MergePatchType, - patchData, "") + context.TODO(), serviceName.Name, types.MergePatchType, patchData, metav1.PatchOptions{}, "") if err != nil { return fmt.Errorf("could not patch service %q: %v", serviceName, err) } @@ -560,7 +571,7 @@ func (c *Cluster) deleteService(role PostgresRole) error { return nil } - if err := c.KubeClient.Services(service.Namespace).Delete(service.Name, c.deleteOptions); err != nil { + if err := c.KubeClient.Services(service.Namespace).Delete(context.TODO(), service.Name, c.deleteOptions); err != nil { return err } @@ -584,7 +595,7 @@ func (c *Cluster) createEndpoint(role PostgresRole) (*v1.Endpoints, error) { } endpointsSpec := c.generateEndpoint(role, subsets) - endpoints, err := c.KubeClient.Endpoints(endpointsSpec.Namespace).Create(endpointsSpec) + endpoints, err := c.KubeClient.Endpoints(endpointsSpec.Namespace).Create(context.TODO(), endpointsSpec, metav1.CreateOptions{}) if err != nil { return nil, fmt.Errorf("could not create %s endpoint: %v", role, err) } @@ -626,7 +637,7 @@ func (c *Cluster) createPodDisruptionBudget() (*policybeta1.PodDisruptionBudget, podDisruptionBudgetSpec := c.generatePodDisruptionBudget() podDisruptionBudget, err := c.KubeClient. PodDisruptionBudgets(podDisruptionBudgetSpec.Namespace). - Create(podDisruptionBudgetSpec) + Create(context.TODO(), podDisruptionBudgetSpec, metav1.CreateOptions{}) if err != nil { return nil, err @@ -647,7 +658,7 @@ func (c *Cluster) updatePodDisruptionBudget(pdb *policybeta1.PodDisruptionBudget newPdb, err := c.KubeClient. PodDisruptionBudgets(pdb.Namespace). - Create(pdb) + Create(context.TODO(), pdb, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("could not create pod disruption budget: %v", err) } @@ -665,7 +676,7 @@ func (c *Cluster) deletePodDisruptionBudget() error { pdbName := util.NameFromMeta(c.PodDisruptionBudget.ObjectMeta) err := c.KubeClient. PodDisruptionBudgets(c.PodDisruptionBudget.Namespace). - Delete(c.PodDisruptionBudget.Name, c.deleteOptions) + Delete(context.TODO(), c.PodDisruptionBudget.Name, c.deleteOptions) if err != nil { return fmt.Errorf("could not delete pod disruption budget: %v", err) } @@ -674,7 +685,7 @@ func (c *Cluster) deletePodDisruptionBudget() error { err = retryutil.Retry(c.OpConfig.ResourceCheckInterval, c.OpConfig.ResourceCheckTimeout, func() (bool, error) { - _, err2 := c.KubeClient.PodDisruptionBudgets(pdbName.Namespace).Get(pdbName.Name, metav1.GetOptions{}) + _, err2 := c.KubeClient.PodDisruptionBudgets(pdbName.Namespace).Get(context.TODO(), pdbName.Name, metav1.GetOptions{}) if err2 == nil { return false, nil } @@ -697,7 +708,8 @@ func (c *Cluster) deleteEndpoint(role PostgresRole) error { return fmt.Errorf("there is no %s endpoint in the cluster", role) } - if err := c.KubeClient.Endpoints(c.Endpoints[role].Namespace).Delete(c.Endpoints[role].Name, c.deleteOptions); err != nil { + if err := c.KubeClient.Endpoints(c.Endpoints[role].Namespace).Delete( + context.TODO(), c.Endpoints[role].Name, c.deleteOptions); err != nil { return fmt.Errorf("could not delete endpoint: %v", err) } @@ -711,7 +723,7 @@ func (c *Cluster) deleteEndpoint(role PostgresRole) error { func (c *Cluster) deleteSecret(secret *v1.Secret) error { c.setProcessName("deleting secret %q", util.NameFromMeta(secret.ObjectMeta)) c.logger.Debugf("deleting secret %q", util.NameFromMeta(secret.ObjectMeta)) - err := c.KubeClient.Secrets(secret.Namespace).Delete(secret.Name, c.deleteOptions) + err := c.KubeClient.Secrets(secret.Namespace).Delete(context.TODO(), secret.Name, c.deleteOptions) if err != nil { return err } @@ -736,7 +748,7 @@ func (c *Cluster) createLogicalBackupJob() (err error) { } c.logger.Debugf("Generated cronJobSpec: %v", logicalBackupJobSpec) - _, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Create(logicalBackupJobSpec) + _, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Create(context.TODO(), logicalBackupJobSpec, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("could not create k8s cron job: %v", err) } @@ -754,9 +766,12 @@ func (c *Cluster) patchLogicalBackupJob(newJob *batchv1beta1.CronJob) error { // update the backup job spec _, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Patch( + context.TODO(), c.getLogicalBackupJobName(), types.MergePatchType, - patchData, "") + patchData, + metav1.PatchOptions{}, + "") if err != nil { return fmt.Errorf("could not patch logical backup job: %v", err) } @@ -768,7 +783,7 @@ func (c *Cluster) deleteLogicalBackupJob() error { c.logger.Info("removing the logical backup job") - return c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Delete(c.getLogicalBackupJobName(), c.deleteOptions) + return c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Delete(context.TODO(), c.getLogicalBackupJobName(), c.deleteOptions) } // GetServiceMaster returns cluster's kubernetes master Service @@ -818,11 +833,13 @@ func (c *Cluster) updateConnPoolDeployment(oldDeploymentSpec, newDeployment *app // worker at one time will try to update it chances of conflicts are // minimal. deployment, err := c.KubeClient. - Deployments(c.ConnectionPool.Deployment.Namespace). - Patch( - c.ConnectionPool.Deployment.Name, - types.MergePatchType, - patchData, "") + Deployments(c.ConnectionPool.Deployment.Namespace).Patch( + context.TODO(), + c.ConnectionPool.Deployment.Name, + types.MergePatchType, + patchData, + metav1.PatchOptions{}, + "") if err != nil { return nil, fmt.Errorf("could not patch deployment: %v", err) } diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index a7c933ae7..eb3835787 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -1,6 +1,7 @@ package cluster import ( + "context" "fmt" batchv1beta1 "k8s.io/api/batch/v1beta1" @@ -140,7 +141,7 @@ func (c *Cluster) syncService(role PostgresRole) error { ) c.setProcessName("syncing %s service", role) - if svc, err = c.KubeClient.Services(c.Namespace).Get(c.serviceName(role), metav1.GetOptions{}); err == nil { + if svc, err = c.KubeClient.Services(c.Namespace).Get(context.TODO(), c.serviceName(role), metav1.GetOptions{}); err == nil { c.Services[role] = svc desiredSvc := c.generateService(role, &c.Spec) if match, reason := k8sutil.SameService(svc, desiredSvc); !match { @@ -166,7 +167,7 @@ func (c *Cluster) syncService(role PostgresRole) error { return fmt.Errorf("could not create missing %s service: %v", role, err) } c.logger.Infof("%s service %q already exists", role, util.NameFromMeta(svc.ObjectMeta)) - if svc, err = c.KubeClient.Services(c.Namespace).Get(c.serviceName(role), metav1.GetOptions{}); err != nil { + if svc, err = c.KubeClient.Services(c.Namespace).Get(context.TODO(), c.serviceName(role), metav1.GetOptions{}); err != nil { return fmt.Errorf("could not fetch existing %s service: %v", role, err) } } @@ -181,7 +182,7 @@ func (c *Cluster) syncEndpoint(role PostgresRole) error { ) c.setProcessName("syncing %s endpoint", role) - if ep, err = c.KubeClient.Endpoints(c.Namespace).Get(c.endpointName(role), metav1.GetOptions{}); err == nil { + if ep, err = c.KubeClient.Endpoints(c.Namespace).Get(context.TODO(), c.endpointName(role), metav1.GetOptions{}); err == nil { // TODO: No syncing of endpoints here, is this covered completely by updateService? c.Endpoints[role] = ep return nil @@ -200,7 +201,7 @@ func (c *Cluster) syncEndpoint(role PostgresRole) error { return fmt.Errorf("could not create missing %s endpoint: %v", role, err) } c.logger.Infof("%s endpoint %q already exists", role, util.NameFromMeta(ep.ObjectMeta)) - if ep, err = c.KubeClient.Endpoints(c.Namespace).Get(c.endpointName(role), metav1.GetOptions{}); err != nil { + if ep, err = c.KubeClient.Endpoints(c.Namespace).Get(context.TODO(), c.endpointName(role), metav1.GetOptions{}); err != nil { return fmt.Errorf("could not fetch existing %s endpoint: %v", role, err) } } @@ -213,7 +214,7 @@ func (c *Cluster) syncPodDisruptionBudget(isUpdate bool) error { pdb *policybeta1.PodDisruptionBudget err error ) - if pdb, err = c.KubeClient.PodDisruptionBudgets(c.Namespace).Get(c.podDisruptionBudgetName(), metav1.GetOptions{}); err == nil { + if pdb, err = c.KubeClient.PodDisruptionBudgets(c.Namespace).Get(context.TODO(), c.podDisruptionBudgetName(), metav1.GetOptions{}); err == nil { c.PodDisruptionBudget = pdb newPDB := c.generatePodDisruptionBudget() if match, reason := k8sutil.SamePDB(pdb, newPDB); !match { @@ -239,7 +240,7 @@ func (c *Cluster) syncPodDisruptionBudget(isUpdate bool) error { return fmt.Errorf("could not create pod disruption budget: %v", err) } c.logger.Infof("pod disruption budget %q already exists", util.NameFromMeta(pdb.ObjectMeta)) - if pdb, err = c.KubeClient.PodDisruptionBudgets(c.Namespace).Get(c.podDisruptionBudgetName(), metav1.GetOptions{}); err != nil { + if pdb, err = c.KubeClient.PodDisruptionBudgets(c.Namespace).Get(context.TODO(), c.podDisruptionBudgetName(), metav1.GetOptions{}); err != nil { return fmt.Errorf("could not fetch existing %q pod disruption budget", util.NameFromMeta(pdb.ObjectMeta)) } } @@ -255,7 +256,7 @@ func (c *Cluster) syncStatefulSet() error { podsRollingUpdateRequired bool ) // NB: Be careful to consider the codepath that acts on podsRollingUpdateRequired before returning early. - sset, err := c.KubeClient.StatefulSets(c.Namespace).Get(c.statefulSetName(), metav1.GetOptions{}) + sset, err := c.KubeClient.StatefulSets(c.Namespace).Get(context.TODO(), c.statefulSetName(), metav1.GetOptions{}) if err != nil { if !k8sutil.ResourceNotFound(err) { return fmt.Errorf("could not get statefulset: %v", err) @@ -404,14 +405,14 @@ func (c *Cluster) syncSecrets() error { secrets := c.generateUserSecrets() for secretUsername, secretSpec := range secrets { - if secret, err = c.KubeClient.Secrets(secretSpec.Namespace).Create(secretSpec); err == nil { + if secret, err = c.KubeClient.Secrets(secretSpec.Namespace).Create(context.TODO(), secretSpec, metav1.CreateOptions{}); err == nil { c.Secrets[secret.UID] = secret c.logger.Debugf("created new secret %q, uid: %q", util.NameFromMeta(secret.ObjectMeta), secret.UID) continue } if k8sutil.ResourceAlreadyExists(err) { var userMap map[string]spec.PgUser - if secret, err = c.KubeClient.Secrets(secretSpec.Namespace).Get(secretSpec.Name, metav1.GetOptions{}); err != nil { + if secret, err = c.KubeClient.Secrets(secretSpec.Namespace).Get(context.TODO(), secretSpec.Name, metav1.GetOptions{}); err != nil { return fmt.Errorf("could not get current secret: %v", err) } if secretUsername != string(secret.Data["username"]) { @@ -434,7 +435,7 @@ func (c *Cluster) syncSecrets() error { pwdUser.Origin == spec.RoleOriginInfrastructure { c.logger.Debugf("updating the secret %q from the infrastructure roles", secretSpec.Name) - if _, err = c.KubeClient.Secrets(secretSpec.Namespace).Update(secretSpec); err != nil { + if _, err = c.KubeClient.Secrets(secretSpec.Namespace).Update(context.TODO(), secretSpec, metav1.UpdateOptions{}); err != nil { return fmt.Errorf("could not update infrastructure role secret for role %q: %v", secretUsername, err) } } else { @@ -577,7 +578,7 @@ func (c *Cluster) syncLogicalBackupJob() error { // sync the job if it exists jobName := c.getLogicalBackupJobName() - if job, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Get(jobName, metav1.GetOptions{}); err == nil { + if job, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Get(context.TODO(), jobName, metav1.GetOptions{}); err == nil { desiredJob, err = c.generateLogicalBackupJob() if err != nil { @@ -611,7 +612,7 @@ func (c *Cluster) syncLogicalBackupJob() error { return fmt.Errorf("could not create missing logical backup job: %v", err) } c.logger.Infof("logical backup job %q already exists", jobName) - if _, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Get(jobName, metav1.GetOptions{}); err != nil { + if _, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Get(context.TODO(), jobName, metav1.GetOptions{}); err != nil { return fmt.Errorf("could not fetch existing logical backup job: %v", err) } } @@ -696,7 +697,7 @@ func (c *Cluster) syncConnectionPool(oldSpec, newSpec *acidv1.Postgresql, lookup func (c *Cluster) syncConnectionPoolWorker(oldSpec, newSpec *acidv1.Postgresql) error { deployment, err := c.KubeClient. Deployments(c.Namespace). - Get(c.connPoolName(), metav1.GetOptions{}) + Get(context.TODO(), c.connPoolName(), metav1.GetOptions{}) if err != nil && k8sutil.ResourceNotFound(err) { msg := "Deployment %s for connection pool synchronization is not found, create it" @@ -710,7 +711,7 @@ func (c *Cluster) syncConnectionPoolWorker(oldSpec, newSpec *acidv1.Postgresql) deployment, err := c.KubeClient. Deployments(deploymentSpec.Namespace). - Create(deploymentSpec) + Create(context.TODO(), deploymentSpec, metav1.CreateOptions{}) if err != nil { return err @@ -755,7 +756,7 @@ func (c *Cluster) syncConnectionPoolWorker(oldSpec, newSpec *acidv1.Postgresql) service, err := c.KubeClient. Services(c.Namespace). - Get(c.connPoolName(), metav1.GetOptions{}) + Get(context.TODO(), c.connPoolName(), metav1.GetOptions{}) if err != nil && k8sutil.ResourceNotFound(err) { msg := "Service %s for connection pool synchronization is not found, create it" @@ -764,7 +765,7 @@ func (c *Cluster) syncConnectionPoolWorker(oldSpec, newSpec *acidv1.Postgresql) serviceSpec := c.generateConnPoolService(&newSpec.Spec) service, err := c.KubeClient. Services(serviceSpec.Namespace). - Create(serviceSpec) + Create(context.TODO(), serviceSpec, metav1.CreateOptions{}) if err != nil { return err diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go index dc1e93954..3c3dffcaf 100644 --- a/pkg/cluster/util.go +++ b/pkg/cluster/util.go @@ -2,6 +2,7 @@ package cluster import ( "bytes" + "context" "encoding/gob" "encoding/json" "fmt" @@ -47,7 +48,7 @@ func (g *SecretOauthTokenGetter) getOAuthToken() (string, error) { // Temporary getting postgresql-operator secret from the NamespaceDefault credentialsSecret, err := g.kubeClient. Secrets(g.OAuthTokenSecretName.Namespace). - Get(g.OAuthTokenSecretName.Name, metav1.GetOptions{}) + Get(context.TODO(), g.OAuthTokenSecretName.Name, metav1.GetOptions{}) if err != nil { return "", fmt.Errorf("could not get credentials secret: %v", err) @@ -278,7 +279,7 @@ func (c *Cluster) waitStatefulsetReady() error { listOptions := metav1.ListOptions{ LabelSelector: c.labelsSet(false).String(), } - ss, err := c.KubeClient.StatefulSets(c.Namespace).List(listOptions) + ss, err := c.KubeClient.StatefulSets(c.Namespace).List(context.TODO(), listOptions) if err != nil { return false, err } @@ -313,7 +314,7 @@ func (c *Cluster) _waitPodLabelsReady(anyReplica bool) error { } podsNumber = 1 if !anyReplica { - pods, err := c.KubeClient.Pods(namespace).List(listOptions) + pods, err := c.KubeClient.Pods(namespace).List(context.TODO(), listOptions) if err != nil { return err } @@ -327,7 +328,7 @@ func (c *Cluster) _waitPodLabelsReady(anyReplica bool) error { func() (bool, error) { masterCount := 0 if !anyReplica { - masterPods, err2 := c.KubeClient.Pods(namespace).List(masterListOption) + masterPods, err2 := c.KubeClient.Pods(namespace).List(context.TODO(), masterListOption) if err2 != nil { return false, err2 } @@ -337,7 +338,7 @@ func (c *Cluster) _waitPodLabelsReady(anyReplica bool) error { } masterCount = len(masterPods.Items) } - replicaPods, err2 := c.KubeClient.Pods(namespace).List(replicaListOption) + replicaPods, err2 := c.KubeClient.Pods(namespace).List(context.TODO(), replicaListOption) if err2 != nil { return false, err2 } diff --git a/pkg/cluster/volumes.go b/pkg/cluster/volumes.go index d92ae6258..a5bfe6c2d 100644 --- a/pkg/cluster/volumes.go +++ b/pkg/cluster/volumes.go @@ -1,11 +1,12 @@ package cluster import ( + "context" "fmt" "strconv" "strings" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -23,7 +24,7 @@ func (c *Cluster) listPersistentVolumeClaims() ([]v1.PersistentVolumeClaim, erro LabelSelector: c.labelsSet(false).String(), } - pvcs, err := c.KubeClient.PersistentVolumeClaims(ns).List(listOptions) + pvcs, err := c.KubeClient.PersistentVolumeClaims(ns).List(context.TODO(), listOptions) if err != nil { return nil, fmt.Errorf("could not list of PersistentVolumeClaims: %v", err) } @@ -38,7 +39,7 @@ func (c *Cluster) deletePersistentVolumeClaims() error { } for _, pvc := range pvcs { c.logger.Debugf("deleting PVC %q", util.NameFromMeta(pvc.ObjectMeta)) - if err := c.KubeClient.PersistentVolumeClaims(pvc.Namespace).Delete(pvc.Name, c.deleteOptions); err != nil { + if err := c.KubeClient.PersistentVolumeClaims(pvc.Namespace).Delete(context.TODO(), pvc.Name, c.deleteOptions); err != nil { c.logger.Warningf("could not delete PersistentVolumeClaim: %v", err) } } @@ -78,7 +79,7 @@ func (c *Cluster) listPersistentVolumes() ([]*v1.PersistentVolume, error) { continue } } - pv, err := c.KubeClient.PersistentVolumes().Get(pvc.Spec.VolumeName, metav1.GetOptions{}) + pv, err := c.KubeClient.PersistentVolumes().Get(context.TODO(), pvc.Spec.VolumeName, metav1.GetOptions{}) if err != nil { return nil, fmt.Errorf("could not get PersistentVolume: %v", err) } @@ -143,7 +144,7 @@ func (c *Cluster) resizeVolumes(newVolume acidv1.Volume, resizers []volumes.Volu c.logger.Debugf("filesystem resize successful on volume %q", pv.Name) pv.Spec.Capacity[v1.ResourceStorage] = newQuantity c.logger.Debugf("updating persistent volume definition for volume %q", pv.Name) - if _, err := c.KubeClient.PersistentVolumes().Update(pv); err != nil { + if _, err := c.KubeClient.PersistentVolumes().Update(context.TODO(), pv, metav1.UpdateOptions{}); err != nil { return fmt.Errorf("could not update persistent volume: %q", err) } c.logger.Debugf("successfully updated persistent volume %q", pv.Name) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 0ce0d026e..9c48b7ef2 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -1,6 +1,7 @@ package controller import ( + "context" "fmt" "os" "sync" @@ -99,7 +100,7 @@ func (c *Controller) initOperatorConfig() { if c.config.ConfigMapName != (spec.NamespacedName{}) { configMap, err := c.KubeClient.ConfigMaps(c.config.ConfigMapName.Namespace). - Get(c.config.ConfigMapName.Name, metav1.GetOptions{}) + Get(context.TODO(), c.config.ConfigMapName.Name, metav1.GetOptions{}) if err != nil { panic(err) } @@ -406,7 +407,7 @@ func (c *Controller) getEffectiveNamespace(namespaceFromEnvironment, namespaceFr } else { - if _, err := c.KubeClient.Namespaces().Get(namespace, metav1.GetOptions{}); err != nil { + if _, err := c.KubeClient.Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{}); err != nil { c.logger.Fatalf("Could not find the watched namespace %q", namespace) } else { c.logger.Infof("Listenting to the specific namespace %q", namespace) diff --git a/pkg/controller/node.go b/pkg/controller/node.go index 8052458c3..be41b79ab 100644 --- a/pkg/controller/node.go +++ b/pkg/controller/node.go @@ -1,6 +1,7 @@ package controller import ( + "context" "fmt" "time" @@ -22,7 +23,7 @@ func (c *Controller) nodeListFunc(options metav1.ListOptions) (runtime.Object, e TimeoutSeconds: options.TimeoutSeconds, } - return c.KubeClient.Nodes().List(opts) + return c.KubeClient.Nodes().List(context.TODO(), opts) } func (c *Controller) nodeWatchFunc(options metav1.ListOptions) (watch.Interface, error) { @@ -32,7 +33,7 @@ func (c *Controller) nodeWatchFunc(options metav1.ListOptions) (watch.Interface, TimeoutSeconds: options.TimeoutSeconds, } - return c.KubeClient.Nodes().Watch(opts) + return c.KubeClient.Nodes().Watch(context.TODO(), opts) } func (c *Controller) nodeAdd(obj interface{}) { @@ -87,7 +88,7 @@ func (c *Controller) attemptToMoveMasterPodsOffNode(node *v1.Node) error { opts := metav1.ListOptions{ LabelSelector: labels.Set(c.opConfig.ClusterLabels).String(), } - podList, err := c.KubeClient.Pods(c.opConfig.WatchedNamespace).List(opts) + podList, err := c.KubeClient.Pods(c.opConfig.WatchedNamespace).List(context.TODO(), opts) if err != nil { c.logger.Errorf("could not fetch list of the pods: %v", err) return err diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 970eef701..c9b3c5ea4 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -1,6 +1,7 @@ package controller import ( + "context" "fmt" "time" @@ -14,7 +15,8 @@ import ( func (c *Controller) readOperatorConfigurationFromCRD(configObjectNamespace, configObjectName string) (*acidv1.OperatorConfiguration, error) { - config, err := c.KubeClient.AcidV1ClientSet.AcidV1().OperatorConfigurations(configObjectNamespace).Get(configObjectName, metav1.GetOptions{}) + config, err := c.KubeClient.AcidV1ClientSet.AcidV1().OperatorConfigurations(configObjectNamespace).Get( + context.TODO(), configObjectName, metav1.GetOptions{}) if err != nil { return nil, fmt.Errorf("could not get operator configuration object %q: %v", configObjectName, err) } diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index 27fd6c956..0defe88b1 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -1,7 +1,9 @@ package controller import ( - "k8s.io/api/core/v1" + "context" + + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" @@ -19,7 +21,7 @@ func (c *Controller) podListFunc(options metav1.ListOptions) (runtime.Object, er TimeoutSeconds: options.TimeoutSeconds, } - return c.KubeClient.Pods(c.opConfig.WatchedNamespace).List(opts) + return c.KubeClient.Pods(c.opConfig.WatchedNamespace).List(context.TODO(), opts) } func (c *Controller) podWatchFunc(options metav1.ListOptions) (watch.Interface, error) { @@ -29,7 +31,7 @@ func (c *Controller) podWatchFunc(options metav1.ListOptions) (watch.Interface, TimeoutSeconds: options.TimeoutSeconds, } - return c.KubeClient.Pods(c.opConfig.WatchedNamespace).Watch(opts) + return c.KubeClient.Pods(c.opConfig.WatchedNamespace).Watch(context.TODO(), opts) } func (c *Controller) dispatchPodEvent(clusterName spec.NamespacedName, event cluster.PodEvent) { diff --git a/pkg/controller/postgresql.go b/pkg/controller/postgresql.go index 5d48bac39..e81671c7d 100644 --- a/pkg/controller/postgresql.go +++ b/pkg/controller/postgresql.go @@ -1,6 +1,7 @@ package controller import ( + "context" "fmt" "reflect" "strings" @@ -43,7 +44,7 @@ func (c *Controller) listClusters(options metav1.ListOptions) (*acidv1.Postgresq var pgList acidv1.PostgresqlList // TODO: use the SharedInformer cache instead of quering Kubernetes API directly. - list, err := c.KubeClient.AcidV1ClientSet.AcidV1().Postgresqls(c.opConfig.WatchedNamespace).List(options) + list, err := c.KubeClient.AcidV1ClientSet.AcidV1().Postgresqls(c.opConfig.WatchedNamespace).List(context.TODO(), options) if err != nil { c.logger.Errorf("could not list postgresql objects: %v", err) } @@ -535,7 +536,7 @@ func (c *Controller) submitRBACCredentials(event ClusterEvent) error { func (c *Controller) createPodServiceAccount(namespace string) error { podServiceAccountName := c.opConfig.PodServiceAccountName - _, err := c.KubeClient.ServiceAccounts(namespace).Get(podServiceAccountName, metav1.GetOptions{}) + _, err := c.KubeClient.ServiceAccounts(namespace).Get(context.TODO(), podServiceAccountName, metav1.GetOptions{}) if k8sutil.ResourceNotFound(err) { c.logger.Infof(fmt.Sprintf("creating pod service account %q in the %q namespace", podServiceAccountName, namespace)) @@ -543,7 +544,7 @@ func (c *Controller) createPodServiceAccount(namespace string) error { // get a separate copy of service account // to prevent a race condition when setting a namespace for many clusters sa := *c.PodServiceAccount - if _, err = c.KubeClient.ServiceAccounts(namespace).Create(&sa); err != nil { + if _, err = c.KubeClient.ServiceAccounts(namespace).Create(context.TODO(), &sa, metav1.CreateOptions{}); err != nil { return fmt.Errorf("cannot deploy the pod service account %q defined in the configuration to the %q namespace: %v", podServiceAccountName, namespace, err) } @@ -560,7 +561,7 @@ func (c *Controller) createRoleBindings(namespace string) error { podServiceAccountName := c.opConfig.PodServiceAccountName podServiceAccountRoleBindingName := c.PodServiceAccountRoleBinding.Name - _, err := c.KubeClient.RoleBindings(namespace).Get(podServiceAccountRoleBindingName, metav1.GetOptions{}) + _, err := c.KubeClient.RoleBindings(namespace).Get(context.TODO(), podServiceAccountRoleBindingName, metav1.GetOptions{}) if k8sutil.ResourceNotFound(err) { c.logger.Infof("Creating the role binding %q in the %q namespace", podServiceAccountRoleBindingName, namespace) @@ -568,7 +569,7 @@ func (c *Controller) createRoleBindings(namespace string) error { // get a separate copy of role binding // to prevent a race condition when setting a namespace for many clusters rb := *c.PodServiceAccountRoleBinding - _, err = c.KubeClient.RoleBindings(namespace).Create(&rb) + _, err = c.KubeClient.RoleBindings(namespace).Create(context.TODO(), &rb, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("cannot bind the pod service account %q defined in the configuration to the cluster role in the %q namespace: %v", podServiceAccountName, namespace, err) } diff --git a/pkg/controller/util.go b/pkg/controller/util.go index 9b7dca063..511f02823 100644 --- a/pkg/controller/util.go +++ b/pkg/controller/util.go @@ -1,6 +1,7 @@ package controller import ( + "context" "encoding/json" "fmt" @@ -50,7 +51,7 @@ func (c *Controller) clusterWorkerID(clusterName spec.NamespacedName) uint32 { } func (c *Controller) createOperatorCRD(crd *apiextv1beta1.CustomResourceDefinition) error { - if _, err := c.KubeClient.CustomResourceDefinitions().Create(crd); err != nil { + if _, err := c.KubeClient.CustomResourceDefinitions().Create(context.TODO(), crd, metav1.CreateOptions{}); err != nil { if k8sutil.ResourceAlreadyExists(err) { c.logger.Infof("customResourceDefinition %q is already registered and will only be updated", crd.Name) @@ -58,7 +59,8 @@ func (c *Controller) createOperatorCRD(crd *apiextv1beta1.CustomResourceDefiniti if err != nil { return fmt.Errorf("could not marshal new customResourceDefintion: %v", err) } - if _, err := c.KubeClient.CustomResourceDefinitions().Patch(crd.Name, types.MergePatchType, patch); err != nil { + if _, err := c.KubeClient.CustomResourceDefinitions().Patch( + context.TODO(), crd.Name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { return fmt.Errorf("could not update customResourceDefinition: %v", err) } } else { @@ -69,7 +71,7 @@ func (c *Controller) createOperatorCRD(crd *apiextv1beta1.CustomResourceDefiniti } return wait.Poll(c.config.CRDReadyWaitInterval, c.config.CRDReadyWaitTimeout, func() (bool, error) { - c, err := c.KubeClient.CustomResourceDefinitions().Get(crd.Name, metav1.GetOptions{}) + c, err := c.KubeClient.CustomResourceDefinitions().Get(context.TODO(), crd.Name, metav1.GetOptions{}) if err != nil { return false, err } @@ -115,7 +117,7 @@ func (c *Controller) getInfrastructureRoles(rolesSecret *spec.NamespacedName) (m infraRolesSecret, err := c.KubeClient. Secrets(rolesSecret.Namespace). - Get(rolesSecret.Name, metav1.GetOptions{}) + Get(context.TODO(), rolesSecret.Name, metav1.GetOptions{}) if err != nil { c.logger.Debugf("infrastructure roles secret name: %q", *rolesSecret) return nil, fmt.Errorf("could not get infrastructure roles secret: %v", err) @@ -161,7 +163,8 @@ Users: } // perhaps we have some map entries with usernames, passwords, let's check if we have those users in the configmap - if infraRolesMap, err := c.KubeClient.ConfigMaps(rolesSecret.Namespace).Get(rolesSecret.Name, metav1.GetOptions{}); err == nil { + if infraRolesMap, err := c.KubeClient.ConfigMaps(rolesSecret.Namespace).Get( + context.TODO(), rolesSecret.Name, metav1.GetOptions{}); err == nil { // we have a configmap with username - json description, let's read and decode it for role, s := range infraRolesMap.Data { roleDescr, err := readDecodedRole(s) diff --git a/pkg/generated/clientset/versioned/clientset.go b/pkg/generated/clientset/versioned/clientset.go index cb72ec50f..5f1e5880a 100644 --- a/pkg/generated/clientset/versioned/clientset.go +++ b/pkg/generated/clientset/versioned/clientset.go @@ -65,7 +65,7 @@ func NewForConfig(c *rest.Config) (*Clientset, error) { configShallowCopy := *c if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 { if configShallowCopy.Burst <= 0 { - return nil, fmt.Errorf("Burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0") + return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0") } configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst) } diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go index 732b48250..d515a0080 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go @@ -25,6 +25,8 @@ SOFTWARE. package fake import ( + "context" + acidzalandov1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" schema "k8s.io/apimachinery/pkg/runtime/schema" @@ -42,7 +44,7 @@ var operatorconfigurationsResource = schema.GroupVersionResource{Group: "acid.za var operatorconfigurationsKind = schema.GroupVersionKind{Group: "acid.zalan.do", Version: "v1", Kind: "OperatorConfiguration"} // Get takes name of the operatorConfiguration, and returns the corresponding operatorConfiguration object, and an error if there is any. -func (c *FakeOperatorConfigurations) Get(name string, options v1.GetOptions) (result *acidzalandov1.OperatorConfiguration, err error) { +func (c *FakeOperatorConfigurations) Get(ctx context.Context, name string, options v1.GetOptions) (result *acidzalandov1.OperatorConfiguration, err error) { obj, err := c.Fake. Invokes(testing.NewGetAction(operatorconfigurationsResource, c.ns, name), &acidzalandov1.OperatorConfiguration{}) diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go index 1ab20dbfc..e4d72f882 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go @@ -25,6 +25,8 @@ SOFTWARE. package fake import ( + "context" + acidzalandov1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" labels "k8s.io/apimachinery/pkg/labels" @@ -45,7 +47,7 @@ var postgresqlsResource = schema.GroupVersionResource{Group: "acid.zalan.do", Ve var postgresqlsKind = schema.GroupVersionKind{Group: "acid.zalan.do", Version: "v1", Kind: "Postgresql"} // Get takes name of the postgresql, and returns the corresponding postgresql object, and an error if there is any. -func (c *FakePostgresqls) Get(name string, options v1.GetOptions) (result *acidzalandov1.Postgresql, err error) { +func (c *FakePostgresqls) Get(ctx context.Context, name string, options v1.GetOptions) (result *acidzalandov1.Postgresql, err error) { obj, err := c.Fake. Invokes(testing.NewGetAction(postgresqlsResource, c.ns, name), &acidzalandov1.Postgresql{}) @@ -56,7 +58,7 @@ func (c *FakePostgresqls) Get(name string, options v1.GetOptions) (result *acidz } // List takes label and field selectors, and returns the list of Postgresqls that match those selectors. -func (c *FakePostgresqls) List(opts v1.ListOptions) (result *acidzalandov1.PostgresqlList, err error) { +func (c *FakePostgresqls) List(ctx context.Context, opts v1.ListOptions) (result *acidzalandov1.PostgresqlList, err error) { obj, err := c.Fake. Invokes(testing.NewListAction(postgresqlsResource, postgresqlsKind, c.ns, opts), &acidzalandov1.PostgresqlList{}) @@ -78,14 +80,14 @@ func (c *FakePostgresqls) List(opts v1.ListOptions) (result *acidzalandov1.Postg } // Watch returns a watch.Interface that watches the requested postgresqls. -func (c *FakePostgresqls) Watch(opts v1.ListOptions) (watch.Interface, error) { +func (c *FakePostgresqls) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { return c.Fake. InvokesWatch(testing.NewWatchAction(postgresqlsResource, c.ns, opts)) } // Create takes the representation of a postgresql and creates it. Returns the server's representation of the postgresql, and an error, if there is any. -func (c *FakePostgresqls) Create(postgresql *acidzalandov1.Postgresql) (result *acidzalandov1.Postgresql, err error) { +func (c *FakePostgresqls) Create(ctx context.Context, postgresql *acidzalandov1.Postgresql, opts v1.CreateOptions) (result *acidzalandov1.Postgresql, err error) { obj, err := c.Fake. Invokes(testing.NewCreateAction(postgresqlsResource, c.ns, postgresql), &acidzalandov1.Postgresql{}) @@ -96,7 +98,7 @@ func (c *FakePostgresqls) Create(postgresql *acidzalandov1.Postgresql) (result * } // Update takes the representation of a postgresql and updates it. Returns the server's representation of the postgresql, and an error, if there is any. -func (c *FakePostgresqls) Update(postgresql *acidzalandov1.Postgresql) (result *acidzalandov1.Postgresql, err error) { +func (c *FakePostgresqls) Update(ctx context.Context, postgresql *acidzalandov1.Postgresql, opts v1.UpdateOptions) (result *acidzalandov1.Postgresql, err error) { obj, err := c.Fake. Invokes(testing.NewUpdateAction(postgresqlsResource, c.ns, postgresql), &acidzalandov1.Postgresql{}) @@ -108,7 +110,7 @@ func (c *FakePostgresqls) Update(postgresql *acidzalandov1.Postgresql) (result * // UpdateStatus was generated because the type contains a Status member. // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). -func (c *FakePostgresqls) UpdateStatus(postgresql *acidzalandov1.Postgresql) (*acidzalandov1.Postgresql, error) { +func (c *FakePostgresqls) UpdateStatus(ctx context.Context, postgresql *acidzalandov1.Postgresql, opts v1.UpdateOptions) (*acidzalandov1.Postgresql, error) { obj, err := c.Fake. Invokes(testing.NewUpdateSubresourceAction(postgresqlsResource, "status", c.ns, postgresql), &acidzalandov1.Postgresql{}) @@ -119,7 +121,7 @@ func (c *FakePostgresqls) UpdateStatus(postgresql *acidzalandov1.Postgresql) (*a } // Delete takes name of the postgresql and deletes it. Returns an error if one occurs. -func (c *FakePostgresqls) Delete(name string, options *v1.DeleteOptions) error { +func (c *FakePostgresqls) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { _, err := c.Fake. Invokes(testing.NewDeleteAction(postgresqlsResource, c.ns, name), &acidzalandov1.Postgresql{}) @@ -127,15 +129,15 @@ func (c *FakePostgresqls) Delete(name string, options *v1.DeleteOptions) error { } // DeleteCollection deletes a collection of objects. -func (c *FakePostgresqls) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { - action := testing.NewDeleteCollectionAction(postgresqlsResource, c.ns, listOptions) +func (c *FakePostgresqls) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionAction(postgresqlsResource, c.ns, listOpts) _, err := c.Fake.Invokes(action, &acidzalandov1.PostgresqlList{}) return err } // Patch applies the patch and returns the patched postgresql. -func (c *FakePostgresqls) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *acidzalandov1.Postgresql, err error) { +func (c *FakePostgresqls) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *acidzalandov1.Postgresql, err error) { obj, err := c.Fake. Invokes(testing.NewPatchSubresourceAction(postgresqlsResource, c.ns, name, pt, data, subresources...), &acidzalandov1.Postgresql{}) diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go index e9cc0de77..80ef6d6f3 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go @@ -25,6 +25,8 @@ SOFTWARE. package v1 import ( + "context" + acidzalandov1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" scheme "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/scheme" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -39,7 +41,7 @@ type OperatorConfigurationsGetter interface { // OperatorConfigurationInterface has methods to work with OperatorConfiguration resources. type OperatorConfigurationInterface interface { - Get(name string, options v1.GetOptions) (*acidzalandov1.OperatorConfiguration, error) + Get(ctx context.Context, name string, opts v1.GetOptions) (*acidzalandov1.OperatorConfiguration, error) OperatorConfigurationExpansion } @@ -58,14 +60,14 @@ func newOperatorConfigurations(c *AcidV1Client, namespace string) *operatorConfi } // Get takes name of the operatorConfiguration, and returns the corresponding operatorConfiguration object, and an error if there is any. -func (c *operatorConfigurations) Get(name string, options v1.GetOptions) (result *acidzalandov1.OperatorConfiguration, err error) { +func (c *operatorConfigurations) Get(ctx context.Context, name string, options v1.GetOptions) (result *acidzalandov1.OperatorConfiguration, err error) { result = &acidzalandov1.OperatorConfiguration{} err = c.client.Get(). Namespace(c.ns). Resource("operatorconfigurations"). Name(name). VersionedParams(&options, scheme.ParameterCodec). - Do(). + Do(ctx). Into(result) return } diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go index 78c0fc390..ca8c6d7ee 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go @@ -25,6 +25,7 @@ SOFTWARE. package v1 import ( + "context" "time" v1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" @@ -43,15 +44,15 @@ type PostgresqlsGetter interface { // PostgresqlInterface has methods to work with Postgresql resources. type PostgresqlInterface interface { - Create(*v1.Postgresql) (*v1.Postgresql, error) - Update(*v1.Postgresql) (*v1.Postgresql, error) - UpdateStatus(*v1.Postgresql) (*v1.Postgresql, error) - Delete(name string, options *metav1.DeleteOptions) error - DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error - Get(name string, options metav1.GetOptions) (*v1.Postgresql, error) - List(opts metav1.ListOptions) (*v1.PostgresqlList, error) - Watch(opts metav1.ListOptions) (watch.Interface, error) - Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.Postgresql, err error) + Create(ctx context.Context, postgresql *v1.Postgresql, opts metav1.CreateOptions) (*v1.Postgresql, error) + Update(ctx context.Context, postgresql *v1.Postgresql, opts metav1.UpdateOptions) (*v1.Postgresql, error) + UpdateStatus(ctx context.Context, postgresql *v1.Postgresql, opts metav1.UpdateOptions) (*v1.Postgresql, error) + Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error + Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.Postgresql, error) + List(ctx context.Context, opts metav1.ListOptions) (*v1.PostgresqlList, error) + Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.Postgresql, err error) PostgresqlExpansion } @@ -70,20 +71,20 @@ func newPostgresqls(c *AcidV1Client, namespace string) *postgresqls { } // Get takes name of the postgresql, and returns the corresponding postgresql object, and an error if there is any. -func (c *postgresqls) Get(name string, options metav1.GetOptions) (result *v1.Postgresql, err error) { +func (c *postgresqls) Get(ctx context.Context, name string, options metav1.GetOptions) (result *v1.Postgresql, err error) { result = &v1.Postgresql{} err = c.client.Get(). Namespace(c.ns). Resource("postgresqls"). Name(name). VersionedParams(&options, scheme.ParameterCodec). - Do(). + Do(ctx). Into(result) return } // List takes label and field selectors, and returns the list of Postgresqls that match those selectors. -func (c *postgresqls) List(opts metav1.ListOptions) (result *v1.PostgresqlList, err error) { +func (c *postgresqls) List(ctx context.Context, opts metav1.ListOptions) (result *v1.PostgresqlList, err error) { var timeout time.Duration if opts.TimeoutSeconds != nil { timeout = time.Duration(*opts.TimeoutSeconds) * time.Second @@ -94,13 +95,13 @@ func (c *postgresqls) List(opts metav1.ListOptions) (result *v1.PostgresqlList, Resource("postgresqls"). VersionedParams(&opts, scheme.ParameterCodec). Timeout(timeout). - Do(). + Do(ctx). Into(result) return } // Watch returns a watch.Interface that watches the requested postgresqls. -func (c *postgresqls) Watch(opts metav1.ListOptions) (watch.Interface, error) { +func (c *postgresqls) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { var timeout time.Duration if opts.TimeoutSeconds != nil { timeout = time.Duration(*opts.TimeoutSeconds) * time.Second @@ -111,87 +112,90 @@ func (c *postgresqls) Watch(opts metav1.ListOptions) (watch.Interface, error) { Resource("postgresqls"). VersionedParams(&opts, scheme.ParameterCodec). Timeout(timeout). - Watch() + Watch(ctx) } // Create takes the representation of a postgresql and creates it. Returns the server's representation of the postgresql, and an error, if there is any. -func (c *postgresqls) Create(postgresql *v1.Postgresql) (result *v1.Postgresql, err error) { +func (c *postgresqls) Create(ctx context.Context, postgresql *v1.Postgresql, opts metav1.CreateOptions) (result *v1.Postgresql, err error) { result = &v1.Postgresql{} err = c.client.Post(). Namespace(c.ns). Resource("postgresqls"). + VersionedParams(&opts, scheme.ParameterCodec). Body(postgresql). - Do(). + Do(ctx). Into(result) return } // Update takes the representation of a postgresql and updates it. Returns the server's representation of the postgresql, and an error, if there is any. -func (c *postgresqls) Update(postgresql *v1.Postgresql) (result *v1.Postgresql, err error) { +func (c *postgresqls) Update(ctx context.Context, postgresql *v1.Postgresql, opts metav1.UpdateOptions) (result *v1.Postgresql, err error) { result = &v1.Postgresql{} err = c.client.Put(). Namespace(c.ns). Resource("postgresqls"). Name(postgresql.Name). + VersionedParams(&opts, scheme.ParameterCodec). Body(postgresql). - Do(). + Do(ctx). Into(result) return } // UpdateStatus was generated because the type contains a Status member. // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). - -func (c *postgresqls) UpdateStatus(postgresql *v1.Postgresql) (result *v1.Postgresql, err error) { +func (c *postgresqls) UpdateStatus(ctx context.Context, postgresql *v1.Postgresql, opts metav1.UpdateOptions) (result *v1.Postgresql, err error) { result = &v1.Postgresql{} err = c.client.Put(). Namespace(c.ns). Resource("postgresqls"). Name(postgresql.Name). SubResource("status"). + VersionedParams(&opts, scheme.ParameterCodec). Body(postgresql). - Do(). + Do(ctx). Into(result) return } // Delete takes name of the postgresql and deletes it. Returns an error if one occurs. -func (c *postgresqls) Delete(name string, options *metav1.DeleteOptions) error { +func (c *postgresqls) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error { return c.client.Delete(). Namespace(c.ns). Resource("postgresqls"). Name(name). - Body(options). - Do(). + Body(&opts). + Do(ctx). Error() } // DeleteCollection deletes a collection of objects. -func (c *postgresqls) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { +func (c *postgresqls) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error { var timeout time.Duration - if listOptions.TimeoutSeconds != nil { - timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + if listOpts.TimeoutSeconds != nil { + timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second } return c.client.Delete(). Namespace(c.ns). Resource("postgresqls"). - VersionedParams(&listOptions, scheme.ParameterCodec). + VersionedParams(&listOpts, scheme.ParameterCodec). Timeout(timeout). - Body(options). - Do(). + Body(&opts). + Do(ctx). Error() } // Patch applies the patch and returns the patched postgresql. -func (c *postgresqls) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.Postgresql, err error) { +func (c *postgresqls) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.Postgresql, err error) { result = &v1.Postgresql{} err = c.client.Patch(pt). Namespace(c.ns). Resource("postgresqls"). - SubResource(subresources...). Name(name). + SubResource(subresources...). + VersionedParams(&opts, scheme.ParameterCodec). Body(data). - Do(). + Do(ctx). Into(result) return } diff --git a/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go b/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go index da7f91669..be09839d3 100644 --- a/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go +++ b/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go @@ -25,6 +25,7 @@ SOFTWARE. package v1 import ( + "context" time "time" acidzalandov1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" @@ -67,13 +68,13 @@ func NewFilteredPostgresqlInformer(client versioned.Interface, namespace string, if tweakListOptions != nil { tweakListOptions(&options) } - return client.AcidV1().Postgresqls(namespace).List(options) + return client.AcidV1().Postgresqls(namespace).List(context.TODO(), options) }, WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { if tweakListOptions != nil { tweakListOptions(&options) } - return client.AcidV1().Postgresqls(namespace).Watch(options) + return client.AcidV1().Postgresqls(namespace).Watch(context.TODO(), options) }, }, &acidzalandov1.Postgresql{}, diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index 75b99ec7c..3a397af7d 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -1,6 +1,7 @@ package k8sutil import ( + "context" "fmt" "reflect" @@ -237,7 +238,7 @@ func SameLogicalBackupJob(cur, new *batchv1beta1.CronJob) (match bool, reason st return true, "" } -func (c *mockSecret) Get(name string, options metav1.GetOptions) (*v1.Secret, error) { +func (c *mockSecret) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.Secret, error) { if name != "infrastructureroles-test" { return nil, fmt.Errorf("NotFound") } @@ -253,7 +254,7 @@ func (c *mockSecret) Get(name string, options metav1.GetOptions) (*v1.Secret, er } -func (c *mockConfigMap) Get(name string, options metav1.GetOptions) (*v1.ConfigMap, error) { +func (c *mockConfigMap) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.ConfigMap, error) { if name != "infrastructureroles-test" { return nil, fmt.Errorf("NotFound") } @@ -283,7 +284,7 @@ func (mock *MockDeploymentNotExistGetter) Deployments(namespace string) appsv1.D return &mockDeploymentNotExist{} } -func (mock *mockDeployment) Create(*apiappsv1.Deployment) (*apiappsv1.Deployment, error) { +func (mock *mockDeployment) Create(context.Context, *apiappsv1.Deployment, metav1.CreateOptions) (*apiappsv1.Deployment, error) { return &apiappsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: "test-deployment", @@ -294,11 +295,11 @@ func (mock *mockDeployment) Create(*apiappsv1.Deployment) (*apiappsv1.Deployment }, nil } -func (mock *mockDeployment) Delete(name string, opts *metav1.DeleteOptions) error { +func (mock *mockDeployment) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error { return nil } -func (mock *mockDeployment) Get(name string, opts metav1.GetOptions) (*apiappsv1.Deployment, error) { +func (mock *mockDeployment) Get(ctx context.Context, name string, opts metav1.GetOptions) (*apiappsv1.Deployment, error) { return &apiappsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: "test-deployment", @@ -318,7 +319,7 @@ func (mock *mockDeployment) Get(name string, opts metav1.GetOptions) (*apiappsv1 }, nil } -func (mock *mockDeployment) Patch(name string, t types.PatchType, data []byte, subres ...string) (*apiappsv1.Deployment, error) { +func (mock *mockDeployment) Patch(ctx context.Context, name string, t types.PatchType, data []byte, opts metav1.PatchOptions, subres ...string) (*apiappsv1.Deployment, error) { return &apiappsv1.Deployment{ Spec: apiappsv1.DeploymentSpec{ Replicas: Int32ToPointer(2), @@ -329,7 +330,7 @@ func (mock *mockDeployment) Patch(name string, t types.PatchType, data []byte, s }, nil } -func (mock *mockDeploymentNotExist) Get(name string, opts metav1.GetOptions) (*apiappsv1.Deployment, error) { +func (mock *mockDeploymentNotExist) Get(ctx context.Context, name string, opts metav1.GetOptions) (*apiappsv1.Deployment, error) { return nil, &apierrors.StatusError{ ErrStatus: metav1.Status{ Reason: metav1.StatusReasonNotFound, @@ -337,7 +338,7 @@ func (mock *mockDeploymentNotExist) Get(name string, opts metav1.GetOptions) (*a } } -func (mock *mockDeploymentNotExist) Create(*apiappsv1.Deployment) (*apiappsv1.Deployment, error) { +func (mock *mockDeploymentNotExist) Create(context.Context, *apiappsv1.Deployment, metav1.CreateOptions) (*apiappsv1.Deployment, error) { return &apiappsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: "test-deployment", @@ -356,7 +357,7 @@ func (mock *MockServiceNotExistGetter) Services(namespace string) corev1.Service return &mockServiceNotExist{} } -func (mock *mockService) Create(*v1.Service) (*v1.Service, error) { +func (mock *mockService) Create(context.Context, *v1.Service, metav1.CreateOptions) (*v1.Service, error) { return &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test-service", @@ -364,11 +365,11 @@ func (mock *mockService) Create(*v1.Service) (*v1.Service, error) { }, nil } -func (mock *mockService) Delete(name string, opts *metav1.DeleteOptions) error { +func (mock *mockService) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error { return nil } -func (mock *mockService) Get(name string, opts metav1.GetOptions) (*v1.Service, error) { +func (mock *mockService) Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.Service, error) { return &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test-service", @@ -376,7 +377,7 @@ func (mock *mockService) Get(name string, opts metav1.GetOptions) (*v1.Service, }, nil } -func (mock *mockServiceNotExist) Create(*v1.Service) (*v1.Service, error) { +func (mock *mockServiceNotExist) Create(context.Context, *v1.Service, metav1.CreateOptions) (*v1.Service, error) { return &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test-service", @@ -384,7 +385,7 @@ func (mock *mockServiceNotExist) Create(*v1.Service) (*v1.Service, error) { }, nil } -func (mock *mockServiceNotExist) Get(name string, opts metav1.GetOptions) (*v1.Service, error) { +func (mock *mockServiceNotExist) Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.Service, error) { return nil, &apierrors.StatusError{ ErrStatus: metav1.Status{ Reason: metav1.StatusReasonNotFound, diff --git a/pkg/util/teams/teams_test.go b/pkg/util/teams/teams_test.go index 51bbcbc31..33d01b75b 100644 --- a/pkg/util/teams/teams_test.go +++ b/pkg/util/teams/teams_test.go @@ -133,11 +133,11 @@ var requestsURLtc = []struct { }{ { "coffee://localhost/", - fmt.Errorf(`Get coffee://localhost/teams/acid: unsupported protocol scheme "coffee"`), + fmt.Errorf(`Get "coffee://localhost/teams/acid": unsupported protocol scheme "coffee"`), }, { "http://192.168.0.%31/", - fmt.Errorf(`parse http://192.168.0.%%31/teams/acid: invalid URL escape "%%31"`), + fmt.Errorf(`parse "http://192.168.0.%%31/teams/acid": invalid URL escape "%%31"`), }, } From 64d816c55601c45afe8dec7a64f737cb19c19f2c Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 31 Mar 2020 11:47:39 +0200 Subject: [PATCH 017/168] add short sleep before redistributing pods (#891) --- e2e/tests/test_e2e.py | 179 ++++++++++++++++++++++-------------------- 1 file changed, 93 insertions(+), 86 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index cc90aa5e2..f6fc8184c 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -69,6 +69,92 @@ class EndToEndTestCase(unittest.TestCase): print('Operator log: {}'.format(k8s.get_operator_log())) raise + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_enable_disable_connection_pool(self): + ''' + For a database without connection pool, then turns it on, scale up, + turn off and on again. Test with different ways of doing this (via + enableConnectionPool or connectionPool configuration section). At the + end turn the connection pool off to not interfere with other tests. + ''' + k8s = self.k8s + service_labels = { + 'cluster-name': 'acid-minimal-cluster', + } + pod_labels = dict({ + 'connection-pool': 'acid-minimal-cluster-pooler', + }) + + pod_selector = to_selector(pod_labels) + service_selector = to_selector(service_labels) + + try: + # enable connection pool + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'enableConnectionPool': True, + } + }) + k8s.wait_for_pod_start(pod_selector) + + pods = k8s.api.core_v1.list_namespaced_pod( + 'default', label_selector=pod_selector + ).items + + self.assertTrue(pods, 'No connection pool pods') + + k8s.wait_for_service(service_selector) + services = k8s.api.core_v1.list_namespaced_service( + 'default', label_selector=service_selector + ).items + services = [ + s for s in services + if s.metadata.name.endswith('pooler') + ] + + self.assertTrue(services, 'No connection pool service') + + # scale up connection pool deployment + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'connectionPool': { + 'numberOfInstances': 2, + }, + } + }) + + k8s.wait_for_running_pods(pod_selector, 2) + + # turn it off, keeping configuration section + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'enableConnectionPool': False, + } + }) + k8s.wait_for_pods_to_stop(pod_selector) + + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'enableConnectionPool': True, + } + }) + k8s.wait_for_pod_start(pod_selector) + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_enable_load_balancer(self): ''' @@ -290,6 +376,10 @@ class EndToEndTestCase(unittest.TestCase): # patch also node where master ran before k8s.api.core_v1.patch_node(current_master_node, patch_readiness_label) + + # wait a little before proceeding with the pod distribution test + time.sleep(k8s.RETRY_TIMEOUT_SEC) + # toggle pod anti affinity to move replica away from master node self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) @@ -349,92 +439,6 @@ class EndToEndTestCase(unittest.TestCase): } k8s.update_config(unpatch_custom_service_annotations) - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_enable_disable_connection_pool(self): - ''' - For a database without connection pool, then turns it on, scale up, - turn off and on again. Test with different ways of doing this (via - enableConnectionPool or connectionPool configuration section). At the - end turn the connection pool off to not interfere with other tests. - ''' - k8s = self.k8s - service_labels = { - 'cluster-name': 'acid-minimal-cluster', - } - pod_labels = dict({ - 'connection-pool': 'acid-minimal-cluster-pooler', - }) - - pod_selector = to_selector(pod_labels) - service_selector = to_selector(service_labels) - - try: - # enable connection pool - k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresqls', 'acid-minimal-cluster', - { - 'spec': { - 'enableConnectionPool': True, - } - }) - k8s.wait_for_pod_start(pod_selector) - - pods = k8s.api.core_v1.list_namespaced_pod( - 'default', label_selector=pod_selector - ).items - - self.assertTrue(pods, 'No connection pool pods') - - k8s.wait_for_service(service_selector) - services = k8s.api.core_v1.list_namespaced_service( - 'default', label_selector=service_selector - ).items - services = [ - s for s in services - if s.metadata.name.endswith('pooler') - ] - - self.assertTrue(services, 'No connection pool service') - - # scale up connection pool deployment - k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresqls', 'acid-minimal-cluster', - { - 'spec': { - 'connectionPool': { - 'numberOfInstances': 2, - }, - } - }) - - k8s.wait_for_running_pods(pod_selector, 2) - - # turn it off, keeping configuration section - k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresqls', 'acid-minimal-cluster', - { - 'spec': { - 'enableConnectionPool': False, - } - }) - k8s.wait_for_pods_to_stop(pod_selector) - - k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresqls', 'acid-minimal-cluster', - { - 'spec': { - 'enableConnectionPool': True, - } - }) - k8s.wait_for_pod_start(pod_selector) - except timeout_decorator.TimeoutError: - print('Operator log: {}'.format(k8s.get_operator_log())) - raise - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_taint_based_eviction(self): ''' @@ -473,6 +477,9 @@ class EndToEndTestCase(unittest.TestCase): } k8s.update_config(patch_toleration_config) + # wait a little before proceeding with the pod distribution test + time.sleep(k8s.RETRY_TIMEOUT_SEC) + # toggle pod anti affinity to move replica away from master node self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) From 6ed10308380cbe87e63c92eed0490bde05811229 Mon Sep 17 00:00:00 2001 From: ReSearchITEng Date: Wed, 1 Apr 2020 10:39:54 +0300 Subject: [PATCH 018/168] TLS - add OpenShift compatibility (#885) * solves https://github.com/zalando/postgres-operator/pull/798#issuecomment-605201260 Co-authored-by: Felix Kunde --- docs/user.md | 13 +++++++++---- manifests/complete-postgres-manifest.yaml | 2 ++ pkg/cluster/k8sres.go | 12 ++---------- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/docs/user.md b/docs/user.md index 8c79bb485..dba157d90 100644 --- a/docs/user.md +++ b/docs/user.md @@ -572,10 +572,15 @@ However, this certificate cannot be verified and thus doesn't protect from active MITM attacks. In this section we show how to specify a custom TLS certificate which is mounted in the database pods via a K8s Secret. -Before applying these changes, the operator must also be configured with the -`spilo_fsgroup` set to the GID matching the postgres user group. If the value -is not provided, the cluster will default to `103` which is the GID from the -default spilo image. +Before applying these changes, in k8s the operator must also be configured with +the `spilo_fsgroup` set to the GID matching the postgres user group. If you +don't know the value, use `103` which is the GID from the default spilo image +(`spilo_fsgroup=103` in the cluster request spec). + +OpenShift allocates the users and groups dynamically (based on scc), and their +range is different in every namespace. Due to this dynamic behaviour, it's not +trivial to know at deploy time the uid/gid of the user in the cluster. +This way, in OpenShift, you may want to skip the spilo_fsgroup setting. Upload the cert as a kubernetes secret: ```sh diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index c82f1eac5..27dfc5f93 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -109,3 +109,5 @@ spec: certificateFile: "tls.crt" privateKeyFile: "tls.key" caFile: "" # optionally configure Postgres with a CA certificate +# When TLS is enabled, also set spiloFSGroup parameter above to the relevant value. +# if unknown, set it to 103 which is the usual value in the default spilo images. diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index c4919c62d..ee46f81e7 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -37,9 +37,6 @@ const ( localHost = "127.0.0.1/32" connectionPoolContainer = "connection-pool" pgPort = 5432 - - // the gid of the postgres user in the default spilo image - spiloPostgresGID = 103 ) type pgUser struct { @@ -990,13 +987,8 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef // configure TLS with a custom secret volume if spec.TLS != nil && spec.TLS.SecretName != "" { - if effectiveFSGroup == nil { - c.logger.Warnf("Setting the default FSGroup to satisfy the TLS configuration") - fsGroup := int64(spiloPostgresGID) - effectiveFSGroup = &fsGroup - } - // this is combined with the FSGroup above to give read access to the - // postgres user + // this is combined with the FSGroup in the section above + // to give read access to the postgres user defaultMode := int32(0640) volumes = append(volumes, v1.Volume{ Name: "tls-secret", From e6eb10d28a8ee843b9389dee2dda7441320f9cd6 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 1 Apr 2020 10:31:31 +0200 Subject: [PATCH 019/168] fix TestTLS (#894) --- pkg/cluster/k8sres_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index e04b281ba..5772f69d1 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -958,6 +958,7 @@ func TestTLS(t *testing.T) { var err error var spec acidv1.PostgresSpec var cluster *Cluster + var spiloFSGroup = int64(103) makeSpec := func(tls acidv1.TLSDescription) acidv1.PostgresSpec { return acidv1.PostgresSpec{ @@ -982,6 +983,9 @@ func TestTLS(t *testing.T) { SuperUsername: superUserName, ReplicationUsername: replicationUserName, }, + Resources: config.Resources{ + SpiloFSGroup: &spiloFSGroup, + }, }, }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) spec = makeSpec(acidv1.TLSDescription{SecretName: "my-secret", CAFile: "ca.crt"}) From b43b22dfcced21098308dd0bf3111716772e715c Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 1 Apr 2020 10:34:03 +0200 Subject: [PATCH 020/168] Call me pooler, not pool (#883) * rename pooler parts and add example to manifest * update codegen * fix manifest and add more details to docs * reflect renaming also in e2e tests --- README.md | 3 +- .../crds/operatorconfigurations.yaml | 22 +-- .../postgres-operator/crds/postgresqls.yaml | 4 +- .../templates/configmap.yaml | 2 +- .../templates/operatorconfiguration.yaml | 4 +- charts/postgres-operator/values-crd.yaml | 22 +-- charts/postgres-operator/values.yaml | 22 +-- docs/reference/cluster_manifest.md | 26 ++-- docs/reference/operator_parameters.md | 36 ++--- docs/user.md | 47 ++++--- e2e/tests/test_e2e.py | 26 ++-- manifests/complete-postgres-manifest.yaml | 14 ++ manifests/configmap.yaml | 20 +-- manifests/operatorconfiguration.crd.yaml | 22 +-- ...gresql-operator-default-configuration.yaml | 22 +-- manifests/postgresql.crd.yaml | 4 +- pkg/apis/acid.zalan.do/v1/crds.go | 26 ++-- .../v1/operator_configuration_type.go | 26 ++-- pkg/apis/acid.zalan.do/v1/postgresql_type.go | 8 +- .../acid.zalan.do/v1/zz_generated.deepcopy.go | 28 ++-- pkg/cluster/cluster.go | 106 +++++++------- pkg/cluster/cluster_test.go | 14 +- pkg/cluster/database.go | 42 +++--- pkg/cluster/k8sres.go | 130 +++++++++--------- pkg/cluster/k8sres_test.go | 118 ++++++++-------- pkg/cluster/resources.go | 76 +++++----- pkg/cluster/resources_test.go | 76 +++++----- pkg/cluster/sync.go | 126 ++++++++--------- pkg/cluster/sync_test.go | 58 ++++---- pkg/cluster/util.go | 28 ++-- pkg/controller/operator_config.go | 62 ++++----- pkg/spec/types.go | 6 +- pkg/util/config/config.go | 30 ++-- pkg/util/constants/pooler.go | 26 ++-- pkg/util/constants/roles.go | 2 +- 35 files changed, 651 insertions(+), 633 deletions(-) diff --git a/README.md b/README.md index a2771efa6..564bb68a1 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,9 @@ pipelines with no access to Kubernetes directly. * Rolling updates on Postgres cluster changes * Volume resize without Pod restarts +* Database connection pooler * Cloning Postgres clusters -* Logical Backups to S3 Bucket +* Logical backups to S3 Bucket * Standby cluster from S3 WAL archive * Configurable for non-cloud environments * UI to create and edit Postgres cluster manifests diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 7e3b607c0..63e52fd7a 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -318,44 +318,44 @@ spec: pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' scalyr_server_url: type: string - connection_pool: + connection_pooler: type: object properties: - connection_pool_schema: + connection_pooler_schema: type: string #default: "pooler" - connection_pool_user: + connection_pooler_user: type: string #default: "pooler" - connection_pool_image: + connection_pooler_image: type: string #default: "registry.opensource.zalan.do/acid/pgbouncer" - connection_pool_max_db_connections: + connection_pooler_max_db_connections: type: integer #default: 60 - connection_pool_mode: + connection_pooler_mode: type: string enum: - "session" - "transaction" #default: "transaction" - connection_pool_number_of_instances: + connection_pooler_number_of_instances: type: integer minimum: 2 #default: 2 - connection_pool_default_cpu_limit: + connection_pooler_default_cpu_limit: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' #default: "1" - connection_pool_default_cpu_request: + connection_pooler_default_cpu_request: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' #default: "500m" - connection_pool_default_memory_limit: + connection_pooler_default_memory_limit: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' #default: "100Mi" - connection_pool_default_memory_request: + connection_pooler_default_memory_request: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' #default: "100Mi" diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index a4c0e4f3a..57875cba7 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -106,7 +106,7 @@ spec: uid: format: uuid type: string - connectionPool: + connectionPooler: type: object properties: dockerImage: @@ -162,7 +162,7 @@ spec: # Note: usernames specified here as database owners must be declared in the users key of the spec key. dockerImage: type: string - enableConnectionPool: + enableConnectionPooler: type: boolean enableLogicalBackup: type: boolean diff --git a/charts/postgres-operator/templates/configmap.yaml b/charts/postgres-operator/templates/configmap.yaml index e8a805db7..64b55e0df 100644 --- a/charts/postgres-operator/templates/configmap.yaml +++ b/charts/postgres-operator/templates/configmap.yaml @@ -20,5 +20,5 @@ data: {{ toYaml .Values.configDebug | indent 2 }} {{ toYaml .Values.configLoggingRestApi | indent 2 }} {{ toYaml .Values.configTeamsApi | indent 2 }} -{{ toYaml .Values.configConnectionPool | indent 2 }} +{{ toYaml .Values.configConnectionPooler | indent 2 }} {{- end }} diff --git a/charts/postgres-operator/templates/operatorconfiguration.yaml b/charts/postgres-operator/templates/operatorconfiguration.yaml index b52b3d664..5df6e6238 100644 --- a/charts/postgres-operator/templates/operatorconfiguration.yaml +++ b/charts/postgres-operator/templates/operatorconfiguration.yaml @@ -34,6 +34,6 @@ configuration: {{ toYaml .Values.configLoggingRestApi | indent 4 }} scalyr: {{ toYaml .Values.configScalyr | indent 4 }} - connection_pool: -{{ toYaml .Values.configConnectionPool | indent 4 }} + connection_pooler: +{{ toYaml .Values.configConnectionPooler | indent 4 }} {{- end }} diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 79940b236..2490cfec6 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -267,24 +267,24 @@ configScalyr: # Memory request value for the Scalyr sidecar scalyr_memory_request: 50Mi -configConnectionPool: +configConnectionPooler: # db schema to install lookup function into - connection_pool_schema: "pooler" + connection_pooler_schema: "pooler" # db user for pooler to use - connection_pool_user: "pooler" + connection_pooler_user: "pooler" # docker image - connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer" # max db connections the pooler should hold - connection_pool_max_db_connections: 60 + connection_pooler_max_db_connections: 60 # default pooling mode - connection_pool_mode: "transaction" + connection_pooler_mode: "transaction" # number of pooler instances - connection_pool_number_of_instances: 2 + connection_pooler_number_of_instances: 2 # default resources - connection_pool_default_cpu_request: 500m - connection_pool_default_memory_request: 100Mi - connection_pool_default_cpu_limit: "1" - connection_pool_default_memory_limit: 100Mi + connection_pooler_default_cpu_request: 500m + connection_pooler_default_memory_request: 100Mi + connection_pooler_default_cpu_limit: "1" + connection_pooler_default_memory_limit: 100Mi rbac: # Specifies whether RBAC resources should be created diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 29f85339d..8d2dd8c5c 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -244,24 +244,24 @@ configTeamsApi: # teams_api_url: http://fake-teams-api.default.svc.cluster.local # configure connection pooler deployment created by the operator -configConnectionPool: +configConnectionPooler: # db schema to install lookup function into - connection_pool_schema: "pooler" + connection_pooler_schema: "pooler" # db user for pooler to use - connection_pool_user: "pooler" + connection_pooler_user: "pooler" # docker image - connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer" # max db connections the pooler should hold - connection_pool_max_db_connections: 60 + connection_pooler_max_db_connections: 60 # default pooling mode - connection_pool_mode: "transaction" + connection_pooler_mode: "transaction" # number of pooler instances - connection_pool_number_of_instances: 2 + connection_pooler_number_of_instances: 2 # default resources - connection_pool_default_cpu_request: 500m - connection_pool_default_memory_request: 100Mi - connection_pool_default_cpu_limit: "1" - connection_pool_default_memory_limit: 100Mi + connection_pooler_default_cpu_request: 500m + connection_pooler_default_memory_request: 100Mi + connection_pooler_default_cpu_limit: "1" + connection_pooler_default_memory_limit: 100Mi rbac: # Specifies whether RBAC resources should be created diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 4400cb666..967b2de5d 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -140,10 +140,10 @@ These parameters are grouped directly under the `spec` key in the manifest. is `false`, then no volume will be mounted no matter how operator was configured (so you can override the operator configuration). Optional. -* **enableConnectionPool** - Tells the operator to create a connection pool with a database. If this - field is true, a connection pool deployment will be created even if - `connectionPool` section is empty. Optional, not set by default. +* **enableConnectionPooler** + Tells the operator to create a connection pooler with a database. If this + field is true, a connection pooler deployment will be created even if + `connectionPooler` section is empty. Optional, not set by default. * **enableLogicalBackup** Determines if the logical backup of this cluster should be taken and uploaded @@ -365,34 +365,34 @@ CPU and memory limits for the sidecar container. memory limits for the sidecar container. Optional, overrides the `default_memory_limits` operator configuration parameter. Optional. -## Connection pool +## Connection pooler -Parameters are grouped under the `connectionPool` top-level key and specify -configuration for connection pool. If this section is not empty, a connection -pool will be created for a database even if `enableConnectionPool` is not +Parameters are grouped under the `connectionPooler` top-level key and specify +configuration for connection pooler. If this section is not empty, a connection +pooler will be created for a database even if `enableConnectionPooler` is not present. * **numberOfInstances** - How many instances of connection pool to create. + How many instances of connection pooler to create. * **schema** Schema to create for credentials lookup function. * **user** - User to create for connection pool to be able to connect to a database. + User to create for connection pooler to be able to connect to a database. * **dockerImage** - Which docker image to use for connection pool deployment. + Which docker image to use for connection pooler deployment. * **maxDBConnections** How many connections the pooler can max hold. This value is divided among the pooler pods. * **mode** - In which mode to run connection pool, transaction or session. + In which mode to run connection pooler, transaction or session. * **resources** - Resource configuration for connection pool deployment. + Resource configuration for connection pooler deployment. ## Custom TLS certificates diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 1ab92a287..c25ca3d49 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -597,39 +597,39 @@ scalyr sidecar. In the CRD-based configuration they are grouped under the * **scalyr_memory_limit** Memory limit value for the Scalyr sidecar. The default is `500Mi`. -## Connection pool configuration +## Connection pooler configuration -Parameters are grouped under the `connection_pool` top-level key and specify -default configuration for connection pool, if a postgres manifest requests it +Parameters are grouped under the `connection_pooler` top-level key and specify +default configuration for connection pooler, if a postgres manifest requests it but do not specify some of the parameters. All of them are optional with the operator being able to provide some reasonable defaults. -* **connection_pool_number_of_instances** - How many instances of connection pool to create. Default is 2 which is also +* **connection_pooler_number_of_instances** + How many instances of connection pooler to create. Default is 2 which is also the required minimum. -* **connection_pool_schema** +* **connection_pooler_schema** Schema to create for credentials lookup function. Default is `pooler`. -* **connection_pool_user** - User to create for connection pool to be able to connect to a database. +* **connection_pooler_user** + User to create for connection pooler to be able to connect to a database. Default is `pooler`. -* **connection_pool_image** - Docker image to use for connection pool deployment. +* **connection_pooler_image** + Docker image to use for connection pooler deployment. Default: "registry.opensource.zalan.do/acid/pgbouncer" -* **connection_pool_max_db_connections** +* **connection_pooler_max_db_connections** How many connections the pooler can max hold. This value is divided among the pooler pods. Default is 60 which will make up 30 connections per pod for the default setup with two instances. -* **connection_pool_mode** - Default pool mode, `session` or `transaction`. Default is `transaction`. +* **connection_pooler_mode** + Default pooler mode, `session` or `transaction`. Default is `transaction`. -* **connection_pool_default_cpu_request** - **connection_pool_default_memory_reques** - **connection_pool_default_cpu_limit** - **connection_pool_default_memory_limit** - Default resource configuration for connection pool deployment. The internal +* **connection_pooler_default_cpu_request** + **connection_pooler_default_memory_reques** + **connection_pooler_default_cpu_limit** + **connection_pooler_default_memory_limit** + Default resource configuration for connection pooler deployment. The internal default for memory request and limit is `100Mi`, for CPU it is `500m` and `1`. diff --git a/docs/user.md b/docs/user.md index dba157d90..67ed5971f 100644 --- a/docs/user.md +++ b/docs/user.md @@ -512,39 +512,38 @@ monitoring is outside the scope of operator responsibilities. See [administrator documentation](administrator.md) for details on how backups are executed. -## Connection pool +## Connection pooler -The operator can create a database side connection pool for those applications, -where an application side pool is not feasible, but a number of connections is -high. To create a connection pool together with a database, modify the +The operator can create a database side connection pooler for those applications +where an application side pooler is not feasible, but a number of connections is +high. To create a connection pooler together with a database, modify the manifest: ```yaml spec: - enableConnectionPool: true + enableConnectionPooler: true ``` -This will tell the operator to create a connection pool with default +This will tell the operator to create a connection pooler with default configuration, through which one can access the master via a separate service -`{cluster-name}-pooler`. In most of the cases provided default configuration -should be good enough. - -To configure a new connection pool, specify: +`{cluster-name}-pooler`. In most of the cases the +[default configuration](reference/operator_parameters.md#connection-pool-configuration) +should be good enough. To configure a new connection pooler individually for +each Postgres cluster, specify: ``` spec: - connectionPool: - # how many instances of connection pool to create - number_of_instances: 2 + connectionPooler: + # how many instances of connection pooler to create + numberOfInstances: 2 # in which mode to run, session or transaction mode: "transaction" - # schema, which operator will create to install credentials lookup - # function + # schema, which operator will create to install credentials lookup function schema: "pooler" - # user, which operator will create for connection pool + # user, which operator will create for connection pooler user: "pooler" # resources for each instance @@ -557,13 +556,17 @@ spec: memory: 100Mi ``` -By default `pgbouncer` is used to create a connection pool. To find out about -pool modes see [docs](https://www.pgbouncer.org/config.html#pool_mode) (but it -should be general approach between different implementation). +The `enableConnectionPooler` flag is not required when the `connectionPooler` +section is present in the manifest. But, it can be used to disable/remove the +pooler while keeping its configuration. -Note, that using `pgbouncer` means meaningful resource CPU limit should be less -than 1 core (there is a way to utilize more than one, but in K8S it's easier -just to spin up more instances). +By default, `pgbouncer` is used as connection pooler. To find out about pooler +modes read the `pgbouncer` [docs](https://www.pgbouncer.org/config.html#pooler_mode) +(but it should be the general approach between different implementation). + +Note, that using `pgbouncer` a meaningful resource CPU limit should be 1 core +or less (there is a way to utilize more than one, but in K8s it's easier just to +spin up more instances). ## Custom TLS certificates diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index f6fc8184c..445067d61 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -70,32 +70,32 @@ class EndToEndTestCase(unittest.TestCase): raise @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_enable_disable_connection_pool(self): + def test_enable_disable_connection_pooler(self): ''' - For a database without connection pool, then turns it on, scale up, + For a database without connection pooler, then turns it on, scale up, turn off and on again. Test with different ways of doing this (via - enableConnectionPool or connectionPool configuration section). At the - end turn the connection pool off to not interfere with other tests. + enableConnectionPooler or connectionPooler configuration section). At + the end turn connection pooler off to not interfere with other tests. ''' k8s = self.k8s service_labels = { 'cluster-name': 'acid-minimal-cluster', } pod_labels = dict({ - 'connection-pool': 'acid-minimal-cluster-pooler', + 'connection-pooler': 'acid-minimal-cluster-pooler', }) pod_selector = to_selector(pod_labels) service_selector = to_selector(service_labels) try: - # enable connection pool + # enable connection pooler k8s.api.custom_objects_api.patch_namespaced_custom_object( 'acid.zalan.do', 'v1', 'default', 'postgresqls', 'acid-minimal-cluster', { 'spec': { - 'enableConnectionPool': True, + 'enableConnectionPooler': True, } }) k8s.wait_for_pod_start(pod_selector) @@ -104,7 +104,7 @@ class EndToEndTestCase(unittest.TestCase): 'default', label_selector=pod_selector ).items - self.assertTrue(pods, 'No connection pool pods') + self.assertTrue(pods, 'No connection pooler pods') k8s.wait_for_service(service_selector) services = k8s.api.core_v1.list_namespaced_service( @@ -115,15 +115,15 @@ class EndToEndTestCase(unittest.TestCase): if s.metadata.name.endswith('pooler') ] - self.assertTrue(services, 'No connection pool service') + self.assertTrue(services, 'No connection pooler service') - # scale up connection pool deployment + # scale up connection pooler deployment k8s.api.custom_objects_api.patch_namespaced_custom_object( 'acid.zalan.do', 'v1', 'default', 'postgresqls', 'acid-minimal-cluster', { 'spec': { - 'connectionPool': { + 'connectionPooler': { 'numberOfInstances': 2, }, } @@ -137,7 +137,7 @@ class EndToEndTestCase(unittest.TestCase): 'postgresqls', 'acid-minimal-cluster', { 'spec': { - 'enableConnectionPool': False, + 'enableConnectionPooler': False, } }) k8s.wait_for_pods_to_stop(pod_selector) @@ -147,7 +147,7 @@ class EndToEndTestCase(unittest.TestCase): 'postgresqls', 'acid-minimal-cluster', { 'spec': { - 'enableConnectionPool': True, + 'enableConnectionPooler': True, } }) k8s.wait_for_pod_start(pod_selector) diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 27dfc5f93..a5811504e 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -19,6 +19,7 @@ spec: - createdb enableMasterLoadBalancer: false enableReplicaLoadBalancer: false +# enableConnectionPooler: true # not needed when connectionPooler section is present (see below) allowedSourceRanges: # load balancers' source ranges for both master and replica services - 127.0.0.1/32 databases: @@ -85,6 +86,19 @@ spec: # - 01:00-06:00 #UTC # - Sat:00:00-04:00 + connectionPooler: + numberOfInstances: 2 + mode: "transaction" + schema: "pooler" + user: "pooler" + resources: + requests: + cpu: 300m + memory: 100Mi + limits: + cpu: "1" + memory: 100Mi + initContainers: - name: date image: busybox diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 67c3368f3..c74a906d1 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -11,16 +11,16 @@ data: cluster_history_entries: "1000" cluster_labels: application:spilo cluster_name_label: cluster-name - # connection_pool_default_cpu_limit: "1" - # connection_pool_default_cpu_request: "500m" - # connection_pool_default_memory_limit: 100Mi - # connection_pool_default_memory_request: 100Mi - connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer:master-5" - # connection_pool_max_db_connections: 60 - # connection_pool_mode: "transaction" - # connection_pool_number_of_instances: 2 - # connection_pool_schema: "pooler" - # connection_pool_user: "pooler" + # connection_pooler_default_cpu_limit: "1" + # connection_pooler_default_cpu_request: "500m" + # connection_pooler_default_memory_limit: 100Mi + # connection_pooler_default_memory_request: 100Mi + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-5" + # connection_pooler_max_db_connections: 60 + # connection_pooler_mode: "transaction" + # connection_pooler_number_of_instances: 2 + # connection_pooler_schema: "pooler" + # connection_pooler_user: "pooler" # custom_service_annotations: "keyx:valuez,keya:valuea" # custom_pod_annotations: "keya:valuea,keyb:valueb" db_hosted_zone: db.example.com diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 4e6858af8..1d5544c7c 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -294,44 +294,44 @@ spec: pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' scalyr_server_url: type: string - connection_pool: + connection_pooler: type: object properties: - connection_pool_schema: + connection_pooler_schema: type: string #default: "pooler" - connection_pool_user: + connection_pooler_user: type: string #default: "pooler" - connection_pool_image: + connection_pooler_image: type: string #default: "registry.opensource.zalan.do/acid/pgbouncer" - connection_pool_max_db_connections: + connection_pooler_max_db_connections: type: integer #default: 60 - connection_pool_mode: + connection_pooler_mode: type: string enum: - "session" - "transaction" #default: "transaction" - connection_pool_number_of_instances: + connection_pooler_number_of_instances: type: integer minimum: 2 #default: 2 - connection_pool_default_cpu_limit: + connection_pooler_default_cpu_limit: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' #default: "1" - connection_pool_default_cpu_request: + connection_pooler_default_cpu_request: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' #default: "500m" - connection_pool_default_memory_limit: + connection_pooler_default_memory_limit: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' #default: "100Mi" - connection_pool_default_memory_request: + connection_pooler_default_memory_request: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' #default: "100Mi" diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 9d609713c..cd739c817 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -121,14 +121,14 @@ configuration: scalyr_memory_limit: 500Mi scalyr_memory_request: 50Mi # scalyr_server_url: "" - connection_pool: - connection_pool_default_cpu_limit: "1" - connection_pool_default_cpu_request: "500m" - connection_pool_default_memory_limit: 100Mi - connection_pool_default_memory_request: 100Mi - connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer:master-5" - # connection_pool_max_db_connections: 60 - connection_pool_mode: "transaction" - connection_pool_number_of_instances: 2 - # connection_pool_schema: "pooler" - # connection_pool_user: "pooler" + connection_pooler: + connection_pooler_default_cpu_limit: "1" + connection_pooler_default_cpu_request: "500m" + connection_pooler_default_memory_limit: 100Mi + connection_pooler_default_memory_request: 100Mi + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-5" + # connection_pooler_max_db_connections: 60 + connection_pooler_mode: "transaction" + connection_pooler_number_of_instances: 2 + # connection_pooler_schema: "pooler" + # connection_pooler_user: "pooler" diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 06434da14..48f6c7397 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -70,7 +70,7 @@ spec: uid: format: uuid type: string - connectionPool: + connectionPooler: type: object properties: dockerImage: @@ -126,7 +126,7 @@ spec: # Note: usernames specified here as database owners must be declared in the users key of the spec key. dockerImage: type: string - enableConnectionPool: + enableConnectionPooler: type: boolean enableLogicalBackup: type: boolean diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index dc552d3f4..cfccd1e56 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -177,7 +177,7 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, }, }, - "connectionPool": { + "connectionPooler": { Type: "object", Properties: map[string]apiextv1beta1.JSONSchemaProps{ "dockerImage": { @@ -259,7 +259,7 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ "dockerImage": { Type: "string", }, - "enableConnectionPool": { + "enableConnectionPooler": { Type: "boolean", }, "enableLogicalBackup": { @@ -1129,32 +1129,32 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, }, }, - "connection_pool": { + "connection_pooler": { Type: "object", Properties: map[string]apiextv1beta1.JSONSchemaProps{ - "connection_pool_default_cpu_limit": { + "connection_pooler_default_cpu_limit": { Type: "string", Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", }, - "connection_pool_default_cpu_request": { + "connection_pooler_default_cpu_request": { Type: "string", Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", }, - "connection_pool_default_memory_limit": { + "connection_pooler_default_memory_limit": { Type: "string", Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", }, - "connection_pool_default_memory_request": { + "connection_pooler_default_memory_request": { Type: "string", Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", }, - "connection_pool_image": { + "connection_pooler_image": { Type: "string", }, - "connection_pool_max_db_connections": { + "connection_pooler_max_db_connections": { Type: "integer", }, - "connection_pool_mode": { + "connection_pooler_mode": { Type: "string", Enum: []apiextv1beta1.JSON{ { @@ -1165,14 +1165,14 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, }, }, - "connection_pool_number_of_instances": { + "connection_pooler_number_of_instances": { Type: "integer", Minimum: &min2, }, - "connection_pool_schema": { + "connection_pooler_schema": { Type: "string", }, - "connection_pool_user": { + "connection_pooler_user": { Type: "string", }, }, diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 3dbe96b7f..6eb6732a5 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -153,18 +153,18 @@ type ScalyrConfiguration struct { ScalyrMemoryLimit string `json:"scalyr_memory_limit,omitempty"` } -// Defines default configuration for connection pool -type ConnectionPoolConfiguration struct { - NumberOfInstances *int32 `json:"connection_pool_number_of_instances,omitempty"` - Schema string `json:"connection_pool_schema,omitempty"` - User string `json:"connection_pool_user,omitempty"` - Image string `json:"connection_pool_image,omitempty"` - Mode string `json:"connection_pool_mode,omitempty"` - MaxDBConnections *int32 `json:"connection_pool_max_db_connections,omitempty"` - DefaultCPURequest string `json:"connection_pool_default_cpu_request,omitempty"` - DefaultMemoryRequest string `json:"connection_pool_default_memory_request,omitempty"` - DefaultCPULimit string `json:"connection_pool_default_cpu_limit,omitempty"` - DefaultMemoryLimit string `json:"connection_pool_default_memory_limit,omitempty"` +// Defines default configuration for connection pooler +type ConnectionPoolerConfiguration struct { + NumberOfInstances *int32 `json:"connection_pooler_number_of_instances,omitempty"` + Schema string `json:"connection_pooler_schema,omitempty"` + User string `json:"connection_pooler_user,omitempty"` + Image string `json:"connection_pooler_image,omitempty"` + Mode string `json:"connection_pooler_mode,omitempty"` + MaxDBConnections *int32 `json:"connection_pooler_max_db_connections,omitempty"` + DefaultCPURequest string `json:"connection_pooler_default_cpu_request,omitempty"` + DefaultMemoryRequest string `json:"connection_pooler_default_memory_request,omitempty"` + DefaultCPULimit string `json:"connection_pooler_default_cpu_limit,omitempty"` + DefaultMemoryLimit string `json:"connection_pooler_default_memory_limit,omitempty"` } // OperatorLogicalBackupConfiguration defines configuration for logical backup @@ -203,7 +203,7 @@ type OperatorConfigurationData struct { LoggingRESTAPI LoggingRESTAPIConfiguration `json:"logging_rest_api"` Scalyr ScalyrConfiguration `json:"scalyr"` LogicalBackup OperatorLogicalBackupConfiguration `json:"logical_backup"` - ConnectionPool ConnectionPoolConfiguration `json:"connection_pool"` + ConnectionPooler ConnectionPoolerConfiguration `json:"connection_pooler"` } //Duration shortens this frequently used name diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 1784f8235..0695d3f9f 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -29,8 +29,8 @@ type PostgresSpec struct { Patroni `json:"patroni,omitempty"` Resources `json:"resources,omitempty"` - EnableConnectionPool *bool `json:"enableConnectionPool,omitempty"` - ConnectionPool *ConnectionPool `json:"connectionPool,omitempty"` + EnableConnectionPooler *bool `json:"enableConnectionPooler,omitempty"` + ConnectionPooler *ConnectionPooler `json:"connectionPooler,omitempty"` TeamID string `json:"teamId"` DockerImage string `json:"dockerImage,omitempty"` @@ -175,10 +175,10 @@ type PostgresStatus struct { // resources) // Type string `json:"type,omitempty"` // -// TODO: figure out what other important parameters of the connection pool it +// TODO: figure out what other important parameters of the connection pooler it // makes sense to expose. E.g. pool size (min/max boundaries), max client // connections etc. -type ConnectionPool struct { +type ConnectionPooler struct { NumberOfInstances *int32 `json:"numberOfInstances,omitempty"` Schema string `json:"schema,omitempty"` User string `json:"user,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 65a19600a..92c8af34b 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -69,7 +69,7 @@ func (in *CloneDescription) DeepCopy() *CloneDescription { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ConnectionPool) DeepCopyInto(out *ConnectionPool) { +func (in *ConnectionPooler) DeepCopyInto(out *ConnectionPooler) { *out = *in if in.NumberOfInstances != nil { in, out := &in.NumberOfInstances, &out.NumberOfInstances @@ -85,18 +85,18 @@ func (in *ConnectionPool) DeepCopyInto(out *ConnectionPool) { return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectionPool. -func (in *ConnectionPool) DeepCopy() *ConnectionPool { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectionPooler. +func (in *ConnectionPooler) DeepCopy() *ConnectionPooler { if in == nil { return nil } - out := new(ConnectionPool) + out := new(ConnectionPooler) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ConnectionPoolConfiguration) DeepCopyInto(out *ConnectionPoolConfiguration) { +func (in *ConnectionPoolerConfiguration) DeepCopyInto(out *ConnectionPoolerConfiguration) { *out = *in if in.NumberOfInstances != nil { in, out := &in.NumberOfInstances, &out.NumberOfInstances @@ -111,12 +111,12 @@ func (in *ConnectionPoolConfiguration) DeepCopyInto(out *ConnectionPoolConfigura return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectionPoolConfiguration. -func (in *ConnectionPoolConfiguration) DeepCopy() *ConnectionPoolConfiguration { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectionPoolerConfiguration. +func (in *ConnectionPoolerConfiguration) DeepCopy() *ConnectionPoolerConfiguration { if in == nil { return nil } - out := new(ConnectionPoolConfiguration) + out := new(ConnectionPoolerConfiguration) in.DeepCopyInto(out) return out } @@ -308,7 +308,7 @@ func (in *OperatorConfigurationData) DeepCopyInto(out *OperatorConfigurationData out.LoggingRESTAPI = in.LoggingRESTAPI out.Scalyr = in.Scalyr out.LogicalBackup = in.LogicalBackup - in.ConnectionPool.DeepCopyInto(&out.ConnectionPool) + in.ConnectionPooler.DeepCopyInto(&out.ConnectionPooler) return } @@ -471,14 +471,14 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { out.Volume = in.Volume in.Patroni.DeepCopyInto(&out.Patroni) out.Resources = in.Resources - if in.EnableConnectionPool != nil { - in, out := &in.EnableConnectionPool, &out.EnableConnectionPool + if in.EnableConnectionPooler != nil { + in, out := &in.EnableConnectionPooler, &out.EnableConnectionPooler *out = new(bool) **out = **in } - if in.ConnectionPool != nil { - in, out := &in.ConnectionPool, &out.ConnectionPool - *out = new(ConnectionPool) + if in.ConnectionPooler != nil { + in, out := &in.ConnectionPooler, &out.ConnectionPooler + *out = new(ConnectionPooler) (*in).DeepCopyInto(*out) } if in.SpiloFSGroup != nil { diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 45ee55300..cde2dc260 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -50,14 +50,14 @@ type Config struct { PodServiceAccountRoleBinding *rbacv1.RoleBinding } -// K8S objects that are belongs to a connection pool -type ConnectionPoolObjects struct { +// K8S objects that are belongs to a connection pooler +type ConnectionPoolerObjects struct { Deployment *appsv1.Deployment Service *v1.Service - // It could happen that a connection pool was enabled, but the operator was - // not able to properly process a corresponding event or was restarted. In - // this case we will miss missing/require situation and a lookup function + // It could happen that a connection pooler was enabled, but the operator + // was not able to properly process a corresponding event or was restarted. + // In this case we will miss missing/require situation and a lookup function // will not be installed. To avoid synchronizing it all the time to prevent // this, we can remember the result in memory at least until the next // restart. @@ -69,7 +69,7 @@ type kubeResources struct { Endpoints map[PostgresRole]*v1.Endpoints Secrets map[types.UID]*v1.Secret Statefulset *appsv1.StatefulSet - ConnectionPool *ConnectionPoolObjects + ConnectionPooler *ConnectionPoolerObjects PodDisruptionBudget *policybeta1.PodDisruptionBudget //Pods are treated separately //PVCs are treated separately @@ -337,24 +337,24 @@ func (c *Cluster) Create() error { c.logger.Errorf("could not list resources: %v", err) } - // Create connection pool deployment and services if necessary. Since we - // need to peform some operations with the database itself (e.g. install + // Create connection pooler deployment and services if necessary. Since we + // need to perform some operations with the database itself (e.g. install // lookup function), do it as the last step, when everything is available. // - // Do not consider connection pool as a strict requirement, and if + // Do not consider connection pooler as a strict requirement, and if // something fails, report warning - if c.needConnectionPool() { - if c.ConnectionPool != nil { - c.logger.Warning("Connection pool already exists in the cluster") + if c.needConnectionPooler() { + if c.ConnectionPooler != nil { + c.logger.Warning("Connection pooler already exists in the cluster") return nil } - connPool, err := c.createConnectionPool(c.installLookupFunction) + connectionPooler, err := c.createConnectionPooler(c.installLookupFunction) if err != nil { - c.logger.Warningf("could not create connection pool: %v", err) + c.logger.Warningf("could not create connection pooler: %v", err) return nil } - c.logger.Infof("connection pool %q has been successfully created", - util.NameFromMeta(connPool.Deployment.ObjectMeta)) + c.logger.Infof("connection pooler %q has been successfully created", + util.NameFromMeta(connectionPooler.Deployment.ObjectMeta)) } return nil @@ -612,11 +612,11 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { } } - // connection pool needs one system user created, which is done in + // connection pooler needs one system user created, which is done in // initUsers. Check if it needs to be called. sameUsers := reflect.DeepEqual(oldSpec.Spec.Users, newSpec.Spec.Users) - needConnPool := c.needConnectionPoolWorker(&newSpec.Spec) - if !sameUsers || needConnPool { + needConnectionPooler := c.needConnectionPoolerWorker(&newSpec.Spec) + if !sameUsers || needConnectionPooler { c.logger.Debugf("syncing secrets") if err := c.initUsers(); err != nil { c.logger.Errorf("could not init users: %v", err) @@ -740,9 +740,9 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { } } - // sync connection pool - if err := c.syncConnectionPool(oldSpec, newSpec, c.installLookupFunction); err != nil { - return fmt.Errorf("could not sync connection pool: %v", err) + // sync connection pooler + if err := c.syncConnectionPooler(oldSpec, newSpec, c.installLookupFunction); err != nil { + return fmt.Errorf("could not sync connection pooler: %v", err) } return nil @@ -796,11 +796,11 @@ func (c *Cluster) Delete() { c.logger.Warningf("could not remove leftover patroni objects; %v", err) } - // Delete connection pool objects anyway, even if it's not mentioned in the + // Delete connection pooler objects anyway, even if it's not mentioned in the // manifest, just to not keep orphaned components in case if something went // wrong - if err := c.deleteConnectionPool(); err != nil { - c.logger.Warningf("could not remove connection pool: %v", err) + if err := c.deleteConnectionPooler(); err != nil { + c.logger.Warningf("could not remove connection pooler: %v", err) } } @@ -869,32 +869,32 @@ func (c *Cluster) initSystemUsers() { Password: util.RandomPassword(constants.PasswordLength), } - // Connection pool user is an exception, if requested it's going to be + // Connection pooler user is an exception, if requested it's going to be // created by operator as a normal pgUser - if c.needConnectionPool() { - // initialize empty connection pool if not done yet - if c.Spec.ConnectionPool == nil { - c.Spec.ConnectionPool = &acidv1.ConnectionPool{} + if c.needConnectionPooler() { + // initialize empty connection pooler if not done yet + if c.Spec.ConnectionPooler == nil { + c.Spec.ConnectionPooler = &acidv1.ConnectionPooler{} } username := util.Coalesce( - c.Spec.ConnectionPool.User, - c.OpConfig.ConnectionPool.User) + c.Spec.ConnectionPooler.User, + c.OpConfig.ConnectionPooler.User) // connection pooler application should be able to login with this role - connPoolUser := spec.PgUser{ - Origin: spec.RoleConnectionPool, + connectionPoolerUser := spec.PgUser{ + Origin: spec.RoleConnectionPooler, Name: username, Flags: []string{constants.RoleFlagLogin}, Password: util.RandomPassword(constants.PasswordLength), } if _, exists := c.pgUsers[username]; !exists { - c.pgUsers[username] = connPoolUser + c.pgUsers[username] = connectionPoolerUser } - if _, exists := c.systemUsers[constants.ConnectionPoolUserKeyName]; !exists { - c.systemUsers[constants.ConnectionPoolUserKeyName] = connPoolUser + if _, exists := c.systemUsers[constants.ConnectionPoolerUserKeyName]; !exists { + c.systemUsers[constants.ConnectionPoolerUserKeyName] = connectionPoolerUser } } } @@ -1224,10 +1224,10 @@ func (c *Cluster) deletePatroniClusterConfigMaps() error { return c.deleteClusterObject(get, deleteConfigMapFn, "configmap") } -// Test if two connection pool configuration needs to be synced. For simplicity +// Test if two connection pooler configuration needs to be synced. For simplicity // compare not the actual K8S objects, but the configuration itself and request // sync if there is any difference. -func (c *Cluster) needSyncConnPoolSpecs(oldSpec, newSpec *acidv1.ConnectionPool) (sync bool, reasons []string) { +func (c *Cluster) needSyncConnectionPoolerSpecs(oldSpec, newSpec *acidv1.ConnectionPooler) (sync bool, reasons []string) { reasons = []string{} sync = false @@ -1264,21 +1264,21 @@ func syncResources(a, b *v1.ResourceRequirements) bool { return false } -// Check if we need to synchronize connection pool deployment due to new +// Check if we need to synchronize connection pooler deployment due to new // defaults, that are different from what we see in the DeploymentSpec -func (c *Cluster) needSyncConnPoolDefaults( - spec *acidv1.ConnectionPool, +func (c *Cluster) needSyncConnectionPoolerDefaults( + spec *acidv1.ConnectionPooler, deployment *appsv1.Deployment) (sync bool, reasons []string) { reasons = []string{} sync = false - config := c.OpConfig.ConnectionPool + config := c.OpConfig.ConnectionPooler podTemplate := deployment.Spec.Template - poolContainer := podTemplate.Spec.Containers[constants.ConnPoolContainer] + poolerContainer := podTemplate.Spec.Containers[constants.ConnectionPoolerContainer] if spec == nil { - spec = &acidv1.ConnectionPool{} + spec = &acidv1.ConnectionPooler{} } if spec.NumberOfInstances == nil && @@ -1291,25 +1291,25 @@ func (c *Cluster) needSyncConnPoolDefaults( } if spec.DockerImage == "" && - poolContainer.Image != config.Image { + poolerContainer.Image != config.Image { sync = true msg := fmt.Sprintf("DockerImage is different (having %s, required %s)", - poolContainer.Image, config.Image) + poolerContainer.Image, config.Image) reasons = append(reasons, msg) } expectedResources, err := generateResourceRequirements(spec.Resources, - c.makeDefaultConnPoolResources()) + c.makeDefaultConnectionPoolerResources()) // An error to generate expected resources means something is not quite // right, but for the purpose of robustness do not panic here, just report // and ignore resources comparison (in the worst case there will be no // updates for new resource values). - if err == nil && syncResources(&poolContainer.Resources, expectedResources) { + if err == nil && syncResources(&poolerContainer.Resources, expectedResources) { sync = true msg := fmt.Sprintf("Resources are different (having %+v, required %+v)", - poolContainer.Resources, expectedResources) + poolerContainer.Resources, expectedResources) reasons = append(reasons, msg) } @@ -1317,13 +1317,13 @@ func (c *Cluster) needSyncConnPoolDefaults( c.logger.Warningf("Cannot generate expected resources, %v", err) } - for _, env := range poolContainer.Env { + for _, env := range poolerContainer.Env { if spec.User == "" && env.Name == "PGUSER" { ref := env.ValueFrom.SecretKeyRef.LocalObjectReference if ref.Name != c.credentialSecretName(config.User) { sync = true - msg := fmt.Sprintf("Pool user is different (having %s, required %s)", + msg := fmt.Sprintf("pooler user is different (having %s, required %s)", ref.Name, config.User) reasons = append(reasons, msg) } @@ -1331,7 +1331,7 @@ func (c *Cluster) needSyncConnPoolDefaults( if spec.Schema == "" && env.Name == "PGSCHEMA" && env.Value != config.Schema { sync = true - msg := fmt.Sprintf("Pool schema is different (having %s, required %s)", + msg := fmt.Sprintf("pooler schema is different (having %s, required %s)", env.Value, config.Schema) reasons = append(reasons, msg) } diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index a1b361642..af3092ad5 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -709,16 +709,16 @@ func TestServiceAnnotations(t *testing.T) { func TestInitSystemUsers(t *testing.T) { testName := "Test system users initialization" - // default cluster without connection pool + // default cluster without connection pooler cl.initSystemUsers() - if _, exist := cl.systemUsers[constants.ConnectionPoolUserKeyName]; exist { - t.Errorf("%s, connection pool user is present", testName) + if _, exist := cl.systemUsers[constants.ConnectionPoolerUserKeyName]; exist { + t.Errorf("%s, connection pooler user is present", testName) } - // cluster with connection pool - cl.Spec.EnableConnectionPool = boolToPointer(true) + // cluster with connection pooler + cl.Spec.EnableConnectionPooler = boolToPointer(true) cl.initSystemUsers() - if _, exist := cl.systemUsers[constants.ConnectionPoolUserKeyName]; !exist { - t.Errorf("%s, connection pool user is not present", testName) + if _, exist := cl.systemUsers[constants.ConnectionPoolerUserKeyName]; !exist { + t.Errorf("%s, connection pooler user is not present", testName) } } diff --git a/pkg/cluster/database.go b/pkg/cluster/database.go index bca68c188..28f97b5cc 100644 --- a/pkg/cluster/database.go +++ b/pkg/cluster/database.go @@ -27,13 +27,13 @@ const ( WHERE a.rolname = ANY($1) ORDER BY 1;` - getDatabasesSQL = `SELECT datname, pg_get_userbyid(datdba) AS owner FROM pg_database;` - createDatabaseSQL = `CREATE DATABASE "%s" OWNER "%s";` - alterDatabaseOwnerSQL = `ALTER DATABASE "%s" OWNER TO "%s";` - connectionPoolLookup = ` - CREATE SCHEMA IF NOT EXISTS {{.pool_schema}}; + getDatabasesSQL = `SELECT datname, pg_get_userbyid(datdba) AS owner FROM pg_database;` + createDatabaseSQL = `CREATE DATABASE "%s" OWNER "%s";` + alterDatabaseOwnerSQL = `ALTER DATABASE "%s" OWNER TO "%s";` + connectionPoolerLookup = ` + CREATE SCHEMA IF NOT EXISTS {{.pooler_schema}}; - CREATE OR REPLACE FUNCTION {{.pool_schema}}.user_lookup( + CREATE OR REPLACE FUNCTION {{.pooler_schema}}.user_lookup( in i_username text, out uname text, out phash text) RETURNS record AS $$ BEGIN @@ -43,11 +43,11 @@ const ( END; $$ LANGUAGE plpgsql SECURITY DEFINER; - REVOKE ALL ON FUNCTION {{.pool_schema}}.user_lookup(text) - FROM public, {{.pool_user}}; - GRANT EXECUTE ON FUNCTION {{.pool_schema}}.user_lookup(text) - TO {{.pool_user}}; - GRANT USAGE ON SCHEMA {{.pool_schema}} TO {{.pool_user}}; + REVOKE ALL ON FUNCTION {{.pooler_schema}}.user_lookup(text) + FROM public, {{.pooler_user}}; + GRANT EXECUTE ON FUNCTION {{.pooler_schema}}.user_lookup(text) + TO {{.pooler_user}}; + GRANT USAGE ON SCHEMA {{.pooler_schema}} TO {{.pooler_user}}; ` ) @@ -278,9 +278,9 @@ func makeUserFlags(rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin return result } -// Creates a connection pool credentials lookup function in every database to -// perform remote authentification. -func (c *Cluster) installLookupFunction(poolSchema, poolUser string) error { +// Creates a connection pooler credentials lookup function in every database to +// perform remote authentication. +func (c *Cluster) installLookupFunction(poolerSchema, poolerUser string) error { var stmtBytes bytes.Buffer c.logger.Info("Installing lookup function") @@ -299,11 +299,11 @@ func (c *Cluster) installLookupFunction(poolSchema, poolUser string) error { currentDatabases, err := c.getDatabases() if err != nil { - msg := "could not get databases to install pool lookup function: %v" + msg := "could not get databases to install pooler lookup function: %v" return fmt.Errorf(msg, err) } - templater := template.Must(template.New("sql").Parse(connectionPoolLookup)) + templater := template.Must(template.New("sql").Parse(connectionPoolerLookup)) for dbname, _ := range currentDatabases { if dbname == "template0" || dbname == "template1" { @@ -314,11 +314,11 @@ func (c *Cluster) installLookupFunction(poolSchema, poolUser string) error { return fmt.Errorf("could not init database connection to %s", dbname) } - c.logger.Infof("Install pool lookup function into %s", dbname) + c.logger.Infof("Install pooler lookup function into %s", dbname) params := TemplateParams{ - "pool_schema": poolSchema, - "pool_user": poolUser, + "pooler_schema": poolerSchema, + "pooler_user": poolerUser, } if err := templater.Execute(&stmtBytes, params); err != nil { @@ -353,12 +353,12 @@ func (c *Cluster) installLookupFunction(poolSchema, poolUser string) error { continue } - c.logger.Infof("Pool lookup function installed into %s", dbname) + c.logger.Infof("pooler lookup function installed into %s", dbname) if err := c.closeDbConn(); err != nil { c.logger.Errorf("could not close database connection: %v", err) } } - c.ConnectionPool.LookupFunction = true + c.ConnectionPooler.LookupFunction = true return nil } diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index ee46f81e7..d5297ab8e 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -35,7 +35,7 @@ const ( patroniPGParametersParameterName = "parameters" patroniPGHBAConfParameterName = "pg_hba" localHost = "127.0.0.1/32" - connectionPoolContainer = "connection-pool" + connectionPoolerContainer = "connection-pooler" pgPort = 5432 ) @@ -72,7 +72,7 @@ func (c *Cluster) statefulSetName() string { return c.Name } -func (c *Cluster) connPoolName() string { +func (c *Cluster) connectionPoolerName() string { return c.Name + "-pooler" } @@ -139,18 +139,18 @@ func (c *Cluster) makeDefaultResources() acidv1.Resources { } } -// Generate default resource section for connection pool deployment, to be used -// if nothing custom is specified in the manifest -func (c *Cluster) makeDefaultConnPoolResources() acidv1.Resources { +// Generate default resource section for connection pooler deployment, to be +// used if nothing custom is specified in the manifest +func (c *Cluster) makeDefaultConnectionPoolerResources() acidv1.Resources { config := c.OpConfig defaultRequests := acidv1.ResourceDescription{ - CPU: config.ConnectionPool.ConnPoolDefaultCPURequest, - Memory: config.ConnectionPool.ConnPoolDefaultMemoryRequest, + CPU: config.ConnectionPooler.ConnectionPoolerDefaultCPURequest, + Memory: config.ConnectionPooler.ConnectionPoolerDefaultMemoryRequest, } defaultLimits := acidv1.ResourceDescription{ - CPU: config.ConnectionPool.ConnPoolDefaultCPULimit, - Memory: config.ConnectionPool.ConnPoolDefaultMemoryLimit, + CPU: config.ConnectionPooler.ConnectionPoolerDefaultCPULimit, + Memory: config.ConnectionPooler.ConnectionPoolerDefaultMemoryLimit, } return acidv1.Resources{ @@ -1851,33 +1851,33 @@ func (c *Cluster) getLogicalBackupJobName() (jobName string) { // // DEFAULT_SIZE is a pool size per db/user (having in mind the use case when // most of the queries coming through a connection pooler are from the same -// user to the same db). In case if we want to spin up more connection pool +// user to the same db). In case if we want to spin up more connection pooler // instances, take this into account and maintain the same number of // connections. // -// MIN_SIZE is a pool minimal size, to prevent situation when sudden workload +// MIN_SIZE is a pool's minimal size, to prevent situation when sudden workload // have to wait for spinning up a new connections. // -// RESERVE_SIZE is how many additional connections to allow for a pool. -func (c *Cluster) getConnPoolEnvVars(spec *acidv1.PostgresSpec) []v1.EnvVar { +// RESERVE_SIZE is how many additional connections to allow for a pooler. +func (c *Cluster) getConnectionPoolerEnvVars(spec *acidv1.PostgresSpec) []v1.EnvVar { effectiveMode := util.Coalesce( - spec.ConnectionPool.Mode, - c.OpConfig.ConnectionPool.Mode) + spec.ConnectionPooler.Mode, + c.OpConfig.ConnectionPooler.Mode) - numberOfInstances := spec.ConnectionPool.NumberOfInstances + numberOfInstances := spec.ConnectionPooler.NumberOfInstances if numberOfInstances == nil { numberOfInstances = util.CoalesceInt32( - c.OpConfig.ConnectionPool.NumberOfInstances, + c.OpConfig.ConnectionPooler.NumberOfInstances, k8sutil.Int32ToPointer(1)) } effectiveMaxDBConn := util.CoalesceInt32( - spec.ConnectionPool.MaxDBConnections, - c.OpConfig.ConnectionPool.MaxDBConnections) + spec.ConnectionPooler.MaxDBConnections, + c.OpConfig.ConnectionPooler.MaxDBConnections) if effectiveMaxDBConn == nil { effectiveMaxDBConn = k8sutil.Int32ToPointer( - constants.ConnPoolMaxDBConnections) + constants.ConnectionPoolerMaxDBConnections) } maxDBConn := *effectiveMaxDBConn / *numberOfInstances @@ -1888,51 +1888,51 @@ func (c *Cluster) getConnPoolEnvVars(spec *acidv1.PostgresSpec) []v1.EnvVar { return []v1.EnvVar{ { - Name: "CONNECTION_POOL_PORT", + Name: "CONNECTION_POOLER_PORT", Value: fmt.Sprint(pgPort), }, { - Name: "CONNECTION_POOL_MODE", + Name: "CONNECTION_POOLER_MODE", Value: effectiveMode, }, { - Name: "CONNECTION_POOL_DEFAULT_SIZE", + Name: "CONNECTION_POOLER_DEFAULT_SIZE", Value: fmt.Sprint(defaultSize), }, { - Name: "CONNECTION_POOL_MIN_SIZE", + Name: "CONNECTION_POOLER_MIN_SIZE", Value: fmt.Sprint(minSize), }, { - Name: "CONNECTION_POOL_RESERVE_SIZE", + Name: "CONNECTION_POOLER_RESERVE_SIZE", Value: fmt.Sprint(reserveSize), }, { - Name: "CONNECTION_POOL_MAX_CLIENT_CONN", - Value: fmt.Sprint(constants.ConnPoolMaxClientConnections), + Name: "CONNECTION_POOLER_MAX_CLIENT_CONN", + Value: fmt.Sprint(constants.ConnectionPoolerMaxClientConnections), }, { - Name: "CONNECTION_POOL_MAX_DB_CONN", + Name: "CONNECTION_POOLER_MAX_DB_CONN", Value: fmt.Sprint(maxDBConn), }, } } -func (c *Cluster) generateConnPoolPodTemplate(spec *acidv1.PostgresSpec) ( +func (c *Cluster) generateConnectionPoolerPodTemplate(spec *acidv1.PostgresSpec) ( *v1.PodTemplateSpec, error) { gracePeriod := int64(c.OpConfig.PodTerminateGracePeriod.Seconds()) resources, err := generateResourceRequirements( - spec.ConnectionPool.Resources, - c.makeDefaultConnPoolResources()) + spec.ConnectionPooler.Resources, + c.makeDefaultConnectionPoolerResources()) effectiveDockerImage := util.Coalesce( - spec.ConnectionPool.DockerImage, - c.OpConfig.ConnectionPool.Image) + spec.ConnectionPooler.DockerImage, + c.OpConfig.ConnectionPooler.Image) effectiveSchema := util.Coalesce( - spec.ConnectionPool.Schema, - c.OpConfig.ConnectionPool.Schema) + spec.ConnectionPooler.Schema, + c.OpConfig.ConnectionPooler.Schema) if err != nil { return nil, fmt.Errorf("could not generate resource requirements: %v", err) @@ -1940,8 +1940,8 @@ func (c *Cluster) generateConnPoolPodTemplate(spec *acidv1.PostgresSpec) ( secretSelector := func(key string) *v1.SecretKeySelector { effectiveUser := util.Coalesce( - spec.ConnectionPool.User, - c.OpConfig.ConnectionPool.User) + spec.ConnectionPooler.User, + c.OpConfig.ConnectionPooler.User) return &v1.SecretKeySelector{ LocalObjectReference: v1.LocalObjectReference{ @@ -1967,7 +1967,7 @@ func (c *Cluster) generateConnPoolPodTemplate(spec *acidv1.PostgresSpec) ( }, }, // the convention is to use the same schema name as - // connection pool username + // connection pooler username { Name: "PGSCHEMA", Value: effectiveSchema, @@ -1980,10 +1980,10 @@ func (c *Cluster) generateConnPoolPodTemplate(spec *acidv1.PostgresSpec) ( }, } - envVars = append(envVars, c.getConnPoolEnvVars(spec)...) + envVars = append(envVars, c.getConnectionPoolerEnvVars(spec)...) poolerContainer := v1.Container{ - Name: connectionPoolContainer, + Name: connectionPoolerContainer, Image: effectiveDockerImage, ImagePullPolicy: v1.PullIfNotPresent, Resources: *resources, @@ -1998,7 +1998,7 @@ func (c *Cluster) generateConnPoolPodTemplate(spec *acidv1.PostgresSpec) ( podTemplate := &v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: c.connPoolLabelsSelector().MatchLabels, + Labels: c.connectionPoolerLabelsSelector().MatchLabels, Namespace: c.Namespace, Annotations: c.generatePodAnnotations(spec), }, @@ -2040,32 +2040,32 @@ func (c *Cluster) ownerReferences() []metav1.OwnerReference { } } -func (c *Cluster) generateConnPoolDeployment(spec *acidv1.PostgresSpec) ( +func (c *Cluster) generateConnectionPoolerDeployment(spec *acidv1.PostgresSpec) ( *appsv1.Deployment, error) { // there are two ways to enable connection pooler, either to specify a - // connectionPool section or enableConnectionPool. In the second case - // spec.connectionPool will be nil, so to make it easier to calculate + // connectionPooler section or enableConnectionPooler. In the second case + // spec.connectionPooler will be nil, so to make it easier to calculate // default values, initialize it to an empty structure. It could be done // anywhere, but here is the earliest common entry point between sync and // create code, so init here. - if spec.ConnectionPool == nil { - spec.ConnectionPool = &acidv1.ConnectionPool{} + if spec.ConnectionPooler == nil { + spec.ConnectionPooler = &acidv1.ConnectionPooler{} } - podTemplate, err := c.generateConnPoolPodTemplate(spec) - numberOfInstances := spec.ConnectionPool.NumberOfInstances + podTemplate, err := c.generateConnectionPoolerPodTemplate(spec) + numberOfInstances := spec.ConnectionPooler.NumberOfInstances if numberOfInstances == nil { numberOfInstances = util.CoalesceInt32( - c.OpConfig.ConnectionPool.NumberOfInstances, + c.OpConfig.ConnectionPooler.NumberOfInstances, k8sutil.Int32ToPointer(1)) } - if *numberOfInstances < constants.ConnPoolMinInstances { - msg := "Adjusted number of connection pool instances from %d to %d" - c.logger.Warningf(msg, numberOfInstances, constants.ConnPoolMinInstances) + if *numberOfInstances < constants.ConnectionPoolerMinInstances { + msg := "Adjusted number of connection pooler instances from %d to %d" + c.logger.Warningf(msg, numberOfInstances, constants.ConnectionPoolerMinInstances) - *numberOfInstances = constants.ConnPoolMinInstances + *numberOfInstances = constants.ConnectionPoolerMinInstances } if err != nil { @@ -2074,9 +2074,9 @@ func (c *Cluster) generateConnPoolDeployment(spec *acidv1.PostgresSpec) ( deployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: c.connPoolName(), + Name: c.connectionPoolerName(), Namespace: c.Namespace, - Labels: c.connPoolLabelsSelector().MatchLabels, + Labels: c.connectionPoolerLabelsSelector().MatchLabels, Annotations: map[string]string{}, // make StatefulSet object its owner to represent the dependency. // By itself StatefulSet is being deleted with "Orphaned" @@ -2088,7 +2088,7 @@ func (c *Cluster) generateConnPoolDeployment(spec *acidv1.PostgresSpec) ( }, Spec: appsv1.DeploymentSpec{ Replicas: numberOfInstances, - Selector: c.connPoolLabelsSelector(), + Selector: c.connectionPoolerLabelsSelector(), Template: *podTemplate, }, } @@ -2096,37 +2096,37 @@ func (c *Cluster) generateConnPoolDeployment(spec *acidv1.PostgresSpec) ( return deployment, nil } -func (c *Cluster) generateConnPoolService(spec *acidv1.PostgresSpec) *v1.Service { +func (c *Cluster) generateConnectionPoolerService(spec *acidv1.PostgresSpec) *v1.Service { // there are two ways to enable connection pooler, either to specify a - // connectionPool section or enableConnectionPool. In the second case - // spec.connectionPool will be nil, so to make it easier to calculate + // connectionPooler section or enableConnectionPooler. In the second case + // spec.connectionPooler will be nil, so to make it easier to calculate // default values, initialize it to an empty structure. It could be done // anywhere, but here is the earliest common entry point between sync and // create code, so init here. - if spec.ConnectionPool == nil { - spec.ConnectionPool = &acidv1.ConnectionPool{} + if spec.ConnectionPooler == nil { + spec.ConnectionPooler = &acidv1.ConnectionPooler{} } serviceSpec := v1.ServiceSpec{ Ports: []v1.ServicePort{ { - Name: c.connPoolName(), + Name: c.connectionPoolerName(), Port: pgPort, TargetPort: intstr.IntOrString{StrVal: c.servicePort(Master)}, }, }, Type: v1.ServiceTypeClusterIP, Selector: map[string]string{ - "connection-pool": c.connPoolName(), + "connection-pooler": c.connectionPoolerName(), }, } service := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: c.connPoolName(), + Name: c.connectionPoolerName(), Namespace: c.Namespace, - Labels: c.connPoolLabelsSelector().MatchLabels, + Labels: c.connectionPoolerLabelsSelector().MatchLabels, Annotations: map[string]string{}, // make StatefulSet object its owner to represent the dependency. // By itself StatefulSet is being deleted with "Orphaned" diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 5772f69d1..4068811c3 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -587,38 +587,38 @@ func TestSecretVolume(t *testing.T) { func testResources(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { cpuReq := podSpec.Spec.Containers[0].Resources.Requests["cpu"] - if cpuReq.String() != cluster.OpConfig.ConnectionPool.ConnPoolDefaultCPURequest { + if cpuReq.String() != cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultCPURequest { return fmt.Errorf("CPU request doesn't match, got %s, expected %s", - cpuReq.String(), cluster.OpConfig.ConnectionPool.ConnPoolDefaultCPURequest) + cpuReq.String(), cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultCPURequest) } memReq := podSpec.Spec.Containers[0].Resources.Requests["memory"] - if memReq.String() != cluster.OpConfig.ConnectionPool.ConnPoolDefaultMemoryRequest { + if memReq.String() != cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultMemoryRequest { return fmt.Errorf("Memory request doesn't match, got %s, expected %s", - memReq.String(), cluster.OpConfig.ConnectionPool.ConnPoolDefaultMemoryRequest) + memReq.String(), cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultMemoryRequest) } cpuLim := podSpec.Spec.Containers[0].Resources.Limits["cpu"] - if cpuLim.String() != cluster.OpConfig.ConnectionPool.ConnPoolDefaultCPULimit { + if cpuLim.String() != cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultCPULimit { return fmt.Errorf("CPU limit doesn't match, got %s, expected %s", - cpuLim.String(), cluster.OpConfig.ConnectionPool.ConnPoolDefaultCPULimit) + cpuLim.String(), cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultCPULimit) } memLim := podSpec.Spec.Containers[0].Resources.Limits["memory"] - if memLim.String() != cluster.OpConfig.ConnectionPool.ConnPoolDefaultMemoryLimit { + if memLim.String() != cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultMemoryLimit { return fmt.Errorf("Memory limit doesn't match, got %s, expected %s", - memLim.String(), cluster.OpConfig.ConnectionPool.ConnPoolDefaultMemoryLimit) + memLim.String(), cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultMemoryLimit) } return nil } func testLabels(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { - poolLabels := podSpec.ObjectMeta.Labels["connection-pool"] + poolerLabels := podSpec.ObjectMeta.Labels["connection-pooler"] - if poolLabels != cluster.connPoolLabelsSelector().MatchLabels["connection-pool"] { + if poolerLabels != cluster.connectionPoolerLabelsSelector().MatchLabels["connection-pooler"] { return fmt.Errorf("Pod labels do not match, got %+v, expected %+v", - podSpec.ObjectMeta.Labels, cluster.connPoolLabelsSelector().MatchLabels) + podSpec.ObjectMeta.Labels, cluster.connectionPoolerLabelsSelector().MatchLabels) } return nil @@ -626,13 +626,13 @@ func testLabels(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { func testEnvs(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { required := map[string]bool{ - "PGHOST": false, - "PGPORT": false, - "PGUSER": false, - "PGSCHEMA": false, - "PGPASSWORD": false, - "CONNECTION_POOL_MODE": false, - "CONNECTION_POOL_PORT": false, + "PGHOST": false, + "PGPORT": false, + "PGUSER": false, + "PGSCHEMA": false, + "PGPASSWORD": false, + "CONNECTION_POOLER_MODE": false, + "CONNECTION_POOLER_PORT": false, } envs := podSpec.Spec.Containers[0].Env @@ -658,8 +658,8 @@ func testCustomPodTemplate(cluster *Cluster, podSpec *v1.PodTemplateSpec) error return nil } -func TestConnPoolPodSpec(t *testing.T) { - testName := "Test connection pool pod template generation" +func TestConnectionPoolerPodSpec(t *testing.T) { + testName := "Test connection pooler pod template generation" var cluster = New( Config{ OpConfig: config.Config{ @@ -668,12 +668,12 @@ func TestConnPoolPodSpec(t *testing.T) { SuperUsername: superUserName, ReplicationUsername: replicationUserName, }, - ConnectionPool: config.ConnectionPool{ - MaxDBConnections: int32ToPointer(60), - ConnPoolDefaultCPURequest: "100m", - ConnPoolDefaultCPULimit: "100m", - ConnPoolDefaultMemoryRequest: "100Mi", - ConnPoolDefaultMemoryLimit: "100Mi", + ConnectionPooler: config.ConnectionPooler{ + MaxDBConnections: int32ToPointer(60), + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", }, }, }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) @@ -686,7 +686,7 @@ func TestConnPoolPodSpec(t *testing.T) { SuperUsername: superUserName, ReplicationUsername: replicationUserName, }, - ConnectionPool: config.ConnectionPool{}, + ConnectionPooler: config.ConnectionPooler{}, }, }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) @@ -702,7 +702,7 @@ func TestConnPoolPodSpec(t *testing.T) { { subTest: "default configuration", spec: &acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, expected: nil, cluster: cluster, @@ -711,7 +711,7 @@ func TestConnPoolPodSpec(t *testing.T) { { subTest: "no default resources", spec: &acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, expected: errors.New(`could not generate resource requirements: could not fill resource requests: could not parse default CPU quantity: quantities must match the regular expression '^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$'`), cluster: clusterNoDefaultRes, @@ -720,7 +720,7 @@ func TestConnPoolPodSpec(t *testing.T) { { subTest: "default resources are set", spec: &acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, expected: nil, cluster: cluster, @@ -729,7 +729,7 @@ func TestConnPoolPodSpec(t *testing.T) { { subTest: "labels for service", spec: &acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, expected: nil, cluster: cluster, @@ -738,7 +738,7 @@ func TestConnPoolPodSpec(t *testing.T) { { subTest: "required envs", spec: &acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, expected: nil, cluster: cluster, @@ -746,7 +746,7 @@ func TestConnPoolPodSpec(t *testing.T) { }, } for _, tt := range tests { - podSpec, err := tt.cluster.generateConnPoolPodTemplate(tt.spec) + podSpec, err := tt.cluster.generateConnectionPoolerPodTemplate(tt.spec) if err != tt.expected && err.Error() != tt.expected.Error() { t.Errorf("%s [%s]: Could not generate pod template,\n %+v, expected\n %+v", @@ -774,9 +774,9 @@ func testDeploymentOwnwerReference(cluster *Cluster, deployment *appsv1.Deployme func testSelector(cluster *Cluster, deployment *appsv1.Deployment) error { labels := deployment.Spec.Selector.MatchLabels - expected := cluster.connPoolLabelsSelector().MatchLabels + expected := cluster.connectionPoolerLabelsSelector().MatchLabels - if labels["connection-pool"] != expected["connection-pool"] { + if labels["connection-pooler"] != expected["connection-pooler"] { return fmt.Errorf("Labels are incorrect, got %+v, expected %+v", labels, expected) } @@ -784,8 +784,8 @@ func testSelector(cluster *Cluster, deployment *appsv1.Deployment) error { return nil } -func TestConnPoolDeploymentSpec(t *testing.T) { - testName := "Test connection pool deployment spec generation" +func TestConnectionPoolerDeploymentSpec(t *testing.T) { + testName := "Test connection pooler deployment spec generation" var cluster = New( Config{ OpConfig: config.Config{ @@ -794,11 +794,11 @@ func TestConnPoolDeploymentSpec(t *testing.T) { SuperUsername: superUserName, ReplicationUsername: replicationUserName, }, - ConnectionPool: config.ConnectionPool{ - ConnPoolDefaultCPURequest: "100m", - ConnPoolDefaultCPULimit: "100m", - ConnPoolDefaultMemoryRequest: "100Mi", - ConnPoolDefaultMemoryLimit: "100Mi", + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", }, }, }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) @@ -822,7 +822,7 @@ func TestConnPoolDeploymentSpec(t *testing.T) { { subTest: "default configuration", spec: &acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, expected: nil, cluster: cluster, @@ -831,7 +831,7 @@ func TestConnPoolDeploymentSpec(t *testing.T) { { subTest: "owner reference", spec: &acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, expected: nil, cluster: cluster, @@ -840,7 +840,7 @@ func TestConnPoolDeploymentSpec(t *testing.T) { { subTest: "selector", spec: &acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, expected: nil, cluster: cluster, @@ -848,7 +848,7 @@ func TestConnPoolDeploymentSpec(t *testing.T) { }, } for _, tt := range tests { - deployment, err := tt.cluster.generateConnPoolDeployment(tt.spec) + deployment, err := tt.cluster.generateConnectionPoolerDeployment(tt.spec) if err != tt.expected && err.Error() != tt.expected.Error() { t.Errorf("%s [%s]: Could not generate deployment spec,\n %+v, expected\n %+v", @@ -877,16 +877,16 @@ func testServiceOwnwerReference(cluster *Cluster, service *v1.Service) error { func testServiceSelector(cluster *Cluster, service *v1.Service) error { selector := service.Spec.Selector - if selector["connection-pool"] != cluster.connPoolName() { + if selector["connection-pooler"] != cluster.connectionPoolerName() { return fmt.Errorf("Selector is incorrect, got %s, expected %s", - selector["connection-pool"], cluster.connPoolName()) + selector["connection-pooler"], cluster.connectionPoolerName()) } return nil } -func TestConnPoolServiceSpec(t *testing.T) { - testName := "Test connection pool service spec generation" +func TestConnectionPoolerServiceSpec(t *testing.T) { + testName := "Test connection pooler service spec generation" var cluster = New( Config{ OpConfig: config.Config{ @@ -895,11 +895,11 @@ func TestConnPoolServiceSpec(t *testing.T) { SuperUsername: superUserName, ReplicationUsername: replicationUserName, }, - ConnectionPool: config.ConnectionPool{ - ConnPoolDefaultCPURequest: "100m", - ConnPoolDefaultCPULimit: "100m", - ConnPoolDefaultMemoryRequest: "100Mi", - ConnPoolDefaultMemoryLimit: "100Mi", + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", }, }, }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) @@ -922,7 +922,7 @@ func TestConnPoolServiceSpec(t *testing.T) { { subTest: "default configuration", spec: &acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, cluster: cluster, check: noCheck, @@ -930,7 +930,7 @@ func TestConnPoolServiceSpec(t *testing.T) { { subTest: "owner reference", spec: &acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, cluster: cluster, check: testServiceOwnwerReference, @@ -938,14 +938,14 @@ func TestConnPoolServiceSpec(t *testing.T) { { subTest: "selector", spec: &acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, cluster: cluster, check: testServiceSelector, }, } for _, tt := range tests { - service := tt.cluster.generateConnPoolService(tt.spec) + service := tt.cluster.generateConnectionPoolerService(tt.spec) if err := tt.check(cluster, service); err != nil { t.Errorf("%s [%s]: Service spec is incorrect, %+v", diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index c0c731ed8..4c341aefe 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -94,37 +94,37 @@ func (c *Cluster) createStatefulSet() (*appsv1.StatefulSet, error) { return statefulSet, nil } -// Prepare the database for connection pool to be used, i.e. install lookup +// Prepare the database for connection pooler to be used, i.e. install lookup // function (do it first, because it should be fast and if it didn't succeed, // it doesn't makes sense to create more K8S objects. At this moment we assume -// that necessary connection pool user exists. +// that necessary connection pooler user exists. // -// After that create all the objects for connection pool, namely a deployment +// After that create all the objects for connection pooler, namely a deployment // with a chosen pooler and a service to expose it. -func (c *Cluster) createConnectionPool(lookup InstallFunction) (*ConnectionPoolObjects, error) { +func (c *Cluster) createConnectionPooler(lookup InstallFunction) (*ConnectionPoolerObjects, error) { var msg string - c.setProcessName("creating connection pool") + c.setProcessName("creating connection pooler") - schema := c.Spec.ConnectionPool.Schema + schema := c.Spec.ConnectionPooler.Schema if schema == "" { - schema = c.OpConfig.ConnectionPool.Schema + schema = c.OpConfig.ConnectionPooler.Schema } - user := c.Spec.ConnectionPool.User + user := c.Spec.ConnectionPooler.User if user == "" { - user = c.OpConfig.ConnectionPool.User + user = c.OpConfig.ConnectionPooler.User } err := lookup(schema, user) if err != nil { - msg = "could not prepare database for connection pool: %v" + msg = "could not prepare database for connection pooler: %v" return nil, fmt.Errorf(msg, err) } - deploymentSpec, err := c.generateConnPoolDeployment(&c.Spec) + deploymentSpec, err := c.generateConnectionPoolerDeployment(&c.Spec) if err != nil { - msg = "could not generate deployment for connection pool: %v" + msg = "could not generate deployment for connection pooler: %v" return nil, fmt.Errorf(msg, err) } @@ -139,7 +139,7 @@ func (c *Cluster) createConnectionPool(lookup InstallFunction) (*ConnectionPoolO return nil, err } - serviceSpec := c.generateConnPoolService(&c.Spec) + serviceSpec := c.generateConnectionPoolerService(&c.Spec) service, err := c.KubeClient. Services(serviceSpec.Namespace). Create(context.TODO(), serviceSpec, metav1.CreateOptions{}) @@ -148,31 +148,31 @@ func (c *Cluster) createConnectionPool(lookup InstallFunction) (*ConnectionPoolO return nil, err } - c.ConnectionPool = &ConnectionPoolObjects{ + c.ConnectionPooler = &ConnectionPoolerObjects{ Deployment: deployment, Service: service, } - c.logger.Debugf("created new connection pool %q, uid: %q", + c.logger.Debugf("created new connection pooler %q, uid: %q", util.NameFromMeta(deployment.ObjectMeta), deployment.UID) - return c.ConnectionPool, nil + return c.ConnectionPooler, nil } -func (c *Cluster) deleteConnectionPool() (err error) { - c.setProcessName("deleting connection pool") - c.logger.Debugln("deleting connection pool") +func (c *Cluster) deleteConnectionPooler() (err error) { + c.setProcessName("deleting connection pooler") + c.logger.Debugln("deleting connection pooler") // Lack of connection pooler objects is not a fatal error, just log it if // it was present before in the manifest - if c.ConnectionPool == nil { - c.logger.Infof("No connection pool to delete") + if c.ConnectionPooler == nil { + c.logger.Infof("No connection pooler to delete") return nil } // Clean up the deployment object. If deployment resource we've remembered // is somehow empty, try to delete based on what would we generate - deploymentName := c.connPoolName() - deployment := c.ConnectionPool.Deployment + deploymentName := c.connectionPoolerName() + deployment := c.ConnectionPooler.Deployment if deployment != nil { deploymentName = deployment.Name @@ -187,16 +187,16 @@ func (c *Cluster) deleteConnectionPool() (err error) { Delete(context.TODO(), deploymentName, options) if !k8sutil.ResourceNotFound(err) { - c.logger.Debugf("Connection pool deployment was already deleted") + c.logger.Debugf("Connection pooler deployment was already deleted") } else if err != nil { return fmt.Errorf("could not delete deployment: %v", err) } - c.logger.Infof("Connection pool deployment %q has been deleted", deploymentName) + c.logger.Infof("Connection pooler deployment %q has been deleted", deploymentName) // Repeat the same for the service object - service := c.ConnectionPool.Service - serviceName := c.connPoolName() + service := c.ConnectionPooler.Service + serviceName := c.connectionPoolerName() if service != nil { serviceName = service.Name @@ -209,14 +209,14 @@ func (c *Cluster) deleteConnectionPool() (err error) { Delete(context.TODO(), serviceName, options) if !k8sutil.ResourceNotFound(err) { - c.logger.Debugf("Connection pool service was already deleted") + c.logger.Debugf("Connection pooler service was already deleted") } else if err != nil { return fmt.Errorf("could not delete service: %v", err) } - c.logger.Infof("Connection pool service %q has been deleted", serviceName) + c.logger.Infof("Connection pooler service %q has been deleted", serviceName) - c.ConnectionPool = nil + c.ConnectionPooler = nil return nil } @@ -816,12 +816,12 @@ func (c *Cluster) GetPodDisruptionBudget() *policybeta1.PodDisruptionBudget { return c.PodDisruptionBudget } -// Perform actual patching of a connection pool deployment, assuming that all +// Perform actual patching of a connection pooler deployment, assuming that all // the check were already done before. -func (c *Cluster) updateConnPoolDeployment(oldDeploymentSpec, newDeployment *appsv1.Deployment) (*appsv1.Deployment, error) { - c.setProcessName("updating connection pool") - if c.ConnectionPool == nil || c.ConnectionPool.Deployment == nil { - return nil, fmt.Errorf("there is no connection pool in the cluster") +func (c *Cluster) updateConnectionPoolerDeployment(oldDeploymentSpec, newDeployment *appsv1.Deployment) (*appsv1.Deployment, error) { + c.setProcessName("updating connection pooler") + if c.ConnectionPooler == nil || c.ConnectionPooler.Deployment == nil { + return nil, fmt.Errorf("there is no connection pooler in the cluster") } patchData, err := specPatch(newDeployment.Spec) @@ -833,9 +833,9 @@ func (c *Cluster) updateConnPoolDeployment(oldDeploymentSpec, newDeployment *app // worker at one time will try to update it chances of conflicts are // minimal. deployment, err := c.KubeClient. - Deployments(c.ConnectionPool.Deployment.Namespace).Patch( + Deployments(c.ConnectionPooler.Deployment.Namespace).Patch( context.TODO(), - c.ConnectionPool.Deployment.Name, + c.ConnectionPooler.Deployment.Name, types.MergePatchType, patchData, metav1.PatchOptions{}, @@ -844,7 +844,7 @@ func (c *Cluster) updateConnPoolDeployment(oldDeploymentSpec, newDeployment *app return nil, fmt.Errorf("could not patch deployment: %v", err) } - c.ConnectionPool.Deployment = deployment + c.ConnectionPooler.Deployment = deployment return deployment, nil } diff --git a/pkg/cluster/resources_test.go b/pkg/cluster/resources_test.go index f06e96e65..2db807b38 100644 --- a/pkg/cluster/resources_test.go +++ b/pkg/cluster/resources_test.go @@ -19,8 +19,8 @@ func boolToPointer(value bool) *bool { return &value } -func TestConnPoolCreationAndDeletion(t *testing.T) { - testName := "Test connection pool creation" +func TestConnectionPoolerCreationAndDeletion(t *testing.T) { + testName := "Test connection pooler creation" var cluster = New( Config{ OpConfig: config.Config{ @@ -29,11 +29,11 @@ func TestConnPoolCreationAndDeletion(t *testing.T) { SuperUsername: superUserName, ReplicationUsername: replicationUserName, }, - ConnectionPool: config.ConnectionPool{ - ConnPoolDefaultCPURequest: "100m", - ConnPoolDefaultCPULimit: "100m", - ConnPoolDefaultMemoryRequest: "100Mi", - ConnPoolDefaultMemoryLimit: "100Mi", + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", }, }, }, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger) @@ -45,31 +45,31 @@ func TestConnPoolCreationAndDeletion(t *testing.T) { } cluster.Spec = acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, } - poolResources, err := cluster.createConnectionPool(mockInstallLookupFunction) + poolerResources, err := cluster.createConnectionPooler(mockInstallLookupFunction) if err != nil { - t.Errorf("%s: Cannot create connection pool, %s, %+v", - testName, err, poolResources) + t.Errorf("%s: Cannot create connection pooler, %s, %+v", + testName, err, poolerResources) } - if poolResources.Deployment == nil { - t.Errorf("%s: Connection pool deployment is empty", testName) + if poolerResources.Deployment == nil { + t.Errorf("%s: Connection pooler deployment is empty", testName) } - if poolResources.Service == nil { - t.Errorf("%s: Connection pool service is empty", testName) + if poolerResources.Service == nil { + t.Errorf("%s: Connection pooler service is empty", testName) } - err = cluster.deleteConnectionPool() + err = cluster.deleteConnectionPooler() if err != nil { - t.Errorf("%s: Cannot delete connection pool, %s", testName, err) + t.Errorf("%s: Cannot delete connection pooler, %s", testName, err) } } -func TestNeedConnPool(t *testing.T) { - testName := "Test how connection pool can be enabled" +func TestNeedConnectionPooler(t *testing.T) { + testName := "Test how connection pooler can be enabled" var cluster = New( Config{ OpConfig: config.Config{ @@ -78,50 +78,50 @@ func TestNeedConnPool(t *testing.T) { SuperUsername: superUserName, ReplicationUsername: replicationUserName, }, - ConnectionPool: config.ConnectionPool{ - ConnPoolDefaultCPURequest: "100m", - ConnPoolDefaultCPULimit: "100m", - ConnPoolDefaultMemoryRequest: "100Mi", - ConnPoolDefaultMemoryLimit: "100Mi", + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", }, }, }, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger) cluster.Spec = acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, } - if !cluster.needConnectionPool() { - t.Errorf("%s: Connection pool is not enabled with full definition", + if !cluster.needConnectionPooler() { + t.Errorf("%s: Connection pooler is not enabled with full definition", testName) } cluster.Spec = acidv1.PostgresSpec{ - EnableConnectionPool: boolToPointer(true), + EnableConnectionPooler: boolToPointer(true), } - if !cluster.needConnectionPool() { - t.Errorf("%s: Connection pool is not enabled with flag", + if !cluster.needConnectionPooler() { + t.Errorf("%s: Connection pooler is not enabled with flag", testName) } cluster.Spec = acidv1.PostgresSpec{ - EnableConnectionPool: boolToPointer(false), - ConnectionPool: &acidv1.ConnectionPool{}, + EnableConnectionPooler: boolToPointer(false), + ConnectionPooler: &acidv1.ConnectionPooler{}, } - if cluster.needConnectionPool() { - t.Errorf("%s: Connection pool is still enabled with flag being false", + if cluster.needConnectionPooler() { + t.Errorf("%s: Connection pooler is still enabled with flag being false", testName) } cluster.Spec = acidv1.PostgresSpec{ - EnableConnectionPool: boolToPointer(true), - ConnectionPool: &acidv1.ConnectionPool{}, + EnableConnectionPooler: boolToPointer(true), + ConnectionPooler: &acidv1.ConnectionPooler{}, } - if !cluster.needConnectionPool() { - t.Errorf("%s: Connection pool is not enabled with flag and full", + if !cluster.needConnectionPooler() { + t.Errorf("%s: Connection pooler is not enabled with flag and full", testName) } } diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index eb3835787..a423c2f0e 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -110,9 +110,9 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { } } - // sync connection pool - if err = c.syncConnectionPool(&oldSpec, newSpec, c.installLookupFunction); err != nil { - return fmt.Errorf("could not sync connection pool: %v", err) + // sync connection pooler + if err = c.syncConnectionPooler(&oldSpec, newSpec, c.installLookupFunction); err != nil { + return fmt.Errorf("could not sync connection pooler: %v", err) } return err @@ -478,12 +478,12 @@ func (c *Cluster) syncRoles() (err error) { userNames = append(userNames, u.Name) } - if c.needConnectionPool() { - connPoolUser := c.systemUsers[constants.ConnectionPoolUserKeyName] - userNames = append(userNames, connPoolUser.Name) + if c.needConnectionPooler() { + connectionPoolerUser := c.systemUsers[constants.ConnectionPoolerUserKeyName] + userNames = append(userNames, connectionPoolerUser.Name) - if _, exists := c.pgUsers[connPoolUser.Name]; !exists { - c.pgUsers[connPoolUser.Name] = connPoolUser + if _, exists := c.pgUsers[connectionPoolerUser.Name]; !exists { + c.pgUsers[connectionPoolerUser.Name] = connectionPoolerUser } } @@ -620,69 +620,69 @@ func (c *Cluster) syncLogicalBackupJob() error { return nil } -func (c *Cluster) syncConnectionPool(oldSpec, newSpec *acidv1.Postgresql, lookup InstallFunction) error { - if c.ConnectionPool == nil { - c.ConnectionPool = &ConnectionPoolObjects{} +func (c *Cluster) syncConnectionPooler(oldSpec, newSpec *acidv1.Postgresql, lookup InstallFunction) error { + if c.ConnectionPooler == nil { + c.ConnectionPooler = &ConnectionPoolerObjects{} } - newNeedConnPool := c.needConnectionPoolWorker(&newSpec.Spec) - oldNeedConnPool := c.needConnectionPoolWorker(&oldSpec.Spec) + newNeedConnectionPooler := c.needConnectionPoolerWorker(&newSpec.Spec) + oldNeedConnectionPooler := c.needConnectionPoolerWorker(&oldSpec.Spec) - if newNeedConnPool { - // Try to sync in any case. If we didn't needed connection pool before, + if newNeedConnectionPooler { + // Try to sync in any case. If we didn't needed connection pooler before, // it means we want to create it. If it was already present, still sync // since it could happen that there is no difference in specs, and all - // the resources are remembered, but the deployment was manualy deleted + // the resources are remembered, but the deployment was manually deleted // in between - c.logger.Debug("syncing connection pool") + c.logger.Debug("syncing connection pooler") // in this case also do not forget to install lookup function as for // creating cluster - if !oldNeedConnPool || !c.ConnectionPool.LookupFunction { - newConnPool := newSpec.Spec.ConnectionPool + if !oldNeedConnectionPooler || !c.ConnectionPooler.LookupFunction { + newConnectionPooler := newSpec.Spec.ConnectionPooler specSchema := "" specUser := "" - if newConnPool != nil { - specSchema = newConnPool.Schema - specUser = newConnPool.User + if newConnectionPooler != nil { + specSchema = newConnectionPooler.Schema + specUser = newConnectionPooler.User } schema := util.Coalesce( specSchema, - c.OpConfig.ConnectionPool.Schema) + c.OpConfig.ConnectionPooler.Schema) user := util.Coalesce( specUser, - c.OpConfig.ConnectionPool.User) + c.OpConfig.ConnectionPooler.User) if err := lookup(schema, user); err != nil { return err } } - if err := c.syncConnectionPoolWorker(oldSpec, newSpec); err != nil { - c.logger.Errorf("could not sync connection pool: %v", err) + if err := c.syncConnectionPoolerWorker(oldSpec, newSpec); err != nil { + c.logger.Errorf("could not sync connection pooler: %v", err) return err } } - if oldNeedConnPool && !newNeedConnPool { + if oldNeedConnectionPooler && !newNeedConnectionPooler { // delete and cleanup resources - if err := c.deleteConnectionPool(); err != nil { - c.logger.Warningf("could not remove connection pool: %v", err) + if err := c.deleteConnectionPooler(); err != nil { + c.logger.Warningf("could not remove connection pooler: %v", err) } } - if !oldNeedConnPool && !newNeedConnPool { + if !oldNeedConnectionPooler && !newNeedConnectionPooler { // delete and cleanup resources if not empty - if c.ConnectionPool != nil && - (c.ConnectionPool.Deployment != nil || - c.ConnectionPool.Service != nil) { + if c.ConnectionPooler != nil && + (c.ConnectionPooler.Deployment != nil || + c.ConnectionPooler.Service != nil) { - if err := c.deleteConnectionPool(); err != nil { - c.logger.Warningf("could not remove connection pool: %v", err) + if err := c.deleteConnectionPooler(); err != nil { + c.logger.Warningf("could not remove connection pooler: %v", err) } } } @@ -690,22 +690,22 @@ func (c *Cluster) syncConnectionPool(oldSpec, newSpec *acidv1.Postgresql, lookup return nil } -// Synchronize connection pool resources. Effectively we're interested only in +// Synchronize connection pooler resources. Effectively we're interested only in // synchronizing the corresponding deployment, but in case of deployment or // service is missing, create it. After checking, also remember an object for // the future references. -func (c *Cluster) syncConnectionPoolWorker(oldSpec, newSpec *acidv1.Postgresql) error { +func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql) error { deployment, err := c.KubeClient. Deployments(c.Namespace). - Get(context.TODO(), c.connPoolName(), metav1.GetOptions{}) + Get(context.TODO(), c.connectionPoolerName(), metav1.GetOptions{}) if err != nil && k8sutil.ResourceNotFound(err) { - msg := "Deployment %s for connection pool synchronization is not found, create it" - c.logger.Warningf(msg, c.connPoolName()) + msg := "Deployment %s for connection pooler synchronization is not found, create it" + c.logger.Warningf(msg, c.connectionPoolerName()) - deploymentSpec, err := c.generateConnPoolDeployment(&newSpec.Spec) + deploymentSpec, err := c.generateConnectionPoolerDeployment(&newSpec.Spec) if err != nil { - msg = "could not generate deployment for connection pool: %v" + msg = "could not generate deployment for connection pooler: %v" return fmt.Errorf(msg, err) } @@ -717,31 +717,31 @@ func (c *Cluster) syncConnectionPoolWorker(oldSpec, newSpec *acidv1.Postgresql) return err } - c.ConnectionPool.Deployment = deployment + c.ConnectionPooler.Deployment = deployment } else if err != nil { - return fmt.Errorf("could not get connection pool deployment to sync: %v", err) + return fmt.Errorf("could not get connection pooler deployment to sync: %v", err) } else { - c.ConnectionPool.Deployment = deployment + c.ConnectionPooler.Deployment = deployment // actual synchronization - oldConnPool := oldSpec.Spec.ConnectionPool - newConnPool := newSpec.Spec.ConnectionPool - specSync, specReason := c.needSyncConnPoolSpecs(oldConnPool, newConnPool) - defaultsSync, defaultsReason := c.needSyncConnPoolDefaults(newConnPool, deployment) + oldConnectionPooler := oldSpec.Spec.ConnectionPooler + newConnectionPooler := newSpec.Spec.ConnectionPooler + specSync, specReason := c.needSyncConnectionPoolerSpecs(oldConnectionPooler, newConnectionPooler) + defaultsSync, defaultsReason := c.needSyncConnectionPoolerDefaults(newConnectionPooler, deployment) reason := append(specReason, defaultsReason...) if specSync || defaultsSync { - c.logger.Infof("Update connection pool deployment %s, reason: %+v", - c.connPoolName(), reason) + c.logger.Infof("Update connection pooler deployment %s, reason: %+v", + c.connectionPoolerName(), reason) - newDeploymentSpec, err := c.generateConnPoolDeployment(&newSpec.Spec) + newDeploymentSpec, err := c.generateConnectionPoolerDeployment(&newSpec.Spec) if err != nil { - msg := "could not generate deployment for connection pool: %v" + msg := "could not generate deployment for connection pooler: %v" return fmt.Errorf(msg, err) } - oldDeploymentSpec := c.ConnectionPool.Deployment + oldDeploymentSpec := c.ConnectionPooler.Deployment - deployment, err := c.updateConnPoolDeployment( + deployment, err := c.updateConnectionPoolerDeployment( oldDeploymentSpec, newDeploymentSpec) @@ -749,20 +749,20 @@ func (c *Cluster) syncConnectionPoolWorker(oldSpec, newSpec *acidv1.Postgresql) return err } - c.ConnectionPool.Deployment = deployment + c.ConnectionPooler.Deployment = deployment return nil } } service, err := c.KubeClient. Services(c.Namespace). - Get(context.TODO(), c.connPoolName(), metav1.GetOptions{}) + Get(context.TODO(), c.connectionPoolerName(), metav1.GetOptions{}) if err != nil && k8sutil.ResourceNotFound(err) { - msg := "Service %s for connection pool synchronization is not found, create it" - c.logger.Warningf(msg, c.connPoolName()) + msg := "Service %s for connection pooler synchronization is not found, create it" + c.logger.Warningf(msg, c.connectionPoolerName()) - serviceSpec := c.generateConnPoolService(&newSpec.Spec) + serviceSpec := c.generateConnectionPoolerService(&newSpec.Spec) service, err := c.KubeClient. Services(serviceSpec.Namespace). Create(context.TODO(), serviceSpec, metav1.CreateOptions{}) @@ -771,12 +771,12 @@ func (c *Cluster) syncConnectionPoolWorker(oldSpec, newSpec *acidv1.Postgresql) return err } - c.ConnectionPool.Service = service + c.ConnectionPooler.Service = service } else if err != nil { - return fmt.Errorf("could not get connection pool service to sync: %v", err) + return fmt.Errorf("could not get connection pooler service to sync: %v", err) } else { // Service updates are not supported and probably not that useful anyway - c.ConnectionPool.Service = service + c.ConnectionPooler.Service = service } return nil diff --git a/pkg/cluster/sync_test.go b/pkg/cluster/sync_test.go index 483d4ba58..45355cca3 100644 --- a/pkg/cluster/sync_test.go +++ b/pkg/cluster/sync_test.go @@ -18,8 +18,8 @@ func int32ToPointer(value int32) *int32 { } func deploymentUpdated(cluster *Cluster, err error) error { - if cluster.ConnectionPool.Deployment.Spec.Replicas == nil || - *cluster.ConnectionPool.Deployment.Spec.Replicas != 2 { + if cluster.ConnectionPooler.Deployment.Spec.Replicas == nil || + *cluster.ConnectionPooler.Deployment.Spec.Replicas != 2 { return fmt.Errorf("Wrong nubmer of instances") } @@ -27,15 +27,15 @@ func deploymentUpdated(cluster *Cluster, err error) error { } func objectsAreSaved(cluster *Cluster, err error) error { - if cluster.ConnectionPool == nil { - return fmt.Errorf("Connection pool resources are empty") + if cluster.ConnectionPooler == nil { + return fmt.Errorf("Connection pooler resources are empty") } - if cluster.ConnectionPool.Deployment == nil { + if cluster.ConnectionPooler.Deployment == nil { return fmt.Errorf("Deployment was not saved") } - if cluster.ConnectionPool.Service == nil { + if cluster.ConnectionPooler.Service == nil { return fmt.Errorf("Service was not saved") } @@ -43,15 +43,15 @@ func objectsAreSaved(cluster *Cluster, err error) error { } func objectsAreDeleted(cluster *Cluster, err error) error { - if cluster.ConnectionPool != nil { - return fmt.Errorf("Connection pool was not deleted") + if cluster.ConnectionPooler != nil { + return fmt.Errorf("Connection pooler was not deleted") } return nil } -func TestConnPoolSynchronization(t *testing.T) { - testName := "Test connection pool synchronization" +func TestConnectionPoolerSynchronization(t *testing.T) { + testName := "Test connection pooler synchronization" var cluster = New( Config{ OpConfig: config.Config{ @@ -60,12 +60,12 @@ func TestConnPoolSynchronization(t *testing.T) { SuperUsername: superUserName, ReplicationUsername: replicationUserName, }, - ConnectionPool: config.ConnectionPool{ - ConnPoolDefaultCPURequest: "100m", - ConnPoolDefaultCPULimit: "100m", - ConnPoolDefaultMemoryRequest: "100Mi", - ConnPoolDefaultMemoryLimit: "100Mi", - NumberOfInstances: int32ToPointer(1), + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", + NumberOfInstances: int32ToPointer(1), }, }, }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) @@ -84,15 +84,15 @@ func TestConnPoolSynchronization(t *testing.T) { clusterDirtyMock := *cluster clusterDirtyMock.KubeClient = k8sutil.NewMockKubernetesClient() - clusterDirtyMock.ConnectionPool = &ConnectionPoolObjects{ + clusterDirtyMock.ConnectionPooler = &ConnectionPoolerObjects{ Deployment: &appsv1.Deployment{}, Service: &v1.Service{}, } clusterNewDefaultsMock := *cluster clusterNewDefaultsMock.KubeClient = k8sutil.NewMockKubernetesClient() - cluster.OpConfig.ConnectionPool.Image = "pooler:2.0" - cluster.OpConfig.ConnectionPool.NumberOfInstances = int32ToPointer(2) + cluster.OpConfig.ConnectionPooler.Image = "pooler:2.0" + cluster.OpConfig.ConnectionPooler.NumberOfInstances = int32ToPointer(2) tests := []struct { subTest string @@ -105,12 +105,12 @@ func TestConnPoolSynchronization(t *testing.T) { subTest: "create if doesn't exist", oldSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, newSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, cluster: &clusterMissingObjects, @@ -123,7 +123,7 @@ func TestConnPoolSynchronization(t *testing.T) { }, newSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{ - EnableConnectionPool: boolToPointer(true), + EnableConnectionPooler: boolToPointer(true), }, }, cluster: &clusterMissingObjects, @@ -136,7 +136,7 @@ func TestConnPoolSynchronization(t *testing.T) { }, newSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, cluster: &clusterMissingObjects, @@ -146,7 +146,7 @@ func TestConnPoolSynchronization(t *testing.T) { subTest: "delete if not needed", oldSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, newSpec: &acidv1.Postgresql{ @@ -170,14 +170,14 @@ func TestConnPoolSynchronization(t *testing.T) { subTest: "update deployment", oldSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{ + ConnectionPooler: &acidv1.ConnectionPooler{ NumberOfInstances: int32ToPointer(1), }, }, }, newSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{ + ConnectionPooler: &acidv1.ConnectionPooler{ NumberOfInstances: int32ToPointer(2), }, }, @@ -189,12 +189,12 @@ func TestConnPoolSynchronization(t *testing.T) { subTest: "update image from changed defaults", oldSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, newSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, cluster: &clusterNewDefaultsMock, @@ -202,7 +202,7 @@ func TestConnPoolSynchronization(t *testing.T) { }, } for _, tt := range tests { - err := tt.cluster.syncConnectionPool(tt.oldSpec, tt.newSpec, mockInstallLookupFunction) + err := tt.cluster.syncConnectionPooler(tt.oldSpec, tt.newSpec, mockInstallLookupFunction) if err := tt.check(tt.cluster, err); err != nil { t.Errorf("%s [%s]: Could not synchronize, %+v", diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go index 3c3dffcaf..405f48f9f 100644 --- a/pkg/cluster/util.go +++ b/pkg/cluster/util.go @@ -415,24 +415,24 @@ func (c *Cluster) labelsSelector() *metav1.LabelSelector { } } -// Return connection pool labels selector, which should from one point of view +// Return connection pooler labels selector, which should from one point of view // inherit most of the labels from the cluster itself, but at the same time // have e.g. different `application` label, so that recreatePod operation will // not interfere with it (it lists all the pods via labels, and if there would // be no difference, it will recreate also pooler pods). -func (c *Cluster) connPoolLabelsSelector() *metav1.LabelSelector { - connPoolLabels := labels.Set(map[string]string{}) +func (c *Cluster) connectionPoolerLabelsSelector() *metav1.LabelSelector { + connectionPoolerLabels := labels.Set(map[string]string{}) extraLabels := labels.Set(map[string]string{ - "connection-pool": c.connPoolName(), - "application": "db-connection-pool", + "connection-pooler": c.connectionPoolerName(), + "application": "db-connection-pooler", }) - connPoolLabels = labels.Merge(connPoolLabels, c.labelsSet(false)) - connPoolLabels = labels.Merge(connPoolLabels, extraLabels) + connectionPoolerLabels = labels.Merge(connectionPoolerLabels, c.labelsSet(false)) + connectionPoolerLabels = labels.Merge(connectionPoolerLabels, extraLabels) return &metav1.LabelSelector{ - MatchLabels: connPoolLabels, + MatchLabels: connectionPoolerLabels, MatchExpressions: nil, } } @@ -510,14 +510,14 @@ func (c *Cluster) patroniUsesKubernetes() bool { return c.OpConfig.EtcdHost == "" } -func (c *Cluster) needConnectionPoolWorker(spec *acidv1.PostgresSpec) bool { - if spec.EnableConnectionPool == nil { - return spec.ConnectionPool != nil +func (c *Cluster) needConnectionPoolerWorker(spec *acidv1.PostgresSpec) bool { + if spec.EnableConnectionPooler == nil { + return spec.ConnectionPooler != nil } else { - return *spec.EnableConnectionPool + return *spec.EnableConnectionPooler } } -func (c *Cluster) needConnectionPool() bool { - return c.needConnectionPoolWorker(&c.Spec) +func (c *Cluster) needConnectionPooler() bool { + return c.needConnectionPoolerWorker(&c.Spec) } diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index c9b3c5ea4..e9fe2f379 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -150,51 +150,51 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.ScalyrCPULimit = fromCRD.Scalyr.ScalyrCPULimit result.ScalyrMemoryLimit = fromCRD.Scalyr.ScalyrMemoryLimit - // Connection pool. Looks like we can't use defaulting in CRD before 1.17, + // Connection pooler. Looks like we can't use defaulting in CRD before 1.17, // so ensure default values here. - result.ConnectionPool.NumberOfInstances = util.CoalesceInt32( - fromCRD.ConnectionPool.NumberOfInstances, + result.ConnectionPooler.NumberOfInstances = util.CoalesceInt32( + fromCRD.ConnectionPooler.NumberOfInstances, int32ToPointer(2)) - result.ConnectionPool.NumberOfInstances = util.MaxInt32( - result.ConnectionPool.NumberOfInstances, + result.ConnectionPooler.NumberOfInstances = util.MaxInt32( + result.ConnectionPooler.NumberOfInstances, int32ToPointer(2)) - result.ConnectionPool.Schema = util.Coalesce( - fromCRD.ConnectionPool.Schema, - constants.ConnectionPoolSchemaName) + result.ConnectionPooler.Schema = util.Coalesce( + fromCRD.ConnectionPooler.Schema, + constants.ConnectionPoolerSchemaName) - result.ConnectionPool.User = util.Coalesce( - fromCRD.ConnectionPool.User, - constants.ConnectionPoolUserName) + result.ConnectionPooler.User = util.Coalesce( + fromCRD.ConnectionPooler.User, + constants.ConnectionPoolerUserName) - result.ConnectionPool.Image = util.Coalesce( - fromCRD.ConnectionPool.Image, + result.ConnectionPooler.Image = util.Coalesce( + fromCRD.ConnectionPooler.Image, "registry.opensource.zalan.do/acid/pgbouncer") - result.ConnectionPool.Mode = util.Coalesce( - fromCRD.ConnectionPool.Mode, - constants.ConnectionPoolDefaultMode) + result.ConnectionPooler.Mode = util.Coalesce( + fromCRD.ConnectionPooler.Mode, + constants.ConnectionPoolerDefaultMode) - result.ConnectionPool.ConnPoolDefaultCPURequest = util.Coalesce( - fromCRD.ConnectionPool.DefaultCPURequest, - constants.ConnectionPoolDefaultCpuRequest) + result.ConnectionPooler.ConnectionPoolerDefaultCPURequest = util.Coalesce( + fromCRD.ConnectionPooler.DefaultCPURequest, + constants.ConnectionPoolerDefaultCpuRequest) - result.ConnectionPool.ConnPoolDefaultMemoryRequest = util.Coalesce( - fromCRD.ConnectionPool.DefaultMemoryRequest, - constants.ConnectionPoolDefaultMemoryRequest) + result.ConnectionPooler.ConnectionPoolerDefaultMemoryRequest = util.Coalesce( + fromCRD.ConnectionPooler.DefaultMemoryRequest, + constants.ConnectionPoolerDefaultMemoryRequest) - result.ConnectionPool.ConnPoolDefaultCPULimit = util.Coalesce( - fromCRD.ConnectionPool.DefaultCPULimit, - constants.ConnectionPoolDefaultCpuLimit) + result.ConnectionPooler.ConnectionPoolerDefaultCPULimit = util.Coalesce( + fromCRD.ConnectionPooler.DefaultCPULimit, + constants.ConnectionPoolerDefaultCpuLimit) - result.ConnectionPool.ConnPoolDefaultMemoryLimit = util.Coalesce( - fromCRD.ConnectionPool.DefaultMemoryLimit, - constants.ConnectionPoolDefaultMemoryLimit) + result.ConnectionPooler.ConnectionPoolerDefaultMemoryLimit = util.Coalesce( + fromCRD.ConnectionPooler.DefaultMemoryLimit, + constants.ConnectionPoolerDefaultMemoryLimit) - result.ConnectionPool.MaxDBConnections = util.CoalesceInt32( - fromCRD.ConnectionPool.MaxDBConnections, - int32ToPointer(constants.ConnPoolMaxDBConnections)) + result.ConnectionPooler.MaxDBConnections = util.CoalesceInt32( + fromCRD.ConnectionPooler.MaxDBConnections, + int32ToPointer(constants.ConnectionPoolerMaxDBConnections)) return result } diff --git a/pkg/spec/types.go b/pkg/spec/types.go index 36783204d..e1c49a1fd 100644 --- a/pkg/spec/types.go +++ b/pkg/spec/types.go @@ -31,7 +31,7 @@ const ( RoleOriginInfrastructure RoleOriginTeamsAPI RoleOriginSystem - RoleConnectionPool + RoleConnectionPooler ) type syncUserOperation int @@ -180,8 +180,8 @@ func (r RoleOrigin) String() string { return "teams API role" case RoleOriginSystem: return "system role" - case RoleConnectionPool: - return "connection pool role" + case RoleConnectionPooler: + return "connection pooler role" default: panic(fmt.Sprintf("bogus role origin value %d", r)) } diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 403615f06..d5464ca7c 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -85,17 +85,17 @@ type LogicalBackup struct { } // Operator options for connection pooler -type ConnectionPool struct { - NumberOfInstances *int32 `name:"connection_pool_number_of_instances" default:"2"` - Schema string `name:"connection_pool_schema" default:"pooler"` - User string `name:"connection_pool_user" default:"pooler"` - Image string `name:"connection_pool_image" default:"registry.opensource.zalan.do/acid/pgbouncer"` - Mode string `name:"connection_pool_mode" default:"transaction"` - MaxDBConnections *int32 `name:"connection_pool_max_db_connections" default:"60"` - ConnPoolDefaultCPURequest string `name:"connection_pool_default_cpu_request" default:"500m"` - ConnPoolDefaultMemoryRequest string `name:"connection_pool_default_memory_request" default:"100Mi"` - ConnPoolDefaultCPULimit string `name:"connection_pool_default_cpu_limit" default:"1"` - ConnPoolDefaultMemoryLimit string `name:"connection_pool_default_memory_limit" default:"100Mi"` +type ConnectionPooler struct { + NumberOfInstances *int32 `name:"connection_pooler_number_of_instances" default:"2"` + Schema string `name:"connection_pooler_schema" default:"pooler"` + User string `name:"connection_pooler_user" default:"pooler"` + Image string `name:"connection_pooler_image" default:"registry.opensource.zalan.do/acid/pgbouncer"` + Mode string `name:"connection_pooler_mode" default:"transaction"` + MaxDBConnections *int32 `name:"connection_pooler_max_db_connections" default:"60"` + ConnectionPoolerDefaultCPURequest string `name:"connection_pooler_default_cpu_request" default:"500m"` + ConnectionPoolerDefaultMemoryRequest string `name:"connection_pooler_default_memory_request" default:"100Mi"` + ConnectionPoolerDefaultCPULimit string `name:"connection_pooler_default_cpu_limit" default:"1"` + ConnectionPoolerDefaultMemoryLimit string `name:"connection_pooler_default_memory_limit" default:"100Mi"` } // Config describes operator config @@ -105,7 +105,7 @@ type Config struct { Auth Scalyr LogicalBackup - ConnectionPool + ConnectionPooler WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to' EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS @@ -213,9 +213,9 @@ func validate(cfg *Config) (err error) { err = fmt.Errorf("number of workers should be higher than 0") } - if *cfg.ConnectionPool.NumberOfInstances < constants.ConnPoolMinInstances { - msg := "number of connection pool instances should be higher than %d" - err = fmt.Errorf(msg, constants.ConnPoolMinInstances) + if *cfg.ConnectionPooler.NumberOfInstances < constants.ConnectionPoolerMinInstances { + msg := "number of connection pooler instances should be higher than %d" + err = fmt.Errorf(msg, constants.ConnectionPoolerMinInstances) } return } diff --git a/pkg/util/constants/pooler.go b/pkg/util/constants/pooler.go index 540d64e2c..52e47c9cd 100644 --- a/pkg/util/constants/pooler.go +++ b/pkg/util/constants/pooler.go @@ -1,18 +1,18 @@ package constants -// Connection pool specific constants +// Connection pooler specific constants const ( - ConnectionPoolUserName = "pooler" - ConnectionPoolSchemaName = "pooler" - ConnectionPoolDefaultType = "pgbouncer" - ConnectionPoolDefaultMode = "transaction" - ConnectionPoolDefaultCpuRequest = "500m" - ConnectionPoolDefaultCpuLimit = "1" - ConnectionPoolDefaultMemoryRequest = "100Mi" - ConnectionPoolDefaultMemoryLimit = "100Mi" + ConnectionPoolerUserName = "pooler" + ConnectionPoolerSchemaName = "pooler" + ConnectionPoolerDefaultType = "pgbouncer" + ConnectionPoolerDefaultMode = "transaction" + ConnectionPoolerDefaultCpuRequest = "500m" + ConnectionPoolerDefaultCpuLimit = "1" + ConnectionPoolerDefaultMemoryRequest = "100Mi" + ConnectionPoolerDefaultMemoryLimit = "100Mi" - ConnPoolContainer = 0 - ConnPoolMaxDBConnections = 60 - ConnPoolMaxClientConnections = 10000 - ConnPoolMinInstances = 2 + ConnectionPoolerContainer = 0 + ConnectionPoolerMaxDBConnections = 60 + ConnectionPoolerMaxClientConnections = 10000 + ConnectionPoolerMinInstances = 2 ) diff --git a/pkg/util/constants/roles.go b/pkg/util/constants/roles.go index 3d201142c..c2c287472 100644 --- a/pkg/util/constants/roles.go +++ b/pkg/util/constants/roles.go @@ -4,7 +4,7 @@ package constants const ( PasswordLength = 64 SuperuserKeyName = "superuser" - ConnectionPoolUserKeyName = "pooler" + ConnectionPoolerUserKeyName = "pooler" ReplicationUserKeyName = "replication" RoleFlagSuperuser = "SUPERUSER" RoleFlagInherit = "INHERIT" From 1249626a6023d9ec9c4e52f4d47971b860a1e91e Mon Sep 17 00:00:00 2001 From: ReSearchITEng Date: Thu, 2 Apr 2020 14:20:45 +0300 Subject: [PATCH 021/168] kubernetes_use_configmap (#887) * kubernetes_use_configmap * Update manifests/postgresql-operator-default-configuration.yaml Co-Authored-By: Felix Kunde * Update manifests/configmap.yaml Co-Authored-By: Felix Kunde * Update charts/postgres-operator/values.yaml Co-Authored-By: Felix Kunde * go.fmt Co-authored-by: Felix Kunde --- .../crds/operatorconfigurations.yaml | 2 ++ charts/postgres-operator/values-crd.yaml | 2 ++ charts/postgres-operator/values.yaml | 2 ++ docs/reference/operator_parameters.md | 6 ++++++ manifests/configmap.yaml | 1 + manifests/operatorconfiguration.crd.yaml | 2 ++ .../postgresql-operator-default-configuration.yaml | 1 + pkg/apis/acid.zalan.do/v1/crds.go | 3 +++ .../acid.zalan.do/v1/operator_configuration_type.go | 1 + pkg/cluster/k8sres.go | 6 +++++- pkg/cluster/util.go | 9 +++++++++ pkg/controller/operator_config.go | 1 + pkg/util/config/config.go | 11 ++++++----- 13 files changed, 41 insertions(+), 6 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 63e52fd7a..7e20c8fea 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -66,6 +66,8 @@ spec: type: boolean etcd_host: type: string + kubernetes_use_configmaps: + type: boolean max_instances: type: integer minimum: -1 # -1 = disabled diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 2490cfec6..37ad6e45e 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -23,6 +23,8 @@ configGeneral: enable_shm_volume: true # etcd connection string for Patroni. Empty uses K8s-native DCS. etcd_host: "" + # Select if setup uses endpoints (default), or configmaps to manage leader (DCS=k8s) + # kubernetes_use_configmaps: false # Spilo docker image docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p2 # max number of instances in Postgres cluster. -1 = no limit diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 8d2dd8c5c..3a701bb64 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -23,6 +23,8 @@ configGeneral: enable_shm_volume: "true" # etcd connection string for Patroni. Empty uses K8s-native DCS. etcd_host: "" + # Select if setup uses endpoints (default), or configmaps to manage leader (DCS=k8s) + # kubernetes_use_configmaps: "false" # Spilo docker image docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p2 # max number of instances in Postgres cluster. -1 = no limit diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index c25ca3d49..9f0e0b67a 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -80,6 +80,12 @@ Those are top-level keys, containing both leaf keys and groups. Patroni native Kubernetes support is used. The default is empty (use Kubernetes-native DCS). +* **kubernetes_use_configmaps** + Select if setup uses endpoints (default), or configmaps to manage leader when + DCS is kubernetes (not etcd or similar). In OpenShift it is not possible to + use endpoints option, and configmaps is required. By default, + `kubernetes_use_configmaps: false`, meaning endpoints will be used. + * **docker_image** Spilo Docker image for Postgres instances. For production, don't rely on the default image, as it might be not the most up-to-date one. Instead, build diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index c74a906d1..fdb11f2fc 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -43,6 +43,7 @@ data: # enable_team_superuser: "false" enable_teams_api: "false" # etcd_host: "" + # kubernetes_use_configmaps: "false" # infrastructure_roles_secret_name: postgresql-infrastructure-roles # inherited_labels: application,environment # kube_iam_role: "" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 1d5544c7c..86051e43b 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -42,6 +42,8 @@ spec: type: boolean etcd_host: type: string + kubernetes_use_configmaps: + type: boolean max_instances: type: integer minimum: -1 # -1 = disabled diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index cd739c817..1f061ef7a 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -5,6 +5,7 @@ metadata: configuration: # enable_crd_validation: true etcd_host: "" + # kubernetes_use_configmaps: false docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p2 # enable_shm_volume: true max_instances: -1 diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index cfccd1e56..4869e9288 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -727,6 +727,9 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation "etcd_host": { Type: "string", }, + "kubernetes_use_configmaps": { + Type: "boolean", + }, "max_instances": { Type: "integer", Description: "-1 = disabled", diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 6eb6732a5..8ed4281f4 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -183,6 +183,7 @@ type OperatorLogicalBackupConfiguration struct { type OperatorConfigurationData struct { EnableCRDValidation *bool `json:"enable_crd_validation,omitempty"` EtcdHost string `json:"etcd_host,omitempty"` + KubernetesUseConfigMaps bool `json:"kubernetes_use_configmaps,omitempty"` DockerImage string `json:"docker_image,omitempty"` Workers uint32 `json:"workers,omitempty"` MinInstances int32 `json:"min_instances,omitempty"` diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index d5297ab8e..1baa4455c 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -672,6 +672,10 @@ func (c *Cluster) generateSpiloPodEnvVars(uid types.UID, spiloConfiguration stri envVars = append(envVars, v1.EnvVar{Name: "ETCD_HOST", Value: c.OpConfig.EtcdHost}) } + if c.patroniKubernetesUseConfigMaps() { + envVars = append(envVars, v1.EnvVar{Name: "KUBERNETES_USE_CONFIGMAPS", Value: "true"}) + } + if cloneDescription.ClusterName != "" { envVars = append(envVars, c.generateCloneEnvironment(cloneDescription)...) } @@ -1406,7 +1410,7 @@ func (c *Cluster) generateService(role PostgresRole, spec *acidv1.PostgresSpec) Type: v1.ServiceTypeClusterIP, } - if role == Replica { + if role == Replica || c.patroniKubernetesUseConfigMaps() { serviceSpec.Selector = c.roleLabelsSet(false, role) } diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go index 405f48f9f..4dcdfb28a 100644 --- a/pkg/cluster/util.go +++ b/pkg/cluster/util.go @@ -510,6 +510,15 @@ func (c *Cluster) patroniUsesKubernetes() bool { return c.OpConfig.EtcdHost == "" } +func (c *Cluster) patroniKubernetesUseConfigMaps() bool { + if !c.patroniUsesKubernetes() { + return false + } + + // otherwise, follow the operator configuration + return c.OpConfig.KubernetesUseConfigMaps +} + func (c *Cluster) needConnectionPoolerWorker(spec *acidv1.PostgresSpec) bool { if spec.EnableConnectionPooler == nil { return spec.ConnectionPooler != nil diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index e9fe2f379..a9c36f995 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -35,6 +35,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur // general config result.EnableCRDValidation = fromCRD.EnableCRDValidation result.EtcdHost = fromCRD.EtcdHost + result.KubernetesUseConfigMaps = fromCRD.KubernetesUseConfigMaps result.DockerImage = fromCRD.DockerImage result.Workers = fromCRD.Workers result.MinInstances = fromCRD.MinInstances diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index d5464ca7c..0a7bac835 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -107,11 +107,12 @@ type Config struct { LogicalBackup ConnectionPooler - WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to' - EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS - DockerImage string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-12:1.6-p2"` - Sidecars map[string]string `name:"sidecar_docker_images"` - PodServiceAccountName string `name:"pod_service_account_name" default:"postgres-pod"` + WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to' + KubernetesUseConfigMaps bool `name:"kubernetes_use_configmaps" default:"false"` + EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS + DockerImage string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-12:1.6-p2"` + Sidecars map[string]string `name:"sidecar_docker_images"` + PodServiceAccountName string `name:"pod_service_account_name" default:"postgres-pod"` // value of this string must be valid JSON or YAML; see initPodServiceAccount PodServiceAccountDefinition string `name:"pod_service_account_definition" default:""` PodServiceAccountRoleBindingDefinition string `name:"pod_service_account_role_binding_definition" default:""` From 64389b8bad2940e829047799850645b02c55da5e Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 3 Apr 2020 16:28:36 +0200 Subject: [PATCH 022/168] update image and docs for connection pooler (#898) --- docs/reference/cluster_manifest.md | 3 ++- docs/reference/operator_parameters.md | 9 ++++++--- docs/user.md | 11 ++++++----- manifests/configmap.yaml | 2 +- .../postgresql-operator-default-configuration.yaml | 2 +- 5 files changed, 16 insertions(+), 11 deletions(-) diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 967b2de5d..f482cc218 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -376,10 +376,11 @@ present. How many instances of connection pooler to create. * **schema** - Schema to create for credentials lookup function. + Database schema to create for credentials lookup function. * **user** User to create for connection pooler to be able to connect to a database. + You can also choose a role from the `users` section or a system user role. * **dockerImage** Which docker image to use for connection pooler deployment. diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 9f0e0b67a..3d31abab4 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -83,7 +83,7 @@ Those are top-level keys, containing both leaf keys and groups. * **kubernetes_use_configmaps** Select if setup uses endpoints (default), or configmaps to manage leader when DCS is kubernetes (not etcd or similar). In OpenShift it is not possible to - use endpoints option, and configmaps is required. By default, + use endpoints option, and configmaps is required. By default, `kubernetes_use_configmaps: false`, meaning endpoints will be used. * **docker_image** @@ -615,11 +615,14 @@ operator being able to provide some reasonable defaults. the required minimum. * **connection_pooler_schema** - Schema to create for credentials lookup function. Default is `pooler`. + Database schema to create for credentials lookup function to be used by the + connection pooler. Is is created in every database of the Postgres cluster. + You can also choose an existing schema. Default schema is `pooler`. * **connection_pooler_user** User to create for connection pooler to be able to connect to a database. - Default is `pooler`. + You can also choose an existing role, but make sure it has the `LOGIN` + privilege. Default role is `pooler`. * **connection_pooler_image** Docker image to use for connection pooler deployment. diff --git a/docs/user.md b/docs/user.md index 67ed5971f..1be50a01a 100644 --- a/docs/user.md +++ b/docs/user.md @@ -527,7 +527,7 @@ spec: This will tell the operator to create a connection pooler with default configuration, through which one can access the master via a separate service `{cluster-name}-pooler`. In most of the cases the -[default configuration](reference/operator_parameters.md#connection-pool-configuration) +[default configuration](reference/operator_parameters.md#connection-pooler-configuration) should be good enough. To configure a new connection pooler individually for each Postgres cluster, specify: @@ -540,7 +540,8 @@ spec: # in which mode to run, session or transaction mode: "transaction" - # schema, which operator will create to install credentials lookup function + # schema, which operator will create in each database + # to install credentials lookup function for connection pooler schema: "pooler" # user, which operator will create for connection pooler @@ -560,11 +561,11 @@ The `enableConnectionPooler` flag is not required when the `connectionPooler` section is present in the manifest. But, it can be used to disable/remove the pooler while keeping its configuration. -By default, `pgbouncer` is used as connection pooler. To find out about pooler -modes read the `pgbouncer` [docs](https://www.pgbouncer.org/config.html#pooler_mode) +By default, [`PgBouncer`](https://www.pgbouncer.org/) is used as connection pooler. +To find out about pool modes read the `PgBouncer` [docs](https://www.pgbouncer.org/config.html#pooler_mode) (but it should be the general approach between different implementation). -Note, that using `pgbouncer` a meaningful resource CPU limit should be 1 core +Note, that using `PgBouncer` a meaningful resource CPU limit should be 1 core or less (there is a way to utilize more than one, but in K8s it's easier just to spin up more instances). diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index fdb11f2fc..954881ed3 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -15,7 +15,7 @@ data: # connection_pooler_default_cpu_request: "500m" # connection_pooler_default_memory_limit: 100Mi # connection_pooler_default_memory_request: 100Mi - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-5" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-6" # connection_pooler_max_db_connections: 60 # connection_pooler_mode: "transaction" # connection_pooler_number_of_instances: 2 diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 1f061ef7a..209e2684b 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -127,7 +127,7 @@ configuration: connection_pooler_default_cpu_request: "500m" connection_pooler_default_memory_limit: 100Mi connection_pooler_default_memory_request: 100Mi - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-5" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-6" # connection_pooler_max_db_connections: 60 connection_pooler_mode: "transaction" connection_pooler_number_of_instances: 2 From 4dee8918bd45e4356e10acf15b9f71bf0a43f9bd Mon Sep 17 00:00:00 2001 From: Leon Albers <5120677+lalbers@users.noreply.github.com> Date: Mon, 6 Apr 2020 14:27:17 +0200 Subject: [PATCH 023/168] Allow configuration of patroni's replication mode (#869) * Add patroni parameters for `synchronous_mode` * Update complete-postgres-manifest.yaml, removed quotation marks * Update k8sres_test.go, adjust result for `Patroni configured` * Update k8sres_test.go, adjust result for `Patroni configured` * Update complete-postgres-manifest.yaml, set synchronous mode to false in this example * Update pkg/cluster/k8sres.go Does the same but is shorter. So we fix that it if you like. Co-Authored-By: Felix Kunde * Update docs/reference/cluster_manifest.md Co-Authored-By: Felix Kunde * Add patroni's `synchronous_mode_strict` * Extend `TestGenerateSpiloConfig` with `SynchronousModeStrict` Co-authored-by: Felix Kunde --- charts/postgres-operator/crds/postgresqls.yaml | 4 ++++ docs/reference/cluster_manifest.md | 6 ++++++ manifests/complete-postgres-manifest.yaml | 2 ++ manifests/postgresql.crd.yaml | 4 ++++ pkg/apis/acid.zalan.do/v1/crds.go | 6 ++++++ pkg/apis/acid.zalan.do/v1/postgresql_type.go | 16 +++++++++------- pkg/cluster/k8sres.go | 8 ++++++++ pkg/cluster/k8sres_test.go | 16 +++++++++------- 8 files changed, 48 insertions(+), 14 deletions(-) diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 57875cba7..f64080ed5 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -218,6 +218,10 @@ spec: type: integer retry_timeout: type: integer + synchronous_mode: + type: boolean + synchronous_mode_strict: + type: boolean maximum_lag_on_failover: type: integer podAnnotations: diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index f482cc218..83dedabd3 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -217,6 +217,12 @@ explanation of `ttl` and `loop_wait` parameters. automatically created by Patroni for cluster members and permanent replication slots. Optional. +* **synchronous_mode** + Patroni `synchronous_mode` parameter value. The default is set to `false`. Optional. + +* **synchronous_mode_strict** + Patroni `synchronous_mode_strict` parameter value. Can be used in addition to `synchronous_mode`. The default is set to `false`. Optional. + ## Postgres container resources Those parameters define [CPU and memory requests and limits](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index a5811504e..3a87254cf 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -67,6 +67,8 @@ spec: ttl: 30 loop_wait: &loop_wait 10 retry_timeout: 10 + synchronous_mode: false + synchronous_mode_strict: false maximum_lag_on_failover: 33554432 # restore a Postgres DB with point-in-time-recovery diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 48f6c7397..f8631f0b7 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -184,6 +184,10 @@ spec: type: integer maximum_lag_on_failover: type: integer + synchronous_mode: + type: boolean + synchronous_mode_strict: + type: boolean podAnnotations: type: object additionalProperties: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 4869e9288..60dd66ba8 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -358,6 +358,12 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ "maximum_lag_on_failover": { Type: "integer", }, + "synchronous_mode": { + Type: "boolean", + }, + "synchronous_mode_strict": { + Type: "boolean", + }, }, }, "podAnnotations": { diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 0695d3f9f..b1b6a36a6 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -118,13 +118,15 @@ type Resources struct { // Patroni contains Patroni-specific configuration type Patroni struct { - InitDB map[string]string `json:"initdb"` - PgHba []string `json:"pg_hba"` - TTL uint32 `json:"ttl"` - LoopWait uint32 `json:"loop_wait"` - RetryTimeout uint32 `json:"retry_timeout"` - MaximumLagOnFailover float32 `json:"maximum_lag_on_failover"` // float32 because https://github.com/kubernetes/kubernetes/issues/30213 - Slots map[string]map[string]string `json:"slots"` + InitDB map[string]string `json:"initdb"` + PgHba []string `json:"pg_hba"` + TTL uint32 `json:"ttl"` + LoopWait uint32 `json:"loop_wait"` + RetryTimeout uint32 `json:"retry_timeout"` + MaximumLagOnFailover float32 `json:"maximum_lag_on_failover"` // float32 because https://github.com/kubernetes/kubernetes/issues/30213 + Slots map[string]map[string]string `json:"slots"` + SynchronousMode bool `json:"synchronous_mode"` + SynchronousModeStrict bool `json:"synchronous_mode_strict"` } //StandbyCluster diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 1baa4455c..8de61bdea 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -49,6 +49,8 @@ type patroniDCS struct { LoopWait uint32 `json:"loop_wait,omitempty"` RetryTimeout uint32 `json:"retry_timeout,omitempty"` MaximumLagOnFailover float32 `json:"maximum_lag_on_failover,omitempty"` + SynchronousMode bool `json:"synchronous_mode,omitempty"` + SynchronousModeStrict bool `json:"synchronous_mode_strict,omitempty"` PGBootstrapConfiguration map[string]interface{} `json:"postgresql,omitempty"` Slots map[string]map[string]string `json:"slots,omitempty"` } @@ -283,6 +285,12 @@ PatroniInitDBParams: if patroni.Slots != nil { config.Bootstrap.DCS.Slots = patroni.Slots } + if patroni.SynchronousMode { + config.Bootstrap.DCS.SynchronousMode = patroni.SynchronousMode + } + if patroni.SynchronousModeStrict != false { + config.Bootstrap.DCS.SynchronousModeStrict = patroni.SynchronousModeStrict + } config.PgLocalConfiguration = make(map[string]interface{}) config.PgLocalConfiguration[patroniPGBinariesParameterName] = fmt.Sprintf(pgBinariesLocationTemplate, pg.PgVersion) diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 4068811c3..0930279d2 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -65,16 +65,18 @@ func TestGenerateSpiloJSONConfiguration(t *testing.T) { "locale": "en_US.UTF-8", "data-checksums": "true", }, - PgHba: []string{"hostssl all all 0.0.0.0/0 md5", "host all all 0.0.0.0/0 md5"}, - TTL: 30, - LoopWait: 10, - RetryTimeout: 10, - MaximumLagOnFailover: 33554432, - Slots: map[string]map[string]string{"permanent_logical_1": {"type": "logical", "database": "foo", "plugin": "pgoutput"}}, + PgHba: []string{"hostssl all all 0.0.0.0/0 md5", "host all all 0.0.0.0/0 md5"}, + TTL: 30, + LoopWait: 10, + RetryTimeout: 10, + MaximumLagOnFailover: 33554432, + SynchronousMode: true, + SynchronousModeStrict: true, + Slots: map[string]map[string]string{"permanent_logical_1": {"type": "logical", "database": "foo", "plugin": "pgoutput"}}, }, role: "zalandos", opConfig: config.Config{}, - result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/11/bin","pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"users":{"zalandos":{"password":"","options":["CREATEDB","NOLOGIN"]}},"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}}}}}`, + result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/11/bin","pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"users":{"zalandos":{"password":"","options":["CREATEDB","NOLOGIN"]}},"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"synchronous_mode":true,"synchronous_mode_strict":true,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}}}}}`, }, } for _, tt := range tests { From 7232326159a9f77766260bc3e7b49f79dd101bd8 Mon Sep 17 00:00:00 2001 From: ReSearchITEng Date: Thu, 9 Apr 2020 10:16:45 +0300 Subject: [PATCH 024/168] Fix val docs (#901) * missing quotes in pooler configmap in values.yaml * missing quotes in pooler configmap in values-crd.yaml * docs clarifications * helm3 --skip-crds * Update docs/user.md Co-Authored-By: Felix Kunde * details moved in docs Co-authored-by: Felix Kunde --- charts/postgres-operator/values-crd.yaml | 5 +++-- charts/postgres-operator/values.yaml | 5 +++-- docs/reference/cluster_manifest.md | 8 ++++++-- docs/user.md | 3 ++- manifests/complete-postgres-manifest.yaml | 3 +++ 5 files changed, 17 insertions(+), 7 deletions(-) diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 37ad6e45e..caa4dda4d 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -277,11 +277,11 @@ configConnectionPooler: # docker image connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer" # max db connections the pooler should hold - connection_pooler_max_db_connections: 60 + connection_pooler_max_db_connections: "60" # default pooling mode connection_pooler_mode: "transaction" # number of pooler instances - connection_pooler_number_of_instances: 2 + connection_pooler_number_of_instances: "2" # default resources connection_pooler_default_cpu_request: 500m connection_pooler_default_memory_request: 100Mi @@ -294,6 +294,7 @@ rbac: crd: # Specifies whether custom resource definitions should be created + # When using helm3, this is ignored; instead use "--skip-crds" to skip. create: true serviceAccount: diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 3a701bb64..e7db249f0 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -254,11 +254,11 @@ configConnectionPooler: # docker image connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer" # max db connections the pooler should hold - connection_pooler_max_db_connections: 60 + connection_pooler_max_db_connections: "60" # default pooling mode connection_pooler_mode: "transaction" # number of pooler instances - connection_pooler_number_of_instances: 2 + connection_pooler_number_of_instances: "2" # default resources connection_pooler_default_cpu_request: 500m connection_pooler_default_memory_request: 100Mi @@ -271,6 +271,7 @@ rbac: crd: # Specifies whether custom resource definitions should be created + # When using helm3, this is ignored; instead use "--skip-crds" to skip. create: true serviceAccount: diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 83dedabd3..0cbcdc08e 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -419,5 +419,9 @@ Those parameters are grouped under the `tls` top-level key. Filename of the private key. Defaults to "tls.key". * **caFile** - Optional filename to the CA certificate. Useful when the client connects - with `sslmode=verify-ca` or `sslmode=verify-full`. Default is empty. + Optional filename to the CA certificate (e.g. "ca.crt"). Useful when the + client connects with `sslmode=verify-ca` or `sslmode=verify-full`. + Default is empty. + + Optionally one can provide full path for any of them. By default it is + relative to the "/tls/", which is mount path of the tls secret. diff --git a/docs/user.md b/docs/user.md index 1be50a01a..bb12fd2e1 100644 --- a/docs/user.md +++ b/docs/user.md @@ -584,7 +584,8 @@ don't know the value, use `103` which is the GID from the default spilo image OpenShift allocates the users and groups dynamically (based on scc), and their range is different in every namespace. Due to this dynamic behaviour, it's not trivial to know at deploy time the uid/gid of the user in the cluster. -This way, in OpenShift, you may want to skip the spilo_fsgroup setting. +Therefore, instead of using a global `spilo_fsgroup` setting, use the `spiloFSGroup` field +per Postgres cluster.``` Upload the cert as a kubernetes secret: ```sh diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 3a87254cf..6de6db11e 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -125,5 +125,8 @@ spec: certificateFile: "tls.crt" privateKeyFile: "tls.key" caFile: "" # optionally configure Postgres with a CA certificate +# file names can be also defined with absolute path, and will no longer be relative +# to the "/tls/" path where the secret is being mounted by default. # When TLS is enabled, also set spiloFSGroup parameter above to the relevant value. # if unknown, set it to 103 which is the usual value in the default spilo images. +# In Openshift, there is no need to set spiloFSGroup/spilo_fsgroup. From a1f2bd05b978b4dac384eb14a06a39f590cf5f57 Mon Sep 17 00:00:00 2001 From: Dmitry Dolgov <9erthalion6@gmail.com> Date: Thu, 9 Apr 2020 09:21:45 +0200 Subject: [PATCH 025/168] Prevent superuser from being a connection pool user (#906) * Protected and system users can't be a connection pool user It's not supported, neither it's a best practice. Also fix potential null pointer access. For protected users it makes sense by intent of protecting this users (e.g. from being overriden or used as something else than supposed). For system users the reason is the same as for superuser, it's about replicastion user and it's under patroni control. This is implemented on both levels, operator config and postgresql manifest. For the latter we just use default name in this case, assuming that operator config is always correct. For the former, since it's a serious misconfiguration, operator panics. --- pkg/cluster/cluster.go | 17 +++++++++++++--- pkg/cluster/cluster_test.go | 34 +++++++++++++++++++++++++++++++ pkg/cluster/resources.go | 5 +++++ pkg/controller/operator_config.go | 5 +++++ pkg/util/config/config.go | 6 ++++++ 5 files changed, 64 insertions(+), 3 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index cde2dc260..51dd368fb 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -877,9 +877,20 @@ func (c *Cluster) initSystemUsers() { c.Spec.ConnectionPooler = &acidv1.ConnectionPooler{} } - username := util.Coalesce( - c.Spec.ConnectionPooler.User, - c.OpConfig.ConnectionPooler.User) + // Using superuser as pooler user is not a good idea. First of all it's + // not going to be synced correctly with the current implementation, + // and second it's a bad practice. + username := c.OpConfig.ConnectionPooler.User + + isSuperUser := c.Spec.ConnectionPooler.User == c.OpConfig.SuperUsername + isProtectedUser := c.shouldAvoidProtectedOrSystemRole( + c.Spec.ConnectionPooler.User, "connection pool role") + + if !isSuperUser && !isProtectedUser { + username = util.Coalesce( + c.Spec.ConnectionPooler.User, + c.OpConfig.ConnectionPooler.User) + } // connection pooler application should be able to login with this role connectionPoolerUser := spec.PgUser{ diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index af3092ad5..432f53132 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -721,4 +721,38 @@ func TestInitSystemUsers(t *testing.T) { if _, exist := cl.systemUsers[constants.ConnectionPoolerUserKeyName]; !exist { t.Errorf("%s, connection pooler user is not present", testName) } + + // superuser is not allowed as connection pool user + cl.Spec.ConnectionPooler = &acidv1.ConnectionPooler{ + User: "postgres", + } + cl.OpConfig.SuperUsername = "postgres" + cl.OpConfig.ConnectionPooler.User = "pooler" + + cl.initSystemUsers() + if _, exist := cl.pgUsers["pooler"]; !exist { + t.Errorf("%s, Superuser is not allowed to be a connection pool user", testName) + } + + // neither protected users are + delete(cl.pgUsers, "pooler") + cl.Spec.ConnectionPooler = &acidv1.ConnectionPooler{ + User: "admin", + } + cl.OpConfig.ProtectedRoles = []string{"admin"} + + cl.initSystemUsers() + if _, exist := cl.pgUsers["pooler"]; !exist { + t.Errorf("%s, Protected user are not allowed to be a connection pool user", testName) + } + + delete(cl.pgUsers, "pooler") + cl.Spec.ConnectionPooler = &acidv1.ConnectionPooler{ + User: "standby", + } + + cl.initSystemUsers() + if _, exist := cl.pgUsers["pooler"]; !exist { + t.Errorf("%s, System users are not allowed to be a connection pool user", testName) + } } diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index 4c341aefe..b38458af8 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -105,7 +105,12 @@ func (c *Cluster) createConnectionPooler(lookup InstallFunction) (*ConnectionPoo var msg string c.setProcessName("creating connection pooler") + if c.ConnectionPooler == nil { + c.ConnectionPooler = &ConnectionPoolerObjects{} + } + schema := c.Spec.ConnectionPooler.Schema + if schema == "" { schema = c.OpConfig.ConnectionPooler.Schema } diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index a9c36f995..07be90f22 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -169,6 +169,11 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur fromCRD.ConnectionPooler.User, constants.ConnectionPoolerUserName) + if result.ConnectionPooler.User == result.SuperUsername { + msg := "Connection pool user is not allowed to be the same as super user, username: %s" + panic(fmt.Errorf(msg, result.ConnectionPooler.User)) + } + result.ConnectionPooler.Image = util.Coalesce( fromCRD.ConnectionPooler.Image, "registry.opensource.zalan.do/acid/pgbouncer") diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 0a7bac835..84a62c0fd 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -218,5 +218,11 @@ func validate(cfg *Config) (err error) { msg := "number of connection pooler instances should be higher than %d" err = fmt.Errorf(msg, constants.ConnectionPoolerMinInstances) } + + if cfg.ConnectionPooler.User == cfg.SuperUsername { + msg := "Connection pool user is not allowed to be the same as super user, username: %s" + err = fmt.Errorf(msg, cfg.ConnectionPooler.User) + } + return } From ea3eef45d95c62ff1f4a4c825744835031a7b54f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thierry=20Sall=C3=A9?= Date: Wed, 15 Apr 2020 09:13:35 +0200 Subject: [PATCH 026/168] Additional volumes capability (#736) * Allow additional Volumes to be mounted * added TargetContainers option to determine if additional volume need to be mounter or not * fixed dependencies * updated manifest additional volume example * More validation Check that there are no volume mount path clashes or "all" vs ["a", "b"] mixtures. Also change the default behaviour to mount to "postgres" container. * More documentation / example about additional volumes * Revert go.sum and go.mod from origin/master * Declare addictionalVolume specs in CRDs * fixed k8sres after rebase * resolv conflict Co-authored-by: Dmitrii Dolgov <9erthalion6@gmail.com> Co-authored-by: Thierry --- .../postgres-operator/crds/postgresqls.yaml | 22 +++ docs/reference/cluster_manifest.md | 12 ++ manifests/complete-postgres-manifest.yaml | 23 +++ manifests/postgresql.crd.yaml | 22 +++ pkg/apis/acid.zalan.do/v1/crds.go | 31 ++++ pkg/apis/acid.zalan.do/v1/postgresql_type.go | 9 + pkg/cluster/k8sres.go | 82 ++++++++- pkg/cluster/k8sres_test.go | 169 ++++++++++++++++++ 8 files changed, 364 insertions(+), 6 deletions(-) diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index f64080ed5..3c666b9ab 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -74,6 +74,28 @@ spec: - teamId - postgresql properties: + additionalVolumes: + type: array + items: + type: object + required: + - name + - mountPath + - volumeSource + properties: + name: + type: string + mountPath: + type: string + targetContainers: + type: array + nullable: true + items: + type: string + volumeSource: + type: object + subPath: + type: string allowedSourceRanges: type: array nullable: true diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 0cbcdc08e..361e32780 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -154,6 +154,18 @@ These parameters are grouped directly under the `spec` key in the manifest. [the reference schedule format](https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#schedule) into account. Optional. Default is: "30 00 \* \* \*" +* **additionalVolumes** + List of additional volumes to mount in each container of the statefulset pod. + Each item must contain a `name`, `mountPath`, and `volumeSource` which is a + [kubernetes volumeSource](https://godoc.org/k8s.io/api/core/v1#VolumeSource). + It allows you to mount existing PersistentVolumeClaims, ConfigMaps and Secrets inside the StatefulSet. + Also an `emptyDir` volume can be shared between initContainer and statefulSet. + Additionaly, you can provide a `SubPath` for volume mount (a file in a configMap source volume, for example). + You can also specify in which container the additional Volumes will be mounted with the `targetContainers` array option. + If `targetContainers` is empty, additional volumes will be mounted only in the `postgres` container. + If you set the `all` special item, it will be mounted in all containers (postgres + sidecars). + Else you can set the list of target containers in which the additional volumes will be mounted (eg : postgres, telegraf) + ## Postgres parameters Those parameters are grouped under the `postgresql` top-level key, which is diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 6de6db11e..23b0b6096 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -12,6 +12,29 @@ spec: volume: size: 1Gi # storageClass: my-sc + additionalVolumes: + - name: data + mountPath: /home/postgres/pgdata/partitions + targetContainers: + - postgres + volumeSource: + PersistentVolumeClaim: + claimName: pvc-postgresql-data-partitions + readyOnly: false + - name: conf + mountPath: /etc/telegraf + subPath: telegraf.conf + targetContainers: + - telegraf-sidecar + volumeSource: + configMap: + name: my-config-map + - name: empty + mountPath: /opt/empty + targetContainers: + - all + volumeSource: + emptyDir: {} numberOfInstances: 2 users: # Application/Robot users zalando: diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index f8631f0b7..c9d60d60a 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -38,6 +38,28 @@ spec: - teamId - postgresql properties: + additionalVolumes: + type: array + items: + type: object + required: + - name + - mountPath + - volumeSource + properties: + name: + type: string + mountPath: + type: string + targetContainers: + type: array + nullable: true + items: + type: string + volumeSource: + type: object + subPath: + type: string allowedSourceRanges: type: array nullable: true diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 60dd66ba8..d693d0e15 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -682,6 +682,37 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, }, }, + "additionalVolumes": { + Type: "array", + Items: &apiextv1beta1.JSONSchemaPropsOrArray{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "object", + Required: []string{"name", "mountPath", "volumeSource"}, + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "name": { + Type: "string", + }, + "mountPath": { + Type: "string", + }, + "targetContainers": { + Type: "array", + Items: &apiextv1beta1.JSONSchemaPropsOrArray{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "string", + }, + }, + }, + "volumeSource": { + Type: "object", + }, + "subPath": { + Type: "string", + }, + }, + }, + }, + }, }, }, "status": { diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index b1b6a36a6..04b70cba8 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -67,6 +67,7 @@ type PostgresSpec struct { PodAnnotations map[string]string `json:"podAnnotations"` ServiceAnnotations map[string]string `json:"serviceAnnotations"` TLS *TLSDescription `json:"tls"` + AdditionalVolumes []AdditionalVolume `json:"additionalVolumes,omitempty"` // deprecated json tags InitContainersOld []v1.Container `json:"init_containers,omitempty"` @@ -98,6 +99,14 @@ type Volume struct { SubPath string `json:"subPath,omitempty"` } +type AdditionalVolume struct { + Name string `json:"name"` + MountPath string `json:"mountPath"` + SubPath string `json:"subPath"` + TargetContainers []string `json:"targetContainers"` + VolumeSource v1.VolumeSource `json:"volume"` +} + // PostgresqlParam describes PostgreSQL version and pairs of configuration parameter name - values. type PostgresqlParam struct { PgVersion string `json:"version"` diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 8de61bdea..e6f53af9d 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -500,7 +500,7 @@ func mountShmVolumeNeeded(opConfig config.Config, spec *acidv1.PostgresSpec) *bo return opConfig.ShmVolume } -func generatePodTemplate( +func (c *Cluster) generatePodTemplate( namespace string, labels labels.Set, annotations map[string]string, @@ -520,6 +520,7 @@ func generatePodTemplate( additionalSecretMount string, additionalSecretMountPath string, volumes []v1.Volume, + additionalVolumes []acidv1.AdditionalVolume, ) (*v1.PodTemplateSpec, error) { terminateGracePeriodSeconds := terminateGracePeriod @@ -559,6 +560,10 @@ func generatePodTemplate( addSecretVolume(&podSpec, additionalSecretMount, additionalSecretMountPath) } + if additionalVolumes != nil { + c.addAdditionalVolumes(&podSpec, additionalVolumes) + } + template := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: labels, @@ -1084,7 +1089,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef annotations := c.generatePodAnnotations(spec) // generate pod template for the statefulset, based on the spilo container and sidecars - podTemplate, err = generatePodTemplate( + podTemplate, err = c.generatePodTemplate( c.Namespace, c.labelsSet(true), annotations, @@ -1104,7 +1109,8 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef c.OpConfig.AdditionalSecretMount, c.OpConfig.AdditionalSecretMountPath, volumes, - ) + spec.AdditionalVolumes) + if err != nil { return nil, fmt.Errorf("could not generate pod template: %v", err) } @@ -1298,6 +1304,69 @@ func addSecretVolume(podSpec *v1.PodSpec, additionalSecretMount string, addition podSpec.Volumes = volumes } +func (c *Cluster) addAdditionalVolumes(podSpec *v1.PodSpec, + additionalVolumes []acidv1.AdditionalVolume) { + + volumes := podSpec.Volumes + mountPaths := map[string]acidv1.AdditionalVolume{} + for i, v := range additionalVolumes { + if previousVolume, exist := mountPaths[v.MountPath]; exist { + msg := "Volume %+v cannot be mounted to the same path as %+v" + c.logger.Warningf(msg, v, previousVolume) + continue + } + + if v.MountPath == constants.PostgresDataMount { + msg := "Cannot mount volume on postgresql data directory, %+v" + c.logger.Warningf(msg, v) + continue + } + + if v.TargetContainers == nil { + spiloContainer := podSpec.Containers[0] + additionalVolumes[i].TargetContainers = []string{spiloContainer.Name} + } + + for _, target := range v.TargetContainers { + if target == "all" && len(v.TargetContainers) != 1 { + msg := `Target containers could be either "all" or a list + of containers, mixing those is not allowed, %+v` + c.logger.Warningf(msg, v) + continue + } + } + + volumes = append(volumes, + v1.Volume{ + Name: v.Name, + VolumeSource: v.VolumeSource, + }, + ) + + mountPaths[v.MountPath] = v + } + + c.logger.Infof("Mount additional volumes: %+v", additionalVolumes) + + for i := range podSpec.Containers { + mounts := podSpec.Containers[i].VolumeMounts + for _, v := range additionalVolumes { + for _, target := range v.TargetContainers { + if podSpec.Containers[i].Name == target || target == "all" { + mounts = append(mounts, v1.VolumeMount{ + Name: v.Name, + MountPath: v.MountPath, + SubPath: v.SubPath, + }) + } + } + } + podSpec.Containers[i].VolumeMounts = mounts + } + + podSpec.Volumes = volumes +} + func generatePersistentVolumeClaimTemplate(volumeSize, volumeStorageClass string) (*v1.PersistentVolumeClaim, error) { var storageClassName *string @@ -1702,7 +1771,7 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) { annotations := c.generatePodAnnotations(&c.Spec) // re-use the method that generates DB pod templates - if podTemplate, err = generatePodTemplate( + if podTemplate, err = c.generatePodTemplate( c.Namespace, labels, annotations, @@ -1721,8 +1790,9 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) { "", c.OpConfig.AdditionalSecretMount, c.OpConfig.AdditionalSecretMountPath, - nil); err != nil { - return nil, fmt.Errorf("could not generate pod template for logical backup pod: %v", err) + nil, + []acidv1.AdditionalVolume{}); err != nil { + return nil, fmt.Errorf("could not generate pod template for logical backup pod: %v", err) } // overwrite specific params of logical backups pods diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 0930279d2..5b55e988c 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -1021,3 +1021,172 @@ func TestTLS(t *testing.T) { assert.Contains(t, s.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "SSL_PRIVATE_KEY_FILE", Value: "/tls/tls.key"}) assert.Contains(t, s.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "SSL_CA_FILE", Value: "/tls/ca.crt"}) } + +func TestAdditionalVolume(t *testing.T) { + testName := "TestAdditionalVolume" + tests := []struct { + subTest string + podSpec *v1.PodSpec + volumePos int + }{ + { + subTest: "empty PodSpec", + podSpec: &v1.PodSpec{ + Volumes: []v1.Volume{}, + Containers: []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{}, + }, + }, + }, + volumePos: 0, + }, + { + subTest: "non empty PodSpec", + podSpec: &v1.PodSpec{ + Volumes: []v1.Volume{{}}, + Containers: []v1.Container{ + { + Name: "postgres", + VolumeMounts: []v1.VolumeMount{ + { + Name: "data", + ReadOnly: false, + MountPath: "/data", + }, + }, + }, + }, + }, + volumePos: 1, + }, + { + subTest: "non empty PodSpec with sidecar", + podSpec: &v1.PodSpec{ + Volumes: []v1.Volume{{}}, + Containers: []v1.Container{ + { + Name: "postgres", + VolumeMounts: []v1.VolumeMount{ + { + Name: "data", + ReadOnly: false, + MountPath: "/data", + }, + }, + }, + { + Name: "sidecar", + VolumeMounts: []v1.VolumeMount{ + { + Name: "data", + ReadOnly: false, + MountPath: "/data", + }, + }, + }, + }, + }, + volumePos: 1, + }, + } + + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + + for _, tt := range tests { + // Test with additional volume mounted in all containers + additionalVolumeMount := []acidv1.AdditionalVolume{ + { + Name: "test", + MountPath: "/test", + TargetContainers: []string{"all"}, + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + } + + numMounts := len(tt.podSpec.Containers[0].VolumeMounts) + + cluster.addAdditionalVolumes(tt.podSpec, additionalVolumeMount) + volumeName := tt.podSpec.Volumes[tt.volumePos].Name + + if volumeName != additionalVolumeMount[0].Name { + t.Errorf("%s %s: Expected volume %v was not created, have %s instead", + testName, tt.subTest, additionalVolumeMount, volumeName) + } + + for i := range tt.podSpec.Containers { + volumeMountName := tt.podSpec.Containers[i].VolumeMounts[tt.volumePos].Name + + if volumeMountName != additionalVolumeMount[0].Name { + t.Errorf("%s %s: Expected mount %v was not created, have %s instead", + testName, tt.subTest, additionalVolumeMount, volumeMountName) + } + + } + + numMountsCheck := len(tt.podSpec.Containers[0].VolumeMounts) + + if numMountsCheck != numMounts+1 { + t.Errorf("Unexpected number of VolumeMounts: got %v instead of %v", + numMountsCheck, numMounts+1) + } + } + + for _, tt := range tests { + // Test with additional volume mounted only in first container + additionalVolumeMount := []acidv1.AdditionalVolume{ + { + Name: "test", + MountPath: "/test", + TargetContainers: []string{"postgres"}, + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + } + + numMounts := len(tt.podSpec.Containers[0].VolumeMounts) + + cluster.addAdditionalVolumes(tt.podSpec, additionalVolumeMount) + volumeName := tt.podSpec.Volumes[tt.volumePos].Name + + if volumeName != additionalVolumeMount[0].Name { + t.Errorf("%s %s: Expected volume %v was not created, have %s instead", + testName, tt.subTest, additionalVolumeMount, volumeName) + } + + for _, container := range tt.podSpec.Containers { + if container.Name == "postgres" { + volumeMountName := container.VolumeMounts[tt.volumePos].Name + + if volumeMountName != additionalVolumeMount[0].Name { + t.Errorf("%s %s: Expected mount %v was not created, have %s instead", + testName, tt.subTest, additionalVolumeMount, volumeMountName) + } + + numMountsCheck := len(container.VolumeMounts) + if numMountsCheck != numMounts+1 { + t.Errorf("Unexpected number of VolumeMounts: got %v instead of %v", + numMountsCheck, numMounts+1) + } + } else { + numMountsCheck := len(container.VolumeMounts) + if numMountsCheck == numMounts+1 { + t.Errorf("Unexpected number of VolumeMounts: got %v instead of %v", + numMountsCheck, numMounts) + } + } + } + } +} From 7e8f6687ebbb4a168a98dbb7d69311bd0d393f30 Mon Sep 17 00:00:00 2001 From: ReSearchITEng Date: Wed, 15 Apr 2020 16:24:55 +0300 Subject: [PATCH 027/168] make tls pr798 use additionalVolumes capability from pr736 (#920) * make tls pr798 use additionalVolumes capability from pr736 * move the volume* sections lower * update helm chart crds and docs * fix user.md typos --- .gitignore | 2 + .../postgres-operator/crds/postgresqls.yaml | 15 ++++++ docs/reference/cluster_manifest.md | 7 +++ docs/user.md | 31 ++++++++++-- manifests/complete-postgres-manifest.yaml | 40 ++++++++-------- manifests/postgresql.crd.yaml | 2 + pkg/apis/acid.zalan.do/v1/crds.go | 3 ++ pkg/apis/acid.zalan.do/v1/postgresql_type.go | 1 + .../acid.zalan.do/v1/zz_generated.deepcopy.go | 29 ++++++++++++ pkg/cluster/k8sres.go | 47 ++++++++++++------- pkg/cluster/k8sres_test.go | 18 +++++-- 11 files changed, 150 insertions(+), 45 deletions(-) diff --git a/.gitignore b/.gitignore index 991fe754f..b407c62f1 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,8 @@ _obj _test _manifests +_tmp +github.com # Architecture specific extensions/prefixes *.[568vq] diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 3c666b9ab..78850ee3b 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -364,6 +364,21 @@ spec: type: string teamId: type: string + tls: + type: object + required: + - secretName + properties: + secretName: + type: string + certificateFile: + type: string + privateKeyFile: + type: string + caFile: + type: string + caSecretName: + type: string tolerations: type: array items: diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 361e32780..c87728812 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -435,5 +435,12 @@ Those parameters are grouped under the `tls` top-level key. client connects with `sslmode=verify-ca` or `sslmode=verify-full`. Default is empty. +* **caSecretName** + By setting the `caSecretName` value, the ca certificate file defined by the + `caFile` will be fetched from this secret instead of `secretName` above. + This secret has to hold a file with that name in its root. + Optionally one can provide full path for any of them. By default it is relative to the "/tls/", which is mount path of the tls secret. + If `caSecretName` is defined, the ca.crt path is relative to "/tlsca/", + otherwise to the same "/tls/". diff --git a/docs/user.md b/docs/user.md index bb12fd2e1..2c1c4fd1f 100644 --- a/docs/user.md +++ b/docs/user.md @@ -585,7 +585,7 @@ OpenShift allocates the users and groups dynamically (based on scc), and their range is different in every namespace. Due to this dynamic behaviour, it's not trivial to know at deploy time the uid/gid of the user in the cluster. Therefore, instead of using a global `spilo_fsgroup` setting, use the `spiloFSGroup` field -per Postgres cluster.``` +per Postgres cluster. Upload the cert as a kubernetes secret: ```sh @@ -594,7 +594,7 @@ kubectl create secret tls pg-tls \ --cert pg-tls.crt ``` -Or with a CA: +When doing client auth, CA can come optionally from the same secret: ```sh kubectl create secret generic pg-tls \ --from-file=tls.crt=server.crt \ @@ -602,9 +602,6 @@ kubectl create secret generic pg-tls \ --from-file=ca.crt=ca.crt ``` -Alternatively it is also possible to use -[cert-manager](https://cert-manager.io/docs/) to generate these secrets. - Then configure the postgres resource with the TLS secret: ```yaml @@ -619,5 +616,29 @@ spec: caFile: "ca.crt" # add this if the secret is configured with a CA ``` +Optionally, the CA can be provided by a different secret: +```sh +kubectl create secret generic pg-tls-ca \ + --from-file=ca.crt=ca.crt +``` + +Then configure the postgres resource with the TLS secret: + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: postgresql + +metadata: + name: acid-test-cluster +spec: + tls: + secretName: "pg-tls" # this should hold tls.key and tls.crt + caSecretName: "pg-tls-ca" # this should hold ca.crt + caFile: "ca.crt" # add this if the secret is configured with a CA +``` + +Alternatively, it is also possible to use +[cert-manager](https://cert-manager.io/docs/) to generate these secrets. + Certificate rotation is handled in the spilo image which checks every 5 minutes if the certificates have changed and reloads postgres accordingly. diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 23b0b6096..b469a7564 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -9,6 +9,24 @@ metadata: spec: dockerImage: registry.opensource.zalan.do/acid/spilo-12:1.6-p2 teamId: "acid" + numberOfInstances: 2 + users: # Application/Robot users + zalando: + - superuser + - createdb + enableMasterLoadBalancer: false + enableReplicaLoadBalancer: false +# enableConnectionPooler: true # not needed when connectionPooler section is present (see below) + allowedSourceRanges: # load balancers' source ranges for both master and replica services + - 127.0.0.1/32 + databases: + foo: zalando + postgresql: + version: "12" + parameters: # Expert section + shared_buffers: "32MB" + max_connections: "10" + log_statement: "all" volume: size: 1Gi # storageClass: my-sc @@ -35,24 +53,6 @@ spec: - all volumeSource: emptyDir: {} - numberOfInstances: 2 - users: # Application/Robot users - zalando: - - superuser - - createdb - enableMasterLoadBalancer: false - enableReplicaLoadBalancer: false -# enableConnectionPooler: true # not needed when connectionPooler section is present (see below) - allowedSourceRanges: # load balancers' source ranges for both master and replica services - - 127.0.0.1/32 - databases: - foo: zalando - postgresql: - version: "12" - parameters: # Expert section - shared_buffers: "32MB" - max_connections: "10" - log_statement: "all" enableShmVolume: true # spiloFSGroup: 103 @@ -148,8 +148,10 @@ spec: certificateFile: "tls.crt" privateKeyFile: "tls.key" caFile: "" # optionally configure Postgres with a CA certificate + caSecretName: "" # optionally the ca.crt can come from this secret instead. # file names can be also defined with absolute path, and will no longer be relative -# to the "/tls/" path where the secret is being mounted by default. +# to the "/tls/" path where the secret is being mounted by default, and "/tlsca/" +# where the caSecret is mounted by default. # When TLS is enabled, also set spiloFSGroup parameter above to the relevant value. # if unknown, set it to 103 which is the usual value in the default spilo images. # In Openshift, there is no need to set spiloFSGroup/spilo_fsgroup. diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index c9d60d60a..1ee6a1ae5 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -341,6 +341,8 @@ spec: type: string caFile: type: string + caSecretName: + type: string tolerations: type: array items: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index d693d0e15..3f4314240 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -513,6 +513,9 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ "caFile": { Type: "string", }, + "caSecretName": { + Type: "string", + }, }, }, "tolerations": { diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 04b70cba8..961051c8d 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -148,6 +148,7 @@ type TLSDescription struct { CertificateFile string `json:"certificateFile,omitempty"` PrivateKeyFile string `json:"privateKeyFile,omitempty"` CAFile string `json:"caFile,omitempty"` + CASecretName string `json:"caSecretName,omitempty"` } // CloneDescription describes which cluster the new should clone and up to which point in time diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 92c8af34b..e6b387ec4 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -47,6 +47,28 @@ func (in *AWSGCPConfiguration) DeepCopy() *AWSGCPConfiguration { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AdditionalVolume) DeepCopyInto(out *AdditionalVolume) { + *out = *in + if in.TargetContainers != nil { + in, out := &in.TargetContainers, &out.TargetContainers + *out = make([]string, len(*in)) + copy(*out, *in) + } + in.VolumeSource.DeepCopyInto(&out.VolumeSource) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AdditionalVolume. +func (in *AdditionalVolume) DeepCopy() *AdditionalVolume { + if in == nil { + return nil + } + out := new(AdditionalVolume) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CloneDescription) DeepCopyInto(out *CloneDescription) { *out = *in @@ -591,6 +613,13 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { *out = new(TLSDescription) **out = **in } + if in.AdditionalVolumes != nil { + in, out := &in.AdditionalVolumes, &out.AdditionalVolumes + *out = make([]AdditionalVolume, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.InitContainersOld != nil { in, out := &in.InitContainersOld, &out.InitContainersOld *out = make([]corev1.Container, len(*in)) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index e6f53af9d..9fb33eab2 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -519,7 +519,6 @@ func (c *Cluster) generatePodTemplate( podAntiAffinityTopologyKey string, additionalSecretMount string, additionalSecretMountPath string, - volumes []v1.Volume, additionalVolumes []acidv1.AdditionalVolume, ) (*v1.PodTemplateSpec, error) { @@ -539,7 +538,6 @@ func (c *Cluster) generatePodTemplate( InitContainers: initContainers, Tolerations: *tolerationsSpec, SecurityContext: &securityContext, - Volumes: volumes, } if shmVolume != nil && *shmVolume { @@ -854,7 +852,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef sidecarContainers []v1.Container podTemplate *v1.PodTemplateSpec volumeClaimTemplate *v1.PersistentVolumeClaim - volumes []v1.Volume + additionalVolumes = spec.AdditionalVolumes ) // Improve me. Please. @@ -1007,8 +1005,10 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef // this is combined with the FSGroup in the section above // to give read access to the postgres user defaultMode := int32(0640) - volumes = append(volumes, v1.Volume{ - Name: "tls-secret", + mountPath := "/tls" + additionalVolumes = append(additionalVolumes, acidv1.AdditionalVolume{ + Name: spec.TLS.SecretName, + MountPath: mountPath, VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ SecretName: spec.TLS.SecretName, @@ -1017,13 +1017,6 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef }, }) - mountPath := "/tls" - volumeMounts = append(volumeMounts, v1.VolumeMount{ - MountPath: mountPath, - Name: "tls-secret", - ReadOnly: true, - }) - // use the same filenames as Secret resources by default certFile := ensurePath(spec.TLS.CertificateFile, mountPath, "tls.crt") privateKeyFile := ensurePath(spec.TLS.PrivateKeyFile, mountPath, "tls.key") @@ -1034,11 +1027,31 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef ) if spec.TLS.CAFile != "" { - caFile := ensurePath(spec.TLS.CAFile, mountPath, "") + // support scenario when the ca.crt resides in a different secret, diff path + mountPathCA := mountPath + if spec.TLS.CASecretName != "" { + mountPathCA = mountPath + "ca" + } + + caFile := ensurePath(spec.TLS.CAFile, mountPathCA, "") spiloEnvVars = append( spiloEnvVars, v1.EnvVar{Name: "SSL_CA_FILE", Value: caFile}, ) + + // the ca file from CASecretName secret takes priority + if spec.TLS.CASecretName != "" { + additionalVolumes = append(additionalVolumes, acidv1.AdditionalVolume{ + Name: spec.TLS.CASecretName, + MountPath: mountPathCA, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: spec.TLS.CASecretName, + DefaultMode: &defaultMode, + }, + }, + }) + } } } @@ -1108,8 +1121,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef c.OpConfig.PodAntiAffinityTopologyKey, c.OpConfig.AdditionalSecretMount, c.OpConfig.AdditionalSecretMountPath, - volumes, - spec.AdditionalVolumes) + additionalVolumes) if err != nil { return nil, fmt.Errorf("could not generate pod template: %v", err) @@ -1614,11 +1626,11 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription) c.logger.Info(msg, description.S3WalPath) envs := []v1.EnvVar{ - v1.EnvVar{ + { Name: "CLONE_WAL_S3_BUCKET", Value: c.OpConfig.WALES3Bucket, }, - v1.EnvVar{ + { Name: "CLONE_WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(description.UID), }, @@ -1790,7 +1802,6 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) { "", c.OpConfig.AdditionalSecretMount, c.OpConfig.AdditionalSecretMountPath, - nil, []acidv1.AdditionalVolume{}); err != nil { return nil, fmt.Errorf("could not generate pod template for logical backup pod: %v", err) } diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 5b55e988c..6e4587627 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -961,6 +961,7 @@ func TestTLS(t *testing.T) { var spec acidv1.PostgresSpec var cluster *Cluster var spiloFSGroup = int64(103) + var additionalVolumes = spec.AdditionalVolumes makeSpec := func(tls acidv1.TLSDescription) acidv1.PostgresSpec { return acidv1.PostgresSpec{ @@ -1000,8 +1001,20 @@ func TestTLS(t *testing.T) { assert.Equal(t, &fsGroup, s.Spec.Template.Spec.SecurityContext.FSGroup, "has a default FSGroup assigned") defaultMode := int32(0640) + mountPath := "/tls" + additionalVolumes = append(additionalVolumes, acidv1.AdditionalVolume{ + Name: spec.TLS.SecretName, + MountPath: mountPath, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: spec.TLS.SecretName, + DefaultMode: &defaultMode, + }, + }, + }) + volume := v1.Volume{ - Name: "tls-secret", + Name: "my-secret", VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ SecretName: "my-secret", @@ -1013,8 +1026,7 @@ func TestTLS(t *testing.T) { assert.Contains(t, s.Spec.Template.Spec.Containers[0].VolumeMounts, v1.VolumeMount{ MountPath: "/tls", - Name: "tls-secret", - ReadOnly: true, + Name: "my-secret", }, "the volume gets mounted in /tls") assert.Contains(t, s.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "SSL_CERTIFICATE_FILE", Value: "/tls/tls.crt"}) From 6a689cdc1c15bef7498cd29c0ad3e79a5f840996 Mon Sep 17 00:00:00 2001 From: Dmitry Dolgov <9erthalion6@gmail.com> Date: Thu, 16 Apr 2020 15:14:31 +0200 Subject: [PATCH 028/168] Prevent empty syncs (#922) There is a possibility to pass nil as one of the specs and an empty spec into syncConnectionPooler. In this case it will perfom a syncronization because nil != empty struct. Avoid such cases and make it testable by returning list of syncronization reasons on top together with the final error. --- pkg/cluster/cluster.go | 3 +- pkg/cluster/sync.go | 64 +++++++++++++++++-------- pkg/cluster/sync_test.go | 101 +++++++++++++++++++++++++++++---------- pkg/cluster/types.go | 5 ++ 4 files changed, 127 insertions(+), 46 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 51dd368fb..83c1bc157 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -741,7 +741,8 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { } // sync connection pooler - if err := c.syncConnectionPooler(oldSpec, newSpec, c.installLookupFunction); err != nil { + if _, err := c.syncConnectionPooler(oldSpec, newSpec, + c.installLookupFunction); err != nil { return fmt.Errorf("could not sync connection pooler: %v", err) } diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index a423c2f0e..361673891 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -111,7 +111,7 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { } // sync connection pooler - if err = c.syncConnectionPooler(&oldSpec, newSpec, c.installLookupFunction); err != nil { + if _, err = c.syncConnectionPooler(&oldSpec, newSpec, c.installLookupFunction); err != nil { return fmt.Errorf("could not sync connection pooler: %v", err) } @@ -620,7 +620,13 @@ func (c *Cluster) syncLogicalBackupJob() error { return nil } -func (c *Cluster) syncConnectionPooler(oldSpec, newSpec *acidv1.Postgresql, lookup InstallFunction) error { +func (c *Cluster) syncConnectionPooler(oldSpec, + newSpec *acidv1.Postgresql, + lookup InstallFunction) (SyncReason, error) { + + var reason SyncReason + var err error + if c.ConnectionPooler == nil { c.ConnectionPooler = &ConnectionPoolerObjects{} } @@ -657,20 +663,20 @@ func (c *Cluster) syncConnectionPooler(oldSpec, newSpec *acidv1.Postgresql, look specUser, c.OpConfig.ConnectionPooler.User) - if err := lookup(schema, user); err != nil { - return err + if err = lookup(schema, user); err != nil { + return NoSync, err } } - if err := c.syncConnectionPoolerWorker(oldSpec, newSpec); err != nil { + if reason, err = c.syncConnectionPoolerWorker(oldSpec, newSpec); err != nil { c.logger.Errorf("could not sync connection pooler: %v", err) - return err + return reason, err } } if oldNeedConnectionPooler && !newNeedConnectionPooler { // delete and cleanup resources - if err := c.deleteConnectionPooler(); err != nil { + if err = c.deleteConnectionPooler(); err != nil { c.logger.Warningf("could not remove connection pooler: %v", err) } } @@ -681,20 +687,22 @@ func (c *Cluster) syncConnectionPooler(oldSpec, newSpec *acidv1.Postgresql, look (c.ConnectionPooler.Deployment != nil || c.ConnectionPooler.Service != nil) { - if err := c.deleteConnectionPooler(); err != nil { + if err = c.deleteConnectionPooler(); err != nil { c.logger.Warningf("could not remove connection pooler: %v", err) } } } - return nil + return reason, nil } // Synchronize connection pooler resources. Effectively we're interested only in // synchronizing the corresponding deployment, but in case of deployment or // service is missing, create it. After checking, also remember an object for // the future references. -func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql) error { +func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql) ( + SyncReason, error) { + deployment, err := c.KubeClient. Deployments(c.Namespace). Get(context.TODO(), c.connectionPoolerName(), metav1.GetOptions{}) @@ -706,7 +714,7 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql deploymentSpec, err := c.generateConnectionPoolerDeployment(&newSpec.Spec) if err != nil { msg = "could not generate deployment for connection pooler: %v" - return fmt.Errorf(msg, err) + return NoSync, fmt.Errorf(msg, err) } deployment, err := c.KubeClient. @@ -714,18 +722,35 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql Create(context.TODO(), deploymentSpec, metav1.CreateOptions{}) if err != nil { - return err + return NoSync, err } c.ConnectionPooler.Deployment = deployment } else if err != nil { - return fmt.Errorf("could not get connection pooler deployment to sync: %v", err) + msg := "could not get connection pooler deployment to sync: %v" + return NoSync, fmt.Errorf(msg, err) } else { c.ConnectionPooler.Deployment = deployment // actual synchronization oldConnectionPooler := oldSpec.Spec.ConnectionPooler newConnectionPooler := newSpec.Spec.ConnectionPooler + + // sync implementation below assumes that both old and new specs are + // not nil, but it can happen. To avoid any confusion like updating a + // deployment because the specification changed from nil to an empty + // struct (that was initialized somewhere before) replace any nil with + // an empty spec. + if oldConnectionPooler == nil { + oldConnectionPooler = &acidv1.ConnectionPooler{} + } + + if newConnectionPooler == nil { + newConnectionPooler = &acidv1.ConnectionPooler{} + } + + c.logger.Infof("Old: %+v, New %+v", oldConnectionPooler, newConnectionPooler) + specSync, specReason := c.needSyncConnectionPoolerSpecs(oldConnectionPooler, newConnectionPooler) defaultsSync, defaultsReason := c.needSyncConnectionPoolerDefaults(newConnectionPooler, deployment) reason := append(specReason, defaultsReason...) @@ -736,7 +761,7 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql newDeploymentSpec, err := c.generateConnectionPoolerDeployment(&newSpec.Spec) if err != nil { msg := "could not generate deployment for connection pooler: %v" - return fmt.Errorf(msg, err) + return reason, fmt.Errorf(msg, err) } oldDeploymentSpec := c.ConnectionPooler.Deployment @@ -746,11 +771,11 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql newDeploymentSpec) if err != nil { - return err + return reason, err } c.ConnectionPooler.Deployment = deployment - return nil + return reason, nil } } @@ -768,16 +793,17 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql Create(context.TODO(), serviceSpec, metav1.CreateOptions{}) if err != nil { - return err + return NoSync, err } c.ConnectionPooler.Service = service } else if err != nil { - return fmt.Errorf("could not get connection pooler service to sync: %v", err) + msg := "could not get connection pooler service to sync: %v" + return NoSync, fmt.Errorf(msg, err) } else { // Service updates are not supported and probably not that useful anyway c.ConnectionPooler.Service = service } - return nil + return NoSync, nil } diff --git a/pkg/cluster/sync_test.go b/pkg/cluster/sync_test.go index 45355cca3..50b5cfaa8 100644 --- a/pkg/cluster/sync_test.go +++ b/pkg/cluster/sync_test.go @@ -2,6 +2,7 @@ package cluster import ( "fmt" + "strings" "testing" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" @@ -17,7 +18,7 @@ func int32ToPointer(value int32) *int32 { return &value } -func deploymentUpdated(cluster *Cluster, err error) error { +func deploymentUpdated(cluster *Cluster, err error, reason SyncReason) error { if cluster.ConnectionPooler.Deployment.Spec.Replicas == nil || *cluster.ConnectionPooler.Deployment.Spec.Replicas != 2 { return fmt.Errorf("Wrong nubmer of instances") @@ -26,7 +27,7 @@ func deploymentUpdated(cluster *Cluster, err error) error { return nil } -func objectsAreSaved(cluster *Cluster, err error) error { +func objectsAreSaved(cluster *Cluster, err error, reason SyncReason) error { if cluster.ConnectionPooler == nil { return fmt.Errorf("Connection pooler resources are empty") } @@ -42,7 +43,7 @@ func objectsAreSaved(cluster *Cluster, err error) error { return nil } -func objectsAreDeleted(cluster *Cluster, err error) error { +func objectsAreDeleted(cluster *Cluster, err error, reason SyncReason) error { if cluster.ConnectionPooler != nil { return fmt.Errorf("Connection pooler was not deleted") } @@ -50,6 +51,16 @@ func objectsAreDeleted(cluster *Cluster, err error) error { return nil } +func noEmptySync(cluster *Cluster, err error, reason SyncReason) error { + for _, msg := range reason { + if strings.HasPrefix(msg, "update [] from '' to '") { + return fmt.Errorf("There is an empty reason, %s", msg) + } + } + + return nil +} + func TestConnectionPoolerSynchronization(t *testing.T) { testName := "Test connection pooler synchronization" var cluster = New( @@ -91,15 +102,15 @@ func TestConnectionPoolerSynchronization(t *testing.T) { clusterNewDefaultsMock := *cluster clusterNewDefaultsMock.KubeClient = k8sutil.NewMockKubernetesClient() - cluster.OpConfig.ConnectionPooler.Image = "pooler:2.0" - cluster.OpConfig.ConnectionPooler.NumberOfInstances = int32ToPointer(2) tests := []struct { - subTest string - oldSpec *acidv1.Postgresql - newSpec *acidv1.Postgresql - cluster *Cluster - check func(cluster *Cluster, err error) error + subTest string + oldSpec *acidv1.Postgresql + newSpec *acidv1.Postgresql + cluster *Cluster + defaultImage string + defaultInstances int32 + check func(cluster *Cluster, err error, reason SyncReason) error }{ { subTest: "create if doesn't exist", @@ -113,8 +124,10 @@ func TestConnectionPoolerSynchronization(t *testing.T) { ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, - cluster: &clusterMissingObjects, - check: objectsAreSaved, + cluster: &clusterMissingObjects, + defaultImage: "pooler:1.0", + defaultInstances: 1, + check: objectsAreSaved, }, { subTest: "create if doesn't exist with a flag", @@ -126,8 +139,10 @@ func TestConnectionPoolerSynchronization(t *testing.T) { EnableConnectionPooler: boolToPointer(true), }, }, - cluster: &clusterMissingObjects, - check: objectsAreSaved, + cluster: &clusterMissingObjects, + defaultImage: "pooler:1.0", + defaultInstances: 1, + check: objectsAreSaved, }, { subTest: "create from scratch", @@ -139,8 +154,10 @@ func TestConnectionPoolerSynchronization(t *testing.T) { ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, - cluster: &clusterMissingObjects, - check: objectsAreSaved, + cluster: &clusterMissingObjects, + defaultImage: "pooler:1.0", + defaultInstances: 1, + check: objectsAreSaved, }, { subTest: "delete if not needed", @@ -152,8 +169,10 @@ func TestConnectionPoolerSynchronization(t *testing.T) { newSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{}, }, - cluster: &clusterMock, - check: objectsAreDeleted, + cluster: &clusterMock, + defaultImage: "pooler:1.0", + defaultInstances: 1, + check: objectsAreDeleted, }, { subTest: "cleanup if still there", @@ -163,8 +182,10 @@ func TestConnectionPoolerSynchronization(t *testing.T) { newSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{}, }, - cluster: &clusterDirtyMock, - check: objectsAreDeleted, + cluster: &clusterDirtyMock, + defaultImage: "pooler:1.0", + defaultInstances: 1, + check: objectsAreDeleted, }, { subTest: "update deployment", @@ -182,8 +203,10 @@ func TestConnectionPoolerSynchronization(t *testing.T) { }, }, }, - cluster: &clusterMock, - check: deploymentUpdated, + cluster: &clusterMock, + defaultImage: "pooler:1.0", + defaultInstances: 1, + check: deploymentUpdated, }, { subTest: "update image from changed defaults", @@ -197,14 +220,40 @@ func TestConnectionPoolerSynchronization(t *testing.T) { ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, - cluster: &clusterNewDefaultsMock, - check: deploymentUpdated, + cluster: &clusterNewDefaultsMock, + defaultImage: "pooler:2.0", + defaultInstances: 2, + check: deploymentUpdated, + }, + { + subTest: "there is no sync from nil to an empty spec", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + EnableConnectionPooler: boolToPointer(true), + ConnectionPooler: nil, + }, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + EnableConnectionPooler: boolToPointer(true), + ConnectionPooler: &acidv1.ConnectionPooler{}, + }, + }, + cluster: &clusterMock, + defaultImage: "pooler:1.0", + defaultInstances: 1, + check: noEmptySync, }, } for _, tt := range tests { - err := tt.cluster.syncConnectionPooler(tt.oldSpec, tt.newSpec, mockInstallLookupFunction) + tt.cluster.OpConfig.ConnectionPooler.Image = tt.defaultImage + tt.cluster.OpConfig.ConnectionPooler.NumberOfInstances = + int32ToPointer(tt.defaultInstances) - if err := tt.check(tt.cluster, err); err != nil { + reason, err := tt.cluster.syncConnectionPooler(tt.oldSpec, + tt.newSpec, mockInstallLookupFunction) + + if err := tt.check(tt.cluster, err, reason); err != nil { t.Errorf("%s [%s]: Could not synchronize, %+v", testName, tt.subTest, err) } diff --git a/pkg/cluster/types.go b/pkg/cluster/types.go index 04d00cb58..199914ccc 100644 --- a/pkg/cluster/types.go +++ b/pkg/cluster/types.go @@ -73,3 +73,8 @@ type ClusterStatus struct { type TemplateParams map[string]interface{} type InstallFunction func(schema string, user string) error + +type SyncReason []string + +// no sync happened, empty value +var NoSync SyncReason = []string{} From 5014eebfb2261d778ac1ed8f8da103e474d85d61 Mon Sep 17 00:00:00 2001 From: ReSearchITEng Date: Thu, 16 Apr 2020 17:47:59 +0300 Subject: [PATCH 029/168] when kubernetes_use_configmaps -> skip further endpoints actions even delete (#921) * further compatibility with k8sUseConfigMaps - skip further endpoints related actions * Update pkg/cluster/cluster.go thanks! Co-Authored-By: Felix Kunde * Update pkg/cluster/cluster.go Co-Authored-By: Felix Kunde * Update pkg/cluster/cluster.go Co-authored-by: Felix Kunde --- pkg/cluster/cluster.go | 18 ++++++++++++++---- pkg/cluster/sync.go | 7 ++++--- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 83c1bc157..74cf6e61d 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -784,8 +784,10 @@ func (c *Cluster) Delete() { for _, role := range []PostgresRole{Master, Replica} { - if err := c.deleteEndpoint(role); err != nil { - c.logger.Warningf("could not delete %s endpoint: %v", role, err) + if !c.patroniKubernetesUseConfigMaps() { + if err := c.deleteEndpoint(role); err != nil { + c.logger.Warningf("could not delete %s endpoint: %v", role, err) + } } if err := c.deleteService(role); err != nil { @@ -1161,11 +1163,19 @@ type clusterObjectDelete func(name string) error func (c *Cluster) deletePatroniClusterObjects() error { // TODO: figure out how to remove leftover patroni objects in other cases + var actionsList []simpleActionWithResult + if !c.patroniUsesKubernetes() { c.logger.Infof("not cleaning up Etcd Patroni objects on cluster delete") } - c.logger.Debugf("removing leftover Patroni objects (endpoints, services and configmaps)") - for _, deleter := range []simpleActionWithResult{c.deletePatroniClusterEndpoints, c.deletePatroniClusterServices, c.deletePatroniClusterConfigMaps} { + + if !c.patroniKubernetesUseConfigMaps() { + actionsList = append(actionsList, c.deletePatroniClusterEndpoints) + } + actionsList = append(actionsList, c.deletePatroniClusterServices, c.deletePatroniClusterConfigMaps) + + c.logger.Debugf("removing leftover Patroni objects (endpoints / services and configmaps)") + for _, deleter := range actionsList { if err := deleter(); err != nil { return err } diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 361673891..43173102b 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -122,10 +122,11 @@ func (c *Cluster) syncServices() error { for _, role := range []PostgresRole{Master, Replica} { c.logger.Debugf("syncing %s service", role) - if err := c.syncEndpoint(role); err != nil { - return fmt.Errorf("could not sync %s endpoint: %v", role, err) + if !c.patroniKubernetesUseConfigMaps() { + if err := c.syncEndpoint(role); err != nil { + return fmt.Errorf("could not sync %s endpoint: %v", role, err) + } } - if err := c.syncService(role); err != nil { return fmt.Errorf("could not sync %s service: %v", role, err) } From 3c91bdeffadb5ec736a63548b5ffa08517c59de8 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Mon, 20 Apr 2020 15:14:11 +0200 Subject: [PATCH 030/168] Re-create pods only if all replicas are running (#903) * adds a Get call to Patroni interface to fetch state of a Patroni member * postpones re-creating pods if at least one replica is currently being created Co-authored-by: Sergey Dudoladov Co-authored-by: Felix Kunde --- .gitignore | 1 + e2e/tests/test_e2e.py | 9 +++++---- pkg/cluster/pod.go | 25 +++++++++++++++++++++++++ pkg/util/patroni/patroni.go | 37 ++++++++++++++++++++++++++++++++++++- 4 files changed, 67 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index b407c62f1..0fdb50756 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,7 @@ _testmain.go /vendor/ /build/ /docker/build/ +/github.com/ .idea scm-source.json diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 445067d61..f46c0577e 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -344,7 +344,6 @@ class EndToEndTestCase(unittest.TestCase): ''' k8s = self.k8s cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' - labels = 'spilo-role=master,' + cluster_label readiness_label = 'lifecycle-status' readiness_value = 'ready' @@ -709,14 +708,16 @@ class K8s: def wait_for_logical_backup_job_creation(self): self.wait_for_logical_backup_job(expected_num_of_jobs=1) - def update_config(self, config_map_patch): - self.api.core_v1.patch_namespaced_config_map("postgres-operator", "default", config_map_patch) - + def delete_operator_pod(self): operator_pod = self.api.core_v1.list_namespaced_pod( 'default', label_selector="name=postgres-operator").items[0].metadata.name self.api.core_v1.delete_namespaced_pod(operator_pod, "default") # restart reloads the conf self.wait_for_operator_pod_start() + def update_config(self, config_map_patch): + self.api.core_v1.patch_namespaced_config_map("postgres-operator", "default", config_map_patch) + self.delete_operator_pod() + def create_with_kubectl(self, path): return subprocess.run( ["kubectl", "create", "-f", path], diff --git a/pkg/cluster/pod.go b/pkg/cluster/pod.go index 9991621cc..a734e4835 100644 --- a/pkg/cluster/pod.go +++ b/pkg/cluster/pod.go @@ -294,6 +294,27 @@ func (c *Cluster) recreatePod(podName spec.NamespacedName) (*v1.Pod, error) { return pod, nil } +func (c *Cluster) isSafeToRecreatePods(pods *v1.PodList) bool { + + /* + Operator should not re-create pods if there is at least one replica being bootstrapped + because Patroni might use other replicas to take basebackup from (see Patroni's "clonefrom" tag). + + XXX operator cannot forbid replica re-init, so we might still fail if re-init is started + after this check succeeds but before a pod is re-created + */ + + for _, pod := range pods.Items { + state, err := c.patroni.GetPatroniMemberState(&pod) + if err != nil || state == "creating replica" { + c.logger.Warningf("cannot re-create replica %s: it is currently being initialized", pod.Name) + return false + } + + } + return true +} + func (c *Cluster) recreatePods() error { c.setProcessName("starting to recreate pods") ls := c.labelsSet(false) @@ -309,6 +330,10 @@ func (c *Cluster) recreatePods() error { } c.logger.Infof("there are %d pods in the cluster to recreate", len(pods.Items)) + if !c.isSafeToRecreatePods(pods) { + return fmt.Errorf("postpone pod recreation until next Sync: recreation is unsafe because pods are being initilalized") + } + var ( masterPod, newMasterPod, newPod *v1.Pod ) diff --git a/pkg/util/patroni/patroni.go b/pkg/util/patroni/patroni.go index bdd96f048..53065e599 100644 --- a/pkg/util/patroni/patroni.go +++ b/pkg/util/patroni/patroni.go @@ -3,6 +3,7 @@ package patroni import ( "bytes" "encoding/json" + "errors" "fmt" "io/ioutil" "net" @@ -11,7 +12,7 @@ import ( "time" "github.com/sirupsen/logrus" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" ) const ( @@ -25,6 +26,7 @@ const ( type Interface interface { Switchover(master *v1.Pod, candidate string) error SetPostgresParameters(server *v1.Pod, options map[string]string) error + GetPatroniMemberState(pod *v1.Pod) (string, error) } // Patroni API client @@ -123,3 +125,36 @@ func (p *Patroni) SetPostgresParameters(server *v1.Pod, parameters map[string]st } return p.httpPostOrPatch(http.MethodPatch, apiURLString+configPath, buf) } + +//GetPatroniMemberState returns a state of member of a Patroni cluster +func (p *Patroni) GetPatroniMemberState(server *v1.Pod) (string, error) { + + apiURLString, err := apiURL(server) + if err != nil { + return "", err + } + response, err := p.httpClient.Get(apiURLString) + if err != nil { + return "", fmt.Errorf("could not perform Get request: %v", err) + } + defer response.Body.Close() + + body, err := ioutil.ReadAll(response.Body) + if err != nil { + return "", fmt.Errorf("could not read response: %v", err) + } + + data := make(map[string]interface{}) + err = json.Unmarshal(body, &data) + if err != nil { + return "", err + } + + state, ok := data["state"].(string) + if !ok { + return "", errors.New("Patroni Get call response contains wrong type for 'state' field") + } + + return state, nil + +} From 21b9b6fcbe5dc03eb52d76f302cfdb4cd36c80de Mon Sep 17 00:00:00 2001 From: Christian Rohmann Date: Mon, 27 Apr 2020 08:22:07 +0200 Subject: [PATCH 031/168] Emit K8S events to the postgresql CR as feedback to the requestor / user (#896) * Add EventsGetter to KubeClient to enable to sending K8S events * Add eventRecorder to the controller, initialize it and hand it down to cluster via its constructor to enable it to emit events this way * Add first set of events which then go to the postgresql custom resource the user interacts with to provide some feedback * Add right to "create" events to operator cluster role * Adapt cluster tests to new function sigurature with eventRecord (via NewFakeRecorder) * Get a proper reference before sending events to a resource Co-authored-by: Christian Rohmann --- .../templates/clusterrole.yaml | 7 ++++ manifests/operator-service-account-rbac.yaml | 7 ++++ pkg/cluster/cluster.go | 32 +++++++++++++++- pkg/cluster/cluster_test.go | 4 ++ pkg/cluster/k8sres_test.go | 32 +++++++++------- pkg/cluster/resources_test.go | 4 +- pkg/cluster/sync.go | 2 + pkg/cluster/sync_test.go | 2 +- pkg/controller/controller.go | 37 ++++++++++++++----- pkg/controller/postgresql.go | 7 +++- pkg/util/k8sutil/k8sutil.go | 2 + 11 files changed, 107 insertions(+), 29 deletions(-) diff --git a/charts/postgres-operator/templates/clusterrole.yaml b/charts/postgres-operator/templates/clusterrole.yaml index 38ce85e7a..0defcab41 100644 --- a/charts/postgres-operator/templates/clusterrole.yaml +++ b/charts/postgres-operator/templates/clusterrole.yaml @@ -42,6 +42,13 @@ rules: - configmaps verbs: - get +# to send events to the CRs +- apiGroups: + - "" + resources: + - events + verbs: + - create # to manage endpoints which are also used by Patroni - apiGroups: - "" diff --git a/manifests/operator-service-account-rbac.yaml b/manifests/operator-service-account-rbac.yaml index 83cd721e7..667941a24 100644 --- a/manifests/operator-service-account-rbac.yaml +++ b/manifests/operator-service-account-rbac.yaml @@ -43,6 +43,13 @@ rules: - configmaps verbs: - get +# to send events to the CRs +- apiGroups: + - "" + resources: + - events + verbs: + - create # to manage endpoints which are also used by Patroni - apiGroups: - "" diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 74cf6e61d..244916074 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -21,8 +21,11 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/tools/reference" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/scheme" "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" @@ -81,6 +84,7 @@ type Cluster struct { acidv1.Postgresql Config logger *logrus.Entry + eventRecorder record.EventRecorder patroni patroni.Interface pgUsers map[string]spec.PgUser systemUsers map[string]spec.PgUser @@ -109,7 +113,7 @@ type compareStatefulsetResult struct { } // New creates a new cluster. This function should be called from a controller. -func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgresql, logger *logrus.Entry) *Cluster { +func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgresql, logger *logrus.Entry, eventRecorder record.EventRecorder) *Cluster { deletePropagationPolicy := metav1.DeletePropagationOrphan podEventsQueue := cache.NewFIFO(func(obj interface{}) (string, error) { @@ -140,7 +144,7 @@ func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgres cluster.teamsAPIClient = teams.NewTeamsAPI(cfg.OpConfig.TeamsAPIUrl, logger) cluster.oauthTokenGetter = newSecretOauthTokenGetter(&kubeClient, cfg.OpConfig.OAuthTokenSecretName) cluster.patroni = patroni.New(cluster.logger) - + cluster.eventRecorder = eventRecorder return cluster } @@ -166,6 +170,16 @@ func (c *Cluster) setProcessName(procName string, args ...interface{}) { } } +// GetReference of Postgres CR object +// i.e. required to emit events to this resource +func (c *Cluster) GetReference() *v1.ObjectReference { + ref, err := reference.GetReference(scheme.Scheme, &c.Postgresql) + if err != nil { + c.logger.Errorf("could not get reference for Postgresql CR %v/%v: %v", c.Postgresql.Namespace, c.Postgresql.Name, err) + } + return ref +} + // SetStatus of Postgres cluster // TODO: eventually switch to updateStatus() for kubernetes 1.11 and above func (c *Cluster) setStatus(status string) { @@ -245,6 +259,7 @@ func (c *Cluster) Create() error { }() c.setStatus(acidv1.ClusterStatusCreating) + c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Create", "Started creation of new cluster resources") if err = c.enforceMinResourceLimits(&c.Spec); err != nil { return fmt.Errorf("could not enforce minimum resource limits: %v", err) @@ -263,6 +278,7 @@ func (c *Cluster) Create() error { return fmt.Errorf("could not create %s endpoint: %v", role, err) } c.logger.Infof("endpoint %q has been successfully created", util.NameFromMeta(ep.ObjectMeta)) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Endpoints", "Endpoint %q has been successfully created", util.NameFromMeta(ep.ObjectMeta)) } if c.Services[role] != nil { @@ -273,6 +289,7 @@ func (c *Cluster) Create() error { return fmt.Errorf("could not create %s service: %v", role, err) } c.logger.Infof("%s service %q has been successfully created", role, util.NameFromMeta(service.ObjectMeta)) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Services", "The service %q for role %s has been successfully created", util.NameFromMeta(service.ObjectMeta), role) } if err = c.initUsers(); err != nil { @@ -284,6 +301,7 @@ func (c *Cluster) Create() error { return fmt.Errorf("could not create secrets: %v", err) } c.logger.Infof("secrets have been successfully created") + c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Secrets", "The secrets have been successfully created") if c.PodDisruptionBudget != nil { return fmt.Errorf("pod disruption budget already exists in the cluster") @@ -302,6 +320,7 @@ func (c *Cluster) Create() error { return fmt.Errorf("could not create statefulset: %v", err) } c.logger.Infof("statefulset %q has been successfully created", util.NameFromMeta(ss.ObjectMeta)) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "StatefulSet", "Statefulset %q has been successfully created", util.NameFromMeta(ss.ObjectMeta)) c.logger.Info("waiting for the cluster being ready") @@ -310,6 +329,7 @@ func (c *Cluster) Create() error { return err } c.logger.Infof("pods are ready") + c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "StatefulSet", "Pods are ready") // create database objects unless we are running without pods or disabled // that feature explicitly @@ -555,6 +575,7 @@ func (c *Cluster) enforceMinResourceLimits(spec *acidv1.PostgresSpec) error { } if isSmaller { c.logger.Warningf("defined CPU limit %s is below required minimum %s and will be set to it", cpuLimit, minCPULimit) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "ResourceLimits", "defined CPU limit %s is below required minimum %s and will be set to it", cpuLimit, minCPULimit) spec.Resources.ResourceLimits.CPU = minCPULimit } } @@ -567,6 +588,7 @@ func (c *Cluster) enforceMinResourceLimits(spec *acidv1.PostgresSpec) error { } if isSmaller { c.logger.Warningf("defined memory limit %s is below required minimum %s and will be set to it", memoryLimit, minMemoryLimit) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "ResourceLimits", "defined memory limit %s is below required minimum %s and will be set to it", memoryLimit, minMemoryLimit) spec.Resources.ResourceLimits.Memory = minMemoryLimit } } @@ -598,6 +620,8 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { if oldSpec.Spec.PostgresqlParam.PgVersion != newSpec.Spec.PostgresqlParam.PgVersion { // PG versions comparison c.logger.Warningf("postgresql version change(%q -> %q) has no effect", oldSpec.Spec.PostgresqlParam.PgVersion, newSpec.Spec.PostgresqlParam.PgVersion) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "PostgreSQL", "postgresql version change(%q -> %q) has no effect", + oldSpec.Spec.PostgresqlParam.PgVersion, newSpec.Spec.PostgresqlParam.PgVersion) //we need that hack to generate statefulset with the old version newSpec.Spec.PostgresqlParam.PgVersion = oldSpec.Spec.PostgresqlParam.PgVersion } @@ -757,6 +781,7 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { func (c *Cluster) Delete() { c.mu.Lock() defer c.mu.Unlock() + c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Delete", "Started deletion of new cluster resources") // delete the backup job before the stateful set of the cluster to prevent connections to non-existing pods // deleting the cron job also removes pods and batch jobs it created @@ -1095,6 +1120,7 @@ func (c *Cluster) Switchover(curMaster *v1.Pod, candidate spec.NamespacedName) e var err error c.logger.Debugf("switching over from %q to %q", curMaster.Name, candidate) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Switchover", "Switching over from %q to %q", curMaster.Name, candidate) var wg sync.WaitGroup @@ -1121,6 +1147,7 @@ func (c *Cluster) Switchover(curMaster *v1.Pod, candidate spec.NamespacedName) e if err = c.patroni.Switchover(curMaster, candidate.Name); err == nil { c.logger.Debugf("successfully switched over from %q to %q", curMaster.Name, candidate) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Switchover", "Successfully switched over from %q to %q", curMaster.Name, candidate) if err = <-podLabelErr; err != nil { err = fmt.Errorf("could not get master pod label: %v", err) } @@ -1136,6 +1163,7 @@ func (c *Cluster) Switchover(curMaster *v1.Pod, candidate spec.NamespacedName) e // close the label waiting channel no sooner than the waiting goroutine terminates. close(podLabelErr) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Switchover", "Switchover from %q to %q FAILED: %v", curMaster.Name, candidate, err) return err } diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index 432f53132..84ec04e3e 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -13,6 +13,7 @@ import ( "github.com/zalando/postgres-operator/pkg/util/k8sutil" "github.com/zalando/postgres-operator/pkg/util/teams" v1 "k8s.io/api/core/v1" + "k8s.io/client-go/tools/record" ) const ( @@ -21,6 +22,8 @@ const ( ) var logger = logrus.New().WithField("test", "cluster") +var eventRecorder = record.NewFakeRecorder(1) + var cl = New( Config{ OpConfig: config.Config{ @@ -34,6 +37,7 @@ var cl = New( k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger, + eventRecorder, ) func TestInitRobotUsers(t *testing.T) { diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 6e4587627..1291d4f47 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -37,7 +37,7 @@ func TestGenerateSpiloJSONConfiguration(t *testing.T) { ReplicationUsername: replicationUserName, }, }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) testName := "TestGenerateSpiloConfig" tests := []struct { @@ -102,7 +102,7 @@ func TestCreateLoadBalancerLogic(t *testing.T) { ReplicationUsername: replicationUserName, }, }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) testName := "TestCreateLoadBalancerLogic" tests := []struct { @@ -164,7 +164,8 @@ func TestGeneratePodDisruptionBudget(t *testing.T) { acidv1.Postgresql{ ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}}, - logger), + logger, + eventRecorder), policyv1beta1.PodDisruptionBudget{ ObjectMeta: metav1.ObjectMeta{ Name: "postgres-myapp-database-pdb", @@ -187,7 +188,8 @@ func TestGeneratePodDisruptionBudget(t *testing.T) { acidv1.Postgresql{ ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 0}}, - logger), + logger, + eventRecorder), policyv1beta1.PodDisruptionBudget{ ObjectMeta: metav1.ObjectMeta{ Name: "postgres-myapp-database-pdb", @@ -210,7 +212,8 @@ func TestGeneratePodDisruptionBudget(t *testing.T) { acidv1.Postgresql{ ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}}, - logger), + logger, + eventRecorder), policyv1beta1.PodDisruptionBudget{ ObjectMeta: metav1.ObjectMeta{ Name: "postgres-myapp-database-pdb", @@ -233,7 +236,8 @@ func TestGeneratePodDisruptionBudget(t *testing.T) { acidv1.Postgresql{ ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}}, - logger), + logger, + eventRecorder), policyv1beta1.PodDisruptionBudget{ ObjectMeta: metav1.ObjectMeta{ Name: "postgres-myapp-database-databass-budget", @@ -368,7 +372,7 @@ func TestCloneEnv(t *testing.T) { ReplicationUsername: replicationUserName, }, }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) for _, tt := range tests { envs := cluster.generateCloneEnvironment(tt.cloneOpts) @@ -502,7 +506,7 @@ func TestGetPgVersion(t *testing.T) { ReplicationUsername: replicationUserName, }, }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) for _, tt := range tests { pgVersion, err := cluster.getNewPgVersion(tt.pgContainer, tt.newPgVersion) @@ -678,7 +682,7 @@ func TestConnectionPoolerPodSpec(t *testing.T) { ConnectionPoolerDefaultMemoryLimit: "100Mi", }, }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) var clusterNoDefaultRes = New( Config{ @@ -690,7 +694,7 @@ func TestConnectionPoolerPodSpec(t *testing.T) { }, ConnectionPooler: config.ConnectionPooler{}, }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) noCheck := func(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { return nil } @@ -803,7 +807,7 @@ func TestConnectionPoolerDeploymentSpec(t *testing.T) { ConnectionPoolerDefaultMemoryLimit: "100Mi", }, }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) cluster.Statefulset = &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: "test-sts", @@ -904,7 +908,7 @@ func TestConnectionPoolerServiceSpec(t *testing.T) { ConnectionPoolerDefaultMemoryLimit: "100Mi", }, }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) cluster.Statefulset = &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: "test-sts", @@ -990,7 +994,7 @@ func TestTLS(t *testing.T) { SpiloFSGroup: &spiloFSGroup, }, }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) spec = makeSpec(acidv1.TLSDescription{SecretName: "my-secret", CAFile: "ca.crt"}) s, err := cluster.generateStatefulSet(&spec) if err != nil { @@ -1112,7 +1116,7 @@ func TestAdditionalVolume(t *testing.T) { ReplicationUsername: replicationUserName, }, }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) for _, tt := range tests { // Test with additional volume mounted in all containers diff --git a/pkg/cluster/resources_test.go b/pkg/cluster/resources_test.go index 2db807b38..9739cc354 100644 --- a/pkg/cluster/resources_test.go +++ b/pkg/cluster/resources_test.go @@ -36,7 +36,7 @@ func TestConnectionPoolerCreationAndDeletion(t *testing.T) { ConnectionPoolerDefaultMemoryLimit: "100Mi", }, }, - }, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger) + }, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger, eventRecorder) cluster.Statefulset = &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ @@ -85,7 +85,7 @@ func TestNeedConnectionPooler(t *testing.T) { ConnectionPoolerDefaultMemoryLimit: "100Mi", }, }, - }, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger) + }, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger, eventRecorder) cluster.Spec = acidv1.PostgresSpec{ ConnectionPooler: &acidv1.ConnectionPooler{}, diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 43173102b..a9a2b5177 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -343,10 +343,12 @@ func (c *Cluster) syncStatefulSet() error { // statefulset or those that got their configuration from the outdated statefulset) if podsRollingUpdateRequired { c.logger.Debugln("performing rolling update") + c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", "Performing rolling update") if err := c.recreatePods(); err != nil { return fmt.Errorf("could not recreate pods: %v", err) } c.logger.Infof("pods have been recreated") + c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", "Rolling update done - pods have been recreated") if err := c.applyRollingUpdateFlagforStatefulSet(false); err != nil { c.logger.Warningf("could not clear rolling update for the statefulset: %v", err) } diff --git a/pkg/cluster/sync_test.go b/pkg/cluster/sync_test.go index 50b5cfaa8..3a7317938 100644 --- a/pkg/cluster/sync_test.go +++ b/pkg/cluster/sync_test.go @@ -79,7 +79,7 @@ func TestConnectionPoolerSynchronization(t *testing.T) { NumberOfInstances: int32ToPointer(1), }, }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) cluster.Statefulset = &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 9c48b7ef2..4e4685379 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -7,24 +7,24 @@ import ( "sync" "github.com/sirupsen/logrus" - v1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/tools/cache" - acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/apiserver" "github.com/zalando/postgres-operator/pkg/cluster" + acidv1informer "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" "github.com/zalando/postgres-operator/pkg/util/ringlog" - - acidv1informer "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/acid.zalan.do/v1" + v1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" ) // Controller represents operator controller @@ -36,6 +36,9 @@ type Controller struct { KubeClient k8sutil.KubernetesClient apiserver *apiserver.Server + eventRecorder record.EventRecorder + eventBroadcaster record.EventBroadcaster + stopCh chan struct{} controllerID string @@ -67,10 +70,21 @@ type Controller struct { func NewController(controllerConfig *spec.ControllerConfig, controllerId string) *Controller { logger := logrus.New() + var myComponentName = "postgres-operator" + if controllerId != "" { + myComponentName += "/" + controllerId + } + + eventBroadcaster := record.NewBroadcaster() + eventBroadcaster.StartLogging(logger.Debugf) + recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: myComponentName}) + c := &Controller{ config: *controllerConfig, opConfig: &config.Config{}, logger: logger.WithField("pkg", "controller"), + eventRecorder: recorder, + eventBroadcaster: eventBroadcaster, controllerID: controllerId, curWorkerCluster: sync.Map{}, clusterWorkers: make(map[spec.NamespacedName]uint32), @@ -93,6 +107,11 @@ func (c *Controller) initClients() { if err != nil { c.logger.Fatalf("could not create kubernetes clients: %v", err) } + c.eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: c.KubeClient.EventsGetter.Events("")}) + if err != nil { + c.logger.Fatalf("could not setup kubernetes event sink: %v", err) + } + } func (c *Controller) initOperatorConfig() { diff --git a/pkg/controller/postgresql.go b/pkg/controller/postgresql.go index e81671c7d..2a9e1b650 100644 --- a/pkg/controller/postgresql.go +++ b/pkg/controller/postgresql.go @@ -11,6 +11,7 @@ import ( "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/cache" @@ -157,7 +158,7 @@ func (c *Controller) acquireInitialListOfClusters() error { } func (c *Controller) addCluster(lg *logrus.Entry, clusterName spec.NamespacedName, pgSpec *acidv1.Postgresql) *cluster.Cluster { - cl := cluster.New(c.makeClusterConfig(), c.KubeClient, *pgSpec, lg) + cl := cluster.New(c.makeClusterConfig(), c.KubeClient, *pgSpec, lg, c.eventRecorder) cl.Run(c.stopCh) teamName := strings.ToLower(cl.Spec.TeamID) @@ -236,6 +237,7 @@ func (c *Controller) processEvent(event ClusterEvent) { if err := cl.Create(); err != nil { cl.Error = fmt.Sprintf("could not create cluster: %v", err) lg.Error(cl.Error) + c.eventRecorder.Eventf(cl.GetReference(), v1.EventTypeWarning, "Create", "%v", cl.Error) return } @@ -274,6 +276,8 @@ func (c *Controller) processEvent(event ClusterEvent) { c.curWorkerCluster.Store(event.WorkerID, cl) cl.Delete() + // Fixme - no error handling for delete ? + // c.eventRecorder.Eventf(cl.GetReference, v1.EventTypeWarning, "Delete", "%v", cl.Error) func() { defer c.clustersMu.Unlock() @@ -304,6 +308,7 @@ func (c *Controller) processEvent(event ClusterEvent) { c.curWorkerCluster.Store(event.WorkerID, cl) if err := cl.Sync(event.NewSpec); err != nil { cl.Error = fmt.Sprintf("could not sync cluster: %v", err) + c.eventRecorder.Eventf(cl.GetReference(), v1.EventTypeWarning, "Sync", "%v", cl.Error) lg.Error(cl.Error) return } diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index 3a397af7d..d7be2f48a 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -45,6 +45,7 @@ type KubernetesClient struct { corev1.NodesGetter corev1.NamespacesGetter corev1.ServiceAccountsGetter + corev1.EventsGetter appsv1.StatefulSetsGetter appsv1.DeploymentsGetter rbacv1.RoleBindingsGetter @@ -142,6 +143,7 @@ func NewFromConfig(cfg *rest.Config) (KubernetesClient, error) { kubeClient.RESTClient = client.CoreV1().RESTClient() kubeClient.RoleBindingsGetter = client.RbacV1() kubeClient.CronJobsGetter = client.BatchV1beta1() + kubeClient.EventsGetter = client.CoreV1() apiextClient, err := apiextclient.NewForConfig(cfg) if err != nil { From f32c615a531426804d031b16967d69a4ec364c16 Mon Sep 17 00:00:00 2001 From: siku4 <44839490+siku4@users.noreply.github.com> Date: Mon, 27 Apr 2020 12:22:42 +0200 Subject: [PATCH 032/168] fix typo in additionalVolume struct (#933) * fix typo in additionalVolume struct Co-authored-by: siku4 --- manifests/complete-postgres-manifest.yaml | 32 ++++++++++---------- pkg/apis/acid.zalan.do/v1/postgresql_type.go | 2 +- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index b469a7564..e701fdfaa 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -31,28 +31,28 @@ spec: size: 1Gi # storageClass: my-sc additionalVolumes: - - name: data - mountPath: /home/postgres/pgdata/partitions - targetContainers: - - postgres - volumeSource: - PersistentVolumeClaim: - claimName: pvc-postgresql-data-partitions - readyOnly: false - - name: conf - mountPath: /etc/telegraf - subPath: telegraf.conf - targetContainers: - - telegraf-sidecar - volumeSource: - configMap: - name: my-config-map - name: empty mountPath: /opt/empty targetContainers: - all volumeSource: emptyDir: {} +# - name: data +# mountPath: /home/postgres/pgdata/partitions +# targetContainers: +# - postgres +# volumeSource: +# PersistentVolumeClaim: +# claimName: pvc-postgresql-data-partitions +# readyOnly: false +# - name: conf +# mountPath: /etc/telegraf +# subPath: telegraf.conf +# targetContainers: +# - telegraf-sidecar +# volumeSource: +# configMap: +# name: my-config-map enableShmVolume: true # spiloFSGroup: 103 diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 961051c8d..e36009208 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -104,7 +104,7 @@ type AdditionalVolume struct { MountPath string `json:"mountPath"` SubPath string `json:"subPath"` TargetContainers []string `json:"targetContainers"` - VolumeSource v1.VolumeSource `json:"volume"` + VolumeSource v1.VolumeSource `json:"volumeSource"` } // PostgresqlParam describes PostgreSQL version and pairs of configuration parameter name - values. From 168abfe37b3e90cf1e3e4cb2cfd15f3c288d9235 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Fischer?= Date: Mon, 27 Apr 2020 17:40:22 +0200 Subject: [PATCH 033/168] Fully speced global sidecars (#890) * implement fully speced global sidecars * fix issue #924 --- .../crds/operatorconfigurations.yaml | 6 + docs/administrator.md | 27 ++ docs/reference/operator_parameters.md | 27 +- docs/user.md | 2 + manifests/operatorconfiguration.crd.yaml | 6 + ...gresql-operator-default-configuration.yaml | 7 +- pkg/apis/acid.zalan.do/v1/crds.go | 11 + .../v1/operator_configuration_type.go | 27 +- .../acid.zalan.do/v1/zz_generated.deepcopy.go | 11 +- pkg/cluster/k8sres.go | 237 ++++++++++-------- pkg/cluster/k8sres_test.go | 199 +++++++++++++++ pkg/cluster/util.go | 19 ++ pkg/controller/controller.go | 5 + pkg/controller/operator_config.go | 3 +- pkg/util/config/config.go | 15 +- 15 files changed, 462 insertions(+), 140 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 7e20c8fea..285d99b40 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -84,6 +84,12 @@ spec: type: object additionalProperties: type: string + sidecars: + type: array + nullable: true + items: + type: object + additionalProperties: true workers: type: integer minimum: 1 diff --git a/docs/administrator.md b/docs/administrator.md index 93adf2eb1..158b733ad 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -507,6 +507,33 @@ A secret can be pre-provisioned in different ways: * Automatically provisioned via a custom K8s controller like [kube-aws-iam-controller](https://github.com/mikkeloscar/kube-aws-iam-controller) +## Sidecars for Postgres clusters + +A list of sidecars is added to each cluster created by the +operator. The default is empty list. + + +```yaml +kind: OperatorConfiguration +configuration: + sidecars: + - image: image:123 + name: global-sidecar + ports: + - containerPort: 80 + volumeMounts: + - mountPath: /custom-pgdata-mountpoint + name: pgdata + - ... +``` + +In addition to any environment variables you specify, the following environment variables are always passed to sidecars: + + - `POD_NAME` - field reference to `metadata.name` + - `POD_NAMESPACE` - field reference to `metadata.namespace` + - `POSTGRES_USER` - the superuser that can be used to connect to the database + - `POSTGRES_PASSWORD` - the password for the superuser + ## Setting up the Postgres Operator UI Since the v1.2 release the Postgres Operator is shipped with a browser-based diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 3d31abab4..259f04527 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -93,9 +93,17 @@ Those are top-level keys, containing both leaf keys and groups. repository](https://github.com/zalando/spilo). * **sidecar_docker_images** - a map of sidecar names to Docker images to run with Spilo. In case of the name - conflict with the definition in the cluster manifest the cluster-specific one - is preferred. + *deprecated*: use **sidecars** instead. A map of sidecar names to Docker images to + run with Spilo. In case of the name conflict with the definition in the cluster + manifest the cluster-specific one is preferred. + +* **sidecars** + a list of sidecars to run with Spilo, for any cluster (i.e. globally defined sidecars). + Each item in the list is of type + [Container](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#container-v1-core). + Globally defined sidecars can be overwritten by specifying a sidecar in the custom resource with + the same name. Note: This field is not part of the schema validation. If the container specification + is invalid, then the operator fails to create the statefulset. * **enable_shm_volume** Instruct operator to start any new database pod without limitations on shm @@ -133,8 +141,9 @@ Those are top-level keys, containing both leaf keys and groups. at the cost of overprovisioning memory and potential scheduling problems for containers with high memory limits due to the lack of memory on Kubernetes cluster nodes. This affects all containers created by the operator (Postgres, - Scalyr sidecar, and other sidecars); to set resources for the operator's own - container, change the [operator deployment manually](../../manifests/postgres-operator.yaml#L20). + Scalyr sidecar, and other sidecars except **sidecars** defined in the operator + configuration); to set resources for the operator's own container, change the + [operator deployment manually](../../manifests/postgres-operator.yaml#L20). The default is `false`. ## Postgres users @@ -206,12 +215,12 @@ configuration they are grouped under the `kubernetes` key. Default is true. * **enable_init_containers** - global option to allow for creating init containers to run actions before - Spilo is started. Default is true. + global option to allow for creating init containers in the cluster manifest to + run actions before Spilo is started. Default is true. * **enable_sidecars** - global option to allow for creating sidecar containers to run alongside Spilo - on the same pod. Default is true. + global option to allow for creating sidecar containers in the cluster manifest + to run alongside Spilo on the same pod. Globally defined sidecars are always enabled. Default is true. * **secret_name_template** a template for the name of the database user secrets generated by the diff --git a/docs/user.md b/docs/user.md index 2c1c4fd1f..d7e6add0a 100644 --- a/docs/user.md +++ b/docs/user.md @@ -442,6 +442,8 @@ The PostgreSQL volume is shared with sidecars and is mounted at specified but globally disabled in the configuration. The `enable_sidecars` option must be set to `true`. +If you want to add a sidecar to every cluster managed by the operator, you can specify it in the [operator configuration](administrator.md#sidecars-for-postgres-clusters) instead. + ## InitContainers Support Each cluster can specify arbitrary init containers to run. These containers can diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 86051e43b..b2496c9c9 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -60,6 +60,12 @@ spec: type: object additionalProperties: type: string + sidecars: + type: array + nullable: true + items: + type: object + additionalProperties: true workers: type: integer minimum: 1 diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 209e2684b..e80bfa846 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -13,8 +13,11 @@ configuration: resync_period: 30m repair_period: 5m # set_memory_request_to_limit: false - # sidecar_docker_images: - # example: "exampleimage:exampletag" + # sidecars: + # - image: image:123 + # name: global-sidecar-1 + # ports: + # - containerPort: 80 workers: 4 users: replication_username: standby diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 3f4314240..bcb35c56c 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -797,6 +797,17 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, }, }, + "sidecars": { + Type: "array", + Items: &apiextv1beta1.JSONSchemaPropsOrArray{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "object", + AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + Allows: true, + }, + }, + }, + }, "workers": { Type: "integer", Minimum: &min1, diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 8ed4281f4..c377a294b 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -8,6 +8,7 @@ import ( "time" "github.com/zalando/postgres-operator/pkg/spec" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -181,18 +182,20 @@ type OperatorLogicalBackupConfiguration struct { // OperatorConfigurationData defines the operation config type OperatorConfigurationData struct { - EnableCRDValidation *bool `json:"enable_crd_validation,omitempty"` - EtcdHost string `json:"etcd_host,omitempty"` - KubernetesUseConfigMaps bool `json:"kubernetes_use_configmaps,omitempty"` - DockerImage string `json:"docker_image,omitempty"` - Workers uint32 `json:"workers,omitempty"` - MinInstances int32 `json:"min_instances,omitempty"` - MaxInstances int32 `json:"max_instances,omitempty"` - ResyncPeriod Duration `json:"resync_period,omitempty"` - RepairPeriod Duration `json:"repair_period,omitempty"` - SetMemoryRequestToLimit bool `json:"set_memory_request_to_limit,omitempty"` - ShmVolume *bool `json:"enable_shm_volume,omitempty"` - Sidecars map[string]string `json:"sidecar_docker_images,omitempty"` + EnableCRDValidation *bool `json:"enable_crd_validation,omitempty"` + EtcdHost string `json:"etcd_host,omitempty"` + KubernetesUseConfigMaps bool `json:"kubernetes_use_configmaps,omitempty"` + DockerImage string `json:"docker_image,omitempty"` + Workers uint32 `json:"workers,omitempty"` + MinInstances int32 `json:"min_instances,omitempty"` + MaxInstances int32 `json:"max_instances,omitempty"` + ResyncPeriod Duration `json:"resync_period,omitempty"` + RepairPeriod Duration `json:"repair_period,omitempty"` + SetMemoryRequestToLimit bool `json:"set_memory_request_to_limit,omitempty"` + ShmVolume *bool `json:"enable_shm_volume,omitempty"` + // deprecated in favour of SidecarContainers + SidecarImages map[string]string `json:"sidecar_docker_images,omitempty"` + SidecarContainers []v1.Container `json:"sidecars,omitempty"` PostgresUsersConfiguration PostgresUsersConfiguration `json:"users"` Kubernetes KubernetesMetaConfiguration `json:"kubernetes"` PostgresPodResources PostgresPodResourcesDefaults `json:"postgres_pod_resources"` diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index e6b387ec4..e2e1d5bd1 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -312,13 +312,20 @@ func (in *OperatorConfigurationData) DeepCopyInto(out *OperatorConfigurationData *out = new(bool) **out = **in } - if in.Sidecars != nil { - in, out := &in.Sidecars, &out.Sidecars + if in.SidecarImages != nil { + in, out := &in.SidecarImages, &out.SidecarImages *out = make(map[string]string, len(*in)) for key, val := range *in { (*out)[key] = val } } + if in.SidecarContainers != nil { + in, out := &in.SidecarContainers, &out.SidecarContainers + *out = make([]corev1.Container, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } out.PostgresUsersConfiguration = in.PostgresUsersConfiguration in.Kubernetes.DeepCopyInto(&out.Kubernetes) out.PostgresPodResources = in.PostgresPodResources diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 9fb33eab2..43190491b 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -462,8 +462,7 @@ func generateContainer( } func generateSidecarContainers(sidecars []acidv1.Sidecar, - volumeMounts []v1.VolumeMount, defaultResources acidv1.Resources, - superUserName string, credentialsSecretName string, logger *logrus.Entry) ([]v1.Container, error) { + defaultResources acidv1.Resources, startIndex int, logger *logrus.Entry) ([]v1.Container, error) { if len(sidecars) > 0 { result := make([]v1.Container, 0) @@ -482,7 +481,7 @@ func generateSidecarContainers(sidecars []acidv1.Sidecar, return nil, err } - sc := getSidecarContainer(sidecar, index, volumeMounts, resources, superUserName, credentialsSecretName, logger) + sc := getSidecarContainer(sidecar, startIndex+index, resources) result = append(result, *sc) } return result, nil @@ -490,6 +489,55 @@ func generateSidecarContainers(sidecars []acidv1.Sidecar, return nil, nil } +// adds common fields to sidecars +func patchSidecarContainers(in []v1.Container, volumeMounts []v1.VolumeMount, superUserName string, credentialsSecretName string, logger *logrus.Entry) []v1.Container { + result := []v1.Container{} + + for _, container := range in { + container.VolumeMounts = append(container.VolumeMounts, volumeMounts...) + env := []v1.EnvVar{ + { + Name: "POD_NAME", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.name", + }, + }, + }, + { + Name: "POD_NAMESPACE", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.namespace", + }, + }, + }, + { + Name: "POSTGRES_USER", + Value: superUserName, + }, + { + Name: "POSTGRES_PASSWORD", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: credentialsSecretName, + }, + Key: "password", + }, + }, + }, + } + mergedEnv := append(container.Env, env...) + container.Env = deduplicateEnvVars(mergedEnv, container.Name, logger) + result = append(result, container) + } + + return result +} + // Check whether or not we're requested to mount an shm volume, // taking into account that PostgreSQL manifest has precedence. func mountShmVolumeNeeded(opConfig config.Config, spec *acidv1.PostgresSpec) *bool { @@ -724,58 +772,18 @@ func deduplicateEnvVars(input []v1.EnvVar, containerName string, logger *logrus. return result } -func getSidecarContainer(sidecar acidv1.Sidecar, index int, volumeMounts []v1.VolumeMount, - resources *v1.ResourceRequirements, superUserName string, credentialsSecretName string, logger *logrus.Entry) *v1.Container { +func getSidecarContainer(sidecar acidv1.Sidecar, index int, resources *v1.ResourceRequirements) *v1.Container { name := sidecar.Name if name == "" { name = fmt.Sprintf("sidecar-%d", index) } - env := []v1.EnvVar{ - { - Name: "POD_NAME", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - APIVersion: "v1", - FieldPath: "metadata.name", - }, - }, - }, - { - Name: "POD_NAMESPACE", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - APIVersion: "v1", - FieldPath: "metadata.namespace", - }, - }, - }, - { - Name: "POSTGRES_USER", - Value: superUserName, - }, - { - Name: "POSTGRES_PASSWORD", - ValueFrom: &v1.EnvVarSource{ - SecretKeyRef: &v1.SecretKeySelector{ - LocalObjectReference: v1.LocalObjectReference{ - Name: credentialsSecretName, - }, - Key: "password", - }, - }, - }, - } - if len(sidecar.Env) > 0 { - env = append(env, sidecar.Env...) - } return &v1.Container{ Name: name, Image: sidecar.DockerImage, ImagePullPolicy: v1.PullIfNotPresent, Resources: *resources, - VolumeMounts: volumeMounts, - Env: deduplicateEnvVars(env, name, logger), + Env: sidecar.Env, Ports: sidecar.Ports, } } @@ -1065,37 +1073,63 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef c.OpConfig.Resources.SpiloPrivileged, ) - // resolve conflicts between operator-global and per-cluster sidecars - sideCars := c.mergeSidecars(spec.Sidecars) + // generate container specs for sidecars specified in the cluster manifest + clusterSpecificSidecars := []v1.Container{} + if spec.Sidecars != nil && len(spec.Sidecars) > 0 { + // warn if sidecars are defined, but globally disabled (does not apply to globally defined sidecars) + if c.OpConfig.EnableSidecars != nil && !(*c.OpConfig.EnableSidecars) { + c.logger.Warningf("sidecars specified but disabled in configuration - next statefulset creation would fail") + } - resourceRequirementsScalyrSidecar := makeResources( - c.OpConfig.ScalyrCPURequest, - c.OpConfig.ScalyrMemoryRequest, - c.OpConfig.ScalyrCPULimit, - c.OpConfig.ScalyrMemoryLimit, - ) + if clusterSpecificSidecars, err = generateSidecarContainers(spec.Sidecars, defaultResources, 0, c.logger); err != nil { + return nil, fmt.Errorf("could not generate sidecar containers: %v", err) + } + } + + // decrapted way of providing global sidecars + var globalSidecarContainersByDockerImage []v1.Container + var globalSidecarsByDockerImage []acidv1.Sidecar + for name, dockerImage := range c.OpConfig.SidecarImages { + globalSidecarsByDockerImage = append(globalSidecarsByDockerImage, acidv1.Sidecar{Name: name, DockerImage: dockerImage}) + } + if globalSidecarContainersByDockerImage, err = generateSidecarContainers(globalSidecarsByDockerImage, defaultResources, len(clusterSpecificSidecars), c.logger); err != nil { + return nil, fmt.Errorf("could not generate sidecar containers: %v", err) + } + // make the resulting list reproducible + // c.OpConfig.SidecarImages is unsorted by Golang definition + // .Name is unique + sort.Slice(globalSidecarContainersByDockerImage, func(i, j int) bool { + return globalSidecarContainersByDockerImage[i].Name < globalSidecarContainersByDockerImage[j].Name + }) // generate scalyr sidecar container - if scalyrSidecar := + var scalyrSidecars []v1.Container + if scalyrSidecar, err := generateScalyrSidecarSpec(c.Name, c.OpConfig.ScalyrAPIKey, c.OpConfig.ScalyrServerURL, c.OpConfig.ScalyrImage, - &resourceRequirementsScalyrSidecar, c.logger); scalyrSidecar != nil { - sideCars = append(sideCars, *scalyrSidecar) + c.OpConfig.ScalyrCPURequest, + c.OpConfig.ScalyrMemoryRequest, + c.OpConfig.ScalyrCPULimit, + c.OpConfig.ScalyrMemoryLimit, + defaultResources, + c.logger); err != nil { + return nil, fmt.Errorf("could not generate Scalyr sidecar: %v", err) + } else { + if scalyrSidecar != nil { + scalyrSidecars = append(scalyrSidecars, *scalyrSidecar) + } } - // generate sidecar containers - if sideCars != nil && len(sideCars) > 0 { - if c.OpConfig.EnableSidecars != nil && !(*c.OpConfig.EnableSidecars) { - c.logger.Warningf("sidecars specified but disabled in configuration - next statefulset creation would fail") - } - if sidecarContainers, err = generateSidecarContainers(sideCars, volumeMounts, defaultResources, - c.OpConfig.SuperUsername, c.credentialSecretName(c.OpConfig.SuperUsername), c.logger); err != nil { - return nil, fmt.Errorf("could not generate sidecar containers: %v", err) - } + sidecarContainers, conflicts := mergeContainers(clusterSpecificSidecars, c.Config.OpConfig.SidecarContainers, globalSidecarContainersByDockerImage, scalyrSidecars) + for containerName := range conflicts { + c.logger.Warningf("a sidecar is specified twice. Ignoring sidecar %q in favor of %q with high a precendence", + containerName, containerName) } + sidecarContainers = patchSidecarContainers(sidecarContainers, volumeMounts, c.OpConfig.SuperUsername, c.credentialSecretName(c.OpConfig.SuperUsername), c.logger) + tolerationSpec := tolerations(&spec.Tolerations, c.OpConfig.PodToleration) effectivePodPriorityClassName := util.Coalesce(spec.PodPriorityClassName, c.OpConfig.PodPriorityClassName) @@ -1188,57 +1222,44 @@ func (c *Cluster) generatePodAnnotations(spec *acidv1.PostgresSpec) map[string]s } func generateScalyrSidecarSpec(clusterName, APIKey, serverURL, dockerImage string, - containerResources *acidv1.Resources, logger *logrus.Entry) *acidv1.Sidecar { + scalyrCPURequest string, scalyrMemoryRequest string, scalyrCPULimit string, scalyrMemoryLimit string, + defaultResources acidv1.Resources, logger *logrus.Entry) (*v1.Container, error) { if APIKey == "" || dockerImage == "" { if APIKey == "" && dockerImage != "" { logger.Warning("Not running Scalyr sidecar: SCALYR_API_KEY must be defined") } - return nil + return nil, nil } - scalarSpec := &acidv1.Sidecar{ - Name: "scalyr-sidecar", - DockerImage: dockerImage, - Env: []v1.EnvVar{ - { - Name: "SCALYR_API_KEY", - Value: APIKey, - }, - { - Name: "SCALYR_SERVER_HOST", - Value: clusterName, - }, + resourcesScalyrSidecar := makeResources( + scalyrCPURequest, + scalyrMemoryRequest, + scalyrCPULimit, + scalyrMemoryLimit, + ) + resourceRequirementsScalyrSidecar, err := generateResourceRequirements(resourcesScalyrSidecar, defaultResources) + if err != nil { + return nil, fmt.Errorf("invalid resources for Scalyr sidecar: %v", err) + } + env := []v1.EnvVar{ + { + Name: "SCALYR_API_KEY", + Value: APIKey, + }, + { + Name: "SCALYR_SERVER_HOST", + Value: clusterName, }, - Resources: *containerResources, } if serverURL != "" { - scalarSpec.Env = append(scalarSpec.Env, v1.EnvVar{Name: "SCALYR_SERVER_URL", Value: serverURL}) + env = append(env, v1.EnvVar{Name: "SCALYR_SERVER_URL", Value: serverURL}) } - return scalarSpec -} - -// mergeSidecar merges globally-defined sidecars with those defined in the cluster manifest -func (c *Cluster) mergeSidecars(sidecars []acidv1.Sidecar) []acidv1.Sidecar { - globalSidecarsToSkip := map[string]bool{} - result := make([]acidv1.Sidecar, 0) - - for i, sidecar := range sidecars { - dockerImage, ok := c.OpConfig.Sidecars[sidecar.Name] - if ok { - if dockerImage != sidecar.DockerImage { - c.logger.Warningf("merging definitions for sidecar %q: "+ - "ignoring %q in the global scope in favor of %q defined in the cluster", - sidecar.Name, dockerImage, sidecar.DockerImage) - } - globalSidecarsToSkip[sidecar.Name] = true - } - result = append(result, sidecars[i]) - } - for name, dockerImage := range c.OpConfig.Sidecars { - if !globalSidecarsToSkip[name] { - result = append(result, acidv1.Sidecar{Name: name, DockerImage: dockerImage}) - } - } - return result + return &v1.Container{ + Name: "scalyr-sidecar", + Image: dockerImage, + Env: env, + ImagePullPolicy: v1.PullIfNotPresent, + Resources: *resourceRequirementsScalyrSidecar, + }, nil } func (c *Cluster) getNumberOfInstances(spec *acidv1.PostgresSpec) int32 { @@ -1803,7 +1824,7 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) { c.OpConfig.AdditionalSecretMount, c.OpConfig.AdditionalSecretMountPath, []acidv1.AdditionalVolume{}); err != nil { - return nil, fmt.Errorf("could not generate pod template for logical backup pod: %v", err) + return nil, fmt.Errorf("could not generate pod template for logical backup pod: %v", err) } // overwrite specific params of logical backups pods diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 1291d4f47..f9e95feef 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -18,6 +18,7 @@ import ( appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" policyv1beta1 "k8s.io/api/policy/v1beta1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" ) @@ -1206,3 +1207,201 @@ func TestAdditionalVolume(t *testing.T) { } } } + +// inject sidecars through all available mechanisms and check the resulting container specs +func TestSidecars(t *testing.T) { + var err error + var spec acidv1.PostgresSpec + var cluster *Cluster + + generateKubernetesResources := func(cpuRequest string, cpuLimit string, memoryRequest string, memoryLimit string) v1.ResourceRequirements { + parsedCPURequest, err := resource.ParseQuantity(cpuRequest) + assert.NoError(t, err) + parsedCPULimit, err := resource.ParseQuantity(cpuLimit) + assert.NoError(t, err) + parsedMemoryRequest, err := resource.ParseQuantity(memoryRequest) + assert.NoError(t, err) + parsedMemoryLimit, err := resource.ParseQuantity(memoryLimit) + assert.NoError(t, err) + return v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: parsedCPURequest, + v1.ResourceMemory: parsedMemoryRequest, + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: parsedCPULimit, + v1.ResourceMemory: parsedMemoryLimit, + }, + } + } + + spec = acidv1.PostgresSpec{ + TeamID: "myapp", NumberOfInstances: 1, + Resources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + }, + Volume: acidv1.Volume{ + Size: "1G", + }, + Sidecars: []acidv1.Sidecar{ + acidv1.Sidecar{ + Name: "cluster-specific-sidecar", + }, + acidv1.Sidecar{ + Name: "cluster-specific-sidecar-with-resources", + Resources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "210m", Memory: "0.8Gi"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "510m", Memory: "1.4Gi"}, + }, + }, + acidv1.Sidecar{ + Name: "replace-sidecar", + DockerImage: "overwrite-image", + }, + }, + } + + cluster = New( + Config{ + OpConfig: config.Config{ + PodManagementPolicy: "ordered_ready", + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + Resources: config.Resources{ + DefaultCPURequest: "200m", + DefaultCPULimit: "500m", + DefaultMemoryRequest: "0.7Gi", + DefaultMemoryLimit: "1.3Gi", + }, + SidecarImages: map[string]string{ + "deprecated-global-sidecar": "image:123", + }, + SidecarContainers: []v1.Container{ + v1.Container{ + Name: "global-sidecar", + }, + // will be replaced by a cluster specific sidecar with the same name + v1.Container{ + Name: "replace-sidecar", + Image: "replaced-image", + }, + }, + Scalyr: config.Scalyr{ + ScalyrAPIKey: "abc", + ScalyrImage: "scalyr-image", + ScalyrCPURequest: "220m", + ScalyrCPULimit: "520m", + ScalyrMemoryRequest: "0.9Gi", + // ise default memory limit + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + + s, err := cluster.generateStatefulSet(&spec) + assert.NoError(t, err) + + env := []v1.EnvVar{ + { + Name: "POD_NAME", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.name", + }, + }, + }, + { + Name: "POD_NAMESPACE", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.namespace", + }, + }, + }, + { + Name: "POSTGRES_USER", + Value: superUserName, + }, + { + Name: "POSTGRES_PASSWORD", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "", + }, + Key: "password", + }, + }, + }, + } + mounts := []v1.VolumeMount{ + v1.VolumeMount{ + Name: "pgdata", + MountPath: "/home/postgres/pgdata", + }, + } + + // deduplicated sidecars and Patroni + assert.Equal(t, 7, len(s.Spec.Template.Spec.Containers), "wrong number of containers") + + // cluster specific sidecar + assert.Contains(t, s.Spec.Template.Spec.Containers, v1.Container{ + Name: "cluster-specific-sidecar", + Env: env, + Resources: generateKubernetesResources("200m", "500m", "0.7Gi", "1.3Gi"), + ImagePullPolicy: v1.PullIfNotPresent, + VolumeMounts: mounts, + }) + + // container specific resources + expectedResources := generateKubernetesResources("210m", "510m", "0.8Gi", "1.4Gi") + assert.Equal(t, expectedResources.Requests[v1.ResourceCPU], s.Spec.Template.Spec.Containers[2].Resources.Requests[v1.ResourceCPU]) + assert.Equal(t, expectedResources.Limits[v1.ResourceCPU], s.Spec.Template.Spec.Containers[2].Resources.Limits[v1.ResourceCPU]) + assert.Equal(t, expectedResources.Requests[v1.ResourceMemory], s.Spec.Template.Spec.Containers[2].Resources.Requests[v1.ResourceMemory]) + assert.Equal(t, expectedResources.Limits[v1.ResourceMemory], s.Spec.Template.Spec.Containers[2].Resources.Limits[v1.ResourceMemory]) + + // deprecated global sidecar + assert.Contains(t, s.Spec.Template.Spec.Containers, v1.Container{ + Name: "deprecated-global-sidecar", + Image: "image:123", + Env: env, + Resources: generateKubernetesResources("200m", "500m", "0.7Gi", "1.3Gi"), + ImagePullPolicy: v1.PullIfNotPresent, + VolumeMounts: mounts, + }) + + // global sidecar + assert.Contains(t, s.Spec.Template.Spec.Containers, v1.Container{ + Name: "global-sidecar", + Env: env, + VolumeMounts: mounts, + }) + + // replaced sidecar + assert.Contains(t, s.Spec.Template.Spec.Containers, v1.Container{ + Name: "replace-sidecar", + Image: "overwrite-image", + Resources: generateKubernetesResources("200m", "500m", "0.7Gi", "1.3Gi"), + ImagePullPolicy: v1.PullIfNotPresent, + Env: env, + VolumeMounts: mounts, + }) + + // replaced sidecar + // the order in env is important + scalyrEnv := append([]v1.EnvVar{v1.EnvVar{Name: "SCALYR_API_KEY", Value: "abc"}, v1.EnvVar{Name: "SCALYR_SERVER_HOST", Value: ""}}, env...) + assert.Contains(t, s.Spec.Template.Spec.Containers, v1.Container{ + Name: "scalyr-sidecar", + Image: "scalyr-image", + Resources: generateKubernetesResources("220m", "520m", "0.9Gi", "1.3Gi"), + ImagePullPolicy: v1.PullIfNotPresent, + Env: scalyrEnv, + VolumeMounts: mounts, + }) + +} diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go index 4dcdfb28a..7559ce3d4 100644 --- a/pkg/cluster/util.go +++ b/pkg/cluster/util.go @@ -530,3 +530,22 @@ func (c *Cluster) needConnectionPoolerWorker(spec *acidv1.PostgresSpec) bool { func (c *Cluster) needConnectionPooler() bool { return c.needConnectionPoolerWorker(&c.Spec) } + +// Earlier arguments take priority +func mergeContainers(containers ...[]v1.Container) ([]v1.Container, []string) { + containerNameTaken := map[string]bool{} + result := make([]v1.Container, 0) + conflicts := make([]string, 0) + + for _, containerArray := range containers { + for _, container := range containerArray { + if _, taken := containerNameTaken[container.Name]; taken { + conflicts = append(conflicts, container.Name) + } else { + containerNameTaken[container.Name] = true + result = append(result, container) + } + } + } + return result, conflicts +} diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 4e4685379..0b3fde5d9 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -178,6 +178,11 @@ func (c *Controller) warnOnDeprecatedOperatorParameters() { c.logger.Warningf("Operator configuration parameter 'enable_load_balancer' is deprecated and takes no effect. " + "Consider using the 'enable_master_load_balancer' or 'enable_replica_load_balancer' instead.") } + + if len(c.opConfig.SidecarImages) > 0 { + c.logger.Warningf("Operator configuration parameter 'sidecar_docker_images' is deprecated. " + + "Consider using 'sidecars' instead.") + } } func (c *Controller) initPodServiceAccount() { diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 07be90f22..c1756604b 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -44,7 +44,8 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.RepairPeriod = time.Duration(fromCRD.RepairPeriod) result.SetMemoryRequestToLimit = fromCRD.SetMemoryRequestToLimit result.ShmVolume = fromCRD.ShmVolume - result.Sidecars = fromCRD.Sidecars + result.SidecarImages = fromCRD.SidecarImages + result.SidecarContainers = fromCRD.SidecarContainers // user config result.SuperUsername = fromCRD.PostgresUsersConfiguration.SuperUsername diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 84a62c0fd..9c2257e78 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -9,6 +9,7 @@ import ( "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util/constants" + v1 "k8s.io/api/core/v1" ) // CRD describes CustomResourceDefinition specific configuration parameters @@ -107,12 +108,14 @@ type Config struct { LogicalBackup ConnectionPooler - WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to' - KubernetesUseConfigMaps bool `name:"kubernetes_use_configmaps" default:"false"` - EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS - DockerImage string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-12:1.6-p2"` - Sidecars map[string]string `name:"sidecar_docker_images"` - PodServiceAccountName string `name:"pod_service_account_name" default:"postgres-pod"` + WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to' + KubernetesUseConfigMaps bool `name:"kubernetes_use_configmaps" default:"false"` + EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS + DockerImage string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-12:1.6-p2"` + // deprecated in favour of SidecarContainers + SidecarImages map[string]string `name:"sidecar_docker_images"` + SidecarContainers []v1.Container `name:"sidecars"` + PodServiceAccountName string `name:"pod_service_account_name" default:"postgres-pod"` // value of this string must be valid JSON or YAML; see initPodServiceAccount PodServiceAccountDefinition string `name:"pod_service_account_definition" default:""` PodServiceAccountRoleBindingDefinition string `name:"pod_service_account_role_binding_definition" default:""` From 0ca30ba3d98a85972e6725da64417f16e26419ae Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Tue, 28 Apr 2020 09:31:41 +0200 Subject: [PATCH 034/168] fix params in function call (#939) Co-authored-by: Sergey Dudoladov --- pkg/cluster/k8sres_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index f9e95feef..d09a2c0aa 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -1299,7 +1299,7 @@ func TestSidecars(t *testing.T) { // ise default memory limit }, }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) s, err := cluster.generateStatefulSet(&spec) assert.NoError(t, err) From 1d009d9595be614d67a72b9bcd520c04dc349aa1 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 28 Apr 2020 16:01:13 +0200 Subject: [PATCH 035/168] bump spilo and pooler version + update docs (#945) --- .../crds/operatorconfigurations.yaml | 2 +- .../templates/operatorconfiguration.yaml | 2 -- charts/postgres-operator/values-crd.yaml | 19 +---------- charts/postgres-operator/values.yaml | 2 +- docs/administrator.md | 8 ++--- .../reference/command_line_and_environment.md | 2 +- docs/reference/operator_parameters.md | 33 ++++++++++--------- manifests/complete-postgres-manifest.yaml | 2 +- manifests/configmap.yaml | 4 +-- manifests/operatorconfiguration.crd.yaml | 2 +- ...gresql-operator-default-configuration.yaml | 12 ++----- pkg/util/config/config.go | 2 +- 12 files changed, 33 insertions(+), 57 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 285d99b40..aeb3d2150 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -305,7 +305,7 @@ spec: type: integer ring_log_lines: type: integer - scalyr: + scalyr: # deprecated type: object properties: scalyr_api_key: diff --git a/charts/postgres-operator/templates/operatorconfiguration.yaml b/charts/postgres-operator/templates/operatorconfiguration.yaml index 5df6e6238..d28d68f9c 100644 --- a/charts/postgres-operator/templates/operatorconfiguration.yaml +++ b/charts/postgres-operator/templates/operatorconfiguration.yaml @@ -32,8 +32,6 @@ configuration: {{ toYaml .Values.configTeamsApi | indent 4 }} logging_rest_api: {{ toYaml .Values.configLoggingRestApi | indent 4 }} - scalyr: -{{ toYaml .Values.configScalyr | indent 4 }} connection_pooler: {{ toYaml .Values.configConnectionPooler | indent 4 }} {{- end }} diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index caa4dda4d..41b8dbefb 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -26,7 +26,7 @@ configGeneral: # Select if setup uses endpoints (default), or configmaps to manage leader (DCS=k8s) # kubernetes_use_configmaps: false # Spilo docker image - docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p2 + docker_image: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115 # max number of instances in Postgres cluster. -1 = no limit min_instances: -1 # min number of instances in Postgres cluster. -1 = no limit @@ -252,23 +252,6 @@ configTeamsApi: # URL of the Teams API service # teams_api_url: http://fake-teams-api.default.svc.cluster.local -# Scalyr is a log management tool that Zalando uses as a sidecar -configScalyr: - # API key for the Scalyr sidecar - # scalyr_api_key: "" - - # Docker image for the Scalyr sidecar - # scalyr_image: "" - - # CPU limit value for the Scalyr sidecar - scalyr_cpu_limit: "1" - # CPU rquest value for the Scalyr sidecar - scalyr_cpu_request: 100m - # Memory limit value for the Scalyr sidecar - scalyr_memory_limit: 500Mi - # Memory request value for the Scalyr sidecar - scalyr_memory_request: 50Mi - configConnectionPooler: # db schema to install lookup function into connection_pooler_schema: "pooler" diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index e7db249f0..7f140f1de 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -26,7 +26,7 @@ configGeneral: # Select if setup uses endpoints (default), or configmaps to manage leader (DCS=k8s) # kubernetes_use_configmaps: "false" # Spilo docker image - docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p2 + docker_image: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115 # max number of instances in Postgres cluster. -1 = no limit min_instances: "-1" # min number of instances in Postgres cluster. -1 = no limit diff --git a/docs/administrator.md b/docs/administrator.md index 158b733ad..ed96b7d35 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -509,9 +509,8 @@ A secret can be pre-provisioned in different ways: ## Sidecars for Postgres clusters -A list of sidecars is added to each cluster created by the -operator. The default is empty list. - +A list of sidecars is added to each cluster created by the operator. The default +is empty. ```yaml kind: OperatorConfiguration @@ -527,7 +526,8 @@ configuration: - ... ``` -In addition to any environment variables you specify, the following environment variables are always passed to sidecars: +In addition to any environment variables you specify, the following environment +variables are always passed to sidecars: - `POD_NAME` - field reference to `metadata.name` - `POD_NAMESPACE` - field reference to `metadata.namespace` diff --git a/docs/reference/command_line_and_environment.md b/docs/reference/command_line_and_environment.md index ec5da5ceb..ece29b094 100644 --- a/docs/reference/command_line_and_environment.md +++ b/docs/reference/command_line_and_environment.md @@ -45,7 +45,7 @@ The following environment variables are accepted by the operator: all namespaces. Empty value defaults to the operator namespace. Overrides the `watched_namespace` operator parameter. -* **SCALYR_API_KEY** +* **SCALYR_API_KEY** (*deprecated*) the value of the Scalyr API key to supply to the pods. Overrides the `scalyr_api_key` operator parameter. diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 259f04527..d98a270e6 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -93,17 +93,18 @@ Those are top-level keys, containing both leaf keys and groups. repository](https://github.com/zalando/spilo). * **sidecar_docker_images** - *deprecated*: use **sidecars** instead. A map of sidecar names to Docker images to - run with Spilo. In case of the name conflict with the definition in the cluster - manifest the cluster-specific one is preferred. + *deprecated*: use **sidecars** instead. A map of sidecar names to Docker + images to run with Spilo. In case of the name conflict with the definition in + the cluster manifest the cluster-specific one is preferred. * **sidecars** - a list of sidecars to run with Spilo, for any cluster (i.e. globally defined sidecars). - Each item in the list is of type + a list of sidecars to run with Spilo, for any cluster (i.e. globally defined + sidecars). Each item in the list is of type [Container](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#container-v1-core). - Globally defined sidecars can be overwritten by specifying a sidecar in the custom resource with - the same name. Note: This field is not part of the schema validation. If the container specification - is invalid, then the operator fails to create the statefulset. + Globally defined sidecars can be overwritten by specifying a sidecar in the + Postgres manifest with the same name. + Note: This field is not part of the schema validation. If the container + specification is invalid, then the operator fails to create the statefulset. * **enable_shm_volume** Instruct operator to start any new database pod without limitations on shm @@ -141,8 +142,8 @@ Those are top-level keys, containing both leaf keys and groups. at the cost of overprovisioning memory and potential scheduling problems for containers with high memory limits due to the lack of memory on Kubernetes cluster nodes. This affects all containers created by the operator (Postgres, - Scalyr sidecar, and other sidecars except **sidecars** defined in the operator - configuration); to set resources for the operator's own container, change the + Scalyr sidecar, and other sidecars except **sidecars** defined in the operator + configuration); to set resources for the operator's own container, change the [operator deployment manually](../../manifests/postgres-operator.yaml#L20). The default is `false`. @@ -215,12 +216,13 @@ configuration they are grouped under the `kubernetes` key. Default is true. * **enable_init_containers** - global option to allow for creating init containers in the cluster manifest to + global option to allow for creating init containers in the cluster manifest to run actions before Spilo is started. Default is true. * **enable_sidecars** - global option to allow for creating sidecar containers in the cluster manifest - to run alongside Spilo on the same pod. Globally defined sidecars are always enabled. Default is true. + global option to allow for creating sidecar containers in the cluster manifest + to run alongside Spilo on the same pod. Globally defined sidecars are always + enabled. Default is true. * **secret_name_template** a template for the name of the database user secrets generated by the @@ -585,11 +587,12 @@ configuration they are grouped under the `logging_rest_api` key. * **cluster_history_entries** number of entries in the cluster history ring buffer. The default is `1000`. -## Scalyr options +## Scalyr options (*deprecated*) Those parameters define the resource requests/limits and properties of the scalyr sidecar. In the CRD-based configuration they are grouped under the -`scalyr` key. +`scalyr` key. Note, that this section is deprecated. Instead, define Scalyr as +a global sidecar under the `sidecars` key in the configuration. * **scalyr_api_key** API key for the Scalyr sidecar. The default is empty. diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index e701fdfaa..f031a5d5b 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -7,7 +7,7 @@ metadata: # annotations: # "acid.zalan.do/controller": "second-operator" spec: - dockerImage: registry.opensource.zalan.do/acid/spilo-12:1.6-p2 + dockerImage: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115 teamId: "acid" numberOfInstances: 2 users: # Application/Robot users diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 954881ed3..cbc55b446 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -15,7 +15,7 @@ data: # connection_pooler_default_cpu_request: "500m" # connection_pooler_default_memory_limit: 100Mi # connection_pooler_default_memory_request: 100Mi - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-6" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-7" # connection_pooler_max_db_connections: 60 # connection_pooler_mode: "transaction" # connection_pooler_number_of_instances: 2 @@ -29,7 +29,7 @@ data: # default_cpu_request: 100m # default_memory_limit: 500Mi # default_memory_request: 100Mi - docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p2 + docker_image: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115 # enable_admin_role_for_users: "true" # enable_crd_validation: "true" # enable_database_access: "true" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index b2496c9c9..53097b96c 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -281,7 +281,7 @@ spec: type: integer ring_log_lines: type: integer - scalyr: + scalyr: # deprecated type: object properties: scalyr_api_key: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index e80bfa846..b5765a0f4 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -6,7 +6,7 @@ configuration: # enable_crd_validation: true etcd_host: "" # kubernetes_use_configmaps: false - docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p2 + docker_image: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115 # enable_shm_volume: true max_instances: -1 min_instances: -1 @@ -117,20 +117,12 @@ configuration: api_port: 8080 cluster_history_entries: 1000 ring_log_lines: 100 - scalyr: - # scalyr_api_key: "" - scalyr_cpu_limit: "1" - scalyr_cpu_request: 100m - # scalyr_image: "" - scalyr_memory_limit: 500Mi - scalyr_memory_request: 50Mi - # scalyr_server_url: "" connection_pooler: connection_pooler_default_cpu_limit: "1" connection_pooler_default_cpu_request: "500m" connection_pooler_default_memory_limit: 100Mi connection_pooler_default_memory_request: 100Mi - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-6" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-7" # connection_pooler_max_db_connections: 60 connection_pooler_mode: "transaction" connection_pooler_number_of_instances: 2 diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 9c2257e78..2bdeb8e1f 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -111,7 +111,7 @@ type Config struct { WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to' KubernetesUseConfigMaps bool `name:"kubernetes_use_configmaps" default:"false"` EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS - DockerImage string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-12:1.6-p2"` + DockerImage string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115"` // deprecated in favour of SidecarContainers SidecarImages map[string]string `name:"sidecar_docker_images"` SidecarContainers []v1.Container `name:"sidecars"` From 0016ebf7df2596fceb8f5b1a0fb8156501fbce97 Mon Sep 17 00:00:00 2001 From: Marcus Portmann Date: Wed, 29 Apr 2020 07:28:35 +0200 Subject: [PATCH 036/168] Allow nodePort value for the postgres-operator-ui service (#928) * Added support for specifying a nodePort value for the postgres-operator-ui service when the type is NodePort Co-authored-by: Marcus Portmann --- charts/postgres-operator-ui/templates/service.yaml | 3 +++ charts/postgres-operator-ui/values.yaml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/charts/postgres-operator-ui/templates/service.yaml b/charts/postgres-operator-ui/templates/service.yaml index 09adff26f..bc40fbbb1 100644 --- a/charts/postgres-operator-ui/templates/service.yaml +++ b/charts/postgres-operator-ui/templates/service.yaml @@ -11,6 +11,9 @@ spec: ports: - port: {{ .Values.service.port }} targetPort: 8081 + {{- if and (eq .Values.service.type "NodePort") .Values.service.nodePort }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} protocol: TCP selector: app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/charts/postgres-operator-ui/values.yaml b/charts/postgres-operator-ui/values.yaml index 148a687c3..dd25864b2 100644 --- a/charts/postgres-operator-ui/values.yaml +++ b/charts/postgres-operator-ui/values.yaml @@ -42,6 +42,9 @@ envs: service: type: "ClusterIP" port: "8080" + # If the type of the service is NodePort a port can be specified using the nodePort field + # If the nodePort field is not specified, or if it has no value, then a random port is used + # notePort: 32521 # configure UI ingress. If needed: "enabled: true" ingress: From cc635a02e328b14daf3817ae2e413cbfb408230f Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Wed, 29 Apr 2020 10:07:14 +0200 Subject: [PATCH 037/168] Lazy upgrade of the Spilo image (#859) * initial implementation * describe forcing the rolling upgrade * make parameter name more descriptive * add missing pieces * address review * address review * fix bug in e2e tests * fix cluster name label in e2e test * raise test timeout * load spilo test image * use available spilo image * delete replica pod for lazy update test * fix e2e * fix e2e with a vengeance * lets wait for another 30m * print pod name in error msg * print pod name in error msg 2 * raise timeout, comment other tests * subsequent updates of config * add comma * fix e2e test * run unit tests before e2e * remove conflicting dependency * Revert "remove conflicting dependency" This reverts commit 65fc09054bf073755b15a3b5fa389e07d4f6f025. * improve cdp build * dont run unit before e2e tests * Revert "improve cdp build" This reverts commit e2a8fa12aa72b47e83901498a849f9eeed36eecc. Co-authored-by: Sergey Dudoladov Co-authored-by: Felix Kunde --- Makefile | 2 +- .../crds/operatorconfigurations.yaml | 2 + charts/postgres-operator/values-crd.yaml | 2 + charts/postgres-operator/values.yaml | 2 + docs/administrator.md | 11 +++ docs/reference/operator_parameters.md | 4 + e2e/Makefile | 2 +- e2e/tests/test_e2e.py | 80 ++++++++++++++++--- go.mod | 16 ++-- go.sum | 67 ++++++++++++---- manifests/configmap.yaml | 1 + manifests/operatorconfiguration.crd.yaml | 2 + ...gresql-operator-default-configuration.yaml | 5 +- pkg/apis/acid.zalan.do/v1/crds.go | 3 + .../v1/operator_configuration_type.go | 1 + pkg/cluster/cluster.go | 15 +++- pkg/cluster/sync.go | 45 +++++++++-- pkg/controller/operator_config.go | 1 + pkg/util/config/config.go | 1 + 19 files changed, 220 insertions(+), 42 deletions(-) diff --git a/Makefile b/Makefile index dc1c790fe..69dd240db 100644 --- a/Makefile +++ b/Makefile @@ -97,4 +97,4 @@ test: GO111MODULE=on go test ./... e2e: docker # build operator image to be tested - cd e2e; make tools test clean + cd e2e; make tools e2etest clean diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index aeb3d2150..71260bdb6 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -62,6 +62,8 @@ spec: type: string enable_crd_validation: type: boolean + enable_lazy_spilo_upgrade: + type: boolean enable_shm_volume: type: boolean etcd_host: diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 41b8dbefb..e6428c478 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -19,6 +19,8 @@ configTarget: "OperatorConfigurationCRD" configGeneral: # choose if deployment creates/updates CRDs with OpenAPIV3Validation enable_crd_validation: true + # update only the statefulsets without immediately doing the rolling update + enable_lazy_spilo_upgrade: false # start any new database pod without limitations on shm memory enable_shm_volume: true # etcd connection string for Patroni. Empty uses K8s-native DCS. diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 7f140f1de..e5cdcee47 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -19,6 +19,8 @@ configTarget: "ConfigMap" configGeneral: # choose if deployment creates/updates CRDs with OpenAPIV3Validation enable_crd_validation: "true" + # update only the statefulsets without immediately doing the rolling update + enable_lazy_spilo_upgrade: "false" # start any new database pod without limitations on shm memory enable_shm_volume: "true" # etcd connection string for Patroni. Empty uses K8s-native DCS. diff --git a/docs/administrator.md b/docs/administrator.md index ed96b7d35..45a328d38 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -458,6 +458,17 @@ from numerous escape characters in the latter log entry, view it in CLI with `PodTemplate` used by the operator is yet to be updated with the default values used internally in K8s. +The operator also support lazy updates of the Spilo image. That means the pod +template of a PG cluster's stateful set is updated immediately with the new +image, but no rolling update follows. This feature saves you a switchover - and +hence downtime - when you know pods are re-started later anyway, for instance +due to the node rotation. To force a rolling update, disable this mode by +setting the `enable_lazy_spilo_upgrade` to `false` in the operator configuration +and restart the operator pod. With the standard eager rolling updates the +operator checks during Sync all pods run images specified in their respective +statefulsets. The operator triggers a rolling upgrade for PG clusters that +violate this condition. + ## Logical backups The operator can manage K8s cron jobs to run logical backups of Postgres diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index d98a270e6..af6e93d25 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -75,6 +75,10 @@ Those are top-level keys, containing both leaf keys and groups. [OpenAPI v3 schema validation](https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#validation) The default is `true`. +* **enable_lazy_spilo_upgrade** + Instruct operator to update only the statefulsets with the new image without immediately doing the rolling update. The assumption is pods will be re-started later with the new image, for example due to the node rotation. + The default is `false`. + * **etcd_host** Etcd connection string for Patroni defined as `host:port`. Not required when Patroni native Kubernetes support is used. The default is empty (use diff --git a/e2e/Makefile b/e2e/Makefile index 77059f3eb..70a2ff4e9 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -44,5 +44,5 @@ tools: docker # install pinned version of 'kind' GO111MODULE=on go get sigs.k8s.io/kind@v0.5.1 -test: +e2etest: ./run.sh diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index f46c0577e..aa6f1205d 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -142,15 +142,6 @@ class EndToEndTestCase(unittest.TestCase): }) k8s.wait_for_pods_to_stop(pod_selector) - k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresqls', 'acid-minimal-cluster', - { - 'spec': { - 'enableConnectionPooler': True, - } - }) - k8s.wait_for_pod_start(pod_selector) except timeout_decorator.TimeoutError: print('Operator log: {}'.format(k8s.get_operator_log())) raise @@ -204,6 +195,66 @@ class EndToEndTestCase(unittest.TestCase): self.assertEqual(repl_svc_type, 'ClusterIP', "Expected ClusterIP service type for replica, found {}".format(repl_svc_type)) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_lazy_spilo_upgrade(self): + ''' + Test lazy upgrade for the Spilo image: operator changes a stateful set but lets pods run with the old image + until they are recreated for reasons other than operator's activity. That works because the operator configures + stateful sets to use "onDelete" pod update policy. + + The test covers: + 1) enabling lazy upgrade in existing operator deployment + 2) forcing the normal rolling upgrade by changing the operator configmap and restarting its pod + ''' + + k8s = self.k8s + + # update docker image in config and enable the lazy upgrade + conf_image = "registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p114" + patch_lazy_spilo_upgrade = { + "data": { + "docker_image": conf_image, + "enable_lazy_spilo_upgrade": "true" + } + } + k8s.update_config(patch_lazy_spilo_upgrade) + + pod0 = 'acid-minimal-cluster-0' + pod1 = 'acid-minimal-cluster-1' + + # restart the pod to get a container with the new image + k8s.api.core_v1.delete_namespaced_pod(pod0, 'default') + time.sleep(60) + + # lazy update works if the restarted pod and older pods run different Spilo versions + new_image = k8s.get_effective_pod_image(pod0) + old_image = k8s.get_effective_pod_image(pod1) + self.assertNotEqual(new_image, old_image, "Lazy updated failed: pods have the same image {}".format(new_image)) + + # sanity check + assert_msg = "Image {} of a new pod differs from {} in operator conf".format(new_image, conf_image) + self.assertEqual(new_image, conf_image, assert_msg) + + # clean up + unpatch_lazy_spilo_upgrade = { + "data": { + "enable_lazy_spilo_upgrade": "false", + } + } + k8s.update_config(unpatch_lazy_spilo_upgrade) + + # at this point operator will complete the normal rolling upgrade + # so we additonally test if disabling the lazy upgrade - forcing the normal rolling upgrade - works + + # XXX there is no easy way to wait until the end of Sync() + time.sleep(60) + + image0 = k8s.get_effective_pod_image(pod0) + image1 = k8s.get_effective_pod_image(pod1) + + assert_msg = "Disabling lazy upgrade failed: pods still have different images {} and {}".format(image0, image1) + self.assertEqual(image0, image1, assert_msg) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_logical_backup_cron_job(self): ''' @@ -594,7 +645,7 @@ class K8s: def wait_for_operator_pod_start(self): self. wait_for_pod_start("name=postgres-operator") - # HACK operator must register CRD / add existing PG clusters after pod start up + # HACK operator must register CRD and/or Sync existing PG clusters after start up # for local execution ~ 10 seconds suffices time.sleep(60) @@ -724,6 +775,15 @@ class K8s: stdout=subprocess.PIPE, stderr=subprocess.PIPE) + def get_effective_pod_image(self, pod_name, namespace='default'): + ''' + Get the Spilo image pod currently uses. In case of lazy rolling updates + it may differ from the one specified in the stateful set. + ''' + pod = self.api.core_v1.list_namespaced_pod( + namespace, label_selector="statefulset.kubernetes.io/pod-name=" + pod_name) + return pod.items[0].spec.containers[0].image + if __name__ == '__main__': unittest.main() diff --git a/go.mod b/go.mod index 8b9f55509..dc6389a1c 100644 --- a/go.mod +++ b/go.mod @@ -4,16 +4,20 @@ go 1.14 require ( github.com/aws/aws-sdk-go v1.29.33 + github.com/emicklei/go-restful v2.9.6+incompatible // indirect + github.com/evanphx/json-patch v4.5.0+incompatible // indirect + github.com/googleapis/gnostic v0.3.0 // indirect github.com/lib/pq v1.3.0 github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d github.com/r3labs/diff v0.0.0-20191120142937-b4ed99a31f5a github.com/sirupsen/logrus v1.5.0 github.com/stretchr/testify v1.4.0 - golang.org/x/tools v0.0.0-20200326210457-5d86d385bf88 // indirect + golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b // indirect gopkg.in/yaml.v2 v2.2.8 - k8s.io/api v0.18.0 - k8s.io/apiextensions-apiserver v0.18.0 - k8s.io/apimachinery v0.18.0 - k8s.io/client-go v0.18.0 - k8s.io/code-generator v0.18.0 + k8s.io/api v0.18.2 + k8s.io/apiextensions-apiserver v0.18.2 + k8s.io/apimachinery v0.18.2 + k8s.io/client-go v11.0.0+incompatible + k8s.io/code-generator v0.18.2 + sigs.k8s.io/kind v0.5.1 // indirect ) diff --git a/go.sum b/go.sum index e8607922b..22be07f7a 100644 --- a/go.sum +++ b/go.sum @@ -46,6 +46,7 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -62,10 +63,14 @@ github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkg github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.6+incompatible h1:tfrHha8zJ01ywiOEC1miGY8st1/igzWB8OmvPgoYX7w= +github.com/emicklei/go-restful v2.9.6+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= +github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -145,6 +150,7 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= @@ -155,10 +161,13 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gnostic v0.0.0-20170426233943-68f4ded48ba9/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.1.0 h1:rVsPeBmXbYv4If/cumu1AzZPwV58q433hvONV1UEZoI= github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0= +github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -174,10 +183,12 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -204,6 +215,7 @@ github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190620125010-da37f6c1e481/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= @@ -216,6 +228,7 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -228,9 +241,11 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= @@ -238,7 +253,9 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= @@ -257,6 +274,7 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q= github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= @@ -264,7 +282,9 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -277,6 +297,7 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -289,7 +310,7 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -361,6 +382,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f h1:25KHgbfyiSm6vwQLbM3zZIe1v9p/3ea4Rz+nnM5K/i4= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190621203818-d432491b9138/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 h1:HmbHVPwrPEKPGLAcHSrMe6+hqSUlvZU0rab6x5EXfGU= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -388,8 +410,8 @@ golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200326210457-5d86d385bf88 h1:F7fM2kxXfuWw820fa+MMCCLH6hmYe+jtLnZpwoiLK4Q= -golang.org/x/tools v0.0.0-20200326210457-5d86d385bf88/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b h1:zSzQJAznWxAh9fZxiPy2FZo+ZZEYoYFYYDYdOrU7AaM= +golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= @@ -431,31 +453,46 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.18.0 h1:lwYk8Vt7rsVTwjRU6pzEsa9YNhThbmbocQlKvNBB4EQ= -k8s.io/api v0.18.0/go.mod h1:q2HRQkfDzHMBZL9l/y9rH63PkQl4vae0xRT+8prbrK8= -k8s.io/apiextensions-apiserver v0.18.0 h1:HN4/P8vpGZFvB5SOMuPPH2Wt9Y/ryX+KRvIyAkchu1Q= -k8s.io/apiextensions-apiserver v0.18.0/go.mod h1:18Cwn1Xws4xnWQNC00FLq1E350b9lUF+aOdIWDOZxgo= -k8s.io/apimachinery v0.18.0 h1:fuPfYpk3cs1Okp/515pAf0dNhL66+8zk8RLbSX+EgAE= -k8s.io/apimachinery v0.18.0/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= -k8s.io/apiserver v0.18.0/go.mod h1:3S2O6FeBBd6XTo0njUrLxiqk8GNy6wWOftjhJcXYnjw= -k8s.io/client-go v0.18.0 h1:yqKw4cTUQraZK3fcVCMeSa+lqKwcjZ5wtcOIPnxQno4= -k8s.io/client-go v0.18.0/go.mod h1:uQSYDYs4WhVZ9i6AIoEZuwUggLVEF64HOD37boKAtF8= -k8s.io/code-generator v0.18.0 h1:0xIRWzym+qMgVpGmLESDeMfz/orwgxwxFFAo1xfGNtQ= -k8s.io/code-generator v0.18.0/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= -k8s.io/component-base v0.18.0/go.mod h1:u3BCg0z1uskkzrnAKFzulmYaEpZF7XC9Pf/uFyb1v2c= +k8s.io/api v0.0.0-20190313235455-40a48860b5ab/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= +k8s.io/api v0.0.0-20190409021203-6e4e0e4f393b/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= +k8s.io/api v0.18.2 h1:wG5g5ZmSVgm5B+eHMIbI9EGATS2L8Z72rda19RIEgY8= +k8s.io/api v0.18.2/go.mod h1:SJCWI7OLzhZSvbY7U8zwNl9UA4o1fizoug34OV/2r78= +k8s.io/apiextensions-apiserver v0.18.2 h1:I4v3/jAuQC+89L3Z7dDgAiN4EOjN6sbm6iBqQwHTah8= +k8s.io/apiextensions-apiserver v0.18.2/go.mod h1:q3faSnRGmYimiocj6cHQ1I3WpLqmDgJFlKL37fC4ZvY= +k8s.io/apimachinery v0.0.0-20190313205120-d7deff9243b1/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= +k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= +k8s.io/apimachinery v0.18.2 h1:44CmtbmkzVDAhCpRVSiP2R5PPrC2RtlIv/MoB8xpdRA= +k8s.io/apimachinery v0.18.2/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= +k8s.io/apiserver v0.18.2/go.mod h1:Xbh066NqrZO8cbsoenCwyDJ1OSi8Ag8I2lezeHxzwzw= +k8s.io/client-go v0.18.2 h1:aLB0iaD4nmwh7arT2wIn+lMnAq7OswjaejkQ8p9bBYE= +k8s.io/client-go v0.18.2/go.mod h1:Xcm5wVGXX9HAA2JJ2sSBUn3tCJ+4SVlCbl2MNNv+CIU= +k8s.io/client-go v11.0.0+incompatible h1:LBbX2+lOwY9flffWlJM7f1Ct8V2SRNiMRDFeiwnJo9o= +k8s.io/client-go v11.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= +k8s.io/code-generator v0.18.2 h1:C1Nn2JiMf244CvBDKVPX0W2mZFJkVBg54T8OV7/Imso= +k8s.io/code-generator v0.18.2/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= +k8s.io/component-base v0.18.2/go.mod h1:kqLlMuhJNHQ9lz8Z7V5bxUUtjFZnrypArGl58gmDfUM= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200114144118-36b2048a9120 h1:RPscN6KhmG54S33L+lr3GS+oD1jmchIU0ll519K6FA4= k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.3/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +k8s.io/kube-openapi v0.0.0-20190603182131-db7b694dc208/go.mod h1:nfDlWeOsu3pUf4yWGL+ERqohP4YsZcBJXWMK+gkzOA4= k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c h1:/KUFqjjqAcY4Us6luF5RDNZ16KJtb49HfR3ZHB9qYXM= k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89 h1:d4vVOjXm687F1iLSP2q3lyPPuyvTUt3aVoBpi2DqRsU= k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0= +sigs.k8s.io/kind v0.5.1 h1:BYnHEJ9DC+0Yjlyyehqd3xnKtEmFdLKU8QxqOqvQzdw= +sigs.k8s.io/kind v0.5.1/go.mod h1:L+Kcoo83/D1+ryU5P2VFbvYm0oqbkJn9zTZq0KNxW68= +sigs.k8s.io/kustomize/v3 v3.1.1-0.20190821175718-4b67a6de1296 h1:iQaIG5Dq+3qSiaFrJ/l/0MjjxKmdwyVNpKRYJwUe/+0= +sigs.k8s.io/kustomize/v3 v3.1.1-0.20190821175718-4b67a6de1296/go.mod h1:ztX4zYc/QIww3gSripwF7TBOarBTm5BvyAMem0kCzOE= +sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e h1:4Z09Hglb792X0kfOBBJUPFEyvVfQWrYT/l8h5EKA6JQ= +sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E= sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index cbc55b446..8719e76a1 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -34,6 +34,7 @@ data: # enable_crd_validation: "true" # enable_database_access: "true" # enable_init_containers: "true" + # enable_lazy_spilo_upgrade: "false" enable_master_load_balancer: "false" # enable_pod_antiaffinity: "false" # enable_pod_disruption_budget: "true" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 53097b96c..2d3e614b9 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -38,6 +38,8 @@ spec: type: string enable_crd_validation: type: boolean + enable_lazy_spilo_upgrade: + type: boolean enable_shm_volume: type: boolean etcd_host: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index b5765a0f4..78312b684 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -3,11 +3,12 @@ kind: OperatorConfiguration metadata: name: postgresql-operator-default-configuration configuration: + docker_image: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115 # enable_crd_validation: true + # enable_lazy_spilo_upgrade: false + # enable_shm_volume: true etcd_host: "" # kubernetes_use_configmaps: false - docker_image: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115 - # enable_shm_volume: true max_instances: -1 min_instances: -1 resync_period: 30m diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index bcb35c56c..36b0904d7 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -761,6 +761,9 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation "enable_crd_validation": { Type: "boolean", }, + "enable_lazy_spilo_upgrade": { + Type: "boolean", + }, "enable_shm_volume": { Type: "boolean", }, diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index c377a294b..4a0abf3ca 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -183,6 +183,7 @@ type OperatorLogicalBackupConfiguration struct { // OperatorConfigurationData defines the operation config type OperatorConfigurationData struct { EnableCRDValidation *bool `json:"enable_crd_validation,omitempty"` + EnableLazySpiloUpgrade bool `json:"enable_lazy_spilo_upgrade,omitempty"` EtcdHost string `json:"etcd_host,omitempty"` KubernetesUseConfigMaps bool `json:"kubernetes_use_configmaps,omitempty"` DockerImage string `json:"docker_image,omitempty"` diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 244916074..2e04cb137 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -470,6 +470,14 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa } } + // lazy Spilo update: modify the image in the statefulset itself but let its pods run with the old image + // until they are re-created for other reasons, for example node rotation + if c.OpConfig.EnableLazySpiloUpgrade && !reflect.DeepEqual(c.Statefulset.Spec.Template.Spec.Containers[0].Image, statefulSet.Spec.Template.Spec.Containers[0].Image) { + needsReplace = true + needsRollUpdate = false + reasons = append(reasons, "lazy Spilo update: new statefulset's pod image doesn't match the current one") + } + if needsRollUpdate || needsReplace { match = false } @@ -501,8 +509,6 @@ func (c *Cluster) compareContainers(description string, setA, setB []v1.Containe checks := []containerCheck{ newCheck("new statefulset %s's %s (index %d) name doesn't match the current one", func(a, b v1.Container) bool { return a.Name != b.Name }), - newCheck("new statefulset %s's %s (index %d) image doesn't match the current one", - func(a, b v1.Container) bool { return a.Image != b.Image }), newCheck("new statefulset %s's %s (index %d) ports don't match the current one", func(a, b v1.Container) bool { return !reflect.DeepEqual(a.Ports, b.Ports) }), newCheck("new statefulset %s's %s (index %d) resources don't match the current ones", @@ -513,6 +519,11 @@ func (c *Cluster) compareContainers(description string, setA, setB []v1.Containe func(a, b v1.Container) bool { return !reflect.DeepEqual(a.EnvFrom, b.EnvFrom) }), } + if !c.OpConfig.EnableLazySpiloUpgrade { + checks = append(checks, newCheck("new statefulset %s's %s (index %d) image doesn't match the current one", + func(a, b v1.Container) bool { return a.Image != b.Image })) + } + for index, containerA := range setA { containerB := setB[index] for _, check := range checks { diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index a9a2b5177..0eb02631c 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -4,17 +4,17 @@ import ( "context" "fmt" - batchv1beta1 "k8s.io/api/batch/v1beta1" - v1 "k8s.io/api/core/v1" - policybeta1 "k8s.io/api/policy/v1beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" "github.com/zalando/postgres-operator/pkg/util/volumes" + appsv1 "k8s.io/api/apps/v1" + batchv1beta1 "k8s.io/api/batch/v1beta1" + v1 "k8s.io/api/core/v1" + policybeta1 "k8s.io/api/policy/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // Sync syncs the cluster, making sure the actual Kubernetes objects correspond to what is defined in the manifest. @@ -252,6 +252,28 @@ func (c *Cluster) syncPodDisruptionBudget(isUpdate bool) error { return nil } +func (c *Cluster) mustUpdatePodsAfterLazyUpdate(desiredSset *appsv1.StatefulSet) (bool, error) { + + pods, err := c.listPods() + if err != nil { + return false, fmt.Errorf("could not list pods of the statefulset: %v", err) + } + + for _, pod := range pods { + + effectivePodImage := pod.Spec.Containers[0].Image + ssImage := desiredSset.Spec.Template.Spec.Containers[0].Image + + if ssImage != effectivePodImage { + c.logger.Infof("not all pods were re-started when the lazy upgrade was enabled; forcing the rolling upgrade now") + return true, nil + } + + } + + return false, nil +} + func (c *Cluster) syncStatefulSet() error { var ( podsRollingUpdateRequired bool @@ -330,6 +352,19 @@ func (c *Cluster) syncStatefulSet() error { } } } + + if !podsRollingUpdateRequired && !c.OpConfig.EnableLazySpiloUpgrade { + // even if desired and actual statefulsets match + // there still may be not up-to-date pods on condition + // (a) the lazy update was just disabled + // and + // (b) some of the pods were not restarted when the lazy update was still in place + podsRollingUpdateRequired, err = c.mustUpdatePodsAfterLazyUpdate(desiredSS) + if err != nil { + return fmt.Errorf("could not list pods of the statefulset: %v", err) + } + } + } // Apply special PostgreSQL parameters that can only be set via the Patroni API. diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index c1756604b..f66eafb1e 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -34,6 +34,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur // general config result.EnableCRDValidation = fromCRD.EnableCRDValidation + result.EnableLazySpiloUpgrade = fromCRD.EnableLazySpiloUpgrade result.EtcdHost = fromCRD.EtcdHost result.KubernetesUseConfigMaps = fromCRD.KubernetesUseConfigMaps result.DockerImage = fromCRD.DockerImage diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 2bdeb8e1f..37ba947d6 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -157,6 +157,7 @@ type Config struct { ProtectedRoles []string `name:"protected_role_names" default:"admin"` PostgresSuperuserTeams []string `name:"postgres_superuser_teams" default:""` SetMemoryRequestToLimit bool `name:"set_memory_request_to_limit" default:"false"` + EnableLazySpiloUpgrade bool `name:"enable_lazy_spilo_upgrade" default:"false"` } // MustMarshal marshals the config or panics From d76203b3f9c690f64aa5c3b3f395c78935de96e8 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 29 Apr 2020 10:56:06 +0200 Subject: [PATCH 038/168] Bootstrapped databases with best practice role setup (#843) * PreparedDatabases with default role setup * merge changes from master * include preparedDatabases spec check when syncing databases * create a default preparedDB if not specified * add more default privileges for schemas * use empty brackets block for undefined objects * cover more default privilege scenarios and always define admin role * add DefaultUsers flag * support extensions and defaultUsers for preparedDatabases * remove exact version in deployment manifest * enable CRD validation for new field * update generated code * reflect code review * fix typo in SQL command * add documentation for preparedDatabases feature + minor changes * some datname should stay * add unit tests * reflect some feedback * init users for preparedDatabases also on update * only change DB default privileges on creation * add one more section in user docs * one more sentence --- .../postgres-operator/crds/postgresqls.yaml | 20 ++ docs/user.md | 169 ++++++++++++- manifests/complete-postgres-manifest.yaml | 11 + manifests/minimal-postgres-manifest.yaml | 2 + manifests/postgresql.crd.yaml | 20 ++ pkg/apis/acid.zalan.do/v1/crds.go | 37 +++ pkg/apis/acid.zalan.do/v1/postgresql_type.go | 50 ++-- .../acid.zalan.do/v1/zz_generated.deepcopy.go | 58 +++++ pkg/cluster/cluster.go | 118 ++++++++- pkg/cluster/cluster_test.go | 89 ++++++- pkg/cluster/database.go | 225 ++++++++++++++++-- pkg/cluster/k8sres.go | 7 + pkg/cluster/sync.go | 138 ++++++++++- pkg/controller/types.go | 3 +- pkg/spec/types.go | 3 + pkg/util/constants/roles.go | 4 + pkg/util/users/users.go | 38 ++- 17 files changed, 927 insertions(+), 65 deletions(-) diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 78850ee3b..fdbcf8304 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -273,6 +273,26 @@ spec: type: object additionalProperties: type: string + preparedDatabases: + type: object + additionalProperties: + type: object + properties: + defaultUsers: + type: boolean + extensions: + type: object + additionalProperties: + type: string + schemas: + type: object + additionalProperties: + type: object + properties: + defaultUsers: + type: boolean + defaultRoles: + type: boolean replicaLoadBalancer: # deprecated type: boolean resources: diff --git a/docs/user.md b/docs/user.md index d7e6add0a..2d9f2be6a 100644 --- a/docs/user.md +++ b/docs/user.md @@ -94,7 +94,10 @@ created on every cluster managed by the operator. * `teams API roles`: automatically create users for every member of the team owning the database cluster. -In the next sections, we will cover those use cases in more details. +In the next sections, we will cover those use cases in more details. Note, that +the Postgres Operator can also create databases with pre-defined owner, reader +and writer roles which saves you the manual setup. Read more in the next +chapter. ### Manifest roles @@ -216,6 +219,166 @@ to choose superusers, group roles, [PAM configuration](https://github.com/CyberD etc. An OAuth2 token can be passed to the Teams API via a secret. The name for this secret is configurable with the `oauth_token_secret_name` parameter. +## Prepared databases with roles and default privileges + +The `users` section in the manifests only allows for creating database roles +with global privileges. Fine-grained data access control or role membership can +not be defined and must be set up by the user in the database. But, the Postgres +Operator offers a separate section to specify `preparedDatabases` that will be +created with pre-defined owner, reader and writer roles for each individual +database and, optionally, for each database schema, too. `preparedDatabases` +also enable users to specify PostgreSQL extensions that shall be created in a +given database schema. + +### Default database and schema + +A prepared database is already created by adding an empty `preparedDatabases` +section to the manifest. The database will then be called like the Postgres +cluster manifest (`-` are replaced with `_`) and will also contain a schema +called `data`. + +```yaml +spec: + preparedDatabases: {} +``` + +### Default NOLOGIN roles + +Given an example with a specified database and schema: + +```yaml +spec: + preparedDatabases: + foo: + schemas: + bar: {} +``` + +Postgres Operator will create the following NOLOGIN roles: + +| Role name | Member of | Admin | +| -------------- | -------------- | ------------- | +| foo_owner | | admin | +| foo_reader | | foo_owner | +| foo_writer | foo_reader | foo_owner | +| foo_bar_owner | | foo_owner | +| foo_bar_reader | | foo_bar_owner | +| foo_bar_writer | foo_bar_reader | foo_bar_owner | + +The `_owner` role is the database owner and should be used when creating +new database objects. All members of the `admin` role, e.g. teams API roles, can +become the owner with the `SET ROLE` command. [Default privileges](https://www.postgresql.org/docs/12/sql-alterdefaultprivileges.html) +are configured for the owner role so that the `_reader` role +automatically gets read-access (SELECT) to new tables and sequences and the +`_writer` receives write-access (INSERT, UPDATE, DELETE on tables, +USAGE and UPDATE on sequences). Both get USAGE on types and EXECUTE on +functions. + +The same principle applies for database schemas which are owned by the +`__owner` role. `__reader` is read-only, +`__writer` has write access and inherit reading from the reader +role. Note, that the `_*` roles have access incl. default privileges on +all schemas, too. If you don't need the dedicated schema roles - i.e. you only +use one schema - you can disable the creation like this: + +```yaml +spec: + preparedDatabases: + foo: + schemas: + bar: + defaultRoles: false +``` + +Then, the schemas are owned by the database owner, too. + +### Default LOGIN roles + +The roles described in the previous paragraph can be granted to LOGIN roles from +the `users` section in the manifest. Optionally, the Postgres Operator can also +create default LOGIN roles for the database an each schema individually. These +roles will get the `_user` suffix and they inherit all rights from their NOLOGIN +counterparts. + +| Role name | Member of | Admin | +| ------------------- | -------------- | ------------- | +| foo_owner_user | foo_owner | admin | +| foo_reader_user | foo_reader | foo_owner | +| foo_writer_user | foo_writer | foo_owner | +| foo_bar_owner_user | foo_bar_owner | foo_owner | +| foo_bar_reader_user | foo_bar_reader | foo_bar_owner | +| foo_bar_writer_user | foo_bar_writer | foo_bar_owner | + +These default users are enabled in the manifest with the `defaultUsers` flag: + +```yaml +spec: + preparedDatabases: + foo: + defaultUsers: true + schemas: + bar: + defaultUsers: true +``` + +### Database extensions + +Prepared databases also allow for creating Postgres extensions. They will be +created by the database owner in the specified schema. + +```yaml +spec: + preparedDatabases: + foo: + extensions: + pg_partman: public + postgis: data +``` + +Some extensions require SUPERUSER rights on creation unless they are not +whitelisted by the [pgextwlist](https://github.com/dimitri/pgextwlist) +extension, that is shipped with the Spilo image. To see which extensions are +on the list check the `extwlist.extension` parameter in the postgresql.conf +file. + +```bash +SHOW extwlist.extensions; +``` + +Make sure that `pgextlist` is also listed under `shared_preload_libraries` in +the PostgreSQL configuration. Then the database owner should be able to create +the extension specified in the manifest. + +### From `databases` to `preparedDatabases` + +If you wish to create the role setup described above for databases listed under +the `databases` key, you have to make sure that the owner role follows the +`_owner` naming convention of `preparedDatabases`. As roles are synced +first, this can be done with one edit: + +```yaml +# before +spec: + databases: + foo: db_owner + +# after +spec: + databases: + foo: foo_owner + preparedDatabases: + foo: + schemas: + my_existing_schema: {} +``` + +Adding existing database schemas to the manifest to create roles for them as +well is up the user and not done by the operator. Remember that if you don't +specify any schema a new database schema called `data` will be created. When +everything got synced (roles, schemas, extensions), you are free to remove the +database from the `databases` section. Note, that the operator does not delete +database objects or revoke privileges when removed from the manifest. + ## Resource definition The compute resources to be used for the Postgres containers in the pods can be @@ -586,8 +749,8 @@ don't know the value, use `103` which is the GID from the default spilo image OpenShift allocates the users and groups dynamically (based on scc), and their range is different in every namespace. Due to this dynamic behaviour, it's not trivial to know at deploy time the uid/gid of the user in the cluster. -Therefore, instead of using a global `spilo_fsgroup` setting, use the `spiloFSGroup` field -per Postgres cluster. +Therefore, instead of using a global `spilo_fsgroup` setting, use the +`spiloFSGroup` field per Postgres cluster. Upload the cert as a kubernetes secret: ```sh diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index f031a5d5b..d436695e8 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -21,6 +21,17 @@ spec: - 127.0.0.1/32 databases: foo: zalando + preparedDatabases: + bar: + defaultUsers: true + extensions: + pg_partman: public + pgcrypto: public + schemas: + data: {} + history: + defaultRoles: true + defaultUsers: false postgresql: version: "12" parameters: # Expert section diff --git a/manifests/minimal-postgres-manifest.yaml b/manifests/minimal-postgres-manifest.yaml index af0add8e6..4dd6b7ee4 100644 --- a/manifests/minimal-postgres-manifest.yaml +++ b/manifests/minimal-postgres-manifest.yaml @@ -15,5 +15,7 @@ spec: foo_user: [] # role for application foo databases: foo: zalando # dbname: owner + preparedDatabases: + bar: {} postgresql: version: "12" diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 1ee6a1ae5..e62204c40 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -237,6 +237,26 @@ spec: type: object additionalProperties: type: string + preparedDatabases: + type: object + additionalProperties: + type: object + properties: + defaultUsers: + type: boolean + extensions: + type: object + additionalProperties: + type: string + schemas: + type: object + additionalProperties: + type: object + properties: + defaultUsers: + type: boolean + defaultRoles: + type: boolean replicaLoadBalancer: # deprecated type: boolean resources: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 36b0904d7..35037ec3c 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -421,6 +421,43 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, }, }, + "preparedDatabases": { + Type: "object", + AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "defaultUsers": { + Type: "boolean", + }, + "extensions": { + Type: "object", + AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "string", + }, + }, + }, + "schemas": { + Type: "object", + AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "defaultUsers": { + Type: "boolean", + }, + "defaultRoles": { + Type: "boolean", + }, + }, + }, + }, + }, + }, + }, + }, + }, "replicaLoadBalancer": { Type: "boolean", Description: "Deprecated", diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index e36009208..5df82e947 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -50,24 +50,25 @@ type PostgresSpec struct { // load balancers' source ranges are the same for master and replica services AllowedSourceRanges []string `json:"allowedSourceRanges"` - NumberOfInstances int32 `json:"numberOfInstances"` - Users map[string]UserFlags `json:"users"` - MaintenanceWindows []MaintenanceWindow `json:"maintenanceWindows,omitempty"` - Clone CloneDescription `json:"clone"` - ClusterName string `json:"-"` - Databases map[string]string `json:"databases,omitempty"` - Tolerations []v1.Toleration `json:"tolerations,omitempty"` - Sidecars []Sidecar `json:"sidecars,omitempty"` - InitContainers []v1.Container `json:"initContainers,omitempty"` - PodPriorityClassName string `json:"podPriorityClassName,omitempty"` - ShmVolume *bool `json:"enableShmVolume,omitempty"` - EnableLogicalBackup bool `json:"enableLogicalBackup,omitempty"` - LogicalBackupSchedule string `json:"logicalBackupSchedule,omitempty"` - StandbyCluster *StandbyDescription `json:"standby"` - PodAnnotations map[string]string `json:"podAnnotations"` - ServiceAnnotations map[string]string `json:"serviceAnnotations"` - TLS *TLSDescription `json:"tls"` - AdditionalVolumes []AdditionalVolume `json:"additionalVolumes,omitempty"` + NumberOfInstances int32 `json:"numberOfInstances"` + Users map[string]UserFlags `json:"users"` + MaintenanceWindows []MaintenanceWindow `json:"maintenanceWindows,omitempty"` + Clone CloneDescription `json:"clone"` + ClusterName string `json:"-"` + Databases map[string]string `json:"databases,omitempty"` + PreparedDatabases map[string]PreparedDatabase `json:"preparedDatabases,omitempty"` + Tolerations []v1.Toleration `json:"tolerations,omitempty"` + Sidecars []Sidecar `json:"sidecars,omitempty"` + InitContainers []v1.Container `json:"initContainers,omitempty"` + PodPriorityClassName string `json:"podPriorityClassName,omitempty"` + ShmVolume *bool `json:"enableShmVolume,omitempty"` + EnableLogicalBackup bool `json:"enableLogicalBackup,omitempty"` + LogicalBackupSchedule string `json:"logicalBackupSchedule,omitempty"` + StandbyCluster *StandbyDescription `json:"standby"` + PodAnnotations map[string]string `json:"podAnnotations"` + ServiceAnnotations map[string]string `json:"serviceAnnotations"` + TLS *TLSDescription `json:"tls"` + AdditionalVolumes []AdditionalVolume `json:"additionalVolumes,omitempty"` // deprecated json tags InitContainersOld []v1.Container `json:"init_containers,omitempty"` @@ -84,6 +85,19 @@ type PostgresqlList struct { Items []Postgresql `json:"items"` } +// PreparedDatabase describes elements to be bootstrapped +type PreparedDatabase struct { + PreparedSchemas map[string]PreparedSchema `json:"schemas,omitempty"` + DefaultUsers bool `json:"defaultUsers,omitempty" defaults:"false"` + Extensions map[string]string `json:"extensions,omitempty"` +} + +// PreparedSchema describes elements to be bootstrapped per schema +type PreparedSchema struct { + DefaultRoles *bool `json:"defaultRoles,omitempty" defaults:"true"` + DefaultUsers bool `json:"defaultUsers,omitempty" defaults:"false"` +} + // MaintenanceWindow describes the time window when the operator is allowed to do maintenance on a cluster. type MaintenanceWindow struct { Everyday bool diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index e2e1d5bd1..5b4d6cdcd 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -570,6 +570,13 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { (*out)[key] = val } } + if in.PreparedDatabases != nil { + in, out := &in.PreparedDatabases, &out.PreparedDatabases + *out = make(map[string]PreparedDatabase, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } if in.Tolerations != nil { in, out := &in.Tolerations, &out.Tolerations *out = make([]corev1.Toleration, len(*in)) @@ -763,6 +770,57 @@ func (in *PostgresqlParam) DeepCopy() *PostgresqlParam { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PreparedDatabase) DeepCopyInto(out *PreparedDatabase) { + *out = *in + if in.PreparedSchemas != nil { + in, out := &in.PreparedSchemas, &out.PreparedSchemas + *out = make(map[string]PreparedSchema, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } + if in.Extensions != nil { + in, out := &in.Extensions, &out.Extensions + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PreparedDatabase. +func (in *PreparedDatabase) DeepCopy() *PreparedDatabase { + if in == nil { + return nil + } + out := new(PreparedDatabase) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PreparedSchema) DeepCopyInto(out *PreparedSchema) { + *out = *in + if in.DefaultRoles != nil { + in, out := &in.DefaultRoles, &out.DefaultRoles + *out = new(bool) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PreparedSchema. +func (in *PreparedSchema) DeepCopy() *PreparedSchema { + if in == nil { + return nil + } + out := new(PreparedSchema) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ResourceDescription) DeepCopyInto(out *ResourceDescription) { *out = *in diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 2e04cb137..387107540 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -9,6 +9,7 @@ import ( "fmt" "reflect" "regexp" + "strings" "sync" "time" @@ -227,6 +228,10 @@ func (c *Cluster) initUsers() error { return fmt.Errorf("could not init infrastructure roles: %v", err) } + if err := c.initPreparedDatabaseRoles(); err != nil { + return fmt.Errorf("could not init default users: %v", err) + } + if err := c.initRobotUsers(); err != nil { return fmt.Errorf("could not init robot users: %v", err) } @@ -343,6 +348,9 @@ func (c *Cluster) Create() error { if err = c.syncDatabases(); err != nil { return fmt.Errorf("could not sync databases: %v", err) } + if err = c.syncPreparedDatabases(); err != nil { + return fmt.Errorf("could not sync prepared databases: %v", err) + } c.logger.Infof("databases have been successfully created") } @@ -649,7 +657,8 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { // connection pooler needs one system user created, which is done in // initUsers. Check if it needs to be called. - sameUsers := reflect.DeepEqual(oldSpec.Spec.Users, newSpec.Spec.Users) + sameUsers := reflect.DeepEqual(oldSpec.Spec.Users, newSpec.Spec.Users) && + reflect.DeepEqual(oldSpec.Spec.PreparedDatabases, newSpec.Spec.PreparedDatabases) needConnectionPooler := c.needConnectionPoolerWorker(&newSpec.Spec) if !sameUsers || needConnectionPooler { c.logger.Debugf("syncing secrets") @@ -766,19 +775,28 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { c.logger.Errorf("could not sync roles: %v", err) updateFailed = true } - if !reflect.DeepEqual(oldSpec.Spec.Databases, newSpec.Spec.Databases) { + if !reflect.DeepEqual(oldSpec.Spec.Databases, newSpec.Spec.Databases) || + !reflect.DeepEqual(oldSpec.Spec.PreparedDatabases, newSpec.Spec.PreparedDatabases) { c.logger.Infof("syncing databases") if err := c.syncDatabases(); err != nil { c.logger.Errorf("could not sync databases: %v", err) updateFailed = true } } + if !reflect.DeepEqual(oldSpec.Spec.PreparedDatabases, newSpec.Spec.PreparedDatabases) { + c.logger.Infof("syncing prepared databases") + if err := c.syncPreparedDatabases(); err != nil { + c.logger.Errorf("could not sync prepared databases: %v", err) + updateFailed = true + } + } } // sync connection pooler if _, err := c.syncConnectionPooler(oldSpec, newSpec, c.installLookupFunction); err != nil { - return fmt.Errorf("could not sync connection pooler: %v", err) + c.logger.Errorf("could not sync connection pooler: %v", err) + updateFailed = true } return nil @@ -949,6 +967,100 @@ func (c *Cluster) initSystemUsers() { } } +func (c *Cluster) initPreparedDatabaseRoles() error { + + if c.Spec.PreparedDatabases != nil && len(c.Spec.PreparedDatabases) == 0 { // TODO: add option to disable creating such a default DB + c.Spec.PreparedDatabases = map[string]acidv1.PreparedDatabase{strings.Replace(c.Name, "-", "_", -1): {}} + } + + // create maps with default roles/users as keys and their membership as values + defaultRoles := map[string]string{ + constants.OwnerRoleNameSuffix: "", + constants.ReaderRoleNameSuffix: "", + constants.WriterRoleNameSuffix: constants.ReaderRoleNameSuffix, + } + defaultUsers := map[string]string{ + constants.OwnerRoleNameSuffix + constants.UserRoleNameSuffix: constants.OwnerRoleNameSuffix, + constants.ReaderRoleNameSuffix + constants.UserRoleNameSuffix: constants.ReaderRoleNameSuffix, + constants.WriterRoleNameSuffix + constants.UserRoleNameSuffix: constants.WriterRoleNameSuffix, + } + + for preparedDbName, preparedDB := range c.Spec.PreparedDatabases { + // default roles per database + if err := c.initDefaultRoles(defaultRoles, "admin", preparedDbName); err != nil { + return fmt.Errorf("could not initialize default roles for database %s: %v", preparedDbName, err) + } + if preparedDB.DefaultUsers { + if err := c.initDefaultRoles(defaultUsers, "admin", preparedDbName); err != nil { + return fmt.Errorf("could not initialize default roles for database %s: %v", preparedDbName, err) + } + } + + // default roles per database schema + preparedSchemas := preparedDB.PreparedSchemas + if len(preparedDB.PreparedSchemas) == 0 { + preparedSchemas = map[string]acidv1.PreparedSchema{"data": {DefaultRoles: util.True()}} + } + for preparedSchemaName, preparedSchema := range preparedSchemas { + if preparedSchema.DefaultRoles == nil || *preparedSchema.DefaultRoles { + if err := c.initDefaultRoles(defaultRoles, + preparedDbName+constants.OwnerRoleNameSuffix, + preparedDbName+"_"+preparedSchemaName); err != nil { + return fmt.Errorf("could not initialize default roles for database schema %s: %v", preparedSchemaName, err) + } + if preparedSchema.DefaultUsers { + if err := c.initDefaultRoles(defaultUsers, + preparedDbName+constants.OwnerRoleNameSuffix, + preparedDbName+"_"+preparedSchemaName); err != nil { + return fmt.Errorf("could not initialize default users for database schema %s: %v", preparedSchemaName, err) + } + } + } + } + } + return nil +} + +func (c *Cluster) initDefaultRoles(defaultRoles map[string]string, admin, prefix string) error { + + for defaultRole, inherits := range defaultRoles { + + roleName := prefix + defaultRole + + flags := []string{constants.RoleFlagNoLogin} + if defaultRole[len(defaultRole)-5:] == constants.UserRoleNameSuffix { + flags = []string{constants.RoleFlagLogin} + } + + memberOf := make([]string, 0) + if inherits != "" { + memberOf = append(memberOf, prefix+inherits) + } + + adminRole := "" + if strings.Contains(defaultRole, constants.OwnerRoleNameSuffix) { + adminRole = admin + } else { + adminRole = prefix + constants.OwnerRoleNameSuffix + } + + newRole := spec.PgUser{ + Origin: spec.RoleOriginBootstrap, + Name: roleName, + Password: util.RandomPassword(constants.PasswordLength), + Flags: flags, + MemberOf: memberOf, + AdminRole: adminRole, + } + if currentRole, present := c.pgUsers[roleName]; present { + c.pgUsers[roleName] = c.resolveNameConflict(¤tRole, &newRole) + } else { + c.pgUsers[roleName] = newRole + } + } + return nil +} + func (c *Cluster) initRobotUsers() error { for username, userFlags := range c.Spec.Users { if !isValidUsername(username) { diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index 84ec04e3e..539038bff 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -13,6 +13,7 @@ import ( "github.com/zalando/postgres-operator/pkg/util/k8sutil" "github.com/zalando/postgres-operator/pkg/util/teams" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" ) @@ -35,7 +36,7 @@ var cl = New( }, }, k8sutil.NewMockKubernetesClient(), - acidv1.Postgresql{}, + acidv1.Postgresql{ObjectMeta: metav1.ObjectMeta{Name: "acid-test", Namespace: "test"}}, logger, eventRecorder, ) @@ -760,3 +761,89 @@ func TestInitSystemUsers(t *testing.T) { t.Errorf("%s, System users are not allowed to be a connection pool user", testName) } } + +func TestPreparedDatabases(t *testing.T) { + testName := "TestDefaultPreparedDatabase" + + cl.Spec.PreparedDatabases = map[string]acidv1.PreparedDatabase{} + cl.initPreparedDatabaseRoles() + + for _, role := range []string{"acid_test_owner", "acid_test_reader", "acid_test_writer", + "acid_test_data_owner", "acid_test_data_reader", "acid_test_data_writer"} { + if _, exist := cl.pgUsers[role]; !exist { + t.Errorf("%s, default role %q for prepared database not present", testName, role) + } + } + + testName = "TestPreparedDatabaseWithSchema" + + cl.Spec.PreparedDatabases = map[string]acidv1.PreparedDatabase{ + "foo": { + DefaultUsers: true, + PreparedSchemas: map[string]acidv1.PreparedSchema{ + "bar": { + DefaultUsers: true, + }, + }, + }, + } + cl.initPreparedDatabaseRoles() + + for _, role := range []string{ + "foo_owner", "foo_reader", "foo_writer", + "foo_owner_user", "foo_reader_user", "foo_writer_user", + "foo_bar_owner", "foo_bar_reader", "foo_bar_writer", + "foo_bar_owner_user", "foo_bar_reader_user", "foo_bar_writer_user"} { + if _, exist := cl.pgUsers[role]; !exist { + t.Errorf("%s, default role %q for prepared database not present", testName, role) + } + } + + roleTests := []struct { + subTest string + role string + memberOf string + admin string + }{ + { + subTest: "Test admin role of owner", + role: "foo_owner", + memberOf: "", + admin: "admin", + }, + { + subTest: "Test writer is a member of reader", + role: "foo_writer", + memberOf: "foo_reader", + admin: "foo_owner", + }, + { + subTest: "Test reader LOGIN role", + role: "foo_reader_user", + memberOf: "foo_reader", + admin: "foo_owner", + }, + { + subTest: "Test schema owner", + role: "foo_bar_owner", + memberOf: "", + admin: "foo_owner", + }, + { + subTest: "Test schema writer LOGIN role", + role: "foo_bar_writer_user", + memberOf: "foo_bar_writer", + admin: "foo_bar_owner", + }, + } + + for _, tt := range roleTests { + user := cl.pgUsers[tt.role] + if (tt.memberOf == "" && len(user.MemberOf) > 0) || (tt.memberOf != "" && user.MemberOf[0] != tt.memberOf) { + t.Errorf("%s, incorrect membership for default role %q. Expected %q, got %q", tt.subTest, tt.role, tt.memberOf, user.MemberOf[0]) + } + if user.AdminRole != tt.admin { + t.Errorf("%s, incorrect admin role for default role %q. Expected %q, got %q", tt.subTest, tt.role, tt.admin, user.AdminRole) + } + } +} diff --git a/pkg/cluster/database.go b/pkg/cluster/database.go index 28f97b5cc..75e2d2097 100644 --- a/pkg/cluster/database.go +++ b/pkg/cluster/database.go @@ -27,9 +27,35 @@ const ( WHERE a.rolname = ANY($1) ORDER BY 1;` - getDatabasesSQL = `SELECT datname, pg_get_userbyid(datdba) AS owner FROM pg_database;` - createDatabaseSQL = `CREATE DATABASE "%s" OWNER "%s";` - alterDatabaseOwnerSQL = `ALTER DATABASE "%s" OWNER TO "%s";` + getDatabasesSQL = `SELECT datname, pg_get_userbyid(datdba) AS owner FROM pg_database;` + getSchemasSQL = `SELECT n.nspname AS dbschema FROM pg_catalog.pg_namespace n + WHERE n.nspname !~ '^pg_' AND n.nspname <> 'information_schema' ORDER BY 1` + getExtensionsSQL = `SELECT e.extname, n.nspname FROM pg_catalog.pg_extension e + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = e.extnamespace ORDER BY 1;` + + createDatabaseSQL = `CREATE DATABASE "%s" OWNER "%s";` + createDatabaseSchemaSQL = `SET ROLE TO "%s"; CREATE SCHEMA IF NOT EXISTS "%s" AUTHORIZATION "%s"` + alterDatabaseOwnerSQL = `ALTER DATABASE "%s" OWNER TO "%s";` + createExtensionSQL = `CREATE EXTENSION IF NOT EXISTS "%s" SCHEMA "%s"` + alterExtensionSQL = `ALTER EXTENSION "%s" SET SCHEMA "%s"` + + globalDefaultPrivilegesSQL = `SET ROLE TO "%s"; + ALTER DEFAULT PRIVILEGES GRANT USAGE ON SCHEMAS TO "%s","%s"; + ALTER DEFAULT PRIVILEGES GRANT SELECT ON TABLES TO "%s"; + ALTER DEFAULT PRIVILEGES GRANT SELECT ON SEQUENCES TO "%s"; + ALTER DEFAULT PRIVILEGES GRANT INSERT, UPDATE, DELETE ON TABLES TO "%s"; + ALTER DEFAULT PRIVILEGES GRANT USAGE, UPDATE ON SEQUENCES TO "%s"; + ALTER DEFAULT PRIVILEGES GRANT EXECUTE ON FUNCTIONS TO "%s","%s"; + ALTER DEFAULT PRIVILEGES GRANT USAGE ON TYPES TO "%s","%s";` + schemaDefaultPrivilegesSQL = `SET ROLE TO "%s"; + GRANT USAGE ON SCHEMA "%s" TO "%s","%s"; + ALTER DEFAULT PRIVILEGES IN SCHEMA "%s" GRANT SELECT ON TABLES TO "%s"; + ALTER DEFAULT PRIVILEGES IN SCHEMA "%s" GRANT SELECT ON SEQUENCES TO "%s"; + ALTER DEFAULT PRIVILEGES IN SCHEMA "%s" GRANT INSERT, UPDATE, DELETE ON TABLES TO "%s"; + ALTER DEFAULT PRIVILEGES IN SCHEMA "%s" GRANT USAGE, UPDATE ON SEQUENCES TO "%s"; + ALTER DEFAULT PRIVILEGES IN SCHEMA "%s" GRANT EXECUTE ON FUNCTIONS TO "%s","%s"; + ALTER DEFAULT PRIVILEGES IN SCHEMA "%s" GRANT USAGE ON TYPES TO "%s","%s";` + connectionPoolerLookup = ` CREATE SCHEMA IF NOT EXISTS {{.pooler_schema}}; @@ -221,43 +247,141 @@ func (c *Cluster) getDatabases() (dbs map[string]string, err error) { } // executeCreateDatabase creates new database with the given owner. -// The caller is responsible for openinging and closing the database connection. -func (c *Cluster) executeCreateDatabase(datname, owner string) error { - return c.execCreateOrAlterDatabase(datname, owner, createDatabaseSQL, +// The caller is responsible for opening and closing the database connection. +func (c *Cluster) executeCreateDatabase(databaseName, owner string) error { + return c.execCreateOrAlterDatabase(databaseName, owner, createDatabaseSQL, "creating database", "create database") } -// executeCreateDatabase changes the owner of the given database. -// The caller is responsible for openinging and closing the database connection. -func (c *Cluster) executeAlterDatabaseOwner(datname string, owner string) error { - return c.execCreateOrAlterDatabase(datname, owner, alterDatabaseOwnerSQL, +// executeAlterDatabaseOwner changes the owner of the given database. +// The caller is responsible for opening and closing the database connection. +func (c *Cluster) executeAlterDatabaseOwner(databaseName string, owner string) error { + return c.execCreateOrAlterDatabase(databaseName, owner, alterDatabaseOwnerSQL, "changing owner for database", "alter database owner") } -func (c *Cluster) execCreateOrAlterDatabase(datname, owner, statement, doing, operation string) error { - if !c.databaseNameOwnerValid(datname, owner) { +func (c *Cluster) execCreateOrAlterDatabase(databaseName, owner, statement, doing, operation string) error { + if !c.databaseNameOwnerValid(databaseName, owner) { return nil } - c.logger.Infof("%s %q owner %q", doing, datname, owner) - if _, err := c.pgDb.Exec(fmt.Sprintf(statement, datname, owner)); err != nil { + c.logger.Infof("%s %q owner %q", doing, databaseName, owner) + if _, err := c.pgDb.Exec(fmt.Sprintf(statement, databaseName, owner)); err != nil { return fmt.Errorf("could not execute %s: %v", operation, err) } return nil } -func (c *Cluster) databaseNameOwnerValid(datname, owner string) bool { +func (c *Cluster) databaseNameOwnerValid(databaseName, owner string) bool { if _, ok := c.pgUsers[owner]; !ok { - c.logger.Infof("skipping creation of the %q database, user %q does not exist", datname, owner) + c.logger.Infof("skipping creation of the %q database, user %q does not exist", databaseName, owner) return false } - if !databaseNameRegexp.MatchString(datname) { - c.logger.Infof("database %q has invalid name", datname) + if !databaseNameRegexp.MatchString(databaseName) { + c.logger.Infof("database %q has invalid name", databaseName) return false } return true } +// getSchemas returns the list of current database schemas +// The caller is responsible for opening and closing the database connection +func (c *Cluster) getSchemas() (schemas []string, err error) { + var ( + rows *sql.Rows + dbschemas []string + ) + + if rows, err = c.pgDb.Query(getSchemasSQL); err != nil { + return nil, fmt.Errorf("could not query database schemas: %v", err) + } + + defer func() { + if err2 := rows.Close(); err2 != nil { + if err != nil { + err = fmt.Errorf("error when closing query cursor: %v, previous error: %v", err2, err) + } else { + err = fmt.Errorf("error when closing query cursor: %v", err2) + } + } + }() + + for rows.Next() { + var dbschema string + + if err = rows.Scan(&dbschema); err != nil { + return nil, fmt.Errorf("error when processing row: %v", err) + } + dbschemas = append(dbschemas, dbschema) + } + + return dbschemas, err +} + +// executeCreateDatabaseSchema creates new database schema with the given owner. +// The caller is responsible for opening and closing the database connection. +func (c *Cluster) executeCreateDatabaseSchema(databaseName, schemaName, dbOwner string, schemaOwner string) error { + return c.execCreateDatabaseSchema(databaseName, schemaName, dbOwner, schemaOwner, createDatabaseSchemaSQL, + "creating database schema", "create database schema") +} + +func (c *Cluster) execCreateDatabaseSchema(databaseName, schemaName, dbOwner, schemaOwner, statement, doing, operation string) error { + if !c.databaseSchemaNameValid(schemaName) { + return nil + } + c.logger.Infof("%s %q owner %q", doing, schemaName, schemaOwner) + if _, err := c.pgDb.Exec(fmt.Sprintf(statement, dbOwner, schemaName, schemaOwner)); err != nil { + return fmt.Errorf("could not execute %s: %v", operation, err) + } + + // set default privileges for schema + c.execAlterSchemaDefaultPrivileges(schemaName, schemaOwner, databaseName) + if schemaOwner != dbOwner { + c.execAlterSchemaDefaultPrivileges(schemaName, dbOwner, databaseName+"_"+schemaName) + c.execAlterSchemaDefaultPrivileges(schemaName, schemaOwner, databaseName+"_"+schemaName) + } + + return nil +} + +func (c *Cluster) databaseSchemaNameValid(schemaName string) bool { + if !databaseNameRegexp.MatchString(schemaName) { + c.logger.Infof("database schema %q has invalid name", schemaName) + return false + } + return true +} + +func (c *Cluster) execAlterSchemaDefaultPrivileges(schemaName, owner, rolePrefix string) error { + if _, err := c.pgDb.Exec(fmt.Sprintf(schemaDefaultPrivilegesSQL, owner, + schemaName, rolePrefix+constants.ReaderRoleNameSuffix, rolePrefix+constants.WriterRoleNameSuffix, // schema + schemaName, rolePrefix+constants.ReaderRoleNameSuffix, // tables + schemaName, rolePrefix+constants.ReaderRoleNameSuffix, // sequences + schemaName, rolePrefix+constants.WriterRoleNameSuffix, // tables + schemaName, rolePrefix+constants.WriterRoleNameSuffix, // sequences + schemaName, rolePrefix+constants.ReaderRoleNameSuffix, rolePrefix+constants.WriterRoleNameSuffix, // types + schemaName, rolePrefix+constants.ReaderRoleNameSuffix, rolePrefix+constants.WriterRoleNameSuffix)); err != nil { // functions + return fmt.Errorf("could not alter default privileges for database schema %s: %v", schemaName, err) + } + + return nil +} + +func (c *Cluster) execAlterGlobalDefaultPrivileges(owner, rolePrefix string) error { + if _, err := c.pgDb.Exec(fmt.Sprintf(globalDefaultPrivilegesSQL, owner, + rolePrefix+constants.WriterRoleNameSuffix, rolePrefix+constants.ReaderRoleNameSuffix, // schemas + rolePrefix+constants.ReaderRoleNameSuffix, // tables + rolePrefix+constants.ReaderRoleNameSuffix, // sequences + rolePrefix+constants.WriterRoleNameSuffix, // tables + rolePrefix+constants.WriterRoleNameSuffix, // sequences + rolePrefix+constants.ReaderRoleNameSuffix, rolePrefix+constants.WriterRoleNameSuffix, // types + rolePrefix+constants.ReaderRoleNameSuffix, rolePrefix+constants.WriterRoleNameSuffix)); err != nil { // functions + return fmt.Errorf("could not alter default privileges for database %s: %v", rolePrefix, err) + } + + return nil +} + func makeUserFlags(rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin bool) (result []string) { if rolsuper { result = append(result, constants.RoleFlagSuperuser) @@ -278,8 +402,67 @@ func makeUserFlags(rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin return result } -// Creates a connection pooler credentials lookup function in every database to -// perform remote authentication. +// getExtension returns the list of current database extensions +// The caller is responsible for opening and closing the database connection +func (c *Cluster) getExtensions() (dbExtensions map[string]string, err error) { + var ( + rows *sql.Rows + ) + + if rows, err = c.pgDb.Query(getExtensionsSQL); err != nil { + return nil, fmt.Errorf("could not query database extensions: %v", err) + } + + defer func() { + if err2 := rows.Close(); err2 != nil { + if err != nil { + err = fmt.Errorf("error when closing query cursor: %v, previous error: %v", err2, err) + } else { + err = fmt.Errorf("error when closing query cursor: %v", err2) + } + } + }() + + dbExtensions = make(map[string]string) + + for rows.Next() { + var extension, schema string + + if err = rows.Scan(&extension, &schema); err != nil { + return nil, fmt.Errorf("error when processing row: %v", err) + } + dbExtensions[extension] = schema + } + + return dbExtensions, err +} + +// executeCreateExtension creates new extension in the given schema. +// The caller is responsible for opening and closing the database connection. +func (c *Cluster) executeCreateExtension(extName, schemaName string) error { + return c.execCreateOrAlterExtension(extName, schemaName, createExtensionSQL, + "creating extension", "create extension") +} + +// executeAlterExtension changes the schema of the given extension. +// The caller is responsible for opening and closing the database connection. +func (c *Cluster) executeAlterExtension(extName, schemaName string) error { + return c.execCreateOrAlterExtension(extName, schemaName, alterExtensionSQL, + "changing schema for extension", "alter extension schema") +} + +func (c *Cluster) execCreateOrAlterExtension(extName, schemaName, statement, doing, operation string) error { + + c.logger.Infof("%s %q schema %q", doing, extName, schemaName) + if _, err := c.pgDb.Exec(fmt.Sprintf(statement, extName, schemaName)); err != nil { + return fmt.Errorf("could not execute %s: %v", operation, err) + } + + return nil +} + +// Creates a connection pool credentials lookup function in every database to +// perform remote authentification. func (c *Cluster) installLookupFunction(poolerSchema, poolerUser string) error { var stmtBytes bytes.Buffer c.logger.Info("Installing lookup function") @@ -305,7 +488,7 @@ func (c *Cluster) installLookupFunction(poolerSchema, poolerUser string) error { templater := template.Must(template.New("sql").Parse(connectionPoolerLookup)) - for dbname, _ := range currentDatabases { + for dbname := range currentDatabases { if dbname == "template0" || dbname == "template1" { continue } diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 43190491b..9b92f2fb5 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1470,6 +1470,13 @@ func (c *Cluster) generateSingleUserSecret(namespace string, pgUser spec.PgUser) return nil } + //skip NOLOGIN users + for _, flag := range pgUser.Flags { + if flag == constants.RoleFlagNoLogin { + return nil + } + } + username := pgUser.Name secret := v1.Secret{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 0eb02631c..0c4c662d4 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -3,6 +3,7 @@ package cluster import ( "context" "fmt" + "strings" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/spec" @@ -108,6 +109,11 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { err = fmt.Errorf("could not sync databases: %v", err) return err } + c.logger.Debugf("syncing prepared databases with schemas") + if err = c.syncPreparedDatabases(); err != nil { + err = fmt.Errorf("could not sync prepared database: %v", err) + return err + } } // sync connection pooler @@ -563,6 +569,7 @@ func (c *Cluster) syncDatabases() error { createDatabases := make(map[string]string) alterOwnerDatabases := make(map[string]string) + preparedDatabases := make([]string, 0) if err := c.initDbConn(); err != nil { return fmt.Errorf("could not init database connection") @@ -578,12 +585,24 @@ func (c *Cluster) syncDatabases() error { return fmt.Errorf("could not get current databases: %v", err) } - for datname, newOwner := range c.Spec.Databases { - currentOwner, exists := currentDatabases[datname] + // if no prepared databases are specified create a database named like the cluster + if c.Spec.PreparedDatabases != nil && len(c.Spec.PreparedDatabases) == 0 { // TODO: add option to disable creating such a default DB + c.Spec.PreparedDatabases = map[string]acidv1.PreparedDatabase{strings.Replace(c.Name, "-", "_", -1): {}} + } + for preparedDatabaseName := range c.Spec.PreparedDatabases { + _, exists := currentDatabases[preparedDatabaseName] if !exists { - createDatabases[datname] = newOwner + createDatabases[preparedDatabaseName] = preparedDatabaseName + constants.OwnerRoleNameSuffix + preparedDatabases = append(preparedDatabases, preparedDatabaseName) + } + } + + for databaseName, newOwner := range c.Spec.Databases { + currentOwner, exists := currentDatabases[databaseName] + if !exists { + createDatabases[databaseName] = newOwner } else if currentOwner != newOwner { - alterOwnerDatabases[datname] = newOwner + alterOwnerDatabases[databaseName] = newOwner } } @@ -591,13 +610,116 @@ func (c *Cluster) syncDatabases() error { return nil } - for datname, owner := range createDatabases { - if err = c.executeCreateDatabase(datname, owner); err != nil { + for databaseName, owner := range createDatabases { + if err = c.executeCreateDatabase(databaseName, owner); err != nil { return err } } - for datname, owner := range alterOwnerDatabases { - if err = c.executeAlterDatabaseOwner(datname, owner); err != nil { + for databaseName, owner := range alterOwnerDatabases { + if err = c.executeAlterDatabaseOwner(databaseName, owner); err != nil { + return err + } + } + + // set default privileges for prepared database + for _, preparedDatabase := range preparedDatabases { + if err = c.execAlterGlobalDefaultPrivileges(preparedDatabase+constants.OwnerRoleNameSuffix, preparedDatabase); err != nil { + return err + } + } + + return nil +} + +func (c *Cluster) syncPreparedDatabases() error { + c.setProcessName("syncing prepared databases") + for preparedDbName, preparedDB := range c.Spec.PreparedDatabases { + if err := c.initDbConnWithName(preparedDbName); err != nil { + return fmt.Errorf("could not init connection to database %s: %v", preparedDbName, err) + } + defer func() { + if err := c.closeDbConn(); err != nil { + c.logger.Errorf("could not close database connection: %v", err) + } + }() + + // now, prepare defined schemas + preparedSchemas := preparedDB.PreparedSchemas + if len(preparedDB.PreparedSchemas) == 0 { + preparedSchemas = map[string]acidv1.PreparedSchema{"data": {DefaultRoles: util.True()}} + } + if err := c.syncPreparedSchemas(preparedDbName, preparedSchemas); err != nil { + return err + } + + // install extensions + if err := c.syncExtensions(preparedDB.Extensions); err != nil { + return err + } + } + + return nil +} + +func (c *Cluster) syncPreparedSchemas(databaseName string, preparedSchemas map[string]acidv1.PreparedSchema) error { + c.setProcessName("syncing prepared schemas") + + currentSchemas, err := c.getSchemas() + if err != nil { + return fmt.Errorf("could not get current schemas: %v", err) + } + + var schemas []string + + for schema := range preparedSchemas { + schemas = append(schemas, schema) + } + + if createPreparedSchemas, equal := util.SubstractStringSlices(schemas, currentSchemas); !equal { + for _, schemaName := range createPreparedSchemas { + owner := constants.OwnerRoleNameSuffix + dbOwner := databaseName + owner + if preparedSchemas[schemaName].DefaultRoles == nil || *preparedSchemas[schemaName].DefaultRoles { + owner = databaseName + "_" + schemaName + owner + } else { + owner = dbOwner + } + if err = c.executeCreateDatabaseSchema(databaseName, schemaName, dbOwner, owner); err != nil { + return err + } + } + } + + return nil +} + +func (c *Cluster) syncExtensions(extensions map[string]string) error { + c.setProcessName("syncing database extensions") + + createExtensions := make(map[string]string) + alterExtensions := make(map[string]string) + + currentExtensions, err := c.getExtensions() + if err != nil { + return fmt.Errorf("could not get current database extensions: %v", err) + } + + for extName, newSchema := range extensions { + currentSchema, exists := currentExtensions[extName] + if !exists { + createExtensions[extName] = newSchema + } else if currentSchema != newSchema { + alterExtensions[extName] = newSchema + } + } + + for extName, schema := range createExtensions { + if err = c.executeCreateExtension(extName, schema); err != nil { + return err + } + } + for extName, schema := range alterExtensions { + if err = c.executeAlterExtension(extName, schema); err != nil { return err } } diff --git a/pkg/controller/types.go b/pkg/controller/types.go index 0d86abec8..b598014c9 100644 --- a/pkg/controller/types.go +++ b/pkg/controller/types.go @@ -1,9 +1,10 @@ package controller import ( - "k8s.io/apimachinery/pkg/types" "time" + "k8s.io/apimachinery/pkg/types" + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" ) diff --git a/pkg/spec/types.go b/pkg/spec/types.go index e1c49a1fd..08008267b 100644 --- a/pkg/spec/types.go +++ b/pkg/spec/types.go @@ -31,6 +31,7 @@ const ( RoleOriginInfrastructure RoleOriginTeamsAPI RoleOriginSystem + RoleOriginBootstrap RoleConnectionPooler ) @@ -180,6 +181,8 @@ func (r RoleOrigin) String() string { return "teams API role" case RoleOriginSystem: return "system role" + case RoleOriginBootstrap: + return "bootstrapped role" case RoleConnectionPooler: return "connection pooler role" default: diff --git a/pkg/util/constants/roles.go b/pkg/util/constants/roles.go index c2c287472..87c9c51ce 100644 --- a/pkg/util/constants/roles.go +++ b/pkg/util/constants/roles.go @@ -14,4 +14,8 @@ const ( RoleFlagCreateDB = "CREATEDB" RoleFlagReplication = "REPLICATION" RoleFlagByPassRLS = "BYPASSRLS" + OwnerRoleNameSuffix = "_owner" + ReaderRoleNameSuffix = "_reader" + WriterRoleNameSuffix = "_writer" + UserRoleNameSuffix = "_user" ) diff --git a/pkg/util/users/users.go b/pkg/util/users/users.go index 112f89b43..345caa001 100644 --- a/pkg/util/users/users.go +++ b/pkg/util/users/users.go @@ -73,26 +73,44 @@ func (strategy DefaultUserSyncStrategy) ProduceSyncRequests(dbUsers spec.PgUserM } // ExecuteSyncRequests makes actual database changes from the requests passed in its arguments. -func (strategy DefaultUserSyncStrategy) ExecuteSyncRequests(reqs []spec.PgSyncUserRequest, db *sql.DB) error { - for _, r := range reqs { - switch r.Kind { +func (strategy DefaultUserSyncStrategy) ExecuteSyncRequests(requests []spec.PgSyncUserRequest, db *sql.DB) error { + var reqretries []spec.PgSyncUserRequest + var errors []string + for _, request := range requests { + switch request.Kind { case spec.PGSyncUserAdd: - if err := strategy.createPgUser(r.User, db); err != nil { - return fmt.Errorf("could not create user %q: %v", r.User.Name, err) + if err := strategy.createPgUser(request.User, db); err != nil { + reqretries = append(reqretries, request) + errors = append(errors, fmt.Sprintf("could not create user %q: %v", request.User.Name, err)) } case spec.PGsyncUserAlter: - if err := strategy.alterPgUser(r.User, db); err != nil { - return fmt.Errorf("could not alter user %q: %v", r.User.Name, err) + if err := strategy.alterPgUser(request.User, db); err != nil { + reqretries = append(reqretries, request) + errors = append(errors, fmt.Sprintf("could not alter user %q: %v", request.User.Name, err)) } case spec.PGSyncAlterSet: - if err := strategy.alterPgUserSet(r.User, db); err != nil { - return fmt.Errorf("could not set custom user %q parameters: %v", r.User.Name, err) + if err := strategy.alterPgUserSet(request.User, db); err != nil { + reqretries = append(reqretries, request) + errors = append(errors, fmt.Sprintf("could not set custom user %q parameters: %v", request.User.Name, err)) } default: - return fmt.Errorf("unrecognized operation: %v", r.Kind) + return fmt.Errorf("unrecognized operation: %v", request.Kind) } } + + // creating roles might fail if group role members are created before the parent role + // retry adding roles as long as the number of failed attempts is shrinking + if len(reqretries) > 0 { + if len(reqretries) < len(requests) { + if err := strategy.ExecuteSyncRequests(reqretries, db); err != nil { + return err + } + } else { + return fmt.Errorf("could not execute sync requests for users: %v", errors) + } + } + return nil } func (strategy DefaultUserSyncStrategy) alterPgUserSet(user spec.PgUser, db *sql.DB) (err error) { From 865d5b41a75a98bbc6c015b67a5f228f0dd2e652 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 29 Apr 2020 17:26:46 +0200 Subject: [PATCH 039/168] set event broadcasting to Infof and update rbac (#952) --- .../templates/clusterrole.yaml | 5 +++++ docs/user.md | 19 +++++++++++++++---- manifests/operator-service-account-rbac.yaml | 5 +++++ pkg/controller/controller.go | 2 +- 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/charts/postgres-operator/templates/clusterrole.yaml b/charts/postgres-operator/templates/clusterrole.yaml index 0defcab41..bd34e803e 100644 --- a/charts/postgres-operator/templates/clusterrole.yaml +++ b/charts/postgres-operator/templates/clusterrole.yaml @@ -49,6 +49,11 @@ rules: - events verbs: - create + - get + - list + - patch + - update + - watch # to manage endpoints which are also used by Patroni - apiGroups: - "" diff --git a/docs/user.md b/docs/user.md index 2d9f2be6a..3683fdf61 100644 --- a/docs/user.md +++ b/docs/user.md @@ -53,8 +53,19 @@ them. ## Watch pods being created +Check if the database pods are coming up. Use the label `application=spilo` to +filter and list the label `spilo-role` to see when the master is promoted and +replicas get their labels. + ```bash -kubectl get pods -w --show-labels +kubectl get pods -l application=spilo -L spilo-role -w +``` + +The operator also emits K8s events to the Postgresql CRD which can be inspected +in the operator logs or with: + +```bash +kubectl describe postgresql acid-minimal-cluster ``` ## Connect to PostgreSQL @@ -736,14 +747,14 @@ spin up more instances). ## Custom TLS certificates -By default, the spilo image generates its own TLS certificate during startup. +By default, the Spilo image generates its own TLS certificate during startup. However, this certificate cannot be verified and thus doesn't protect from active MITM attacks. In this section we show how to specify a custom TLS certificate which is mounted in the database pods via a K8s Secret. Before applying these changes, in k8s the operator must also be configured with the `spilo_fsgroup` set to the GID matching the postgres user group. If you -don't know the value, use `103` which is the GID from the default spilo image +don't know the value, use `103` which is the GID from the default Spilo image (`spilo_fsgroup=103` in the cluster request spec). OpenShift allocates the users and groups dynamically (based on scc), and their @@ -805,5 +816,5 @@ spec: Alternatively, it is also possible to use [cert-manager](https://cert-manager.io/docs/) to generate these secrets. -Certificate rotation is handled in the spilo image which checks every 5 +Certificate rotation is handled in the Spilo image which checks every 5 minutes if the certificates have changed and reloads postgres accordingly. diff --git a/manifests/operator-service-account-rbac.yaml b/manifests/operator-service-account-rbac.yaml index 667941a24..266df30c5 100644 --- a/manifests/operator-service-account-rbac.yaml +++ b/manifests/operator-service-account-rbac.yaml @@ -50,6 +50,11 @@ rules: - events verbs: - create + - get + - list + - patch + - update + - watch # to manage endpoints which are also used by Patroni - apiGroups: - "" diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 0b3fde5d9..26b6b1b87 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -76,7 +76,7 @@ func NewController(controllerConfig *spec.ControllerConfig, controllerId string) } eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(logger.Debugf) + eventBroadcaster.StartLogging(logger.Infof) recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: myComponentName}) c := &Controller{ From 5af4379118a41664f404a95c40ff17a872f15e04 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 30 Apr 2020 09:58:07 +0200 Subject: [PATCH 040/168] [UI] add toggle for connection pooler (#953) * [UI] add toggle for connection pooler * remove team service logger * fix new.tag.pug and change port in Makefile --- ui/Dockerfile | 5 ++-- ui/Makefile | 2 +- ui/app/src/edit.tag.pug | 1 + ui/app/src/new.tag.pug | 25 ++++++++++++++++- ui/app/src/postgresql.tag.pug | 9 ++++++ ui/manifests/deployment.yaml | 2 ++ ui/manifests/ui-service-account-rbac.yaml | 1 + ui/operator_ui/main.py | 34 +++++++++++++++++++++-- ui/operator_ui/spiloutils.py | 28 ++++++++++++++++--- ui/requirements.txt | 2 +- ui/start_server.sh | 2 ++ 11 files changed, 100 insertions(+), 11 deletions(-) create mode 100644 ui/start_server.sh diff --git a/ui/Dockerfile b/ui/Dockerfile index 3e1ae8756..5ea912dbc 100644 --- a/ui/Dockerfile +++ b/ui/Dockerfile @@ -1,7 +1,7 @@ FROM alpine:3.6 MAINTAINER team-acid@zalando.de -EXPOSE 8080 +EXPOSE 8081 RUN \ apk add --no-cache \ @@ -29,6 +29,7 @@ RUN \ /var/cache/apk/* COPY requirements.txt / +COPY start_server.sh / RUN pip3 install -r /requirements.txt COPY operator_ui /operator_ui @@ -37,4 +38,4 @@ ARG VERSION=dev RUN sed -i "s/__version__ = .*/__version__ = '${VERSION}'/" /operator_ui/__init__.py WORKDIR / -ENTRYPOINT ["/usr/bin/python3", "-m", "operator_ui"] +CMD ["/usr/bin/python3", "-m", "operator_ui"] diff --git a/ui/Makefile b/ui/Makefile index e7d5df674..29c8d9409 100644 --- a/ui/Makefile +++ b/ui/Makefile @@ -36,4 +36,4 @@ push: docker push "$(IMAGE):$(TAG)$(CDP_TAG)" mock: - docker run -it -p 8080:8080 "$(IMAGE):$(TAG)" --mock + docker run -it -p 8081:8081 "$(IMAGE):$(TAG)" --mock diff --git a/ui/app/src/edit.tag.pug b/ui/app/src/edit.tag.pug index 9029594bd..c1d94e589 100644 --- a/ui/app/src/edit.tag.pug +++ b/ui/app/src/edit.tag.pug @@ -137,6 +137,7 @@ edit o.spec.numberOfInstances = i.spec.numberOfInstances o.spec.enableMasterLoadBalancer = i.spec.enableMasterLoadBalancer || false o.spec.enableReplicaLoadBalancer = i.spec.enableReplicaLoadBalancer || false + o.spec.enableConnectionPooler = i.spec.enableConnectionPooler || false o.spec.volume = { size: i.spec.volume.size } if ('users' in i.spec && typeof i.spec.users === 'object') { diff --git a/ui/app/src/new.tag.pug b/ui/app/src/new.tag.pug index fe9d78226..6293a6c7a 100644 --- a/ui/app/src/new.tag.pug +++ b/ui/app/src/new.tag.pug @@ -239,6 +239,18 @@ new | | Enable replica ELB + tr + td Enable Connection Pool + td + label + input( + type='checkbox' + value='{ enableConnectionPooler }' + onchange='{ toggleEnableConnectionPooler }' + ) + | + | Enable Connection Pool (using PGBouncer) + tr td Volume size td @@ -493,6 +505,9 @@ new {{#if enableReplicaLoadBalancer}} enableReplicaLoadBalancer: true {{/if}} + {{#if enableConnectionPooler}} + enableConnectionPooler: true + {{/if}} volume: size: "{{ volumeSize }}Gi" {{#if users}} @@ -516,13 +531,14 @@ new - {{ odd }}/32 {{/if}} + {{#if resourcesVisible}} resources: requests: cpu: {{ cpu.state.request.state }}m memory: {{ memory.state.request.state }}Mi limits: cpu: {{ cpu.state.limit.state }}m - memory: {{ memory.state.limit.state }}Mi{{#if restoring}} + memory: {{ memory.state.limit.state }}Mi{{/if}}{{#if restoring}} clone: cluster: "{{ backup.state.name.state }}" @@ -542,6 +558,7 @@ new instanceCount: this.instanceCount, enableMasterLoadBalancer: this.enableMasterLoadBalancer, enableReplicaLoadBalancer: this.enableReplicaLoadBalancer, + enableConnectionPooler: this.enableConnectionPooler, volumeSize: this.volumeSize, users: this.users.valids, databases: this.databases.valids, @@ -552,6 +569,7 @@ new memory: this.memory, backup: this.backup, namespace: this.namespace, + resourcesVisible: this.config.resources_visible, restoring: this.backup.state.type.state !== 'empty', pitr: this.backup.state.type.state === 'pitr', } @@ -598,6 +616,10 @@ new this.enableReplicaLoadBalancer = !this.enableReplicaLoadBalancer } + this.toggleEnableConnectionPooler = e => { + this.enableConnectionPooler = !this.enableConnectionPooler + } + this.volumeChange = e => { this.volumeSize = +e.target.value } @@ -892,6 +914,7 @@ new this.odd = '' this.enableMasterLoadBalancer = false this.enableReplicaLoadBalancer = false + this.enableConnectionPooler = false this.postgresqlVersion = this.postgresqlVersion = ( this.config.postgresql_versions[0] diff --git a/ui/app/src/postgresql.tag.pug b/ui/app/src/postgresql.tag.pug index be7173dbe..9edae99d3 100644 --- a/ui/app/src/postgresql.tag.pug +++ b/ui/app/src/postgresql.tag.pug @@ -92,6 +92,8 @@ postgresql .alert.alert-success(if='{ progress.masterLabel }') PostgreSQL master available, label is attached .alert.alert-success(if='{ progress.masterLabel && progress.dnsName }') PostgreSQL ready: { progress.dnsName } + .alert.alert-success(if='{ progress.pooler }') Connection pooler deployment created + .col-lg-3 help-general(config='{ opts.config }') @@ -122,9 +124,11 @@ postgresql jQuery.get( '/postgresqls/' + this.cluster_path, ).done(data => { + this.progress.pooler = false this.progress.postgresql = true this.progress.postgresqlManifest = data this.progress.createdTimestamp = data.metadata.creationTimestamp + this.progress.poolerEnabled = data.spec.enableConnectionPooler this.uid = this.progress.postgresqlManifest.metadata.uid this.update() @@ -160,6 +164,11 @@ postgresql this.progress.dnsName = data.metadata.name + '.' + data.metadata.namespace } + jQuery.get('/pooler/' + this.cluster_path).done(data => { + this.progress.pooler = {"url": ""} + this.update() + }) + this.update() }) }) diff --git a/ui/manifests/deployment.yaml b/ui/manifests/deployment.yaml index 6138ca1a8..ccaecd312 100644 --- a/ui/manifests/deployment.yaml +++ b/ui/manifests/deployment.yaml @@ -44,6 +44,8 @@ spec: value: "http://postgres-operator:8080" - name: "OPERATOR_CLUSTER_NAME_LABEL" value: "cluster-name" + - name: "RESOURCES_VISIBLE" + value: "False" - name: "TARGET_NAMESPACE" value: "default" - name: "TEAMS" diff --git a/ui/manifests/ui-service-account-rbac.yaml b/ui/manifests/ui-service-account-rbac.yaml index 2e09797a0..d4937b5a2 100644 --- a/ui/manifests/ui-service-account-rbac.yaml +++ b/ui/manifests/ui-service-account-rbac.yaml @@ -39,6 +39,7 @@ rules: - apiGroups: - apps resources: + - deployments - statefulsets verbs: - get diff --git a/ui/operator_ui/main.py b/ui/operator_ui/main.py index 5a3054f0e..a294ae081 100644 --- a/ui/operator_ui/main.py +++ b/ui/operator_ui/main.py @@ -25,7 +25,7 @@ from flask import ( from flask_oauthlib.client import OAuth from functools import wraps from gevent import sleep, spawn -from gevent.wsgi import WSGIServer +from gevent.pywsgi import WSGIServer from jq import jq from json import dumps, loads from logging import DEBUG, ERROR, INFO, basicConfig, exception, getLogger @@ -44,6 +44,7 @@ from .spiloutils import ( create_postgresql, read_basebackups, read_namespaces, + read_pooler, read_pods, read_postgresql, read_postgresqls, @@ -80,6 +81,7 @@ OPERATOR_CLUSTER_NAME_LABEL = getenv('OPERATOR_CLUSTER_NAME_LABEL', 'cluster-nam OPERATOR_UI_CONFIG = getenv('OPERATOR_UI_CONFIG', '{}') OPERATOR_UI_MAINTENANCE_CHECK = getenv('OPERATOR_UI_MAINTENANCE_CHECK', '{}') READ_ONLY_MODE = getenv('READ_ONLY_MODE', False) in [True, 'true'] +RESOURCES_VISIBLE = getenv('RESOURCES_VISIBLE', True) SPILO_S3_BACKUP_PREFIX = getenv('SPILO_S3_BACKUP_PREFIX', 'spilo/') SUPERUSER_TEAM = getenv('SUPERUSER_TEAM', 'acid') TARGET_NAMESPACE = getenv('TARGET_NAMESPACE') @@ -312,6 +314,7 @@ DEFAULT_UI_CONFIG = { def get_config(): config = loads(OPERATOR_UI_CONFIG) or DEFAULT_UI_CONFIG config['read_only_mode'] = READ_ONLY_MODE + config['resources_visible'] = RESOURCES_VISIBLE config['superuser_team'] = SUPERUSER_TEAM config['target_namespace'] = TARGET_NAMESPACE @@ -397,6 +400,22 @@ def get_service(namespace: str, cluster: str): ) +@app.route('/pooler//') +@authorize +def get_list_poolers(namespace: str, cluster: str): + + if TARGET_NAMESPACE not in ['', '*', namespace]: + return wrong_namespace() + + return respond( + read_pooler( + get_cluster(), + namespace, + "{}-pooler".format(cluster), + ), + ) + + @app.route('/statefulsets//') @authorize def get_list_clusters(namespace: str, cluster: str): @@ -587,6 +606,17 @@ def update_postgresql(namespace: str, cluster: str): spec['volume'] = {'size': size} + if 'enableConnectionPooler' in postgresql['spec']: + cp = postgresql['spec']['enableConnectionPooler'] + if not cp: + if 'enableConnectionPooler' in o['spec']: + del o['spec']['enableConnectionPooler'] + else: + spec['enableConnectionPooler'] = True + else: + if 'enableConnectionPooler' in o['spec']: + del o['spec']['enableConnectionPooler'] + if 'enableReplicaLoadBalancer' in postgresql['spec']: rlb = postgresql['spec']['enableReplicaLoadBalancer'] if not rlb: @@ -1006,7 +1036,7 @@ def init_cluster(): def main(port, secret_key, debug, clusters: list): global TARGET_NAMESPACE - basicConfig(level=DEBUG if debug else INFO) + basicConfig(stream=sys.stdout, level=(DEBUG if debug else INFO), format='%(asctime)s %(levelname)s: %(message)s',) init_cluster() diff --git a/ui/operator_ui/spiloutils.py b/ui/operator_ui/spiloutils.py index 33d07d88a..8d1996fb5 100644 --- a/ui/operator_ui/spiloutils.py +++ b/ui/operator_ui/spiloutils.py @@ -1,7 +1,7 @@ from boto3 import client from datetime import datetime, timezone from furl import furl -from json import dumps +from json import dumps, loads from logging import getLogger from os import environ, getenv from requests import Session @@ -18,6 +18,15 @@ session = Session() OPERATOR_CLUSTER_NAME_LABEL = getenv('OPERATOR_CLUSTER_NAME_LABEL', 'cluster-name') +COMMON_CLUSTER_LABEL = getenv('COMMON_CLUSTER_LABEL', '{"application":"spilo"}') +COMMON_POOLER_LABEL = getenv('COMMONG_POOLER_LABEL', '{"application":"db-connection-pooler"}') + +logger.info("Common Cluster Label: {}".format(COMMON_CLUSTER_LABEL)) +logger.info("Common Pooler Label: {}".format(COMMON_POOLER_LABEL)) + +COMMON_CLUSTER_LABEL = loads(COMMON_CLUSTER_LABEL) +COMMON_POOLER_LABEL = loads(COMMON_POOLER_LABEL) + def request(cluster, path, **kwargs): if 'timeout' not in kwargs: @@ -85,6 +94,7 @@ def resource_api_version(resource_type): return { 'postgresqls': 'apis/acid.zalan.do/v1', 'statefulsets': 'apis/apps/v1', + 'deployments': 'apis/apps/v1', }.get(resource_type, 'api/v1') @@ -149,7 +159,7 @@ def read_pod(cluster, namespace, resource_name): resource_type='pods', namespace=namespace, resource_name=resource_name, - label_selector={'application': 'spilo'}, + label_selector=COMMON_CLUSTER_LABEL, ) @@ -159,7 +169,17 @@ def read_service(cluster, namespace, resource_name): resource_type='services', namespace=namespace, resource_name=resource_name, - label_selector={'application': 'spilo'}, + label_selector=COMMON_CLUSTER_LABEL, + ) + + +def read_pooler(cluster, namespace, resource_name): + return kubernetes_get( + cluster=cluster, + resource_type='deployments', + namespace=namespace, + resource_name=resource_name, + label_selector=COMMON_POOLER_LABEL, ) @@ -169,7 +189,7 @@ def read_statefulset(cluster, namespace, resource_name): resource_type='statefulsets', namespace=namespace, resource_name=resource_name, - label_selector={'application': 'spilo'}, + label_selector=COMMON_CLUSTER_LABEL, ) diff --git a/ui/requirements.txt b/ui/requirements.txt index 5d987416c..7dc49eb3d 100644 --- a/ui/requirements.txt +++ b/ui/requirements.txt @@ -1,5 +1,5 @@ Flask-OAuthlib==0.9.5 -Flask==1.1.1 +Flask==1.1.2 backoff==1.8.1 boto3==1.10.4 boto==2.49.0 diff --git a/ui/start_server.sh b/ui/start_server.sh new file mode 100644 index 000000000..e2c3980cc --- /dev/null +++ b/ui/start_server.sh @@ -0,0 +1,2 @@ +#!/bin/bash +/usr/bin/python3 -m operator_ui From be208b61f1f795bd79490832dc798cf262ba0cf8 Mon Sep 17 00:00:00 2001 From: Petr Barborka Date: Thu, 30 Apr 2020 17:10:16 +0200 Subject: [PATCH 041/168] Fix S3 backup list (#880) * Fix S3 backup list Co-authored-by: Petr Barborka --- ui/operator_ui/spiloutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/operator_ui/spiloutils.py b/ui/operator_ui/spiloutils.py index 8d1996fb5..ea347a84d 100644 --- a/ui/operator_ui/spiloutils.py +++ b/ui/operator_ui/spiloutils.py @@ -322,7 +322,7 @@ def read_basebackups( f=configure_backup_cxt, aws_instance_profile=use_aws_instance_profile, s3_prefix=f's3://{bucket}/{prefix}{pg_cluster}{suffix}/wal/', - )._backup_list(detail=True) + )._backup_list(detail=True)._backup_list(prefix=f"{prefix}{pg_cluster}{suffix}/wal/") ] From d52296c3235956ea4cb34f78f6e4cf0aa374d09d Mon Sep 17 00:00:00 2001 From: Rafia Sabih Date: Mon, 4 May 2020 14:46:56 +0200 Subject: [PATCH 042/168] Propagate annotations to the StatefulSet (#932) * Initial commit * Corrections - set the type of the new configuration parameter to be array of strings - propagate the annotations to statefulset at sync * Enable regular expression matching * Improvements -handle rollingUpdate flag -modularize code -rename config parameter name * fix merge error * Pass annotations to connection pooler deployment * update code-gen * Add documentation and update manifests * add e2e test and introduce option in configmap * fix service annotations test * Add unit test * fix e2e tests * better key lookup of annotations tests * add debug message for annotation tests * Fix typos * minor fix for looping * Handle update path and renaming - handle the update path to update sts and connection pooler deployment. This way no need to wait for sync - rename the parameter to downscaler_annotations - handle other review comments * another try to fix python loops * Avoid unneccessary update events * Update manifests * some final polishing * fix cluster_test after polishing Co-authored-by: Rafia Sabih Co-authored-by: Felix Kunde --- .../crds/operatorconfigurations.yaml | 4 ++ charts/postgres-operator/values-crd.yaml | 9 ++- charts/postgres-operator/values.yaml | 3 + docs/reference/cluster_manifest.md | 2 +- docs/reference/operator_parameters.md | 6 ++ e2e/tests/test_e2e.py | 63 ++++++++++++++++--- manifests/configmap.yaml | 1 + manifests/operatorconfiguration.crd.yaml | 4 ++ ...gresql-operator-default-configuration.yaml | 3 + pkg/apis/acid.zalan.do/v1/crds.go | 34 ++++++---- .../v1/operator_configuration_type.go | 1 + .../acid.zalan.do/v1/zz_generated.deepcopy.go | 5 ++ pkg/cluster/cluster.go | 3 +- pkg/cluster/cluster_test.go | 33 +++++++++- pkg/cluster/k8sres.go | 6 +- pkg/cluster/resources.go | 21 +++++++ pkg/cluster/sync.go | 32 ++++++++++ pkg/controller/operator_config.go | 1 + pkg/controller/postgresql.go | 4 +- pkg/util/config/config.go | 1 + 20 files changed, 205 insertions(+), 31 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 71260bdb6..ffcef7b4a 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -117,6 +117,10 @@ spec: type: object additionalProperties: type: string + downscaler_annotations: + type: array + items: + type: string enable_init_containers: type: boolean enable_pod_antiaffinity: diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index e6428c478..98a399c8b 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -67,6 +67,11 @@ configKubernetes: # keya: valuea # keyb: valueb + # list of annotations propagated from cluster manifest to statefulset and deployment + # downscaler_annotations: + # - deployment-time + # - downscaler/* + # enables initContainers to run actions before Spilo is started enable_init_containers: true # toggles pod anti affinity on the Postgres pods @@ -262,11 +267,11 @@ configConnectionPooler: # docker image connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer" # max db connections the pooler should hold - connection_pooler_max_db_connections: "60" + connection_pooler_max_db_connections: 60 # default pooling mode connection_pooler_mode: "transaction" # number of pooler instances - connection_pooler_number_of_instances: "2" + connection_pooler_number_of_instances: 2 # default resources connection_pooler_default_cpu_request: 500m connection_pooler_default_memory_request: 100Mi diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index e5cdcee47..5578a5ed1 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -63,6 +63,9 @@ configKubernetes: # annotations attached to each database pod # custom_pod_annotations: "keya:valuea,keyb:valueb" + # list of annotations propagated from cluster manifest to statefulset and deployment + # downscaler_annotations: "deployment-time,downscaler/*" + # enables initContainers to run actions before Spilo is started enable_init_containers: "true" # toggles pod anti affinity on the Postgres pods diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index c87728812..576031543 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -165,7 +165,7 @@ These parameters are grouped directly under the `spec` key in the manifest. If `targetContainers` is empty, additional volumes will be mounted only in the `postgres` container. If you set the `all` special item, it will be mounted in all containers (postgres + sidecars). Else you can set the list of target containers in which the additional volumes will be mounted (eg : postgres, telegraf) - + ## Postgres parameters Those parameters are grouped under the `postgresql` top-level key, which is diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index af6e93d25..a81cabfc4 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -200,6 +200,12 @@ configuration they are grouped under the `kubernetes` key. of a database created by the operator. If the annotation key is also provided by the database definition, the database definition value is used. +* **downscaler_annotations** + An array of annotations that should be passed from Postgres CRD on to the + statefulset and, if exists, to the connection pooler deployment as well. + Regular expressions like `downscaler/*` etc. are also accepted. Can be used + with [kube-downscaler](https://github.com/hjacobs/kube-downscaler). + * **watched_namespace** The operator watches for Postgres objects in the given namespace. If not specified, the value is taken from the operator namespace. A special `*` diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index aa6f1205d..18b9852c4 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -428,7 +428,7 @@ class EndToEndTestCase(unittest.TestCase): k8s.api.core_v1.patch_node(current_master_node, patch_readiness_label) # wait a little before proceeding with the pod distribution test - time.sleep(k8s.RETRY_TIMEOUT_SEC) + time.sleep(30) # toggle pod anti affinity to move replica away from master node self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) @@ -465,21 +465,24 @@ class EndToEndTestCase(unittest.TestCase): pg_patch_custom_annotations = { "spec": { "serviceAnnotations": { - "annotation.key": "value" + "annotation.key": "value", + "foo": "bar", } } } k8s.api.custom_objects_api.patch_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_custom_annotations) + # wait a little before proceeding + time.sleep(30) annotations = { "annotation.key": "value", "foo": "bar", } self.assertTrue(k8s.check_service_annotations( - "cluster-name=acid-service-annotations,spilo-role=master", annotations)) + "cluster-name=acid-minimal-cluster,spilo-role=master", annotations)) self.assertTrue(k8s.check_service_annotations( - "cluster-name=acid-service-annotations,spilo-role=replica", annotations)) + "cluster-name=acid-minimal-cluster,spilo-role=replica", annotations)) # clean up unpatch_custom_service_annotations = { @@ -489,6 +492,40 @@ class EndToEndTestCase(unittest.TestCase): } k8s.update_config(unpatch_custom_service_annotations) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_statefulset_annotation_propagation(self): + ''' + Inject annotation to Postgresql CRD and check it's propagation to stateful set + ''' + k8s = self.k8s + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + + patch_sset_propagate_annotations = { + "data": { + "downscaler_annotations": "deployment-time,downscaler/*", + } + } + k8s.update_config(patch_sset_propagate_annotations) + + pg_crd_annotations = { + "metadata": { + "annotations": { + "deployment-time": "2020-04-30 12:00:00", + "downscaler/downtime_replicas": "0", + }, + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_crd_annotations) + + # wait a little before proceeding + time.sleep(60) + annotations = { + "deployment-time": "2020-04-30 12:00:00", + "downscaler/downtime_replicas": "0", + } + self.assertTrue(k8s.check_statefulset_annotations(cluster_label, annotations)) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_taint_based_eviction(self): ''' @@ -528,7 +565,7 @@ class EndToEndTestCase(unittest.TestCase): k8s.update_config(patch_toleration_config) # wait a little before proceeding with the pod distribution test - time.sleep(k8s.RETRY_TIMEOUT_SEC) + time.sleep(30) # toggle pod anti affinity to move replica away from master node self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) @@ -694,10 +731,18 @@ class K8s: def check_service_annotations(self, svc_labels, annotations, namespace='default'): svcs = self.api.core_v1.list_namespaced_service(namespace, label_selector=svc_labels, limit=1).items for svc in svcs: - if len(svc.metadata.annotations) != len(annotations): - return False - for key in svc.metadata.annotations: - if svc.metadata.annotations[key] != annotations[key]: + for key, value in annotations.items(): + if key not in svc.metadata.annotations or svc.metadata.annotations[key] != value: + print("Expected key {} not found in annotations {}".format(key, svc.metadata.annotation)) + return False + return True + + def check_statefulset_annotations(self, sset_labels, annotations, namespace='default'): + ssets = self.api.apps_v1.list_namespaced_stateful_set(namespace, label_selector=sset_labels, limit=1).items + for sset in ssets: + for key, value in annotations.items(): + if key not in sset.metadata.annotations or sset.metadata.annotations[key] != value: + print("Expected key {} not found in annotations {}".format(key, sset.metadata.annotation)) return False return True diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 8719e76a1..537055b18 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -30,6 +30,7 @@ data: # default_memory_limit: 500Mi # default_memory_request: 100Mi docker_image: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115 + # downscaler_annotations: "deployment-time,downscaler/*" # enable_admin_role_for_users: "true" # enable_crd_validation: "true" # enable_database_access: "true" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 2d3e614b9..23b5ff0fc 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -93,6 +93,10 @@ spec: type: object additionalProperties: type: string + downscaler_annotations: + type: array + items: + type: string enable_init_containers: type: boolean enable_pod_antiaffinity: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 78312b684..9ae1b3b26 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -31,6 +31,9 @@ configuration: # custom_pod_annotations: # keya: valuea # keyb: valueb + # downscaler_annotations: + # - deployment-time + # - downscaler/* enable_init_containers: true enable_pod_antiaffinity: false enable_pod_disruption_budget: true diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 35037ec3c..ad1b79a45 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -21,48 +21,48 @@ const ( // PostgresCRDResourceColumns definition of AdditionalPrinterColumns for postgresql CRD var PostgresCRDResourceColumns = []apiextv1beta1.CustomResourceColumnDefinition{ - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Team", Type: "string", Description: "Team responsible for Postgres cluster", JSONPath: ".spec.teamId", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Version", Type: "string", Description: "PostgreSQL version", JSONPath: ".spec.postgresql.version", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Pods", Type: "integer", Description: "Number of Pods per Postgres cluster", JSONPath: ".spec.numberOfInstances", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Volume", Type: "string", Description: "Size of the bound volume", JSONPath: ".spec.volume.size", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "CPU-Request", Type: "string", Description: "Requested CPU for Postgres containers", JSONPath: ".spec.resources.requests.cpu", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Memory-Request", Type: "string", Description: "Requested memory for Postgres containers", JSONPath: ".spec.resources.requests.memory", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Age", Type: "date", JSONPath: ".metadata.creationTimestamp", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Status", Type: "string", Description: "Current sync status of postgresql resource", @@ -72,31 +72,31 @@ var PostgresCRDResourceColumns = []apiextv1beta1.CustomResourceColumnDefinition{ // OperatorConfigCRDResourceColumns definition of AdditionalPrinterColumns for OperatorConfiguration CRD var OperatorConfigCRDResourceColumns = []apiextv1beta1.CustomResourceColumnDefinition{ - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Image", Type: "string", Description: "Spilo image to be used for Pods", JSONPath: ".configuration.docker_image", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Cluster-Label", Type: "string", Description: "Label for K8s resources created by operator", JSONPath: ".configuration.kubernetes.cluster_name_label", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Service-Account", Type: "string", Description: "Name of service account to be used", JSONPath: ".configuration.kubernetes.pod_service_account_name", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Min-Instances", Type: "integer", Description: "Minimum number of instances per Postgres cluster", JSONPath: ".configuration.min_instances", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Age", Type: "date", JSONPath: ".metadata.creationTimestamp", @@ -888,6 +888,14 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, }, }, + "downscaler_annotations": { + Type: "array", + Items: &apiextv1beta1.JSONSchemaPropsOrArray{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "string", + }, + }, + }, "enable_init_containers": { Type: "boolean", }, diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 4a0abf3ca..d3a9f6ec2 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -62,6 +62,7 @@ type KubernetesMetaConfiguration struct { PodRoleLabel string `json:"pod_role_label,omitempty"` ClusterLabels map[string]string `json:"cluster_labels,omitempty"` InheritedLabels []string `json:"inherited_labels,omitempty"` + DownscalerAnnotations []string `json:"downscaler_annotations,omitempty"` ClusterNameLabel string `json:"cluster_name_label,omitempty"` NodeReadinessLabel map[string]string `json:"node_readiness_label,omitempty"` CustomPodAnnotations map[string]string `json:"custom_pod_annotations,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 5b4d6cdcd..5879c9b73 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -180,6 +180,11 @@ func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfigura *out = make([]string, len(*in)) copy(*out, *in) } + if in.DownscalerAnnotations != nil { + in, out := &in.DownscalerAnnotations, &out.DownscalerAnnotations + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.NodeReadinessLabel != nil { in, out := &in.NodeReadinessLabel, &out.NodeReadinessLabel *out = make(map[string]string, len(*in)) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 387107540..31b8fa155 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -711,8 +711,7 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { updateFailed = true return } - - if !reflect.DeepEqual(oldSs, newSs) { + if !reflect.DeepEqual(oldSs, newSs) || !reflect.DeepEqual(oldSpec.Annotations, newSpec.Annotations) { c.logger.Debugf("syncing statefulsets") // TODO: avoid generating the StatefulSet object twice by passing it to syncStatefulSet if err := c.syncStatefulSet(); err != nil { diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index 539038bff..4562d525e 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -28,19 +28,48 @@ var eventRecorder = record.NewFakeRecorder(1) var cl = New( Config{ OpConfig: config.Config{ - ProtectedRoles: []string{"admin"}, + PodManagementPolicy: "ordered_ready", + ProtectedRoles: []string{"admin"}, Auth: config.Auth{ SuperUsername: superUserName, ReplicationUsername: replicationUserName, }, + Resources: config.Resources{ + DownscalerAnnotations: []string{"downscaler/*"}, + }, }, }, k8sutil.NewMockKubernetesClient(), - acidv1.Postgresql{ObjectMeta: metav1.ObjectMeta{Name: "acid-test", Namespace: "test"}}, + acidv1.Postgresql{ObjectMeta: metav1.ObjectMeta{Name: "acid-test", Namespace: "test", Annotations: map[string]string{"downscaler/downtime_replicas": "0"}}}, logger, eventRecorder, ) +func TestStatefulSetAnnotations(t *testing.T) { + testName := "CheckStatefulsetAnnotations" + spec := acidv1.PostgresSpec{ + TeamID: "myapp", NumberOfInstances: 1, + Resources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + }, + Volume: acidv1.Volume{ + Size: "1G", + }, + } + ss, err := cl.generateStatefulSet(&spec) + if err != nil { + t.Errorf("in %s no statefulset created %v", testName, err) + } + if ss != nil { + annotation := ss.ObjectMeta.GetAnnotations() + if _, ok := annotation["downscaler/downtime_replicas"]; !ok { + t.Errorf("in %s respective annotation not found on sts", testName) + } + } + +} + func TestInitRobotUsers(t *testing.T) { testName := "TestInitRobotUsers" tests := []struct { diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 9b92f2fb5..534ae7b8e 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -6,6 +6,7 @@ import ( "fmt" "path" "sort" + "strconv" "github.com/sirupsen/logrus" @@ -1182,12 +1183,15 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef return nil, fmt.Errorf("could not set the pod management policy to the unknown value: %v", c.OpConfig.PodManagementPolicy) } + annotations = make(map[string]string) + annotations[rollingUpdateStatefulsetAnnotationKey] = strconv.FormatBool(false) + statefulSet := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: c.statefulSetName(), Namespace: c.Namespace, Labels: c.labelsSet(true), - Annotations: map[string]string{rollingUpdateStatefulsetAnnotationKey: "false"}, + Annotations: c.AnnotationsToPropagate(annotations), }, Spec: appsv1.StatefulSetSpec{ Replicas: &numberOfInstances, diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index b38458af8..3528c46f4 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -853,3 +853,24 @@ func (c *Cluster) updateConnectionPoolerDeployment(oldDeploymentSpec, newDeploym return deployment, nil } + +//updateConnectionPoolerAnnotations updates the annotations of connection pooler deployment +func (c *Cluster) updateConnectionPoolerAnnotations(annotations map[string]string) (*appsv1.Deployment, error) { + c.logger.Debugf("updating connection pooler annotations") + patchData, err := metaAnnotationsPatch(annotations) + if err != nil { + return nil, fmt.Errorf("could not form patch for the deployment metadata: %v", err) + } + result, err := c.KubeClient.Deployments(c.ConnectionPooler.Deployment.Namespace).Patch( + context.TODO(), + c.ConnectionPooler.Deployment.Name, + types.MergePatchType, + []byte(patchData), + metav1.PatchOptions{}, + "") + if err != nil { + return nil, fmt.Errorf("could not patch connection pooler annotations %q: %v", patchData, err) + } + return result, nil + +} diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 0c4c662d4..697fc2d05 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -3,6 +3,7 @@ package cluster import ( "context" "fmt" + "regexp" "strings" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" @@ -358,6 +359,8 @@ func (c *Cluster) syncStatefulSet() error { } } } + annotations := c.AnnotationsToPropagate(c.Statefulset.Annotations) + c.updateStatefulSetAnnotations(annotations) if !podsRollingUpdateRequired && !c.OpConfig.EnableLazySpiloUpgrade { // even if desired and actual statefulsets match @@ -397,6 +400,30 @@ func (c *Cluster) syncStatefulSet() error { return nil } +// AnnotationsToPropagate get the annotations to update if required +// based on the annotations in postgres CRD +func (c *Cluster) AnnotationsToPropagate(annotations map[string]string) map[string]string { + toPropagateAnnotations := c.OpConfig.DownscalerAnnotations + pgCRDAnnotations := c.Postgresql.ObjectMeta.GetAnnotations() + + if toPropagateAnnotations != nil && pgCRDAnnotations != nil { + for _, anno := range toPropagateAnnotations { + for k, v := range pgCRDAnnotations { + matched, err := regexp.MatchString(anno, k) + if err != nil { + c.logger.Errorf("annotations matching issue: %v", err) + return nil + } + if matched { + annotations[k] = v + } + } + } + } + + return annotations +} + // checkAndSetGlobalPostgreSQLConfiguration checks whether cluster-wide API parameters // (like max_connections) has changed and if necessary sets it via the Patroni API func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration() error { @@ -939,6 +966,11 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql } } + newAnnotations := c.AnnotationsToPropagate(c.ConnectionPooler.Deployment.Annotations) + if newAnnotations != nil { + c.updateConnectionPoolerAnnotations(newAnnotations) + } + service, err := c.KubeClient. Services(c.Namespace). Get(context.TODO(), c.connectionPoolerName(), metav1.GetOptions{}) diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index f66eafb1e..4eed44924 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -73,6 +73,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.PodRoleLabel = fromCRD.Kubernetes.PodRoleLabel result.ClusterLabels = fromCRD.Kubernetes.ClusterLabels result.InheritedLabels = fromCRD.Kubernetes.InheritedLabels + result.DownscalerAnnotations = fromCRD.Kubernetes.DownscalerAnnotations result.ClusterNameLabel = fromCRD.Kubernetes.ClusterNameLabel result.NodeReadinessLabel = fromCRD.Kubernetes.NodeReadinessLabel result.PodPriorityClassName = fromCRD.Kubernetes.PodPriorityClassName diff --git a/pkg/controller/postgresql.go b/pkg/controller/postgresql.go index 2a9e1b650..c243f330f 100644 --- a/pkg/controller/postgresql.go +++ b/pkg/controller/postgresql.go @@ -487,7 +487,9 @@ func (c *Controller) postgresqlUpdate(prev, cur interface{}) { if pgOld != nil && pgNew != nil { // Avoid the inifinite recursion for status updates if reflect.DeepEqual(pgOld.Spec, pgNew.Spec) { - return + if reflect.DeepEqual(pgNew.Annotations, pgOld.Annotations) { + return + } } c.queueClusterEvent(pgOld, pgNew, EventUpdate) } diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 37ba947d6..d5c4b6671 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -34,6 +34,7 @@ type Resources struct { SpiloPrivileged bool `name:"spilo_privileged" default:"false"` ClusterLabels map[string]string `name:"cluster_labels" default:"application:spilo"` InheritedLabels []string `name:"inherited_labels" default:""` + DownscalerAnnotations []string `name:"downscaler_annotations"` ClusterNameLabel string `name:"cluster_name_label" default:"cluster-name"` PodRoleLabel string `name:"pod_role_label" default:"spilo-role"` PodToleration map[string]string `name:"toleration" default:""` From 76d43525f7463fb7598ea83736052e9d4ee91325 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 4 May 2020 16:23:21 +0200 Subject: [PATCH 043/168] define more default values for opConfig CRD (#955) --- charts/postgres-operator/values.yaml | 2 +- manifests/configmap.yaml | 10 +++---- pkg/controller/operator_config.go | 42 ++++++++++++++-------------- pkg/util/config/config.go | 2 +- pkg/util/util.go | 14 ++++++++-- 5 files changed, 39 insertions(+), 31 deletions(-) diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 5578a5ed1..cb8e29081 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -109,7 +109,7 @@ configKubernetes: # Postgres pods are terminated forcefully after this timeout pod_terminate_grace_period: 5m # template for database user secrets generated by the operator - secret_name_template: '{username}.{cluster}.credentials' + secret_name_template: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}" # group ID with write-access to volumes (required to run Spilo as non-root process) # spilo_fsgroup: "103" diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 537055b18..0a740e198 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -50,16 +50,16 @@ data: # inherited_labels: application,environment # kube_iam_role: "" # log_s3_bucket: "" - # logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup" + logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup" # logical_backup_s3_access_key_id: "" - # logical_backup_s3_bucket: "my-bucket-url" + logical_backup_s3_bucket: "my-bucket-url" # logical_backup_s3_region: "" # logical_backup_s3_endpoint: "" # logical_backup_s3_secret_access_key: "" - # logical_backup_s3_sse: "AES256" - # logical_backup_schedule: "30 00 * * *" + logical_backup_s3_sse: "AES256" + logical_backup_schedule: "30 00 * * *" master_dns_name_format: "{cluster}.{team}.{hostedzone}" - # master_pod_move_timeout: 10m + # master_pod_move_timeout: 20m # max_instances: "-1" # min_instances: "-1" # min_cpu_limit: 250m diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 4eed44924..389240c09 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -33,28 +33,28 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result := &config.Config{} // general config - result.EnableCRDValidation = fromCRD.EnableCRDValidation + result.EnableCRDValidation = util.CoalesceBool(fromCRD.EnableCRDValidation, util.True()) result.EnableLazySpiloUpgrade = fromCRD.EnableLazySpiloUpgrade result.EtcdHost = fromCRD.EtcdHost result.KubernetesUseConfigMaps = fromCRD.KubernetesUseConfigMaps - result.DockerImage = fromCRD.DockerImage + result.DockerImage = util.Coalesce(fromCRD.DockerImage, "registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115") result.Workers = fromCRD.Workers result.MinInstances = fromCRD.MinInstances result.MaxInstances = fromCRD.MaxInstances result.ResyncPeriod = time.Duration(fromCRD.ResyncPeriod) result.RepairPeriod = time.Duration(fromCRD.RepairPeriod) result.SetMemoryRequestToLimit = fromCRD.SetMemoryRequestToLimit - result.ShmVolume = fromCRD.ShmVolume + result.ShmVolume = util.CoalesceBool(fromCRD.ShmVolume, util.True()) result.SidecarImages = fromCRD.SidecarImages result.SidecarContainers = fromCRD.SidecarContainers // user config - result.SuperUsername = fromCRD.PostgresUsersConfiguration.SuperUsername - result.ReplicationUsername = fromCRD.PostgresUsersConfiguration.ReplicationUsername + result.SuperUsername = util.Coalesce(fromCRD.PostgresUsersConfiguration.SuperUsername, "postgres") + result.ReplicationUsername = util.Coalesce(fromCRD.PostgresUsersConfiguration.ReplicationUsername, "standby") // kubernetes config result.CustomPodAnnotations = fromCRD.Kubernetes.CustomPodAnnotations - result.PodServiceAccountName = fromCRD.Kubernetes.PodServiceAccountName + result.PodServiceAccountName = util.Coalesce(fromCRD.Kubernetes.PodServiceAccountName, "postgres-pod") result.PodServiceAccountDefinition = fromCRD.Kubernetes.PodServiceAccountDefinition result.PodServiceAccountRoleBindingDefinition = fromCRD.Kubernetes.PodServiceAccountRoleBindingDefinition result.PodEnvironmentConfigMap = fromCRD.Kubernetes.PodEnvironmentConfigMap @@ -64,31 +64,31 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.ClusterDomain = util.Coalesce(fromCRD.Kubernetes.ClusterDomain, "cluster.local") result.WatchedNamespace = fromCRD.Kubernetes.WatchedNamespace result.PDBNameFormat = fromCRD.Kubernetes.PDBNameFormat - result.EnablePodDisruptionBudget = fromCRD.Kubernetes.EnablePodDisruptionBudget - result.EnableInitContainers = fromCRD.Kubernetes.EnableInitContainers - result.EnableSidecars = fromCRD.Kubernetes.EnableSidecars + result.EnablePodDisruptionBudget = util.CoalesceBool(fromCRD.Kubernetes.EnablePodDisruptionBudget, util.True()) + result.EnableInitContainers = util.CoalesceBool(fromCRD.Kubernetes.EnableInitContainers, util.True()) + result.EnableSidecars = util.CoalesceBool(fromCRD.Kubernetes.EnableSidecars, util.True()) result.SecretNameTemplate = fromCRD.Kubernetes.SecretNameTemplate result.OAuthTokenSecretName = fromCRD.Kubernetes.OAuthTokenSecretName result.InfrastructureRolesSecretName = fromCRD.Kubernetes.InfrastructureRolesSecretName - result.PodRoleLabel = fromCRD.Kubernetes.PodRoleLabel + result.PodRoleLabel = util.Coalesce(fromCRD.Kubernetes.PodRoleLabel, "spilo-role") result.ClusterLabels = fromCRD.Kubernetes.ClusterLabels result.InheritedLabels = fromCRD.Kubernetes.InheritedLabels result.DownscalerAnnotations = fromCRD.Kubernetes.DownscalerAnnotations - result.ClusterNameLabel = fromCRD.Kubernetes.ClusterNameLabel + result.ClusterNameLabel = util.Coalesce(fromCRD.Kubernetes.ClusterNameLabel, "cluster-name") result.NodeReadinessLabel = fromCRD.Kubernetes.NodeReadinessLabel result.PodPriorityClassName = fromCRD.Kubernetes.PodPriorityClassName - result.PodManagementPolicy = fromCRD.Kubernetes.PodManagementPolicy + result.PodManagementPolicy = util.Coalesce(fromCRD.Kubernetes.PodManagementPolicy, "ordered_ready") result.MasterPodMoveTimeout = time.Duration(fromCRD.Kubernetes.MasterPodMoveTimeout) result.EnablePodAntiAffinity = fromCRD.Kubernetes.EnablePodAntiAffinity - result.PodAntiAffinityTopologyKey = fromCRD.Kubernetes.PodAntiAffinityTopologyKey + result.PodAntiAffinityTopologyKey = util.Coalesce(fromCRD.Kubernetes.PodAntiAffinityTopologyKey, "kubernetes.io/hostname") // Postgres Pod resources - result.DefaultCPURequest = fromCRD.PostgresPodResources.DefaultCPURequest - result.DefaultMemoryRequest = fromCRD.PostgresPodResources.DefaultMemoryRequest - result.DefaultCPULimit = fromCRD.PostgresPodResources.DefaultCPULimit - result.DefaultMemoryLimit = fromCRD.PostgresPodResources.DefaultMemoryLimit - result.MinCPULimit = fromCRD.PostgresPodResources.MinCPULimit - result.MinMemoryLimit = fromCRD.PostgresPodResources.MinMemoryLimit + result.DefaultCPURequest = util.Coalesce(fromCRD.PostgresPodResources.DefaultCPURequest, "100m") + result.DefaultMemoryRequest = util.Coalesce(fromCRD.PostgresPodResources.DefaultMemoryRequest, "100Mi") + result.DefaultCPULimit = util.Coalesce(fromCRD.PostgresPodResources.DefaultCPULimit, "1") + result.DefaultMemoryLimit = util.Coalesce(fromCRD.PostgresPodResources.DefaultMemoryLimit, "500Mi") + result.MinCPULimit = util.Coalesce(fromCRD.PostgresPodResources.MinCPULimit, "250m") + result.MinMemoryLimit = util.Coalesce(fromCRD.PostgresPodResources.MinMemoryLimit, "250Mi") // timeout config result.ResourceCheckInterval = time.Duration(fromCRD.Timeouts.ResourceCheckInterval) @@ -115,8 +115,8 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.AdditionalSecretMountPath = fromCRD.AWSGCP.AdditionalSecretMountPath // logical backup config - result.LogicalBackupSchedule = fromCRD.LogicalBackup.Schedule - result.LogicalBackupDockerImage = fromCRD.LogicalBackup.DockerImage + result.LogicalBackupSchedule = util.Coalesce(fromCRD.LogicalBackup.Schedule, "30 00 * * *") + result.LogicalBackupDockerImage = util.Coalesce(fromCRD.LogicalBackup.DockerImage, "registry.opensource.zalan.do/acid/logical-backup") result.LogicalBackupS3Bucket = fromCRD.LogicalBackup.S3Bucket result.LogicalBackupS3Region = fromCRD.LogicalBackup.S3Region result.LogicalBackupS3Endpoint = fromCRD.LogicalBackup.S3Endpoint diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index d5c4b6671..d8c92ba3e 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -83,7 +83,7 @@ type LogicalBackup struct { LogicalBackupS3Endpoint string `name:"logical_backup_s3_endpoint" default:""` LogicalBackupS3AccessKeyID string `name:"logical_backup_s3_access_key_id" default:""` LogicalBackupS3SecretAccessKey string `name:"logical_backup_s3_secret_access_key" default:""` - LogicalBackupS3SSE string `name:"logical_backup_s3_sse" default:"AES256"` + LogicalBackupS3SSE string `name:"logical_backup_s3_sse" default:""` } // Operator options for connection pooler diff --git a/pkg/util/util.go b/pkg/util/util.go index 46df5d345..5701429aa 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -147,7 +147,7 @@ func Coalesce(val, defaultVal string) string { return val } -// Yeah, golang +// CoalesceInt32 works like coalesce but for *int32 func CoalesceInt32(val, defaultVal *int32) *int32 { if val == nil { return defaultVal @@ -155,6 +155,14 @@ func CoalesceInt32(val, defaultVal *int32) *int32 { return val } +// CoalesceBool works like coalesce but for *bool +func CoalesceBool(val, defaultVal *bool) *bool { + if val == nil { + return defaultVal + } + return val +} + // Test if any of the values is nil func testNil(values ...*int32) bool { for _, v := range values { @@ -166,8 +174,8 @@ func testNil(values ...*int32) bool { return false } -// Return maximum of two integers provided via pointers. If one value is not -// defined, return the other one. If both are not defined, result is also +// MaxInt32 : Return maximum of two integers provided via pointers. If one value +// is not defined, return the other one. If both are not defined, result is also // undefined, caller needs to check for that. func MaxInt32(a, b *int32) *int32 { if testNil(a, b) { From bb3d2fa678afd579184a6ee81d13fba9e03459dd Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 5 May 2020 12:52:54 +0200 Subject: [PATCH 044/168] Bump v1.5.0 (#954) * bump to v1.5.0 * update helm charts and docs * update helm charts and packages * update images for spilo, logical-backup and pooler --- charts/postgres-operator-ui/Chart.yaml | 6 +- charts/postgres-operator-ui/index.yaml | 27 ++++++- .../postgres-operator-ui-1.5.0.tgz | Bin 0 -> 3786 bytes .../templates/deployment.yaml | 4 + charts/postgres-operator-ui/values.yaml | 10 ++- charts/postgres-operator/Chart.yaml | 4 +- charts/postgres-operator/index.yaml | 76 ++++++------------ .../postgres-operator-1.2.0.tgz | Bin 6799 -> 0 bytes .../postgres-operator-1.3.0.tgz | Bin 19063 -> 0 bytes .../postgres-operator-1.4.0.tgz | Bin 42200 -> 14223 bytes .../postgres-operator-1.5.0.tgz | Bin 0 -> 15843 bytes charts/postgres-operator/values-crd.yaml | 10 +-- charts/postgres-operator/values.yaml | 8 +- docs/index.md | 31 +++++-- manifests/complete-postgres-manifest.yaml | 2 +- manifests/configmap.yaml | 2 +- manifests/postgres-operator.yaml | 2 +- ...gresql-operator-default-configuration.yaml | 4 +- pkg/controller/operator_config.go | 2 +- pkg/util/config/config.go | 2 +- ui/manifests/deployment.yaml | 2 +- 21 files changed, 105 insertions(+), 87 deletions(-) create mode 100644 charts/postgres-operator-ui/postgres-operator-ui-1.5.0.tgz delete mode 100644 charts/postgres-operator/postgres-operator-1.2.0.tgz delete mode 100644 charts/postgres-operator/postgres-operator-1.3.0.tgz create mode 100644 charts/postgres-operator/postgres-operator-1.5.0.tgz diff --git a/charts/postgres-operator-ui/Chart.yaml b/charts/postgres-operator-ui/Chart.yaml index a6e46ab3e..13550d67e 100644 --- a/charts/postgres-operator-ui/Chart.yaml +++ b/charts/postgres-operator-ui/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 name: postgres-operator-ui -version: 1.4.0 -appVersion: 1.4.0 +version: 1.5.0 +appVersion: 1.5.0 home: https://github.com/zalando/postgres-operator description: Postgres Operator UI provides a graphical interface for a convenient database-as-a-service user experience keywords: @@ -14,8 +14,6 @@ keywords: maintainers: - name: Zalando email: opensource@zalando.de -- name: siku4 - email: sk@sik-net.de sources: - https://github.com/zalando/postgres-operator engine: gotpl diff --git a/charts/postgres-operator-ui/index.yaml b/charts/postgres-operator-ui/index.yaml index 0cd03d6e5..114e6a4d7 100644 --- a/charts/postgres-operator-ui/index.yaml +++ b/charts/postgres-operator-ui/index.yaml @@ -1,9 +1,32 @@ apiVersion: v1 entries: postgres-operator-ui: + - apiVersion: v1 + appVersion: 1.5.0 + created: "2020-05-04T16:36:04.770110276+02:00" + description: Postgres Operator UI provides a graphical interface for a convenient + database-as-a-service user experience + digest: ff373185f9d125f918935b226eaed0a245cc4dd561f884424d92f094b279afe9 + home: https://github.com/zalando/postgres-operator + keywords: + - postgres + - operator + - ui + - cloud-native + - patroni + - spilo + maintainers: + - email: opensource@zalando.de + name: Zalando + name: postgres-operator-ui + sources: + - https://github.com/zalando/postgres-operator + urls: + - postgres-operator-ui-1.5.0.tgz + version: 1.5.0 - apiVersion: v1 appVersion: 1.4.0 - created: "2020-02-24T15:32:47.610967635+01:00" + created: "2020-05-04T16:36:04.769604808+02:00" description: Postgres Operator UI provides a graphical interface for a convenient database-as-a-service user experience digest: 00e0eff7056d56467cd5c975657fbb76c8d01accd25a4b7aca81bc42aeac961d @@ -26,4 +49,4 @@ entries: urls: - postgres-operator-ui-1.4.0.tgz version: 1.4.0 -generated: "2020-02-24T15:32:47.610348278+01:00" +generated: "2020-05-04T16:36:04.768922456+02:00" diff --git a/charts/postgres-operator-ui/postgres-operator-ui-1.5.0.tgz b/charts/postgres-operator-ui/postgres-operator-ui-1.5.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..6d64ee3b55582f3420da6c1d78ad883affe7a09e GIT binary patch literal 3786 zcmV;*4mI%~iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PK8ibK5wQa6a=_^wQ5(&16H0)SGQq<+=4Z-jr(-M`b(N+S{6P zL1at9m;x98l{{mk$5dtI;H^}3yBp5N>E-ZStXrS}abm5PXG z-Z%HvoZKIzkc56fAt`48%zTR^N%_-vd#-0iC__OLWnK+(Zh?0>x4`>TNCcnJ2qh30 z3zAG|NMfK&p%{@6VZ;Rx2sxV}qsWv65D`U&M505aLmY`>Mni;Dq5!;2P*7wcS`$9S z047Q$GHA8NR87*M8}e!E6Nw3ncx#Dwt!uoy;UbcOwuIZ)O+kW$NmFy?9GFrw%T zH6&4jGhHDQ8gpw(C{u(o6b2Da35^4dgh`%?5dWSJz>UxXU_=k|zq40X_G{Iyva-eo z<%1?Qt1rmouK&vYKO=F9^3epqhW+31`}@uP-|P9i{r?nY4_@Jjq_Ki*(am;U#T2f! z2bU8nf$EL^@1wVGoDmmOq7+6jqA_ZKR~W`bAk2uMWEi6a#bJn$5GgUzLV?2_PX#EP zCNWVcUCR=f#55$wJY^~X-?H|!v>4HGDiBf%bV|l(nZJPrAaG2j5_duQCO%(sK_55`~ck`BMN&q-a^W*gFbCGx0TO30&0R zHY3(}E+=T@67UhI2{v&ku7#~@Ku<@&xB`g^2#a!5dXY7<7eF#RRKggUrW!_&O3KCt z?#v_ou}Vfj}c<5hnT% zQ!hZ}8$r`xq3MRGZV;L2;r*#WQsO-toVswzBno9(6-uwVN>yN(puoRaP`*4$=z9?d zU~AnGw3eAUIPeZUyVBw~PNhO|MyB|N3{kfYvsaGp)iwmp*Wm+|bQog*_CH80O`RfQ zj7qnb2?=#olt~JEiMsh@3YOM1R(FYRl8@nZpfM&~r~vE(@4&YU-qZ8eumueYx=;yc zJi-Cj3t(D8NSJ1OneUM*AF={;L@|!4xa&0N|p#|z*B=m@BS3DI?#7}P6_3RSMW zF|LRbDD7!xT2U}MT|R|2Y7IwaI~iJkQfFyZAka?Fna zQtIL@nIXwzsS1Ne_^|Ed=?*3Z8h3V;I=Ly>WaZtbX8W;GS`dl?!h!#*Vi z_V2Csb|o(>?Y~kxx6a;Oo?N)7VE zbC}U-U_1cr$7yTq1F{(Ej7UXkOdz!i8JGoB#D`ZYyAr|>;_)xKJ=6%3^63BM38)n@Xu0aWs^7Nq%^*P@n0~~UX}3+ zej45x!n``W(6HJ@)%NDku5~&Bkx9r5Nytr{N#>45=4mOop>eD&U2Saw(eAb$d_LOk z=v+8v3-ZuWN!p^aByRz;7GNv-$8v$`|CzZ2FSlRf$~a51WLRoiWii&;+XoLY5tOMB z*ngGIU!~o^mi7A(rd#*iV8+6`A-N;|kR`3h3(hKwTN3#v%ECB}HZz6$bGZn0v}R4< zOOvRus+I7UtW)MRXx_Jn2KDVleL#B$P zYQhEmWFiyp^?@^bK#T2oXcB8mj%35o9Mi{fdOfYbKA1(_hp;}xqt z2!nrX7s4Q#U_5o@q-6xpHtll>RU%j($&@fM#>g4o8GNNy$>Xo<8&W0}VIkI=o#nz< zq;Ym`L=yU1@H8=`a7}+zA=cm!j?0|moP4qB}wE8Oo}KRrLjcyAZ)8fReHT&!gPOc zl+2HXp$nOAO+ z;RN$}Yei*|-vsk{uvD$c80Pb(ZU&Y7rlmHEG?zer=c)w{KDI?;TM5-((suEax)w>oYKF(uBZo*QtnX)}&$1 z6Gwx=)%)`|%}JSDlts!<%tI2JWQ`()W(BHugOl^4%XjBjM}yNZ;H)e?vYB-2X-4&N zufl%(=KaOx$@$gU(c6=&H%GsmJhY=`cGB7|&rdGiy+1!bxw!gpdU5*8o0Ba&YUVbr z?cnn0{PoGDnaRcA=y+TFjr^$f=ueK`UaZ0S*=f87_|MWC$VJ&w#jhLI=>6%{@w>Bs zoW9<^_xn}7M?92QF=f|wU>BLoOJdR0TUt#}18Y%@tYu6{)v{l#M9Sn+4Y{lK zV{)@rHpun)!+R=ubQ|DQqSy*>5%;^wk7uMEtYjNc8sH&jiQxU7=poIzI8o=s;}OS@uOcNd8BWyxH+HQcP?9KXueL4ygx#<(7`$crv5 z|4JM4zH09|X5#P}KBin%EWyh7vKQ5iY29S;duc+ow)p|zmquP2RFTuI`_NCzLo+T# zIe{$AyZl+{#njYoTPJtL%$yrm?~sJo$fBn2mhbhmH9KSlXy#LD&_Z-nAZxT)O|$PU zSN|wwgZ?js^zW(v{jS&i{&&B>-{0x~rznl@e`_Z?yO{4I#(XOwG$)T%RU!&CW2lsL zS%_gNyq0S@!Xo$yKQ7@gpHtbz*HiXVs-Iha6IFIpyjeM>W$-1vwg~iH9uJm6+FujL zeIypkK0{kEDMO_B+_L`UnqoWopXaN7r5e~E|Gj>|6NTdI%ZtpWAw3U|M%McrTZWJ-Sy6h zI5z%0{FDeB(OZnN(7<1QcHte1?|?JYQV$6d6cAH}uI0YExVlhWptT3bd^+U}J{(^_ zM1{26F;y+|Hygg?4nK*O`CGi0j9dCo@l(!NYk^4ntZ8DteUjF5SKcJnb9YFtt>>%P zd%wGM|4XO8+y75e8j2m65;O}nt>TO#zq-~YS8t7bqx9SShLQm6w%7I?uj_QYOaCC~ zdO_#V^*yiGIrQ6yfAiXb=VgaOzsGe`2Jco^P8mo#MyUcg!~@a`+u<r z>x>Tk{$aP%8+mOq9AUptdYutzk9uCz@AdY>-hL-K?2%zaGK1sccUb9Wi+ijJe=)zy zD*IW3HFsKN54C%(%21uT+ba8&-ERdZ&4|`>R5spoW&Fh2T~~`YYi`y7sHIOl@ZKxk zBo*-z5vrXyhS^*-{$zk=O&;8WS-V>?t9N*K*y(nAKPInw9(p($?R&j`)a!Tq`(e}z z5BGch-u`Gf-0z16k>`^zjM_cY9qyBm9Qebq+a^fD!@eK=Ie4{fCa?c4>9xa9HNhpl zPP;z{V|Q0Gw&vIJE~fR|+U|Dyhy8+Gzqgv%m0j7DAF%vy00030|11aSEC5yj0N5d2 A1^@s6 literal 0 HcmV?d00001 diff --git a/charts/postgres-operator-ui/templates/deployment.yaml b/charts/postgres-operator-ui/templates/deployment.yaml index da0280e61..6247ec933 100644 --- a/charts/postgres-operator-ui/templates/deployment.yaml +++ b/charts/postgres-operator-ui/templates/deployment.yaml @@ -41,6 +41,10 @@ spec: value: "http://localhost:8081" - name: "OPERATOR_API_URL" value: {{ .Values.envs.operatorApiUrl }} + - name: "OPERATOR_CLUSTER_NAME_LABEL" + value: {{ .Values.envs.operatorClusterNameLabel }} + - name: "RESOURCES_VISIBLE" + value: {{ .Values.envs.resourcesVisible }} - name: "TARGET_NAMESPACE" value: {{ .Values.envs.targetNamespace }} - name: "TEAMS" diff --git a/charts/postgres-operator-ui/values.yaml b/charts/postgres-operator-ui/values.yaml index dd25864b2..90e9daa66 100644 --- a/charts/postgres-operator-ui/values.yaml +++ b/charts/postgres-operator-ui/values.yaml @@ -8,7 +8,7 @@ replicaCount: 1 image: registry: registry.opensource.zalan.do repository: acid/postgres-operator-ui - tag: v1.4.0 + tag: v1.5.0 pullPolicy: "IfNotPresent" rbac: @@ -25,8 +25,8 @@ serviceAccount: # configure UI pod resources resources: limits: - cpu: 300m - memory: 3000Mi + cpu: 200m + memory: 200Mi requests: cpu: 100m memory: 100Mi @@ -36,12 +36,14 @@ envs: # IMPORTANT: While operator chart and UI chart are idendependent, this is the interface between # UI and operator API. Insert the service name of the operator API here! operatorApiUrl: "http://postgres-operator:8080" + operatorClusterNameLabel: "cluster-name" + resourcesVisible: "False" targetNamespace: "default" # configure UI service service: type: "ClusterIP" - port: "8080" + port: "8081" # If the type of the service is NodePort a port can be specified using the nodePort field # If the nodePort field is not specified, or if it has no value, then a random port is used # notePort: 32521 diff --git a/charts/postgres-operator/Chart.yaml b/charts/postgres-operator/Chart.yaml index 89468dfa4..cd9f75586 100644 --- a/charts/postgres-operator/Chart.yaml +++ b/charts/postgres-operator/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 name: postgres-operator -version: 1.4.0 -appVersion: 1.4.0 +version: 1.5.0 +appVersion: 1.5.0 home: https://github.com/zalando/postgres-operator description: Postgres Operator creates and manages PostgreSQL clusters running in Kubernetes keywords: diff --git a/charts/postgres-operator/index.yaml b/charts/postgres-operator/index.yaml index 53181d74a..63c7b450d 100644 --- a/charts/postgres-operator/index.yaml +++ b/charts/postgres-operator/index.yaml @@ -2,11 +2,33 @@ apiVersion: v1 entries: postgres-operator: - apiVersion: v1 - appVersion: 1.4.0 - created: "2020-02-20T17:39:25.443276193+01:00" + appVersion: 1.5.0 + created: "2020-05-04T16:36:19.646719041+02:00" description: Postgres Operator creates and manages PostgreSQL clusters running in Kubernetes - digest: b93ccde5581deb8ed0857136b8ce74ca3f1b7240438fa4415f705764a1300bed + digest: 43510e4ed7005b2b80708df24cfbb0099b263b4a2954cff4e8f305543760be6d + home: https://github.com/zalando/postgres-operator + keywords: + - postgres + - operator + - cloud-native + - patroni + - spilo + maintainers: + - email: opensource@zalando.de + name: Zalando + name: postgres-operator + sources: + - https://github.com/zalando/postgres-operator + urls: + - postgres-operator-1.5.0.tgz + version: 1.5.0 + - apiVersion: v1 + appVersion: 1.4.0 + created: "2020-05-04T16:36:19.645338751+02:00" + description: Postgres Operator creates and manages PostgreSQL clusters running + in Kubernetes + digest: f8b90fecfc3cb825b94ed17edd9d5cefc36ae61801d4568597b4a79bcd73b2e9 home: https://github.com/zalando/postgres-operator keywords: - postgres @@ -23,50 +45,4 @@ entries: urls: - postgres-operator-1.4.0.tgz version: 1.4.0 - - apiVersion: v1 - appVersion: 1.3.0 - created: "2020-02-20T17:39:25.441532163+01:00" - description: Postgres Operator creates and manages PostgreSQL clusters running - in Kubernetes - digest: 7e788fd37daec76a01f6d6f9fe5be5b54f5035e4eba0041e80a760d656537325 - home: https://github.com/zalando/postgres-operator - keywords: - - postgres - - operator - - cloud-native - - patroni - - spilo - maintainers: - - email: opensource@zalando.de - name: Zalando - name: postgres-operator - sources: - - https://github.com/zalando/postgres-operator - urls: - - postgres-operator-1.3.0.tgz - version: 1.3.0 - - apiVersion: v1 - appVersion: 1.2.0 - created: "2020-02-20T17:39:25.440278302+01:00" - description: Postgres Operator creates and manages PostgreSQL clusters running - in Kubernetes - digest: d10710c7cf19f4e266e7704f5d1e98dcfc61bee3919522326c35c22ca7d2f2bf - home: https://github.com/zalando/postgres-operator - keywords: - - postgres - - operator - - cloud-native - - patroni - - spilo - maintainers: - - email: opensource@zalando.de - name: Zalando - - email: kgyoo8232@gmail.com - name: kimxogus - name: postgres-operator - sources: - - https://github.com/zalando/postgres-operator - urls: - - postgres-operator-1.2.0.tgz - version: 1.2.0 -generated: "2020-02-20T17:39:25.439168098+01:00" +generated: "2020-05-04T16:36:19.643857452+02:00" diff --git a/charts/postgres-operator/postgres-operator-1.2.0.tgz b/charts/postgres-operator/postgres-operator-1.2.0.tgz deleted file mode 100644 index bd725688c2d5b4a1f6942dede4e49830ed3e59c1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6799 zcmV;A8gS(wiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PHa(mwK;ad(BIS?PXPJpDcM@-uTc;PEz}}wx+-t zkVG^COaKgJTG9UYt!`j04k=NVth^U!U^o64{W$aXQ(^VPQ%qrbKnlWvA z3zjC|-P1D|3z>?L zroS89xvk~S{gXT_6F+lpgpwne4+EBE<)6deZf_98+(cSrmR_BBZh;S;TOiV$S#E&I z7*ZzLl$)Z|*?+x-D9MfG+JMfb6mkke1^_?iW3DB)+yq=sh2$fcDw`$2Or?ATGix(5 z>i4I@&hl|DQfdD)OPGvRzs0V>IbU2T9h*_mh2m?_h4OnBqD19!S28Q+91Ah4l@vi2 zOePW)q)bT5gyfoF@RW&Ug!QFSxsLeXeA~U42LO^GhJSajs^aIL}pVTxrf%~)F1=kL>eygs}n&E&nZCz=~BZ6^qJ-~$O4mLv->8IQzd0ZanZnk_(0fG|ACljMv?np@KgcEJCH zG&dH;93@Xq0Gv3AK}fJOP7}~%EaF&6^GPLJ&Px&06RIDp$n+y6Bc56M;RQmU8aMqN zkC>3tF1^AY8HYqy$u8?=Dh_s9xFTV9pz8%ex;XWu{gE$RiT5n!L6E8VP)cPP62gq& z@(N$QWn-SWzk*1~iI{$5ddlqxI!E;9JC+dvr(AN)5^#lbO<*8nnx!0tudn^u6>~Tt z*IQt60g_)p%q$x-!y!{KTnIZ;xrIce!lMLAf|;d|@>JG@Pp>$KG|U z&UuG`a~s8ojpPxbVys5lKT#S^ob>I%t2DC<$PG8};|t@(GaMhC^$@&`;&7%6_SJzr zLs<>6iq5%qWDdap89E-mmT&vSsU+~b!{KPyJLqQrMu1cH31ps*xh7_l#xfakgB(<9 zJFmrh&>g}LAeAFc0FVk9mQ_a3=?)#XsgQR7%08_GnsF^u3}bFDIG0FY!=s$Yrg_GM z2F-uVxv|F0QYLih>WttZy$OU_$fyYb-;@CM1`a~RBxIV;xwH^*jq?N|hNGw^o;^U# zxsFtt3Y&6i9lLQGy04*cBeW`{9*>|iVTs|Lz>i0Upp1xl#I&2iV4Fj4986avtaE`O zhM~_D{k6)ikO*eROTz$6#x5j*&a{fSF=g+OE>W&Emo`CWzfk%dtpn)n(|oID7C7r@ zjNDxgY;LMK_md&Bf@oj=j@`l^F-;21vqVI0+7@F|a)2Am;(J)s9^n+0@p#b$Zt{$4 zEJKvT-i>)F6~_w@QrLJLfGb#oR@zP;^U6kZ=*N7*^2EaN`!k3Y8cj&rA77M=3>P!5 zYxEVR(HB5bCUiANCG#gLVhO=#aV(r3fP@4eFk{42@)$dTBUiMe#=r82c(j0;cuC2! zl1?LM+#zK8+JIK|tpHXhQJqU_jUbs^OC#`>7lF5GI!(AC(aEG0z$O!<@&bh>!YUaf zQ?aRJXQtv1%NNC~c~FOXa{Ll8Hr(Pr?}R*@Gl5vZ0F;Jd6J9Kg&Pkny<2-h{q7uu* z5o$t6Ipqaj)j7+2qM5Nek8G|vXobzY>4(OB#*86Dn?N-Shg3LpFvBK4{g!l_g}YSP z&Cx^{ix~M4w=-r*!i+fxIpbR3kW92n8w`*#DJI-lkBKi=azP;gtVlWPF`Qv0k8p}l z@&r;b^%@c-79)r_GOnFmnrO!3JVBa*RAlt;_;N(!RLyzlVUJ*UkOqZ>TkZ)fy&fbW zi%8BQNH8;%Co$5FB|JJ&F&vlt2rr2r zA3Kghn=l@`DK$|#Wyq+d9eL>o`l_47W5>FvQnZLspk*pi(*>OK1+~?hb_jcEYK29u z@SNj3YZ`xOXCNl+Nm80wWD^0GbD@<)r3Q1R1+s`!AS%nL^GX7rbG<+eyj+gMTD8#0 z)|6${7#21!Hi(^ZUNkk5ks#(OB&6ozWm9{HsJ#UnIn|QTEuW@%+fEvN~mxVIC>XjVWi!_fpvQlEc zr?#hBK%w+7d!L(&}7#`;{4xN_MqUwy-)LdnOQx3MYo3 zkDxOg?0NRMm~lHJDsc~_bWaK~g75{JWnwN8KIJi_PHs7<)%&K3DK+&4v(bzjKb`#0 z6^hY&Qd2SKvht_UDxz9(SiT!U=SioK>Q36QB&q5cw4Dkc1#MPmqgQ4Zw%=Bmu14|h zM8&7P447bRM<+iy)42jrcB1m+&|7d8Oc-T(X?gBBQbbu^paB&BxCJO*Zv?}Ew~@UW zd@H29aqRILz7};zUxHfG+9vGY2^;}G)6m>aXo8@5q0*Y=#>i_cDF4xyA2pQf#f(e1 zV8RxTU0Z%QtrqiyTORv~O@EkRmI`ouV3mpX`iWkzPg&p8@t%pp1tN|;e<$BJN zm5vXyCC5pu`M&CWEe5tpJ#V`Fk$BgKUd{)v;iGnNeB`I<0 zLjw}}BxQqfy%tWKF-07B93sdO#N!_S#Bl61VZM=c* z41y>(R;8h@5!N8OE&+7T`Qp=vruyfn=X}B3Z&p~f-Z3cJw2O={^^){78Guf$S4M7O z*i^mt&V#pa5l&dZY4{Fb(5&}buWi?tX*FC^Q*|q|99thzY8V2wt~L>?TJ_zGrD@up z-&YF%60PE^p0R8yigT{Q_x@?LPh_t!KjO6u170CWnQtRh#}_Ve2OIt z%2#f=H9VR4DRqp+MvVcUb3ImuI|z`dsi_>Wnhiyt!$eUdD>JD_LDS$~CINk4zTQpTygi44V4u3fV zrQ!9_iKph_h54XgN13DJVHM4Bfo^!jE)3R~Mj6zM>c@z2s&Z+IE)}j*6%Z=iQ7Sfq z4r;cpd)IZ2oAWxDomJ2f6dS8KFA(tHCnDiyVJuHep;VYpkGwIyrsXhWb~b`eAKUN8 zLi=`+6q-+wY|SBfb@sAw2#{kgOz4V@fal!^N0}wvVFl#P;X7yKAcUNhi$_|pw8v8R zSxI(bXjRY~LxuKexP(5=v;*kphJvN!t5qK7*&dAZ=-h>ra^s|xnTaezZmP{fkt_=S z*})&7*$Z9GO@aM#_!bPGqCaXJQ~h%Iw)@I?Hb~zlz&blpXi+N(zpAl!irdayIg>@i zl9wzx&piW<={NXb4lgGn0(3Gq0PGUh3ti!`D2#r=00skig8#YsN73cN6g36wcmPFf zi0LD5G@w^zAe)A9o@Ojb9=F?TE?;zRUcd15yRLqxO@GIC9vjU~o=}{0xt}$nr;D!p z-OY7EBfxU2Qq*CEiHe<_#;A(8}UY|g## zn$DcnI@L^CA99RTN=7G2nUmLLu+K4}R|geHRG1XSuIYqI3stbIxHt-O&S))bjNK_B zzonM@*a(aH7O5vCkfk+?LgnQKnZ)x(31&qEq6XB2EE7$O$Gi1jE&+p|PT$sN7t8Go z#iw5}cvLPZpthp`o|9u4OT)&RV72@yM&Vp*IavoRyBRw7LU>w=Gd_k(Zjt&bNw z?z@D-A7&OD$(RPQBR+dNTg_J}n(KGX?*IZ-@^q55>znIJ;F+uyRh_0Q!JUpr#2TpIXr=#M+zJt-tRNR#+=KY;wrJ&1X+-1^=Zn>OL ztn*i!%NF`U_1lc#fBqK{c&}n!tTTNP0w~M6(_F;7kG=02 zp16$~{Q&~Heb=I?=>>b;KQi0ioC-5$+-5NzKSiHrgxVT~N@Nr0cFWbBA1*JUcghpa4DY=! z-on)tOu2PVQ#XMQ%d%cmBq0Q>T*l5iEhKv6?Uv#B+Jse*$xnj>v|bh=kik#(zkWEYXaQ;7R}LD!@{~t4|p@ zm`m1aMCU~UU#{!p)%6jg=35pEp{p>cMW62N2mXF`jn3WMc+oPW%J3$UFOfWdKcWlkw)hzE-yFyc zFhbsAy8)34qrRezn3QxaK9;mC%B(H05%CjKLHY}IWc0)4<6FbCq=|#l&?gnHjkmW( zFw;WX33UEyx_>pDC1|eQ+i={v-%FfWteY|K#BDsDO-J*6ZDdOdKN2!Z^7saJ=shmU z--Jn{0oS2l<4}|Ue)l7CCvmA~c~^=YlZ|>tre^<1a_ef&hLgx~2n&<*;3e}8Z3{Lk)ScYFTtVV=v&x_;1O7WI4(>FO$| z9}c8XEVnbI#b*jg^v++90Y4v(xn;xPT*!C?M~-PuRl$2KOHR54#c>PG*m*PDSuXOp&DShs8*B}}AC@R!gmoBE;# zZo=J>hnBfyvhZhhjQEUZ{oyZX?3sQUC9UMIYHX_Yc(b(i;x5cslY^I+HpI4%hm(94-Y^k{FF~dZrAB+ zcS30Ga0CEtN9>NoH@VyTiEfAk-rSN9?0{6Xg`u|BmA0vPxj`0hbs}UUB#(m~vZvg6 zxl-3JozY&3nk8aRdtZBZXQ$e+rFAP;--0^0R{j^>)EzH_xk>&%AMP)&|2!M)ZRP(% zJU;Nwe=BuB{i>1rO(V++p%VK=k=4@vI+)@K zL~#Q9xa5cxqQ4e#?kxwazgaO4uyo=Rpi`heQ|i1kf=I_*0+J+w$vACU_SXU|lTj8C zn8N0-XihH^VeF4fAKtp)2SAqOnkA#9l>z`G-H@Z9X{?XF=?$34lKjvAY7Aco^ybk!r_5Z={^7{W^c(B#~5AifbZ)U1!-f=nl z4Y~Rai^?x?X$y(z5Bi%CUD;-eBNl{7%4{@yTchiJQT9tZSzd_bP^*nR!CTfA0Jn=E zfPd&V5R<1q$kcFeAJJAY_)G7j$EEj@8BWK$J4->S&SU7o?_g$Z_u!ey)2fMb-`{;S zx>B)hf`D_(YGf*C)V%U_wcoc?ru6$>)M3_|Xim=iWTP)(q}82vY`(U?{&TyltCi)?k(1hW%B>pJUCVXkxSZc@z83vI`tbfg-nh=-8b#t|7 z@Z`hU$JeK?&cY8TuTBp?emD&s3O}75eth%cefZ(!f4(~Uc#nA0=Tl=Py3MfFfzgs- zhl91;zkE4#Rj_?+bi1uq?G5G#l$Ro|f>_>Dx(ZU`AmED1_3tYI)DJGMs9*cMMzGc8 z#`0;@s+koBRcljXZN)oA&Hg;kb@u;yt#`XNZjApO443Tx-Q8!~^M4QW+@uXwdgn){ z$F1d{P9qr5pAS31buobLQqdPXBFzgGZn?k%?h=>v)s|Lz_Fbj(#%Dm$H1-<^^}RT-)+6*o6s|4yRGJ9zc7g zt3;rF2hepGtX%u@P-1ss+=h!_1=-4LZf>!-RiQl-L>Lk&LLxVR(sE69PEJPQ-5n+rujsC z;x6A5{LAAWe2~ckl%$%75@dA$s^q<(cYGF}S*3Zf14k-Nm4u&<&L9@r1ih)Sefqyc zKj@7=>puOzcrlyy@qfjiW-j{`Ab(vVT?`sL>6wcxc+wlQ^WaI(rdjah|AHO(nQ5VN z18?|ARc+ z{D1$9`Ty2NW$Nhwt;)IypYuODF<^PG;kTL=usgi>q<|jepEe~R-G+NF2?5=`lK!gO z5p?`7#~l6-%cfs1Y2f}gW#B(<%D`%yM431czG~(`|Bmj>yXW+QZqD3m^1xDbe;cU- zy|%7#L-l5xQ*fJ8aGO(bn^W*sIR!g6M28V{Rjuu@bHk!G;dN;6o` ze2#o)!@t0WX04hGAB?nsfAu5Oy(P&p5}mmC6n37b4u(E<;aCk(m|% zU%eiA8g@z$dR27iIvDMc)4N*8X+uD(BTnzGi#Xi?szmF6LQUJGjvFL(+<{+aT!QAw zf)diGgeT?t;>x`waCCb7)QN33Ix|+p+B${Ycf9ohX>uV9f7)~iox9>S4u0wQnc8XH x>gWWzpeW{8#4(qx`&+gdDYqFZw;3t7&-U3q+vgsi{|^8F|NrJQ?uP)P004?pXx9J$ diff --git a/charts/postgres-operator/postgres-operator-1.3.0.tgz b/charts/postgres-operator/postgres-operator-1.3.0.tgz deleted file mode 100644 index 460fed53286e21075290ed1d7cdcc1778100f95c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 19063 zcmV)XK&`(YiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PH;nd=y3ZD4=p62ndL@A)$njyX107AV)JK0Rp6ugc|VJyPdmR zvbTHe?j<=A2wgfz?;usAO9zoIO`3=(y*DWWf(SzXzn$H^+iM{S$oGA}KdV2z?9RM- zGxO%ndvD&nXQf%r#6VU_TOosSG!qNKO{ok z$nhJ!SBFJt!i#~B*M&fSkB#LprdY@;x8>fs|B(l?5}hE!5;Ua+8EOS)wL1P(t2C+* z1rAvQLs&WVDoLakkSJ0MFfb6~APX=G2NsONOpujIrF3Wy45W?aAj1O2Mo|Q10tBT1 zpq)(*847ZcRhVfD)B-cdSy^pxu!-QzHoeL~TY~#wBu3$Mum^!E(xE+*W^h)kPy*?z zLJ1t-mB2vKHe5+z9FYO}LKw%;6roT8)=H4H!h#VLhY=KH5DaL+2vW<}r&!v?7~p4O z+bSF?0H81)!*0SWh47arQbJ7xuTY)Ia2@S8KL4KUKLaCekbRvD;3exnEId5iZT*L< zLqguJ|2OeqMk7HHoLvjD`zj1HWh6{dnQS7{+F;cx0KhU?9Rp2>LZMA*fE34Iq*5&w zje&Zbi7$uqKXfE*G7*$X3;a072K^KOKnl~7P={k2rpH*Q!wd$_cbI>LgPpce2gvm9erIoGBNEhQCB2V({T4@L`GX~3;Yb*NUYicnhB5zb}}46e(- zNCL+M_T`QiYqsbzXwt?nY3Dc2fa}aO%V~ih|F;Fp(h(HPVHCf)KPV?MKEB&n^X#+;e zzLS_9lB^bBRx3#ugnkNK&RK@%SwcZA5c!RB4EWjC8E6MX^cW;7jq5OqBb=1tXT(U> zy$DXQj16rQI=u}yLC(u?6F4+rE^SzGy?~yPW-J&dlldPcCi>kla6r`2#bFs ze&iryAt;Q4IunB#pz9p7(7-@W=m`fcRubc&7SzwSv5?_k_EkB$qcSiMhkQp!wmx5# zZ%Ailhzx>+CWy<2HxuIwW{4xWtQ9lxFazt0^wOTj@Rs~yFhB+^9X+C2i0k^$6x0G- zuYy^a#Y#ec3v$ka1rrE|#&Dfps_!|c3`7LwyPOCIr&!k+tnchB1Yy9)mY5;khL|BKywK^<8(m>}l~`RVRQH6(YDD(PQ?!o2 zMgHSiN^diy^XtoER|-FsHiqmL8CMeD)D3NfT?fNdsLM@1jO$iN;@baA}QyU3+ z_69;iKp_8*IE<3am_*~r&><%ATZR#{k(|z8wdqL0Lhwjx{Mojp=*6;x1=2PS zbpmldNJ7N-GBJYFal%)=S&LWzu@tXg5%&?k5EKWQ42;wQ4eR>FQAZOZd=UmwXD~xU zx@SF?uP*Hg=yzn1-0N6|2z7~3X#u~b;d-7WvNkJZWM3Szwt>OX7KKEAE#P^%LRbZS zhixR8Br@4JV*<@3F_486hZ+=?NdAjOz>H=Ie#vS9S~{L`GXNZB;#C$THd8bXQy>Wq z9L*p}#!BN+6h(8WcdYcPy|`KOe=!8j5S%@l#8@@~_0P{w!RRrAN7on!)$$ct$Yc-( z7-cZfHj48oEa?lBjU*KuP4egpD5Ujx^V0pxA5ZyT+P~P~gv8X?6cv}ny};S)h9tyTs7jFDg1&u$3G#4p7AQ&Jhgq#ES6PjRz*m;&4)}Q>q2kN<0Tt-m*YCe^ z5&aK8`K^DQ8Io4Wuqw{_N~VC9tp8Ar$NFy+8rtaX`hOEoc6M+e$RI3Qz(Nif2@-O4 zD{N-rhs9uqS`ZlAx37XPrHIY4ViXQIGemnFppEFa0ojWcVmT!*h7nXgt61qQ$BSz~ zB^aD}mH||q1mhE5GBE=(41q(C!|$w=0YqrfPr{O7Ga893;HPu~gA|T_3z$Zu9T{Mb zHjFlsw1eI@j3kT%#CdHQ^+BcR3x6J#^%7~ued zGLSa>CFG!L;23^am}oXYe(1|6l;;4V4T+~tM5Q9h3#E{YfoPIx5-Kc^ z!*Gnlv<|hHZ_YQf^2Y0Up(;RX4;Mw_(92M}^dI%QVBmOVlVCYWal)ybfy4-l8+YdV<0U%JgqGBNH6mn&BIll}C0$;Y$%E0}n*?;<|B1iJ_@pw4UYHvT(q|N{3iL zIK+AtVwADc>};fI{I4h)1(g3qqA8jh@Dll79Tw*1{}-wbYZUrc{(lqCYmxswMgNup zy}Z=`UWW!ilQ0++cn1puCk*eeb`Yt#3F1`eqq0`L@9>J^1H&*3QJ=e{90*}lV< z#w)kakEhD4)Bn$zyfkc98rO+jODA$Y-79fAzQc4^%tE?Nu;_FPasWZ$km5jSex;Gq zPyx%zPRj~bL2{@nU~#$QV*#s3F3kmQLFV~az&gkky5R6-&Wi;dCDBFAqqy=_HmQ91 zKZC!PdjK!t{~@6v5$^s!;p(^N|8L}RmH!P4E))NG(qb^m1VJ4u#+ida9lf1j)!Jr2 z)rue?t7=Wsdf>;?NpK*d6hD!I>qrG4Csandx>6}8*H@(6x6kW%@>~Bnw34*l0;yNL z4*0V3zpxMw|DQ%-Z_oeU$Rn5NFsqeyYV$FU*>{oa{@+SyMH6XRD>TTQtywKlqx&MF zuQI~Kf^i0Od)c_Yu8}QDCmq*U1k_|RkKoCis>N4@(;$HOAoq0EW)YQ47x6X#xq^T? zs#FmdRZEP8N&GJ2=cbhUH2^A*16VT_8Xm#gEY2nzem}3HQLo3O1wlaA-ekihjghN^ zx_7zHt&BtNYb=K;Yr=JLzD_bK3L$y#v_R%Q`zwC-?5am+rGHJ#;=B(h zzEL_?zuYrPHbUY(AV0o@*dg>vC0%v^eftJ``6yj=0Db#vJw9`o2{}ExG?FB{V&PH4 zdH+I=9HcT{ZM{nUk4{Wz71vr9AC;s_h>DMOdjztGUP`v04GUbY)Zfj*+;G!UH}<6#c=b8$!u z&QSY_dW2U9Z%xHTS43Qvqc`4~pMe(un{fZLfR^4cp;Ed_Z~_2m%*L5@9Gwm+o$T^^ zQMTPTX7ToBmlnGxK}V4c^4;=%-)Z&$$~o={7u@hIR&~8}Ur0;12m{Mxbu^EytQM>#9M})k+|NBNBSNp%)#USsVKn@F`n-j=e zU!Pas+uyjyiRb!P@k2$JD4Kz-1WuTGBf;h+|I5Vi-Km`&h(hg_})#OnER>%O7prA^jib>I>a5MuI z{vevRSZE4#icSGI!LSOIiQt0Ke+BdvDt#X&82wjzVKxQx|B?P=GpJxEAVJ1Kcb{2B zpo-13Dgsq{EL{<(;w)B0U_XUF=!7u@ZDT=POf0KVSsA()G;j(PfkP}BuaVbM9%7voD3xqCkByJ#R5Cl!75*EmEn8k{8Ynrd;@U+HJ ztTDaIg&hEdOR6a3762JC-2VDIXHg!+Bs`F^S>(m|CVaCTfipVT zbkV@OTW6hT;;z<2ci=9==d0U@4&~7hUB39l(^s5UueJ3wO+t+FR$Ugxwcu49QC$7vWUxt?h2By#i+|3&Q@=} zOFqIx`^3lFDwe_Vh$j6ifIMSP7ZsW#Gp#4l)%p=+`6ByQdW$?ND`8s z1$ZPSXC9q_#8|e7ljs`ey(rb)bv9>Io7V!a$lY=Wgt{r-zkO9CeFVQREGVyG@(T?F z1=Kmzm75NL-SLJES~w7B*r0{@ zDlIlCwRL>E9z;Q~d2K^c6nr>YsC>_s3r$)+g6S89qj?Y7Usy=q;`hG-jH09AJte>^ z!YK+mZk|itY875N6jUO|2KV2f7P)0xDeOq+my4ySI*j*>Ero{WBd<&0X~}z%m%_l2 zyRdnyQfNwYPQIiHSfnfth3cFSI65oFnF|lhnPaQ4Rnu*Hs3R~767~vRo1{&$bTWml zoym8;NVVOa3%yk<^wj2;oTmr_lW$_4SBF=`G0!K%&#Qy-%mwrsV6v}1`ciJ8SC(LM zx(7B02o@xl?rcN1R}Hem`nriPxRl<`v&|xi#qtb!Mdk7CiS!BuN9I}kTDm0Sgy9x~ z64Xs2Em)L`ZXe`v)FIO}59gJ@c_SYnPU`$-n=xcrJG*#&F6Y5(D0r>*XNXt+$#9A6r&KGWfbnkJeI=C*_jLc|Fn1F+vneYeE9#$(gnPF1hD-5e>Li` zFpv0;>bLiQ-^}yx`TxpE=zqfL*NHHC>Gbq_iY4?U6n%F^htKHtUL8L9F;eluX&f$VQSJblf1@J&ln(7>nZ-R@ za1MsbSN_8Q6PU-)VSabn`n&l;>y|yxeXSeqbWomN5(rnZKH58 z_k-6Kg-ug{~{;m<84ws~2HqHPQVDGso9${-3){Ny^x)NvHz(Ir$^C3RkOad$NQ zKj_=cPm=iOYj$aNV}amb^<$RDgYq##-bYD*>y1gszdLy3OFHOV)5J)U&V+ahB7|g& z^6ALx&7eZrND|YN&MedN$A_imB{nL#%94|&Rz~*dj_0Vm1za;s>d(P$z; zijbBJgsUPN2D9J1ZXrqyZob&cBM}v;3(Oxl$&B82Jo)@Vj zBO{dRP^CIDRUM(#L~6quDMK1-Lqfb(OAu2;5Uh|uS6Y&CWIQU`ywkaOm<((-0xt|5 zNarVS@D72q>yocM>zD@|m?48^Ktc@AnY<(gEH;)Cl4Tf3jDa{HC^X&ZbOoeCyCjVA zp!mO4uKh&EpK(RFjN=l~#d}@fS8?Kz?|?ikO<^G<3vL6i0wTq1=~T$uK<&l5P0ezZ^D{Qo|DNjZ(B_&DFqXFvW4>5V-z7cid++>CAfU&7GOD zB<<;B`7dnxF^X)E%fEW$A)mP@pY}D0Q;YJXh|O7SPNcR(xqRMQkUUo=@6UAXP7cPEfX(A zTX^!l`x2?r_`LBFsS3;ITX;U-B7A;Rhxq=a_Wdc;%dH!X!7Px=*PrL#WTlK3VXy$C z=}p2b$}NZds7tbCdijJ7wcS7D>$-?46k&*cBPd`Zn0>u{aads8pZxITAA{9aB(5$z z{vZY#2n$963gc`HMgk)(n8kqJ4lF@{W_XRqR|%g`ahh|1WV7=6D`#dPbil(26Tz`6 zUod>aMj@{V7U-b?v#~;gLRl!$=%kJg;sR!j1r~xN36!RrGkfZ@5>#>906ZZRJl9bH zq~T)*&W4df5OWq_3>1!eAlpRH?rBQI+H-YCU{N}4 z9#he1Au6*bA_G#Qd|CLJGIN}j)dmN$w>pJBy6`b!`H4zOkPf}A9HncR zPWO+{X!O=n{t}cASA~}{SP7FNRnE__$M+<*w+$nGf6XGi8Hl6|Wko?q7`lI@F9~#P z7Btv6UxE@}LOQDRStN;pM#$t_&`G%|&_GjozBWN`tK7!I*I^JWY(T!kSlYcG@8ru& z-5eDCl4%kGBZ-+joCO@a1izm#7)}^C05O9Z(3}}Eo<>}sXG%r1X)A%=ZpB!ZA-&PI z6141^6qS;a+&;zYSBlQRc0ywQ^~G4gdaoa&lVhV&V`EzSXgS}y35ogE7V0vzSDKbu zPEKqen{V4b>r1Wsgdj$R^ZG3=p-pUZT&jyDx)~koYnfjtjS5zov9dicRjA5}WL!{rqbuBzmu( zU);V|pfFMY|Lm*js@hlRW4e^Q4v3JH08|LaXWZi$^_m$^MCL%5UVRsje%xE!z4 zs!)~S*(2_@THqHNqK;ICG*+rLsp>{rjYg{pRW;IthlHxbLNyITLbV~{iM6+@MXVz- zx(Ja)MmRysXXO`=M4xOHFgA*Eu6KfxnH_~@2oq#EEx^?wjnpBAMh2rg(ijFqBO+j< zMj>Iwa9j-|8{-C}AwsQ(P!p++3=a*}ghm)N;fByq1J(!+HHPYq2pFVH1O>IgL~~X_ z7ckQnsC5`FOay1P=~V{W5{#}b;B>IJn{7zL$)p+Fc|S}nCI0DBRtXFwZNrrm^3|0p z2rg9OUr2>Uzmd0_+%-@Mpamm@VryTxKgMIWyj2qKEpJc za7|=mi9Y|+3-w=mJo$e<16jm2^5_5R@KALl_xr!$jo#+}dm~S+kso}Fe&jVM)(~rx z9Aan_AJN_(8LroK3c*vtiEb&85s95TMZ|X|dtnrAPwhZAYXeD3vD?iEw%UgAe>dGW zZ%Ut)`s_pPzPJNB_L>u9?{V(Q(Y6W6&(^PBw7u%Mn6TpQiIeE$sYsHwwSKAWS5>?@BG>0nor8Sccfg|8|QcQ zT=mbr3x8YhZMk@E$K4k@cD%Uo_k}Gxta*R;d{U##-O>YopM9pwqHWdAY+8Qx@bd{x zgDSRQErZAVPc2t`GQ2ux*Ex#*X!n_zZusQSXJt_X%FRv#Kh5k)_22(Va=qUs&Y1V} zZ@F8C1neJu^;G}E3qNKQE#E8F;+kPa-3@0-bWa#^#eZtxjJiA8FU%>SN!Ms@{k3@K z;Rn5glZiY{yQ!l(ZrihBN2IQMK#dt4jt;q$essu#>?-qOuJ-O6+VEQS((0v?K8~u= zzFvh&3);6iIQYu+o_|anjt_pcu*0S?PtNC@-a7UDV&iKM&COi-&86$dha?AQ{FRXx zv8T!~w(FDb3+DVB)@Z=9HD^nnEW57X(iKN$_&w*K^LDogzjGAy2L$$==Ejc}JpVGd`|!Vlu11Yp@+7G(tXg5`s0$Aix~{PWI^rD1DI@B6UPAD2sC-lM5rYu2NWz8ihXx~Rs;Bm9vT{$K zqcwJx1C#vYi;YuMo!qh2(VJy!tY0!Nv)F;=%S%k}txPRrpn@AuX`|aUf9AeL_8$hn zcPRJZrqw?jO0LJ zy?y68%h&<6hkkXW((`o(mXG_u_(Qv0#oGU%sx#p5+7TPo@ZOiI4lBZ^L_`_4B|WXa zy!egV$8(k%D>we#JnBr3vLy9Qm zU4E{4@#D$&vyPRZ9*jy4ik`UZ+-Q4)N_$Ixl>BQ}_5KaJ4C-_$q2|K_t=JoTb9O%1 zz4QJv&9F+J#3>#Qc&y529qMu6t?NFlAE+<2!_+%yX3hSCOVkQ6 z>`r~~Ns|Hoo!8F)VcUaYS!?@G&x+gs?f3+L)0x`%(^LIBJz8F@Q~eIJ+VvU@rgd-U z|HtGi12**1#&4{BZ2y8v+v`zkU^-o#71UD_Z^wGg8dzRXhzUagJQ$ZbZ z=2DA$gPYFO4Cu1G*u#?R(qE)D=0bMWYw>(CH)F__)L#!Bi}_^aCx4gUpfSen z92NON6rhyB`w zXeQNH|Fm!Jh5Kt#<31*)@0}Yt^6c&(F4cZ9{M{cv{-^VSjp?mFKHsnLk9Wu_%e2?W z<2Bej7aJYy+Uvp}S3ca49JFE0IK_$@`Qnyt zAb5DKT202ZoDouUOM~zZTDb4)g*EIq-+%hej6H_Eo4-z8JFLT87}()>$(avJt~)V3 zjOS0Q5_H@HV!*7nu>%d%(QK2it4}m1e2Q3*}E$ zXxZeaTiyFSn^bjN+l+O|>sM)OX4m^@xT5~hxnt*Qs^7Lh-Z-^)6aT>83iZH0 zN=}D*RfXnP)e91+$gfX4TQs6U_@k0*j*Y6;V%9%LD~&joot&b+{_TUsM+4_|>)qx1 zYEjqY7LU7?aB5YZCAU6VJ4;ckq@|N-erIEq_2)ZHUD4|P-IzU#zbiAfXL{@UEh3L^ zdN(O!-h(Sg)2x--e**1<{cdD(OxMrnzVmV3#*zU#y2P#l8@}9HHK*Z%?|R?9|LOOY zZEbV|yB$AO@AkP;)!%7x{OFl@^T4_GuYQ_-^@n}w4FmFyO{o<=@z$PE?Kbw@S$Ry4 zw0@13SAIA+t>&}g@ws2w2X<_z)c!d!t#pl&{v)dVv|vb$3*7QA+UG8Nr{RLDeGZSG zG^X5|fa1%#_F8$=Zsq!%37fwknC|Zk@3g(D$}> z_}6=jhHvUP`23@0s}%F9jy*hU{~*JN$9DCx?>e{M79F^u=HHWc&FFBnp5LQ=ZSSo- zSEA{qtwg!Q%j_dsZNHo`a`b9q-yrpz)ti#8tgbs^Pe`lt&A!Q5vSGxY;O9;Dj=C{2 zr_)dI!v{9}qi%!Vx1x(zcv=Sz>b$8J=KEbLEvS8S zGTgtj>-7b7(v;)7TD}`_x6HtRMx7RZwX*y#V6*BS?(pB2jt^V(^!noKIXlkKow}?$ zrw%x^cI2qwN8|qttJ*(ddFNIX_G{|a$A<@Ppvxz8*-ktut2?^vuWl!n-IPG3$r z+OtN>*u0fHz8(ozv~Ktr{CxGoVYRPa&KXg+^nuRQrtqfIo;0wH+S6+EJG0qlorX5~ zVBFVBmQ-Ig{lnNfyJ9-{H;i0J_8xz5pZ)QTk0+R}EcgGq%;KRf8@As(B5_2cuaE@ z+daqs#&@55_X#(-WkUCE$}FN;rEiIPqv{SBvvKsp{ew^XmwAzXqv3?Ajm-lRhCOR~ z;Zsm0p(TM&S}}ch`IuG}#}93|;8s~~afAMGRU4nZH!3ZxW9Pwbht(*X8(Dne#Ye}6 zFTQqh$lr5yt*6buj8!_+djOTSTUF+lsjaI1JnGioVZFa$h>@yFD^e3e%yKFj#>;=@<`mJ=2L-7NiGfyg)nKQ3Nuj(n!nJ4R?9k55VIbUzPwn?(N zL&H+rzFD0);`HI}krQHfkDn5xJ<_cBn468B^eh`(-|t;6G@{t;3cr-@adB0^sD3+& zy;nSIVBV&`TOYf+^W*T6zh@PTO&@mCpAMVb*$8F~?LFHV6jyV@>2#HjqdwpE$D=JP z`Yn#n-J4mn+M)Q_Ni(@kbHe8>9=YP#*+aIL{@auQ4us&DTMT|KHs zrOHtkl^32R&(tp&w`a+?-2<()$CogYhqwMQ{qW2a+7s=&)nZ1awry3v?X(Fe;i@C* zuR6b9sZHwCkN2#&Ik;`yBF(KC~w z7^=40^~3a@>;7qQCFaDT)6HY^GM=SYYw==B%34*;C&jM}95_DiX>_gl>gz7Vez0ML z--TblZCr2hwpLh+WkX(Y_bV!f>ATLXy+-Z7ypvyi@!toI?f2Ip`u6g%Rhr!0GIhAQ z)0p{J0SIsbd*mjO#SH^^GADV?M)&UF`et=cRhv-rj#H?R|)DsvRr$ zR;s7()@aqOgmUF=_L=DyG#59o+C8AgPyUm;Dw4-^8UMui<^Hsk#UH6kcA2ikR#e=3 zV1B>cormpHn%lb8>(l(7W15FwpX`n+{+t@?zisM|{_3b_%Z9C(^?lQ(-&DVQB(dA8&~_(qzn4K zOI9ts_bIe1%f8pVU--agist1XKJ9tl7(TE@7jJ)5xp$jQE^ zp5-a9o=@+!$jPz&vS!Zh8mBiBdH3$FIlJ(A?vcmuZ#n%ux7_`l)R60^mbVB!eQu^g zV?1M2*K83x$2j=Zr$gG}6>nr5JTa@5D)_6+0~HUKxwkd=+|Cn~+HbNYq%;~(_Taal zr_`xkVp;be4r`Tl`dqmhU2eq+ZQ5{dt-q@0TzCZtA@uuywTqeeP~AHKNRo=eHx~81|RjGkNU!V^b%UZ+d$E9KU7<>sP#A zN_$rE$rSbTjHxvvHB%>9Hr{z}+tr|`XH8qxANurY>3$c>YA0>@^}WY?w@+jSp?V zpSiL6eD!^0LRy|PYS4@d|TH1#Hx$D~Hv|Vc! z3?KB<(fXBEjNY)JRmky{r%G`zdTq`PpHK8XfdtCw&UMc?N()0?w!x--#*i8`wR2%XJa3AU!2$XpO~eC&);l(^uwpQxw-Vs ze~#SknVJ3K9ya^n!Oi`S8Y(}^`RL}Yz^`kSU62^GL9=j=Nzvxd!@6kerA zT22}BdH7g(B4dv&z1q|tADr)Nn0Dyw@AGm;Cw<&+I@|2=Ty3JZ;q)UV^6*VD*Ft7j zJk@36S=F9)$tCr%rN>;paeZRi7r|?ejp}!`>!@03pB&vW_v+&jchyyir|R)O~~u4(mt1#EJ5%O>>*^_+)O_cp%&&x2MQ?qpUOIxyr!WcdxVe%W3! zar~jDmpANMac6Y>yUBYa>P<;i>!NP8%3E@B#(-$kg!O4%Hjp*zwqMZhk10okW;~tu z9<}rAmtRcYgn#>o$u{cADC6^s!KSOXUKmGjfq#GaH2KTr`nex|ar(|L=LTob&{+1A zIP&9&_x@?lu{Bb9oLrv9En46E$+i8rBetxp)5jL@!>r-IeRBWwxHE|-!^>PfZ7$w@ zX~)iM2WR}+WnY(lH&2p_jE|JzgGQe`kTUK_-@jX~JJ*uCJ*{8Wy34Pu>_vX_C~M8y zN0(GTc9}3}%9Jbha-u)qIrQsx}^;?%0VFP=R+GUayO#T_qxd67Hr`scUv*5Azg z>HfiAif=4e?z6i8OiXM=oxGWz+0uN+TB4Yx_3bgc8*MuMbT~Dv__;IVKf2vNb?uO^ z)}NfTeo5A*sLHg>5BIAxc2JFP7oV|jN)7Wbrq|UcWb3jsO^Z*hUTACB`TRib z#!fG)ocp;(@Ssg`FIv>P8@w{FX|>yyt9j8)oBwe*x1sH`=ihw!G~WN9;;RbJV!u4` z(+lmiZX*+NFSq<2q%FF1%Q~)N+QnIQYGibI-l*M#h3`Dxuw(l@fd9B~$l2f99&c8; z+T`WSCtEsw)fn4|-G3Un>cgcUe^D`l`fB~0oLyROqLF>FZbJPT$@}juqbl7y7aP~3 zVP20vSJe1^%fjXxiRZa_f3KGNBk59fEG@(P~@>6$STWL-R zu87?p+pgBR%@^MPZ1R=qb>}af^lmZ#D`AOKsVI$R;hksS>{sE(i*1hyyzC$F;q|Gi zDxWrL*|->WXa`+w_DOOWwV_&0vwGdPe!u$a-uExEBoY!M|nH$K6skfGi%j}ymI%$Vrk4zf^1otLN55HON>5TF{elw2nuRJ*J02!8@Hvjj&9UiyScKv+Vm87Mk zo9;;arE;+iBSHV%8xbE(SloQy&sMtDu}&dR+TEMKci~^-t0wH-(xhF~lEa1@JMQ%~ zl7C&8w*UAy!bS$-2>aY^BYnZ{rT+kpRr%2n-dN{!{-NM z3r@_+AirO~?FGAK>GH#sD~{~9VPLJy56V=W|Mji5*1r~|9eVNf@a|8Cj4+((vNLno z=Z{9PXRWpqy8Ej)##ddjqSf*vljCn5jh_GIgj2Qy%RfEQiL0e|tdR%^+ja_H{nVo1n-8rzG2qyE?2AF`&TPwx5gPSBEs% z(3N=b!ITY|M?0=N+2H=8zecX_f9gTtkB6pwn{(n?$;=JiPd^Cxbx5g6Wfyl?U-Pr< zg-c`R_Iv-R?UQBQpFK#@XJTiDj_g_Tw*%=EwR&U6ubNe2Mns!ezQ3C4aHx1_`F530 zCcPN-eQ?KT4`-A*c{A_YFTVtjedpZVy!Vehp8P29!f!h|<+a#%{7l{kF2wpMVS8xH zjcZGhTTb4)yZQN!%$e)1=a!4ByC|9HmuNW?mRD}*`^*2)4Z8Dp-uw0&r`jJ}x8>r` zA8y&QH*4m|!H+3pkk*uvFnmMv!zHdx-9K^tdq0k?z+&v zs(9w@wS>SM=t+W@@Cm>jkoqR+4}6Lr_Igjwxe?A zoU;oCs@5H>H7EIxJs0*|SX$@f>SfH}#Ku zo4MS(%X0lM*JAeXs`Jn3ceX9*{K;q4?A^u=>b9^#qhYz3%dSFl2)KFUyIM;h;4H|fBKx2pJ^IjM?HZ1?4~%EfeBYuZ1}X!}v0 zj@)(aCwImj-_mU#5&hXx|Ac!hMm9cM?3_5)V)OpfhhhH@AJ6k& z;R=<~z~HaT2`K;b-_WoyO{m-XZ+Jvl=-cz(H}N>4_%qN%upDF8I)15~?jiEH$qcm8 zEWy!yQOrQ#UVSq-%%lYwYLx~#16ge(nIz(-9@jf>AaEKy*5iqNO>~WAc?4Xw(+;K9T4z z6D%a05B3)kBPf#+z2bW$#8PCHG^NCpRvK6Md*I3stiK~Xt;L=!4sAPV7fK{x7N}6T zg?|Crefd}Ig&^7dUkc%hQ!2)oAg2X>l1o&y{N_`1ats^xb1(^hA28mL9+D?wkjS5E= z9s`5xWC3j@zjuGYuQ?82lpRno(;55DO5+GBZR3RdOTvHvngXoZ;=Ft-CG9p_bfW*P z6Ex%uINxDP41Os1a0(HN1qUe=h_zTbJFu~k1??KMq8Am2iB3`Rz&Qhsa{Fn4pC1C5 zVkJmg45=qfu|kTlCuYIOcg+N&Rf7Dnlm^_YREKKSstBc39ib3!w2C)35w%M(wRjSC z4{xmkN;PN(C|VdD1&XGnb5zj+rCI<Cz;f0T}~r zu@D?jxdOF8_iU&-juvx|03$}SP$ABQl?NnVlM|Q+;9O^(6(9;lT(ZmwUjUSXWBQ4| z1PT%V;4qV)(3o?67}~}Wlo-xa;-P@GVwn_b-pbGh$g+;{;YZFO1h6Lg)sRUu=_qJn z7{B_Rt6OxZ)br%;n09pSMO^nCr7JN04(~Zfj7t6oj#iD#uii0tge*$3?7L2?t(ETL zxdL~%$fL34`=*BumaVsoVcSs1Y%64B&>X&yBIZtqP62%}=PigyNC7x);pu`l@@rcn zbC8J~${nLi$h}0Rj7CCDtz&7Bv;kf3@|Ruy5mOfcjAaQE1##iNJan`p`0TSXsBXCq zq_NUEJT(0Um@Xh?7v)8W@^O+AimWA+W>ZLPF1=Io@7%i}%c8)yh2f}4h_0*4d9kqO zgygG2_Ij5PW&VQGDy9{j_c7-yz4%Hmkh;LGS+_8ie4C7o0+>O-L=TNL14U*_u)-dJ zE_8|P-Xln-gXeR!$;3Cn?;jY&5kShZ&+i=kU^&R&D&!gvA-bpJm$-o(W>QQ`9#a-_ z{Qu$!Dk=jb_(29CfM$T&6&g;kj19>%I=u}?o3(Ri5{E(JF9y${AZg0Pf0jl;z@Ft7 zi!dM3&L`dEbwWogN{R`KWDBVQe*F`ak-=Dwu^BiU0|7(xhzJZ0qi|rwSQa@SD};;V z^1>*O8mp7#ZkH#wP`VaYqQb(olLU(b6o@MZ1t8`*xE@NknRpJv&=!{kWWgxHD5NFu zcb1c_B7fwBF2C!;zkyT+!O#@q`NI4O0sRmQV{Itjwb4cb3try2u7;il0>m;zCAfVNz)v>i9qT{eczNwz$ruBmzR|0?+{eHZjV^nL`0br$Y+u zEtr4+{|)7+15rtF2o{amII|8F^f~tq9waeg$2KNq_ZQ@@n3f=&Q8idO;%NH+hC%Cme01N!nyb5s~?}JXnB2h;k@j<~pes z3V;s$Ty+6n^y4w{^B7V=+K$Jpg6JzFIgo)dgs{5#L8YxW)Onf`xvIoWc}!&0#O#C? z%!*`9fs#ZONQkZ=U35T=Sdj{aL=+C=AWzOLuLYruD6V21D|XviTW6INFq%PRhUm)f z=VJr%;F=(SCu1=OHybQPv1c2IEun0S;^rgBy5eO=e>XT7!;=SNAtr4!0fbN2&3$ZyiFeGP)IvQ}e`+M0wWz?;cKxu(rwj_S^RXHe7 zWndrA@}*A6;u`$1V~r;B=#>b zeT)SZO(_|g<|NUtfk+tyN$ZTPn9oQZq7f)2(+oK?8aQd+aO^A+tKp#&yzES55F|7~ z99RV2Cct#Qm+ycMVJTA#H!rL&aV{mXMBJmq2sDzu&5Na|7Or!=(*nOhKWSqS*bhdM z&N?hFzDlYpbOr}<$h~~~p1ZkFLwc7)-Qb=h_^&5#0_7{N)(wTJt2JN+yC?V{|B4qT353nfnRt83T)lD&uX9^ta)}A9%P;_#Pt0seVA`G_+jY+Vz zK*M?nCPLU2q!`NPi-#zsIdTQ&z+4l;x?;}J2*3P$z|lf{DM2XW=ag5G91(I9&M>Ju zRL(c@UZ>ZJjCuyNUt$ggUETx;fGU#;q;PByFBLTKY|&+IT=;<=nz0OmMk+H&V>r-@ z5g;T&bk7;!kJLew;)2Ieq6+>c+SO4XTp;mkhoeDUUW;EidSn%XxQ+*@1-M=%D)r(8 zIDZet%4#L;g7ScNO^+6lWX_60y^Fdf*6^WnWr_M#A8%Tii}oV5%-OBbt*>bNMslIB z+dG9}N#1exT~s?nIhco|(>vmCcAgXf;pc&9Q zI!PSIs7yAIX>G6yBN*j0I|(NqAOOo``5GpJ70ATt20XqN+D36wha{!RX}OXVQB+I| z{P-CuM_c%P zTaw6m5;EH?7$xT7Ld5K_ok_Bfq#W_@9oq?CN{8VVfab|w+ z5Raw;94A-&CXeiv0u#4l7M;rhxRYf$EgbwrU^tEp60!ji=Rn%*1-lM1=kjI6G(Tt? z=Cw?bzlwHJF?5A6QSC)rxIibFX2M{0j5!7Qy}}G-(}JjjpoF+?UXmO;&(xd~A&jDA zA@}?)TID57#GZ*pc`?WmodKClhTu3z@w0_da!Y%VFfzQr=y{r;K%u5MXy6qj$0&$- ztPy>p)=|f#!#{x-3q^u%D@ZE?EjM=<36oq9lNZqH{0oh6j$h#5@I=e&L9*1$r znyTd~1vSf4r3a-;Oh8)nh`4n&h7=r7w86nfEFCJ*CnaVjRH9Wz#by{(uE&oGe~@Cp zNIOzad2PWYHwkkzO$w7JMrskf8c>X2@sLR5L;?^|C`*cS7|fB?P-sA^fL0%Rh0vCm zv!GJT&cX{&U)mzY+74Tso2kkJfLL8_ic|YJKrr6SUHQTAw86#8Y;XkxK=g(KS=qsI(d(c?tQ(-0|CGCwRFelYW({(tsf{kBk&uIOP!H(ZzAJ2Wo(|kji$OvB3shY9s7n{ zSUBMRqSsT&Sg z>hm-(9{RmmtLAV>ED$(I4P@@N$%}A%X;SjAm zOHN;Q2n60VssQ`;B7P$u!h?vXPoM7kkac^?MGz+z;Lqxot7k#6wNaNkc1L*4oD4^o zP1Li%na%9k5*t9xi02Ah-|QUgqVOu!nZZ6`^`5_AHRDmlakLrVtd5Af5kNU3_lm>M zAzQI+WsTiJ--_ZeeLj0@_!B`&aXSIYw(Pi>Nch@Nhx#6h@PKq=BOI{1YB?POQJFtfgGgfDv_TCd2z4N$dyr=6l8EJGeNnj6xj3EUo@ikGm*b$Em^ai4I!PU zTCoDb<&GRj0(a0;k4T%2TL?CZw<*)wL$F)$R0MP8g4ag_BfJg{&3zt6i8=*=m7vnb z_D;s(=v-ke{6%^8SCEK*h9Pxq^&JW*C;Z^VuBRFLX>@Z~N9>)9a@Io51 z2Amjd_EhlU>F*m)o)p&sv7~)@d3rtH6(31g-bfrE2HPD1k2?c2E>LanJo(t+k=kWu z4vr&$92pk(8wia0a87pk9`be(H2A$N&&E@*`M6Rx7H4x|JxlOg8wdPXSfoPG?N~`9 zH?Qls@-aXojnODcYHj*4*YJo9DziA6h`pZ&kyHlB?A$vd1TqNJFA{x_DUuHw5{QH$ zZ?2<_kg-qaei_r_dyB;&5?bRHz8<&3buAZ|+v#tTlL>@~I14W%aPVa;(gLdjx=>Rw z4coh(bZCBdMcumxX|Ca8m^lZmf+ATWGLQ|} zK}@gU;i;pOQE)%L1Y3jy17U$MqBvWPrg|d}K`a=$djfZp@M)SK**WhKhEBf&r65sJ z5qjjugdnV-YEe;7mAX+hGJUzI=c54uzvEnP9Cak5aKP>b(fDXMZj>ldpTx9Q&%m&% zLlA#=J`ID(U6gzo|L8=B?Rg{W3e_jj!T@OY2WRk=cPwD^Csphst6UK=?RT*0m<3A0 zJ|a)ej~m_uz|)`b>40f4yM-x+!{N<04^puxsESDJIgYsKg0H3aT7)3@1y%;x?y-G@#G`s^k@9Kf*A4#(T_wZ z?=>0Er>Jr~Z;*&gIJ1S)+T?Nmy4V}aJP?3h=i@RD#)^^(04(qxn z3)&p80TF_3rPVvn7ugb6e7H2f^&Q^}Pb|zhlVA@w*TOJj1I&}$cf1wAQyfc&!6pf3 z@470N#;!nD(9AeobEnJJ!zP12^FN0O5E;KwN`nyK$Q>(^NgFz;8!2EaV7I_Noqtjq z!B2w#JuhYy>_vvcNii8gF_vZrox5pde?sCSpobq{RQ@?7(LcWj; zx(=`v6ypSs$3ZX>=Q~u2(gj)t)s#3|1lLPRID!QeLPfwm(6P5;3EP^$p_be1gJKc^ zrbXQDzLwe=LmBC&mu`Yx#A1%_Bg))lFLA==vW7Mv68y#t0aMjj^Znf{4jhq@4C<{W zxg8JM$RTVjwz*vPCxui2{B+V1;}m0&$cwgQs<=mQoE*VN{jt;!Fcm^~jy58-Jhev` z%~1|U<|HIgXl@EF!i_Yzzi)DDW_}T<7T;<^!EXLcctt|e38~h;=gId^pF$Z0O>G*g zMvX?krH%^tS!2LRsV2`R=GBvQ&i| zZB(fR7`e9D2S+yIfK{9P#$*_D?tvMW(Sm@MH7oDt6El-mG&DOrpN9iV@_ghm(hNVw zi1G}Npg>3WBU8zMB7|+yRlA1dXwQMGT7$GCD4909EH9%Z9ZW8~1FWl{LL;^L9z9?W z^P%$FjZX{m@^u2x3Op!{j8yRJUBDPgbHq%u4T9Hk-P{$);aTl|7Jx8akUdFQ_o8k>9OH`7vYOT^2O9u)>`4(#>)VC2a7lCh})!K^3S7#oe zcVLyHC4AES%qRXqLqS|1b*KO3o;K5$)9YKrZcXLCNF6q%f~&|cd~Zc03+qK9f~-70 zqWW9XxXEo&YdQx)d{d+sOtkVjjAqA2{OPGoofu$^aC*hb48!oL`&jrLK*$U5Gl+B z2ckha8TqI(crDoR$GjoQR!uE|j3NZ+SHgS0aI=G&5B|Qlg_K8I?u~r@0xmpUSf=0{ z9hz20cat#+>+d6dr}JbcYRE5$kP-eT_Ixshmh3=3sQJDc`i;rUo#(G+`H=J8#@&T! zqbAlCjQX!atznL3HiN^pS~oExYjbs>#+Ed&pKFP$#`FEACEfrnicO8gRg%F33z$W; zHW12-rypIEGrX1^wxP+*rxbdUHLKclAuaGun&`i8lZ4CM{wn4f{`PD#p^~+N?1UH$ zSdp^Q4{VYV+rb&0{e8(5m;!Ty#KVUMLXc2v`2>q7lefxo-a9Z||GRj$w)v zfK?{=F<4(BQ*vRpVejAe-l-@-%ehy0SM%yjD{b42JL$jUs*%)1m&!3YuB@;bekuWw zcTm~bHtnp~;5ZWfod*1kSF}4YJzAQRx&QYoL2gr;5qI=LPwRmrfXSxa4Za-MpSN*V> zU>j~YsSp?9i_FUzh%mILr{%(85qBzlFCdTb@4)eo5WOZOD%K7V`F&{fSXc^IfZl*p z*XnXG;>1#6-*I8lpXa;v z2D{kfIXXnYJ6cItXTkA;bvl(*@iKFfHUaemYS+{}kJxBqvNoCO6)TuJZ4Rw&IW>4~ z0seF+_W;BXuJ7#ITpI1q_d@^Ru)5`JgKo@=eAfKi;WGGm$pS?+zu(+mG$W$RMOUF* z{dEBo533Jq0;gk;C7h2sHG8z25Fkd%+$e2^V1G01_}<(|=#vUJ=>nc3&nXPRQSkA{ zJ&ar`Ej?k_A|S$6A$*7xU7>X6?zIj+7)~_5BuqPpfj^HN-kpiR4C-cUk1TR`#cu}H ziy-3&x_8?tH^T*i=sXJV33w#Idnv}BGPFU^$jpeWFDnqdY%vj)aFx8I1| z))p|6L{KEKlwfF|ILS{IuwraWvOg=_KNfY$Ej;T3pz`0STK8PK6`>{G46m~TASxWOGgooTO{qlk(+{UE#AXbKZ(GVB1yh=NM+ibuo1 zWe#jDSa@Oy=rs9N3T4sFjS1Tu+!4X{1ra{7iTQv`kK=F0CpC+F8*e4iL`$di;EhM$Dy-bgrUIttw>QSHoUBrrBL?h=| z5yXZ-ilKzP?@c}}=Ny_3oXDBRGSDQY9X|kP5*)wYUsHLZOK?t+kM zTSth2b>?v~K;)<1(O?hHl#+FWD22C4EXPPC+S^z>xKTrU$15z>!6EU}gcDn^jf%KO zfejd;LrNf<7LwAkX}s zQ)FDr46XPcs5?{t`&I)g4J0uXRHcnli}8-}Tk^=7I)A+2k9s9P(j`0S{c#Vc9{479 z)iSlI>b^5$bp3L_+q_lLir;Bq5Au0EJrB@ia`TI#*^p?x4!Jro+1X@4y{=hOfc2Ra zA)(xKzBiLp@fT|DpuGFJA{>~OML1FepB1xo6$}?HSCh?eufo71H>H%`gdgoj$D%6+ z(gdZq z(51NZ6!utcsnjD5xutc5URdxOEFEki)8H|dkDop#Nc$|Dz8(gO;f!{cETV=e!BGdi zXCx%7KW-a6&r>$-&CYidOP!}!q4!+yK(Ep_D=Y~X^G{AUK#(^nT}dSaUC{>H1p?4o zpoht_#L1W$2uE}&SPY|pbTDBOxs`cv_^1jd&;6CX6jbmk((sfotP!-0h+Q>Nj z+v6Glj#(a@X(K$9$EWmS(rdIt>nP!Zh4}tlkq?M5wy(boJW?WZK6$Vzc)vMtg|w;| zMVSLu8u%ZsXgZH<;Yb4JT{ar^G5SP^v0y#<%paq5{VP6xd4|O{UioNVk?^Z#U3lTg zwI*4BV@;Pb4jw3b*|8#Q8#FjvGj)1z z?tjGqRg3XCnD=&c9`WzOUvq7%qk%UB?oM}oV>?AfWv_ckn4R>TTI>5VWx1{)uVR$qvPqDDF9dl0i9l?5>KgushpHWHeA65680$-8na# z+Ppv`p*-P!fak9O*Qd_(ubZsCkV|Wk6^bOf+t-u=I4=5@7^{IvvkSjIfki%oel~ve zc0HQ$>_jg0`*!^=I2!Q_p#pcd4-cn!sg=4u8VNZmOC=I6deu}@nu~3EJyA8y1ma$t zul=}<2n(-~$aw}6?sSQN(JS>HesfLK&_qGRVWr-AgQ_VV zybb%SLH`A@e_`BX@1BnzBL7o5AUTF#%5jE&2Tw$C`1DR=8OYZ7%~pAu@7T4asI&4y zKzSmT&6zt_t?6PpT}`y_YmbDL*Azy82krIR1hC{Brh7z$6GtMMt0r}=B%z&w(Wu%j zH&^M^<6)-SxOMkrMLmINTT<*Q_gBY$67j8q{n#XUiGR|h1eV}~=HRB{Dz#m0^mE+z z`T95|rvP(i9lM?=fW(L3Hp2&e^jmFr`9!AF702wA>ZK ze8%Zml+>&osM;-35IIq5b|P|0o3h%g*LLcX?s9y?BC>KBNqdFRD!hi`V?tLUgK5x_ ztrWL)Ldw6;#rgKk=tJUaO$_DQ6Gq_oSd>DOjp3-K`1e3>LTLRVU_A0x$HQKZ+Q->< z_}Cji5pXQE@@bisL;e1QS}G7fdKltX+o6n%(6dY2`+Tu}9!y~{BJ2|Pn?PH6ph4-| z9o@Ye&97Q(l6!|(X5D~>0i6nptfPUym}#!u7zNyucHxga7A1N8t@0=XZ5uZVfAM)i zgTLyA)Ji1SH3L8x&dq=k!yVSo+*D)+~JmttBN4= zaCp6NniJUe@Joj0{0{aD(RGlOHcvY~1N?bhR(#;&iJ4FYbWjLnv!zg`H)H+52@Tf{ z@^!fuXzEnblZl|>+1tG zeaf{ur;|Pr#kLjWPjyGg7q+yL;R*gjEtu)H$=L?1`HJQHXbmIl%7!Nzr}k7%Yhbw6 z63{CT@yefPp{oOQ_06Ae%oE702l8S96dFB+-vBaa=;dyMAn!v3+zop)XU{sT`tNoK^XD@$2>No}>Olbm#2nql`GvFkIDQ=4D*r_5>WeF3Crh z^`ed!$W%jW_rQ721n>N{TcBE*Yj!+Pkk=;)=##9V1`NmLTJ|qi18faw0nv%_{HVU! zb;yeT`ZDABi*uPArVt~D6P^{nF)B&xGDQhhfRTumvkJB`OzD97caPlKziPI8FJ%?n zj6tqe1BZsxNWVXhj%dFxlmts9!u76~GlkZg@)j<>Z8@5ERJ0JjPcEL$7;Kh{F3ibY z1!Dji1Y1I;JSX{A}D8(l) zuEiQaYZQAdFSy1F9Gpo^qsxNozQ)uWdk#VBF~#d(U9`loD(>KvJ=%V|`dY0z2X1st z_4WF3czAnpwDbFV7wp0J1~__0_gU3cLe=d)+3V@*KSJNsGSw>B@9HAsgc9$MUAU(O zyE6kX4lYkX?*MDR%%&=S)}K9>QLVdGCB8pQh%5$;G?R|B!nJT|kc;lM=s+p$b?8CS zI!ARBH@@{^1bCiz7WdwD1~p{(ko&s{+Lj(^^fXY=(F<-6-8V@C>r%b8S{fvPHyDpV z`LZS7Mj-KVlRRGe4|{Ve#BAL>?CpUk*%;b%R98#MWKj<%%p1E1lbe){MRSH%rmAx* zp3^&0b?Li9(7I`n`W*(l!<5wYzgv-itv*y2JKnu6!R9seBfC4cwAE&I-8xH%bq4z{ zAcGwM1Nb;xFHzPI)9|xuhKlFZX^I5BaxP-w9eA9+v1?dyw-~lARfia(RU#@pe<<@| zcw2N>?tQ)Ts-7u;$L1-YX%dS)Z6K>xz}EXaArIitug|$6A8B`&k^ry!kDa}n@HWf8 z8rf^=6#au5&;X;cJEQ}=EmjeN7_dHSyV=^mQnFg>DVhuyyxSv;&cKb2%^x3+p7nE3 zpZZYnH%w6f3KO|+y*=cO<&=H&5gjM(0B3X&l!;(7vC-*ZZfRE^z6Vbhj0vJI`U1s{ zVSkTae1FY@%IJV1i~Eq46yuHKFK!e9)hMa+5saGJoYT%&xpH@ zp)XrSXyjxhnEJ9loks#has`iUv!WrIHZ+&yzIJmgV$a(-<5OAK1$@tf?b$F|pLM}u z9^qv#x37PKlm>$XT!fs8R=IDG-#x5(1GwSM*g$TVH1)ala8~fpne7($V*F}YJ_T$R z_s$1g7Ki(m=l;6w%7RW<51$$6UTsOZv<38qKYPFO{up!PT=rXH z2^YMpX-MPrlA9fZB>H5Z^gn6*F$bs$dxeWz5`TttqTMZ#c(lWq`dSn-dWN_Ls zkOF62oiLTb(%Ja$`MBgYo`vK1R@3=b1nz0u+x2jwHPc@@G*a7X{J5!^W=uCDSo}A%rl%HzX#_IpCE1^Zt6Sul z(Mad2&-E*9=NBkj=6?`?1agaq+Xg&6ZV(at3uVoa+~EdZ;^rYYMR&M=wnAUsddF>l z<3Jp_49pViO~<7STp} zZ8%JZ5fLfGG##&rkX3HRGmXoUwTtNE=gt%xIeATW9B3X^SGn~%L9ss3T`~Nt3>;KV zU|hkf_Za}wuacRj8eiH3y609ezT$DD=+z(Vn6p8=xT2^xrIRKhu9qxe*m4&K_xjgGp7AJ$DfTO0 z*R48l z52@EZEjisSDB}u~yMME{{QT{8EcSMA&djBeg{Y#n!B?F-kH!j`a-cPSk8;9TX^t3U zh%DM*Qmp|r`~%b5(qK~nq=d^8;gv2vr$9=uNvyB&Er#YnAsIGuJr$(;ZFyw-Y*x#v zb!c8MwL5gONw-wXOl9PE4rE)c{8g7NGJw+||7J?74XKwMR6VrPW~(ZcZMxWfr_(R) z9<)UK4|Edc@$eR8CSTvvV1V)}f{Vtde8PH`T)^u=c1=ODWzM;Aq!$XWT|`F@fWxh) z5`${{-gk`|OPE3xaB2{;kj*D=MDh@QHJUwT)B2d&4-MMp+;$&Oo^`*Q`auS31Nw*u zc!WwqVQ7U@bzr11enV_Lm2CwL%8eS}Hqw_jy>hyeRaLNu*vvgFE11%cuekG7o+atw zGR`urjw;w{r>;`r$w)Ey7q}j8es9jgwt&u~?|trl>F3;-<^8Gq(#P-PCEl9r%tQY6 zxh=C-Yo$kwleE*+;zM3O^Tqz=eQlBe>7A17)6v=K_WBsI-=$jTUst@0Le^@DmxEtF zS<8!MuWYV9uEEA)QbW@Y=EG0E(`{i>&b~9^Q^~r4j11ukCV6?N>FN2wVQ?7-#a0hW zc+-VUhs)iSp9BxZRMpaBISo~Sb>k0FgrZ{jm=g%{E|6}|JXo?oc@>)yoI0pwJRnsz z?pq+0o$cwuY^rbQOw4Vj;-Q&RLra_EV+u2)Qn^yYs}5=&fd9P(9;`QsalEA_&b-{8 zWSP`fD|hDM8uesry=QppUYQfW#$464m!c4N{1ZpeIhE8c<&~jslHC+hOYK>UJ$M8Wgy2c}ezU@-*g#i8CWuyY`c_ zJ1dTg-rtv45;ddl+9C08{y+@=wl)9`*I8u>o1P)`>5>5)XbtUI`>HYw#KX};i6&^0*YUF+;2 z*zxmg(w=Hm&*;$w$j+d_hk^G@#_p#5ryE&|0SW{~P*$pSghk?PAWp7DXny|~KsnCL zBG%%7Me)cZq5@s7hn&Mj36j+a9{xy^@*x4jhuvo3CEd1SP8#WT=f3~Ix>+{G z+e4sOrZWSIkUmq~MmKW3QOKlZ_gAVZ);u+Te6Ae9AB~!>u$9UBjfsMbbx{n zu>h-ptjo=bo?1K1D!00`6)~<;g{R3^`Rp?hiL(l z;ML&Jti7k4|54f{e@(s46g82MnuHgQShJv>t9REGk>G_l-vRM352A3uhcN;QU|0l$ z+5MIg6k%Iv!?KS?%?FqZMTN2wz1PcE|E&EZCU3K^c~jW*{e}59dXnU;#h>r>nCjb> zxbIG^c;nh0y2rOr!z=ae?yTJvM6Jr^c<$q)@AKx zpk)F77Clp8QKGMBlm$dA$!&pwVbv;2pkjR6nS4m$2h_Q9$0T)7Rd?GxDP^}cd)w{0 zwlaoT?{fDcM|`_HFQt|R-Y;Bno5yv*?4T0y`#{j@?sGF!_K-RJ09Lbd0^`H$>=!lq zjEtQ|f;w-)C~{dM12qCV{h*#W+qaCSuPbc6=z4ed{PCxSb||jP@^nv^X4u~z*XBQ# z+YS}JOpruFOb4kk(EPtwl$0U+{Iu?nDAv*f;c-sG>hkH+;&(G2lr%{j{tO=>H)96E z590du_*RaX00=i%u7n=V^pBuN07K04JJP~i0a|5$M(7EiRkIBIP#9>U9KyfYxRBB= zioPl<=Pr?d&VldcA_Tn|*m(OnbWO6-fG78y0qt6xA{oLR zrUcUHC#Hl^PX2)^)-2t>a+);6bfwido1ou}o$1rS?UuxJ!#S{mz(sfaNbOl5TK_c1 z%zXt-;B$2SzKnDR_?WdapSZ>F(HM+3Xd^Jcqzj;nL|`eXMoO2pGRwUYbIh84kSAg0BMGCo4GEjRZJ;Ed6d9f77-@+T z)~sYQH)UfH1}6&pc7Vn_uN{L zJiO(!e7Ckc)6>+IxfR}6borW%nLGLbMyX0GxjSlAb}MUw)b4?KY|sW-q^=Abtjt!> zOb>27TD}E6j<)KBWg7@eXzP7+JO72}DTpVS1D?UPpZgHV8jO#)7PA#OoEd-1hCU!j zs#HLvb5hQ!=Dxr**P(9ZtBnMe+(L6PXOl@3IZ5LckT3l)AV2@B6Ks|&+=5a?m3m-D zwUCJlo8*}WmmC7X%A6ZnWqxpy;qfHf?Mt=Gg~6I)U@HU{qQwGFAfJ8s%7ikmX2`Vd z851B7fZ%WUXO8~$0qgto&(~cCX`raJiupGTMHR(u(zZ`xmV(oRl%3spNa?Og zrvNq?=9iR>VB}042`6!k(Msr+{?L+&>%c6ry>k###H%$>b>j(*p6tvx0cr>ge|@3L zz$U?dUvk}5JI7d?^Lg2n^&@3v!)vS9Xi%-f66@MCRd7xRHPinrZp@UiW_6ctkYbh9 zJ#<1oMV$w;ff^eK;am281zk7IQ_^UB9fX3sgMBt->ToweNa>i87)H6wX=wVpYM!ub ze=>;n_*pjrYF66cWP=X6;RR)O+?E-9JM1OZp0UQHwByhCyKBJ0P_FGliOpNwm~t&B@Nsc>t^flRG)6n^T(R~dx`z&!1-p{joMn; z3ktG~ey*;5_O4ELU$up&>CTipy|uG0e|THr@rAoRlUl2;rhbbatjO+9_4n(iV_-+S zpYN}Q`^wCJ=T}Bh0$B-{(d>77ynHnm^jvGE+*h7rKGbu@FE2@O=>fC?WK$~`3eC~J1 z)w5evIe(yxn|QQD=02Cq0Ws$M4Ks(4UxXe_eO8bBd2C!C-t>_acz4Pk^yBX)DBW1) zNwH~y=O~WpNCq~eL;aTR2spVLC8hc2sd;PUr>tD76=^V<(atviIV z+t-nN|CRuD@!8iC3m(gMJi4cEw+`&*tqO4xH4PW-%t**vv$vsN2gQd3TRHo8r0&jm z2Hhb}zht{TG7<2L3|5Q9;N<5S_{JnSM|4r>5ZMK!N&L+wjRL$9xL32uoK+!_(i8i} zElO@Mw#=Zwf)>E%xG_hveM$X)w{tXWCl?8R&&FGyB{?0jX`ih==O~Ew%{%H2lA&1* zC+$@X+vYZMk_=pjQLOWL`fYpden(Mw9OUx*3qn;o+qkbQn{X+<&26+WZW*!E))bm1EtyPlAOhnchNV$S6&y8GKM;IeZa{Ty8+U$({bFI|<$nH?8& z7aBX=>GDYDJzXqBHA=leyNZ<5qgd30{6Z^>xUL~V5Egpuj4%QWpt}0!; z{_C{jvHIFPp6@<)3QE9JMipxeUSv~<>ItBmi;G9%91!MwKj|BYL+ty7uXk z$DyodxtoR3Da=N+t2yjWmwaEbo||uwOt(dL_eI~UuJ?~L4&3(!-5XdhTek=#*j#Qs zrY%D0A9%9g8!JQ658uL(Jf2;Wl@uiFKhe@A1jB+VRtz3l()tQ;s=@6z+An?mpykWV zb|~$jMja9m^9m6-XiA>H>M+%g25Hhh`Hxs!8e1}-vgTjkz68)(2<3^ z{Et#JQU)n;W={$g>x5>#SUaa%%4D za$lTL@S{hbgolfWxB{JSLiZ$UCdIl`sw%_Cd2Xp(u(P{kOgeu~?NRLzd89&=G4y@6{O<>1SlQbS0OX>Ubn(muoL%=aGp zbVmdFgEzf=9F8P>+(YBxMUKnR!rO0`LK$r%NR@kY-8NklWxgd_z+1{ve){rr%zRq3 zw>)iUz=F!4D^=9oGB{inls2Tip~Viz`50BH{gg?`gg|>BX00fDUUq%?#|N3Yy)lp& zs1lM*;x=N$j9mnl*e8#37G;KzC7QcB%`bZvsyySEK^3EtQ;sLIE9Vm1j(bx=-kSDP1*O znZ2yL+m*$K-7h3a`C z?Z4y|Em;#O2n;Tn@E=%1P(3b4-0lzYbp1wS88N=Qozu4W^MWrWMrgD!w2<#=rNT`i zO+Et9e6!Lah5pwCf*BL<8|}Z1n^#H~%$X`HR|;E&T7j(jbBd(oL#QSW6u^UW{g{BN zbDUn!v{4)7VWihyFC65*@^Hs8qEJWJ5eOh42$L^sLpubdn_{6@IPRv~Q*6@91-D4% zVmntM=HXZ-w7Yty#%pLK&#PkN1oAvkDT9mt7NK5=q`~=Z!esw*RH+n0O&O4UI*dk$ zA}c+7)MZ7FBt{;i;x|QdhX+8Q4i$+>sFUH0;Br&)qmU2pPtyAR1AZ-|Qz$~lUKsrK zamNx(fiDQn@V89^E~t;>OC{X=A&i>pD1-Ag2ePuPMl?&xdblc+hUkZSJ?GrhISR9- zD**s;H7c^@hpsjTF{PzXYd#9SG0l;f4mF)GmTp*pEvAOBt?nYD(p)BJ`4cJC(_lhl z;TKjj#G)5sBDfp2S9hnQB6vMQ{R7USjE*YH(s_t(xfv^^)I~TreTsWL-hQ=Cabf>< z*Uwp0JYg4HEw!3xD^;Ez=|zVW*nF@xjE|qAtX=@CN_d;XU@g)DEE!IjQM^mcdx|p8 zazGsB&5G6c)#DX@NeO;NS@@KSry_!A<86cL#Y{1joz_UlV2#J63tyGLIa(2%o;Lh!)tNXvePg?) zaPh-}nWilui)eU0460RMC9gfq3(DtflV7b%GMx97y$ULUiRmG;-zX8FWWUCN5E< zy-twa>5W4ryBWL4z71C~T{C%5zT$O}QSTLl8!@HwK^gjx3 z&;5t!#%dhF|E=%@{vU;>HNdYZdK{=6lHR9g6~7JT0*_}@{vUsL{y+ZiLFGUG&MLH7 z#D@Cs~WhT`e$hyhFh#}A;O~~eoSCpsOj4-Q_OOCLN zQ`6=1BO318&`H=T;e1lfqdZrgR5`_T`hT(A)lb?5XYP*K5c%lXCMGGA+*+a3s=qqN z4^n05vN@!7_$UwvyRpg-yNbZZ4euyDf}LINjQDla&{_i4%92O43a2#GE@|u1^w6)EpH$EV1jI(^AdRI<%z3;JEm8ai(r|_Scn0h5!lUR#7jjc#=^a|6 zWiVi~PFe@=3`d%G5*(ew&Qd@A$JLqs$JKHF*VSdO*(qd~UQ6v_fO|hM4dNv&h~lXA zgC59KoT!8+JL;xv$S*5}Xnsx`E#=No;5)3&qa+I;2qQ^DP`t9xoG+k3nU&(kD_=jC z6x6d~=azFde3c%CHtG_<&9bLv1w?<#m{u>dO5KHIX^>|dZnbUFCajg&Yi5bSz)GP; zHk;4kJx_;;BPr74|F9B}mNm(S+AN->T3UKG>7(b?#^($v6O>|3_lkyGOl)}P?-y2{ z(_?De4eR$4gaqHn$sZH7V;V%i2)e43?GymBNF?w05c|6+6jb1ng;Ar>6V~*iJ*lf* zLM~xdc}y7^^iYO+<(c3iI{2SE6<5hJW{3j_5SFt|>s<^ZV8ji&@> zOcPyb`}b5w2eRSC&1QE|#^4atp`=3m`6Yvwllwoo9m}x4W*1o2(!k4E&n(SiL&g%@ zAQC!vl$X007px-f?}W(o`9L|4MtYQw91Y4~6Y9E;Q&t1qX6RC+HdJ19WD0aznakl; z1!;!vwZQFJO?UjzHUY@p&>+=;D*wfDF9NJ?43kNVQX-jf*BDNnGvSNKY{~(!Jj1i) z0t_`++c<)!_tmbHR>naul;xO}q{-ZwaIlvM28&AMo;>TMLVA5HJZPPLN=M^4bG|(+ zQJtV2BN;E)zqT#q)83sO$Q-f|cbG>5#(5NZ!MqzR+CZiz;VGoN*6uTILl97oBE87l zw9@6TOLr|Q$9%2}BL)aT1sH!2&Rg=fs2e?%X}6k2K0L?d4?Xs5KPsh<=S~O^dxk*m zJpauWGha>s_gRe1Hs@z8H^<_tvRcV_WDt4Gc-LRqWko+!qe!1iGY{a@zNLPAM>enH{n9a6X%PfAS7l}|T+ zK2;jIBy+M2)3To5n^@i0`pkc_?F0!D%I>T=W}IPdj~P=qc2~3SxQ(KiGIV!E7k-Fm zfXCvt8A=lG5%vC#*wi}+UbGeJ?&BTnsxA5G+z-kG8?&^m2|CS1E#T~IM79? zYP+5fn|{0sF2T*kN~Bm*L4a@qM5KH7lEs6&zD#UN5Zd0E`MnGeZ6?OWf=siD%E!(64?T1LkpdXu~u zM{JuJjM^n`QS7(^H`FfxfN!hH^0vxa2{LSQ*jrijZm`UU9Az7HDQ^48Pa{vWXlx}U zaJ5&R;jR|(Uq2_LcfI+~&qWDZC-%Qv|MPQ`k31*!|CgUT`M2tQ=l9JO44nf)r-TPg Jflz>g{2%o}nxOyy literal 42200 zcmV)HK)t^oiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMX7ToXzAIF4OoZ>VP*!GcIaLa4DIO+}>$h+?_8$!?OB&2HS? z08tc0?AUwndiLJ+?ES=k-r0Na?d<$N+fpDwK+k)>|DXHb$2aWGJoC)VGf$rx13}SR z5~0L|0U;rpAU#8L5J^i+kREgTPLEV7mHK*nv;QxZN^}0d)K})`B$NAk$$jK+Fqr?8NxUUe5sXl3 z5;f55t7x8DKon0cKusbLjZgr>VW5X_NQ+QHsn|{(ff_SXG(u8P(XDX#eo}&%OSWAk2tR-;)94z5czteZ1v4*1wO; zTlRDP|A+_GLs~>30)RxcC`FSdh547nvLYlb0ZU+lEodMplqQ&>kQ#;U5H$caq-FRC zOVa=tj2IS8V5r)p050L0NP>&nM6t{T!Y~t{42T-lm;i(Wh$JBsAT)r* zFxrS=v51;PXi6e-1^iE-H&QfEAq?j+vH&=C1OpTYv<_j{E@gn!2vdnfG!{;iXl6kY zl<~z!fr*EEr&bNJx(`%$Kj-mMg~Lq}CAxg#c6oV8nnCCOv}F0yFnC z8eo>Ghs1Yy%d^AzY16 z456BZDlaT0Kr90-0Zwp3BLYB=;!1NB1#l6|IBfMO{tZB(6uUqT2#FFfP$6_Og5b>f zQi$5f4jYLWAe00o(#MEUG{rFk2ql$V9R={we+>j>!qv6_m^Wqsaw!Kv4dK8*B1tS& zA|$gMff{0FkFfU8Y#=t?o6?|3_JGL-75EOGhdYnpg2myDiPoc zT0t`_Y&_-`Rggm;Ml_HSqd{n7EPx3{Q)7my{!Qlu>du6=vec z2sMPU@F5sRIdKCpmcIgsqEIc4z)UBYnG@O}RJ{>3OQQl*jAzJ@!8}e1PNQTXvtMnX zS^6dbvmPRADzR#W878hJGx}oY9f!R|N&twaA+?Tc34zo)8}7<2A|pi;dZmGYm5{Y~ zaY;l>kb>3oAdf`Eq~d<7c#7tjF2&Aah*=ae&5%YMKx(de6`~I>!<08M5`Y;PGnE?%n6kj+xowdhJ+}ZG^%MMi2#ydP~mtLV{jQDiegP67L;va zC0kg@!BpBVzCM`Pwy>Bj%uPRrQZ$1RzdUFiM6>Kzg#Z-SAtcJsuZAS_HmgMs;iv|o zXdV;3oV_LEf#69fN#HEg=H?C6Ba9}%@F>ctW|*YLhygvSUG@>_P5RGtZQfM%6izZ+YYF-2&m|X--IWl9E zcUZ+SB@IF9A!c=$75;ww?DrQNU=>Htj8G<-#Xu7V0wc60kcgPrwsNg?26m41tQ2Y~ z6A@-{+iK~Irp92Bd*zt*VNP4MqA-vIktnx>nL#BCMz-?=4v-{-G?{rNgNao&B`2Nq zkijyCf^ot$$aIEtcd(5ZSmTCOC}@aA80w`M`-ITsQp6~}+m_l&gJ2pFBuiB+8JS!h zNR?dd5x6l_OCe+uszxYJ1cwa-iqlk6z9nXD3-rjTrpz0DEwGgUq*fC~oaP9Hm5%_T zK>OIJNY8|zh>n70&5=K=PpC{5WkBEoJg?>klLxKjK`SkwbFYe=XymUQd#!K_!9t-1 z0_Mh%=`v8EIL!2;fZc?1BIvM_>`?Qq*zK|H#6cvp9!Wilvr2)6Ak_@*U?u=*n2ypq zloGfj-(hocrIv)$NKVb$W_6nt0#;Gu%~jr7w6BC%u@j{LN&%Ym6ZE(K67!Oms7VAy zn6-;hP1v@zBw;jw@KA0ICNly+46)pt8BxxDqktQu{-PvD%Z&Wwq$WfGoWR8-LC}Jt z+>~E+C`KqX6boGeTx3!?KbFZlgx0a6B#2(Ka2EJB10Q892{Z}C5G?`&JtrtQsFwG( ziiBCYOonK+4xxDd#ZR4}%khF5& z)f@G|npzDcLl~HeQ~PppcHU688bmK2zTYfZ9^t}6wjBvC9|mcWS$$#*ON?FDjP6V5 zt!yM-XBWIke0hb9@|CPZaF7h4wBSO?EoV5s2qPH6Df+DT%>2dZ&P-K{wk6{`?hrb| zRau_Q$S#4gb2=Fhnf=27SQMubG6}-$x)>muVO=zpqdh@40wFP>wgxL}`a-aYYfONx z0CFmqddjN2gx6whuOt7{k-tAZ%TWS?wy; zw}-$=2BZSODhZMTF>@zT6Z%~CG{ck)2F%2nj-0&D*&=I`QM^HCV`Q5(ft-eR3TIsR z^{_I!c>yNat3qqE$$;eOAFIjnItepl7BfgteJ%av%}m|`V&EtRIQH%20<_D!hVpEf zFOyfx)`p0AS_Q4kT6MANt1V(y9FglMj8nGeNfV2T(vLBm`b+w8WWZ2C|0V^;L+HCB zM1+3lfr*$YV;>z6qi$Fz5QPDZsEk@Ig9<;=7GD=lA()0=J)ul7h0TN}A*70+5Do&s z2rXqX)UC8U_OlWr*hcL7Wy~#VjJTR}m!Y(YGnRR|1uUwajldO@P77TKZ)L-h7!;g#LG>IZ+{Z3#*I+UVWg@eIOWz=X8YquT$RmoRo6;)|2&=^Z~JYXw8 zoUy?$9-1I6O)@jc5Ex@kV1!o5R_8#w*z#JI?{i|FQ@Q;&#YM^msel`lOffaIY6GxR`sgD9?~-6mTfD4-hk`Ap04|IW z6LasxoGs7UMG&pCKx05bDk2FH5a7XUPz<3=6piT3Oh{1g+<0@rLCZ=5MC%m5#gl2@ z6GlnCU1lgqM9U1D?G%K?1`8$&Lovyek}IYFh*7L&W*Nf7G7FIKpa@Q%VIUY;Il?22 zLV5|Lhx!sYluWVN0KDEK2nri7f!T&mG^!9K3h7y!(qcfy%7MzLPUKvhMvCK7lnyl* z7;3Uwg(zkc=*N}$p`_$Wu4aCLbqVSSC`7AAaGH+-b_wbz4&!VuW_)b{T9YH|uO|r{ z^d?lipCq@rX&@wF6AQ%;dG1pNT0SGQ68rg?KF%uUQ7VG8X%hl%xkoN;4 zW`@lW9w&qC0OAWPfQwuTq*CC{{EsDH$Vd}fw9`arro~Z)}bdKq0&6*%1tz%>lAH#Grhd3Ekfpv&;Lcjp&l{U!` zmb?!X*AUibfG6us_wr(j9s zVSZW#<%4sNu>l6AEW@sJDu`7H%t*5n%lR<*lrg@sj(kvtqrPMvs@9q35l0xor9-Ty znx#1u=i-8Pya|>_u4QVuNwIRCT0T)Owpj2T<0C^1^H3qI=RcW5X&S+qiHC5Tpbs}N zj1HSSq{Sgb5;US_tTpo>Fvze3&o|HZN{Bae)LlTD&ao_Wz_)f9X;7`Tc4}=LvV)p) zo~^(eP_7F(DA0u!!Zbv)PJac2VSKX;k>^@E0&@|-va3aDBr%SZ6`r0NC=n5}pTv*> zmGHhT36-Rl*oM7DqzY0i`1BS;jcOR>R&bui1P6s!916k;wDFB_zZ8IBG9rL7XeyQ5z75wJG;?zzm!?J{L?F=NvFN79(MchHF@9NFg-mnqomQ zMn41013(-c)Y^~ENlsQlHDVY?pu7<&;JP^s$A**G0IVyBDL`7Bg`Pb*7-9nf zi*W)*JY9tto*2SmF@)2o*c|AygzNkDXAp`bFofjYF5eR;MrzP(`?R?QPm$#}r2yS~ zG6+NxFd{^N3xxo~6~$U33L~CO@5K~?aY<;NBIfT{_D_OjJN#d#c;@-!&i@rim-38^ ziVKUC&?)pcw(+(6e=lFJ9QnV#azCG+`M*Eni6cM~LULhkh|ZnZ4s?(?x5tCAvlNWb zVpfEj#(-FCj$yUz*AGZy5DbARL=q{y1^xR2EkbjKoEQTxkij6aT~7=Irn1@Z=8_8u z3F|2U641ZD%m4Or`ak#Nwf>bl1T!EcC7})9v=!hh>)%VBYyJCq`TflQ{~=GmexB|i z3DqlDjYorG2yHSTE%nT>sC9?}xO?{RFJelG!cq*Zos~nEQnUNTM0`0h?^}1gip7?4 zY^JA#+i=FP3?PZ;HlHvhql8SNR7Ul})xcNI{zUb$MvVqd0WM+-Fa(F$-yEhPoYDrF z+u)Ka`WPXM(I#O=muLGR5p_X0=xlkKX@ltx1yqO{V$4o1=w@-GvOU+J2*#&xuqvC7 zn1fQljkkIEr~;!HQU*lbR3Zx30Fq;m91z~hrMLm))k>PPCWZ79hyoCcJdbTY23b%+ zEn|af%;PG2G>@%N#$kbc4L(L$AT?w=qG*}_`L)8+JSH3z8+l^E)#h!15j2n}PHTWm zBTC$ea>)UWYqua2=jeA1O3az!l<)bpHH#vxX4X8yZYZwCjPO^;LDJMb{I)RJ*#LQ= zZ$+UD2jKUtG1Ms}Ntqb|r&Z^xXKZ;Fhx1}hHr#K1XCg>tLvDi6JmVq(^87hG%XPAd>~q_v)V~*4%AR0F@OwcTaqvu*dY^hAru?uk=Uv@U{eA3 z*W7%vKe^m3tDkBD;~f|5OD!8i60?DG_J@H%iT%ax31ok=cslL?8^X*N!hEpP3fF=m zKobB%XfqyaLD=dL2*wb~oZ*r`Y=jJIyQAkSq^nxxlG^Vml0qXm#pc`D?J%M?k|=Fr zlfF@#94^*s<_nrzvBkznA&lf_WZCx-W%=a)-?Sb4EBxQrTb5)0x0h7*^ZbV&^ZYpe z&$Q%NPqnA!^1S({q<}DtV*<=CB3a0E# zG(;P@*FsA+PWK-rY-mE@?*z$Is@kP0uY z;#UqkDRd$i?wr6tAUMaid^R#^-mjETL22HNWJg@G4dqlA_z*EfA^^fD!fdBSn~y^I zAJivk0))hL4mIyovDw4Ih2`@r$yosTL0iDkde*p^EqT1pZMfmnW@Su8^5Q`ce`IYlEl&84-eF$mS? zz|GFMaA;bvv%<+5B})_*CUAts5H{hE9#sPaNhG1nvO{rg;ZXhB2=bf&xB>#lO#D`N z3hm41Gnw=FTrD~(w_k|cgoVQZCICrb7{@%=voF4RxO1tt4hJ_M%Yy9=k~0Sq#jX3n z@<0@4$efKwbG-&k#r&h}iKHNE#|kjNJUKm?+fa#e@$;NXCRPUve7o}PVEF9wToy+* z$Zwf5o*k+QoF)m3vfbN~4~!v$!HzaB?DeDy$ZK*;y9H|@Ws+k%wh51Ruu}2;&nYZO z0%w4H14=o^REyA7yVdNd9PAEpZns@NIiqFon|+DK`wgQ*gDDB<%&l-8WH2B&#jzK* z6P|pxa<$3rskKYwmJR>he*b(Al(iCKn?XXElda7Cvd!G*Hxd?TE%2^l2i(tkS@Jkp z?{UqQm@l@sk2M!M4z>K;STd~%jeQ+OE?vuP*8cz0n*P7{6x9?}2nociB+^5p-5*Wnuv1N|ER_wkeE%>VKE>HqyP51)8}^fAi- z>!T*EPnygr5G-t8C}N-6=fzX4vJ?IvXY$;zG1w6akRHo18~o9oAQhlEjNmlz;u)gd z((Gw~`7CRd+LXTv!s!wDEN;EWKc7{E-KhC)!D<6xKI^c1r}GcrYO8U6M~UAJmaDk< ze@uqu$^S|CyPO04)%&inozjoQt8$PWUXc(WQN zVT^2#{I{=4<|t!UCH<{oS~X$2IKL+u6^4-PcbX$}hyCS0=k98*&I${Q{hiCP^bM{k zmQXCmR`YMfmep^anQk>g!qFm33BE(@D~WK{2OwzuN_1z3bXeeXGw77@IZU$hmjk z(u|#L^;S#5Va1=e2mj=y|2vj?1#9Cg`oC9h{ol*W=coPuL!Pfm1L2~#5NjF;i3!c+ z4tBB0G4)K6x$t&`T?jv2#f3LSY|oV`pwAd^XIVL}mcx1M%~eXYxuyj;zJs7(vse(X zG>?h-bRza>`@E#Hs1IXMs>>whTcNY64S$xR7Y}4b5s{(bmOmU5ABn%Y6In{nteM^Z9O1KKMVM zEL?ao@U{4_uUwWh{^RTY^Zc(L^4P|Ia~up}KNB>km=KZxf986AyRiA;IgYv4zl0eo zRErZN@}1iF+WPmA<<$RtWYVAhpC9tDr#R3E$=#B`O*^-hOl3qd_O>(wq)vpi2z&8% zoDO9#e`S>zKI?~(URlBp0HGUxiT8{`E?TBv!0ydB~1fP``$MUWp+pNmLEb}|uM z1QD+zh(s3!P?NBj6}A{=w#N8N1asnm3tPp6y%Y9*)@+0wrRWZh%XH-SQ37DVj3k6P z8U-$bjpAA(29XX5ab}H69dA)m12g#eAh`d$zA{xLSRNcZneWw0mAt^>;+4d zyX1&{!CfkD;7%v!?=EI<-mtp=lD#CB+sX{si!!6xdsHO0AtCF_HLTj^tzI!3)XxVF zY&o_`n_JGTFU__hfIRTAqM_W^YFi=5D{ErQHkPlU82Qy3OY@DHqw7np@QXlkak$k5 z?E-epWNQ&JH+m8L-GY`Ihs>LrSz&7DDu6tY)uXu5`hIu;tsonU;`giDz!A8CM7T;L z4%H(R4e1T6!^1Q!o6Vg7aOaM&2WSJ~#6i~fU?cn}%tt@iV1xm|gQCOZ z<*{lVqBrL%ap-Y3ABlu$K6mG$!Ya*xS@BkXS{S0$%iIeS@OL)SC}Ndw!*UO+T!dW2 z#61XZ)LR#0nsCf=IL>Hh(}e?{GdgF%xay%4%WAM+%X(c}Rpb_>0*GY*xHA8? z1TaboR|wh^{FhrG5z2(Cg-e6;+j9~zKqy(zS`?D=qE#hL@Ryh7dz$z=^KGtgaxgTx zD3y7vMWlVY&9e{51es`+1ZG?QD{~>U`M^hA^NqNpWd?vbmz2GBRk#8tcd5Mk#XRio zE`_Y0GaUTQT@u7JOz0sr7YF!iUxey8^B?nt7sxlA)uH zAtXv8@K>oBB1y<(=PODh`oc^#f1@fhvU(y3F<r9fj3~|LWqoZO{M!WVC{?AP zy)wWmo9kT*Tt5fpD`~<&V1(Ah-icQjXb7H!k_3BYjnbSJUnoeNH=6~-9h_5ODE72Y z;SmEDf`kzgfmy0Bxi~6ifJg|#5G;2A21y~AQ>n%viYnwJ68!yQg=5b4Y&Od&f!_sO zp}Un65Zg`u+?H=el14~9ibFJ_)RK@Iai9u*6_nNTQ`+t`DF_e)iV;c;#rbj^*0!&m z3Dqqp0u`L&TT++*JEq2+Cj~1l=VlaS7#-!%<+~?&sdD_S1WL8RXkXj^YUh9u08oG? zzq<8+o9R<~z|GuxNP58iWFGSVO#_=YY?5cYyutB5GqzBxrX`mT_W=#Y6 zSKY&+Fm3+#vth|9wbyci7Z$TvS+S)|hETfD+8L`j zBN)P}&EEux73LfW*8Q9W6pU@*FgfUki?RihL_H9w!*5zDd~^u=y4^)>+~9suY-# z9No%a(9Y#~$9{ozhFGVHecUfwE|CVUbo#BVqo&T z(*E1w6>`k;xRw6fK^2k)^K;+*cYsNK`!os{-+!|zl-k3+shcOK0`;Jp<#^S!pLhQp zdJw<2#9o_)ur5?at&-n7QfOfzoGQ=HQo-O@?RxnxUCQff|H5`K$F2R}0>*AvzYZby z9sC7WRBbNAFSwfP;DURHfWEgo)$Wb_vIPvH(1~Wnyh*9u4N1~j?axI0gvojm` z|87-Z%u(i|q0&o7_*P7$6bKK;RUr!Vo@pmrwW%>1a3M zlNlokqg92&RxMg5K<3>j4?nh3Me*j!-gcrq>L~Sryrxmk;x=^@qGX7i9;h=oKjAC;>w{TbC*E$qpBuNCP z0cFC~ywqnV*FvV|U_h=ev4tf<=lK+ObHo1!bDNn-;{W{4Da|=pAoy4Fn61wPbTC5p zM~Q=Lk4ad6H}lA^bkMPh#9!SR%MwBiggZ=RwdV=IjTi>0Fhl`p(um~v=)gtr73Yez zuASOlCJyM?15TCofP28HUfrc)Z{|-gpYBqzPtT@3BtN#0Ctt{mc`NTO6?^r}sp+dR zJCUb6ELZyz&&{K?^!WNj0*m!taJ%greAeIWi2p9mX-clt5TqJmqdiKBHWk9RM9G!O z5LUQ&5iRijZ!|EwOT_^_n>7`?Nm*h{m35cOyn0&x?k@A{$(CX0Pu8<(_c*$z8}o~M z$NcdI)-Q2o&>)W$ws$2JFVtZh$px85M0dNIzIs}l2 z7D+L1n`#m68bd}{ikU$HT!<04#k2_la=APJ#Bw*W0w0NQQ$Xt=nq`ux21O_Uu>;wc zz!3mZ0Kx&G(NG9&*}Y2>pmSV^T};AKAiaU%MY4bZU$M+fEDMN}`6}cA3LihQ)L$W$ z+N~Bz&wWrMU00SLD zvNK@5=axujG%27rQZ##=Dx$_95`h7Vv(wFn1wbNV5|mK}6!W*lwx4VOFuV|(acqtW zvtQToR-6bndTxEzSwSJhaDnr00THXy#}{xnP|oajX3k*5Kq6|0MsgC^zaga^IL_^o zUT)wF`XLT;^Z)no#jI&8)Qm2!R-Oua|~ZD zUD^KVePw@?t&lKq7n6UdPz19eVAX#Yr!r`jIu-Qef_)T5e{*%QCPB$o8cFAE%o* z+E$Sd`DY(mbgtvL^Iuu8rN5I=mz2ZP!8en~FeT;oOnen>?#XlSOMpb~aK}r4#5<2~ zK6!leb@)vtb^J-@_>-5NT{jR3=@B~5M3jG%m6CRp!F-UWJq@ogyBzYOF2R>+=Mx)^ z&G{kE6P$6o_NOAYN zTb)lG65ZKMTtEj=K#yV=N-;Jat;=oC3Q&2m0T@C?8LlG%tc4G$X(NPjmsn5$A`$M= z8qNSz6R=~5*7%VK8-)T$ih)EZ20Tb6j2O&Vlscmp0T2!Y9YK<8pa{b4+0YzRoS+%L zXaYI}GiZz$gNDUN4G@wfNT5e33eqA}bI!Uaa31U2t3v=w;FyWUly_S2#;g`iLU3Nc zEci_6Xxcz2JUwlri7=sNud-Dm23ok&mQQ5jPw^A03BAEcBVsd?5Swj5Pggwx;V7;Z z6B@Cj^;Eb75CLt$FM@Ehp3^zNdIFefP$8f%LK4gxQ6Ye#_1xNs0F%~1oWDR#kX%fV zJ<5&eTK0#Q;|brix)TT^P(!#tcS5)bq5uR87=+@2_QQ07eajvJCuSX@4xtKlj1U|tppf1Ls{nZc{0M~&t8=+6`S#S=-0WwsPJzQF4kj!wHBAf>5qqmvYCAXM5J;^91g%5J+zw=i@vK7n)5Sn#GC&kXV)kfT41yD) zgJNT2I>y@liW7O)j*QB?K7S6e%I?RIn6RL@u+U%!E$3M`GAhs7TwRi|OVdKjF;N}E z@@(5-eW7)S5cm`6?0yT6Y!?<29_OItylO^9tg_VcbC8D+nIUT5YO zTxH>s-L0QIbrrPlci4(vc!j*c%6M+$kD>_@(jsiAv*0O^Mpblv^t69_|4;7yUnmYE zDc`>7%j)^c{V(29dCvP^y#2i8KlgwAh$qLzPFA~+b5RDjCn-k-!0q5NzmiG3C0u3? zFYy$>#Y^fX6-&LuUeY+3pTf&a;VqZ=`gwZ?$bIDg&7?Agl#f{3+ge1KPe$h zv+^nH4M==WHU&r{j$76oid(rIxn@u;LeUB!ll%L~Ah}wN$o&wNzs4U@!*V|#janv! z<*>iXTdkJM{Cw0>4FV$p-rh1;6#xbJdBMJrm&(_h>!yy-BMP&dLW|Nmqe`MC^q%Y{ z1(@)(w~vjrDUt~iw(JJuOYwi&lobOtMi^l+&SvHc6}W^c{x8;OWq-4|ZPtl^Vu0u& zl)H)o=X@cpdE6vsw%i4#7E`ht?`k{eGqR% z1xcmeGQ?jB`T0s=Umsr|xu4w2=O+pNOC%__PlEFNlAu>UNl@nFFY^!hFG&!VN&RF} zwVzre3($BYUcSDFpP$rQ;{(f(0DoAmQTxhN2qF)V1^9S*$-R8lav!yqmm2bey)<4b z&Ht}S&|EE1Ya$4LFS%E1E%Soun@y!eRG&g<`OSy_CEqN9{hhTcA$1PE+_Ld3nTkKvi zweK2LzY`sXJwZD6p6g-idG+0?_K`6kH*a3Dr`EVo?_wRD$|2oOe|ftK?UJN6o~-}ib~x7^dRgC5Pf+;z$B+LyPjx|8*JVhfL| zfs{UDg6p)3#ik&4=I+0W6O|5J4($d{`E_;*KB(fH?%>R<1bpDpnlX+4o-}j8xxdqP z4RJd<`p(6HS&OTYqTrHFfp^ub8*RBj8@zju){c0=-z0;?QSF^iy%F0$ut`=0IW8*5- z7jbx2r3!<*GJ%k=uY+MO~l z8nvF%ozeYl*`z($8Qy-*hw3znigcRgKQgG@T6tvEQ?SR)uHmo38+Fg>Q2AWd<@Hw_ zdF|TBbxy*E8zqA0{^Iw!?$~ML&PsOet5<7SXqTgsM;iAEN!_-(K*{qSl?dAI|7OwW z@t!?K{Nr&aXxy@Q(e061RrY0Ge<@P#Lez7%?oMxabi}j)aNlE3N-NsFzSn$8K0a$!;?+_H7w>&X{eDH4_hc)#@|2PF(NiYYz53Rd-)SFxfT2X`HCm zl+JBVJ*ZH3^RjWtPRClUDn6r+IIf%;_w=9IPPu>KtRqWIhcZf@NISl5-JugPjp_Rj zGp~bPWik?u{8q;4+9ap9B`X9UXb8_Zxou;mKBHe1cNx0yx5jf?wmz|KOxz67OcJfT zqfhPaYvvQK8{HSx8#_-9jkqy;&)Bv@=VcBz9oRT?Z?BmlM4gUZ8m^=UH5fYoWcAOR zj;$J3T63twey5IyBn<~;Z5+8(hCCZD>9pEss&9~Hcl3uktBT!!d^UB3#@YXoF7tBF z3K+gT;Gc;NHl*v6*<-Hkt1pWXo!Af>ieGeA$C3kU|8cWl*K_r5RGac5<#ciUWoDvB z$fW&ON1K{fKU}s_%wH*W1~%(DIR0W}y;sNDQ1=g~?t6J)-;0m(;njbMXm_!`a#Ciw z8uHbNud6RUU88lU#}a<1}d~RP|Sn z%&hR)LvOW6+P(gO4LPyn!LUR94`nt8nc3Q> z?65e0TDre+;O8my%pp7C{yK3wwC1Rq?<;SSYr^+s29yrC)$RJYjvL$8E;r7rY2(XX z$ByvqLIyQ(TG=k(p==h^c&uS$)XU;AvNyj}Y2@rW+}~^U>>e|8hiMUF=c7OK9A(~U*0upq39&5Gxbwx&k^zW>v z`mlWFLG|J7zr}1E-f14<-sxv?tj^EFWtPf z+nriF1Cl)Y3|g@Cc)P8j`fM3?z4FB>!OhP+?9un*H2%^pnKs<$32X^xVGW4hc!3O7S%7IkJm2jqN%a@ zTKu%tZC*SLJ-BpdxoN!;+cpUdIJ>QAbkc&Cw@-C9ICrdxm{8NxfSAyPU*{F6mc6xv zo02HLf6$ikyK1F2TeP#!;}^gD?rdzQOz(E~V&lhG%hV|nc=pug2wnO-)BH0t?i@Oj z*vu{a^wj!3lO7(-?69@hKIbt#yASYR<@_q6d%ce%BGTrY(mOX3D_&3PUbb!t*O4{O zEE-bxI=yOG$F!A2nk~B1H*3P=F%>ts6(C>EWpmeUi@v?C(a3|+ zHrHA%PhGZUy^~9It-h$igDJ?-eF^s#HS8{)kf7f==xMohH^2DB^Vd}V6Kt0hp|jrK zJUe{JhkHx!rS82<#CP3vRpxeawfwI<%k1vmxl2a- z;dLvd1r%F+=txc_SnGX*S@3WjlM@g!$ zj;&tF{caRr-j+*=5+#lQ zy8HQ?9jgZ{jYvD3T(9sEITx*@*)F=m!(*|>wt#vMpEG?-9agJtb{J|k<^dBypT-RhH>aqZhQX+M49 z1!V0>+59f0s<(@qR_);G2N~_lc3!(;tG|fM(uor@uQYYpy?DyUze%DKdC&3_<&!4!|Dl9wcV_faPFFRs4O6t8Ub~`Q>jQ|-O`0O_x(Qj^oh93 zlilxro~>Fu?>KxJIWrR9HYO1FE@#FdnDdO9C!W@^yX6L^|UWF zDyOVb{kl@#zj9Hx*r1vha3iP^kb(To3xvY`Z z3k;j9$+-B-koIuZ`$@;o&#o`=oS%HGYF4>tyF9P%J72xyHe+P0-=GS|*Z&&ZutD*a zJq~3l#0~r2z7tY$^=d`;5%ijq##e!}UNpRQ@imsTbk!ceKeDgv`twuT?;k(?o%AK} z0A>0+ttd5q%#oq9hG$#~JCxRLyvOOxvDd@1AG+^}*}EdDR=~6lf1;uH!ip+aKe~16 zk+#nk_qMf<^?kah%*b;0KR@=Jt3F!s;FPh~PEVU$xy7Z0b6r{{D$fr%#Z+%j7_Z^R*k1g6X8T#Q=*#S2yC?;?DtK{3md#1H5_gTMXWPFLBW%sJ? zt-5>NpvTu#rG`kKA8Gz^0OAy(pZMZ_AI}tdktYWh7Q1<+->uU3%NrkEzQ68Tog?L> z!B^Egii|J2@b?9yr%j^8t)^@gA6qG2Tlq!R*q{$(jqvOJcW-syzhTjc!Dmi2slIyj zmMv|hXM-=6p}+Lrp5`;Vk#6;sONPtauXUK!b>&Rivjh4Tb0XTl>aT}8hyts4wSW1p zWYYbXlB3r$!WR~~Fkg8qdXm$Ce_Y8XeU@oWT}IZKIJ$3}JIxahFV6CPx+QRW@!e|Jzv+ebk}8FQeWf!52}`?ZMfLabCPk+)`f#Ss;=2|Ozsjqb6zhucjULfAQ&Mn=Mb0YYa`7o)4(JW%i$YN<>XK@!{5%{i~mh zZt^tduy5n3aWZAl!#3H=F3cPhqMf+8d)F;ky+$1ub$CAYl*h~u3rgbqu8bcxWgEQy zxz?EZE>rXQhNt$eW(CiU^*L-nl+~ueXKIQIQ(iQ8mqH~vx8A*S2J<|2a zgA3Ra%^R`L;L#V3#g04K|9$YLtHJc+=>uvtT6KF(Z*2LSlnoo-+?4#$b>iTuQ*Sp; z4f%E7(6_tq?eL_E;cK&pef)TG>f`Jid%yhoC2icjUms_0evp0U#qmFjZLL_bb)!#{ zqT1jW9wa6Q>z){jJL%g#9&^BN+oca9@ZrU-UY=0t@xZu^L*{S3FnRN`lx;!IgwX|d zX*hOp-StZ^o3_PV@SJ`6*O4*sRO7bj8;AFEy^7tLr&UHA7Li(73)Kc{8C z-_Yn|8u1QmmR`stXTua}qIxlu! ztJsG-<>~DX_bfW0;h>}b`h>R^ih4}!SaITx_u_?FtK zEgSdP_4~RzhfCc&d~?RBg^Kr6yHdz>=c%iE{L@b1u??7dH4e#q{Xvi5v-#TVXTgl4(!YM+1)<+pt(KkArgAGGX<2Ngfe ztlaZ&%}7`0jPPTacfalnANB9_wu2(!*WtILSB!44xBH*YPFqHSfob=BD@|P5>c}|* zQU7$j^j(K%3lA^;XF{#W!#kRH2wIk>pAGp8M?@ zOP8y<@VAHU4gW0Ze&WlAtOLIc8L7V9bzk!EU*C+Rt{9Bxl`qz9ji|MHb(>Wur$jtB z6|!*r#EZsbtA07(^SJT$)>+r|zdL^p{P6rs;MJWKuGO8B@vP#tUGBRF@0j)QZROoY zYLo8y?frirzOZyn*1n2O)~>MiEpUD0u{Y0}6**T? zvidjW$DM7~p1*VGC3LWdonYoUYN_rOq=KPfS zzHfUjesxHHeeIX^*DtNO@}}9ZcZM|Gl7PM}J#|a+sm_}&G=1^rpHZ6!UVQ2P$BC)y zQ_tTmk-VkHrI*sbhLo9HVQHt$^;-8^ydreofKsQ7HCOid_%d3R3|$^Ns#l4>k0s*x zx~-k(x2#T%4AHGAwGQueqL^3Z4$c>%zhwUI+4g3xg zZ?dody*EBP@W|QA*;{C-;Z5Woui&j4%V0Y$JbSwR^WNlHo9?Am3~#g~1|1NkzwDh| zacHSkpOk~2yw5IWx_`0b@l88!oGZU$$KjM&qcYy&8V`jwHgd$4R$0aGOglPhbICt4 zo0RETb^E;2ZR8D>MqSx-Q~ofO~iBwXk5z}vBDi9MIR4^6!(xmNAd@muv$#~m#? zY0-?~RbV=M`R!fvl2e`MBrG8^>u+AAC9mn9eRA)s8%f#OgIiqsl$M?O zKKsu$$ioLttB!3iGxt@XLe|T)=Ejy8&TE_R{Aba?dbj8LHUd|M_VX-Lq`T<1&0<4csD==CqY$n`+{*q+`qL#O*!`n~tv*4bm8p4f5k^&mX!a^lN9F?3q( zWyN-FUfn0;jb>)7LHO@^d_aljes`)p?s>7}+_JTlr#&xr>3`m|HFWU)y;m=^lXqRV zKtExx^?zxjd!4(aURjhb*>t@A+?eMFuOGa=qG7c<6-pi-_^iq7>jvpxjjLW3 z{`)ksVO4tj^?I>oR);tIYs|%~4Mz1U@wnODN*;|Leb#*GKJ}kx-P?WK;`Lzejv>`e zuYO(sWfS?O^So!%od31lnPnU}9Ue={%>)NJnV+VIzT*YsATJp-f z&a<1|8meqpZe+{-i(;d$PI>`}CIp-c>okLUJ+e}g3_9t5c+=!#CF}+vmC6?Rr2qb=Al2n`<@Q!u@yh?cc2~-K#e@yP}g*cJ^KQ(_u~v z{_9}a|ASBd;=l5LG5+iABlFE2|MmVE|NSA)x5R%-rB)03$88=7N|zqKvBCU|aXW64 z;h?R&VUePX#%28S{I4CecXVIBrQhSHjRLBa?x|?^40K-9q11Ru#Eju?yZo9KO)nk% zW%ukWyDlX*D0AeW{<{NL1nzpAwbLYjb@xO+;_6QA;)JXNt>eSHHyGT%gET89oR7V%8?hfcXb&SrrS4a(6|djS`QdI`i^u-*CWlg z+_}0fdwi)4Rc3?zRlFCMsMWiKW|QJe+zRjIl}c1dC>GK+;I(`BBIj{#F=eIFO1D5+ zsi>55=Dv>OCbwTq>nfI8H2MzMU-VF$PQ{U;#TPrxn;lWHmWUkJc^X>wTv7Sqgao}| z?TG7t`0hH^x_kP^UJcvapx2ffj|`y^g!nJZZiC_`YUC zo=K2O_9y~t+9a!yfuSGFp=h1`u&K%)6 z>{TG$Q14ZFQh3=<{)b9_uIAKQbarC&Fm;_Jhr9l^zg)>9>&Fx=I_b}G>E+tE_-|}b z*GsW4efm7-=i&`@)LjBDED?ztRgWyCE?KlLO%2}H7Rea%w;Gy&Sh)Zvom%MVem;Y5X_}$d$-V?Jn)W2S%@#yr4OZ#b67_?peI<9!#Kcebaj9=LV>*(CBkD^&bm6AK0PnD}s zYxtv@!*3+5s~IwJOrI*HN2R$AxDC_|+%g9%st&F-?@^r%r}ngtykDl2IKIlQdCuof zZeBO;hVpIafb9#tq?LxHwr#h$m2xuCPKyjLm!8ph`0-(WRd1Ek40qkSvby}awAiCj zug}am^nA)Em9%w9aov^+x($Ey;Z(JAMSHACPndSFeei&?WLA2Ks-=e=GR*#U`Llqf zn)LQxMac|#JvJT45n+Mhy8-MM?KKGz`ua+&peLQ82=bL)#X23_t zPVq(8xi_1?;Y_)xAf_1=?z z7l-y7nKG@xm7&ubG~IdX!RgvPA=BipJ)6e;lX19r=-VT)8^`Pk`*JIxJt1H8>d|pk z7g6A3-=VL&Rf*X;xv^VT6Swh(*-t)5js}Z|kBlf&e(dv?vzx@$E@OIIGB~(3$R6vT zIpL3Kp|_5nYS8#a_o0;36+E9u%Td<0VT!+h<>w-xEf4by`<#jk5ATm4B~Pt<>~#ndgd@fAryCr?ypHQY)A2 zq?Lw*Y}dA&x<>M(%>86|{N+bohn6;u=!6#8z4%7oIgQr)HiKbho%Nds?LF;0Bs5KY z-Lx&ZYl+lh`i9dRdj0LME_$MoY}}A)!+xnYrQMfNVS6$)Wdr4qq*7H zxuI{YcZFC`rhJ_fApB7KTb-(fQNJaxe7$_v)`XX7vf?Ky&1yO0Kv&-)Z%&?!+IwF% zxb20SrOv;so^S-NRI(-_J>@&{m7B+dvXe$PtX?PL<$aOcqBkk8&U`*zKkB{m;_kn$ zXB;^9SGDR>5AIUc_B@*%;u?JAc6RB}ZC=HHNvj&=eqvNc#_0~l) zNfa zd2Jc>L8a&n1?Z?e$G5@vhXs0Qz-%l8ENy`@d!^8GBLPXA1sDSXl0YF2I2v$ff&~G3 zI~tXkgcQCkL74z+g@y(yMVt{Z0(%sIh&neb4509i;6Nrpl0XlEjpgxV0fvwlu`R$L z^QBz}nlL)XRr=@{Td9oC0tFg;l669=gF2#ZR`B_i;prF?0gB6h;>6YEZJ`mgG6cmL zfnQx#S(o*5gC?|F7>e`t+l>-Mu@PW`+*t7Wbd@&~WHaj+;Q)d%kT(1!_#PBy}C@nizzAk~GJF8mt$0Y9Bz zp(gA<&2j&`p8WQI(R{+tBz&dgfS20;H5%9bKQu(GiFmvJzlkS1TfRT=`majRXL*Jo zKO97#FdJv48KMu;X;tZskpQ2e)YUjcw<=T-m!fOj1N0WX*mfuNWnX;HD}V8n)AT3KXPj@q6$2u5aHskb9wj(sIX zO2-JU)e=Oe%-BfbSP`=ltr^;8MT4M3b`e*biOja%Vx#-j~*EK`FUv%b;E*L-5TJb76^c3F!hZfvi8Y;8fcP)22MCsadv5O%m|^z3>Pt>!kq>O9;O&Nd5-cwvTIT=70=(&rrd+?c(z#EWuzg2D;P^lvsJ z6CB-|;TxBgM|MKtOA#dl4@C9ix^YK|p{ZW9p5@oFaKOV#hgd*3#CjECl(Ew6Y@})Y zuP7P?l>bGdDViGa68T>p7Ut&v7pe|x6#7>Fe-qDZk^enK|CR#1yww0+hXz2CFc}&} z9|y$i@|g|=0G^82D-J$u)Axb~4yhpm@D!J#3V%S+=m^vdCFYOz8X)owB=NFf|KzsquAk^}oi;i}K{l|0Qke6`es} z#{V0s!`#n*!Zl%U`Tv`EM8iAm?brk4jxxZ{;V9$hz9Bf-zQdQsE4R;&r^>9;|IeAc zG;CHH*NI$9CvrXAD{(r$!*o~7Lb^?`=yVHm072o9;y`GArIFK60n5rx%L-OOa;Pd` zak=AT0jo$Z%>`~j=J{8^I>;5e;P7S6iv=Ae(M8RpxbjstseJiAgTI!0059SHA)z4= z?*2dF>bK|rZ{%^6{|yW-6aRVAVlc`CK^-f`nS($by`5jx+Gar2iXb7YYE9C5;K$QR za3G=-Kaqm#NChA#R7SeGQYk0bSESsx&+B;dTmLw;lC<3dsaL!X__FiAun-UbpGIMC z&;Q=YBbVqftCe+X^D&OucaiJ<-%4mj6KPp1G{~H-SuIeb`y!&RGQz}yaRzgH*|@&0 zku6Fm9oJU`)MPV{;K`h-#aD&XAb|KF_jJ}~5tU3A@iqXtf`B@zR1p_dON@m{{4V0> zrj+_M04k6JSThzH9>Lly&L$jwKd+-vug9bXK|t8vWWywlk*kBcce&54j6?2gEQcv; z!gX=JPBJPAA$jk#K;}OCD}MIusz+y~g@t~5C_k@Y7(#^NybmY7Q94(@+%riwLgGCj zKfZ+6A@oWmU3LI{`v!aYC|z{`efw%XK697}IX$~Hk|eug;ZehR|3Z!&q%vM@y-NL$ zPE2SO*IE}Jm846EijQ@C2LRn0F1*I?YpqHRCMBk%woZ;s(IqCuCP$?vChG(gIwnV@ z#w8}`5?i*7jZS@yc*xh2%Q2EC!8$%V3058AllmQVmm(a>I}5sh@z$0{?Eo@%Cny7P}`w zN0AKj-SU0kY4!lhIqnG;-0&?{b-i?7NK3c~1IuJ}G@~;atiAzw+Ql6R$nSUOhk;j1 zxd;Up!h;mhWG)SxAey#VXbNes9+}`LB>IMpIJqqip{ht0#$k}T@k3_ zELKHeKZQT&gfRqdV?kU@EUQpi8M+rVa0(THLo8S*$I!k0cXoZ}$JR!S255%a~T2TgO7}cJYyir5EO+qr6j?=9K{4N@P)f$ zvUA7^jF~UHhYY`Xf>OpwK_+F`@sn+Wj3;{W1VvbE7C_l7ddTpII$~6!&D0UUR~(l7 zk#&i2DMKsDg`&g@gf4I-ZXjq71Wlz97RYj##fo%mny=^Zw8l}aF}=%$9RP$&swm_Z z02wmf1D)_lAw30Sloe7@NpYPtDF!pNIL@jC^l0R|u8>Z1dfNyF;#%1;Q@U^^N&3^t z5M?ht7bXB!h9^djfU@&e#64tZQ69u3JdmVjLX7fOT^7c*;8h(_UJHt=YaCdwwdCtkR#8}# zTA)+|;LrcJ8NdmaM;%wV75tZ5aE(P<(4M5D4^o)kqRfTb<%5m&3CAnqzLpsPJ_0v-l*+GP{Nonhu~7Bhj{{!0 zOK#zgy&#isU)-W0`>gI2FHm4QDKH+gh|7iU3X-tJsLLMCR&TybKEg!%#K+qzmcj9e zCjBaaJY!B56`CV7b)f+el##($jHr%^1UAb}wHE z4qA#b)nY_v6fv95fR5m?h3x@F;}8YbCMd|VFIxxyHUaxOsw`};+(VTYx_-U^%!>jI zQW*q8Qz)*v!!5E%kURsV7Z!>)sZ)BCz|SSzx}UF7R*b<&5|W+;cqAof9-V>2Shk3h z=o;p|DAnC{HfL0u*8;A{-Es$nx+&hjeN`lV1ivpVD6e7i3k?JX%r~-vEyrhV@6wr< zc>9Is_)gcg{~S~E%nVSNVf5whP~e{AC(3!obuTgp#J9p-SLK&obGtTfbOd9 zc(yu7)3-s326gJ^+pZM*{mr)Q5|sXxn+|~8@rDgrI1p&qpoRD`3RAi>0VKjQ5N!g@)!MuS?-+$$OHQ!oZQcuz9OeXi9QUzN89R zq%01F>YNWaIxEJR3lGehW2>-L(`|aFBQOgR_6l8_q)oDPGKH<3$#=d;wcVWyy;Upp z)aI9*rw9X+Z(^QThgZZg&nLsrtAp~)1@szVvade+Qf{GFmSA$a2Q~-@79^MMY(uwK z4YI@fx`{8il-|y>%_4}!@(g)J=2`n%x+LO+;TD1t)J-ETSd@!yALMb= zA=5Mu=as;C?dr=RSTT#v=b@M@2w(E0$dHSw%ZtjHV-$qmyg56dm7zIk;AI;}FWzA5 zhyt86Rp{{(>RSl``2x=5zb!60=uu~53MZF)y0jM<|9mAODO;XTTznZ&d?nQ@G$6O! zdxfem`RBiuu@Hcv*(u7dCsu#c_6}LIZ`Rx)He58(O04|fVJdM&=09pFk)gxuxsWLo zqY#~C6z;`5mcq-~nG5{?w0Gj$=ih#O`2Wh%1-yC$u>AdhHR`Z1kNA)3xA%YF%=7R0 z|H?_|f5Pe4i72-tV>Bws>s0(i@#4HN0tl!1=lP?As$XaQN zCG;c|eRoBN&*=7E9X|OnQt`rR94>27?f{v8qays24((-`#XVba4u;BE%n{*3@-Kdg zb8CXbsg?Pb!w^BF zf#6^DW0uE*@-af*M@fL|jY-JAJ9y+vI_O)|#7L6Pgm?-fgk+5J>B#EMphDS564R5; zEYtGGho$8uHY&Nwl9Q%ZM)v5A=cv2m-SM2zv=C(&|7U1;T8J{dM}zLFH(MxJEELMW z)ue?eLwmR@Oa{`+xm!L40Phno%%gnwc=<#ErxB#^>hCL9>|gm9;AiU- zNf-&p0t^jgADV(F$t(qEqmhN2^Y$)BgN~`uUNMQy!Yo#v7pWs7Bb4e;r8+WI9ii1k zYQq~TLmF#CLcCT>5K}}DtdKxgT9R^PJSy6})46z<3~V+6FAN<>=O=IQ4uP}llCM4M zmLJZKEyd(uIHkK2TWf(||fjA&2G~MWQ1*Ai}B#iQ)_`g-I{Y1x~aYeX{ z;}X%udtKjGapIBhfIKWsVId?7ZUe6ZBE@X!RLI*v?%wVEoZ-;QKm*1>cLMu2c4;5y z4_-;n6F854v`^ps|2}+4IgO?GIN!}@LA~m^t_9qyL`qbZ#cBS<!Lce|Fnq#BA+HD)=%E3# zu|k4ESt!xyq>c{a0%nW_7J?)Rl%|_Ad+M_iRB_t?JRuW2*HHna;bR8QhLJ)La~5C> z6p|JQ3ZQ|;eM6Mv$3S!n1z?tyfh-RmWYRVg=QT>R%>)5P;lNBY3_2*nC~rEn0F|OS zfHE|iA!#+*NFEKRjT%6Pp&4L-EQ^^S+eFarX-dS}b9G2yQ95lNQ_*N4Dzhdc15%=V zS@@YUbDWjc1_!&ICgQXKr7|=?D<|bN6#c=(D1J%+;s8Ssp%lCn& zE`eNF2w0nuhrntz+ zCS)c|=E8F!B%;6FHs5ZW|AB4uwNb3EAUFSg2LvccL#ve5-aiJ3f{ z1suBszn?J}P8c`-- ziq5}wLSp{)#aO_4uOFk6W1~`IV_N!XIp4YoiTTzR>N2!fnwDBlPHZ2WZ`(fWORf8a zAV!7r`YkS@O>A;ps*jfQtC^6PUtN+m5#HoLC1{`6IxfLSd%kNWB>Ju=i)rcgdva`2 z`?zS7^3g}TFN2Vf_%axd3%6dsrgTh-P41Wyo9v_g{A(v9das{f+}gs6EP}(i{K;2V zA^NCiSqT|NGyCK^2Nd=M9{?m7K{>{hp{ue>vFzeMXeaW7e zrFgviJfGZxjiUDxRc~o0SGs^9Iw=>P?g}>Bks0Z;1?RA zj#P#;R;o3r>PA|PMym-`HPVEKgsQ_rH4Q^TwISk(wYRH9tRphI2$4lbI6=#20c9ff8H6J$9pz||p*)FFmO2BSLC7zRTlB4DFNAz{XFTn!@|;|8N4 zLam2T6RD024-M6XMi?~VhR{$0)(8(ZhU$$77^F-D1+~CLb5=nYFw+*Obr>#81ZTGC zRR-D;jIJ%z-QNQFkfk++-N zHBbql1tWyy6qH~c`Ao!3ulCh;PU z@5_7uxE#qV3qNEwhV&RNM+qe&MHOf2gGyw$Qm6@K$IOpD!!(+3O=M$_hFoxC1-( zniFL2aqh{{wh774)~{c*z3RA_u;T5Dm4V%ky?DHw>ZvN*pu^;4-&J9gE2U^Yn}5vw z=aBgtcsrFn-m&V*9|O;}n69>Dm!4ej{MqB0Ps+S^q+Hn>=Xdm6_0PQve_QWuxp;2J z-4{D{ytwf9g)KX*d4KnOQlre>(gS{>>qvgRR6;Z zKV}py-z(PQnqfuV4QEPpPZ)B=e`?^2x;xr0%qgKs*Jy72wRq>@2fc%ni9AiasiQh> z+p}Uvq^^5FjTs$|4!M+mbjXA3D)VBl_U;_o@LKiK>ZOxDj;hkWUWG~v+P674_{#L2 ze@q;X4}P?;!=^D$&gY!oI`#cx<7*Gi&0P7-rR&FsBnM~wm5~>*r^+z4>yz#a=KLJi zXuz{IXG@+eyRP5T6-Q?HKZ?D4es&%^`O29yi;HdS_u}5&vX9?y_b{*L#tqwM<(0cX zY-0T10mY8y1dci}vcum?J0BWQQ;h|8N)1VA)n!O**E*LfjVjgiw~w!ux?lIR$@TXS zZyU4rlc?*5u5A6b{`6`pOZ^WrE)*|SWpbU;Ayvoj#@}iG?(yl$&*J-sG~GYp%Ba@U z(~g(U*q%2ytWo73s@F|SC^ob4h^RKJGzk@t;z5_X#666wn|8Qe`JXF(Q)}tIKmF_a z&+hv4V#$_sK5g{8#@MOjj;prrtXcJon9loEzZ(O~W^~)=hh@(Fy=2tp#*Y>}|1!Ay z@V|ntMvYtYB&jW|T4Cp?3l9{!t zScfq2J%M5R!s@d;mrGyXqp4nN)}xQU8-2;TsLbON1DZy6{c!b&!<#bjvfJ~jYHNN!nSQo= zNu58o?7C?FwaE=*UaS5ok4xdR$84%+obW?v)!u`ur}k>Ha!;S5HFlN*lladwJ*~dH_>J4gbCwz_H~!r` z>P(NaB=t??UlVGt8EDq!jXArsmO5T>XiZEEb*i!U1ih=EDQ6*c*Fuc0Sm>^Zqlecy zH*r*%Dw-APe^y#}tjcE{>T%(%>prX>s4um{)H`Tq&HjT+)Cw`|PJQr6lL7vn*UtZ8 z+k;_QYx_^nirfF~_ym8`ncDc%Q~f(VT3)PE{SLF*^%@PPb#LeY$K)ylHuTcQZ>)W6 z|AI=}>s4Iv;N2nhb_Q1UPg%VLH!ZsK(ZMQvmfDlP=)?R|K^<}CQj2?oo6gh>=(4@o z#GzQl&x1xCj+-^~a?^~>^C}$NUA|Y#Hj!6LetshIV9va0tF>dv6_Ms4hqm1O;)lLJ zjH(?yeWu^~PF9Bt9sStbX)qg}RmfhcymeF{}G@^ADGs zMok}>ivM}<2@`WU^^pArHnwNxxc7z(PP<#G;ntdUz{oa-{n~_RCe>H}v~TW(`)g9; zJ|?E`of|pw?Cu{f)qXMj-5)>xr}Kf0>8(FL->>nHcgQNswAaVuHP|~B8y)Q0>%t#b zKHQNUv|-IS#flpHnQkBZef9Cu$&0)049aZ#;z`}6+79aKQ~voNczCT^O~$mG5mIwY zgYXVoxbN$QHS9OvfBMagJ%+uTzfN8|tixOw*x`7|nGZ{@J25@ugKVPLkl5va%)Hz* z{nK2t3Dmn2mpr{IUh$nN>{pXw7QV+WD*5fO6cBiD$-}nq&SAHHR_43zV|I00cXauBh~FS9?KzO~m+q5h-pWpr%vqhe$laVbzgKh`w5 zJsB66HMo2F#@Qo>&1gQQ_oBgTudSv_E$+Qz?BP{A8I(udH%k#$A!8mlxJO*& z?e!Y}eY^cs{kMI7h4-!BblpAW(A4+VOj_TNXhu)Wo{_aKW@YcpF8x2<@z<0gq2Jw_ zRQ=bHVPhk|P1b+Ec>UsTSF3&(nGw``z`Vr=+iV1tW~s>w;+2p5N-TOS7RCQe2 zjCINDS7~cz*ZXL=qW;jiW9MqB-?l&AIJI{Z|G?hm7I29Q^}s($PKSC`h2~e)3lgZv zuTMN%G@?QHqmpZmjjGmS);~upjX0K_oT9$|?SsWf1Lt+?-R1jgQP<-ZkGqv{YE_*j zw?0`rOHr$&rITrXXJeK1=Q~YZ(dz!)m_3WXD>Jocdh7ZvB9Cu+Hz{M@gDXeVtd-k; z0_}wTZe(&y*U#s^^KstBk^wrp#I6AwzT8?hr{RL{df&eP>GzdwZFB>>9Y0m?_PJ8k z-)V9D=$Ux)z`6FXewu#uhkfY{1M-edsTDr))}B%AHul_Ec}$PAevOw`emFR-=Ck4P zxnJ1_c5JBB{y8zNbd8e!BdYwgU`UM%-10Bl=PrAv;exAu4v(KSrrer<;>)`BT6xrN z<@%fno4+5J?(Yomw7uqz*~GLO_`0vdqAR3c9`aGC$(q{f+uQ#5*L#bGZ|XSs{G(>8 z6!WT%Jv?jwAj62qcJ;CEI=9{y9k`+9-;;LD=y0^2-=lqP@2xynqUoirM7hJu>?2xj zznn30^lDIZx~w~=4mh=Tpf-&v7YVfm${vsaUWSJr0U%zC%$rCZNVUrssNvqsCy8Bw?NfzH&X@TSwAG_Z}@(`xiPv)N{yhBoDCz!4*_y4-g;-M`Yw%zATJAyRarDFegHQUGd69mj;e@J<%>xpKJ!^X5Q&1(LC4o;`F@1OW zm{t|X4{f;MR#|RwgZ^<<8=t*5DlM#I=fQ1<)hL@AS$yHeN5_UQzIJiQ-*a`Xr_H~N zRXWsr0F|{{Rpytet*ZV!>ek+2y}x0Ik*Z27QYw8Ec&0;O#K5w>*00DuX&W=+p?2!5 zjz>TGxZ2Xl;E(>eP~Y#O5erIe_f_iD#IfJ0zh87U#E=m&Frn7zdwW|x%lPu+>4{&G zhiYfp9)D~7V4OkK{>F&W{jZ<>p!Kx6XR(r34qa?wx&j8zoH(*|w^eKRCO_^Pmw}l( z9<3HVI;Nb%yg z9ksstt#pq=@dKVSPb!z0Gp|Lj>M75eC+nXbut&5xUvIm%NwT>^!&2M6S)DoJ^x^K2 z6JmFdpAw`!(yaKHn~k3IEE`AG^Bq zyKHv7oqb)1?EsoFKn_08kq4?QJ zGr3K3!sjg>x#HQ`L$;Rw+nJ@wKVw6NkC-*9dA z{X96RV*JRxTONI`dS}p_g!8|csvNviX3t`sIxAHbFn9g@VzVb0sn_B8uwjJXgm_tMH`S1Y_fd;imKcBi!d zBy88IV;NtJ>omFbjUf|bKEsDy?ECQNrFz@m-hV0WeTZ(V9V_=%s;BSPXw|KRa^-FI znduia7dNlkJ)p)<{*${ZlE-uz|HSy^{ZoVShOLZzwYU9|O&J&f|Z?Yw%G#XI$;J2Tr)Tv!!S@$0fYn65S zT)7%uZp8|1+Hh{=5!=HSGw;{AeCkg!WAXCsjeg8KyY-iUa=(A}&ub)Vb zb+rS1?rtwNqRfrwww3T(7vJ;-46Yk&W9h{|k=g#i=#V?)BzWl+B4{g7nxv~0u^?hYRTAnj( zdFRX0^S_@rdg??@*?jU^<$-0&RpswjOo@70+J^tR>)PeCU27H$AN14F`ju9U-msxn z$nln^N^viGZO#p!RoA@Y>}l(n&F9-U#GQ@{5TlfCnZ>JvuyX?3+p`rd_yBkpc!F|EY575qx%AC{j@<2;nf>A(Hv8bg z&HauVDnH8k=;p1!uWOZEkQlTUXv4s9I^C9NjVZ>f;f2*0uW({5-nY#4X)V`rUZE=0v|%f%h-2 zY4v^uY;tzXCiMvQoQG5QHopJQgH{{vWL6nEFyusJ`3ae zqCej`^zpXqTY}l*)T+EMo;^D<<#yi19WQ=)kvs1C=eP6L-^}~z{=r|0Z!A~tv%3FG zOl(D+yqTWa(tO8SqL`)i?J>I>Z94sQI5n*JxijNGy4^o@?U1k5pPaOQN!F&Q%CyZ7 z_p38@P>pXFpRsRBJ{dgg*ykgr*VQLv>#{RVi%+dyXlvN{{6Ou-PA{sQ`?*H&piOZv zTGYB5yfUw8wcD1fdC^Uq|8Y3Cq3yHh-+cKr-v6NDs|wFzzdZ5N3+=RSBNK8jxBMQY zExL5eI<8{c#aVS~WORAnsNIBx?>yeHWBWaT|G03-+27k9Z&tb55 ze;T>!!=)d8Q89x0YW>u#q^{J{VpEhdQxEOV42VHIU zNpcvqp;}I}dfm5vzxwLl_b=_eH2vs&?cY{f*}KftKm^Mw^>Fd~xij zXLpa-p0>EwbzbELQ!>9R`{aj5c{}btc$(KUYt@Ooa`(exY2!EMbG;*?)5a1e_fci|M)kO%CkO{@4o-H2?hF1KYau8&Vhj`Rwzbv0tW}6AnMa=Lcg8PRz<6zhA%Y1-oVG z^23!Yj_kK#V6Ds#%2b^H^{uwnzZRt(dhzt|?oWq|Fr4YKGjrJIk4CU(t+o@o`>Qv` zS6#89)$${g<8K~~p8w^9Q?>)kKRwaopzX@WnddFvSAO2&=^rmzocpfq`5Lna-z#^1 zYv8s)TV~#RTmeTUxZ8f4ja_H{nVo1n-8rzG2qyE?2AF`&TPwx5gPSBEs%(3N=b!ITY| zM?0=N+2H=8zecX_f9gTtkB6pwn{(n?$;=JiPd^Cxbx5g6Wfyl?U-Prr`A8y&QH*4m| z!H+3pkk*uvFnmMv!zHdx-9K^tdq0k?z+&vs(9w@wS>SM=t+W@@Cm>jkoqR+4}6Lr_Igjwxe?AoU;oCs@5H> zH7EIxJs0*|SX$@f>SfH}#Kuo4MS(%X0lM z*JAeXs`Jn3ceX9*{K;q4?A^u=>b9^#qhYz3%dSFl2)KFUyIM;h;4H|fBKx2pJ^IjM?HZ1?4~%EfeBYuZ1}X!}v0j@)(aCwImj z-_mU#5&hXx|Ac!hMm9cM?3_5)V)OpfhhhH@AJ6k&;R=<~z~HaT z2`K;b-_WoyO{m-XZ+Jvl=-cz(H}N>4_%qN%upDF8I)15~?jiEH$qcm8EWy!yQOrQ# zUVSq-%%lYwYLx~#16ge(nIz(-9@jf>AaEKy*5iqNO>~WAc?4Xw(+;K9T4z6D%a05B3)k zBPf#+z2bW$#8PCHG^NCpRvK6Md*I3stiK~Xt;L=!4sAPV7fK{x7N}6Tg?|Crefd}I zg&^7dUkc%hQ!2)oAg2X>l1o&y{N_`1ats^xb1(^hA28mL9+D?wkjS5E=9s`5xWC3j@ zzjuGYuQ?82lpRno(;55DO5+GBZR3RdOTvHvngXoZ;=Ft-CG9p_bfW*P6Ex%uINxDP z41Os1a0(HN1qUe=h_zTbJFu~k1??KMq8Am2iB3`Rz&Qhsa{Fn4pC1C5VkJmg45=qf zu|kTlCuYIOcg+N&Rf7Dnlm^_YREKKSstBc39ib3!w2C)35w%M(wRjSC4{xmkN;PN( zC|VdD1&XGnb5zj+rCI<Cz;f0T}~ru@D?jxdOF8 z_iU&-juvx|03$}SP$ABQl?NnVlM|Q+;9O^(6(9;lT(ZmwUjUSXWBQ4|1PT%V;4qV) z(3o?67}~}Wlo-xa;-P@GVwn_b-pbGh$g+;{;YZFO1h6Lg)sRUu=_qJn7{B_Rt6OxZ z)br%;n09pSMO^nCr7JN04(~Zfj7t6oj#iD#uii0tge*$3?7L2?t(ETLxdL~%$fL34 z`=*BumaVsoVcSs1Y%64B&>X&yBIZtqP62%}=PigyNC7x);pu`l@@rcnbC8J~${nLi z$h}0Rj7CCDtz&7Bv;kf3@|Ruy5mOfcjAaQE1##iNJan`p`0TSXsBXCqq_NUEJT(0U zm@Xh?7v)8W@^O+AimWA+W>ZLPF1=Io@7%i}%c8)yh2f}4h_0*4d9kqOgygG2_Ij5P zW&VQGDy9{j_c7-yz4%Hmkh;LGS+_8ie4C7o0+>O-L=TNL14U*_u)-dJE_8|P-Xln- zgXeR!$;3Cn?;jY&5kShZ&+i=kU^&R&D&!gvA-bpJm$-o(W>QQ`9#a-_{Qu$!Dk=jb z_(29CfM$T&6&g;kj19>%I=u}?o3(Ri5{E(JF9y${AZg0Pf0jl;z@Ft7i!dM3&L`dE zbwWogN{R`KWDBVQe*F`ak-=Dwu^BiU0|7(xhzJZ0qi|rwSQa@SD};;V^1>*O8mp7# zZkH#wP`VaYqQb(olLU(b6o@MZ1t8`*xE@NknRpJv&=!{kWWgxHD5NFucb1c_B7fwB zF2C!;zkyT+!O#@q`NI4O0sRmQV{Itjwb4cb3t7GXED(f-SQOG0`0z&Bm&;b87G0Mi7LjgypLkjIJn1BHP4dtiJK_cTfd4^vn9BrjZ+GIx&k@>bfSb#!^awuQsI;j>4fDZgzbpc-V<1z8` z7*avnj>oKm=qn>RkbyCTu)6s{rL8v9d72Wrs>DorOk~x>?1UD~ieycJl0+3qh^`=A zbU=+*kqU)G6b|DcPtGi_1)+>6u3{Z4cH3E7XO$E%nn7fS=*sTrV*~QwnjnBDV=)Ie z8!Sb!XB&tup=^ud<|D|u;$=sFH#iu>lLuoVD72lSVDd&Aue@PKel?;{DB=neVLOWJ zOblj#?t-NYu{ky{Bxi^^8gRJ#d)Yo^)UA_1X@Ot1B!2W&IVezNU?2{8QX*MZHfLqh z2@3CvexWQweTCsfzd>A#Kqr|z2U7|m_xYg}R2CKlNLTqJ_AfAfj0F@;DH)pPB+;*d zNErl4>x`_J&qy7j5hy0p3^_9zIBDN->?{(i;h__}>`Y`3Bs4)BSOnfCz;wQs?|=?r zDN_tLFRU+dE+w%<+@r(@G?Ko}i>0U*u5-N80>3~%X=4!B4@Q#CIxH`~N~$V!1_yG; zy?pzgySY$9dY44w$?{N-{wUCp5Mj{)!0I)1jVMIQ5;+1tjWD4Gt4%^fys21lP*Pe! zpbin$7*SIx6jCk;6M?!lA%|2H)ZCID^4P@_{0xXoj}=k@2-L~a7MbuYN^Fu*LM*R! z*>*F{kOG+)!AZ)uCqohTJsgry5c81y#Q()hPkdG9KC2c-&L?!9Q}X-LiTYI^Z(5j(_9C^+*{#s6uW0*5a-p!>JB49M-f{L_R69gD zn1`d&JK}G6>P&@bRkR-D+pJi|ce`@Q(=olG;m<`lkoD6QEoYlzA90FxlSiFGKsGNa z_f=&J z809oO2`3&P0Lx_g8YY7k$i(ReJiZp%MsZSyB&Eq|xsnu7R7?x}_$^E+yi*D)o1kxC zoY@JD2ZibB3@9NWf;SQ*WbG^mEe^&b$w>jC#ON|AJ6b;bZ4C#Vt%f<@4i8T{eD^E+Z)klzaiF*9` zA39c}6KWO|Sm&trfQ2Sr6BSR6I!Cov#tI4yKfbO2<>ZLWxC~8!UbJ3RhdtFe#~OA` zUL+pLNirfUO|_VT2Ix^q1UW$5&3R>s1y+*|w^^(hN%Ev17S{>Ib*^VWelI`#5$h|3 z`hL=QMG4%l9kheE7|&)r623VOq~t>_keJvJ2C_C1nV&sjImXvwR|>zCHimRSlr(x^ zV@N?#p=eGF9vSRidz4hgnFk^&Ey>|!Ac{#j1M8r4&jUuG*$vEtmpmM1P@=54RNq_O zUCzC?E>(4BXxWLPqOQjkf_rq0vZ5O`@nl`r$FApSVjz0Z;2Mo)qq1gQAJLphqVBGb zBWK<7Rn>j;bPs}?v-yYS54vwx{l2fh`rg%N>KF5$IBVI`+=8zZUprHL@Gt$Ve%3A9 zCUhPzou;wdcXW1cy5=YE&zW%Yt&>jq>gg{`UD`SKjgAw3`R1%;FLwTL)DAvj^ukpq zo;YswmO~4#{Zi|xOV@uiw0_5vl}T*}W>1^>>(@FTe`oj3<@ayBH zf82gXuKk4Ln(vKYHR@i(Y?2=ld+UAj#N37DQ-A%? z%5Oh*-9T@v+}g6{;NtZUFFNCv;@%Ad*UumO@elWoC(mE;y(@QJ^|1fV9n%;5?1kGV zpML$F&gmZ>%3bv4%$1M-?O!gbv_3g$=jdBjUj6F4W$!((bq!y!d3#ISqd$JT@||6a z?znqi`PGwW>}$HV@`H&(cmC^|ec%1b!Y$KIT6~3n*0Qs{kUeK`+PEA4e8OMx8>jtXy=_e_i`oTjsLug&!|l(7U?g?U`}TI5Brs=EkNamrQ*0 zl%DH1ytjSX8{K0jtZ7GQL`GI-B>YY=%a$oO^X3!AKR*4Q_a{$1XWEz#*WGdF#w9au-@SdZ`>hkFJho)h zhK^B_U!B08e+=zMFa7(H9b=|WdWdhG_Pu3y?7r05-TA_ApZ?km=imo7lY{R~ef^UK z-`_v+q}SM~1fxW+nb^`ZZH=8g|`?!i}{{QiV} z=>b^Yf0hnAiA?kBr`?lcC|p8oKgPj&VNlTW(f-OKLTwU8bkezfr5EgyeS z`Q*`|Uw!@4-~H!hqeqSU^was?1Di%=M*p$@>eaDV^s4>)Z|XK=k+r$=hq?s#@PGfU zt-XEj@W21oeopJ$V}Jkc|KmkHTv8Wtb5f#0@=88_mrJfn%&9vGbLvt=P68&VWH1!l z)Tw(Z_Z*ee4ktV4J`sPKwDDmHqK$ zm^sRt9cGJWoB1{gqE5v#FBGMRum9lp<`x%+GWNBL9~b*61c455no1k zjy8vZqwXH<>g^V2kjgSTFCczL&+-fAT!3BbL^t3Q9~WF=+1#-P;^%oJc}NryusMma zUqL<@L=Kj?fCWJTa}+8IC32}pjX)6dktln}BOc?G1YVRq%iLj~Koe;@uwb9qYPI?v zbyQ>=_2|n&b7-*_$O;Mt5opCcVQv&f-Ccb-0GGC7m6-rJHz>%yfVvDh%g;7JTHr$r;!*hc{;14 zD;klO{?;-yEXgdj-aIN|SCEX^i2#5Eb%>3*_8Ul6Z@SQkdWesbeCdKA3^LwNxX7cq zVEBn)n>&OHp!xB(haxfg^M$z?w=up%R(%f(hc}Kg7)}D0+S+SGdl&16-t6dmE1I1L z0pT!4$rjdDAhEYDR1o#4bP5@y2d2>CD}fE?$t}j3eTNir=t|VRx(_+b z1HE8HFzZOo(O{YIlyqtOYGgCU(lhh1X}HYBuENKTLv?&Yu5!o-3qeahabOX!ZxJ0< zxP;hBha*%yAs<~29my&mUsT+vF{>bx1xAQzIi(dPMG0i3$;E5P!n5H^z)6?^TG&w} z$pGP%DmSccE)&m zq{uiiewGRy9*C28dYEn{VL9dQB`cMH0E7BINx;ueWK_vXQiuxS`n1YZ|3t}9#R--vo@KB{PRv~m$oKAw05kur(>{?N{%rC*$ zkrXud0*poPM-=olL|O*QqF7N+P-#U(0PLUoMUI8!p)EsB5NE)Mc5v8tP=JL{cW^7H z>cN&8Y-voZ+8#H$+3LYrHCS)EON9iEjK#<@mdb+{2%>(Oa0=R7@1 z)X}qCVKE%avZV%&xc9LSikxs9#w76q7fYg3uI^6M)WaOq9a{(LRMs&|>mXDIPC@&x zBI6#0v@5PCHXGF#$_7qBCu0!gIoOG00drZY0$xYas&N5;J1gKRK&6>llT7sk1Yt4y zc7H`AP}P%4Q6!cG)LB3Q#ELa(oiNKzW?Mm2doixmOzBmnVT1rw<26wesIee?fZ9Yf z6W<9K^`)3)D1qTe!AGaL@WdF0nuM@zv(T4XQB*KOSfF$J&RgAlVaKYKQI@XlLFFHX zN|HQ84#K!_EldM#8E7kkP9z#_;czyNj7~i&+7J>A4Va_l2unr<>N~KGqgtm>HW{ADS6O$QF{s4O z)WfF!te>x>+=&%In~ zi7YF@MS(gF<&kd(bPB}^9iT2L5eIoXm}t!8c{=5Yv(rH=?J^O@p_#pm@<5Xml;{BQ zlimi@0cw_G#n1An;mn385^H~tT{oE{pk9dD0#L{4nl&jjfPQWIq-R9WdYIE|CYtcv z)pNcsCKG_fxZ<1#s%M?GTXqme)mTGoLyTeA(5d86K$&^4VWw_0rQuOAWeP^aS9?p# zDw@XnJj_+spo%{OTE^T_Yxby5z0gBxO$|6GhFj4oVvVaeU4WDj+uF^#6|7uztL`o6Xb$~bQ1q)DJAk}Ks z@c?D2wc;}DpaOQWZzpY!l?*$b=9yN;?$v$hQs!$rjo(0xk zP>n<4SOA$vP9bNSKRKIeJsMts2Z3ABMW#*whEr4(oG`^-RaqXV26apGO{NaNv|hDd zRpmyMWhEfP)~eDxld!<ebdqLDh7b4olXI8JTXnLW#FG zO2^VUO=_LhY^6M-UZdlSVRcKAyKIJvYACl^s{?3*Nmq`Fk7(s1F%qr%Fq~vzOnr7T zkai@RNFC&mLRcyRD@M*q(@YBD7L65k!?1kSb`B6;V1j4_7fdp>OM!6_piTRGa99lH{%`>_)k0q7%^*OV2Rms{5q4SR}SmHp7&P4!w)})oIXbE;C?MZzohMMr8fEgt_K&dKV7Vzt- z0?sTbDz1|bXVkDz!}6#BLReNm%j(izmq}Qb6EuLr1vWq;1`K?WxZ4P6igCnXSD^#&~N>x_yO(3L@9>0x<9!mJHDjNNQ&0k%F!3ScwEFdBm6IaB*G7%IsoYcY%DX0n_TJ z+WnmQ=n_^iZIjySGqz!BW49CA{h?7nbxafGh$7l*R}Vx5HBuornP?74vp_+~azZb_ zt_!MQhFf~LRee0z@Y&~sW`0)BZ;0g8p}pM-QbEhRYsrW?#=BYlq+(GgIkQZQTC12@ z2qz+R)iFgaRxDqylGX3CA$Ma$qq(eTuf+BUl~U7ho~b8CIMYtWEWt4K=lj7@}U<7z!{|W1%rZokmj%QCK*jX)x2MF}@sO zbbjy3v`50BEJMgKC!bC6!6@Vgsx7X4e}Fx!>ZcWnr%U=duo2jCRQn|*5VHqSj-b0( z>a+mHf^R7ou!`PLoEo=mx{u0nI81NUN^{hr^<=P2?Q$%0pFoHylhn2-?Vvs)*e=(r z`jaEuWFxc`qXU@Q@6Dl4#EDs%Ir9NiFA&wXALLX@;=^uXKYfrhTT2G=%-X{$SU+cy z*icu+Qc&z}bpDtw;gm>+Z*hzqP|bX`!l!7$XEK^jk_K};^!vphbJQ9V*kAex}8Pd!p> zN6uMS;;Eln9Krl3vYOd(=W$qDt_HwNAF1(JueEnLjM|Rls#M4E2!TdwIIcQe!|^k~ zV#ajSju)_+`n-fVlCpn`y5@4g< z9s0CmBpMj?RgL>>V8B^xeh(Pg)G!R~|8r{S@}kO61d+N~sGz_~0BknG8i%HjI&?Op ziiXg2wI4F&d2k3UtMy4zLng!1O?2I<){emrLZuF#`lL5D(T}KC zpBmdbd~DpZ?TCIB5ucQ*6F+rm-iUgS;-`^`_iEM{*?>7&pE;ztF*;_=V&BPPU(#%R z7MuLgV)GC%w3fMqn?~3lk|zl1Qqh+&h2~81UgXjGi-CdF%puXK{pVr;LSMENQin9d w>a&8l`ZtHo8Tj0%J~FNr{Kx*8@3Ggh*Rj{JSIz5x0{{U3|GrFW?*Nbi0N<|8z5oCK diff --git a/charts/postgres-operator/postgres-operator-1.5.0.tgz b/charts/postgres-operator/postgres-operator-1.5.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..f575f7cfd04c55f39bf421afd6ce0a63ddb654db GIT binary patch literal 15843 zcmZ|WQ*^RvM%~m! z)tv8B#BnefApaRaS`d0u88sGj8AUDyUtUf#b}beQbxvD7bzUxIEiEoZZ3jD3CktN< zRcApNO9w}gKmE4e$6d`d=L_%jEVP5}WaSdfRLIb;kzDX2p z=vcCgAT*lmY=ePHCgfcwRPED5aWBS+=U$t`x~Ja+F95%AB?*a1TdK8McyY$_@M0_sw=hanm`YSR zmso3wO*L)cGghQ46U718y?~-EpE$=g$&bs?lW1 zH#NZ#huIfisF%~^A>dvno)5Iem><+_9~@(xAu$X?-cP;=B_prA4V@%;Da#X0LH&oV zTvwPVn!*w4sxst^ugrI+Z7+5*k{B%%0#JYO@_g`8<{}4fPb$BgG4Z8zAvo}B|B#3J zkQ8_5;W?OPlNf>S3TWfwa@Y1vd{hiukgN(#s$``hGl0`=+;1lNgaP_WRsmB4IQX53psHQ_9H&Rqep~C zGRwVB#32?V3fq^!I5`QO)-RvXBB+D{8%+_$rN97m@ehp{hoqa*>NX}W60F9dg`P_e z{zRBx+A}#;Y|!L#l7-qA%b(&#JnS#Ra2!SJP(MgPG`t@n<)BDxSe{;XT_Zy`MMHN< z7Ou8YFVZcGlzT@J9(=fQO3P`o2;ihr=a1Sx&sfSbb->FAhWzC%&Qo?W=_>!4?|^7_ z&Jgw#G=O;-{E0!RDDEi_=}>^;HFrR8H8cnsDgG?lbgo~V61i6y;wVaD;;8QgQS{t9&-2+DP$OqUNCZO3T2BGstAs${taS zrjaXsC!>=_o(5$-G&it|Nl($Cm?cM9g4~UR^5EFOkSeK;Pu1&1MSufENuktC)W!2+ zcF~AhD2mavYN@=)CO)n<+gLLk>ii65TeA}pL}2lfQQYTnL!&;cvGFRH*j}ndXxual zXIRn9fDr@oI+q76Umytb9vbU$tq>igHF4zu!hs*?Mx(ZxWb2 zr#+yz*vt*xk7I&3t=p4%bn&vPM9k&~4vjmTLPdMKlVKyEo+B6TEm1Mqo!lZ86QmDv$@q_#vYq z!&)W8w3?kllj^_~P9?%}sG$~7sZKKpk__X1f@L&LraH7*N?*-rltGPkIkrX5DIt-t z%AZrPV+Pk64@EK&W$A;8DWGll+b2a@Z{j|p0T*GWcjP1FCqw?QH9uAlG+0Ir6%zeA zXUu~P4%)-{{+ zlN@O}5?r%OAc1?Cz#1(0sC zc4KKToYg7-F!>DugEbnd3%4>Y8^JT2Y_+2jfDjxOf|fDJgWGWvKl%%q zx;yWde9B3t<|p2|;wZPQ3)$k-SU1>=3)7joW|L zQ7jbZ5@Qb|TFbch64PQA$!A8C#8dv>ptT;Dl1a68?bh9R9f2n70gHt%9Va8LZeo)N z@a%H*@AtmaUVl4RtPz78cMb#G*ti?Q=$*4)T03+hw;{`O-4s1Z$l6VQf|WBM@|Z8zocUV?hKLGc!hOJaMu&s54|%(j zo!On&d3Zi-JeY?l4+j-@a!|3O6-&T_Ah!D$uVd!P`A1vwP>-;XvlnWGbF>6VIm_xb zVX+Wv)Z6^oOHg_3ZAj0~(LUl~x`bh2t4oE5=XadNdVy^--U#qO{Cg87nfcS#zqoxR zuCOgZ7lcMDQ(beYZk@r7z^a>lDIO7y1p!ayq5?mJ?ODo(hh@C18Jk;dS|=H*v2^%1 z;j^h@xWN{S<{NI2`z-m|SwscL6#B6`ebrvX1bqG0nopA}vQKzdu*FCH5MD%Qv;onN zFLIHdqg65%YxDL^(SFMx*4eo2JA79P@2axZJ-MkEyQXyYzp9u}Ia=do6xx^?F)ZHT zSIw*RY*fWp?zPT`>@Z8AXw7x|*V_>+o~7Y+a^yywU3bnmhtS@#G16b^O2U?+kPbyJ2>FeIQ*Fd41`pF((Nk)cZByDi`bPXvI-278x$*J};(YEQmb*s_oRt$wkA@P2wFW zSMaqkiz4S3W>p#;(SFD{I8XPj{Bi&13-jZrT0ekMb4b3tp;{=7B-HT&FaA@nX`6bw zP_~3MgrL}FnVgt)){xmkb}kmWBG2+(Og~x=wyLAbfzi~VBgOW1HN3_bsG9p!Sm0YR zfNHW%(Oy3#DMk0C&X=jGVKNT)I8N6n8Js|A%YYxk-r9}wX!%nBQ1utLyu`u&Q@Xe_0E+(H{ELz5&$LzWuA1+FnGgH6OmDu@S|7nbX;DLChem_2Hu|CgTmyN}+05HmMpil6uJrIPYtNxnZnJw1)B+An9QYsC$I@l;h zYQRDN=9;QhKjep>+i0$?!&nnV^r)+v2YCUpg_rQX!C}scz=Z$L3pkz3MgN!}N)Wo<{Dauhq->0Tx$1k4C(;iLMc8kpkJd z?UfBXNB#%z99t2UGv>07>4i0PQ2f}i&V~W@CNs5jGD&MhnUy@o9z#)Ol*@ zHsx;y$zW^rqot}}MNX6}C=-n?KB@R%8uNW8 z1QQ}~6>mD&#F^emy|tZ4ieDld#J0BR1pVjR$^1*7x6&;lXV%Y9W1$D$NBAu`D^;6X z&(MbGxff{EWQl7+(u5iK7Z#}J`=aya&`XOfxp4YB>nq=q zc9c`dKW3FE$E3q_9VTGA&HkoJx?N1ZiHdPG)JQUx*c*08-68Gv8Fk<*zk?q_Vp->H zD7D1?IwSZ?<9vC%uUo6}WV$DbSs{CFI5vkZ7k%#N5m@lxW|k<&is0ucWjmuhrKbiA zHrkC&4C07sG6#w9YC3)LycZ+mk?EuKqN{bqnK-#((B1;LN38Vf@k$6S?chPD!H#m2 zo79j9mhH^i6Zd|eXOHjYIASXXdb{Jz?+R2vNNnc$sMp&7D*TLFh8tD>P++UqQja}u zV;4$v#z$UcZT$s3WhI?14xXq^qlo-5VQZ@|jf`N^`m9KwfMT5zVba+x3SxmV{Flk7 zcfY>cUp%LQVFoT>KaiuG-B|f~PJ**)&G%)!qkON89$VfP2 zz@XEvH_n^LNt^ibMLR$m`hoE~{v|%MmfC65$oE%2MQOiDaeQ;uo!^jV?yQifCbNbj zItz;`0?#lD2VD*+Ky?hMR&4`@9cO`T$UoUObuRr9~F-#h>+^Trn#&qU~5 zH5jNg$WW8K4MY{4ve97|x;9Leb^!#wpd?wscqSpM6*ZpaAY9!2zZ%d>=Rm>b@BNo$ zN2INjJNbso^dUuC#GX-V9)fHUcC=+$oB%KJgNL;#+a*Q*NXp>iZ%#21C;t2-X~(8E z|Ejx6On%9^5bs6iq9|KT2c^T~-`QArcR<}~o*&=pb7aHGDN(y5DBK+IVnjb0)M%Y; z)@zGxx+Z_@_1leNmDZ|tuLVN$Y)}=gJv6`}^qcwDoWE<>W^u7dhKCJ+Hd({i^Q`Xo zEQ=S<4hjG`SGbw*@F5z(G#l~!y6{@ty@&$qxJOl^yn7#aU(tbz=QH&lF^m;;P;vk8nVUst+71hc!f>MUpNRRw$T)lVEjVr3TCNR z^HOeKyiXexhx+mCE%IA1E(Y;y6Am;yg1BO)BUa!P;a>dsVYBzn05Eq z`ckf&CU$1MH@!MPJNZs8NQuu!W@C_MqE*kR zl%g&w*-^r_upniHH|QmQ5{?@Tdu$C+1m~q@+B**RWsuYbNuclCd6zb!-T(L;9sH{< z3^cEldD;)|3JP3HB|Z`SrQ`YS4D=av`#yVfae4FjK4UGgk8*Qs= zKv`Hp-BcO+A&y&0;xddW9Pl0Qmq|gTleFjm7c1~9)=I(-*&L;mX}-lMI}seB>W_tQ zXOx6;cath|;MaEGArA5fgFf(x_Xqokq_Wj3A6%E&7coMi_6G1VeE7HRm+<)GyE~BI zLq0V*<&QcZdiUfca#8DV7bU6l-tUb@LYg(-FIIJ@!J6r!`8aQd?xAzp>slU~X;2klUOM1g*P> za7mLgT^m<<%BUNERKZ!Nix$CcR;jD=bJq<FFrWo^4M8 zg;hlsT^bXmT$Ylm{*v7fT5W%436KSIGO2t*|JsmkcPzdf-6;3lp8Ug_5Ty_oseP9W zYr9D>!WY#C2WTVLGBDgZd%=s#N`Sy?P!vg#7&Gs$N2Ar#bR3A8OMaemNnz6gW@4 zqUKSGS>AMb&#w3kvc+`!F$|CTwB{*D&~gZfYdPLtD&XLTrwIKU@c!QPnxeHX*R{%yiI&Y9V@|fFfAwR$C<=AbX4oC^WF1-zxjOIy61B~1*{dX~ zHv&4~Tw&s$O$`8ps z%!MJVRg-zAJpdSC7iV$)^iav}J>Kc@p)JkOP|HFCa#iHW`a~;JC@-&T0hp+h!zY{+u$Fy@b{0yM;J71Ohv!@o{Fx&i2Pqv|PAyNe+t5$6e z9K~5F3)3I%vf+ULBaj!6*2n0dffi}nCgUn5p62Q$9}2nX(Lq>KE4AoC*;aZhu+oVF z`!N;a*E@>DA4yk-ta){GWg9`J)&243d(?2HNFUle!LqnVyarm*=JHki-aH2^yA}sz z*B%0ML%Lg8D5c48fon#+_UK~|rY1b0=DyR2XYb4NGvmPi6bL7ciK;o9Z5loPNIeZ5 zvN42XtsB1d+qll!rA@U0|G88QN|N~xFZx{6bfpAdQ(o466&%~V8i*TmT8+5?`7;0E zei2gjmD>|0q`lkp;Yf&;t$jc>wsoTf@>iMmCxJDtY@`);4SL{8yb$xcxte|%r%Gkq6fJQV!jr98Gy3Q z3SU2=1mNJk{xn#C8k}gY_;(-S)XOPE^Lzj6s5JIVSb&c}XbYHl|0((cq*r8Cb_$7h z0dLAONV$+9t_~_?KGz*bxZ2C|M;e%X?mkteW*M3xJN-_*+C5mol77M2x!b?hod(u# z928^@H|_xagA5XYd_+Wo_rM7sh2Mzp(Zngg8}MG8@fzVG!mfXj&Fuu?DnFZ%ea~h0 z5q4DY-*xN?M-H?M9#AyQ)_21Y1&VSc$j~i)w=7@QoN&(gg}Pm7Hf37W>+TFr+sx6u zEGM+<-`&XSv;um_tODQm66me~DZG~hR`-|POM2}^C}r~R!ihu_$dl7=K+$v{F|lB9 zEHE&wvJMz6@UR+8f&o~;6%C$m$`ck3E?_XRO$-w0A6(9g*~`sRRKY$k1Ll{cH5AVX z{{oY^oW2u+0V{ly^LTp-$hF@3U^a+(Jp!X&2OiuX6!idwH-o2vk2HymO9u?-k(|L= zJ65+>p1=;gZ7;g|Nf{nGr$%&g&*%Q8GqqU~c5bD{dRV z6vxmw$mj3N^NEh3ZYQqQ2{_eE6TKtfTBdq4=w-%=%i?yY-DOXSiD0#1f4$x{rxuIb zJ5>%zj{}EdlAJd`hsMhwH`v{6iA~jQpp$8@kaq1JjT*;_%T`RMi9WqhiDjDt&^gmD zx4l^mxG!8Y2E4fy4(0>i(=5>0nxKCbd~mq<`D(dCYdcYO6M7{OF2Nj3Y4-J#stuev zi8C`VYuk5d*LL|MdzR;_t@lKGs)P5f&&uYij!^~KuCRN*mrQrGDim~_fmVa2Grf*} z0G!p^1M2@q6}W(d@Bd&uI?~N*2e0}&MhN!(RX@XG^OS~9lRUr`sF>MJ$mK160o8B^ zc8#lO%@XOx8^eX&2ddF~&_JlxTl#fpJT}~|@STh4f6Ex6`FAkf?sMVpefT%+_nB(J zIH4hVQNd@>>+9ct{S5<#`S(aP9)lhd(et;s;kZz&nmk#o7N3X)W#(1<>b55ML7gqn zt_1zl@7CA)A1@caP1cCt&QHD9V?#Ax`$yrXp1`xW^Na6JY2WF;gXwXmIpm_XW+PCb zLSue=#+w3u@#cYFuO?MbgpLp-n$=m;38lnRd)UT z26+9asF1ai0sH*(Py6{3+x)kZ*w)2T%%8j@7n~n- zZbhrWcwc!E6T7v+0P`X|%8Ums4p?Tyby)4JMMH*jkJuR%WpulQ=ZoYs85!9z z$I6u2(}7_=+H7y5ae2wcM0B#;RbVk%-}AJ-SkO9iqoesVl%0AoB7p|)ng#7wO(EqN zc?jj34~GRhbWMD6v*=GUo3!>Kp79pAvDeeXl}p2zv84J0-CX$H(OW{raChePm@yfres8J^*W{$1gcMpMZKUi=Xcp)>fG-6qtol?m$!zvRN}de-CkLxW`+xbIYl>5nq7r%Q-(oIgwv+;cJYaxJdFroCL8Hh5 zrK!#4zbAaSPm{k|qcLy4bXg>BEdx!-QWf;n4#$4c+#FoLZN$h5Q_P_c;&3d@yKz{1 zIQa43t?hP`QPuS2YCsy{4Rkos^F6Tx(+AoPFIrL~1@N?~+x7fMbQD~+H3Y{CvCQ-B zgchs_q#0otKuJ%hbd1%SL^W}x=$`MlcNw*SJlI^unqlnCL8Y+Y=az_eb*JvojU`ry zLqi)r|iFd%L|D|R}@h;OP?a+y-2Rn ztjw034!AknexVcCt8=y1zQi7njErXU_azUMb4NJiyOzig#B{RD+X3X?jDl6*gXW)6 zZKOS6c2VWXR+)Z|7{4~r#F$>R951#IqMp@etU!p;^PFcs)~+A@y{Bq8QRP|cU1!q+ zo7ZaZOY9FK*DKE3?`hgv=Xy)fN5i?+Q1U+C3}ZJ9>BI=o2Z)3eXRzBFm(#EDL!rF3 zx`@X0pjUq03h?+DP%?@=-+)$i?ON0|a&O`3;maE!7UdzSQvxu`lTsC7lPcUtswtMH z*Pa&D9n~I3C#n8q#4D~&jSK#(YsYVtnl-QP5{*P0i$!u@=;Eky8qa5c=592qeCzyI z2BJcn-WmFzgeP0hQR_TNUv#+_87^r+!N#0MQ7bGEY}HbLEKeeq4r{kV?O$FDkTqZi zk=edmcf~O$?Z}g`T3&IYCdhI4bEq`|4ch71SrekcJtxTjDyfE6u!tegR^UpIf>rKI zN#p#UFmo2h1zLj8TD>^XxxPKOU0iRplhVdJ!5r2-NT|Nz?I3w{Sdgk1)yyd6dR@Sv zPI*;r#C=9-HEI>{eFWV4V*C(#f6#yWx^eE`FIgD1hBkivZh1We>TEa6$^7kN%w${N z>%B1i&yqc30o`h424?4J_VbFX1#5#3P59q>-~r)X>8&4PG2p$TiQvww@!!deme(oL=^ z^m21gw@Ew?zY0fa@@b?ge~cIE^mM~;OJz25XCUF}ZG^EaW&agp=aH=L%4~6N?nWjM zz8t8Y9#7Ly7~qUR%XRlJoo^q)wbq&EhW4T!&ztR7Quv=1siybQHfwIe*sxDZH1`Yv zbUmC-pXK$XZ%l<9zOp0kxzE<=6d(A}C>peRpfK-dwRunXx^Z;pN!AAFmi`Vg$Z7_M z4zxvv1gjM?sQOMY-muYRrw%oz=J|~C{`K`sJKR(5+rWd}?c>2lERn_h4M%%-q$lf? zwYfWwwY{mS^B#9=8!o}E#!|S))b9xLMZNSjAxl4V6Mt^{-#l9U2q0%hCV0I)cw$kh zu2)7bxDvS~c07MOm{84+dQ=zteFePj#Rc^@?N1f2T05mk2>$f4lum?cH<3k6lbEBdirjT< zQUG>9#3E9x%(!ffrKNM=Rq|5Nnt>xZxZ?AK7lF^5Py_N5M_>N96lEygxK{@4B{t{@hIhP~ywU|9ZG5q?LajfB&Jz=SH`?U(**a4sU>opgh zTo(}{)D+eEx4TT{g9|d$?KxK9+o_nXF0(?BTV<@e;&|P9wS2yB-Xk&NS#Tdh((53o z-}`&v=z+Ji(L@Y<~dwo zt4G?W0al4Gp|8Q?1w58iNn%`0l$w>*+Iu!Maiuc>TF4~c&!oIi+9>WDH0C4$@9iJn zSua?!aVW-#H&UJ5y|Y=sUTaT$*uwX&g0!piERf`E@GL}Ld5kJ$5U$Mva`O2}3Ri~< zyD6dAG$c>iZf=%*uc}`l^&hEV3qxk=rPMDK(aMrlX&20p)wMQzVv{VhyQx2oFuW#W z(pL9iGU<`*V-q5t94KLQKN@gp-`61JpXQ}%kTuJQlMk7?!Lxbh&a^XCj|l$qb^kfX z`N{0l0ZTRY`936_L&t#fReYB9`8Pygs>(NwV~2|BAlO^LO%nRoE(~PthJtnY5y7v# zO`DslD<-9y39%>Wc6Nley*Ewvj#@Lzw*@xf4^=q4>pe`mzEPXn?GLE(mGaQLT7wVz z`IU3)AN#?SDc?;@;!SIY%nwsopO+nL7E!!!HAaRUg! zaY1nde}H^2NCyjFrN+?1OSk@m&qSRn`xx8NozR0wVPE^TQD!*YKi3(uyIwHy(w=ag zW3&okws)`t6Aj<}qwxeLUwV0PlyQ-N#9HIRck6WcuXlXWB8_|V>@iVD>1iGpAjSUM zcxHcaAKnW(7H<$Cm=NJLnECtLvkTZu2M)pW8F;&4|NfWhbD25R203US|C%mE^Hwf6 zfS2gc_4hh^{imh(>jv=q>F-|VuJIsp&XA9ao&njppArf+`vdFBYadGf9NtqW2o&T6 z4PBf6`-ZxQi_0CFp{+17-zDKKDer{Zlp|lc1W6s#1G+ry2-pEe^MwYAt^o}>?#IPP z8*1tXv1#TIvd(VgSdu1S)cQU=MsZhYAeT3yPCPAm^g5+_?f=KtgRE& zJ-yPu)X?ER5OVOCxld+KJG{UwKX45hjIy)8=$(F`U){o_uAN__#}&ZFz$;HomxG2S z!SU{OP`>3LP{@E9jZ9cdZ?;~iF6iQ&Nn>>tu6NRwBu{p=tx%NJtU&nvi|kLj%z#dy zOye`o8yVtRUO59cJz1-spa3R}IZl>K9K8(MCu+tz&v+}V{DiuKxmT~ku-&CPZ|Bib zwXP#JdmU+PZn5F2T3)^J)=hyx(#lguOQ&UyE}KHDh_x9Awx zj6m|5Qm$+L z>*b=9KV#w_3*%U@hB)95b06=9Gat2G@rsSgVS{B02tB2kukb~%!9eARahd0li$U>` z`WbwgK$L-3m-}@Al2LvyW@`7+7>{iGPfys*w#-@EF8PGQb`Va zI4m6Fz=~y9bW?m_gt2at zqJjXe=e{no$#Mg<=Q3FsNHf$(phwwrB44{{|Iw@c>&zfcs>HavWx~aPBBdK|Yscw% z9^qGurU7B+-iKLJp9+5tu98WGEt)i02_hQ5cx8Qlh}#nswI?8cTTq-^AX$OQ0Du(t z8na-@W0`mWkebwX@=#Sn3_aoY6fG=r21e~ITAvdJMg^4t1AcY~u>U^r68hGa@!CR~ zP2CAd$yIi9-_8E$Ov!cU%XOZQOpE=d;41ui%8inXEY-|K_`CBi_WNq5sj)%)>uN2x zVn!o#2mc?}Q?zq1k-rVYrtc0VHvI)7Pv|??p{={_{Clq>myYc@P*5$%mf5g8A%B{5 zC~p}>{z1EE43P!5vQsC8c*?Byl60?Sd(F#`so~_Sf|}Gx2HhWVI=iBVc*>)v`)?Oz zhi^GXN0x)8@0Jqe)A6&g8$oCAmrU*vQhLND9I2p3u#E9u>jtrG)(n>;p4=u&%sLr% zo&v0~UQ^KyJz5Kg-le#+*+e%9@&V@_hx9a+Sc~sfz!-1VGH&k;WgKkIxQ`b6v0DuU zjUa(EM_SBp7eH1X_b$wGkMZCMm0<`~90L6C5!E)*V(p+|o`Ip^K)>B*PRmLAez1##~)4k8TqRLU{hO1!;CJu;;#&5;l9BBw(Iqej%O_xDpjeY&c|2a z>($KTRYpeUS^&^gcQnCZ`Bpa>=np)Xgad!-)m(&BQI{)tHDdYUP5|msJpl zfRK<#K$u8jvRP~Ol!27|3?@gFcY(S;i89brM$qi5!e1ec)n&p*V+YDiE?ZdM^1lJJ z24Q$BrUymkyL>m~sxzj?l{5`>3`N$Z#?Mox4xppU74JkV0Q9f(SC4w1t<(TU9g4%= zd)`DlpcgHathgXAP>s^5Hmw~BmZ(8COd1-RxqvuO9xZPmtB*iImIo1eWGe2fbZSN% z4Y4ql#t)Y;(gKf&z}!tBfE7n7T6%&RU?fyXaCpW{ZTj?S(xPl6l>D1mA}o520w#@< zoEAf0OdFaRP(Vy{t>5#pE)cY(o(c3zrGxQ(Qjr^~`2>z06p;ZF3y`CfAAqbx-dmOZ zE$$H+hI`fjEv%~srdv%z{{ibg2OY1BiTlWT_2{0nl~)BL7cv>T-u!u_k0jeli9VL; zN3a78ei98R_n|woMDuHFjDLMBHsRg-Mu^I8FcbB$TyqnqU95kenorMrV!WwwK#wk$2 zo=;TE?LD&=5j_c0Bg)JRt3Gz$801qeBvhRgOlNN|W&wzs+&y7ZT>o!09ozp8P0M}e z{|}m`C>1XZYI_<)Kf}JEihp`_jO=H3SkKe5jY}HDrt_6wHO6QLUD^xz)_p3cY~}u zBcd5(t0FLKegw!ixU-L1QL%woa0zmzLTukt8`{Ju{#aPUmiUu^Z}=#W9{nzUsH@Gb zHH&QBT~^&q{&O1cKd8Cplzm=|ECQbxr(4B%6>F5NZE9?Z)TKKt4mUp#tae_p_aOoQ z`;=_lJFn&p33oJ6hEZ$b{qz*z*BQ!bh1$tqYx#RVv0@zSgNsy&LlNtA$DlGpy-#Hr z@l-hDA?kLs$kXHH8}4FtpQTB7z{LV^leRIH!1Sz`EVt~^$zg_7BUnV}96e;P^y9zi zvW&D5>IT5;&krScCfB^~W<1_e6X{Wg&?m751c~6h0I>@WEBMY^F!y5JYWO|J;zcZg zMydp)&^wt(WB0niA^*142>JEF=veI!U(Tg6BK2akf=@5@c{Txc!W_4Zu6ubGxk?F{ zp~=*s=_Uq<#)7By_(T5b0NU7EdGFZQ1|uMULJ^jEiCYq`%5UG;r!`tI#mO{RGYC*j zJ+B(a>WrPkNQSh3N*PL|;h$2rNTm*!h}`nHVqeJ&>afIpg-D6?b$yXT#8NeX0aQ?T zW`SWAfyNA*T1(crDx2k+HX*RoAxq9K;|6q;7Byoxj?ObgwfyHsP^&h#I7h<3Xa!DY&>);W2Cqs!#N(mrI_i#=oa zQfh_E3huuH9;V-NvG2xbyzBM3D}^z8F}r*2?QX z+|uz$*YiFJ{^O4^qt_bDW)Rn4z=!wkl4NEzDJ;h$KyWQ|xa3X4cz<~8$qNoLZY_KF z#R+;|%uRys1@TOs1Q1srpUF{}CBO%$cYFL7J^qzJxl0W`!B_vsWqE_jeO>Qzi@wc& zn9SpJ9l@`BeYRkHKwY^4&@+Ge*P-gzK1mY2elW~w#PoSH?c$OO$)cIgRz&BK(2O~mPRg- zV*u(nAQACW)x5)N^~+&xrctm{!=IEIHIx%OZqGXV9zRFQc#YXp|3h0*Ad z$z8?P$b-+(Xj-|J?{uEwO8>VFw!%z>f;Hdj)9T}^|FKA@lMC9nWEn6y)^7 zd>_APb1owF&fp#*%keH#0|Po9IlVRaTmHiEx;Al8;&I z{pppKDQ0*Ehoe^$k}0RD9!Mh!X1>CS^WM(aj&Vwp^Az7$jm@Sgt?4g3a~R2m+YYAU zsETVklI2FpEF%BlW&RS(VFj$DIun_)YG<7JM}Gq~doopZOkUu6C+M0b_cKq7otx6Q zZPs|&Gq`@`RqxtwbTaHb>3Pm>*!BfwEIsg;y9;@*GJ0e#n7{^J2d*E3teUK*C)_m& zI*XSvlJ`Z+f>`D(U4(3_s*ojkkR509U2!n_i&SG!6y#`nSh+J}ulW97;*v2L$dPa*L2yL{Fg2AhD{` zc4YCQ?g2ge-N$7*r&fs8O@ty*q6|?4%P0?06lBt4s<&P)C`@+>8I)~oDv{D1b;coo zafw3wy7`#IREoFPjiL~#x!tlk_RH}a=yLXRaj}8T9>pogOh`uGn)Fd*c>&?u!O5{P z`v0QdaLoUp-rfHP^&0*U>Yb_5FLXl8Wai}LKh%=`$9d=G(qzthw3ECOIH1dQiyXKB z4-})W2xTg=3C4Q~2IN~+)x_r?T^s{Sv<`z`bT zt9XO|qj)v{DPH%!m*dP@DH^87aOeq!6*Cz;`}Gwr9}n+p>Ug90?BFABgB|&@LEQj;0(7f zab5S|s+1#>3Op!Bf(xo373Ed^R_Bt7V<)B2Zv4lzkfv;OXFUwhMGay*yCz^<|DhS# z6V3K++J|>*xH&7UAUi(KX#{95`qEnb31yECq#ufg6qr zxwvFu1l^vyos-o`A}k$~<3Z%FQUPuLdrq12Y&B`UQge8oYC=G^$!bAcN-moGi99iA zd6iWtoqd{9c8K9qY?mgR8>R@?UksKvh8(qW_7b=cr814g9|7(ocg? zXhC*g`hOIcMWnVOTxgO{UM=ZwcR!|#WSHLZh=1!jo~R9y3X}@cm+cJlfNF{}y<4HT zA&m;hy$F1%E1d`8QC2IDox>n0e%JiD;ZJE1+21|m{5vT-y%}sjwQmsFf*>aSY}+;6 zlE5(c`&*5iY(2^d>+CBfy@(b(TO9C>?{a)g5zmndf4f*5jyenwT{R8{c)IC%tSA{z zkyZxlCSyHRA!fV`Vj$I;;aA|^A$>q{5Z0gkb94j(D|JNQ!t%J(OTjg~6g+AvmG}^c zQC6apBX zK{nYVI0-Iq;V+6+u(XIKay0SxqZQ|B!Ib{T*>*e!{I9dk(Kj*RJ1`r?9q{7Qz1}k| z$4YZrerjw$Xx*u2@MZ-33ZXIHD4#~EwakzE{t4CIMd;XmU2{m_wRi$$I=(l?^1rf{ zU*Laa?R7S`DcgU^+A&Qy`<>!oqguxe^RM;YY2C7c{AP)br}r4~{eTa!I}iuVg*C zO#%*KJK(K%Rxq%b--rOKkpdkMTa@e%@ literal 0 HcmV?d00001 diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 98a399c8b..4f57dd642 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -1,7 +1,7 @@ image: registry: registry.opensource.zalan.do repository: acid/postgres-operator - tag: v1.4.0 + tag: v1.5.0 pullPolicy: "IfNotPresent" # Optionally specify an array of imagePullSecrets. @@ -28,7 +28,7 @@ configGeneral: # Select if setup uses endpoints (default), or configmaps to manage leader (DCS=k8s) # kubernetes_use_configmaps: false # Spilo docker image - docker_image: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115 + docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p3 # max number of instances in Postgres cluster. -1 = no limit min_instances: -1 # min number of instances in Postgres cluster. -1 = no limit @@ -67,7 +67,7 @@ configKubernetes: # keya: valuea # keyb: valueb - # list of annotations propagated from cluster manifest to statefulset and deployment + # list of annotations propagated from cluster manifest to statefulset and deployment # downscaler_annotations: # - deployment-time # - downscaler/* @@ -214,7 +214,7 @@ configAwsOrGcp: # configure K8s cron job managed by the operator configLogicalBackup: # image for pods of the logical backup job (example runs pg_dumpall) - logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup" + logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:master-58" # S3 Access Key ID logical_backup_s3_access_key_id: "" # S3 bucket to store backup results @@ -265,7 +265,7 @@ configConnectionPooler: # db user for pooler to use connection_pooler_user: "pooler" # docker image - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-7" # max db connections the pooler should hold connection_pooler_max_db_connections: 60 # default pooling mode diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index cb8e29081..2a6a181f5 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -1,7 +1,7 @@ image: registry: registry.opensource.zalan.do repository: acid/postgres-operator - tag: v1.4.0 + tag: v1.5.0 pullPolicy: "IfNotPresent" # Optionally specify an array of imagePullSecrets. @@ -28,7 +28,7 @@ configGeneral: # Select if setup uses endpoints (default), or configmaps to manage leader (DCS=k8s) # kubernetes_use_configmaps: "false" # Spilo docker image - docker_image: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115 + docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p3 # max number of instances in Postgres cluster. -1 = no limit min_instances: "-1" # min number of instances in Postgres cluster. -1 = no limit @@ -203,7 +203,7 @@ configAwsOrGcp: # configure K8s cron job managed by the operator configLogicalBackup: # image for pods of the logical backup job (example runs pg_dumpall) - logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup" + logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:master-58" # S3 Access Key ID logical_backup_s3_access_key_id: "" # S3 bucket to store backup results @@ -257,7 +257,7 @@ configConnectionPooler: # db user for pooler to use connection_pooler_user: "pooler" # docker image - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-7" # max db connections the pooler should hold connection_pooler_max_db_connections: "60" # default pooling mode diff --git a/docs/index.md b/docs/index.md index 87b08deb2..d0b4e4940 100644 --- a/docs/index.md +++ b/docs/index.md @@ -37,9 +37,10 @@ in some overarching orchestration, like rolling updates to improve the user experience. Monitoring or tuning Postgres is not in scope of the operator in the current -state. Other tools like [ZMON](https://opensource.zalando.com/zmon/), -[Prometheus](https://prometheus.io/) or more Postgres specific options can be -used to complement it. +state. However, with globally configurable sidecars we provide enough +flexibility to complement it with other tools like [ZMON](https://opensource.zalando.com/zmon/), +[Prometheus](https://prometheus.io/) or more Postgres specific options. + ## Overview of involved entities @@ -70,12 +71,26 @@ Please, report any issues discovered to https://github.com/zalando/postgres-oper ## Talks -1. "Building your own PostgreSQL-as-a-Service on Kubernetes" talk by Alexander Kukushkin, KubeCon NA 2018: [video](https://www.youtube.com/watch?v=G8MnpkbhClc) | [slides](https://static.sched.com/hosted_files/kccna18/1d/Building%20your%20own%20PostgreSQL-as-a-Service%20on%20Kubernetes.pdf) +- "PostgreSQL on K8S at Zalando: Two years in production" talk by Alexander Kukushkin, FOSSDEM 2020: [video](https://fosdem.org/2020/schedule/event/postgresql_postgresql_on_k8s_at_zalando_two_years_in_production/) | [slides](https://fosdem.org/2020/schedule/event/postgresql_postgresql_on_k8s_at_zalando_two_years_in_production/attachments/slides/3883/export/events/attachments/postgresql_postgresql_on_k8s_at_zalando_two_years_in_production/slides/3883/PostgreSQL_on_K8s_at_Zalando_Two_years_in_production.pdf) -2. "PostgreSQL and Kubernetes: DBaaS without a vendor-lock" talk by Oleksii Kliukin, PostgreSQL Sessions 2018: [video](https://www.youtube.com/watch?v=q26U2rQcqMw) | [slides](https://speakerdeck.com/alexeyklyukin/postgresql-and-kubernetes-dbaas-without-a-vendor-lock) +- "Postgres as a Service at Zalando" talk by Jan Mußler, DevOpsDays Poznań 2019: [video](https://www.youtube.com/watch?v=FiWS5m72XI8) -3. "PostgreSQL High Availability on Kubernetes with Patroni" talk by Oleksii Kliukin, Atmosphere 2018: [video](https://www.youtube.com/watch?v=cFlwQOPPkeg) | [slides](https://speakerdeck.com/alexeyklyukin/postgresql-high-availability-on-kubernetes-with-patroni) +- "Building your own PostgreSQL-as-a-Service on Kubernetes" talk by Alexander Kukushkin, KubeCon NA 2018: [video](https://www.youtube.com/watch?v=G8MnpkbhClc) | [slides](https://static.sched.com/hosted_files/kccna18/1d/Building%20your%20own%20PostgreSQL-as-a-Service%20on%20Kubernetes.pdf) -4. "Blue elephant on-demand: Postgres + Kubernetes" talk by Oleksii Kliukin and Jan Mussler, FOSDEM 2018: [video](https://fosdem.org/2018/schedule/event/blue_elephant_on_demand_postgres_kubernetes/) | [slides (pdf)](https://www.postgresql.eu/events/fosdem2018/sessions/session/1735/slides/59/FOSDEM%202018_%20Blue_Elephant_On_Demand.pdf) +- "PostgreSQL and Kubernetes: DBaaS without a vendor-lock" talk by Oleksii Kliukin, PostgreSQL Sessions 2018: [video](https://www.youtube.com/watch?v=q26U2rQcqMw) | [slides](https://speakerdeck.com/alexeyklyukin/postgresql-and-kubernetes-dbaas-without-a-vendor-lock) -5. "Kube-Native Postgres" talk by Josh Berkus, KubeCon 2017: [video](https://www.youtube.com/watch?v=Zn1vd7sQ_bc) +- "PostgreSQL High Availability on Kubernetes with Patroni" talk by Oleksii Kliukin, Atmosphere 2018: [video](https://www.youtube.com/watch?v=cFlwQOPPkeg) | [slides](https://speakerdeck.com/alexeyklyukin/postgresql-high-availability-on-kubernetes-with-patroni) + +- "Blue elephant on-demand: Postgres + Kubernetes" talk by Oleksii Kliukin and Jan Mussler, FOSDEM 2018: [video](https://fosdem.org/2018/schedule/event/blue_elephant_on_demand_postgres_kubernetes/) | [slides (pdf)](https://www.postgresql.eu/events/fosdem2018/sessions/session/1735/slides/59/FOSDEM%202018_%20Blue_Elephant_On_Demand.pdf) + +- "Kube-Native Postgres" talk by Josh Berkus, KubeCon 2017: [video](https://www.youtube.com/watch?v=Zn1vd7sQ_bc) + +## Posts + +- "How to set up continuous backups and monitoring" by Pål Kristensen on [GitHub](https://github.com/zalando/postgres-operator/issues/858#issuecomment-608136253), Mar. 2020. + +- "Postgres on Kubernetes with the Zalando operator" by Vito Botta on [has_many :code](https://vitobotta.com/2020/02/05/postgres-kubernetes-zalando-operator/), Feb. 2020. + +- "Running PostgreSQL in Google Kubernetes Engine" by Kenneth Rørvik on [Repill Linpro](https://www.redpill-linpro.com/techblog/2019/09/28/postgres-in-kubernetes.html), Sep. 2019. + +- "Zalando Postgres Operator: One Year Later" by Sergey Dudoladov on [Open Source Zalando](https://opensource.zalando.com/blog/2018/11/postgres-operator/), Nov. 2018 diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index d436695e8..e626d6b26 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -7,7 +7,7 @@ metadata: # annotations: # "acid.zalan.do/controller": "second-operator" spec: - dockerImage: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115 + dockerImage: registry.opensource.zalan.do/acid/spilo-12:1.6-p3 teamId: "acid" numberOfInstances: 2 users: # Application/Robot users diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 0a740e198..4314b41d3 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -29,7 +29,7 @@ data: # default_cpu_request: 100m # default_memory_limit: 500Mi # default_memory_request: 100Mi - docker_image: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115 + docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p3 # downscaler_annotations: "deployment-time,downscaler/*" # enable_admin_role_for_users: "true" # enable_crd_validation: "true" diff --git a/manifests/postgres-operator.yaml b/manifests/postgres-operator.yaml index 4b254822c..e7a604a2d 100644 --- a/manifests/postgres-operator.yaml +++ b/manifests/postgres-operator.yaml @@ -15,7 +15,7 @@ spec: serviceAccountName: postgres-operator containers: - name: postgres-operator - image: registry.opensource.zalan.do/acid/postgres-operator:v1.4.0 + image: registry.opensource.zalan.do/acid/postgres-operator:v1.5.0 imagePullPolicy: IfNotPresent resources: requests: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 9ae1b3b26..049e917f6 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -3,7 +3,7 @@ kind: OperatorConfiguration metadata: name: postgresql-operator-default-configuration configuration: - docker_image: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115 + docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p3 # enable_crd_validation: true # enable_lazy_spilo_upgrade: false # enable_shm_volume: true @@ -92,7 +92,7 @@ configuration: # log_s3_bucket: "" # wal_s3_bucket: "" logical_backup: - logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup" + logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:master-58" # logical_backup_s3_access_key_id: "" logical_backup_s3_bucket: "my-bucket-url" # logical_backup_s3_endpoint: "" diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 389240c09..41d701fe2 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -37,7 +37,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.EnableLazySpiloUpgrade = fromCRD.EnableLazySpiloUpgrade result.EtcdHost = fromCRD.EtcdHost result.KubernetesUseConfigMaps = fromCRD.KubernetesUseConfigMaps - result.DockerImage = util.Coalesce(fromCRD.DockerImage, "registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115") + result.DockerImage = util.Coalesce(fromCRD.DockerImage, "registry.opensource.zalan.do/acid/spilo-12:1.6-p3") result.Workers = fromCRD.Workers result.MinInstances = fromCRD.MinInstances result.MaxInstances = fromCRD.MaxInstances diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index d8c92ba3e..348452193 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -112,7 +112,7 @@ type Config struct { WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to' KubernetesUseConfigMaps bool `name:"kubernetes_use_configmaps" default:"false"` EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS - DockerImage string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115"` + DockerImage string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-12:1.6-p3"` // deprecated in favour of SidecarContainers SidecarImages map[string]string `name:"sidecar_docker_images"` SidecarContainers []v1.Container `name:"sidecars"` diff --git a/ui/manifests/deployment.yaml b/ui/manifests/deployment.yaml index ccaecd312..8da564322 100644 --- a/ui/manifests/deployment.yaml +++ b/ui/manifests/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: postgres-operator-ui containers: - name: "service" - image: registry.opensource.zalan.do/acid/postgres-operator-ui:v1.4.0 + image: registry.opensource.zalan.do/acid/postgres-operator-ui:v1.5.0 ports: - containerPort: 8081 protocol: "TCP" From 62bde6faa24b75973781d012983eafdb9a6dd863 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 8 May 2020 13:02:27 +0200 Subject: [PATCH 045/168] fix env var in UI chart (#967) * fix env var in UI chart * re-include 1.4.0 in helm chart index * fix import in UI main.py and updated images --- charts/postgres-operator-ui/index.yaml | 8 ++++---- .../postgres-operator-ui-1.5.0.tgz | Bin 3786 -> 3799 bytes .../templates/clusterrole.yaml | 1 + .../templates/deployment.yaml | 6 +++--- charts/postgres-operator-ui/values.yaml | 2 +- ui/manifests/deployment.yaml | 2 +- ui/operator_ui/main.py | 1 + 7 files changed, 11 insertions(+), 9 deletions(-) diff --git a/charts/postgres-operator-ui/index.yaml b/charts/postgres-operator-ui/index.yaml index 114e6a4d7..2da53497d 100644 --- a/charts/postgres-operator-ui/index.yaml +++ b/charts/postgres-operator-ui/index.yaml @@ -3,10 +3,10 @@ entries: postgres-operator-ui: - apiVersion: v1 appVersion: 1.5.0 - created: "2020-05-04T16:36:04.770110276+02:00" + created: "2020-05-08T12:07:31.651762139+02:00" description: Postgres Operator UI provides a graphical interface for a convenient database-as-a-service user experience - digest: ff373185f9d125f918935b226eaed0a245cc4dd561f884424d92f094b279afe9 + digest: d7a36de8a3716f7b7954e2e51ed07863ea764dbb129a2fd3ac6a453f9e98a115 home: https://github.com/zalando/postgres-operator keywords: - postgres @@ -26,7 +26,7 @@ entries: version: 1.5.0 - apiVersion: v1 appVersion: 1.4.0 - created: "2020-05-04T16:36:04.769604808+02:00" + created: "2020-05-08T12:07:31.651223951+02:00" description: Postgres Operator UI provides a graphical interface for a convenient database-as-a-service user experience digest: 00e0eff7056d56467cd5c975657fbb76c8d01accd25a4b7aca81bc42aeac961d @@ -49,4 +49,4 @@ entries: urls: - postgres-operator-ui-1.4.0.tgz version: 1.4.0 -generated: "2020-05-04T16:36:04.768922456+02:00" +generated: "2020-05-08T12:07:31.650495247+02:00" diff --git a/charts/postgres-operator-ui/postgres-operator-ui-1.5.0.tgz b/charts/postgres-operator-ui/postgres-operator-ui-1.5.0.tgz index 6d64ee3b55582f3420da6c1d78ad883affe7a09e..4443e15bc739899f370f7c71a2d45e78172c09a1 100644 GIT binary patch delta 3770 zcmV;r4n^_G9oHR@L4WXk=CA0bpRJn74k?n>n^~3T*5i0nu1y@5?PP0jYsv+YEeT@^ zU;t2#6dNz5QVS2Y>J$rS~l*lZuERysz%7 zJ-I(fAt`-_LQ>8`nEMt<)9R=1_FT`3QAUEM%Dg%&xCP!6+yZY;AQgO0W0XK(B1k%= z5lMhDgqs zm0_ndp=z3q+<%DAIv+_wSj;;sylY+K?F|>P3@r!BTFZf|-hqr-4nzshVuuk$=cplx z5}fG@nbL$?GeVgnl%X()a7JhnY9vhZOhowiVgPQ8762o9nE#!>vhrW+c9oSkHYgu7 zsd;lj9(VoM?*BPSGL(-d0JiM^eSgqv@BdzZxBs7{?0>-v9Fr_jkT1H~uIre>wf5k0 zN+nRe(f@t;`js>0Vn&q07{)X~4e$b^gb0K=5tNJ)l%P0_5K`h=MuE=A1TFJ7v;YK7s8r%Mte)KE)^yD_a${}) zQ!c6E8h?~TG+y0WdRGD{GSRY6Z!RaMLfu*bX_h2sJfYET2=>YNl&dp=5}C3sON>aQ z2eJnjDMoZmQNqm>)f5Gs|8jT)0*kE}zL))a^Y5{0n^#Zw4MWN2B1@H>nmGyM%{ z30ySbwjl@g1_s6I2q=jrQb2^{AK6`V1d3bsm!oQ|8!Aeg+qzxcLiMJ=^4}m~q zWPdTH`VUh-gDSRzromF*jZobnGSkD`6N99}dw6!@!U>ZolxbC}zWOTFfnkaQ|6)P) z@-U@uMG}IoHAvW5W$AF}4L!To;wZ_aLUBrF_==2Bw~g{wj_%bq1TEI#9hG#HU3M6| zk_IJRsDx7<;~CcrU|K>%m}Yxf?2$SjvJ!MmF^TKAn>8A(;r0^8&%g!HF;E3lqU&;D zs8<*n5D;23&+`7M)Vn6f&bTsYL)Z^`fxki#%JkYu*K9Au7~>FZWLh%D`98I+0)IwR ztEw4m>RSMWF|LR$kp4P_mFNVPGmT3&@Qk6o*Pb0Ees zWuM>0&JxtPo^Bp+)KUZ7cX{wZL%x}#5Hv>kq>UvOL4V&o(%D5e9@7tCJ4yJc1ubw}vn;PA@d9wobLB`IBp%j6viQGDi||6K9fzi;??S3T|kUXd74C znLxC=tp{I>_I`9OU9u&4=(r+nSy_>{gxLsiFZw5PiRu5DxrEQ}zkkHFaaLr>u{5;G za;%NF4<2ADC{tsw|0g3K#q#OWKSSoYoe%BJz)vMM)NK zXA1YHY7v@f&6>dHCQ)NmEe(p`q-ImYt_APsP>FUp?>j%!#NN0-b^ghpi$FDe0)c6w zjSc&6HrQA8gI4Za3xD8mT(;={B2*xFf?pmE-lG5gUcVLp4ZMEP-|7FSD0g>t%^*f3 za*N>KVqw(}hfEd8)RYVQ(ZnR&>!C5==e~7KSsX$U<~ipHS~FB6CW?gB35o9Mv*Kx; zfYbKA1-T$8A{FaB2!sFLE`&ie#boBnX~zhj`?N14RE1!5B!4r)$OL0&bZhWcTBV4* zZf;1KRD?y?Y<5}*W058Kxe-a}OTn|$kis!h?PkQg7EoQDfW0>%nK_D4QIW5(d5kf^ z;&Fn?{7uMST|T1fNihqHEwmcg80TDz8AKFDL#jBl`Ls0S24wyCj54W zOvxOxA_~V@B7adm2>ZHGR$lPuFx}l5rSpC1ocqiF5oIxDlP^#M+vNYg*NXoHf#36X z^8YEy?_qC=1P& z=S`c!ir=R={lk?l@_(r`zakrCi~R2e!G7ER@9yvH|9>YbbrG-arlmB0Q7wH#ivP)F za80^x8DcHkV&rCuQJ%hHvRz~dz6BsL!ANnD({rZn;#V~SKO%|0WU6Hk#pr5Wpi9yi z=%WtxO9q=4g=7|jtvw?f_TY>NGDC%ee#HMWK;=4^=M@d~b!yAxqwrdt< zYgaXpXMZ<^VCzX0R+xF^mf20PSajA@mbp!^ScEIpicDa!Sn1}hmesV*K3ehrb*@w4-)T(#9^&k1yW5 zJ%2wszPNgKa&hv@tK&^nYWLJmX=?3;S{Ijx=P!>h&0H?d4v#kV(C(m>7qxK)m&b>% zFE-%(_q%ny$2^i(31!!IXqS1*D}vG0 zn_5j#18ZT8tYZvF)vGcL=vq!#C_T^qOK-H;SIU1_hWLh);1_K`onvwnDjotnM83fz-83$ zsyLjHcDR4OKDx>Z>;e))f~9}%~ics7H%`KE<1PbNy`@hugos|s!sv7<$r(d z?|UoxA3=BL|2;)%r3RGiU8JHe&-1l6b6T9~-}{t#*<8*elYbjNZy!9nCabn~`Y&-d zM{&ewDG>i@0nGs!C|Ft&eecj@7%*5dnywA9**@Ct4 zs4|;d_o1Iwhi2T0Y65wJclEQ@i>YZ^w|`FIj+r^P z?A{THu93xU|E<{TryF+28qnOY)S#v4s6jSpvz}((TCV<4$`<`!3hCcd|NC99{r&G? z;QKrM{}iS5{cqz$XBYQ<#JF!QjOG*(t6EH^>{>m^HkXlYxlpa z=|m@t3w(?|w(b93&|kU#A=rKY^CV>t&WKVdn7OJUUvs#bB7;#zlYf}938W;tCKHs^ z4G=O*jWq*uib(>Kgpcy0Ysw}+gTTaGtYW;x+IzxcYY!MsjDHV5rUJ+G1IBr1;4eS9 z@P;L~z?o^OhXg4KNGL%PFzB`8zrFpz&i{Lo(o*cm zl%RR2X&q;j`PGd!g?ek<8>QbCcawx*2VUSgUeED{m%bkcL4VjCx(D81c+l_n{l9rZ z=z00^(C=~Gl)-ziE2j)3ouE`947y%$FdPiJ-2*=w9r)v5)Ef_nez&*p6EckYBY#BV zcswMrf6yhP;9%q*_|dQ*ce`FR*v}1)hu>kPn=S9LD*eUcF01@!1J>MWl|MA@wW>mO z=5DL}SAM@0n156w+RRhgde4>d6B~D3E#GXoSqGqzLh-uzJfFOwu`hNW9VAiUcy#BkS*A7F~1~ZrRI_rKXoZVf|*_vUiYnV2& kYk%N-{y`;MzqOv(m0j7D@2~uC00030|6@Qfpa50?0FnrN&j0`b delta 3757 zcmV;e4pQ;g9m*Y$L4R;Q^H=oJ&sNQ3LyFX!ZC2&E^*G*?YZFIhJK5UXnsPy8OTw4} z7yy**IJ)0{1@J{AB}H=LaWXf;2b&@q4WQBJH#CuOsm21O!xI!lap9!2bvz+Lxp!n5 zKYNVgd7jtrcFlj!^P2y?e!u<9@AP|Juif>!ooAlk>-gR?@P8hq_YEbLiil_4H}}wuIZ)O+kW$NmFy?9GFrw%TH6&4j zGhHDQ8gpw(C{u(o6b2Da35^4dgh`%?5dWSJz>UxXU_=k|zq40X_G{Iyva-eo<%1?Q zt1rmouK&vYKO=F9^3epqhW+31`}@uP-|P9i{r?nY4}V_ah@`QCY|+hjUBwixwFj3I zDuL>a{_mr=Z=4YqQ=$|`FrqPPfL9pCL?FzFpkx@M1jS*9kPs;`(n5j59Zv-)oF*|* zC|%1En8Y+B$2?^!0N=9qw6qw}aVii}3Uo@wXqmr(1t4%tr4n~R`Q$FPrfas58*u}e za7h){pnoK!(el>PyAnW=v6g*ia{(l291nO*!@B_N)6p4M1A!8mvMoytNvMag2Nwy3 zbVO0Y%>>m11)TqKbPNLXO%8Ix(>T&|4+Rp1kp=lv07|52S-IFd3PUsTHE0Q3)ZjKF z)_5)_Xyg*`5vU0^aVV~Zt!qF}N5HrOi3$jda(`5Mku|axKr%d3!WfyR8b**x%Ekuo zD2-!NU#w>#59^Dll?0C#)UnDZJ?)rIsWMLhVUh;W_PnY2HN~k}y|(AQrDjDxrYLK$ zeiPO=u+8pI&OS&B*-U11K7BiQcYb+vb{W9GCN#!ULqMoaA47@vr^OF}KqF)kCi)Lk zFMmMg8$r`xq3MRGZV;L2;r*#WQsO-toVswzBno9(6-uwVN>yN(puoRaP`*4$=z9?d zU~AnGw3eAUIPeZUyVBw~PNhO|MyB|N3{kfYvsaGp)iwmp*Wm+|bQog*_CH80O`RfQ zj7qnb2?=#olt~JEiMsh@3YOM1R(FYRl7Elkbf7UNT&Mu-1Mk4M3*OW7*02Q)3c64U zXFS3I*9%};LP(fqdztT%Dj%`}bVM0>=yB0_X^+f(g-enHbb7j0^|} zt&^u&|5WN-lVN9E8MFa(+P${FL=npL+DO-IFT@C=0BmGhCPvvlwXGaRQ>&~QX@BZl z0E98Fh&H-p0QYml;d^kM4pA`FVw3V#(MK2)DFa|*%5GNzs#rQC)OLxT^$alKuoNi^EDMZ$2~`Y^(IiGcyu7~$_Z(v+65Ud3z`Sf2nNP|5p=?6BN=_$=ADr8}@&@>-o+7-+$ll?)Lvv zl>7VEbC}U-U_1cr$7yTq1F{(Ej7UXkOdz!i8JGoB#D`ZYk&OHnFz1TVK=;>tKnvVUY)YFcG6*4o<# z4=@pwsS((JmCj$K-N2Uh`w*sE_uOE{!n+~4BmR&jt;Y+_DvMhZ`6tT4IE^+lh5K{4 z2z9h(P2fwDsIaP*26=E&v#DX%ocA-RSi7Cu&d)Tl*Dg?%f3oL1Pz|3!U=nL%!~UBM z_LcphmHXZT_&b*k`hPzU7ziHY*N1~Q=zqW0?bY>v+w1$g`0taH`}?Y95JM8Wc?fVm zx2lIjri!F$!Ug?gA`|ZQfid7`zI9Dm6hI#AIp;B2Q&c1(iUj2eiSFr(;%SwD)Aqgv znIOsI6{|f6gMVum!XTPpJay%yWdzSQ?Q;oLB3K^DlrS>J$bT8$8GNNy$>Xo<8&W0} zVIkI=o#nz}+zA=cm!j?0|moP4qB}wE8Oo}KRrLjcyAZ)8f zReHT&!gPOclz+^Rtz)*A|3k_m%En)z1~$q6j@O9)wA+5Cx0C-*QGO@+|4YU!J2mjA zYJkUhjw3z)LPmXIT`YiQwfya?Ak*P*7%HQVGW_+!#Yb6awm5ZK7gqdZh}R#!Y>@vE zCNaO8BKwwfunqFR*KRM_|K0XZ{y#~nig;TaPOX`5=YR6tepRl%U6bU0ayeX)a2tkL ziMSZqnP8Zuu$YV&8GvsANQ^O5Tx9f|YTNitg}{$U;;)%%(L+AE3K!@S*9Q8iL;ae; zWHYHki`WWyc|h#*r`DCj5rF9TGpgLz)kLf@vkOlAsSWps5PXPr%oC-O*QkOfGNL8{*sTeYSWWC^ z?dma5MinLTD~!qA1%{kOGJswUMA0eoRF$=UZGV^*w{KDI?;TM5-((suEax)w>oYKF z(uBZo*QtnX)}&$16Gwx=)%)`|%}JSDlts!<%tI2JWQ`()W(BHugOl^4%XjBjM}yNZ z;H)e?vYB-2X-4&Nufl%(=KaOx$@$gU(c6=&H%GsmJhY=`cGB7|&rdGiy+1!bxw!gp zdVg{H%bSxeJ8I@Ot?l6Q==}A`rJ2da;OKZ;{Ehsm_2^HI-d?Q1`Ppf_2l&s@8^}f3 zQpK+u*697|)$zNtf1JMFzW4i8y+=HhS21PRc3>Bo%S&R>)mvIkPy=gGjjUx%NY%1m ztVGJ>RhB?{C6%CTY-<7N&z*a3?%ucOYkykhnd9UP>uL_&{`#ukG7C4ESeKnc_k?AG|5xM~e$%G_oAN(89j}@HvA;Y2d79El4Sy)s zJ5NYmoabv#<}5$c-};n!(Ok|#lZ6|+Y#uzjCbzb7`Y&-hLvhTf2@weP|9|Pbe%JAA zaF0nshBT&%qICb64l(y!-PcqVhrv!GY{^jFaB)r9coD7~q9T4hC7{{_Zxu2lHS4fh zuqktmPcu5Lj&ClAwToKnHkx`jRDVsDxU7=poIzI8o=s;}OS@uOcNd8BWyxH+HQcP? z9KXueL4ygx#<(7`$crv5|4JM4zH09|X5#P}KBin%EWyh7vKQ5iY29S;duc+ow)p|z zmquP2RFTuI`_NCzLo+T#Ie{$AyZl+{#njYoTPJtL%$yrm?~sJo$fBn2mVfW{vo$+p z1!(3|YS2P-R3K}#SxvL=Em!|2WrO}Ng!J#J|NXAl{Qh^pzu({K|EDO8?|*A2I=h(f zBgT9yAv7nCR#hSjHDjohbXkaDDZG|zIl?0N2|q63FrQP|#n)5zQmUU@eiK!8RJ>U^ zre*LYy|xJST^#*dYJCezzh2 z+kU&h`~K%iN#6Qrpoii7`MuE!(lo2SZdSh@dQO(!~LT;OB$v1$MJ+Wn>b zAN<|-KTlHjU_g{Y!OT?+*_y-61Q`rd8b_3kAtB*48KW$3fRJfotbZAh6O3aR$9$L_ zTT?cE0Rm%lv5N5$EAI)5tUX{jHvT>Qln5NrTa2>Mz+ZlL;T?u8wFk$1I^_&L9A7|0g|yr;RW0*38@}ZZKZ%z4TfCTzTl!D&Q_fgxfk^wT zX=1*8lGbxq-Xzv@cYjE(t>>%Pd%wGM|4XO8+y75e8j2m65;O}nt>TO#zq-~YS8t7b zqx9SShLQm6w%7I?uj_QYOaCC~dO_#V^*yiGIrQ6yfAiXb=YM5~L%+v$QwHx=S56s7 zI!37iIK%_e3)|r^>V${Ac0>-m(Cds2{QhCL(;InhG8|#QPkNmZX^(nd)bI87!`^-; zI_!~QL^6Zp;dfZ+W{Z2Q3V$)b%PRX>gEeG`t;$fHx!WrHmECUzCe4V}b5u6o zb7lO*+Fe(RH-Bqx)&Z!cPdxD6E8Qd&@e&cLoj8WsTs8h=fM!h|+=5xVTQRG5czD?9 zc6&c2uX-MOI2!GHy?)f|cl-Nc)C&*yd;Q-2XgJ*OhX;}8lQ4|hJ<=WSlaL(v!?4>X zNW#OuAN@IawQMG@|1RmZ!%#KBCB06&KL}%YS2MQe*Cz5VruE#~?sof!{eoS;x0>0N XUD=f%u>5ZT00960EC=W;09F717N1$~ diff --git a/charts/postgres-operator-ui/templates/clusterrole.yaml b/charts/postgres-operator-ui/templates/clusterrole.yaml index 4f76400ec..57a1a6365 100644 --- a/charts/postgres-operator-ui/templates/clusterrole.yaml +++ b/charts/postgres-operator-ui/templates/clusterrole.yaml @@ -38,6 +38,7 @@ rules: - apiGroups: - apps resources: + - deployments - statefulsets verbs: - get diff --git a/charts/postgres-operator-ui/templates/deployment.yaml b/charts/postgres-operator-ui/templates/deployment.yaml index 6247ec933..00610c799 100644 --- a/charts/postgres-operator-ui/templates/deployment.yaml +++ b/charts/postgres-operator-ui/templates/deployment.yaml @@ -1,5 +1,5 @@ -apiVersion: "apps/v1" -kind: "Deployment" +apiVersion: apps/v1 +kind: Deployment metadata: labels: app.kubernetes.io/name: {{ template "postgres-operator-ui.name" . }} @@ -44,7 +44,7 @@ spec: - name: "OPERATOR_CLUSTER_NAME_LABEL" value: {{ .Values.envs.operatorClusterNameLabel }} - name: "RESOURCES_VISIBLE" - value: {{ .Values.envs.resourcesVisible }} + value: "{{ .Values.envs.resourcesVisible }}" - name: "TARGET_NAMESPACE" value: {{ .Values.envs.targetNamespace }} - name: "TEAMS" diff --git a/charts/postgres-operator-ui/values.yaml b/charts/postgres-operator-ui/values.yaml index 90e9daa66..2fdb8f894 100644 --- a/charts/postgres-operator-ui/values.yaml +++ b/charts/postgres-operator-ui/values.yaml @@ -8,7 +8,7 @@ replicaCount: 1 image: registry: registry.opensource.zalan.do repository: acid/postgres-operator-ui - tag: v1.5.0 + tag: v1.5.0-dirty pullPolicy: "IfNotPresent" rbac: diff --git a/ui/manifests/deployment.yaml b/ui/manifests/deployment.yaml index 8da564322..4161b4fc1 100644 --- a/ui/manifests/deployment.yaml +++ b/ui/manifests/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: postgres-operator-ui containers: - name: "service" - image: registry.opensource.zalan.do/acid/postgres-operator-ui:v1.5.0 + image: registry.opensource.zalan.do/acid/postgres-operator-ui:v1.5.0-dirty ports: - containerPort: 8081 protocol: "TCP" diff --git a/ui/operator_ui/main.py b/ui/operator_ui/main.py index a294ae081..dc2450b9f 100644 --- a/ui/operator_ui/main.py +++ b/ui/operator_ui/main.py @@ -7,6 +7,7 @@ gevent.monkey.patch_all() import requests import tokens +import sys from backoff import expo, on_exception from click import ParamType, command, echo, option From a5bb8d913c149e6b16d43e910f52bc9e52ffdba1 Mon Sep 17 00:00:00 2001 From: Damiano Albani Date: Tue, 12 May 2020 09:20:09 +0200 Subject: [PATCH 046/168] Fix typo (#965) --- pkg/util/retryutil/retry_util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/util/retryutil/retry_util.go b/pkg/util/retryutil/retry_util.go index f8b61fc39..868ba6e98 100644 --- a/pkg/util/retryutil/retry_util.go +++ b/pkg/util/retryutil/retry_util.go @@ -27,7 +27,7 @@ func (t *Ticker) Tick() { <-t.ticker.C } func Retry(interval time.Duration, timeout time.Duration, f func() (bool, error)) error { //TODO: make the retry exponential if timeout < interval { - return fmt.Errorf("timout(%s) should be greater than interval(%v)", timeout, interval) + return fmt.Errorf("timeout(%s) should be greater than interval(%v)", timeout, interval) } tick := &Ticker{time.NewTicker(interval)} return RetryWorker(interval, timeout, tick, f) From 852f29274ab0fa9ce79354798393fe7ab9176a05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ask=20Bj=C3=B8rn=20Hansen?= Date: Tue, 12 May 2020 01:05:42 -0700 Subject: [PATCH 047/168] Fix typo in error message (#969) --- pkg/cluster/pod.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cluster/pod.go b/pkg/cluster/pod.go index a734e4835..44b2222e0 100644 --- a/pkg/cluster/pod.go +++ b/pkg/cluster/pod.go @@ -331,7 +331,7 @@ func (c *Cluster) recreatePods() error { c.logger.Infof("there are %d pods in the cluster to recreate", len(pods.Items)) if !c.isSafeToRecreatePods(pods) { - return fmt.Errorf("postpone pod recreation until next Sync: recreation is unsafe because pods are being initilalized") + return fmt.Errorf("postpone pod recreation until next Sync: recreation is unsafe because pods are being initialized") } var ( From 8ff7658ed3caf1d93605a8310bda309d1e622753 Mon Sep 17 00:00:00 2001 From: Christian Rohmann Date: Wed, 13 May 2020 14:55:54 +0200 Subject: [PATCH 048/168] Fix pooler delete (#960) deleteConnectionPooler function incorrectly checks that the delete api response is ResourceNotFound. Looks like the only consequence is a confusing log message, but obviously it's wrong. Remove negation, since having ResourceNotFound as error is the good case. Co-authored-by: Christian Rohmann --- pkg/cluster/cluster.go | 2 +- pkg/cluster/resources.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 31b8fa155..de6578f69 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -1345,7 +1345,7 @@ func (c *Cluster) deleteClusterObject( objType, namespacedName) if err = del(name); err != nil { - return fmt.Errorf("could not Patroni delete cluster object %q with name %q: %v", + return fmt.Errorf("could not delete Patroni cluster object %q with name %q: %v", objType, namespacedName, err) } diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index 3528c46f4..5c35058c2 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -191,7 +191,7 @@ func (c *Cluster) deleteConnectionPooler() (err error) { Deployments(c.Namespace). Delete(context.TODO(), deploymentName, options) - if !k8sutil.ResourceNotFound(err) { + if k8sutil.ResourceNotFound(err) { c.logger.Debugf("Connection pooler deployment was already deleted") } else if err != nil { return fmt.Errorf("could not delete deployment: %v", err) @@ -213,7 +213,7 @@ func (c *Cluster) deleteConnectionPooler() (err error) { Services(c.Namespace). Delete(context.TODO(), serviceName, options) - if !k8sutil.ResourceNotFound(err) { + if k8sutil.ResourceNotFound(err) { c.logger.Debugf("Connection pooler service was already deleted") } else if err != nil { return fmt.Errorf("could not delete service: %v", err) From 3a49b485e5c407d0dfa1d3f6cb97421db54b8cef Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 14 May 2020 11:34:02 +0200 Subject: [PATCH 049/168] delete secrets of system users too (#974) --- pkg/cluster/cluster.go | 9 --------- pkg/cluster/cluster_test.go | 31 ------------------------------- 2 files changed, 40 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index de6578f69..9538b4ab1 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -822,10 +822,6 @@ func (c *Cluster) Delete() { } for _, obj := range c.Secrets { - if doDelete, user := c.shouldDeleteSecret(obj); !doDelete { - c.logger.Warningf("not removing secret %q for the system user %q", obj.GetName(), user) - continue - } if err := c.deleteSecret(obj); err != nil { c.logger.Warningf("could not delete secret: %v", err) } @@ -1300,11 +1296,6 @@ func (c *Cluster) Unlock() { c.mu.Unlock() } -func (c *Cluster) shouldDeleteSecret(secret *v1.Secret) (delete bool, userName string) { - secretUser := string(secret.Data["username"]) - return (secretUser != c.OpConfig.ReplicationUsername && secretUser != c.OpConfig.SuperUsername), secretUser -} - type simpleActionWithResult func() error type clusterObjectGet func(name string) (spec.NamespacedName, error) diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index 4562d525e..1f6510e65 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -12,7 +12,6 @@ import ( "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" "github.com/zalando/postgres-operator/pkg/util/teams" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" ) @@ -334,36 +333,6 @@ func TestInitHumanUsersWithSuperuserTeams(t *testing.T) { } } -func TestShouldDeleteSecret(t *testing.T) { - testName := "TestShouldDeleteSecret" - - tests := []struct { - secret *v1.Secret - outcome bool - }{ - { - secret: &v1.Secret{Data: map[string][]byte{"username": []byte("foobar")}}, - outcome: true, - }, - { - secret: &v1.Secret{Data: map[string][]byte{"username": []byte(superUserName)}}, - - outcome: false, - }, - { - secret: &v1.Secret{Data: map[string][]byte{"username": []byte(replicationUserName)}}, - outcome: false, - }, - } - - for _, tt := range tests { - if outcome, username := cl.shouldDeleteSecret(tt.secret); outcome != tt.outcome { - t.Errorf("%s expects the check for deletion of the username %q secret to return %t, got %t", - testName, username, tt.outcome, outcome) - } - } -} - func TestPodAnnotations(t *testing.T) { testName := "TestPodAnnotations" tests := []struct { From 0fa61a6ab374dfcce19fd0b1f49a86d9c2a495c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steffen=20P=C3=B8hner=20Henriksen?= Date: Mon, 25 May 2020 16:32:33 +0200 Subject: [PATCH 050/168] Changed order of sidecar env vars (#980) * Changed order of sidecar env vars * Cleaned up test code --- pkg/cluster/k8sres.go | 2 +- pkg/cluster/k8sres_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 534ae7b8e..e1093b72b 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -531,7 +531,7 @@ func patchSidecarContainers(in []v1.Container, volumeMounts []v1.VolumeMount, su }, }, } - mergedEnv := append(container.Env, env...) + mergedEnv := append(env, container.Env...) container.Env = deduplicateEnvVars(mergedEnv, container.Name, logger) result = append(result, container) } diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index d09a2c0aa..78f088389 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -1394,7 +1394,7 @@ func TestSidecars(t *testing.T) { // replaced sidecar // the order in env is important - scalyrEnv := append([]v1.EnvVar{v1.EnvVar{Name: "SCALYR_API_KEY", Value: "abc"}, v1.EnvVar{Name: "SCALYR_SERVER_HOST", Value: ""}}, env...) + scalyrEnv := append(env, v1.EnvVar{Name: "SCALYR_API_KEY", Value: "abc"}, v1.EnvVar{Name: "SCALYR_SERVER_HOST", Value: ""}) assert.Contains(t, s.Spec.Template.Spec.Containers, v1.Container{ Name: "scalyr-sidecar", Image: "scalyr-image", From 2b0def5bc8b131343d6f77c69a2dc4a12ae7bd61 Mon Sep 17 00:00:00 2001 From: alfredw33 Date: Wed, 3 Jun 2020 11:33:48 -0400 Subject: [PATCH 051/168] Support for GCS WAL-E backups (#620) * Support for WAL_GS_BUCKET and GOOGLE_APPLICATION_CREDENTIALS environtment variables * Fixed merge issue but also removed all changes to support macos. * Updated test to new format * Missed macos specific changes * Added documentation and addressed comments * Update docs/administrator.md * Update docs/administrator.md * Update e2e/run.sh Co-authored-by: Felix Kunde --- charts/postgres-operator/values-crd.yaml | 6 + charts/postgres-operator/values.yaml | 6 + docs/administrator.md | 51 ++++++++ docs/reference/operator_parameters.md | 14 ++ manifests/configmap.yaml | 2 + manifests/operatorconfiguration.crd.yaml | 4 + ...gresql-operator-default-configuration.yaml | 2 + .../v1/operator_configuration_type.go | 2 + pkg/cluster/k8sres.go | 11 ++ pkg/cluster/k8sres_test.go | 121 ++++++++++++++++++ pkg/controller/operator_config.go | 2 + pkg/util/config/config.go | 2 + 12 files changed, 223 insertions(+) diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 4f57dd642..4be39325f 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -202,12 +202,18 @@ configAwsOrGcp: # AWS region used to store ESB volumes aws_region: eu-central-1 + # GCP credentials that will be used by the operator / pods + # gcp_credentials: "" + # AWS IAM role to supply in the iam.amazonaws.com/role annotation of Postgres pods # kube_iam_role: "" # S3 bucket to use for shipping postgres daily logs # log_s3_bucket: "" + # GCS bucket to use for shipping WAL segments with WAL-E + # wal_gs_bucket: "" + # S3 bucket to use for shipping WAL segments with WAL-E # wal_s3_bucket: "" diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 2a6a181f5..54461d141 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -200,6 +200,12 @@ configAwsOrGcp: # S3 bucket to use for shipping WAL segments with WAL-E # wal_s3_bucket: "" + # GCS bucket to use for shipping WAL segments with WAL-E + # wal_gs_bucket: "" + + # GCP credentials for setting the GOOGLE_APPLICATION_CREDNETIALS environment variable + # gcp_credentials: "" + # configure K8s cron job managed by the operator configLogicalBackup: # image for pods of the logical backup job (example runs pg_dumpall) diff --git a/docs/administrator.md b/docs/administrator.md index 45a328d38..e2c2e01eb 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -518,6 +518,57 @@ A secret can be pre-provisioned in different ways: * Automatically provisioned via a custom K8s controller like [kube-aws-iam-controller](https://github.com/mikkeloscar/kube-aws-iam-controller) +## Google Cloud Platform setup + +To configure the operator on GCP there are some prerequisites that are needed: + +* A service account with the proper IAM setup to access the GCS bucket for the WAL-E logs +* The credentials file for the service account. + +The configuration paramaters that we will be using are: + +* `additional_secret_mount` +* `additional_secret_mount_path` +* `gcp_credentials` +* `wal_gs_bucket` + +### Generate a K8 secret resource + +Generate the K8 secret resource that will contain your service account's +credentials. It's highly recommended to use a service account and limit its +scope to just the WAL-E bucket. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: psql-wale-creds + namespace: default +type: Opaque +stringData: + key.json: |- + +``` + +### Setup your operator configuration values + +With the `psql-wale-creds` resource applied to your cluster, ensure that +the operator's configuration is set up like the following: + +```yml +... +aws_or_gcp: + additional_secret_mount: "pgsql-wale-creds" + additional_secret_mount_path: "/var/secrets/google" # or where ever you want to mount the file + # aws_region: eu-central-1 + # kube_iam_role: "" + # log_s3_bucket: "" + # wal_s3_bucket: "" + wal_gs_bucket: "postgres-backups-bucket-28302F2" # name of bucket on where to save the WAL-E logs + gcp_credentials: "/var/secrets/google/key.json" # combination of the mount path & key in the K8 resource. (i.e. key.json) +... +``` + ## Sidecars for Postgres clusters A list of sidecars is added to each cluster created by the operator. The default diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index a81cabfc4..691e2f262 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -451,6 +451,20 @@ yet officially supported. present and accessible by Postgres pods. At the moment, supported services by Spilo are S3 and GCS. The default is empty. +* **wal_gs_bucket** + GCS bucket to use for shipping WAL segments with WAL-E. A bucket has to be + present and accessible by Postgres pods. Note, only the name of the bucket is + required. At the moment, supported services by Spilo are S3 and GCS. + The default is empty. + +* **gcp_credentials** + Used to set the GOOGLE_APPLICATION_CREDENTIALS environment variable for the pods. + This is used in with conjunction with the `additional_secret_mount` and + `additional_secret_mount_path` to properly set the credentials for the spilo + containers. This will allow users to use specific + [service accounts](https://cloud.google.com/kubernetes-engine/docs/tutorials/authenticating-to-cloud-platform). + The default is empty + * **log_s3_bucket** S3 bucket to use for shipping Postgres daily logs. Works only with S3 on AWS. The bucket has to be present and accessible by Postgres pods. The default is diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 4314b41d3..8ec850bae 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -45,6 +45,7 @@ data: # enable_team_superuser: "false" enable_teams_api: "false" # etcd_host: "" + # gcp_credentials: "" # kubernetes_use_configmaps: "false" # infrastructure_roles_secret_name: postgresql-infrastructure-roles # inherited_labels: application,environment @@ -100,6 +101,7 @@ data: # team_api_role_configuration: "log_statement:all" # teams_api_url: http://fake-teams-api.default.svc.cluster.local # toleration: "" + # wal_gs_bucket: "" # wal_s3_bucket: "" watched_namespace: "*" # listen to all namespaces workers: "4" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 23b5ff0fc..d02ff1682 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -216,10 +216,14 @@ spec: type: string aws_region: type: string + gcp_credentials: + type: string kube_iam_role: type: string log_s3_bucket: type: string + wal_gs_bucket: + type: string wal_s3_bucket: type: string logical_backup: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 049e917f6..a7ca8c4ee 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -88,8 +88,10 @@ configuration: # additional_secret_mount: "some-secret-name" # additional_secret_mount_path: "/some/dir" aws_region: eu-central-1 + # gcp_credentials: "" # kube_iam_role: "" # log_s3_bucket: "" + # wal_gs_bucket: "" # wal_s3_bucket: "" logical_backup: logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:master-58" diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index d3a9f6ec2..2dd0bbb50 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -111,6 +111,8 @@ type LoadBalancerConfiguration struct { type AWSGCPConfiguration struct { WALES3Bucket string `json:"wal_s3_bucket,omitempty"` AWSRegion string `json:"aws_region,omitempty"` + WALGSBucket string `json:"wal_gs_bucket,omitempty"` + GCPCredentials string `json:"gcp_credentials,omitempty"` LogS3Bucket string `json:"log_s3_bucket,omitempty"` KubeIAMRole string `json:"kube_iam_role,omitempty"` AdditionalSecretMount string `json:"additional_secret_mount,omitempty"` diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index e1093b72b..ef20da062 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -714,12 +714,23 @@ func (c *Cluster) generateSpiloPodEnvVars(uid types.UID, spiloConfiguration stri if spiloConfiguration != "" { envVars = append(envVars, v1.EnvVar{Name: "SPILO_CONFIGURATION", Value: spiloConfiguration}) } + if c.OpConfig.WALES3Bucket != "" { envVars = append(envVars, v1.EnvVar{Name: "WAL_S3_BUCKET", Value: c.OpConfig.WALES3Bucket}) envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))}) envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""}) } + if c.OpConfig.WALGSBucket != "" { + envVars = append(envVars, v1.EnvVar{Name: "WAL_GS_BUCKET", Value: c.OpConfig.WALGSBucket}) + envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))}) + envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""}) + } + + if c.OpConfig.GCPCredentials != "" { + envVars = append(envVars, v1.EnvVar{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: c.OpConfig.GCPCredentials}) + } + if c.OpConfig.LogS3Bucket != "" { envVars = append(envVars, v1.EnvVar{Name: "LOG_S3_BUCKET", Value: c.OpConfig.LogS3Bucket}) envVars = append(envVars, v1.EnvVar{Name: "LOG_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))}) diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 78f088389..ff830a1f5 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -20,9 +20,17 @@ import ( policyv1beta1 "k8s.io/api/policy/v1beta1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" ) +// For testing purposes +type ExpectedValue struct { + envIndex int + envVarConstant string + envVarValue string +} + func toIntStr(val int) *intstr.IntOrString { b := intstr.FromInt(val) return &b @@ -93,6 +101,119 @@ func TestGenerateSpiloJSONConfiguration(t *testing.T) { } } +func TestGenerateSpiloPodEnvVars(t *testing.T) { + var cluster = New( + Config{ + OpConfig: config.Config{ + WALGSBucket: "wale-gs-bucket", + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) + + expectedValuesGSBucket := []ExpectedValue{ + ExpectedValue{ + envIndex: 14, + envVarConstant: "WAL_GS_BUCKET", + envVarValue: "wale-gs-bucket", + }, + ExpectedValue{ + envIndex: 15, + envVarConstant: "WAL_BUCKET_SCOPE_SUFFIX", + envVarValue: "/SomeUUID", + }, + ExpectedValue{ + envIndex: 16, + envVarConstant: "WAL_BUCKET_SCOPE_PREFIX", + envVarValue: "", + }, + } + + expectedValuesGCPCreds := []ExpectedValue{ + ExpectedValue{ + envIndex: 14, + envVarConstant: "WAL_GS_BUCKET", + envVarValue: "wale-gs-bucket", + }, + ExpectedValue{ + envIndex: 15, + envVarConstant: "WAL_BUCKET_SCOPE_SUFFIX", + envVarValue: "/SomeUUID", + }, + ExpectedValue{ + envIndex: 16, + envVarConstant: "WAL_BUCKET_SCOPE_PREFIX", + envVarValue: "", + }, + ExpectedValue{ + envIndex: 17, + envVarConstant: "GOOGLE_APPLICATION_CREDENTIALS", + envVarValue: "some_path_to_credentials", + }, + } + + testName := "TestGenerateSpiloPodEnvVars" + tests := []struct { + subTest string + opConfig config.Config + uid types.UID + spiloConfig string + cloneDescription *acidv1.CloneDescription + standbyDescription *acidv1.StandbyDescription + customEnvList []v1.EnvVar + expectedValues []ExpectedValue + }{ + { + subTest: "Will set WAL_GS_BUCKET env", + opConfig: config.Config{ + WALGSBucket: "wale-gs-bucket", + }, + uid: "SomeUUID", + spiloConfig: "someConfig", + cloneDescription: &acidv1.CloneDescription{}, + standbyDescription: &acidv1.StandbyDescription{}, + customEnvList: []v1.EnvVar{}, + expectedValues: expectedValuesGSBucket, + }, + { + subTest: "Will set GOOGLE_APPLICATION_CREDENTIALS env", + opConfig: config.Config{ + WALGSBucket: "wale-gs-bucket", + GCPCredentials: "some_path_to_credentials", + }, + uid: "SomeUUID", + spiloConfig: "someConfig", + cloneDescription: &acidv1.CloneDescription{}, + standbyDescription: &acidv1.StandbyDescription{}, + customEnvList: []v1.EnvVar{}, + expectedValues: expectedValuesGCPCreds, + }, + } + + for _, tt := range tests { + cluster.OpConfig = tt.opConfig + + actualEnvs := cluster.generateSpiloPodEnvVars(tt.uid, tt.spiloConfig, tt.cloneDescription, tt.standbyDescription, tt.customEnvList) + + for _, ev := range tt.expectedValues { + env := actualEnvs[ev.envIndex] + + if env.Name != ev.envVarConstant { + t.Errorf("%s %s: Expected env name %s, have %s instead", + testName, tt.subTest, ev.envVarConstant, env.Name) + } + + if env.Value != ev.envVarValue { + t.Errorf("%s %s: Expected env value %s, have %s instead", + testName, tt.subTest, ev.envVarValue, env.Value) + } + } + } +} + func TestCreateLoadBalancerLogic(t *testing.T) { var cluster = New( Config{ diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 41d701fe2..2a3a01fd4 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -111,6 +111,8 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.AWSRegion = fromCRD.AWSGCP.AWSRegion result.LogS3Bucket = fromCRD.AWSGCP.LogS3Bucket result.KubeIAMRole = fromCRD.AWSGCP.KubeIAMRole + result.WALGSBucket = fromCRD.AWSGCP.WALGSBucket + result.GCPCredentials = fromCRD.AWSGCP.GCPCredentials result.AdditionalSecretMount = fromCRD.AWSGCP.AdditionalSecretMount result.AdditionalSecretMountPath = fromCRD.AWSGCP.AdditionalSecretMountPath diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 348452193..d15c7caa7 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -126,6 +126,8 @@ type Config struct { WALES3Bucket string `name:"wal_s3_bucket"` LogS3Bucket string `name:"log_s3_bucket"` KubeIAMRole string `name:"kube_iam_role"` + WALGSBucket string `name:"wal_gs_bucket"` + GCPCredentials string `name:"gcp_credentials"` AdditionalSecretMount string `name:"additional_secret_mount"` AdditionalSecretMountPath string `name:"additional_secret_mount_path" default:"/meta/credentials"` DebugLogging bool `name:"debug_logging" default:"true"` From 9acdcd8bbf9723fd3c20b1cd9ad6604f481d9f9c Mon Sep 17 00:00:00 2001 From: Kamil Solecki Date: Thu, 4 Jun 2020 16:49:22 +0200 Subject: [PATCH 052/168] Make selector match labels defined in the deployment (#1001) Currently, the deployment manifest specifies two labels: `name` and `team`. This fixes the service not matching the deployed pods by chosing a correct selector. --- ui/manifests/service.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/manifests/service.yaml b/ui/manifests/service.yaml index 989ec041e..7b820ca92 100644 --- a/ui/manifests/service.yaml +++ b/ui/manifests/service.yaml @@ -8,7 +8,7 @@ metadata: spec: type: "ClusterIP" selector: - application: "postgres-operator-ui" + name: "postgres-operator-ui" ports: - port: 80 protocol: "TCP" From 3c352fb46020b67129069258f2c06e17b9ef20fb Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 5 Jun 2020 11:14:17 +0200 Subject: [PATCH 053/168] bump pooler image and more coalescing for CRD config (#1004) Co-authored-by: Felix Kunde --- charts/postgres-operator-ui/index.yaml | 8 ++-- .../postgres-operator-ui-1.5.0.tgz | Bin 3799 -> 3512 bytes charts/postgres-operator/index.yaml | 8 ++-- .../postgres-operator-1.5.0.tgz | Bin 15843 -> 15565 bytes charts/postgres-operator/values-crd.yaml | 4 +- charts/postgres-operator/values.yaml | 4 +- manifests/configmap.yaml | 5 +- manifests/operatorconfiguration.crd.yaml | 20 ++++++++ ...gresql-operator-default-configuration.yaml | 4 +- manifests/postgresql.crd.yaml | 32 +++++++++++++ pkg/controller/operator_config.go | 44 +++++++++--------- pkg/util/config/config.go | 2 +- pkg/util/util.go | 44 ++++++++++++++++++ 13 files changed, 136 insertions(+), 39 deletions(-) diff --git a/charts/postgres-operator-ui/index.yaml b/charts/postgres-operator-ui/index.yaml index 2da53497d..5a7b42d80 100644 --- a/charts/postgres-operator-ui/index.yaml +++ b/charts/postgres-operator-ui/index.yaml @@ -3,10 +3,10 @@ entries: postgres-operator-ui: - apiVersion: v1 appVersion: 1.5.0 - created: "2020-05-08T12:07:31.651762139+02:00" + created: "2020-06-04T17:06:37.153522579+02:00" description: Postgres Operator UI provides a graphical interface for a convenient database-as-a-service user experience - digest: d7a36de8a3716f7b7954e2e51ed07863ea764dbb129a2fd3ac6a453f9e98a115 + digest: c91ea39e6d51d57f4048fb1b6ec53b40823f2690eb88e4e4f1a036367b9fdd61 home: https://github.com/zalando/postgres-operator keywords: - postgres @@ -26,7 +26,7 @@ entries: version: 1.5.0 - apiVersion: v1 appVersion: 1.4.0 - created: "2020-05-08T12:07:31.651223951+02:00" + created: "2020-06-04T17:06:37.15302073+02:00" description: Postgres Operator UI provides a graphical interface for a convenient database-as-a-service user experience digest: 00e0eff7056d56467cd5c975657fbb76c8d01accd25a4b7aca81bc42aeac961d @@ -49,4 +49,4 @@ entries: urls: - postgres-operator-ui-1.4.0.tgz version: 1.4.0 -generated: "2020-05-08T12:07:31.650495247+02:00" +generated: "2020-06-04T17:06:37.152369987+02:00" diff --git a/charts/postgres-operator-ui/postgres-operator-ui-1.5.0.tgz b/charts/postgres-operator-ui/postgres-operator-ui-1.5.0.tgz index 4443e15bc739899f370f7c71a2d45e78172c09a1..d8527f293f9ad6864a4c19af3b7980964c1bb0f7 100644 GIT binary patch delta 3487 zcmV;Q4Pf%u9k?5iJAXWXbKAO+`J10&OJ8T+OGAp3EhXhl?#|C~Gnyuj#&*)_b>3(o zawXvz1Q-C6?dRw|`whS!q9`eH64z<2!XGwCEEd4-Vt;X|Oej5-s63IOBw9!>r=61- zk=nl{DSPq|EeL|3e{^L32SL#OAN2d(C*fhg-|q&!UU>K<2!DHrVgCsP4>J3XmMcx< zli-`@YH#ibX(Xc`P%0{T1oO}#SytbMe$Nk_1XV0)rtPC)DJ}4>loohDfJ}-xO;7=W zsU+Eq#)JXo8s&t<2ooWJKrHwiIYq7=fP`o=CJH^GJmM*oa~dP$3MJr8hLR$W(V2-9 zBbaHOsi@PLQhz^fI?do5(5BFBTO*YTk4_aLW^2Qd?Q;&Gzs z91SGVQgG9uGRlOL63R899Hm8sDWNPfNVpQYjPbwA1^5X%0Gya*{!j79Deg9%DyJB1 zQGU^;=Bo^OSpIK3|8v4}R1c;AcAWpi@OiI&{(HUs`G0?ub^xz%LUN{|V7fi7n}ou5 z4&ZV|6;N}~|9<@TjW-c8C0b(w6Uxv4yuz3fi7+RUk}*RCTEG|~BT8Xnl!Aa;kxS5+ zW{hZ5zT-&D7>&t^$hnRnbesdDEhcoDON5*PP017;`!{j`Bu=T)@;0ik{N>T~?J@Eb zVF5FtsDBm)l*BYyKU(Hg0%$Tdy3ZUgFQHQ3IsjSD*ibMU-$vjLCTBtqB`W0FbsRY+ zv02CgTx1y22}K3hGt@JbaQ@5j2}mrDI;fe*Sz;&&G0`3T)Vh4J~YY`qQ%y%0WI?1)T?PhwsiWkIyb6_}h#!tnCEE#sRXFct5CaBmzy4 zCx4ikKU_ZpT^ zvVEG!DQYuXWh6FTNue*?HR_g|sYKe)SbyJUW=Oe)!O&pLgwzqZFM=1LTgjf;w}Gu_ zP|<}dI1>pDg<*gl2{GYD>{WS2nsUf1&2NT3s-8fL`w70S?X z7&#D-#xkeH{8X7!Q()&pTeJ}zc6;4$jUtpA-YDN5FT@0s2wdbwGbY73b)6E%P=BlL znHcIj0EBa)iE+JD1a}K7;RkS)k5O_o%9DytHAl>dQW5Ye3@Z9EpMU@7L3e-t zAEVvfb)LeUrjhmiO&n+Zu$QSJOFY8}o^}=s$3Qu!H<@u}bt|f{?y>Kba2_N$q5Siw z*js|y(9dTB1QOXt+R^FTSrpasK7=p zn3#CVMg%{NZ!KY7on07MX{&CJ+;D zo5h!_y&s*ckZegFI;lxpb=KsqV73C>i~dY4G5v?#OZ4*oEN-l`CVxwTWyPv2*Shlb zix-$l%Jl@?KPvB!%57mQ=KT_(+jiX&#wxBMWeENyEM3hMoHdwR6Zt2~W0ohog~I>2 zX2L33%M2kHYedS2Ny?gzEAQ*hia^nWG$+9R-K&Wr7UL%PDI6 zpHWp@WJJd^dzEL28L+kh*WEO0(wc4!@wG*KFKneyd=BF}S+Y#6V@fb%;xs(|;=KDEM=j?(VG7`PZ^J_t*bp$`i_`UttDz>Hou^mH+8>!=uB!{(p@2 zPtyOt6vDDM0}pBj1jBQjnEh8e=`(zB0M^a&w{t<|-NUy3e{{I_{~x6_Re!wko0i)ARn7VpE&gw=gB#jy z#}XUa7OOWijEnLWTkWDE2ps^03}Y>1LC@6q#cvt}en1j`%~q=!%GEWbz%;h9&<9=W z*DSVZ8c7;~Ya$~T4q!+mNl~MuzvF*dpo$Ondd&iTo7uAUC}LSVSS`vcg~lz%x^)i} z)lCt&W`9$q7iJ&%Rdo|A7M%^9Rc#Y27SUR_CR11})}|RYs+!iuEK8o6^epN%8ha`B zQ}41Ue|QnR2v;BgGAXo(1&hGFJQ+5ZB9SDNqf$dD#<+@krgiojb<{>i^eh6mGeg4k z%x(8>UIXP+Q^H;$BexeA3!bP5dMhBBrYLe<_kV^f%WQ;r%XW9?ffj!!DO-BYb-mXo z;M5ieLD*fTBJz2ghNVm#4~L`o=Wp7ZvK1)HLLVk#!fc5~RXDQ&)w|*8`SInu^U?8e z@EM#9)+3)Qf3>)1b={k=pS*d0ad~<^Iy-)QI(l>b%juU+)UHX|I_3H4#k=?CC#M&q z4}XJ;!7p!4w^6A*Q@f<8bs8E|TppjlKE1SixfmXwY@4AyL8~rm>kckYkKbNw!THH+ zJqP&5+7l?1*;>c1JGk_IFgkg6_Lsry`_F#2Y4$|KYQ!kNawE5@TV7L)zB$x-h6dOu zYvdj4K2F=57Z$oq`rM31IK<2AW$&SPt`HV!B)`j^jC3+a7;bA|F=fXl4ksC+r2+-Rd< z0k~mzBc)AGxV0zP*7Ik-zrCxsF2ZdgHfEHV;^K!VF$F}}9df9&Q?Axl^#@l~|=^W*WNHZc4 z-2Z$T`r(lmxZt0VjEpIxnxgXmnvb!JT>aNnS1*ITR@#!Iz83O|^64^NIYv#w)sleb z5WLaIt=?RPEs{+|a6BmJv^l@IB!AXEYiY(9>RnSkTa&UzeRBcXEO@q)HNEYIcYU-( zT(YGI>$XUc%x0%CStuM`n74-&+0`K~^F^lb4^=`c~9J70F`G37*5?>)t z+VNX?*3Y(_kPV7Yl{Y&3yzc|%eH&>sugq9AatbTXP_5~TB*R*Yt+aB4C-4(~T*F~Mq;jk8rrfns zKfnGOs-CENvT;r8;A?hmnSboNJQ=Q~wZA5p`#>UAe1NuLQ}ly8G{c9;F??kZ6sP+n;JEcn;Sy&<|gA!9IQVqCnsrhNJgBr^NMDmF@NJSRMH4uIp-#`o}3CUHVgU%6aOm>&$uTkI9wu)Yoa|JpEti06q{&MXq4*>Qp&?CdF?U zYsaSvl1>39#cz&3SFuR2^8@)58_z%G3En*3?>~iM`}^PCVQ(M*JxXgCcH~-8EPidE z{;7`$+-}efyrAy|N0(tg>h+@F&<|f61<(KdJna4{=te={{un?~zi<1tZ~L}y{}%uN N|NrzppZ)++002&u)<*yU delta 3776 zcmV;x4nOg@8`m9>JAZs_bK5wQ@O$p1e)|=`7m<_{*@?%=+z20RifA-|Mx)=*M9QU_2$T*_Q4qz2lhMx6 zlnCYCl3DV@V-(NxyneT9{(GL+{_pks!4Lj^zt;tc~Qx-r>6d4hT4v`LVB#JqW5Hg7Z@I#7%B8$+P z@)?FORVtNXr++h{YMPDQh|fA7NkUl6J1e|vUE}Qy7qJX22g+K@fvVntj9Lyv3D07O z5k=>yA&C;4=?a-FoezFgH(DmIr=s|p@3sbf`7dfEw{QDvS0qBIL3@VuG%HN%-% z!@%=iQ?st`Gn5TjzYXgf*mn2Fr|+bNY%X&;pS(VMbAEYvdKtpMrZmAyPe7y%AVZ0_ zCx7J+fk0zqF{b(tQ$K?$wu7d@Qs0eG-5@g4!`l;sq{4f6cH+VblPHvFRjR)FD%F8u ziUR*)LG|)5rEf(Nf~_@3*jZ)iaOe#^yVl|;$)rMYN@nh=T)iwkz*5MtM zbd+ES_CH7>O`RfQf=aiRDT#DdoXZP)g@3xmWJ;FSG*-8%Zc>cl$saW8m-~>635TL1<)~2 z1yiEya$%@f7#R={S~Jh`{;AZvCdbaWGH65C4|;*WLJ`XJ+DO-IFT@z*5Nu>xGJnST zKDDg^MpLV*8EfiW0E98Fh&H`s2zLv^;d^kMjZiSu!jtk&*+-NRDMMfr%08?GREcy* zq^%P>?-^jqt8L_%A0cGa#SdhbCg^f8F;$=qq^S(yKeqj=UerWC|7s`z4oHkd&LO(B zqY|0d)X?wcXq+7TjFi~Fwes7Q^?&wXrJXycZ!V87T=hYHO&{C*zo6f3*?$9X=l?xT zx#TcMLC3eCrbMk;enuc_5-BYvMaDwQ0wY->m4IV(h|vts@9w}o#{`K)x6&7|Sil4| z-dKW@fKAfWZOSE$K{qa$4(zq&r%_hm!eU{6YnuIEDqHsd)fAHyg>+T=<$tc=mi-@e zJ-@yG2i{<}|DU4V-F2S9oX$dH`)fZ=+hNaVnk>;2LwMF%EG!LW9e+r*F{@f$g)xs^ ztAKML#xZ4|-^I=n)VQ8*9&prB1Kf9c@IgbqnWGRiM);(SB^E({-#pUUMK&JO4`4e> zWXNLkoDcRW_h3lXhjvnK;eUNblH6=0X={Iil- z`Q(iuDUDZPJQs|$KV>|EA4j)_FfUFoG_1BxwWax!Yn_ZiE=>H;AAb5gb9uD53|NUOS75@#qe$e0P|EDN- zcXiDmMkI2J;NN0l)enbE70J|;3;NN-B;4zvG2rLEbxm0uLJ{UU=LuRfR3s*fgw+X& z?&-7QX`O)6_PzzVASogh>pcjA|K2WyK{Ul==E`Zu2%h`2FMlLdg}}#vr~Iwt^`>hjEoGX)0GS z#h51ic7{yJ9DlPS3ddO@Q9TIzx=~hM@aHhy-5I6xed(P0%l{E&F=dl4Py^fK|Gw9X z{{(^G^LFz8Da!98|9{DuWv2!nRSob2&vC5hU&^G$z^a&x@{R^E!tw_W{OdszGAXnWC*?mAThy6agoz=rtRWaH3B~(iN9p3We>&Z zYFwa8(irHY4)se0n-_&-7J{ukBOCVMj0iGAg@S&>|1v=3I+*7b4fJ(t%jBc*W$s`( zD5qc=yMJhFS2d7lH-%v9NflO@dF7VbO|V#W)>M|cO|V#mE7gijV6j-~=B$?0w9;mo z@>Hj1p081xOFo}!mU;Ta&>Q*<2!K>@#Uq}CU|$}c)rTUG#FU|wXM&HgiFvA2`Vv*x zMn=>$1iLdulBlWOu3bL{%BZ3wd4UPJy}*dGSbv7lYk(*^L!POs)^7~6=He~O-JJu9 z|C`K`rPW;Jdwl{%O>z+UL6eHe=4~1lJaKq-cJ=oBReMq<17(@$lkkWnCPkx6oLPhF z&Drt!;pLn2tHZOC&)}>rJ+irUo5@AZaj(OE^y=-!EY|+t5=7=96z+9c23g9 zE`QIDFW$U8KRUj+dUtYh^2@8^O;l?4)J|z??S@(xmxt#sk1x$!F3t{*Hucc%pp_T3 zaR!&ihp#U-;QZvY-UIw+io9`o zHP4sT&aUDp^Bn(|Cx=j@Lb)ORF#LpPAhE5P~C8G zP1$4_t{kBvelsPY-UL4=WJYc_VSn>rQ|=s}i zxT@KLwejUI>bcUo$@2Hggc@y&1HjLXyfLUUn_KsxpH_!v+=^-fd4hNKvwzl$scBlb zPT`K3Ik)WI5s9vm#clts*z2bocE}pg+^^K2rRb6Rgx|bGOUEy3N1%i3_s%g z6&&V6D!cq{%3ewJbE~hRs(+5kH*3eV3cjM(mchQuqqCK;_LszRABn~C575?3$`EO> zu&h71rnsN{FY;BtPz`L6|6ae_lK+7p40hlDJV|Nhf3%W;*JpyX)kJylzsvP_M1AvA z*$r#=zpLp)CyWbxj6SyQ|6b5vx&I;9egE?$We?7XQYe_Ysv%!zONWQtO#JBG=Y4SItuE+5cF3@Qw(| zGYKazj-}Ev*N|7N*!8Sxl9e;}J;?f$1M#t%<+|5GsNwd23N{lU)v zdy>*p?8ua$d8la}XO#KXjW&gPYuy{A-xhb1gkT3=;5lB;@qdPwz8?lb*d4kD-e7pp z@Av(`c|quT`SH;2aov=`d#@{}3?!YPR3QwyUT`oR47%L|KN=nQ<6zVq4~KrYx9<}& zjQS&gMB;cnB(Z*tY`9qmppioHz-FPazuO-~aW6U;^!mNQcr+UH zqhaj%B#PpoN4lc{iOA3&McsfPi4OXH{O4fSs+qk0yE&xS4nx%jm-IU8ekYvWUC-H? qVXJGHHnMAf;Cuc-C0oC>p4pXM*_H3F{BHmN0RR7EKro;HRsaA~Hh;VT diff --git a/charts/postgres-operator/index.yaml b/charts/postgres-operator/index.yaml index 63c7b450d..3c62625a1 100644 --- a/charts/postgres-operator/index.yaml +++ b/charts/postgres-operator/index.yaml @@ -3,10 +3,10 @@ entries: postgres-operator: - apiVersion: v1 appVersion: 1.5.0 - created: "2020-05-04T16:36:19.646719041+02:00" + created: "2020-06-04T17:06:49.41741489+02:00" description: Postgres Operator creates and manages PostgreSQL clusters running in Kubernetes - digest: 43510e4ed7005b2b80708df24cfbb0099b263b4a2954cff4e8f305543760be6d + digest: 198351d5db52e65cdf383d6f3e1745d91ac1e2a01121f8476f8b1be728b09531 home: https://github.com/zalando/postgres-operator keywords: - postgres @@ -25,7 +25,7 @@ entries: version: 1.5.0 - apiVersion: v1 appVersion: 1.4.0 - created: "2020-05-04T16:36:19.645338751+02:00" + created: "2020-06-04T17:06:49.416001109+02:00" description: Postgres Operator creates and manages PostgreSQL clusters running in Kubernetes digest: f8b90fecfc3cb825b94ed17edd9d5cefc36ae61801d4568597b4a79bcd73b2e9 @@ -45,4 +45,4 @@ entries: urls: - postgres-operator-1.4.0.tgz version: 1.4.0 -generated: "2020-05-04T16:36:19.643857452+02:00" +generated: "2020-06-04T17:06:49.414521538+02:00" diff --git a/charts/postgres-operator/postgres-operator-1.5.0.tgz b/charts/postgres-operator/postgres-operator-1.5.0.tgz index f575f7cfd04c55f39bf421afd6ce0a63ddb654db..6e1a48ab772c479d991b518a7a5164311b93a632 100644 GIT binary patch literal 15565 zcmYM*Q*b8H8ZO}2nb@}NWMbR4ZCihA+qP}noN$7PJ()Q9PxjuY&gqM;zF5^)UG=T? zzE2ayz@UTt=K|7z&>2gqFq=xrbIN)1aG0=ZFq^4zSnH_ra4KqOaLQ}i*%&*Rd8;Wq z@=KZ9*@Im5d3)|~rc>_)Zd435th(yB%<7QKde~5|2x_amTbt^eV787(tph#$hh|#YU*r?Y6 zsAq?otuDdIP}MiGgOFx3fI|d9Zux7$$eyv|B5V{4w|tq8yR`;0ib_@5Dtb3qki{*xFuVBLe2{6hp53IQlNaB((pA$6V&wE0$a{+WyT`YKV ztAlfs!5(mnbSsTxV5geIn}UsQtOk_CSm%cDn1&GZijzo_wMR;mGO*lQT31HCO|4Z4 zd8*9^?@>);oC*?tM3UiFL-E2SK`4?QLjQbp{QFF{n)Z`Se=Rl~poPk?3V?~C$g)TZ z`UO5opAtFF0!x!lu2Yx@nh{i-D~|H9P8_{X`r&*)CvszX9($A+tK=C?W}FT9q(@P} zq$csC9z|7bB&Ieu-ZK(2wSmHGf{5~tH9=+Tr`AFFyE$O44E6WqF=CP!;H)532&FCc zgIfVdG9!EDaZx*W8gd!}33kUF(3!5p9=psjoDwPQmqf`KlN((_;&FV!n zFAF1ii#`&_Wael75Mh!3=$(df?<*STET7^JIz);SQXQs&Ujj_>iJ)Se2#3hY<5JO- z<2yer#KO?9{~Jkt@1BPt7_SkE6h}=39n8TW91;?RaSBmUa9HCc$3H`YW2T4X=6#FTxp?bzYj3Y}3Xwc0ULqyqgJ~fQ}k-S>+Vs zNkgD1i`wcJk_fD$&DHMhBEt(xHN%uN?vhZ7C?D4=++vFLz42%@20;c(IQd!st zR-5BOPqbv$oGGv+z_in{aCad%LxvLKN!V4*64y3fUO!PuVz3pQ==*1lA4y3M&%yl5 zvR>v5vF`W=hve&Ff>sb2BvAy(+6RVsbP+K>h>Kw)t9m&fKY5xJ7>sz@9H^su#n z0vUq{xY@rph`O^J$deyQJusPRzmrhy?*HO7XP)h&wG@-lk|T6)YfWoMTf+ULv2+%YVo^9tg@Kjgo|QWn zgrr(yo4G+n`5k!&$0^RExSaWu3GS%?CzTxw)kZt-_z;GDx&`7uh>V9sbv&x*K+S4h zayTSL*nI%mQl?g}`4>Vq$ao~0G)lfg(Fd3o>8yJHy$WVDSUI7-wEaD+V3QJ)qG1gQ zsZHtL((wzXYNk0mhyW-32z=G=EL8AMX-pfO3FB@HX8LQm3vkpXGpK@VtusQH&1xTK zt@e81R7uhdRMEb!Uuqi~sM+@NbPcNUVaM9UZ1v>mqzDvYuZm5`GHmKfGhzPfn;XoZ ztky4<$7%4s)lS~d*>iNKTC^4{B$1S^!`A&`yM1;URYRkxwp)t}(k`+*va8K&s#2z4 zDyT|INsvOr71{t3UbbWkC-U0t;ItyV1yoR9wiu&Kc!%och$n4)3YU@Ozok=#Q?lsig05rV2m}f5!dco%0DXxYXc6sNyU^*8Cu1eB}>|oZ~(kT@U zOE(noOV<4yi?(eXPwY>H8ewSi{1vei*iUlx(f+LP2QeApXnoVdKEE5q$z`=iE#Tcm zF{|rvAwf3N^l!Fxz7T^M4A%>U3L)jvZz}YxmuSOUZWb9%)NY-z4bv@>)^U-$DEyLE zJcXvKG+^4ID`^F`60q_u!1KRsEOu(>i)++rNyBWjn_Tj6c>kOO4FN#DA51}t^JWQ;T$Fk24de(_YJ4hs)9i(Ryp zdhri~eQvxQ`U7t#ZufKu#2qmy@Z>SWj7(|5NgjL>5v}%vp5Pb92*G|X0w_s9_8YGo zqy21Ws)6Nc6716@Q9M^MT82Ob~lw&1y8VXwNR9tEaS_ZP`OiUa1jXO@|Kl&kj7 zKGbfD`G~IwHPmp!wJ&SIu0s~(yA0e1NjXeM!6@kOxlNa&Pkk){gGBf-;ND=|BSOL1 z20dL#Pi@ZVJ?CB<$|gVxLcT;Ct;HFJJ1uL;tJa$!VcXZ>_wm+u~M z`Y5D@b%gwt+C&5cxkPp?^=H^P>c*ATR#6Jgw8&toJ$14e9?`4@7+=qDpLQwW8>Aer z!`2lT0~dKMKnpkZyDGtK+9;I+5-qx-E!>oXj2DGuICnu(tGGl)f|V)b`a4lnBx2aT z2!24_R>Irj9ZyGB4KIM?nG#kDOt2R@W0HBcg`CNIwJs*>$3fx!4PuLYk0{@Vs}=V0 zo&1xuDF{)@*QEtn06CkCqF|F5}+s}UQe9Y@DAXcDR?AEl0_IDZTh8mGfqEN>y9>T6p z<2KcFzO)FDP`ncNc@h%#Nj+9$XZaYovcig0QJW|Ogqp5ecRDk#u5{bNSZK8mP`Kui zsDQ7!-`GflqOER9T!!{jl{EuJ&1f9%-#A=@cwii*H9KC=kuLXK^&QkTZs_WAxO|p5 zLywNQK%EA<>mQto2J=z;=*o8-%HmqvkFrEvfndzXvS(=~a4kr~uVx;?_FMbN%>bsM z(NNa8GdH0fY7JsUWhu&Ec2Jd>NsRP%Dv4G$9t$++cxjZ6>36VbVxmGQIZ+~@o??Rk zekEkulrB9ccW}=o3*ntt>RC~2-Z7{}dQgPhQY@M1Z*uk5Cvj}!BBz4585WrQHZjoXE*d$)rPGO#XXa%;|1RYE#*Kq?=tPRLbX7~#ID3f$$72r zhT3IV(Gh><$k{mLf>}Y-P;A4G55y%KU&=|5Wy_xMJPa9tRpF1H2s73wfGpPcqbeMy zmr5#uzp5%BEIZ6%)b{Ueyw-6IrR5aSH!{f&IJF+rqm$K|hqnTOQ`}@|g2BF1#YJgC zP9Eh!X+GSoSH{xOc~Yf6C0o7z9?*#{D~t$`RBYJo59qmZhPr8x8nbzK?oBA9Sy+Sg zN0WG%Lh}jx*4|LWX3+`Qr8M!r-@791kXwLDH?vEEIXZ>Ez=nroGweN0d|I)#RoW3a z`&0;?ecvpgT*X9T1zjVh1>U-~@zN4H;N@^|gJT2oY2{v*e3G6riU&1}YcFfP&&cl* zWIs3d7TlQ-ICHre)>Xa^rNuaU!lA zzpfUl>UMAqv71kZz|ASWFt9O-WqNh(EEjtTM6$uzDJ6@)biTZ{knYMR&P|wInF8Zj zpDa1(m`REE?+CquGy0GX=a|3#4iiy;hh{-Q_FI0v;4;DrpWj+rN?@(@%d=VVyy7Kd zG`MKOQZ!}b5-yz%!tD<7>aGC z2u?z4Z#r`nD=7T&ONBETy6!}nDYZNcc^n6#JWco*B79Eh&G`mkV$mHqMIQQM5r^I4@CyETxrZubkl#R*qh#%M`A>p z7uOlOQFZiI>SA66Y#P*mj*!)q#Jg{-(2IEbDZ=CCPIT0>oO7jIUyvKMtZ@e&8TXGZ z0uOR?xQT_$R3ua4-IONDAg>M^(X@Dnruj|4cAF-=yLw&qS%Y#g)fHWc(?YH2KFq+L z>VGx!WI~~3gR8x8mtlA!*@v*=!x=x0tm$UZ9?H!Jzh#>K-u+0$J|;A;-oR%VYcg~x zr11;0Vl}Tq*++pUN7$01}#bkST&_2|jBvjCd@ld3!bT{=@7Z0w?dfdP7UUP~to#y{b-E5@akj@)1&yC8!RzwLy_B%&%N= zW7Bt*uip`9`&IlRyCuRM$Kqh-!6ex4zToJXHVK@$Fm;n^X;h){msG0IB{A=IIuDgw zOk_v6ixytVu&y*>c}nr`cg zMze?20STB*`wFzXM#S`XCk)D=!^Rq%yB}|`qs$B;bJFNUKR4vSzXWBmsSXBew8^>E zb{a3e55vn121S?DN&g_q`*NToV(_d(JK1lVursFV?s^8f+WWaMTvrb+^0ik)=PRZG zto?1-sn760eB&F#MR!-Y_|auT=ZdA!{7uc913soVJa!gG)-vndy)I1paKfuI^uJHg zSW$MoWke4>&|^&ke%Ed)3OE%m&Zx#DdzYlU3K2aMDsNZN0VL?9+WjqwJVZ3{URUQC zsvcSlIYc4&xzUrLfQlCeiL+Bf(BNl!@^xQxI*1@F@0Qr`Ql!3Tzyq|# z+|d3Of9CxTlAa5uIG~`Q-EpoMombG^qr810ylZL=rMZX<^+_EaV}{SuLp9)uSn|}R zU1x+odNZ$`*ef$HTjZ!t| zY5?fwmiIL$g+VTKnTP4&AorOD#QB`fWZV$>_I%#to4@ZDq)=1nJ(~G6)CAEz0f+9) z-GKu;fv*q$NoW3^s{(Pl%kHI`#%SUq^-7QVj9$1iQ&4=?fe&TiWxSR|D&}l854!G# zy{6ON5|dW@V+P6!)=~Wa2l!G@7oU|(1tL@Mmkn*OX@X2$fPR+ zY~>W$EhoJ3wjl2caN>p3;7}A)*CGr~sfvqg9#FS}zF^o%K%vT{eM8Zy{Hjk?Wk`?g zDv#|hx1}MsI6RPDcMz9DrKB4-E^`%8Hjb6SS&~y1!L7`Ysu)5wHU`0r!+M82;)L5> zWLP-evJ+}PGZX5*C%94FP=Xlf`9d{W$?bZ9A@qWu9D6FNAdM44-!Ar0>WY|x^7B66 z*|AtRxO{upY^?3GzIL>_8l>p7$mWs6WXS8_g#8cJ(q*3evr>+~FGo`* zxQD>Q!3s5gG0K2Rlb$p$QwrZCO0o+BO{ZFUf|C%m+5j4-C%ava(>5ON#;3X@gq2Ep z2kmb^sSh0;-g4$f{gIKLM(>me=u8>xNc0xnmTQVj+jBVN9X+YZ>aQkc$kpJw#l2b9 zOuhq;XeCerq=VG{1A5uDMDH!jVl&LA+(RiI*6a@I)Q=a3NNL`Tlv!_Z$Apy5l19e;IQ8k;)WaK!J$ z_t_oo0IYAQUx}qjSM7txf20r}sWGnpVbysM(k#bNWp*r!m8XP(p%}E%eE7w24(uBS ztt$jdN;1*j4?ID?kCafo0kIb}%Ym|Mxm1)!nIQ3`eFFzrJuRBdkHO9kRSfalw#Vge z$ID{QS9Ju+pFK?E=~)5xy0m$z{Fp>Zv&jzQMb)q>W+v!!(#3VFye3K`Fr}@9T^4`V zp+%6Y{Lga-J3N*>io<(OAR!{s8s(;Es0ZNsD3R!kpI3Pf;`^mj08-)!Qw?yrdUbRt zl{R#~olorQe^8Hn!g++bQRz3-nf^BV4Tv_0v%0zb8?g?U=x}@2?0K)FW2OeVEVO5F z(@gBoNN-%!OHemPe&~eC=B$s13~3Tcbn$6OXLQp!h?s<5dz+K9&y}kLM7ALBU6zm3e+M>E${;qFgQ1g`f zR}3S+6F}`Fho*IM0iyqmD?Ij3`6dO4+$y7G?rHU~GagU4twLeVd+px5XCQW;)oDwcv0(1>=o<|7yF3aUI}++&<9HLvbalGkani^eyF?vy0iP;ivfv_SmL>k3+iP>2yJ%a*bU5KAb)j; zXIFi)W?#IBeZ-)7^V!L>+_m!yhV?fO9C26A*1=J7^VtR^V*K18e9cp(&(9DeHcxlCE-|q4vX)oiGyQumt?k~z_bT}G}N@dRlk%-8V zCS`y?;j}Lz0z&>*pun_hIuM+Y_fU`?Jw63TI8dP}S5PuApWeteptBI?lpV?C4$fOh>U$3TF4kZwYthOJ-(F@ z0%R-`?-7!uF)@@b1HcHY)}_D31Q6`LuHjSn8lEZw9I~!ytX=5O(_!cA%6qv;U9s)j zQgeChGresCC$4=pZSz}kWdR%Pg}#}^t0FWi7MWZIP4D^%UX}nmKV)iS-co~qc7w=| ze`E&^O6JPL`~7g%1Z01zW)RB8>G7Vg+Hgec*)qD2clGTDw0#P8c>z2Aa((q2$pSy` zds;sDmXnIOHeR-$s_pL{O#D7iF6^!h_44%z3=Q>Oal?q}-LSi3Hh}IPE}ubOo$po6 z0kRz&a6DbcvGom%d;R=959+-gbIf#x%-Z4l=ZjVSdC}mDetd|~b^bNw^>lFSDO+pX z)oTW(&?P_j#n487;>n|xsu%&nr*EDmKpco8!r$gkF`F+{+HAPvi?hO|2>s}(W zmsnp1@-FSy$KhTBfsH4?o&n&)`PW3r&zjC)`A6Tw`rwqBm293))b%5eO%5M1G60q8 z{Zq6QkHk*6hC%6(O1oxFrmx|d3|CpLJCbcVlviC=3a2hc&SshX!oy@3?e;utz*atV zDU{kt7tuPGUrzO(#v44HVMGK!D!=hh?ZkHQ%Iz^%h@IaRGt5@4sd&^yKXUrZr*?km z5+1sOt9k`L#Zt0mige+N<-qQNREI^_Myxtm1>Q@tI58dpEhhHfVT{&ZZwoJgY<}L% zUJpR;)QWqt4dzO!p8e130GFiX9jwz26&8F(S1RGNmztrPV2kSSLtUd4pk9%24X>)S zQMNsv68)%o*gtT(I|klUC%`Qd!)~1CJhon5 z>ZZg>(015HhvNMtE8h4eyy05yazIA{6~z-p${wS3fr&F~&FWjfLbj|MlDaKvD|Zf!dV!7u+&4hdCtm_<5?g~vRjN*wr6}Dv&x`+TGRt-IScHu+O)n%; z9H%bTbMs-NeZMe_3#j0Fo&0`yHxalT5qwf~&Uu6vqNIrIo?Z7|NAt8k)piVVi~sv<==IKS@NuL8qu&QY zhn9red=wnS3>~a|1;$of0hI^H_6G5V^R<4FnE=kl+XIpYTs(0NN~{*97LM)5cVxjosmZq3eJt#Y}v2Ne?W4G12jH%A+q^b{$@NG@HZatqF6D5~jJ>hO5 ztU-0|Kb2x5WH1Hs=+7tj#1O>lX=1gigpC`AruR>|O*2Ing|WA|@t4pPmx`urS%Tc? zZ1&hhE-@?Uug@Q0YMTf%Wg!F3@VVcG+Y8M>YM3n&;ddkFx1tlG<_h(ioP{80i_~36 z0?W))7H)@=y%i8VVZJ)oW9d!g_LqUdWd6SRfnur{W}IKK??^x`xwGw6;zQR*2|8fn z1<_j6ZLAt6T_98SqD|S3Ocv%(f5(N?LOHM|Lx&Umu5#8xcfEnIV82IJqZU>>);i9l z{wk)<;2)MgJX7mav;kg%2=8_;9WTpIDOJSzsf7Kq#9a!~L6n2bgOyf)<#$H?U;V~b zg&D8zT0}OGmWU2i2ZPLgH>{h7nx9aCPvc^$-hlWmU}e(J3-gFm#4YHIW| zX-|`K>Q?%VZVS0Kn_)HYYa~9Nhk})c!rCL0Ny|1SM0CeS08*6dqQyJv)S`LfNm19} zL6X<|){sc)N7GX#SgB z3UjYdGj1X6!Nh1RREz8#>)LY4M0NWjlUsSlm~y)Y2$fe@+C>H;U7M$v+h3{ccT>!Z z&G;2BPFhFtH)9tKzn*WH4!G`54Tl{)0?$8D18@ZdUz(pc`UHXI);*kgmq+b-GS5D@ z<(^aD>wE*}P+lx<8&1|1EHgG7_xyo$`p`Mhc+vLvpr62|4Y1bNf0iY8TlO~IM{$rj z>P-y5AF5q#Zsib`s3;Vi`|!yoC%6D!#?G-ES2gJICo3zklbXxSXW=pkOFTE)`E~g= zPR+SCk?fY|{V)|?p6VNsnKjgC!SNcbC< z{Z-TBbXA#tH85kG#{VSpY=e&~wC1^>J*fN?EViY}xqbm&&?jV8cRM{sE&93dsI;A0 zW}OykgT!^$>-Qq8s@v~Z(}Uk;ekiWvU}L3o-?{s_EUNlqD%deSf7>q0J&77tW7aX0DE(97LNPv0s#7Z2VTxpQ&UvR z`KluyGDEl(wmQ7o(H513*FgZTn=8RmL7wM3Ya^k#9V2V6*AQwd=Oh%=7c9el<*D%ZaxI89!p(5^6inFC2KvWJ;%7Cv z1aaJi{3-oRo0putJE~4jWc)GelNaC&TM);sC$K*xDrFJ@h}~p)T2{F9OVLV~3YbIfMa) z^w#`JDfm=ibnMyF++@A2W_@g1B&(Jy9vh6U?-fHV_oWA)&(# z`3DgQddoRPo+U{1BBL4xQM~_1n1YAYX%5Jilpe7sMZI~>m8d^6>cJ@@{+E4b=agrf zgNHXqFxL}vbkT8fFj4I`;ok6za*od)f9)rm;d+6S!%WXBV@!6!4sGE?iuYP(xEqcCg7yuMR z3;@3G&w+$qOurXCM+%_#4jOz19tk^?_R+T@I-muSdOrgek?N?FA;upb8V^1YV^^4 z-oIKkcD&0jv_5R??iK(~j~aXnJOjVRZNZL&tXZN8FL{J>XD1`nCcH^_b2ug)2@&9y z4b-fjFC3}Pe*gZh%f9YG=(fqXCF&AicDU^(6D6dDcr2NY41gSlHCd@5Zib;I!+AS+ zD@93ZLE=^j#zq|Ye450&@f~q&h>$D^b(D-?>Km$ADs;FHv$@TJ0OPCxlc$Li>7mj$ zv%zRN33K>4u9UuXk+ieyj;t0EDUf;;=0d}}Kb=l=gjCU>cw=f0+{Ib2eb@!h8PM%} zShBh4lT%auRU5ar`k<}tl_S82HJMn|sb^s=?!$42%U9>q?mx#dRBnm!bq8U+{^k6~ zXGV8xn!AB$H!MQZ#8;PNT61<~=WBqYLzj@((a*>3^O}%lQg|r-8pAhf28tvbt-cg2 z2HV85aIT0O4OJcnP8!)*K|~CVB81xU=eIRjw}3AEeQVHFa4px2p?qzliTJsz*T;V- zMdR~Up0-a@;mHN@QI1^SI`Hl<6nlD~fF~hWWgufXl5qZvz6-YK*g^jBbr-k8!Rc^M zWzbA=dnt);id=@Or7yTdZsv(({-9$o`~u0pz3V@bys&}u`hf;3Xt>a_5P0c>a#@pk?*-q0r`e+|OEOp<$6XLd}r`54t$a?5d1FO9J% zAf@5dAbbAf)eM%;&x%MNrO6`eBDkkVj9Hgxm9`{I?@)WA985g6?BZb`)1g7^VeCTI zr*2~qOPcwV-Z)F^AZyL$pi1b@*eHZSByscw+>SwTn5}@~3wXSw6$8u|eWbwwU&;dsH%IE=(}0s0M@IQUH@doF(s zlpg@N=j;aguR6w8uSohf&RWwSTKpdluGc9_As%DJ=Jt%C<9KeB9FtaTm2;*(^d*2g z%n?l6gXNm)I*no9Y>w?(Nm+Wyn)E5um?hlsYyvkKFx(>;3M~tKNvr8c6q;;$r4M|T z_~9qiIwes`*~Dga>G0!RwB!-^^H`ZB+t4LAUC86_Z&@mcO*pk~WXi+SC$@(5(e@D3r zUQW>V^*qbJ%3e-#=TCA}Nhpd1o93w6UokdMLhH10e%#q=i37P&p47Ql2jceGAyxHJ zxkL`hb5|z^)7K# zfl^o9z(WO;pXc2&yyU5Y$f{+(=nz?#q%`7&>wL!7<0%*-%Pyt72;d%S#o#hEm3mhO z8I#6G2`W2W#`#F;k_-EQtZ^joT52~;o|7=Ixf4pW+jMEr6j^8uu;h@e*GmN{ia1Jy zLj6{8C@Fu-BZ=!y|LGDgds{V2H6j1B(b*>S`Fx%^x_`O2?!31hPTvR!Z0hZow&b$@ z@N#kY@bmY0>n{6K=Du%8UsreOvff#4Kck7|xRBnytFzy}2RB4MZS~>t^%a<>pJ&+7 z+FN__9%JrK##tgi9rE2=^zsDmFB|yQJ!4(}%Vigh+i#d>+|Mt-H!Lu0ZR>#>eveUS ze)#PjXE-7F9XWk63-s7n2YO#WW6k}`_#?lqcedkv*t+qWE zVU4#GdsVaxc&_?9G^{eP{fO}SK*|Ojk9>v(a4%GU?2jA>Q+!RtWH3Dy90vXzSCzhU zEWs^SH2l_VgUQ4PFz(OL@8FJ3$A|`cNC^@VOcKycK9Z8qp^k?AKi@1G1#uC&14pkm z6czn}wq$KscXvgRgWs=(mJGG|0%j<_0*OwiLO*{c`jfTNdlk@TvIF7bvF6AMB9C3IG1q5svvka6rLF`4Ac2sLMx){$%XoBz|nOJQ+} z52gWrtJX7@wrS%8r3dJz;vy!}<=W&%Dxi!r_vx;4hP z(gOQEq{h38Hzi>wc|*nwnMXynS%6RO zD8cYuvYw}iUGb}xc$<_7))bc^CdhyXUm~FLEONtbf!y}YhqY3>2ti{&-Wo2lNv58F z&VW4Q-^iBqlH9(zm$H57^x|M?p!BJdRHH6{?$}LjL0?ddD9;^xeHr5}QY9rfI(8EL zGpx*Pz<*wc6XuyZRZ3kS;(&hp*XPp`S9oq*><=kf;b05I`64Ws{A8-L2{tH3wkK_i zo2-RunuFgc{;8ksTI78=oFl`zF`CaXPFgwSxy#jvifcBTxr65^+M)KBe6`2dtW* z()?R5NyrS>xQx5;DbE^Ru8slpE{v?`7q&T4^pz|W`QW~)o;e$TE3xL$(&1oG&8}!f zG+Tv$*e7V`iRKX-&8D(&8-Vf9zD=UkY#NpNGysIqOo~(1=$i$)@A2^uF#JmP%pDBq zezceY-7~n}doUeCZ5oH{uhPT)FS50tNxotX&2`h+OL8^LnN^pq`hUFE@|)LM ze)HNqAJ*^utF2$rBF*o9*{S%CW}-az6=4pm_U>pzyA7bIg?5ScB3+#WITbYJrg>@2 zXNfP@hGI-R@(k0pxt+I8(G^iKkOMVA-J}g#B{JG< zPKwG_V>qMeRjTSIZMn6!5k_9S+5+OQtQ;-dxSe8st;g{>iAPs|lLpn9mRj$2%XuFU zI|?%wj1p3QmSK^wi7Vje%dT$C``K;@dCgQYatu8WOKYH|x{rlYgc{Evc-HDb5NsSKjABv7wDq!5+5B?rtcEl7x0e-Cb}4P*DBPjk zCMM*tu$9HtX|%5+E@$6im3+!;|F7;go)T68kIJGDrbU z%`8+SxSSVJyO;o!6f68zE|yh3a#l~LX{FKYsK#bWrh>+u?Ly#E$D;b2J-)a&9^~I> zww~01MD8WWVt_vOGRIYqET%zM#;`}KY;%818n_gm=-oOSU8Wc`-*}yNk#T&@I<$G{ zUCF^Ep)VN9{paf%}$x^k~^CS-w;1ogvosu#1Dw=OEtwUHCtsbmU5w$NNfZEFUcqd+e& z9pg&Dwb;~+*oL+8Nml&{&hp#7;+c1^WZq*FAc*0(PN4_P_!4J%6mC2)&C*!ibiq4S z@N>9eypEsCr^)jxi6GBkOE{oeWjtE&v!@^ONcsEz!<2PZB{fB3o#;rVkyRLVinN$9kHge4 zUj-9t6E6e6mQ1JBf-p?w#BBkGDnE&GV!TKD$mRnTlYqm^QY($&Z%N;$o#HNelru5^$6jScqp@Lp3avfh+cVX^o1>D z1EeEz2_8b34GKPxT_U!Giz>qcKjC&1GO)xo@wV`27`GY;e~GalP019qbaDyoY4spU zM39X&a+2CSu48( z-~D|tK(KVD6<)NcMMRVGx827UEnS`pn5!5oQxML064$EGUansENO8vmK^&=O4ZZz- zK+(c{$f_qTRyJ9oN5F@|j8u?jxmOAfT=WrP{JmuQZ$8kST1PXQ89$k);NCVD-haJy z_n^zK-9<%mR?$=^?DEE8yuR5I@I92Cw7=F|jT{xmX9_>vJ}_ZfW28;y?GNHS%h!1D zU)hvHJMsI}LFlNvwnncbJtTtnX*%S>1N=1$mJpUOI=;)CVM0)vp0Gd|3gyAJZsPY{$G;1e%N#hJ$+MnDR1088Z}Ak z9>0?PMBx7aF=|88e=+J!>o-RA{0~NzN@upN2PFiuI{=a;D?NW>yQm;Yo6h-oc|!vf zSgYqjk)^@+9}*9HE>FeoLlriCVh|G6`iYwi^nHi?GP{QlK-* zok9-wo{O%ll;bn|NY^zkbN)mJ@4p81?%SZ! zpbe^h8&t|~gPO@+@sq1|0GR`Ba*T)i&Nd`Xatwmk6zSDP_++cef$LPv5!sCmG-v&4 z4PPYGx;C9~*Bx=GoMOZuSwc{+&)phr{um#on+JDqPBC6MTN%o0uI>VM)iqmFtI&wl z37a-FiH+FeN3B}PwZlau*mO2`gv{QV+TU6-aS@Sns*5_F{iVxe2Zc~?ffHEJa&D^m z*2*G)A%&(Ec82%(6tIdH2Zd~U%5uw-!YMD27Tw`n?zCibW?gP)#f1FJB+#kO z0Fia;g_RqUHUapUCVsz-80QiSt-0}=Adfg@RbE#L1I=?aQJuWQd0g);461Z3+8iK^ zMzjs4hPi#2E~ychf0C^}Pj}qgCrv*|{@d|E>Qn4oEtNy4!RyODa&md+I3IOxrjzj^ zTZ$plHy8zNy?~gkt8F1yZ_WwT3RaUun z0`cPYj)*g>UE~0m&KEZ}eTLVL@&0u>m;j{+zQCpPkVQSzu>seFWtr^Er@br#s!Jol zsp+w;`tUd{6>&l|@iHmS`VZE5?Snzb)vpj-MI@ zW-94d7s*oy`;4asVLzE5HBAJDih0CZ^Q-c3lY9S^;@;YQFzc(-#eJss>B6KR`ol={D&g|{mPORCbg#j?ITWzUY z(Mb&$5TXYQ97Jg%pIg{JRW;e618ql)M_nuJ$Mn%Kf`az@+2S;A0T)+t_HziXp=D(A z7rK@UY3Nx@w>u?rJY%7=*)y%YX~UgQ+WzsD^RvM%~m! z)tv8B#BnefApaRaS`d0u88sGj8AUDyUtUf#b}beQbxvD7bzUxIEiEoZZ3jD3CktN< zRcApNO9w}gKmE4e$6d`d=L_%jEVP5}WaSdfRLIb;kzDX2p z=vcCgAT*lmY=ePHCgfcwRPED5aWBS+=U$t`x~Ja+F95%AB?*a1TdK8McyY$_@M0_sw=hanm`YSR zmso3wO*L)cGghQ46U718y?~-EpE$=g$&bs?lW1 zH#NZ#huIfisF%~^A>dvno)5Iem><+_9~@(xAu$X?-cP;=B_prA4V@%;Da#X0LH&oV zTvwPVn!*w4sxst^ugrI+Z7+5*k{B%%0#JYO@_g`8<{}4fPb$BgG4Z8zAvo}B|B#3J zkQ8_5;W?OPlNf>S3TWfwa@Y1vd{hiukgN(#s$``hGl0`=+;1lNgaP_WRsmB4IQX53psHQ_9H&Rqep~C zGRwVB#32?V3fq^!I5`QO)-RvXBB+D{8%+_$rN97m@ehp{hoqa*>NX}W60F9dg`P_e z{zRBx+A}#;Y|!L#l7-qA%b(&#JnS#Ra2!SJP(MgPG`t@n<)BDxSe{;XT_Zy`MMHN< z7Ou8YFVZcGlzT@J9(=fQO3P`o2;ihr=a1Sx&sfSbb->FAhWzC%&Qo?W=_>!4?|^7_ z&Jgw#G=O;-{E0!RDDEi_=}>^;HFrR8H8cnsDgG?lbgo~V61i6y;wVaD;;8QgQS{t9&-2+DP$OqUNCZO3T2BGstAs${taS zrjaXsC!>=_o(5$-G&it|Nl($Cm?cM9g4~UR^5EFOkSeK;Pu1&1MSufENuktC)W!2+ zcF~AhD2mavYN@=)CO)n<+gLLk>ii65TeA}pL}2lfQQYTnL!&;cvGFRH*j}ndXxual zXIRn9fDr@oI+q76Umytb9vbU$tq>igHF4zu!hs*?Mx(ZxWb2 zr#+yz*vt*xk7I&3t=p4%bn&vPM9k&~4vjmTLPdMKlVKyEo+B6TEm1Mqo!lZ86QmDv$@q_#vYq z!&)W8w3?kllj^_~P9?%}sG$~7sZKKpk__X1f@L&LraH7*N?*-rltGPkIkrX5DIt-t z%AZrPV+Pk64@EK&W$A;8DWGll+b2a@Z{j|p0T*GWcjP1FCqw?QH9uAlG+0Ir6%zeA zXUu~P4%)-{{+ zlN@O}5?r%OAc1?Cz#1(0sC zc4KKToYg7-F!>DugEbnd3%4>Y8^JT2Y_+2jfDjxOf|fDJgWGWvKl%%q zx;yWde9B3t<|p2|;wZPQ3)$k-SU1>=3)7joW|L zQ7jbZ5@Qb|TFbch64PQA$!A8C#8dv>ptT;Dl1a68?bh9R9f2n70gHt%9Va8LZeo)N z@a%H*@AtmaUVl4RtPz78cMb#G*ti?Q=$*4)T03+hw;{`O-4s1Z$l6VQf|WBM@|Z8zocUV?hKLGc!hOJaMu&s54|%(j zo!On&d3Zi-JeY?l4+j-@a!|3O6-&T_Ah!D$uVd!P`A1vwP>-;XvlnWGbF>6VIm_xb zVX+Wv)Z6^oOHg_3ZAj0~(LUl~x`bh2t4oE5=XadNdVy^--U#qO{Cg87nfcS#zqoxR zuCOgZ7lcMDQ(beYZk@r7z^a>lDIO7y1p!ayq5?mJ?ODo(hh@C18Jk;dS|=H*v2^%1 z;j^h@xWN{S<{NI2`z-m|SwscL6#B6`ebrvX1bqG0nopA}vQKzdu*FCH5MD%Qv;onN zFLIHdqg65%YxDL^(SFMx*4eo2JA79P@2axZJ-MkEyQXyYzp9u}Ia=do6xx^?F)ZHT zSIw*RY*fWp?zPT`>@Z8AXw7x|*V_>+o~7Y+a^yywU3bnmhtS@#G16b^O2U?+kPbyJ2>FeIQ*Fd41`pF((Nk)cZByDi`bPXvI-278x$*J};(YEQmb*s_oRt$wkA@P2wFW zSMaqkiz4S3W>p#;(SFD{I8XPj{Bi&13-jZrT0ekMb4b3tp;{=7B-HT&FaA@nX`6bw zP_~3MgrL}FnVgt)){xmkb}kmWBG2+(Og~x=wyLAbfzi~VBgOW1HN3_bsG9p!Sm0YR zfNHW%(Oy3#DMk0C&X=jGVKNT)I8N6n8Js|A%YYxk-r9}wX!%nBQ1utLyu`u&Q@Xe_0E+(H{ELz5&$LzWuA1+FnGgH6OmDu@S|7nbX;DLChem_2Hu|CgTmyN}+05HmMpil6uJrIPYtNxnZnJw1)B+An9QYsC$I@l;h zYQRDN=9;QhKjep>+i0$?!&nnV^r)+v2YCUpg_rQX!C}scz=Z$L3pkz3MgN!}N)Wo<{Dauhq->0Tx$1k4C(;iLMc8kpkJd z?UfBXNB#%z99t2UGv>07>4i0PQ2f}i&V~W@CNs5jGD&MhnUy@o9z#)Ol*@ zHsx;y$zW^rqot}}MNX6}C=-n?KB@R%8uNW8 z1QQ}~6>mD&#F^emy|tZ4ieDld#J0BR1pVjR$^1*7x6&;lXV%Y9W1$D$NBAu`D^;6X z&(MbGxff{EWQl7+(u5iK7Z#}J`=aya&`XOfxp4YB>nq=q zc9c`dKW3FE$E3q_9VTGA&HkoJx?N1ZiHdPG)JQUx*c*08-68Gv8Fk<*zk?q_Vp->H zD7D1?IwSZ?<9vC%uUo6}WV$DbSs{CFI5vkZ7k%#N5m@lxW|k<&is0ucWjmuhrKbiA zHrkC&4C07sG6#w9YC3)LycZ+mk?EuKqN{bqnK-#((B1;LN38Vf@k$6S?chPD!H#m2 zo79j9mhH^i6Zd|eXOHjYIASXXdb{Jz?+R2vNNnc$sMp&7D*TLFh8tD>P++UqQja}u zV;4$v#z$UcZT$s3WhI?14xXq^qlo-5VQZ@|jf`N^`m9KwfMT5zVba+x3SxmV{Flk7 zcfY>cUp%LQVFoT>KaiuG-B|f~PJ**)&G%)!qkON89$VfP2 zz@XEvH_n^LNt^ibMLR$m`hoE~{v|%MmfC65$oE%2MQOiDaeQ;uo!^jV?yQifCbNbj zItz;`0?#lD2VD*+Ky?hMR&4`@9cO`T$UoUObuRr9~F-#h>+^Trn#&qU~5 zH5jNg$WW8K4MY{4ve97|x;9Leb^!#wpd?wscqSpM6*ZpaAY9!2zZ%d>=Rm>b@BNo$ zN2INjJNbso^dUuC#GX-V9)fHUcC=+$oB%KJgNL;#+a*Q*NXp>iZ%#21C;t2-X~(8E z|Ejx6On%9^5bs6iq9|KT2c^T~-`QArcR<}~o*&=pb7aHGDN(y5DBK+IVnjb0)M%Y; z)@zGxx+Z_@_1leNmDZ|tuLVN$Y)}=gJv6`}^qcwDoWE<>W^u7dhKCJ+Hd({i^Q`Xo zEQ=S<4hjG`SGbw*@F5z(G#l~!y6{@ty@&$qxJOl^yn7#aU(tbz=QH&lF^m;;P;vk8nVUst+71hc!f>MUpNRRw$T)lVEjVr3TCNR z^HOeKyiXexhx+mCE%IA1E(Y;y6Am;yg1BO)BUa!P;a>dsVYBzn05Eq z`ckf&CU$1MH@!MPJNZs8NQuu!W@C_MqE*kR zl%g&w*-^r_upniHH|QmQ5{?@Tdu$C+1m~q@+B**RWsuYbNuclCd6zb!-T(L;9sH{< z3^cEldD;)|3JP3HB|Z`SrQ`YS4D=av`#yVfae4FjK4UGgk8*Qs= zKv`Hp-BcO+A&y&0;xddW9Pl0Qmq|gTleFjm7c1~9)=I(-*&L;mX}-lMI}seB>W_tQ zXOx6;cath|;MaEGArA5fgFf(x_Xqokq_Wj3A6%E&7coMi_6G1VeE7HRm+<)GyE~BI zLq0V*<&QcZdiUfca#8DV7bU6l-tUb@LYg(-FIIJ@!J6r!`8aQd?xAzp>slU~X;2klUOM1g*P> za7mLgT^m<<%BUNERKZ!Nix$CcR;jD=bJq<FFrWo^4M8 zg;hlsT^bXmT$Ylm{*v7fT5W%436KSIGO2t*|JsmkcPzdf-6;3lp8Ug_5Ty_oseP9W zYr9D>!WY#C2WTVLGBDgZd%=s#N`Sy?P!vg#7&Gs$N2Ar#bR3A8OMaemNnz6gW@4 zqUKSGS>AMb&#w3kvc+`!F$|CTwB{*D&~gZfYdPLtD&XLTrwIKU@c!QPnxeHX*R{%yiI&Y9V@|fFfAwR$C<=AbX4oC^WF1-zxjOIy61B~1*{dX~ zHv&4~Tw&s$O$`8ps z%!MJVRg-zAJpdSC7iV$)^iav}J>Kc@p)JkOP|HFCa#iHW`a~;JC@-&T0hp+h!zY{+u$Fy@b{0yM;J71Ohv!@o{Fx&i2Pqv|PAyNe+t5$6e z9K~5F3)3I%vf+ULBaj!6*2n0dffi}nCgUn5p62Q$9}2nX(Lq>KE4AoC*;aZhu+oVF z`!N;a*E@>DA4yk-ta){GWg9`J)&243d(?2HNFUle!LqnVyarm*=JHki-aH2^yA}sz z*B%0ML%Lg8D5c48fon#+_UK~|rY1b0=DyR2XYb4NGvmPi6bL7ciK;o9Z5loPNIeZ5 zvN42XtsB1d+qll!rA@U0|G88QN|N~xFZx{6bfpAdQ(o466&%~V8i*TmT8+5?`7;0E zei2gjmD>|0q`lkp;Yf&;t$jc>wsoTf@>iMmCxJDtY@`);4SL{8yb$xcxte|%r%Gkq6fJQV!jr98Gy3Q z3SU2=1mNJk{xn#C8k}gY_;(-S)XOPE^Lzj6s5JIVSb&c}XbYHl|0((cq*r8Cb_$7h z0dLAONV$+9t_~_?KGz*bxZ2C|M;e%X?mkteW*M3xJN-_*+C5mol77M2x!b?hod(u# z928^@H|_xagA5XYd_+Wo_rM7sh2Mzp(Zngg8}MG8@fzVG!mfXj&Fuu?DnFZ%ea~h0 z5q4DY-*xN?M-H?M9#AyQ)_21Y1&VSc$j~i)w=7@QoN&(gg}Pm7Hf37W>+TFr+sx6u zEGM+<-`&XSv;um_tODQm66me~DZG~hR`-|POM2}^C}r~R!ihu_$dl7=K+$v{F|lB9 zEHE&wvJMz6@UR+8f&o~;6%C$m$`ck3E?_XRO$-w0A6(9g*~`sRRKY$k1Ll{cH5AVX z{{oY^oW2u+0V{ly^LTp-$hF@3U^a+(Jp!X&2OiuX6!idwH-o2vk2HymO9u?-k(|L= zJ65+>p1=;gZ7;g|Nf{nGr$%&g&*%Q8GqqU~c5bD{dRV z6vxmw$mj3N^NEh3ZYQqQ2{_eE6TKtfTBdq4=w-%=%i?yY-DOXSiD0#1f4$x{rxuIb zJ5>%zj{}EdlAJd`hsMhwH`v{6iA~jQpp$8@kaq1JjT*;_%T`RMi9WqhiDjDt&^gmD zx4l^mxG!8Y2E4fy4(0>i(=5>0nxKCbd~mq<`D(dCYdcYO6M7{OF2Nj3Y4-J#stuev zi8C`VYuk5d*LL|MdzR;_t@lKGs)P5f&&uYij!^~KuCRN*mrQrGDim~_fmVa2Grf*} z0G!p^1M2@q6}W(d@Bd&uI?~N*2e0}&MhN!(RX@XG^OS~9lRUr`sF>MJ$mK160o8B^ zc8#lO%@XOx8^eX&2ddF~&_JlxTl#fpJT}~|@STh4f6Ex6`FAkf?sMVpefT%+_nB(J zIH4hVQNd@>>+9ct{S5<#`S(aP9)lhd(et;s;kZz&nmk#o7N3X)W#(1<>b55ML7gqn zt_1zl@7CA)A1@caP1cCt&QHD9V?#Ax`$yrXp1`xW^Na6JY2WF;gXwXmIpm_XW+PCb zLSue=#+w3u@#cYFuO?MbgpLp-n$=m;38lnRd)UT z26+9asF1ai0sH*(Py6{3+x)kZ*w)2T%%8j@7n~n- zZbhrWcwc!E6T7v+0P`X|%8Ums4p?Tyby)4JMMH*jkJuR%WpulQ=ZoYs85!9z z$I6u2(}7_=+H7y5ae2wcM0B#;RbVk%-}AJ-SkO9iqoesVl%0AoB7p|)ng#7wO(EqN zc?jj34~GRhbWMD6v*=GUo3!>Kp79pAvDeeXl}p2zv84J0-CX$H(OW{raChePm@yfres8J^*W{$1gcMpMZKUi=Xcp)>fG-6qtol?m$!zvRN}de-CkLxW`+xbIYl>5nq7r%Q-(oIgwv+;cJYaxJdFroCL8Hh5 zrK!#4zbAaSPm{k|qcLy4bXg>BEdx!-QWf;n4#$4c+#FoLZN$h5Q_P_c;&3d@yKz{1 zIQa43t?hP`QPuS2YCsy{4Rkos^F6Tx(+AoPFIrL~1@N?~+x7fMbQD~+H3Y{CvCQ-B zgchs_q#0otKuJ%hbd1%SL^W}x=$`MlcNw*SJlI^unqlnCL8Y+Y=az_eb*JvojU`ry zLqi)r|iFd%L|D|R}@h;OP?a+y-2Rn ztjw034!AknexVcCt8=y1zQi7njErXU_azUMb4NJiyOzig#B{RD+X3X?jDl6*gXW)6 zZKOS6c2VWXR+)Z|7{4~r#F$>R951#IqMp@etU!p;^PFcs)~+A@y{Bq8QRP|cU1!q+ zo7ZaZOY9FK*DKE3?`hgv=Xy)fN5i?+Q1U+C3}ZJ9>BI=o2Z)3eXRzBFm(#EDL!rF3 zx`@X0pjUq03h?+DP%?@=-+)$i?ON0|a&O`3;maE!7UdzSQvxu`lTsC7lPcUtswtMH z*Pa&D9n~I3C#n8q#4D~&jSK#(YsYVtnl-QP5{*P0i$!u@=;Eky8qa5c=592qeCzyI z2BJcn-WmFzgeP0hQR_TNUv#+_87^r+!N#0MQ7bGEY}HbLEKeeq4r{kV?O$FDkTqZi zk=edmcf~O$?Z}g`T3&IYCdhI4bEq`|4ch71SrekcJtxTjDyfE6u!tegR^UpIf>rKI zN#p#UFmo2h1zLj8TD>^XxxPKOU0iRplhVdJ!5r2-NT|Nz?I3w{Sdgk1)yyd6dR@Sv zPI*;r#C=9-HEI>{eFWV4V*C(#f6#yWx^eE`FIgD1hBkivZh1We>TEa6$^7kN%w${N z>%B1i&yqc30o`h424?4J_VbFX1#5#3P59q>-~r)X>8&4PG2p$TiQvww@!!deme(oL=^ z^m21gw@Ew?zY0fa@@b?ge~cIE^mM~;OJz25XCUF}ZG^EaW&agp=aH=L%4~6N?nWjM zz8t8Y9#7Ly7~qUR%XRlJoo^q)wbq&EhW4T!&ztR7Quv=1siybQHfwIe*sxDZH1`Yv zbUmC-pXK$XZ%l<9zOp0kxzE<=6d(A}C>peRpfK-dwRunXx^Z;pN!AAFmi`Vg$Z7_M z4zxvv1gjM?sQOMY-muYRrw%oz=J|~C{`K`sJKR(5+rWd}?c>2lERn_h4M%%-q$lf? zwYfWwwY{mS^B#9=8!o}E#!|S))b9xLMZNSjAxl4V6Mt^{-#l9U2q0%hCV0I)cw$kh zu2)7bxDvS~c07MOm{84+dQ=zteFePj#Rc^@?N1f2T05mk2>$f4lum?cH<3k6lbEBdirjT< zQUG>9#3E9x%(!ffrKNM=Rq|5Nnt>xZxZ?AK7lF^5Py_N5M_>N96lEygxK{@4B{t{@hIhP~ywU|9ZG5q?LajfB&Jz=SH`?U(**a4sU>opgh zTo(}{)D+eEx4TT{g9|d$?KxK9+o_nXF0(?BTV<@e;&|P9wS2yB-Xk&NS#Tdh((53o z-}`&v=z+Ji(L@Y<~dwo zt4G?W0al4Gp|8Q?1w58iNn%`0l$w>*+Iu!Maiuc>TF4~c&!oIi+9>WDH0C4$@9iJn zSua?!aVW-#H&UJ5y|Y=sUTaT$*uwX&g0!piERf`E@GL}Ld5kJ$5U$Mva`O2}3Ri~< zyD6dAG$c>iZf=%*uc}`l^&hEV3qxk=rPMDK(aMrlX&20p)wMQzVv{VhyQx2oFuW#W z(pL9iGU<`*V-q5t94KLQKN@gp-`61JpXQ}%kTuJQlMk7?!Lxbh&a^XCj|l$qb^kfX z`N{0l0ZTRY`936_L&t#fReYB9`8Pygs>(NwV~2|BAlO^LO%nRoE(~PthJtnY5y7v# zO`DslD<-9y39%>Wc6Nley*Ewvj#@Lzw*@xf4^=q4>pe`mzEPXn?GLE(mGaQLT7wVz z`IU3)AN#?SDc?;@;!SIY%nwsopO+nL7E!!!HAaRUg! zaY1nde}H^2NCyjFrN+?1OSk@m&qSRn`xx8NozR0wVPE^TQD!*YKi3(uyIwHy(w=ag zW3&okws)`t6Aj<}qwxeLUwV0PlyQ-N#9HIRck6WcuXlXWB8_|V>@iVD>1iGpAjSUM zcxHcaAKnW(7H<$Cm=NJLnECtLvkTZu2M)pW8F;&4|NfWhbD25R203US|C%mE^Hwf6 zfS2gc_4hh^{imh(>jv=q>F-|VuJIsp&XA9ao&njppArf+`vdFBYadGf9NtqW2o&T6 z4PBf6`-ZxQi_0CFp{+17-zDKKDer{Zlp|lc1W6s#1G+ry2-pEe^MwYAt^o}>?#IPP z8*1tXv1#TIvd(VgSdu1S)cQU=MsZhYAeT3yPCPAm^g5+_?f=KtgRE& zJ-yPu)X?ER5OVOCxld+KJG{UwKX45hjIy)8=$(F`U){o_uAN__#}&ZFz$;HomxG2S z!SU{OP`>3LP{@E9jZ9cdZ?;~iF6iQ&Nn>>tu6NRwBu{p=tx%NJtU&nvi|kLj%z#dy zOye`o8yVtRUO59cJz1-spa3R}IZl>K9K8(MCu+tz&v+}V{DiuKxmT~ku-&CPZ|Bib zwXP#JdmU+PZn5F2T3)^J)=hyx(#lguOQ&UyE}KHDh_x9Awx zj6m|5Qm$+L z>*b=9KV#w_3*%U@hB)95b06=9Gat2G@rsSgVS{B02tB2kukb~%!9eARahd0li$U>` z`WbwgK$L-3m-}@Al2LvyW@`7+7>{iGPfys*w#-@EF8PGQb`Va zI4m6Fz=~y9bW?m_gt2at zqJjXe=e{no$#Mg<=Q3FsNHf$(phwwrB44{{|Iw@c>&zfcs>HavWx~aPBBdK|Yscw% z9^qGurU7B+-iKLJp9+5tu98WGEt)i02_hQ5cx8Qlh}#nswI?8cTTq-^AX$OQ0Du(t z8na-@W0`mWkebwX@=#Sn3_aoY6fG=r21e~ITAvdJMg^4t1AcY~u>U^r68hGa@!CR~ zP2CAd$yIi9-_8E$Ov!cU%XOZQOpE=d;41ui%8inXEY-|K_`CBi_WNq5sj)%)>uN2x zVn!o#2mc?}Q?zq1k-rVYrtc0VHvI)7Pv|??p{={_{Clq>myYc@P*5$%mf5g8A%B{5 zC~p}>{z1EE43P!5vQsC8c*?Byl60?Sd(F#`so~_Sf|}Gx2HhWVI=iBVc*>)v`)?Oz zhi^GXN0x)8@0Jqe)A6&g8$oCAmrU*vQhLND9I2p3u#E9u>jtrG)(n>;p4=u&%sLr% zo&v0~UQ^KyJz5Kg-le#+*+e%9@&V@_hx9a+Sc~sfz!-1VGH&k;WgKkIxQ`b6v0DuU zjUa(EM_SBp7eH1X_b$wGkMZCMm0<`~90L6C5!E)*V(p+|o`Ip^K)>B*PRmLAez1##~)4k8TqRLU{hO1!;CJu;;#&5;l9BBw(Iqej%O_xDpjeY&c|2a z>($KTRYpeUS^&^gcQnCZ`Bpa>=np)Xgad!-)m(&BQI{)tHDdYUP5|msJpl zfRK<#K$u8jvRP~Ol!27|3?@gFcY(S;i89brM$qi5!e1ec)n&p*V+YDiE?ZdM^1lJJ z24Q$BrUymkyL>m~sxzj?l{5`>3`N$Z#?Mox4xppU74JkV0Q9f(SC4w1t<(TU9g4%= zd)`DlpcgHathgXAP>s^5Hmw~BmZ(8COd1-RxqvuO9xZPmtB*iImIo1eWGe2fbZSN% z4Y4ql#t)Y;(gKf&z}!tBfE7n7T6%&RU?fyXaCpW{ZTj?S(xPl6l>D1mA}o520w#@< zoEAf0OdFaRP(Vy{t>5#pE)cY(o(c3zrGxQ(Qjr^~`2>z06p;ZF3y`CfAAqbx-dmOZ zE$$H+hI`fjEv%~srdv%z{{ibg2OY1BiTlWT_2{0nl~)BL7cv>T-u!u_k0jeli9VL; zN3a78ei98R_n|woMDuHFjDLMBHsRg-Mu^I8FcbB$TyqnqU95kenorMrV!WwwK#wk$2 zo=;TE?LD&=5j_c0Bg)JRt3Gz$801qeBvhRgOlNN|W&wzs+&y7ZT>o!09ozp8P0M}e z{|}m`C>1XZYI_<)Kf}JEihp`_jO=H3SkKe5jY}HDrt_6wHO6QLUD^xz)_p3cY~}u zBcd5(t0FLKegw!ixU-L1QL%woa0zmzLTukt8`{Ju{#aPUmiUu^Z}=#W9{nzUsH@Gb zHH&QBT~^&q{&O1cKd8Cplzm=|ECQbxr(4B%6>F5NZE9?Z)TKKt4mUp#tae_p_aOoQ z`;=_lJFn&p33oJ6hEZ$b{qz*z*BQ!bh1$tqYx#RVv0@zSgNsy&LlNtA$DlGpy-#Hr z@l-hDA?kLs$kXHH8}4FtpQTB7z{LV^leRIH!1Sz`EVt~^$zg_7BUnV}96e;P^y9zi zvW&D5>IT5;&krScCfB^~W<1_e6X{Wg&?m751c~6h0I>@WEBMY^F!y5JYWO|J;zcZg zMydp)&^wt(WB0niA^*142>JEF=veI!U(Tg6BK2akf=@5@c{Txc!W_4Zu6ubGxk?F{ zp~=*s=_Uq<#)7By_(T5b0NU7EdGFZQ1|uMULJ^jEiCYq`%5UG;r!`tI#mO{RGYC*j zJ+B(a>WrPkNQSh3N*PL|;h$2rNTm*!h}`nHVqeJ&>afIpg-D6?b$yXT#8NeX0aQ?T zW`SWAfyNA*T1(crDx2k+HX*RoAxq9K;|6q;7Byoxj?ObgwfyHsP^&h#I7h<3Xa!DY&>);W2Cqs!#N(mrI_i#=oa zQfh_E3huuH9;V-NvG2xbyzBM3D}^z8F}r*2?QX z+|uz$*YiFJ{^O4^qt_bDW)Rn4z=!wkl4NEzDJ;h$KyWQ|xa3X4cz<~8$qNoLZY_KF z#R+;|%uRys1@TOs1Q1srpUF{}CBO%$cYFL7J^qzJxl0W`!B_vsWqE_jeO>Qzi@wc& zn9SpJ9l@`BeYRkHKwY^4&@+Ge*P-gzK1mY2elW~w#PoSH?c$OO$)cIgRz&BK(2O~mPRg- zV*u(nAQACW)x5)N^~+&xrctm{!=IEIHIx%OZqGXV9zRFQc#YXp|3h0*Ad z$z8?P$b-+(Xj-|J?{uEwO8>VFw!%z>f;Hdj)9T}^|FKA@lMC9nWEn6y)^7 zd>_APb1owF&fp#*%keH#0|Po9IlVRaTmHiEx;Al8;&I z{pppKDQ0*Ehoe^$k}0RD9!Mh!X1>CS^WM(aj&Vwp^Az7$jm@Sgt?4g3a~R2m+YYAU zsETVklI2FpEF%BlW&RS(VFj$DIun_)YG<7JM}Gq~doopZOkUu6C+M0b_cKq7otx6Q zZPs|&Gq`@`RqxtwbTaHb>3Pm>*!BfwEIsg;y9;@*GJ0e#n7{^J2d*E3teUK*C)_m& zI*XSvlJ`Z+f>`D(U4(3_s*ojkkR509U2!n_i&SG!6y#`nSh+J}ulW97;*v2L$dPa*L2yL{Fg2AhD{` zc4YCQ?g2ge-N$7*r&fs8O@ty*q6|?4%P0?06lBt4s<&P)C`@+>8I)~oDv{D1b;coo zafw3wy7`#IREoFPjiL~#x!tlk_RH}a=yLXRaj}8T9>pogOh`uGn)Fd*c>&?u!O5{P z`v0QdaLoUp-rfHP^&0*U>Yb_5FLXl8Wai}LKh%=`$9d=G(qzthw3ECOIH1dQiyXKB z4-})W2xTg=3C4Q~2IN~+)x_r?T^s{Sv<`z`bT zt9XO|qj)v{DPH%!m*dP@DH^87aOeq!6*Cz;`}Gwr9}n+p>Ug90?BFABgB|&@LEQj;0(7f zab5S|s+1#>3Op!Bf(xo373Ed^R_Bt7V<)B2Zv4lzkfv;OXFUwhMGay*yCz^<|DhS# z6V3K++J|>*xH&7UAUi(KX#{95`qEnb31yECq#ufg6qr zxwvFu1l^vyos-o`A}k$~<3Z%FQUPuLdrq12Y&B`UQge8oYC=G^$!bAcN-moGi99iA zd6iWtoqd{9c8K9qY?mgR8>R@?UksKvh8(qW_7b=cr814g9|7(ocg? zXhC*g`hOIcMWnVOTxgO{UM=ZwcR!|#WSHLZh=1!jo~R9y3X}@cm+cJlfNF{}y<4HT zA&m;hy$F1%E1d`8QC2IDox>n0e%JiD;ZJE1+21|m{5vT-y%}sjwQmsFf*>aSY}+;6 zlE5(c`&*5iY(2^d>+CBfy@(b(TO9C>?{a)g5zmndf4f*5jyenwT{R8{c)IC%tSA{z zkyZxlCSyHRA!fV`Vj$I;;aA|^A$>q{5Z0gkb94j(D|JNQ!t%J(OTjg~6g+AvmG}^c zQC6apBX zK{nYVI0-Iq;V+6+u(XIKay0SxqZQ|B!Ib{T*>*e!{I9dk(Kj*RJ1`r?9q{7Qz1}k| z$4YZrerjw$Xx*u2@MZ-33ZXIHD4#~EwakzE{t4CIMd;XmU2{m_wRi$$I=(l?^1rf{ zU*Laa?R7S`DcgU^+A&Qy`<>!oqguxe^RM;YY2C7c{AP)br}r4~{eTa!I}iuVg*C zO#%*KJK(K%Rxq%b--rOKkpdkMTa@e%@ diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 4be39325f..14287fdaf 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -45,7 +45,7 @@ configGeneral: # example: "exampleimage:exampletag" # number of routines the operator spawns to process requests concurrently - workers: 4 + workers: 8 # parameters describing Postgres users configUsers: @@ -271,7 +271,7 @@ configConnectionPooler: # db user for pooler to use connection_pooler_user: "pooler" # docker image - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-7" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-8" # max db connections the pooler should hold connection_pooler_max_db_connections: 60 # default pooling mode diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 54461d141..cce0b79c8 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -44,7 +44,7 @@ configGeneral: # sidecar_docker_images: "" # number of routines the operator spawns to process requests concurrently - workers: "4" + workers: "8" # parameters describing Postgres users configUsers: @@ -263,7 +263,7 @@ configConnectionPooler: # db user for pooler to use connection_pooler_user: "pooler" # docker image - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-7" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-8" # max db connections the pooler should hold connection_pooler_max_db_connections: "60" # default pooling mode diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 8ec850bae..963aea96b 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -15,7 +15,7 @@ data: # connection_pooler_default_cpu_request: "500m" # connection_pooler_default_memory_limit: 100Mi # connection_pooler_default_memory_request: 100Mi - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-7" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-8" # connection_pooler_max_db_connections: 60 # connection_pooler_mode: "transaction" # connection_pooler_number_of_instances: 2 @@ -95,6 +95,7 @@ data: secret_name_template: "{username}.{cluster}.credentials" # sidecar_docker_images: "" # set_memory_request_to_limit: "false" + # spilo_fsgroup: 103 spilo_privileged: "false" super_username: postgres # team_admin_role: "admin" @@ -104,4 +105,4 @@ data: # wal_gs_bucket: "" # wal_s3_bucket: "" watched_namespace: "*" # listen to all namespaces - workers: "4" + workers: "8" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index d02ff1682..364ea6d5a 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -11,6 +11,26 @@ spec: singular: operatorconfiguration shortNames: - opconfig + additionalPrinterColumns: + - name: Image + type: string + description: Spilo image to be used for Pods + JSONPath: .configuration.docker_image + - name: Cluster-Label + type: string + description: Label for K8s resources created by operator + JSONPath: .configuration.kubernetes.cluster_name_label + - name: Service-Account + type: string + description: Name of service account to be used + JSONPath: .configuration.kubernetes.pod_service_account_name + - name: Min-Instances + type: integer + description: Minimum number of instances per Postgres cluster + JSONPath: .configuration.min_instances + - name: Age + type: date + JSONPath: .metadata.creationTimestamp scope: Namespaced subresources: status: {} diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index a7ca8c4ee..ab6a23113 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -19,7 +19,7 @@ configuration: # name: global-sidecar-1 # ports: # - containerPort: 80 - workers: 4 + workers: 8 users: replication_username: standby super_username: postgres @@ -128,7 +128,7 @@ configuration: connection_pooler_default_cpu_request: "500m" connection_pooler_default_memory_limit: 100Mi connection_pooler_default_memory_request: 100Mi - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-7" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-8" # connection_pooler_max_db_connections: 60 connection_pooler_mode: "transaction" connection_pooler_number_of_instances: 2 diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index e62204c40..73382ae5b 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -11,6 +11,38 @@ spec: singular: postgresql shortNames: - pg + additionalPrinterColumns: + - name: Team + type: string + description: Team responsible for Postgres CLuster + JSONPath: .spec.teamId + - name: Version + type: string + description: PostgreSQL version + JSONPath: .spec.postgresql.version + - name: Pods + type: integer + description: Number of Pods per Postgres cluster + JSONPath: .spec.numberOfInstances + - name: Volume + type: string + description: Size of the bound volume + JSONPath: .spec.volume.size + - name: CPU-Request + type: string + description: Requested CPU for Postgres containers + JSONPath: .spec.resources.requests.cpu + - name: Memory-Request + type: string + description: Requested memory for Postgres containers + JSONPath: .spec.resources.requests.memory + - name: Age + type: date + JSONPath: .metadata.creationTimestamp + - name: Status + type: string + description: Current sync status of postgresql resource + JSONPath: .status.PostgresClusterStatus scope: Namespaced subresources: status: {} diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 2a3a01fd4..bfb0e6dcc 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -38,11 +38,11 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.EtcdHost = fromCRD.EtcdHost result.KubernetesUseConfigMaps = fromCRD.KubernetesUseConfigMaps result.DockerImage = util.Coalesce(fromCRD.DockerImage, "registry.opensource.zalan.do/acid/spilo-12:1.6-p3") - result.Workers = fromCRD.Workers + result.Workers = util.CoalesceUInt32(fromCRD.Workers, 8) result.MinInstances = fromCRD.MinInstances result.MaxInstances = fromCRD.MaxInstances - result.ResyncPeriod = time.Duration(fromCRD.ResyncPeriod) - result.RepairPeriod = time.Duration(fromCRD.RepairPeriod) + result.ResyncPeriod = util.CoalesceDuration(time.Duration(fromCRD.ResyncPeriod), "30m") + result.RepairPeriod = util.CoalesceDuration(time.Duration(fromCRD.RepairPeriod), "5m") result.SetMemoryRequestToLimit = fromCRD.SetMemoryRequestToLimit result.ShmVolume = util.CoalesceBool(fromCRD.ShmVolume, util.True()) result.SidecarImages = fromCRD.SidecarImages @@ -58,7 +58,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.PodServiceAccountDefinition = fromCRD.Kubernetes.PodServiceAccountDefinition result.PodServiceAccountRoleBindingDefinition = fromCRD.Kubernetes.PodServiceAccountRoleBindingDefinition result.PodEnvironmentConfigMap = fromCRD.Kubernetes.PodEnvironmentConfigMap - result.PodTerminateGracePeriod = time.Duration(fromCRD.Kubernetes.PodTerminateGracePeriod) + result.PodTerminateGracePeriod = util.CoalesceDuration(time.Duration(fromCRD.Kubernetes.PodTerminateGracePeriod), "5m") result.SpiloPrivileged = fromCRD.Kubernetes.SpiloPrivileged result.SpiloFSGroup = fromCRD.Kubernetes.SpiloFSGroup result.ClusterDomain = util.Coalesce(fromCRD.Kubernetes.ClusterDomain, "cluster.local") @@ -71,14 +71,14 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.OAuthTokenSecretName = fromCRD.Kubernetes.OAuthTokenSecretName result.InfrastructureRolesSecretName = fromCRD.Kubernetes.InfrastructureRolesSecretName result.PodRoleLabel = util.Coalesce(fromCRD.Kubernetes.PodRoleLabel, "spilo-role") - result.ClusterLabels = fromCRD.Kubernetes.ClusterLabels + result.ClusterLabels = util.CoalesceStrMap(fromCRD.Kubernetes.ClusterLabels, map[string]string{"application": "spilo"}) result.InheritedLabels = fromCRD.Kubernetes.InheritedLabels result.DownscalerAnnotations = fromCRD.Kubernetes.DownscalerAnnotations result.ClusterNameLabel = util.Coalesce(fromCRD.Kubernetes.ClusterNameLabel, "cluster-name") result.NodeReadinessLabel = fromCRD.Kubernetes.NodeReadinessLabel result.PodPriorityClassName = fromCRD.Kubernetes.PodPriorityClassName result.PodManagementPolicy = util.Coalesce(fromCRD.Kubernetes.PodManagementPolicy, "ordered_ready") - result.MasterPodMoveTimeout = time.Duration(fromCRD.Kubernetes.MasterPodMoveTimeout) + result.MasterPodMoveTimeout = util.CoalesceDuration(time.Duration(fromCRD.Kubernetes.MasterPodMoveTimeout), "10m") result.EnablePodAntiAffinity = fromCRD.Kubernetes.EnablePodAntiAffinity result.PodAntiAffinityTopologyKey = util.Coalesce(fromCRD.Kubernetes.PodAntiAffinityTopologyKey, "kubernetes.io/hostname") @@ -91,15 +91,15 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.MinMemoryLimit = util.Coalesce(fromCRD.PostgresPodResources.MinMemoryLimit, "250Mi") // timeout config - result.ResourceCheckInterval = time.Duration(fromCRD.Timeouts.ResourceCheckInterval) - result.ResourceCheckTimeout = time.Duration(fromCRD.Timeouts.ResourceCheckTimeout) - result.PodLabelWaitTimeout = time.Duration(fromCRD.Timeouts.PodLabelWaitTimeout) - result.PodDeletionWaitTimeout = time.Duration(fromCRD.Timeouts.PodDeletionWaitTimeout) - result.ReadyWaitInterval = time.Duration(fromCRD.Timeouts.ReadyWaitInterval) - result.ReadyWaitTimeout = time.Duration(fromCRD.Timeouts.ReadyWaitTimeout) + result.ResourceCheckInterval = util.CoalesceDuration(time.Duration(fromCRD.Timeouts.ResourceCheckInterval), "3s") + result.ResourceCheckTimeout = util.CoalesceDuration(time.Duration(fromCRD.Timeouts.ResourceCheckTimeout), "10m") + result.PodLabelWaitTimeout = util.CoalesceDuration(time.Duration(fromCRD.Timeouts.PodLabelWaitTimeout), "10m") + result.PodDeletionWaitTimeout = util.CoalesceDuration(time.Duration(fromCRD.Timeouts.PodDeletionWaitTimeout), "10m") + result.ReadyWaitInterval = util.CoalesceDuration(time.Duration(fromCRD.Timeouts.ReadyWaitInterval), "4s") + result.ReadyWaitTimeout = util.CoalesceDuration(time.Duration(fromCRD.Timeouts.ReadyWaitTimeout), "30s") // load balancer config - result.DbHostedZone = fromCRD.LoadBalancer.DbHostedZone + result.DbHostedZone = util.Coalesce(fromCRD.LoadBalancer.DbHostedZone, "db.example.com") result.EnableMasterLoadBalancer = fromCRD.LoadBalancer.EnableMasterLoadBalancer result.EnableReplicaLoadBalancer = fromCRD.LoadBalancer.EnableReplicaLoadBalancer result.CustomServiceAnnotations = fromCRD.LoadBalancer.CustomServiceAnnotations @@ -114,7 +114,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.WALGSBucket = fromCRD.AWSGCP.WALGSBucket result.GCPCredentials = fromCRD.AWSGCP.GCPCredentials result.AdditionalSecretMount = fromCRD.AWSGCP.AdditionalSecretMount - result.AdditionalSecretMountPath = fromCRD.AWSGCP.AdditionalSecretMountPath + result.AdditionalSecretMountPath = util.Coalesce(fromCRD.AWSGCP.AdditionalSecretMountPath, "/meta/credentials") // logical backup config result.LogicalBackupSchedule = util.Coalesce(fromCRD.LogicalBackup.Schedule, "30 00 * * *") @@ -132,20 +132,20 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur // Teams API config result.EnableTeamsAPI = fromCRD.TeamsAPI.EnableTeamsAPI - result.TeamsAPIUrl = fromCRD.TeamsAPI.TeamsAPIUrl - result.TeamAPIRoleConfiguration = fromCRD.TeamsAPI.TeamAPIRoleConfiguration + result.TeamsAPIUrl = util.Coalesce(fromCRD.TeamsAPI.TeamsAPIUrl, "https://teams.example.com/api/") + result.TeamAPIRoleConfiguration = util.CoalesceStrMap(fromCRD.TeamsAPI.TeamAPIRoleConfiguration, map[string]string{"log_statement": "all"}) result.EnableTeamSuperuser = fromCRD.TeamsAPI.EnableTeamSuperuser result.EnableAdminRoleForUsers = fromCRD.TeamsAPI.EnableAdminRoleForUsers result.TeamAdminRole = fromCRD.TeamsAPI.TeamAdminRole - result.PamRoleName = fromCRD.TeamsAPI.PamRoleName - result.PamConfiguration = fromCRD.TeamsAPI.PamConfiguration - result.ProtectedRoles = fromCRD.TeamsAPI.ProtectedRoles + result.PamRoleName = util.Coalesce(fromCRD.TeamsAPI.PamRoleName, "zalandos") + result.PamConfiguration = util.Coalesce(fromCRD.TeamsAPI.PamConfiguration, "https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees") + result.ProtectedRoles = util.CoalesceStrArr(fromCRD.TeamsAPI.ProtectedRoles, []string{"admin"}) result.PostgresSuperuserTeams = fromCRD.TeamsAPI.PostgresSuperuserTeams // logging REST API config - result.APIPort = fromCRD.LoggingRESTAPI.APIPort - result.RingLogLines = fromCRD.LoggingRESTAPI.RingLogLines - result.ClusterHistoryEntries = fromCRD.LoggingRESTAPI.ClusterHistoryEntries + result.APIPort = util.CoalesceInt(fromCRD.LoggingRESTAPI.APIPort, 8080) + result.RingLogLines = util.CoalesceInt(fromCRD.LoggingRESTAPI.RingLogLines, 100) + result.ClusterHistoryEntries = util.CoalesceInt(fromCRD.LoggingRESTAPI.ClusterHistoryEntries, 1000) // Scalyr config result.ScalyrAPIKey = fromCRD.Scalyr.ScalyrAPIKey diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index d15c7caa7..01057f236 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -150,7 +150,7 @@ type Config struct { EnablePodDisruptionBudget *bool `name:"enable_pod_disruption_budget" default:"true"` EnableInitContainers *bool `name:"enable_init_containers" default:"true"` EnableSidecars *bool `name:"enable_sidecars" default:"true"` - Workers uint32 `name:"workers" default:"4"` + Workers uint32 `name:"workers" default:"8"` APIPort int `name:"api_port" default:"8080"` RingLogLines int `name:"ring_log_lines" default:"100"` ClusterHistoryEntries int `name:"cluster_history_entries" default:"1000"` diff --git a/pkg/util/util.go b/pkg/util/util.go index 5701429aa..ff1be4e68 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -147,6 +147,30 @@ func Coalesce(val, defaultVal string) string { return val } +// CoalesceStrArr returns the first argument if it is not null, otherwise the second one. +func CoalesceStrArr(val, defaultVal []string) []string { + if len(val) == 0 { + return defaultVal + } + return val +} + +// CoalesceStrMap returns the first argument if it is not null, otherwise the second one. +func CoalesceStrMap(val, defaultVal map[string]string) map[string]string { + if len(val) == 0 { + return defaultVal + } + return val +} + +// CoalesceInt works like coalesce but for int +func CoalesceInt(val, defaultVal int) int { + if val == 0 { + return defaultVal + } + return val +} + // CoalesceInt32 works like coalesce but for *int32 func CoalesceInt32(val, defaultVal *int32) *int32 { if val == nil { @@ -155,6 +179,14 @@ func CoalesceInt32(val, defaultVal *int32) *int32 { return val } +// CoalesceUInt32 works like coalesce but for uint32 +func CoalesceUInt32(val, defaultVal uint32) uint32 { + if val == 0 { + return defaultVal + } + return val +} + // CoalesceBool works like coalesce but for *bool func CoalesceBool(val, defaultVal *bool) *bool { if val == nil { @@ -163,6 +195,18 @@ func CoalesceBool(val, defaultVal *bool) *bool { return val } +// CoalesceDuration works like coalesce but for time.Duration +func CoalesceDuration(val time.Duration, defaultVal string) time.Duration { + if val == 0 { + duration, err := time.ParseDuration(defaultVal) + if err != nil { + panic(err) + } + return duration + } + return val +} + // Test if any of the values is nil func testNil(values ...*int32) bool { for _, v := range values { From fe7ffaa11247ea150551764ad2a1d0682c95fae6 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 9 Jun 2020 10:27:57 +0200 Subject: [PATCH 054/168] trigger rolling update when securityContext of PodTemplate changes (#1007) Co-authored-by: Felix Kunde --- pkg/cluster/cluster.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 9538b4ab1..275a51042 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -455,6 +455,12 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa needsRollUpdate = true reasons = append(reasons, "new statefulset's pod template metadata annotations doesn't match the current one") } + if !reflect.DeepEqual(c.Statefulset.Spec.Template.Spec.SecurityContext, statefulSet.Spec.Template.Spec.SecurityContext) { + match = false + needsReplace = true + needsRollUpdate = true + reasons = append(reasons, "new statefulset's pod template security context in spec doesn't match the current one") + } if len(c.Statefulset.Spec.VolumeClaimTemplates) != len(statefulSet.Spec.VolumeClaimTemplates) { needsReplace = true reasons = append(reasons, "new statefulset's volumeClaimTemplates contains different number of volumes to the old one") From fa6929f0288549a060c98ce1925ce6ec8b987688 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 11 Jun 2020 12:23:39 +0200 Subject: [PATCH 055/168] do not block rolling updates with lazy spilo update enabled (#1012) * do not block rolling updates with lazy spilo update enabled * treat initContainers like Spilo image Co-authored-by: Felix Kunde --- docs/reference/operator_parameters.md | 2 +- pkg/cluster/cluster.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 691e2f262..f8189f913 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -76,7 +76,7 @@ Those are top-level keys, containing both leaf keys and groups. The default is `true`. * **enable_lazy_spilo_upgrade** - Instruct operator to update only the statefulsets with the new image without immediately doing the rolling update. The assumption is pods will be re-started later with the new image, for example due to the node rotation. + Instruct operator to update only the statefulsets with new images (Spilo and InitContainers) without immediately doing the rolling update. The assumption is pods will be re-started later with new images, for example due to the node rotation. The default is `false`. * **etcd_host** diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 275a51042..1585618a6 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -488,7 +488,6 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa // until they are re-created for other reasons, for example node rotation if c.OpConfig.EnableLazySpiloUpgrade && !reflect.DeepEqual(c.Statefulset.Spec.Template.Spec.Containers[0].Image, statefulSet.Spec.Template.Spec.Containers[0].Image) { needsReplace = true - needsRollUpdate = false reasons = append(reasons, "lazy Spilo update: new statefulset's pod image doesn't match the current one") } From 3d976ebe8b05a268579f884bd4a12c45443a118b Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 12 Jun 2020 15:09:59 +0200 Subject: [PATCH 056/168] include volume in list of required fields (#1016) Co-authored-by: Felix Kunde --- charts/postgres-operator/crds/postgresqls.yaml | 1 + manifests/postgresql.crd.yaml | 1 + pkg/apis/acid.zalan.do/v1/crds.go | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index fdbcf8304..6df2de723 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -73,6 +73,7 @@ spec: - numberOfInstances - teamId - postgresql + - volume properties: additionalVolumes: type: array diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 73382ae5b..1d42e7254 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -69,6 +69,7 @@ spec: - numberOfInstances - teamId - postgresql + - volume properties: additionalVolumes: type: array diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index ad1b79a45..43410ed3b 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -132,7 +132,7 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, "spec": { Type: "object", - Required: []string{"numberOfInstances", "teamId", "postgresql"}, + Required: []string{"numberOfInstances", "teamId", "postgresql", "volume"}, Properties: map[string]apiextv1beta1.JSONSchemaProps{ "allowedSourceRanges": { Type: "array", From 0e3fb9ec43c15448c01e7748531cc884ef8219dc Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 16 Jun 2020 10:51:49 +0200 Subject: [PATCH 057/168] update dependencies (#1019) Co-authored-by: Felix Kunde --- go.mod | 27 ++++++++-------- go.sum | 97 +++++++++++++++++++--------------------------------------- 2 files changed, 44 insertions(+), 80 deletions(-) diff --git a/go.mod b/go.mod index dc6389a1c..041f90706 100644 --- a/go.mod +++ b/go.mod @@ -3,21 +3,18 @@ module github.com/zalando/postgres-operator go 1.14 require ( - github.com/aws/aws-sdk-go v1.29.33 - github.com/emicklei/go-restful v2.9.6+incompatible // indirect - github.com/evanphx/json-patch v4.5.0+incompatible // indirect - github.com/googleapis/gnostic v0.3.0 // indirect - github.com/lib/pq v1.3.0 + github.com/aws/aws-sdk-go v1.32.2 + github.com/lib/pq v1.7.0 github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d - github.com/r3labs/diff v0.0.0-20191120142937-b4ed99a31f5a - github.com/sirupsen/logrus v1.5.0 - github.com/stretchr/testify v1.4.0 - golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b // indirect + github.com/r3labs/diff v1.1.0 + github.com/sirupsen/logrus v1.6.0 + github.com/stretchr/testify v1.5.1 + golang.org/x/mod v0.3.0 // indirect + golang.org/x/tools v0.0.0-20200615222825-6aa8f57aacd9 // indirect gopkg.in/yaml.v2 v2.2.8 - k8s.io/api v0.18.2 - k8s.io/apiextensions-apiserver v0.18.2 - k8s.io/apimachinery v0.18.2 - k8s.io/client-go v11.0.0+incompatible - k8s.io/code-generator v0.18.2 - sigs.k8s.io/kind v0.5.1 // indirect + k8s.io/api v0.18.3 + k8s.io/apiextensions-apiserver v0.18.3 + k8s.io/apimachinery v0.18.3 + k8s.io/client-go v0.18.3 + k8s.io/code-generator v0.18.3 ) diff --git a/go.sum b/go.sum index 22be07f7a..b3d154b98 100644 --- a/go.sum +++ b/go.sum @@ -26,8 +26,8 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.29.33 h1:WP85+WHalTFQR2wYp5xR2sjiVAZXew2bBQXGU1QJBXI= -github.com/aws/aws-sdk-go v1.29.33/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= +github.com/aws/aws-sdk-go v1.32.2 h1:X5/tQ4cuqCCUZgeOh41WFh9Eq5xe32JzWe4PSE2i1ME= +github.com/aws/aws-sdk-go v1.32.2/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= @@ -46,7 +46,6 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -63,14 +62,10 @@ github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkg github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.6+incompatible h1:tfrHha8zJ01ywiOEC1miGY8st1/igzWB8OmvPgoYX7w= -github.com/emicklei/go-restful v2.9.6+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= -github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -150,7 +145,6 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= @@ -161,13 +155,10 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gnostic v0.0.0-20170426233943-68f4ded48ba9/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.1.0 h1:rVsPeBmXbYv4If/cumu1AzZPwV58q433hvONV1UEZoI= github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0= -github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -183,12 +174,10 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= +github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -199,8 +188,9 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -208,14 +198,13 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= -github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.7.0 h1:h93mCPfUSkaul3Ka/VG8uZdmW1uMHDGxzu0NWHuJmHY= +github.com/lib/pq v1.7.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190620125010-da37f6c1e481/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= @@ -228,7 +217,6 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -241,11 +229,9 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= @@ -253,9 +239,7 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= @@ -268,23 +252,20 @@ github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6T github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/r3labs/diff v0.0.0-20191120142937-b4ed99a31f5a h1:2v4Ipjxa3sh+xn6GvtgrMub2ci4ZLQMvTaYIba2lfdc= -github.com/r3labs/diff v0.0.0-20191120142937-b4ed99a31f5a/go.mod h1:ozniNEFS3j1qCwHKdvraMn1WJOsUxHd7lYfukEIS4cs= +github.com/r3labs/diff v1.1.0 h1:V53xhrbTHrWFWq3gI4b94AjgEJOerO1+1l0xyHOBi8M= +github.com/r3labs/diff v1.1.0/go.mod h1:7WjXasNzi0vJetRcB/RqNl5dlIsmXcTTLmF5IoH6Xig= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q= -github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -297,12 +278,12 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= @@ -338,6 +319,8 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -382,7 +365,6 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f h1:25KHgbfyiSm6vwQLbM3zZIe1v9p/3ea4Rz+nnM5K/i4= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190621203818-d432491b9138/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 h1:HmbHVPwrPEKPGLAcHSrMe6+hqSUlvZU0rab6x5EXfGU= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -410,8 +392,8 @@ golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b h1:zSzQJAznWxAh9fZxiPy2FZo+ZZEYoYFYYDYdOrU7AaM= -golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200615222825-6aa8f57aacd9 h1:cwgUY+1ja2qxWb2dyaCoixaA66WGWmrijSlxaM+JM/g= +golang.org/x/tools v0.0.0-20200615222825-6aa8f57aacd9/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= @@ -453,46 +435,31 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.0.0-20190313235455-40a48860b5ab/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= -k8s.io/api v0.0.0-20190409021203-6e4e0e4f393b/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= -k8s.io/api v0.18.2 h1:wG5g5ZmSVgm5B+eHMIbI9EGATS2L8Z72rda19RIEgY8= -k8s.io/api v0.18.2/go.mod h1:SJCWI7OLzhZSvbY7U8zwNl9UA4o1fizoug34OV/2r78= -k8s.io/apiextensions-apiserver v0.18.2 h1:I4v3/jAuQC+89L3Z7dDgAiN4EOjN6sbm6iBqQwHTah8= -k8s.io/apiextensions-apiserver v0.18.2/go.mod h1:q3faSnRGmYimiocj6cHQ1I3WpLqmDgJFlKL37fC4ZvY= -k8s.io/apimachinery v0.0.0-20190313205120-d7deff9243b1/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= -k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= -k8s.io/apimachinery v0.18.2 h1:44CmtbmkzVDAhCpRVSiP2R5PPrC2RtlIv/MoB8xpdRA= -k8s.io/apimachinery v0.18.2/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= -k8s.io/apiserver v0.18.2/go.mod h1:Xbh066NqrZO8cbsoenCwyDJ1OSi8Ag8I2lezeHxzwzw= -k8s.io/client-go v0.18.2 h1:aLB0iaD4nmwh7arT2wIn+lMnAq7OswjaejkQ8p9bBYE= -k8s.io/client-go v0.18.2/go.mod h1:Xcm5wVGXX9HAA2JJ2sSBUn3tCJ+4SVlCbl2MNNv+CIU= -k8s.io/client-go v11.0.0+incompatible h1:LBbX2+lOwY9flffWlJM7f1Ct8V2SRNiMRDFeiwnJo9o= -k8s.io/client-go v11.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= -k8s.io/code-generator v0.18.2 h1:C1Nn2JiMf244CvBDKVPX0W2mZFJkVBg54T8OV7/Imso= -k8s.io/code-generator v0.18.2/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= -k8s.io/component-base v0.18.2/go.mod h1:kqLlMuhJNHQ9lz8Z7V5bxUUtjFZnrypArGl58gmDfUM= +k8s.io/api v0.18.3 h1:2AJaUQdgUZLoDZHrun21PW2Nx9+ll6cUzvn3IKhSIn0= +k8s.io/api v0.18.3/go.mod h1:UOaMwERbqJMfeeeHc8XJKawj4P9TgDRnViIqqBeH2QA= +k8s.io/apiextensions-apiserver v0.18.3 h1:h6oZO+iAgg0HjxmuNnguNdKNB9+wv3O1EBDdDWJViQ0= +k8s.io/apiextensions-apiserver v0.18.3/go.mod h1:TMsNGs7DYpMXd+8MOCX8KzPOCx8fnZMoIGB24m03+JE= +k8s.io/apimachinery v0.18.3 h1:pOGcbVAhxADgUYnjS08EFXs9QMl8qaH5U4fr5LGUrSk= +k8s.io/apimachinery v0.18.3/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= +k8s.io/apiserver v0.18.3/go.mod h1:tHQRmthRPLUtwqsOnJJMoI8SW3lnoReZeE861lH8vUw= +k8s.io/client-go v0.18.3 h1:QaJzz92tsN67oorwzmoB0a9r9ZVHuD5ryjbCKP0U22k= +k8s.io/client-go v0.18.3/go.mod h1:4a/dpQEvzAhT1BbuWW09qvIaGw6Gbu1gZYiQZIi1DMw= +k8s.io/code-generator v0.18.3 h1:5H57pYEbkMMXCLKD16YQH3yDPAbVLweUsB1M3m70D1c= +k8s.io/code-generator v0.18.3/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c= +k8s.io/component-base v0.18.3/go.mod h1:bp5GzGR0aGkYEfTj+eTY0AN/vXTgkJdQXjNTTVUaa3k= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200114144118-36b2048a9120 h1:RPscN6KhmG54S33L+lr3GS+oD1jmchIU0ll519K6FA4= k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v0.3.3/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= -k8s.io/kube-openapi v0.0.0-20190603182131-db7b694dc208/go.mod h1:nfDlWeOsu3pUf4yWGL+ERqohP4YsZcBJXWMK+gkzOA4= -k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c h1:/KUFqjjqAcY4Us6luF5RDNZ16KJtb49HfR3ZHB9qYXM= -k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= +k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6 h1:Oh3Mzx5pJ+yIumsAD0MOECPVeXsVot0UkiaCGVyfGQY= +k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89 h1:d4vVOjXm687F1iLSP2q3lyPPuyvTUt3aVoBpi2DqRsU= k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0= -sigs.k8s.io/kind v0.5.1 h1:BYnHEJ9DC+0Yjlyyehqd3xnKtEmFdLKU8QxqOqvQzdw= -sigs.k8s.io/kind v0.5.1/go.mod h1:L+Kcoo83/D1+ryU5P2VFbvYm0oqbkJn9zTZq0KNxW68= -sigs.k8s.io/kustomize/v3 v3.1.1-0.20190821175718-4b67a6de1296 h1:iQaIG5Dq+3qSiaFrJ/l/0MjjxKmdwyVNpKRYJwUe/+0= -sigs.k8s.io/kustomize/v3 v3.1.1-0.20190821175718-4b67a6de1296/go.mod h1:ztX4zYc/QIww3gSripwF7TBOarBTm5BvyAMem0kCzOE= -sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e h1:4Z09Hglb792X0kfOBBJUPFEyvVfQWrYT/l8h5EKA6JQ= -sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E= sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= From 0c6655a22dffce9c2388e6ca6a69bc29c26e7c6b Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 17 Jun 2020 13:32:16 +0200 Subject: [PATCH 058/168] skip creation later to improve visibility of errors (#1013) * try to emit error for missing team name in cluster name * skip creation after new cluster object * move SetStatus to k8sclient and emit event when skipping creation and rename to SetPostgresCRDStatus Co-authored-by: Felix Kunde --- pkg/apis/acid.zalan.do/v1/marshal.go | 6 ++-- pkg/cluster/cluster.go | 41 ++++------------------------ pkg/cluster/sync.go | 4 +-- pkg/controller/controller.go | 11 ++++++++ pkg/controller/postgresql.go | 19 ++++++++++--- pkg/util/k8sutil/k8sutil.go | 30 ++++++++++++++++++++ 6 files changed, 67 insertions(+), 44 deletions(-) diff --git a/pkg/apis/acid.zalan.do/v1/marshal.go b/pkg/apis/acid.zalan.do/v1/marshal.go index d180f784c..336b0da41 100644 --- a/pkg/apis/acid.zalan.do/v1/marshal.go +++ b/pkg/apis/acid.zalan.do/v1/marshal.go @@ -102,7 +102,7 @@ func (p *Postgresql) UnmarshalJSON(data []byte) error { } tmp.Error = err.Error() - tmp.Status = PostgresStatus{PostgresClusterStatus: ClusterStatusInvalid} + tmp.Status.PostgresClusterStatus = ClusterStatusInvalid *p = Postgresql(tmp) @@ -112,10 +112,10 @@ func (p *Postgresql) UnmarshalJSON(data []byte) error { if clusterName, err := extractClusterName(tmp2.ObjectMeta.Name, tmp2.Spec.TeamID); err != nil { tmp2.Error = err.Error() - tmp2.Status = PostgresStatus{PostgresClusterStatus: ClusterStatusInvalid} + tmp2.Status.PostgresClusterStatus = ClusterStatusInvalid } else if err := validateCloneClusterDescription(&tmp2.Spec.Clone); err != nil { tmp2.Error = err.Error() - tmp2.Status = PostgresStatus{PostgresClusterStatus: ClusterStatusInvalid} + tmp2.Status.PostgresClusterStatus = ClusterStatusInvalid } else { tmp2.Spec.ClusterName = clusterName } diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 1585618a6..44c3e9b62 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -5,7 +5,6 @@ package cluster import ( "context" "database/sql" - "encoding/json" "fmt" "reflect" "regexp" @@ -181,34 +180,6 @@ func (c *Cluster) GetReference() *v1.ObjectReference { return ref } -// SetStatus of Postgres cluster -// TODO: eventually switch to updateStatus() for kubernetes 1.11 and above -func (c *Cluster) setStatus(status string) { - var pgStatus acidv1.PostgresStatus - pgStatus.PostgresClusterStatus = status - - patch, err := json.Marshal(struct { - PgStatus interface{} `json:"status"` - }{&pgStatus}) - - if err != nil { - c.logger.Errorf("could not marshal status: %v", err) - } - - // we cannot do a full scale update here without fetching the previous manifest (as the resourceVersion may differ), - // however, we could do patch without it. In the future, once /status subresource is there (starting Kubernetes 1.11) - // we should take advantage of it. - newspec, err := c.KubeClient.AcidV1ClientSet.AcidV1().Postgresqls(c.clusterNamespace()).Patch( - context.TODO(), c.Name, types.MergePatchType, patch, metav1.PatchOptions{}, "status") - if err != nil { - c.logger.Errorf("could not update status: %v", err) - // return as newspec is empty, see PR654 - return - } - // update the spec, maintaining the new resourceVersion. - c.setSpec(newspec) -} - func (c *Cluster) isNewCluster() bool { return c.Status.Creating() } @@ -257,13 +228,13 @@ func (c *Cluster) Create() error { defer func() { if err == nil { - c.setStatus(acidv1.ClusterStatusRunning) //TODO: are you sure it's running? + c.KubeClient.SetPostgresCRDStatus(c.clusterName(), acidv1.ClusterStatusRunning) //TODO: are you sure it's running? } else { - c.setStatus(acidv1.ClusterStatusAddFailed) + c.KubeClient.SetPostgresCRDStatus(c.clusterName(), acidv1.ClusterStatusAddFailed) } }() - c.setStatus(acidv1.ClusterStatusCreating) + c.KubeClient.SetPostgresCRDStatus(c.clusterName(), acidv1.ClusterStatusCreating) c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Create", "Started creation of new cluster resources") if err = c.enforceMinResourceLimits(&c.Spec); err != nil { @@ -630,14 +601,14 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { c.mu.Lock() defer c.mu.Unlock() - c.setStatus(acidv1.ClusterStatusUpdating) + c.KubeClient.SetPostgresCRDStatus(c.clusterName(), acidv1.ClusterStatusUpdating) c.setSpec(newSpec) defer func() { if updateFailed { - c.setStatus(acidv1.ClusterStatusUpdateFailed) + c.KubeClient.SetPostgresCRDStatus(c.clusterName(), acidv1.ClusterStatusUpdateFailed) } else { - c.setStatus(acidv1.ClusterStatusRunning) + c.KubeClient.SetPostgresCRDStatus(c.clusterName(), acidv1.ClusterStatusRunning) } }() diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 697fc2d05..e49bd4537 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -32,9 +32,9 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { defer func() { if err != nil { c.logger.Warningf("error while syncing cluster state: %v", err) - c.setStatus(acidv1.ClusterStatusSyncFailed) + c.KubeClient.SetPostgresCRDStatus(c.clusterName(), acidv1.ClusterStatusSyncFailed) } else if !c.Status.Running() { - c.setStatus(acidv1.ClusterStatusRunning) + c.KubeClient.SetPostgresCRDStatus(c.clusterName(), acidv1.ClusterStatusRunning) } }() diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 26b6b1b87..6011d3863 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -25,6 +25,7 @@ import ( typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" + "k8s.io/client-go/tools/reference" ) // Controller represents operator controller @@ -442,6 +443,16 @@ func (c *Controller) getEffectiveNamespace(namespaceFromEnvironment, namespaceFr return namespace } +// GetReference of Postgres CR object +// i.e. required to emit events to this resource +func (c *Controller) GetReference(postgresql *acidv1.Postgresql) *v1.ObjectReference { + ref, err := reference.GetReference(scheme.Scheme, postgresql) + if err != nil { + c.logger.Errorf("could not get reference for Postgresql CR %v/%v: %v", postgresql.Namespace, postgresql.Name, err) + } + return ref +} + // hasOwnership returns true if the controller is the "owner" of the postgresql. // Whether it's owner is determined by the value of 'acid.zalan.do/controller' // annotation. If the value matches the controllerID then it owns it, or if the diff --git a/pkg/controller/postgresql.go b/pkg/controller/postgresql.go index c243f330f..a41eb0335 100644 --- a/pkg/controller/postgresql.go +++ b/pkg/controller/postgresql.go @@ -421,14 +421,25 @@ func (c *Controller) queueClusterEvent(informerOldSpec, informerNewSpec *acidv1. } if clusterError != "" && eventType != EventDelete { - c.logger. - WithField("cluster-name", clusterName). - Debugf("skipping %q event for the invalid cluster: %s", eventType, clusterError) + c.logger.WithField("cluster-name", clusterName).Debugf("skipping %q event for the invalid cluster: %s", eventType, clusterError) + + switch eventType { + case EventAdd: + c.KubeClient.SetPostgresCRDStatus(clusterName, acidv1.ClusterStatusAddFailed) + c.eventRecorder.Eventf(c.GetReference(informerNewSpec), v1.EventTypeWarning, "Create", "%v", clusterError) + case EventUpdate: + c.KubeClient.SetPostgresCRDStatus(clusterName, acidv1.ClusterStatusUpdateFailed) + c.eventRecorder.Eventf(c.GetReference(informerNewSpec), v1.EventTypeWarning, "Update", "%v", clusterError) + default: + c.KubeClient.SetPostgresCRDStatus(clusterName, acidv1.ClusterStatusSyncFailed) + c.eventRecorder.Eventf(c.GetReference(informerNewSpec), v1.EventTypeWarning, "Sync", "%v", clusterError) + } + return } // Don't pass the spec directly from the informer, since subsequent modifications of it would be reflected - // in the informer internal state, making it incohherent with the actual Kubernetes object (and, as a side + // in the informer internal state, making it incoherent with the actual Kubernetes object (and, as a side // effect, the modified state will be returned together with subsequent events). workerID := c.clusterWorkerID(clusterName) diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index d7be2f48a..5cde1c3e8 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -6,10 +6,13 @@ import ( "reflect" b64 "encoding/base64" + "encoding/json" batchv1beta1 "k8s.io/api/batch/v1beta1" clientbatchv1beta1 "k8s.io/client-go/kubernetes/typed/batch/v1beta1" + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + "github.com/zalando/postgres-operator/pkg/spec" apiappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" policybeta1 "k8s.io/api/policy/v1beta1" @@ -156,6 +159,33 @@ func NewFromConfig(cfg *rest.Config) (KubernetesClient, error) { return kubeClient, nil } +// SetPostgresCRDStatus of Postgres cluster +func (client *KubernetesClient) SetPostgresCRDStatus(clusterName spec.NamespacedName, status string) (*acidv1.Postgresql, error) { + var pg *acidv1.Postgresql + var pgStatus acidv1.PostgresStatus + pgStatus.PostgresClusterStatus = status + + patch, err := json.Marshal(struct { + PgStatus interface{} `json:"status"` + }{&pgStatus}) + + if err != nil { + return pg, fmt.Errorf("could not marshal status: %v", err) + } + + // we cannot do a full scale update here without fetching the previous manifest (as the resourceVersion may differ), + // however, we could do patch without it. In the future, once /status subresource is there (starting Kubernetes 1.11) + // we should take advantage of it. + pg, err = client.AcidV1ClientSet.AcidV1().Postgresqls(clusterName.Namespace).Patch( + context.TODO(), clusterName.Name, types.MergePatchType, patch, metav1.PatchOptions{}, "status") + if err != nil { + return pg, fmt.Errorf("could not update status: %v", err) + } + + // update the spec, maintaining the new resourceVersion. + return pg, nil +} + // SameService compares the Services func SameService(cur, new *v1.Service) (match bool, reason string) { //TODO: improve comparison From 6869c2cf1b0d1fc5fcafe08e4e3450b0e24c02b1 Mon Sep 17 00:00:00 2001 From: Jan Mussler Date: Tue, 23 Jun 2020 10:16:40 +0200 Subject: [PATCH 059/168] Added image to readme, added/rewrote features. (#1031) * Added image to readme, added/rewrote features. --- README.md | 24 +- docs/diagrams/neutral_operator.excalidraw | 3499 +++++++++++++++++++++ docs/diagrams/neutral_operator.png | Bin 0 -> 869518 bytes 3 files changed, 3514 insertions(+), 9 deletions(-) create mode 100644 docs/diagrams/neutral_operator.excalidraw create mode 100644 docs/diagrams/neutral_operator.png diff --git a/README.md b/README.md index 564bb68a1..f65a97a23 100644 --- a/README.md +++ b/README.md @@ -8,25 +8,27 @@ -The Postgres Operator enables highly-available [PostgreSQL](https://www.postgresql.org/) +The Postgres Operator delivers an easy to run highly-available [PostgreSQL](https://www.postgresql.org/) clusters on Kubernetes (K8s) powered by [Patroni](https://github.com/zalando/spilo). -It is configured only through manifests to ease integration into automated CI/CD -pipelines with no access to Kubernetes directly. +It is configured only through Postgres manifests (CRDs) to ease integration into automated CI/CD +pipelines with no access to Kubernetes API directly, promoting infrastructure as code vs manual operations. ### Operator features -* Rolling updates on Postgres cluster changes -* Volume resize without Pod restarts -* Database connection pooler -* Cloning Postgres clusters -* Logical backups to S3 Bucket +* Rolling updates on Postgres cluster changes, incl. quick minor version updates +* Live volume resize without pod restarts (AWS EBS, others pending) +* Database connection pooler with PGBouncer +* Restore and cloning Postgres clusters (incl. major version upgrade) +* Additionally logical backups to S3 bucket can be configured * Standby cluster from S3 WAL archive * Configurable for non-cloud environments +* Basic credential and user management on K8s, eases application deployments * UI to create and edit Postgres cluster manifests +* Works well on Amazon AWS, Google Cloud, OpenShift and locally on Kind ### PostgreSQL features -* Supports PostgreSQL 9.6+ +* Supports PostgreSQL 12, starting from 9.6+ * Streaming replication cluster via Patroni * Point-In-Time-Recovery with [pg_basebackup](https://www.postgresql.org/docs/11/app-pgbasebackup.html) / @@ -55,6 +57,10 @@ production for over two years. For a quick first impression follow the instructions of this [tutorial](docs/quickstart.md). +## Supported setups of Postgres and Applications + +![Features](docs/diagrams/neutral_operator.png) + ## Documentation There is a browser-friendly version of this documentation at diff --git a/docs/diagrams/neutral_operator.excalidraw b/docs/diagrams/neutral_operator.excalidraw new file mode 100644 index 000000000..f9e48aec1 --- /dev/null +++ b/docs/diagrams/neutral_operator.excalidraw @@ -0,0 +1,3499 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://excalidraw.com", + "elements": [ + { + "id": "HJnnb_r0hPUKoBEINbers", + "type": "ellipse", + "x": 273, + "y": 517.75, + "width": 121, + "height": 32, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 10898020, + "version": 258, + "versionNonce": 298931548, + "isDeleted": false + }, + { + "id": "tCDf1dMVyFkty_0jKAnZs", + "type": "line", + "x": 273, + "y": 531.75, + "width": 0, + "height": 91, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1834520924, + "version": 237, + "versionNonce": 1077299676, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + 0, + 91 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "nA3ZdlWP2zjjNACKfYs-d", + "type": "line", + "x": 395, + "y": 532.75, + "width": 0, + "height": 89, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1434407004, + "version": 289, + "versionNonce": 789098076, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + 0, + 89 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "vtgct6qIZTm4sYOD92wKg", + "type": "ellipse", + "x": 274, + "y": 602.75, + "width": 121, + "height": 34, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 2141653860, + "version": 264, + "versionNonce": 1327137500, + "isDeleted": false + }, + { + "id": "mOLA3EYJz1RciiXTcNzKd", + "type": "text", + "x": 305, + "y": 654.25, + "width": 56, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1191437788, + "version": 171, + "versionNonce": 640281700, + "isDeleted": false, + "text": "pod-0", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "baseline": 18 + }, + { + "id": "LKNTYzb6pb0XqqRf2YNv9", + "type": "ellipse", + "x": 539, + "y": 523.25, + "width": 121, + "height": 32, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 223989476, + "version": 308, + "versionNonce": 453278684, + "isDeleted": false + }, + { + "id": "75R3P1ZFskWD8-1ssBxzK", + "type": "line", + "x": 539, + "y": 537.25, + "width": 0, + "height": 91, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1763311964, + "version": 287, + "versionNonce": 1651949540, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + 0, + 91 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "RT5N8ktBKNNZFEGC5HrUk", + "type": "line", + "x": 663.5, + "y": 538.25, + "width": 0, + "height": 89, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1317425764, + "version": 340, + "versionNonce": 966842852, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + 0, + 89 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "dfEllTQv2I7GjGNlxLtO7", + "type": "ellipse", + "x": 540, + "y": 608.25, + "width": 121, + "height": 34, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 25785820, + "version": 314, + "versionNonce": 886907748, + "isDeleted": false + }, + { + "id": "jsYpTmNMxbY44mytnrs1Q", + "type": "ellipse", + "x": 735, + "y": 521.25, + "width": 121, + "height": 32, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 197655268, + "version": 290, + "versionNonce": 1837380828, + "isDeleted": false + }, + { + "id": "D5XP-OpR0GnxMkHaVvFR2", + "type": "line", + "x": 735, + "y": 535.25, + "width": 0, + "height": 91, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1895077212, + "version": 269, + "versionNonce": 1135285988, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + 0, + 91 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "GSsk0CDMtzw5RPe6jYwQG", + "type": "line", + "x": 857, + "y": 536.25, + "width": 0, + "height": 89, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 911146596, + "version": 325, + "versionNonce": 1902197084, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + 0, + 89 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "MYgWwh6xIpAnWKGvUPxaR", + "type": "ellipse", + "x": 736, + "y": 606.25, + "width": 121, + "height": 34, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1492224476, + "version": 296, + "versionNonce": 997104228, + "isDeleted": false + }, + { + "id": "Mgil_EoL7vCEANAAQbeT9", + "type": "text", + "x": 220, + "y": 686.25, + "width": 166, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 573888220, + "version": 166, + "versionNonce": 1814670812, + "isDeleted": false, + "text": "spilo-role=master", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "baseline": 18 + }, + { + "id": "yeRW34kgnJTZIZlScfwrn", + "type": "text", + "x": 523.5, + "y": 689.25, + "width": 165, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1364782300, + "version": 245, + "versionNonce": 116764132, + "isDeleted": false, + "text": "spilo-role=replica", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "baseline": 18 + }, + { + "id": "dyn57jMr5lf-PlaHz_aQC", + "type": "text", + "x": 579, + "y": 667.25, + "width": 44, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1855903580, + "version": 176, + "versionNonce": 1324323420, + "isDeleted": false, + "text": "pod-1", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "baseline": 18 + }, + { + "id": "H3QMgY9OZFFTnObVeqsin", + "type": "text", + "x": 775, + "y": 659.25, + "width": 56, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1891081700, + "version": 214, + "versionNonce": 122573156, + "isDeleted": false, + "text": "pod-2", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "baseline": 18 + }, + { + "id": "xTZVls7LlMTme9sH-DYxP", + "type": "text", + "x": 720.5, + "y": 691.25, + "width": 165, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1213607260, + "version": 314, + "versionNonce": 411363036, + "isDeleted": false, + "text": "spilo-role=replica", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "baseline": 18 + }, + { + "id": "02J48ELeakCl3ignRYIBB", + "type": "draw", + "x": 994, + "y": 889.75, + "width": 456, + "height": 202, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fab005", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 83904484, + "version": 269, + "versionNonce": 1271301348, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + 2, + -10 + ], + [ + 0, + -13 + ], + [ + 0, + -15 + ], + [ + -6, + -21 + ], + [ + -19, + -27 + ], + [ + -32, + -27 + ], + [ + -47, + -14 + ], + [ + -54, + -13 + ], + [ + -59, + -19 + ], + [ + -81, + -32 + ], + [ + -98, + -36 + ], + [ + -128, + -38 + ], + [ + -142, + -35 + ], + [ + -152, + -31 + ], + [ + -168, + -21 + ], + [ + -183, + -4 + ], + [ + -184, + -4 + ], + [ + -195, + -19 + ], + [ + -200, + -23 + ], + [ + -207, + -26 + ], + [ + -219, + -27 + ], + [ + -283, + -2 + ], + [ + -296, + 8 + ], + [ + -299, + 12 + ], + [ + -300, + 26 + ], + [ + -299, + 28 + ], + [ + -299, + 25 + ], + [ + -312, + 25 + ], + [ + -323, + 27 + ], + [ + -335, + 32 + ], + [ + -343, + 38 + ], + [ + -361, + 65 + ], + [ + -368, + 80 + ], + [ + -371, + 97 + ], + [ + -369, + 105 + ], + [ + -366, + 110 + ], + [ + -352, + 118 + ], + [ + -344, + 119 + ], + [ + -336, + 118 + ], + [ + -316, + 109 + ], + [ + -310, + 104 + ], + [ + -309, + 101 + ], + [ + -308, + 115 + ], + [ + -305, + 130 + ], + [ + -296, + 144 + ], + [ + -282, + 159 + ], + [ + -274, + 163 + ], + [ + -262, + 164 + ], + [ + -240, + 163 + ], + [ + -210, + 153 + ], + [ + -173, + 139 + ], + [ + -137, + 118 + ], + [ + -134, + 115 + ], + [ + -129, + 121 + ], + [ + -114, + 144 + ], + [ + -98, + 154 + ], + [ + -86, + 157 + ], + [ + -61, + 157 + ], + [ + -36, + 153 + ], + [ + -16, + 147 + ], + [ + -8, + 143 + ], + [ + -6, + 136 + ], + [ + -5, + 112 + ], + [ + -6, + 106 + ], + [ + 3, + 119 + ], + [ + 8, + 122 + ], + [ + 17, + 124 + ], + [ + 26, + 123 + ], + [ + 57, + 111 + ], + [ + 74, + 100 + ], + [ + 80, + 92 + ], + [ + 83, + 77 + ], + [ + 76, + 57 + ], + [ + 69, + 43 + ], + [ + 83, + 31 + ], + [ + 85, + 23 + ], + [ + 73, + 3 + ], + [ + 67, + -5 + ], + [ + 47, + -15 + ], + [ + 25, + -12 + ], + [ + 0, + 0 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "VZp483EWEr7EJ_FlHJk0a", + "type": "text", + "x": 665.75, + "y": 931.5, + "width": 386, + "height": 35, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fab005", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 224990820, + "version": 195, + "versionNonce": 1620426212, + "isDeleted": false, + "text": "External Storage: S3, GCS", + "fontSize": 28, + "fontFamily": 1, + "textAlign": "left", + "baseline": 25 + }, + { + "id": "BrLh-5pM2Jhx-5_Vep5-G", + "type": "arrow", + "x": 393, + "y": 729.75, + "width": 262, + "height": 163, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fab005", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 310475876, + "version": 138, + "versionNonce": 1240221796, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + 262, + 163 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "UfPfV9MFcDCPYI-jf7seb", + "type": "text", + "x": 381.5, + "y": 814.75, + "width": 190, + "height": 25, + "angle": 0.45141580316417595, + "strokeColor": "#000000", + "backgroundColor": "#fab005", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [ + "3u9rzicVh0QO2xN6fmhVp" + ], + "seed": 1292594020, + "version": 326, + "versionNonce": 1344659420, + "isDeleted": false, + "text": "Nightly Basebackup", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "baseline": 18 + }, + { + "id": "SmcG2YbgL-8v4clAUp0xh", + "type": "text", + "x": 346.75, + "y": 846.25, + "width": 225, + "height": 25, + "angle": 0.4312915734727083, + "strokeColor": "#000000", + "backgroundColor": "#fab005", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [ + "3u9rzicVh0QO2xN6fmhVp" + ], + "seed": 1782203356, + "version": 325, + "versionNonce": 361602020, + "isDeleted": false, + "text": "Write Ahead Log (WAL)", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "baseline": 18 + }, + { + "id": "_Kyp73xUT4mahN8JdoGcS", + "type": "diamond", + "x": 277, + "y": 412.75, + "width": 112, + "height": 37, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#228be6", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 270461276, + "version": 139, + "versionNonce": 144016476, + "isDeleted": false + }, + { + "id": "Bmee_A3CCXFMs_Jo4fDwh", + "type": "diamond", + "x": 638, + "y": 404.75, + "width": 127, + "height": 34, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#228be6", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1791880796, + "version": 172, + "versionNonce": 414744420, + "isDeleted": false + }, + { + "id": "AwIhDdIZFowEQWpatIS_J", + "type": "text", + "x": 265, + "y": 376.25, + "width": 146, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#228be6", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 446241244, + "version": 143, + "versionNonce": 1285331164, + "isDeleted": false, + "text": "Master Service", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "baseline": 18 + }, + { + "id": "zYWhLoCY5QQJkDhjKtZMM", + "type": "text", + "x": 623.5, + "y": 374.25, + "width": 149, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#228be6", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1243730268, + "version": 137, + "versionNonce": 1270543076, + "isDeleted": false, + "text": "Replica Service", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "baseline": 18 + }, + { + "id": "IPdSpq4-kBLUQkvNDjyXi", + "type": "arrow", + "x": 334.5, + "y": 452.25, + "width": 4.965440128473574, + "height": 57.669431607425224, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#228be6", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1700066276, + "version": 127, + "versionNonce": 935235548, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + -4.965440128473574, + 57.669431607425224 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "05IcxTgNq-2qmCLEERhJz", + "type": "arrow", + "x": 698, + "y": 448.75, + "width": 93, + "height": 59, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#228be6", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1235196764, + "version": 112, + "versionNonce": 1831007844, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + -93, + 59 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "OzTNSlxHW3EHNyqPewfNh", + "type": "arrow", + "x": 705, + "y": 444.75, + "width": 90, + "height": 70, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#228be6", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1509319268, + "version": 124, + "versionNonce": 330562916, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + 90, + 70 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "0ocAfgcF_lOPK9OGRgqVZ", + "type": "text", + "x": 947.75, + "y": 215, + "width": 300, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#228be6", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1404120284, + "version": 182, + "versionNonce": 1435274980, + "isDeleted": false, + "text": "K8s account/network boundary", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "baseline": 18 + }, + { + "id": "MY4EMfFbYicpuS02Xsrz5", + "type": "rectangle", + "x": 306, + "y": 206.75, + "width": 91, + "height": 41, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#4c6ef5", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1814514020, + "version": 128, + "versionNonce": 614182244, + "isDeleted": false + }, + { + "id": "ymXiakDGTbDBDdbumOMAM", + "type": "text", + "x": 276.5, + "y": 172.25, + "width": 143, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#4c6ef5", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 111950308, + "version": 140, + "versionNonce": 108395612, + "isDeleted": false, + "text": "Load Balancer", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "baseline": 18 + }, + { + "id": "KbmHbQP5KiwSuGmdHvMPG", + "type": "rectangle", + "x": 655.25, + "y": 206.5, + "width": 91, + "height": 41, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#4c6ef5", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 571247452, + "version": 180, + "versionNonce": 252207332, + "isDeleted": false + }, + { + "id": "EpO40F5rsuuBJFSu77u5n", + "type": "text", + "x": 625.75, + "y": 172, + "width": 143, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#4c6ef5", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 995715172, + "version": 192, + "versionNonce": 438990044, + "isDeleted": false, + "text": "Load Balancer", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "baseline": 18 + }, + { + "id": "JI02M9qfU4tMF5xh8ZxUg", + "type": "line", + "x": 408, + "y": 224.75, + "width": 238, + "height": 2, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#4c6ef5", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1266200156, + "version": 141, + "versionNonce": 9610340, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + 238, + 2 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "zTkeS6HMU9Ne-W4IQppVG", + "type": "line", + "x": 756, + "y": 225.75, + "width": 177, + "height": 0, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#4c6ef5", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 2023140068, + "version": 117, + "versionNonce": 44254172, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + 177, + 0 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "UqDNg7adJeN1tJ7o6cuMM", + "type": "line", + "x": 299, + "y": 223.75, + "width": 172, + "height": 4, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#4c6ef5", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1114792676, + "version": 124, + "versionNonce": 1594390500, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + -172, + 4 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "zsE606bqp5qqQi7xyFI6R", + "type": "arrow", + "x": 343, + "y": 254.75, + "width": 4, + "height": 111, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#4c6ef5", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 650104412, + "version": 137, + "versionNonce": 1304848476, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + -4, + 111 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "yVLFLudgVoRUdAxFNJC_z", + "type": "arrow", + "x": 698, + "y": 255.75, + "width": 0.8342433616519429, + "height": 119.78342345680107, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#4c6ef5", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 954257116, + "version": 131, + "versionNonce": 645389156, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + 0.8342433616519429, + 119.78342345680107 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "lM8YhhEb6z3bDEJuu4m3d", + "type": "rectangle", + "x": 1058, + "y": 285.75, + "width": 102, + "height": 50, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fd7e14", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 837465444, + "version": 206, + "versionNonce": 34667740, + "isDeleted": false + }, + { + "id": "6vuRc6aOqhdoEuttAeKt2", + "type": "rectangle", + "x": 1044.75, + "y": 270.75, + "width": 102, + "height": 50, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fd7e14", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1395971300, + "version": 187, + "versionNonce": 556715748, + "isDeleted": false + }, + { + "id": "-fAixshWpoLjWfYDN2XQn", + "type": "rectangle", + "x": 1074, + "y": 299.75, + "width": 102, + "height": 50, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fd7e14", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 684139868, + "version": 244, + "versionNonce": 1762984284, + "isDeleted": false + }, + { + "id": "9ly5PAEyfbUB3QeBMxvhA", + "type": "arrow", + "x": 1034, + "y": 339.75, + "width": 249, + "height": 72, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fd7e14", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 27470428, + "version": 132, + "versionNonce": 532370020, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + -249, + 72 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "YhujSfYUqGuzRLIsXy9jA", + "type": "arrow", + "x": 1038, + "y": 287.75, + "width": 613, + "height": 95, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fd7e14", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 726845916, + "version": 227, + "versionNonce": 1243903452, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + -613, + 95 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "nGE3aIbk_78-jtgJnzjfR", + "type": "text", + "x": 773, + "y": 281.25, + "width": 138, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fd7e14", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 43455324, + "version": 126, + "versionNonce": 820733412, + "isDeleted": false, + "text": "read &writes ", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "baseline": 18 + }, + { + "id": "pMb4NsyES3HkiEmKW13wZ", + "type": "text", + "x": 910.5, + "y": 375.25, + "width": 89, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fd7e14", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1546616164, + "version": 115, + "versionNonce": 844192348, + "isDeleted": false, + "text": "read only", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "baseline": 18 + }, + { + "id": "CplzIegzDmsR0AWIsy2Nk", + "type": "ellipse", + "x": 1356.625, + "y": 684.25, + "width": 121, + "height": 32, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#868e96", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 617074276, + "version": 371, + "versionNonce": 1506609508, + "isDeleted": false + }, + { + "id": "FlifalHMUV7XU10a9nWsU", + "type": "line", + "x": 1356.625, + "y": 698.25, + "width": 0, + "height": 91, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1194179036, + "version": 335, + "versionNonce": 1013821148, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + 0, + 91 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "ZML-phKfbEV-wWs8DNoCS", + "type": "line", + "x": 1478.625, + "y": 699.25, + "width": 0, + "height": 89, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1460167140, + "version": 387, + "versionNonce": 1303264484, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + 0, + 89 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "sxeHgJbMHMUfe7pY2nz37", + "type": "ellipse", + "x": 1357.625, + "y": 769.25, + "width": 121, + "height": 34, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#868e96", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1167203932, + "version": 377, + "versionNonce": 1493922652, + "isDeleted": false + }, + { + "id": "XsWj_GN-Vna0UzrJZdvtD", + "type": "text", + "x": 1236.875, + "y": 475, + "width": 404, + "height": 35, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fd7e14", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 83159388, + "version": 129, + "versionNonce": 899524964, + "isDeleted": false, + "text": "Clone from External Storage", + "fontSize": 28, + "fontFamily": 1, + "textAlign": "left", + "baseline": 25 + }, + { + "id": "SR_Mx08J0VFGQwsyXJVBl", + "type": "diamond", + "x": 1365.375, + "y": 579.5, + "width": 112, + "height": 37, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1095611996, + "version": 189, + "versionNonce": 377268188, + "isDeleted": false + }, + { + "id": "11TagJjN0nNn4FtwrFBke", + "type": "text", + "x": 1353.375, + "y": 543, + "width": 146, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#228be6", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 206859620, + "version": 186, + "versionNonce": 451367908, + "isDeleted": false, + "text": "Master Service", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "baseline": 18 + }, + { + "id": "-JJEKxE4FMwgRCKN8xPNm", + "type": "arrow", + "x": 1425.375, + "y": 621.5, + "width": 5, + "height": 48, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#228be6", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1669237468, + "version": 158, + "versionNonce": 1758678108, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + -5, + 48 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "N5Rd_bvTstfjJ4_Fudzbh", + "type": "arrow", + "x": 1096.375, + "y": 888.75, + "width": 237.5, + "height": 90, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fd7e14", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1399054820, + "version": 73, + "versionNonce": 1910595804, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + 237.5, + -90 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "v3KmUjaQRCHwnpY1QivMD", + "type": "text", + "x": 1085.125, + "y": 813.75, + "width": 205, + "height": 25, + "angle": 5.92323678115218, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1552100828, + "version": 184, + "versionNonce": 537307876, + "isDeleted": false, + "text": "Restore point in time", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "baseline": 18 + }, + { + "id": "bp0xDgg7BJ_M54RKiISdr", + "type": "text", + "x": 1198.875, + "y": 290, + "width": 210, + "height": 35, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 217533284, + "version": 62, + "versionNonce": 1228465500, + "isDeleted": false, + "text": "Your Application", + "fontSize": 28, + "fontFamily": 1, + "textAlign": "left", + "baseline": 25 + }, + { + "id": "6spCo6ScZkWngCoTAcCiW", + "type": "ellipse", + "x": 947.625, + "y": 1154.5, + "width": 121, + "height": 32, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 905272292, + "version": 401, + "versionNonce": 1143724508, + "isDeleted": false + }, + { + "id": "F6nmJkeYAfpVLmj-qHhol", + "type": "line", + "x": 947.625, + "y": 1168.5, + "width": 0, + "height": 91, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1401537628, + "version": 380, + "versionNonce": 1216102884, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + 0, + 91 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "4ccPjDIHyolwYrRjePIZQ", + "type": "line", + "x": 1069.625, + "y": 1169.5, + "width": 0, + "height": 89, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1445478244, + "version": 432, + "versionNonce": 1036795484, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + 0, + 89 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "7n9Va4aCv2frULofssYe1", + "type": "ellipse", + "x": 949.875, + "y": 1245.75, + "width": 121, + "height": 34, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 789108956, + "version": 447, + "versionNonce": 48391524, + "isDeleted": false + }, + { + "id": "bwfIiq16JgZiB3KxNuUSE", + "type": "text", + "x": 979.625, + "y": 1291, + "width": 56, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1160950500, + "version": 314, + "versionNonce": 1592529628, + "isDeleted": false, + "text": "pod-0", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "baseline": 18 + }, + { + "id": "5cQrHFOQIIiWj1g6iz-0u", + "type": "text", + "x": 919.625, + "y": 1321.75, + "width": 166, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1655790940, + "version": 331, + "versionNonce": 1673009380, + "isDeleted": false, + "text": "spilo-role=master", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "baseline": 18 + }, + { + "id": "r715B9Xh8NK3tgDrHa_ta", + "type": "diamond", + "x": 949.125, + "y": 1405.75, + "width": 112, + "height": 37, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#228be6", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1638053476, + "version": 297, + "versionNonce": 1713198940, + "isDeleted": false + }, + { + "id": "IUD9gKNFwwDGssTCQMivE", + "type": "text", + "x": 940.875, + "y": 1453, + "width": 146, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#228be6", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 131420636, + "version": 301, + "versionNonce": 378231908, + "isDeleted": false, + "text": "Master Service", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "baseline": 18 + }, + { + "id": "olLquFzP1bWExaxUnEtOC", + "type": "rectangle", + "x": 827.625, + "y": 1581.5, + "width": 357.50000000000006, + "height": 122.49999999999991, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 312740956, + "version": 279, + "versionNonce": 623521764, + "isDeleted": false + }, + { + "id": "2T8bRODpwVvJ_LB346Tys", + "type": "rectangle", + "x": 843.875, + "y": 1599, + "width": 93.75, + "height": 90, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#12b886", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 154310628, + "version": 158, + "versionNonce": 1035743324, + "isDeleted": false + }, + { + "id": "15CHgGbfgXivIS1LM9pn0", + "type": "rectangle", + "x": 956.25, + "y": 1597.25, + "width": 93.75, + "height": 90, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#12b886", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 259570396, + "version": 197, + "versionNonce": 799183716, + "isDeleted": false + }, + { + "id": "jH3mVWOPIu4Z23BhJup1k", + "type": "rectangle", + "x": 1070.75, + "y": 1597.75, + "width": 93.75, + "height": 90, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#12b886", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 291171428, + "version": 182, + "versionNonce": 902364, + "isDeleted": false + }, + { + "id": "2NVzp0qsCAL-Wg6lDjMT1", + "type": "text", + "x": 1209.875, + "y": 1626.25, + "width": 318, + "height": 35, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#12b886", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1993947100, + "version": 172, + "versionNonce": 608339684, + "isDeleted": false, + "text": "PGBouncer Deployment", + "fontSize": 28, + "fontFamily": 1, + "textAlign": "left", + "baseline": 25 + }, + { + "id": "MFTqi-5KJ9TBfh7uqNVkE", + "type": "arrow", + "x": 1003.3728133276104, + "y": 1572.2587801109535, + "width": 3.0021866723894846, + "height": 89.75878011095347, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#12b886", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 893334116, + "version": 134, + "versionNonce": 473483612, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + 3.0021866723894846, + -89.75878011095347 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "H4Mnirn-bNpmp8P3r8e3l", + "type": "arrow", + "x": 1006.375, + "y": 1397.5, + "width": 1.25, + "height": 46.25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#12b886", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1660277988, + "version": 121, + "versionNonce": 2006452836, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + -1.25, + -46.25 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "Ti-szwL_0r7pK6bVA0cpb", + "type": "rectangle", + "x": 791.3125, + "y": 1836.25, + "width": 102, + "height": 50, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fd7e14", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1569973084, + "version": 397, + "versionNonce": 1972321756, + "isDeleted": false + }, + { + "id": "hrkuTxEdhUF4slXQn3d8p", + "type": "rectangle", + "x": 778.0625, + "y": 1821.25, + "width": 102, + "height": 50, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fd7e14", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1620283492, + "version": 378, + "versionNonce": 1550917092, + "isDeleted": false + }, + { + "id": "F5w_L9hECvkc5DtRnBhAw", + "type": "rectangle", + "x": 807.3125, + "y": 1850.25, + "width": 102, + "height": 50, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fd7e14", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1349982172, + "version": 435, + "versionNonce": 891495004, + "isDeleted": false + }, + { + "id": "0JIhGYPnVdHMjSdyBHj6E", + "type": "text", + "x": 1345.9375, + "y": 1919.25, + "width": 500, + "height": 35, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#15aabf", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1745609700, + "version": 321, + "versionNonce": 2077999460, + "isDeleted": false, + "text": "Your Application at large pod counts", + "fontSize": 28, + "fontFamily": 1, + "textAlign": "left", + "baseline": 25 + }, + { + "id": "ZHbFN5wfBx1NalW6UbZeM", + "type": "rectangle", + "x": 976.5, + "y": 1841.25, + "width": 102, + "height": 50, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fd7e14", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 349222500, + "version": 431, + "versionNonce": 298219228, + "isDeleted": false + }, + { + "id": "3PnSvfGnPGd5XNGLNcRBH", + "type": "rectangle", + "x": 963.25, + "y": 1826.25, + "width": 102, + "height": 50, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fd7e14", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1720144348, + "version": 412, + "versionNonce": 757462244, + "isDeleted": false + }, + { + "id": "x83a6lJgB-m58TYSJr7m8", + "type": "rectangle", + "x": 992.5, + "y": 1855.25, + "width": 102, + "height": 50, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fd7e14", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 632572388, + "version": 469, + "versionNonce": 403389276, + "isDeleted": false + }, + { + "id": "yM7IY0uff7LBVk-kR8nsa", + "type": "rectangle", + "x": 1164, + "y": 1840, + "width": 102, + "height": 50, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fd7e14", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 649289700, + "version": 428, + "versionNonce": 769505380, + "isDeleted": false + }, + { + "id": "fNOgZlF9boua0Bgb1tpPt", + "type": "rectangle", + "x": 1150.75, + "y": 1825, + "width": 102, + "height": 50, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fd7e14", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1486749788, + "version": 409, + "versionNonce": 1917022172, + "isDeleted": false + }, + { + "id": "paHFy_DHE_S0Pf7eoTgTF", + "type": "rectangle", + "x": 1180, + "y": 1854, + "width": 102, + "height": 50, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fd7e14", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 142984036, + "version": 466, + "versionNonce": 1506872292, + "isDeleted": false + }, + { + "id": "of-4scEuTkHGGLqyZpKil", + "type": "rectangle", + "x": 915.25, + "y": 1956.25, + "width": 102, + "height": 50, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fd7e14", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1682303844, + "version": 428, + "versionNonce": 813765724, + "isDeleted": false + }, + { + "id": "1k7q_Kt5-BRgsBAWNpyUz", + "type": "rectangle", + "x": 902, + "y": 1941.25, + "width": 102, + "height": 50, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fd7e14", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1542188252, + "version": 409, + "versionNonce": 1661857636, + "isDeleted": false + }, + { + "id": "0D93-v_7WWIol3r7Oedv9", + "type": "rectangle", + "x": 931.25, + "y": 1970.25, + "width": 102, + "height": 50, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fd7e14", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 448454372, + "version": 466, + "versionNonce": 278276316, + "isDeleted": false + }, + { + "id": "i9XufVTPGOfxMO3Yr8pKR", + "type": "rectangle", + "x": 1107.75, + "y": 1957.5, + "width": 102, + "height": 50, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fd7e14", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1850083044, + "version": 425, + "versionNonce": 165831396, + "isDeleted": false + }, + { + "id": "lsBHDW4ED1LbgRpMwVXqp", + "type": "rectangle", + "x": 1094.5, + "y": 1942.5, + "width": 102, + "height": 50, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fd7e14", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 103631196, + "version": 406, + "versionNonce": 1175760220, + "isDeleted": false + }, + { + "id": "9Gv1xBz0PDtcIYrABxd-9", + "type": "rectangle", + "x": 1123.75, + "y": 1971.5, + "width": 102, + "height": 50, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fd7e14", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 892045924, + "version": 463, + "versionNonce": 1423730276, + "isDeleted": false + }, + { + "id": "DpB7bLHnenCrJH0UrF2wE", + "type": "arrow", + "x": 818.875, + "y": 1809.25, + "width": 151.25, + "height": 92.5, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#12b886", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 600387420, + "version": 99, + "versionNonce": 1112449500, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + 151.25, + -92.5 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "hhcSLd8LaNTFKrVkn7cpx", + "type": "arrow", + "x": 1011.375, + "y": 1814.25, + "width": 6.25, + "height": 98.75, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#12b886", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 475116516, + "version": 107, + "versionNonce": 1221196260, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + -6.25, + -98.75 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "O-cD7TO6FEdMCeX8PHLiN", + "type": "arrow", + "x": 1196.375, + "y": 1811.75, + "width": 145, + "height": 90, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#12b886", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1661665244, + "version": 99, + "versionNonce": 2086012508, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + -145, + -90 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "3v6ZCgcVGuyCcReUPeom4", + "type": "arrow", + "x": 1003.875, + "y": 1141.875, + "width": 40, + "height": 85, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#12b886", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 658333276, + "version": 94, + "versionNonce": 549002596, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + -40, + -85 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "Lqxtc4vXdq0aP4yP3vrbo", + "type": "text", + "x": 1307.375, + "y": 1155.461956521739, + "width": 49.250000000000036, + "height": 74.94565217391309, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#12b886", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 246194532, + "version": 192, + "versionNonce": 956095196, + "isDeleted": false, + "text": "...", + "fontSize": 59.68022440392712, + "fontFamily": 1, + "textAlign": "left", + "baseline": 53 + }, + { + "id": "MpgPzae1IIzHDBhZ2dAsQ", + "type": "ellipse", + "x": 1153.125, + "y": 1147.625, + "width": 121, + "height": 32, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 321763812, + "version": 437, + "versionNonce": 100126948, + "isDeleted": false + }, + { + "id": "jz6wcBLdv5F7JTNEPJoUG", + "type": "line", + "x": 1153.125, + "y": 1161.625, + "width": 0, + "height": 91, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1259226716, + "version": 416, + "versionNonce": 1389698908, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + 0, + 91 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "QxnIa0hyHn9qELemjPAEE", + "type": "line", + "x": 1275.125, + "y": 1162.625, + "width": 0, + "height": 89, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1755635044, + "version": 468, + "versionNonce": 1209275492, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + 0, + 89 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "87YsZ0WhoaM_0KQpdMPcy", + "type": "ellipse", + "x": 1154.125, + "y": 1232.625, + "width": 121, + "height": 34, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 112007900, + "version": 443, + "versionNonce": 36754396, + "isDeleted": false + }, + { + "id": "5uRjADtUCimVL1nnCSAMG", + "type": "text", + "x": 1137.625, + "y": 1313.625, + "width": 165, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1516998884, + "version": 374, + "versionNonce": 349956068, + "isDeleted": false, + "text": "spilo-role=replica", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "baseline": 18 + }, + { + "id": "UdAXrAQCc8cnIf7FmuG9G", + "type": "text", + "x": 1193.125, + "y": 1291.625, + "width": 44, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 8216412, + "version": 305, + "versionNonce": 622983260, + "isDeleted": false, + "text": "pod-1", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "baseline": 18 + }, + { + "id": "iwGdpD-yA5HRQQQa5oObs", + "type": "ellipse", + "x": 414.5, + "y": 1190.625, + "width": 121, + "height": 32, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 863881572, + "version": 523, + "versionNonce": 811420516, + "isDeleted": false + }, + { + "id": "coxnHZBDXMLYv6wioEitM", + "type": "line", + "x": 414.5, + "y": 1204.625, + "width": 0, + "height": 91, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 589885148, + "version": 502, + "versionNonce": 125471964, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + 0, + 91 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "cAt5dmyINZTv98xAyZP8b", + "type": "line", + "x": 536.5, + "y": 1205.625, + "width": 0, + "height": 89, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 860290276, + "version": 554, + "versionNonce": 753286884, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + 0, + 89 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "3xYBhCRPLLXNqfYMftBXC", + "type": "ellipse", + "x": 416.75, + "y": 1281.875, + "width": 121, + "height": 34, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1180611420, + "version": 569, + "versionNonce": 718049628, + "isDeleted": false + }, + { + "id": "N0zhhdNd_2qdGLiaFXa5k", + "type": "text", + "x": 446.5, + "y": 1327.125, + "width": 56, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1520287844, + "version": 436, + "versionNonce": 587514468, + "isDeleted": false, + "text": "pod-0", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "baseline": 18 + }, + { + "id": "1P4LkT1SjnEcULq5T-RN9", + "type": "text", + "x": 386.5, + "y": 1357.875, + "width": 166, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 2003298268, + "version": 456, + "versionNonce": 241829340, + "isDeleted": false, + "text": "spilo-role=master", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "baseline": 18 + }, + { + "id": "mdHM232x-I25hYtSee4Ge", + "type": "diamond", + "x": 416, + "y": 1441.875, + "width": 112, + "height": 37, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#228be6", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 22025188, + "version": 419, + "versionNonce": 1770444260, + "isDeleted": false + }, + { + "id": "ZJlUU3rT957t1RdM5RInK", + "type": "text", + "x": 407.75, + "y": 1489.125, + "width": 146, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#228be6", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 173444188, + "version": 423, + "versionNonce": 2003102300, + "isDeleted": false, + "text": "Master Service", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "baseline": 18 + }, + { + "id": "B1Qvi5HNMM_roG6ON0BQD", + "type": "arrow", + "x": 473.25, + "y": 1433.625, + "width": 1.25, + "height": 46.25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#12b886", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1433139044, + "version": 243, + "versionNonce": 1299872100, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + -1.25, + -46.25 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "XDPXWDobzcFq4D7R2gqh4", + "type": "arrow", + "x": 692.625, + "y": 1046.125, + "width": 178.44955960164782, + "height": 137.8884688600433, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#12b886", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1705785180, + "version": 76, + "versionNonce": 948911836, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + -178.44955960164782, + 137.8884688600433 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "83GpNN-rHKKsQhs40OAmS", + "type": "text", + "x": 547.875, + "y": 1088.625, + "width": 97, + "height": 25, + "angle": 5.608444364956034, + "strokeColor": "#000000", + "backgroundColor": "#12b886", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 728359772, + "version": 135, + "versionNonce": 1244507364, + "isDeleted": false, + "text": "continuous", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "baseline": 18 + }, + { + "id": "0iYzRksu9bFd4FRhNS4vL", + "type": "rectangle", + "x": 356.375, + "y": 1152.3750000000002, + "width": 228.75000000000003, + "height": 381.2499999999999, + "angle": 0, + "strokeColor": "#5f3dc4", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1912617828, + "version": 132, + "versionNonce": 2054882140, + "isDeleted": false + }, + { + "id": "_w-R0RLuSZGXtjYndbBuF", + "type": "text", + "x": 205.125, + "y": 1332.375, + "width": 225, + "height": 35, + "angle": 4.723624462642652, + "strokeColor": "#5f3dc4", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 339311844, + "version": 143, + "versionNonce": 1093562468, + "isDeleted": false, + "text": "Standby Cluster", + "fontSize": 28, + "fontFamily": 1, + "textAlign": "left", + "baseline": 25 + }, + { + "id": "_MObJgJ_wonJNtPP1p6ZX", + "type": "rectangle", + "x": 430.25, + "y": 1656.625, + "width": 102, + "height": 50, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fd7e14", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1745745252, + "version": 505, + "versionNonce": 2141117532, + "isDeleted": false + }, + { + "id": "ers2ZQYACj6DmohzS1V61", + "type": "rectangle", + "x": 417, + "y": 1641.625, + "width": 102, + "height": 50, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fd7e14", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1610711772, + "version": 486, + "versionNonce": 1938859876, + "isDeleted": false + }, + { + "id": "a7YApsVVLIgOEWz-wRw-R", + "type": "rectangle", + "x": 446.25, + "y": 1670.625, + "width": 102, + "height": 50, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fd7e14", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1022470372, + "version": 543, + "versionNonce": 1220210908, + "isDeleted": false + }, + { + "id": "-HPd9HbQfjaSSnyVtxzAI", + "type": "arrow", + "x": 468.1411898687482, + "y": 1625.0761543437839, + "width": 5.733810131251801, + "height": 83.95115434378386, + "angle": 0, + "strokeColor": "#5f3dc4", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1743894116, + "version": 51, + "versionNonce": 1766726372, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + 5.733810131251801, + -83.95115434378386 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "74_FKllXVIzRwRKskBVca", + "type": "text", + "x": 238.625, + "y": 1657.375, + "width": 143, + "height": 50, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 990652900, + "version": 76, + "versionNonce": 778073436, + "isDeleted": false, + "text": "Read only app\n(e.g. migration)", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "baseline": 43 + }, + { + "id": "nhSbj-rOoBSaObbxkDC06", + "type": "text", + "x": -17.8215120805408, + "y": 418.7138510177598, + "width": 297.4346399775327, + "height": 35.408885711611035, + "angle": 4.728012709005166, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 107991652, + "version": 133, + "versionNonce": 1041309284, + "isDeleted": false, + "text": "Postgres Deployment", + "fontSize": 28.32710856928882, + "fontFamily": 1, + "textAlign": "left", + "baseline": 25 + }, + { + "id": "P6l-7fAr8MMc9KsB2kZw_", + "type": "text", + "x": 409.625, + "y": 30.375, + "width": 1036.2000000000005, + "height": 55, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 2073692900, + "version": 209, + "versionNonce": 1038182876, + "isDeleted": false, + "text": "Zalando Postgres Operator : Supported Setups", + "fontSize": 43.99999999999998, + "fontFamily": 1, + "textAlign": "left", + "baseline": 39 + }, + { + "id": "bZZTypDxRVen7t6-5jXqf", + "type": "arrow", + "x": 404.8103958096076, + "y": 598.9661255675087, + "width": 125.49452508728018, + "height": 1.4560642183292885, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1915235036, + "version": 95, + "versionNonce": 2115434724, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + 125.49452508728018, + -1.4560642183292885 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "g1TuJyldd9lGKOVeuxdCc", + "type": "text", + "x": 424.125, + "y": 563.625, + "width": 67, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "seed": 1359012196, + "version": 11, + "versionNonce": 173002332, + "isDeleted": false, + "text": "stream", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "baseline": 18 + } + ], + "appState": { + "viewBackgroundColor": "#ffffff" + } +} \ No newline at end of file diff --git a/docs/diagrams/neutral_operator.png b/docs/diagrams/neutral_operator.png new file mode 100644 index 0000000000000000000000000000000000000000..b8f807d639e30ee419205b3b3d8c484f4422bc4d GIT binary patch literal 869518 zcmafbbySpX_pXG5QUfC0(nupi*U*TBARyA6LwC0@AkvM5NOz}nBi$ej-Q95>-}u(= zoU^`j{+q>mYVUjRYhQcc*G-s`qBQ0+;%85uJi(Nefv7xrf-?W)3F{D{ zPbig+Y6&p4^xKem3MYGLGzqe|){>`x6L=M+}qq*Vxa6xqFaBzEG&q|PPn$bM%UEDmqbbI=M5z}3`i);i!;xRiYawrr{J(Aa^PtR`4W9p}!GFEY?A#*UAF}*&Q5zu@^q_t-SmQ0a;zkJQuoKT>H|H|L&;c`O4dvf8X<e2njep+_#yDR9T(=@(mYd5iFAaiMxeZ`U;l?uZG<#OZAjRGs1%;;o(B#Y&kubbZ+m8DJsSQK9>0Bn_cY=DS=|T_ zVqr;xMcqm;$>+*X`a$@?H?^?naoZj4Wv|P{vR>-&z}v%C5e2^W*p;*KcWf01JpWsC z|9+%DnlX$D3L?+}H(hU}tk6(g%hWyG>~{D+XJw%N&$a9ufNx>-jF*A5lz!!Mde`d- z*tN^v9(Fh3Y%Q*Jo9Q0iOrL`{y$+i%gS%dSv@&aS+f3sX1lA1GXA|W=U;bxdkS*~AEOKd z+vtUl!?IYYYxXJzu#D^fcG~5l%QE!nCo1;s(7bIc8C{0w$@dF5u$=#S1TNta^4Jbh z>Zbj`YXa~4UEQb+)r*DyB$_DWpI2b~^NIzB5wfj(Ul<9m%@3b>LcaU&il?U;kDgcS zY(9K@ytd8f>Z7gyM@T)mYUVY4-#jp0NZ$7|BvZSb~&qW*(Rgbsg#Y$4X89SAZF z+N04ZZ*IZ+v+=I&((-QNm44<@(n1E}beH6(guLG8lOH-B@0TBW?)E9Ij($mC2T5QH zRn?Kdnb8Xl~HL5%;&GGlr#YI z+P|mR@GEVu=+XWIGK87{*3k?sXaC93Eh0beo!G(RaMp$&rak02NX-BauOLxeiMrmO zPlsUMu7prvoem2eQl`mmYv{VJ1hGHfouWRF!O+%_um#y8C`7z}kBs2`vpQ$b{alvZ z=(WAC*Cj;?ElcX@?k-y&)6C31=z2M6c%4t)8uvy@Aif1%?c4g02;XnN!@k8t@P54N z5lNmKk+Hff^|@!hyW14OrcZY@xay#IU?0VyOR=gr0-PEMm~8SYc$4ElmWTqDAc4)X zGyxDFqcsmS(}d9 zkGj~r%g?wsl1$UfRzEYpRKy#0@_YJ{@NC@a?(b;xOUI=&Lq%BqlZQ*y25Np<0bsy- zOW_0$ekeAb-%GSR;KQ*1*WZ@ue$Btmc)ZJF<9fsR+y{W1Ai!&EM~ZjdB(@VmmYkO+ ziTc4Wd#~{R&F5}vgsz%$@NRk~@+)>abeyznU=3exwhXUUhJn>VQ!BfJYD@xJ>K%c)8y0*LDCFX&bd@Zd9{<9~Jut08spW z4dz7^VgmkT`Y&>XEcv`dgU{d6JmDnEXwsy15Dx*e+}0)Q+Qs&0X>z{N+0m^u`|tpp z*r-(7mg{x^!Dc4%q4$(;zT)%gw+FV^WpX>=+OYgxD7GjO0y$HUje3>7mc)DA!6~o- zNV1(V%Z7da>KOaI@%~~y-0Ea>47(;kV{!5cNW&nEmX7Hscs*7X1$ir)JJRR4i-2G@ z_sk$W$MUtrlROt{1QRsNcY?oWXG*Mc$BliK&DO`iTZFfC*>YLkK;lCW6b@>{lIPiGoR+Emz*3T+X*U3Y zw3EIk{`~ce3jnuPllZRJVxqj+(^nwKIMxixk!Aej3S2OL@ybC?`KwKlN4~u#lMJtm zquSPcYshdM6S)YGQ5|BaWZa{5CkRv}UZ*L`7(fjsi7r zPYo7*#8tA|U9Pt?0#Yai83z|j@Tl#mJj#p3s(w8l!B4MfuCTO)zX+M~9^V-V>+;Xp z2X3Vm{Y+h`);6U8tN3B@tN?ajaUAlr( zJQCV8Y=3Xhcil{LnG$m1YzW0z@_D>RYDVb!Y+c^<@-L#Gm~g%V;GtzAz@57Lw>{F| z;$%J!Fy^JIfM03KLnq*MrN-c?x|WuU>e3)=&)-OIN&DVPC;RTU%Z!M+M80bs$K+io z+k(Fz_(3W9q#uA6VI0M~gD>`_jDF*a0>x-o;TPe$cUd^@QsJPPUKbUcSdz{3?KsdC zBvV4@wpUtMWw1+8VE;vLH_40a99mcGan4S!0Y&AtG5Ap03Xl6yT~Ydo4fW9D<#y38$EiNgk&uWg|kHIyF16-0(2B2?LfmLX{9!77JHUWxJRx&AF zlEw5VmrvWJ0PA=c2%u}WYiY%!R*atX^+x?n&7$*s5CFF}6RGFp??gT!Gi*!^Soti; z<^iCXEDpX8?&Dl1C-3wv)3$;Otp7x%Amy3aN zNlF)D{8bVP$h}!CIh7?Fu~4J>N9$^D(w)$#Ft@>51<#Ah49qiAqPJ3EX;tsZ3^<9t zfLV5Cxkd!>Y?{xfvIKEJbFuC6m{d2@tq)G3ueNv%_eO+otwhaU6}SOh*oxVZ$EDzL z&s2n&GrH;k_X^0VPYc-qZ&4YgkFu9DXBTsc((yWkoC6rU;inhkY@HjY&eVC}-Bda! zR6Ba%F&#AeCi_t!qoT;=A+^O?!+cTJci9rSMa1ogYy9yD|913S^Ko`i%#= zOr|Y5DYOkivPQEMBae+*SA;l83nDH>BVe9n9KYT3E%GYyDpc}|EbN3kgv@y+G^#iT zDRZ*}SDuBVN}e<6>+v5;wg|Xv7V5+qAYZa= z=K=nMqsYIVBRbo+pwR)U7?5*z3il|z9B^_27}|?_XB~hpi7U`51wM!b{Q8Kh6Hv#R zbE?3OxJ!$rGleFHHGb+05DqlvqO$hTXY&cV-fpAe3EtUx>FyTqW4dOn z(C|J`)i5lJ%B9|QD?S+p8V`ykS2!#zQabaQAKQCMqW1$`Zwgc#sY6Yb+#GKgy_M?0 zHh<;x76rhn?zspeApku@MBr>NTl!=Ij_vJVp{NOe974SidAuH3&8Wbx)#}Je_iiQ^ z`~+_9Z)^f;atDxvd{sijI~7szegV>-nA>g>L!9hjd{DHC1h}11rwgZO%PzPwk#27< z3ZtNlK)0#1Ld6~x$9Tv4BqrlMUsqU-*xFV9hZar+tM6xI1`@A3l%a4K?@L+%P`(}c z7i%e#1?q5?v3N6on%zJ=MTQCxiE*tx1yrQUpk6hN!6vKzcG3n2_Od>@loz|7UQ0mX zq+0lteXcnPX#V7&!@<|nB1u)TaR%On#95b7K6mgxE%M7-3!_lGP#YTSRMTcoeKqO_ z(B#%@3ESF`f}jQBj9t^h5->md_!bd4?wA)vl@ z-H0VYF_hhb4!h)P_kuSY^$*t-bi8wn65QSX*nd8YeQ9!QI=}3()qOrvxdV`Viz^^h zbxs@o+CqWS@O|su$7*e}hmz{MlYx+Vo5o_eufX-5wF%Vi_siB?7pjqff{0{3onq@K z;hU|j1^XoBxjSi*8|8QXo-T&uK$<(OE!OOuR^xfz>Qgkk*B*vAIAQ$7fZ1mNR73H2 zAZ-RnoiQR953~rqQK`Y`p#}}sN52(giz@I}NC*)cJ+P~Qzxo_J&h(mjcK4Xuqp9`b zymaNc?v?mTTTf_7>(y#_Bel*71C87pVq;pnGhX|HCCBl%lInPxM8@4bC;9KZnEs3? zf-31=ugtJSDOyE!5Q{Mp3)(r?TIfEUBy<2|p6AS=lj|U5;kZ{wa%vl(8+c2A*Ty=A z03z#0P6FO5l#=R5+cI_^;2GD&M)k(0%CpJ@qN3L&brM|gc#_cif_f?A;ViH1zMKE6 zNy}=L6rsOnwq@ZYpRAAc_B)L!`>ecq&U7-Dh4S7F&&S(k-R+~)!JN_#e3Ok9<{QDqu9`;9Ka;x)j~;PGPCenPY)gdx9P|#HwHVd-qNB zQls&vyl)p!FSG>{dIvM}UQnac4x(Y0_w+8n`8S1!k?_XuzivzTC~;=(5GiWj2yHZT zTHd2jCDx~VUChq&!AkCHQg+38Gq}LneeX*Qz@vR@>*pO zUXtkdzeCgc-H^Q_p5(MMJHI7paWACli|m(EGi#E9@9Az0vsv4RG?ap?uR<2)6Fz^DPg6k9+%h$s$pDWa=BXmH(H%L=;s`HR)ff;9Ac*)GqyOC5=UyU-%PVH$h zc=R}htJ1QS2K7qSm^F+5Jz7!SyQ@!Xw`=qwn&5ZPi8!L9LZe^5Cv3Z*oc8-Q254=u zKdgFsgNtdsp*D9yNu&#=MwrzGV1qFF#rx7xTh;eYHOj0SKjIwctRmW+FDXzaL&2AUh8!`!A0Ng}Od&z0* z{7m(@x;>um`V?BKa`mh>ybUrf!1;o?Xh~BD5aF6C# z`b;Ufm%K47%3Z#hs{Nc9;GQMf98p-%V^~Zx&R44_6<2EK7_*Q<1i@mXID0rfzc!5S$T&MAr~;vZii&qgLrV-RkB~bie57AZUI@GklU)2J|+FN7qB&< z^KlQa%g(M~C8yLFEeoPc!%eerWSjMNH_T-D@!|5h^o!teq)q9#-5*ng&|oSPr4nYI z&I)(0u^Dpu*H}-fJHw7>;ZID(XyC$P7@4-uH$NXq)(}Tx%h6Z!A0F3*zpl}+v)-IF zJfgb|FRFd-+AbSrh^^dEV26b!TpZuOHrE#Y;;`?o?oYS*Jj9+$(S3^YfSJ@~wC)m$ zG41dL6b38nKV*396I^8|tcX`zpWJZ{m0ajCpB-cK+Lr&*+}Ji2ky`cQ$`8DVAxkjT zglqSJk`vNd1n4U;(5W-`K22^pm$+PvoG(XA?kdNc&b+DA-5|>ox`Y&4W|EA|=#)NR z^d3I4TJtWTjpR;_0VbWb(78ALdRA2#Qa%|M3N*L%i|4hY$2<2zyJUVZjpiru7F}IF zx-2?{U&d-3g0Af}-l4b2VV377De!UjG88sWX<6crEZz}G9LlGMpQqm*UzJJinZJrG zzbGFmXs5K~TX%LN$TwYVc^JQhtCyg2cCU2YG998DQ60BH)CCUF_u~^xm(X7f1nBUe z?+uM4xfw04c@?r8J=KS~Aj)wW07IO~GCc+eX~~Cl4jSX8T03jdmP)DB8$(*Wq?W;A zX1 zGL4=Dr zvIVS&jMLV>gUU}e!m{W?umew&YtPDXwIBTX%R#DkREj%}_LiM1hV`Cm7!qj@$uP#u z6el5!LZoHu)bEQlw^I_{7b~fovfRiHU=I5o$EH7G7mfaz!$Q%@k8hXic@YeH6=ygt z(Bzd&lRfhhOlmh^sM9I#Te>M@Vs-3o!!6wJw5lfy<9Y7qf1BcFk`8{8I{+R0n)HP{ z!!kf*q%stzAS`|u=4SAi%B*}C!8_u)_8?{!9oa-^>-&r;r5_8@a);A3=JF4c95gph2B4icg0U~Sw}iLF*B z{s@G4$@6CS;WZ*d=$cS~Qz&252&9A~P&+*zDidUkjDWnUj^tmUU0K9a#7FoD`?81~ zJk#dX$JLS!Dr<9waRz~2qg%+rShA(E-Y*D`_^%LsQP&Z@qxu!#(Lf~e{>P<)6|F#o z1MTk|7WwZ4(;|qdYFMrpx!pH!H}Ga598aF>pcVd^+Zl_M6SY{$Lvb8BP6G{3)7u)- zPq)`9dC6!FJq0%{pVoC*qx%~QnKW+*Fbq|g?3-;ge$$j1vU(@$nl9TkKEX_Y9OKaWt?T$Zi>qYFzIjZ)lr1$i9w1B;!Uhi$-~$a zQFa~eF}J1~y(S%;1pYx_0f3OsecQAJwm_8M`Svy!*Df_Mz~hONG$7<3`Xc+NRx(!u zg2p2AkIQ?cc_?Hc5hr#qnp7Q3Wp?qzZ=M>(EHPSm{=Pw5zQ=y=**pTRp@K)p3DRq; zO4&`I^`0xr^+&r*+-kA`-_gpzlR&jJ8?;ZJXj0jCIEWL4l|Ub%qo?ONTwSIXSiSJbrq+y&21tw zR_*IHcif*`4LLe&8` zoal>K?1lIWIsHfWecRraBra0$b^VD)3C!$B#$$J7p)hV`_D8HGjn3kOM@0v8BX_gA zA>LHKhAXlOLTm8;1^J9^(vM_v>`^BEW7X(DYYkgX7fT{b{GU&X%=1@z;Uo4S*dA88 z@wP+oIVq~M#ThERNkt+@axLH3C<kG1_n*v47t1D|{2 z`d88PhZh{77%Yf1S;xzNHqQhS+!mKlw0|cJH{tzKF61_!M){@J6I-z1)_wXz{X5%X z1llID|AagFk(K;rK*UDXnnU*Kz|$xGnY~S))#*yU-$Ewfi!pme6NCiq2AaQ+5Gi)W znlVhLDE#YA?7R1(?l+Q@n(H3WQK(PZc}ny*%5=Kf4lP5uiZ481Jpo)JLJuoKel9Cg z%lK*D7v1HQHY&N<7HudIXkYsaRnKB(UXH*-as1_66{qp4HEvxmbE>>(b~^>xUv9J{ z-3m|;T<(s^vwdo^tYDi8=hPR*p3xUsa8VoPA&;?5K5P8qvR8-Y;*Po1-RPCgcYgTK zm^qC)b7qS1>wz>dVpK!EVwZT*OY;Cl`As^=C0URzIC=r*XQ+=Q)?|vMVcJcS?0!NK zHVUnRBb(D1BAW70XlZ2IM?sjVZ*nw=!7$L?)^fDTY9~iDdR)9kU#0raabHRy{8ye4 zv4N7-o2MWQgeR!h$+TV0M!~u2@m{|0z%83$n7t+%j*)EF_l;0JG&8WP3Nzk3G*&tN z`{0#m&Yf;LRf3KD1e^c+X8VaV>x#p=@S6^67*Xb~dNNI|s zvc}fPlbJyLr*|-WGbnuSj`1Ytz8e~(8$2dVg!Pcxk?F-+7$^>KZuXiP&R6Uex^G50 z;D*Ro2A4^zW0NLtBZ<#4l9Ic`rISB-62r%*C1YkaWsbB0f9+o#!#F-p*f5EJd@3eoeN}xHt-Ox zzR*rpBIaY2;72W+gmTxys?~1dfbGfNM!&phaa4mLW z`|{fFSDyoJbaPVCmx9ZgOzWOoA~iII>JQ8w5vItOP@N@?i`chaH8G0(<_x~M&%*!g z2=-#Z;y+uXt(+m0;LG?Qb!oK;o_Gg}MpBZ%Oi3@zbU0&M+9N6laiTONW(Wk))FnLM z!KyPZYbM<7?p3AOF3MJCntaM5yK%$LT{o#!1ZZ`#+UX}-37cKM%8 zAuTE(zMt{<&;WZK7KomZ)fX}}>wNmD*kI)z7|Ha>H6D5j{|g*n=rE1qWW@qipEKAF zXQhDHxCZ*5XY}b9q-x>`0TIcnOL3F-f!!?6Nf?=%Ln%W#vihZs$Tby2LV;-UYA=$jo&_dt86S+V8<;ut_rLMd~xk?JP0`X}N%%n$+2tE&2$QFbbsYhY= z)2{mFk0FJPaTuZGIz(Ua74p{>TS;d;t>7*o9Hy`Mf-?!NW6wjY32jl6`w<`ZB-9ZS zoF~+{1A8hhByP)1DvHKa-h`J$ODkY`zM=Q9^}%mOsSEtF$7*UuL`W-{Lk@?Ub4?Rj zTcYSWn9A-urJ%#RpD-S(i+)jVTFZ!AgdPH{GD4y#Rdf(Hu|pVmv_yimoHgDZc1s5j z+-)5UO*CyxZ`di9X%&~`1mp_0VmU6GFFoT*?bLO6PFjxE$FN~)wb;Dm;Zd$50wR zllZyGZ3=JJ6b&h&E?r&y5xS&Pa3Zy{Qz zvV>1GZf$A}wgsX}I;_OIi2h)4TMM`Nw`?Uo8hPs?`D02vIAQfe11DvUVX2d_}8|}Z9&n3;{P?A)_V{7olQcozoW5+ z40;NstWm9DaOtnm<=hNK0U9ikSc}1IHjoI8c>%xUcm#;_HkON}QV@c=Irs6~uRj-ICDpDn0n>sEq8ux91 z5J1inx&`kg2OXB>wIXX@&UslrgI zs<0ww4NhoQbalJu6t1mhnkhL?cno65CvJ!?>AqXM;ho2@->F!UY*0uA{MavQE9Igf zyNNmX495yY+?Om7`nL35y|bpmil|eDdXVItVL*~_nAS;SBl_~srW$ByG19IynlZ>X zp4gvGs3x>ViWIUYM!6;!x(dh+zBdz{9{+|I42gr;xHzvE7JG@g#lXR-VuEpK4FON2L?EGsDoc~+l{?)4&x2#A!* z>8Ujx2ALhwpIK{<2&#^sn7Q@o1Udw6Iv36V#ebn&T@EIdLvNVR-v|Q!M+2;xUUJ=3NB_~ts|8;Zjbu=Frv-#igzl?tYq}SN6A{49LAX1$%$pDGTkfN;INKD{q6JpS|6q81WZ zogNr6q!!NqgPQzeqUFVSFBU@pT>ujrrRPdpuAq#CQNl06mc<7Fw@OpF`84mFZKv7E z-3|%ObSQU*=?qv!+8|H&;*Uw zAt{Tt#rt+8d7vKf&p*0>_HRn+)K!b{4FeKIXpC&NRJR(q#jn7wV7Jp*H*V;6UDsr^>aewo z)xU0*q?4SC=+-(7;7WI@AS+bi#5aAl%8Qle`8Kl0La5 zL8{Vum&kOhP%A6{ycrb=(+dVPA39F@8DF z{OKUaR`qaNa?oaxJS8}}S7A;OB8a~A`lR_`A@OwUOb{wK^(lRKLAog; z-uUT8vwjNx{12_KCa-A*@h0AP>BX2_AH&+6@Rl1Js!?(bcPARX;!d6Z9cDHDX{Bhl z05a<5M=z>NBTRT2flV40n=U^vUF}}sHbJ3W0)De|?zBpiu0YdY9!mtO3WelXwoYA* zwb)j3vYqUqRrp>d7Mgn2jm8SxgWuND<1w(|#~wxvK6mGcBQne?m9zmv$BO9s$2qX5 z>-0&AbzbT&1ec$RrSX3~fk*qr$9E2u6pOVKr2^(H72jdcmDn*(t#Qb>ntudk4zeHWnJeMt-fYQ+fmnXt%2b(f z3UABG>uGRv`fhI~wa}alYhIL${YWs?IS?57%F3Ki(BoUC!G=R!s`AAo-s$#clP@Wp z7@6e#Yum*^p0rlf<3;7tQrnh88(|OhnmNzYp%5%t6J~!EA&FnP?NY^}7)u@FEI&*% zU~4sQefH@ejf_uII~(}0VSwnXE3t$G<~^{yIUBBq#C%0Z70Q4a+*erPXo=Z7QkZaf zp_bSH`&$;5`4Ujwm13gH=qcb)c7*S-lRD)`!z3At6ulfbK~1rk7Oz%U1WP6x86xq& zQNB*nu;gSEkm7q0(Q686SU9oQH*%O9C|SwIlm4Vt*}8X?>aio(U(0COKm5k?E(4h6 z=jK72#gffm$TzP=7Pt!$tVY_I(6~c7G=~#=Nh72#`hgu_@Y!QM9vZ6pYpTp8%y{X5 z)xmCbi9nay2o+`X4Acr_Rn$*;Q7HmR4CWzia}iW>_QkpD$p!Ik-?Yh_b?Y`Q(5KNv zP{|yX78Po>;BN<%s8YY?DorwoB*x>wU9>+!e)3jS6io|Sva`K=FL_tuqU~-GO-6ue z%IXQqM%Va3zjs7Z!Ob8m<(6ZDxyg+HDnhSDmN=lOXu>fjiEFjExT>yU+jUZ?Gt|m?!bL+)ym21t*OJhTGtO^ z3>$My&%3wJ%H3*Ws7AaJ;@?`IgN@|!)+77abMJVZAMY;-a#K_XZpq@w5_s&|1JOI& zDK~z~38q+_YrbYqNFzrL@gRlICm)RG`d%HBxx@EiG4g9? zM9?LuqJ0kCaphCY{jWv=;`=N^QFY$UZ==^6#y?MWLSf$T@)f4a&5XSqPQ8@&19WW7 zu^p!J0KuJyhUS({F~a$9{`}zu%=PB-AH|+K0BTLbuPFUEWPPI~YnV< z_Mpu80HKLn8IuXdDqh+KPp@seA)w#u(17-Z!&>^}WYw4Ic?AxepAOmm_-C?psm zA{{V~nv?P}`k0oXco(_prZq+E7hm0!blwnWhok2kLniTyK07DQ7r}evW~dQ~$Y#`) z{6ss#CyFwDX~bx29JONV_Nf>HGIy>JHlumsC0nk-r(%_9qi!7@n$r;p$PsY zFb@j#jL^N``GC~^j@zHHP8eXzCRxX@uEvkNFe|fkzH&zu{MK&h0;1H#xKYC(f)Jtvyfzb&xKCmX^V(5%GUsr`n9K^A5I7jNYsMfQWCf8o`7y{lS zgjtjJ{VPxCKcdi+4wpJp5ADLRK^ZKCcTLxrv)^OcAXnQpxt7X;FSA}u3+;+$kHw>X z2>ty>Yn*SUZhPNJ)m?s)84boNp#sMaiKQ2}+xDJF`F^{z&>D{A$#CN1of4nY59FNc z(RMBB5@vfoaQdUj)MT=rWZTqAc-VO=sIF8;&=lu=A8vU3Fj_HgpDA0P2&_JU(F{h7C!-MZV7IcF4%gHoI;l^ifno*_B`lR{rGc@s%>V z3YmEqMB8%O7A$znMM2-(llaIvaO-fqAYRnokNAC6SVJDipudwO@nosLQLT!bq;!7T zM@J=vqwK&5o>UHq#a?*TXHD+#ecYYf-l*g~yzcW*W=Tnmq8Gt_NFNJF{;+o1)p-9$ zVUdhzLq8SKl>GD-g0?XqG*+ax48I~f)DW~Bw;)5iF&gf&I=!d&0y5S7?|o7Aa3i zH{~om*;lNJ)?c;#?2uNe;-u%p%BXU|$I|BfaHZD;l6%j+iZ}cqxJ!+11avxaggldbFnP4G#O#Iw_EEJ zHKC#Ak2woses~dEjv5oV1bPv@G4ad#cSc`-bDQxH*CwU7rdZc7Zm<&@uUMQ9TAH+0 z*W_}Ow$A}u7(!L(vmY01-Rg_#5L&5DgH)p3UTPJOWopZ7mqI8%FpfZ8f;1z*Z3gch z1--ctjSk-1s{iUwODyK3f9{J>G11GQ_bIH~DW3mLijew}Zr(@Q+PWxERv$zD>L4A| zPocKykdEjRG@l@MnI)iN|CIde_8X(L^5IIlRacYypGmSN&+o$+l!L}=)tBYH!{KG| z<6kUm;|;kZk*MMoo%a4a;8yN87@f%9gYhYadY^1D=sep<&wV7f?{j`A8NWe zeb9h&IlaG$afqLPTCKx2sJOtqzZXz6%XlgoCVYh~pxy-I z+9AUu?KZ}WXo_2|d_DKW=~(MttGWowll0JvRk;ckp56=C6t*PvDs?l> zLVdHoF`lVd1J_aTc#;R--ymjJ0hq={M{C00GW!WY__}dU1&lV)5h3@~@n4K%1b7&^ zLVG9o$wl03%b0$HB5{gOP)_SzUy39`U&QjMo^Wzh%Y_^%r<&gJ5SwapIguJL7gwnq zA14zlGN(2w8?AJ_Jjkz%3f*H`pFY9$^qf%LU~nnw`D%hD4Ega@%paOaJZ#ZfQl)@v zNFy#NWvfd31tWG#sRKb0J&T(w(u?y0w38#4Cdl3jtOhP^;-#E-z)1Wsgl5i70eeXI zw)T{f&W80D7PGymJiz_Xn`4R0^KA51kbUN?#YmV;o0F4iYM(XY* z;BC7rDg)^b#sET<^-DlZH^uxEHz5^&GAz(vq!y+mJt(EnjX+Rbl&VGFLvk4e4fLI0>=1(yE-(ae zu>}QZCy%w8Pq7$g>5{J!C%pm_^eqk6$_BY`5{?^ImQWEp?(k_|2zEw?Gt3si*0|nK zIM7c$y_f|8)sJbLE5f6c#i3@BZ+q>~RWZ*GG-MQ0%Y{|Z100Yjbaqz0`M4CI@oqqF z*Zz?Tnnz>_LunI=bR7|pOj^(VxJ@RVDN4N8c}PgmLl+wE`z-M0j0|S&$_8uJ9aqU| zf*&8?1=WGErtIa=X>6jO4hS%aq2z=1x&;iAo2Q3%DU@;jsCX*y(dV|S;*0~8{dxC4 zY91y695_Bs)l^#>X^`a5FqU!5 zM*gbeeMt`sK#JDShP~NAN1{#!nQl$0bup~?A+;2}L#3$j?Cch%w(|iqLNjralBM2k zQj8z<$_rIsr&@spE#h9u_rSIwHQL4LW?*|e@Wy8Nc>Gr`$koFwfsxR{ zu4f^bin$v7AbNq}oQjIrb60p~d%0S?aaV{l@j8aI`=lfd9J#z4UqVoC5YW!{DupGh zUySeBp4hXwi8Pp8elYgP3aNj2V*gFed*BTkHiU3`_x`JZHqKL=(cSYs`KSzOmg(@- zw>}}2Sb3i;LhGw5F8YU)oIanFFT|)FG<#AQ`}59!Sx>kiUK{DP5p^{0I%$c7h|%b| zWMO|K#UAEQttX&&g8tX{2 z+7zv%9mko;H<)uv-*du}mv+CBb zGRm(L1nY0pV8kuKwn*cKmXd>e z#WWl57OVFJ(U`K3ots&fnsQDh2|m3ynxkUkK3&j<$=*hRnn2UUh-SSJE3*2wWSZxB4dBav4!!`ybLS5Oy#anmy+$C#_f%SogeoRtOG4@tV+lI0#nH(s z(0hNhakOWERs?`!JNmO6ko}VCDXk87uXQ}%tAO#EXf&A|j_ADnQ^o^kTZME}5$IAx zwL&kA_zMuj$c$W182irH22%!#Zj5 zA__;G*|{0`48+e&#CO{L*t4Po$S%9d||spA5C7{ z6pm9R*{YS!dmgb#2d9%~uC6*=zm?lp87E4>xk*f9{NNfq!Gtf&GZ z;g#U;F>^z*jYa0q)q*|OQs!qmcs-;0rfgX@nup_@SVmIEX-bxRO}lm_$7DeYKdWqQ z^Jcs6sits?epcpAK4gS0BO?MPn=Gb>xLbkg3v4?N84(R#mvZs>lJy4~$1dh=MT$&K z?}SMIoIzck|1K97Y*T_jb8fSYAzvH!cTryZ)Be)|?Z2}06a6p@kT8maDLw3*MdeLHWJT`&EpeVPSeWn5ZCsoUEY$Y+((s476nROni{afoZ>3 z)ixqlyF*NNIiKNz^7qlG!ffZnNd%p6mBsmCD2Cl?3|}xD6ND4}mBEZNqJY2tN=TE~ zPWXz9!p9iGozwXh8WHJO8%6Osvamc%3H%*L)$A2c8#2iscExmup-{@yN0O z$saVG4vV|Orpc$DIEbzB4I6Xf7G7-suzwh`35W?f&JD)n7@@OpZdI%8`56euQA872 zz#fHWu0r8At2&f&Ed|QXUvNe8(bUdsjpx;dtPIobn5cb6QHc^8WR|RmYchm~pPERbiMmcBPJV?NoQbU+j$JA)@oZ&x^G9U* z{h0jGV=&8C9!aVF)7OgH-&Z4?@u&9WiFfO?w!@H8zJ`D4pOb50H#!xz2XB|~`!c^{ zSlcXA#u|2Ouo>j*PhnQn;>e!1{IDAEz#sNrVjMthJ%KyIZ=`=HaCxE?t$wN}3C2N| zfG{oLjYX_6_51j^<>T_mi^A)Yqdi1WoE_O?J z*TTO^o{|qWjW{`A66U+ys=MC&JGB4v-vX-t96D!0^z=Pid9S_`V#P|j>J}C3{scNl zRkR}5RxaLY^IE@n${T92I`L!u*Ip;`5mnePiCCLrE5=ktPX*z#UlNkC1f;w}-g~uN z>X9QMY|4X+EQ}q5*NNRf6?A!r38c9c86(33LVE4{H{SCI%ghpZeRNljEZM2knq`gB zql>{Ab>SvddWiQkOosgzpC>@zMT@ElsPZB9jfhMjK9#9PfIbr?PQEFB>sjf zZIGTcwFhS4lVlU%Bv1j=R(?;bDo=4lEEP!l@OB0?1!aBW?%FZEkIEI zIz#~*kqDYuspGH=^->wGpeXL=sZHO)`ygI)?tUGVOTc_O%~X(>uHU zbHj^IwZn~gw~`F9T+#!9v{QPB`s|2^I4Gd0-7tM*fvzR{rcIT|H$2EPowQRPB^*;^ zojvN19!03kF2ji!TI~dR#(OfSj{5tt`IB%7(Gg?G-pXT9 zi^iCPlOvpbt={z;@~HM>ql8AE*$;9_+y=d4E%}l!Kb!3?1>yC#%S+{gh>s7j>ym`w zZfa%a5MgPhu7LNiLLwwc*NgH~I{;a{no;T`SD*mtw@_E3C~iZrQ{PSTTxDsJ*X4MP zn+b`gwNa&0KQybfYD_LjDV}7#kb!S-Ii^^;2J8THNMqqaWVs?ceY-v9o*OpHOYN5Y zbSd8fDgs0QJqS@^-2FLmHEqLzhI}&k24d^aGgr-BXnc2d97M=()_wZ)xODmHf{u^} zx*`knqp;cOT=6kKc1PcZ1t#Ip;l*Nu`eNx{bT|S7IBOQYa-auqcMULxv+NJ)1I(%mgB-KlhUBP~+WU6PX0-6`GOoq}{D{m#p^_rC9YKkHe~ z^L~5Z9hG~%OWiKt24zlA^a^qM2Te8vXWPtukTA4`94b-QAB(&3q^*JlofNWN=5COtxaMdxO+UFA0Bx+iytfHR% z*}}SpM@v=a)hy+v1j=Eid~smez}CI;45(H9XY*z#q^AhRS-w-O4gbs`u|_@K17s zx4g(7uenAqfZfMLbr@b3A(trZ`Tf^ds_MctqvcRvT+sHak*m3b+$apJ%^k*E zn`7#yQM?tRhlf&P1WMrcz?!+bof%Gt*f66PjIMVtj+6bUcP{%-vBLZyMo|VEy%#hR#lk&Xec@e>f6>mYThK(r6FLa~8o?U@qFCES) z;oC*<<=wL{!1VMUrf_^7xYkGBY`;mqaI-kyK6@KR=G{xt49r%Vi?J={pFR#Rk?T0x zDsB;@HI2Q0T>l}m&iv20%K_%@V|GJCHJF#*1V3s$7CsuQ$eX2b$NK~1QdJP9P^ozU|2`OZ{es}IuHZ)ccQ25l0f-YWkrzo3X0Z4U%5Vu;2|!x9+Uv=(`ZS1NW=@&-)dJ-Y=&mq6WBMiIvTGZAA8^ z5BjyhpB>~yK_AGB+)bZ>gN+8Pxz?*_9dH9JSKPl}CI3qz($AjgGqS)0D3FuH<^(6l z#feUX#I>DHaw-M2|3th@Z`m^DznT5YpCQZW5i-vTGd_dab+6AL-f(jcl0U6F$TkRA zAue_?zY{?t+~880^!FqGiwe>YyFDED^l?O8!}U%KE`J(#RM=eJZcP z^V;TM92Nw&M=WpPW-RF;4|D3mHueQbhQj?jFW_f`V7_;dFC#3EZ&+lm+ke#X7+6~~ zVc$LG3K~!O4W((NAOe2q=j5Y`;L!B@%XN<6M!a?b+=b#=C~g)^l=m^(A)+v*1Fl)p#Z%(_x4Q-POs4pXVYzqm!t_J?+0BDWiKI-B>2ny;+uy9$J9S+OUOx%0KLJt z#C?-`6}U=i?yZ2i5)qh3RaR@I_Sb^x5qC>kJdec`r5Ht4oB=B+QMjAK`+Q21{v8-< zPyF(7zEJT11cvSx(;Vz|B_0nraiZ5BXZ*MQN27m%43s7AOF`VB3zHZNZO2LF|NE;d zeYn2~0DfAVnKUmbvjC8Hj!OA&hMb#Gw|o5tfg^eqU?U=PVB2b%&x>q&hyz|kasXp* z+f!T%B5dg0wL@rX7VK8pTdt&!jiH5=P>@R70Q2wyMmDs(NsV(I7>kMVfH})@f1D7U zZ+(PXS#@lF^hN$guLxKJy3VmqU-`8x$~xNs zBSrJ}q1JZ24H)c+XnW`=1h243D@)U^2ZzD{c_y1Ui~33LWX?b8=sJe(50-`X}l z2#ZT)7vg_rIKivbA%OuD4_S`A(sqens3TA0#~8@l;Ml~7&g%u!2Jw%-Is(|cgV=p zy_}EK)&MUc06x8bvEag?1Llbv>=LLxV3@b4@Wa%0x7{W>dj`d{%yChCCxHzWPV&0H zYKJ0ac$7on9l+t50*oe1 z46P5kKf#dD{|M#}R)T@7{BQMx3RTXcAfylBpS+m}0cW@cU^UW(PMUE3`wxSU_bbO` zR=;J2?HmX!+z6ngYmb2^kADDOirxa0HdTAvh5Kv=53dlcB_1$mikb$IA}}6trBFD9 zF37!G-?TRDS8H;d^pFPMNih$ytO9ref}$+1y0G$1MT=M{NZP9oS<-g*ZAq;)X zt=$Z;+O>gL#xrUS065Kpa^Dm4-V%`=f%Vf5xD;WT2!{bXi)38*cQi$<1!nfP_?Fwv z(w+PQ|MhTwJ}JgdG)o~+Ea@D?X9>khMSkT4R`=@5pNQbOeTR+-4*N~<-;W|4S~^tU zKi%{Xl?ePm9Au?*0hkQ~u6LyX@N4yYw*-kLWBOm0`Hx`ur>b-f3`&|01Ae6cHP_l9 zjb~$$zgKe!&;8PNL|s5tiHi6$g#P(k|N1O}G-zC_fV1WJ*Od6TpZM3e`3M2i=)dOp zzyA3ze(ww$Of$YD`$iUf3|u5*^mA{#s9a>_y6*r z|B0jjpNONb5-U*jL|uA z0zH4ZUgI^w){E%?vL_NeO{oCKR}uz=7{k z5<>v@3@j*rZ1L?TaHD~$ud%WlfTH4@0CEEDEaOtBD(Z%@rAKY>0pR3c)8o0a3}i9& zO%K+JRY3Q|F-o@WvH?UT_EYBHN$3%FSS`2xDjO(&^*8LqB6t{gv6tfm{y=)Q=eU^BZwIjHWh2iY)qinH1-IfM0PJB%VFlUUq9=lYy=qTfKawM3MhR#_Cnn-Qv^dj z%ZBO!NKQrNb-xRZx4B_{V?zOB@Z1KVWc>g!v+?9t^N}CDFTetuWb6a-@L1^pQ@<}z z5}-0sIYfLr=C|?zC5l63WvB+pU8Q;X*XtCp;|E#te%vE|g$x&geZ1m>pp+))o{xGxYz34h~@TW%H)+W=5^>j)GlqTfvp0gx>e)2p&*xwsFNI2ZtM zIRGeXnWQ)Nuc431RaysNr;xhwo!@b6r~9-44Q75~HSNP%;BPf7fWVc42k_QJB08Y@ zU15U(UbLnAuhoEOVWV*KP*`i=>r|}Y64j)VqN85?fzWRe=-(HfbTKgeTw&+h9pYO7 zeoOxM9^KRwUtIX1)NxFysp;(em-K2vg&!{L+G1! zw`Z&i=u?oE>&cQf3F@MWv*uGmq9u8*L8d6nOowMLv0`C?oZ1A29lbC3v&wqddQw@H zc4{8HV0tJ@kv24XfMaSg-1986^YG!k#LF*QmalGR8Hi2_g)LCwQSwH{OZdkkV9we1 zi&smJ;s4b<@LmEM*DkMp_{e?&FFRn6#I)$?8Ul**y^Qn~eNfXMPK}R5&O>n)rw5Ur!1j5uo#w{^%c_BC?rMH}<3Vy6P8kDanr0$fxIg4={|Y z0CcLySPd<778y=63Js}-YL*m~oKPk8>~!XID~P%n0u_oY8VIX-^_FnV3sHvnZuY7# z)}d15sMtq^YG6gxloz!P=e^kOM}4(x<&SMd(N(=$+~5yP(B0DEr@vv@fO`WRlFV&J zNtSG*hwJq+M!48k`&fEQDb7pkb7>VRch{PK71xNo51{XYPd^;Y7r?xCss-=*u_4B5 zfWIcdE4vcH12lzIbL|_AW1%jI*?uESpdTVVBlw{6YghXL=LeXJfu8pOnO$n*{s6dC z5b#KCp?7oljPJ-hgJug5#3!aaZ_7Z*;57u#wgW}vX9uv-H0*~ITaA95o`Zp>k=wfm zvlDR{0MC>7CZMieX!NrIH1q&y&H(i;HG~a@XS=2gXF05>Vu;K8g$Yyi|HRq<^9C&(jD5@(1@dclHH%Kv{;HBR<}6Nl_0Zb;4=25^+Y7@H;5E%D7d+d`MU9e|EgDZZces>Kc{9fD_RDJJkcP^$ZO{UWX-}M zrOhMM(JzDUTubg(k-V^8!k7mdhT5Rimj291!f2ZKN)`|*cfd}6kW4^O_ zGg?s?IqI|Z@i8JKH^oU~^ybB=S=FQpbzcZ=?aHT6&!v-PLf7rDB1TUP2ccBIIJ^l1 zm1MBOtj1|tQC=`ZW7%(&K1K*FlO5%S&S+Q@$Uuqs6RnexW-0oML$ete9$_kKx_-J1 zhIBnKAl}=zfyjWa0>t(Vhxq{h+tso53VgD?gO-PT2JnCVK|+EW3UO|)&K5(R^~7@q zxvUok0t5~tZ-lZpxg50sctk(&kKF+=zLUr!5JH-e6Z5gK%NI(Z+I9jU6bIdSfns;K zw3~&k{M*KC%LDM#zP?xkS4K|*24z%tdPSRY6<(z|W07%hc^3-gec*u-Ss>>2x^RKlOF0#K~Y_6k#wyZ4^>|^Ht zM>b+f{Z4+H`e*x+)K&NFntAZ%H7rD5@vCEl^~USDEu1dgD0vlhw68*Q-OwHiZ5z?D zT4xfO@6_lpvRjDzL-_1CxKLWQuTUigsuitXY>_8oR9SrW!*~uwTt2>=Qpjw1i8ulx zM-Im{K*`r}9?Xt#pnWgETX-Z^oETxRSDkZsaq zj!&EiJKl%z)?v?!3ZR#Bqh<7;^wVp{!zXHJT9k>@Ud)*XRO_dBo({4HKfIQM@qYcj zA?ba|43haD=5&Hs8Cah@X&&`*3BB!A36av9+SBN&cLF~v`%3QaX8D26my&4*@`}hr z2h?J8K1`Q`TJH?VbDb2KB5VR!!Xpb33jLz0DXJ-fTAf?$hw#YWuEvP6cpMTObC%h) zJd}yH(||3p`XA7`Hlf*JyrkV#kPjxf(CnxsyxSJ*WX=4m9v^g1+8v$Mh2}Ti>-mhT{FhHBNET8;5{fY=ybZh7*D%*ApWw- zA@XsZ9ol??oNEKLpFks*N7B>0QK~MBsc}d=w*0~f(@Pqho^o{FW_=9jULz>3;{r-UwZ7lM+J`_5@+&gsSJJt z#KWMo7310d?@Af|R>vgS|V4Uirk@(b$)SFTIqhq*R^m(~F+*@J#Dd7;E zk5H(;JCuQ6v*c=SOHVR`!hkYer7}eg%PX->@=~M$d`fUg0qgWTW(yF4R>63-G94Pz zN;2e5F#2NJ2DYCHc>FkmB@UXW(-83`hy zG}AotXJq?rT;GolpuGV`m=RtPu4Scs=GVA;-pzp?4E@{N_PU5svDq?!!RLvH?6S4h zSvT2yzyy;eBZQ_w^CLzcWLpypn%pCW~{pxtlCl9g1L%o6atN5~M3o_4W ziIyB&C<`n7$&t!z&;}htws;K?^UwOFL58H9Y_ra1>A~xy+)tVr#7v-E<{&Bfus3hn z`^4>1Az{#+Upr{gp$f6cYXXG`B3SDlA66hN2aYlje~tjVUBu9OZJqK z7&~+?-*NhVjhZ$}i~3^?^%CSy#h|o(&GSd9*G8mR5Uam_xxWQiQakI%gHSrY*Vln! z7K}|~7d{Z6;X5@%jvjp?{e-$DAc`+~)D7Jit<4I!?vM!5y4zf(!LvkdX!Tok+}SXB`dhUow0`dGX!-G&#NS z2r%Wf28d@r!)7DTd|yWC5R4{YlAb+cKM)E(YoS=?Ki?3d`+gn^qsr5%t1OH70JP5YocdBR0zhrP{ViZ zx7KWUJsdQ>U)r%q!?csbgs;2~-ImQ+CdOKFt&xXhlAB{FBoUrbc{zjpGn(OWPtWJ| zH)nHTfcvwFC9e%LFiq>d?W%pWCCw&E^#V-pa_Q67QRyp&J$=F(wMc~TcoFBe(6&LK z@IR4RBE_=2m)x<=0&u1M#Dh$NOyY15En(jzcS0Nu99prLpoK!%FvqFI3W8Cjp2xx3 zG%yIiD`~zz#g$_DJYyg2X;nB_m^N6b3f=GrNm0SzBF~~YFmwl-loDq}u6EBA|7FxI z69rMXaPSuemo`&91#zIyA=taOJD3oR`NzNAjpr>^`H@te>}~s zoQ9Tg8B|5)vH#49#+YYjE1#lp6s_(=Q{%jHTVog5a_+gIS*gK|^%rV)U|3*oF$%dR%po`Dqb`hKu-=D3t>lT5# zOd%GCL!GsZ`7+3?*B|z=Rg@ePI|r6(Lv#u&@!gpw4=A4aqE)gj@)lbdHnlQ_boaY6 zIC8Gp7})TFTITlh9Y8!Y)s@hRi$yTvkNIv;lqF1xU6cGQs|RyhR)o{|u6hVEixw;7O~z9ZEPZ+(#(8AD2L zPk&KCI^wO+AJZ%R$K+p55H)C5g()Lxk)Bb1qgBM$A5 zJG4qt5o3!^P>v-zAckq<5ZQa%SSXK1!ZLF9qYv7f&*kcS4|C=v2pley<02FM>IM+2 z7^^Ho6g_1Y2^o^>h(Qq*%*`5#3T5)RDsJB7XNX$Dl)f~{X$EpDzeKY18aZkxD@3nE zliN9Df4Yh0vG*nDmZ4#{5g;m;_h_to+kY7dYBYX%mx%m=vRzc2xmLmHMF8vFW?x9N zv%#~d#i#8ikPwb>Mbh2yLN@04xu%TUCRl&e^a)bIc>=dR`IvS?{7gJc7Gd`(_^CHc zHQS3KPL#XU>Pn1B!7pnUat(G#wj-|mYA5zwJGnl6?66(uprGt^?XMr^lJ256e`+q4 z8`}DGY8Y(VYakjc9G~ybjP;8EkoY{5hW$S6Wi+&wsch-k_?8W4{dI!R>38s!E|A0B<#YAZ9)J^B-goN^H?CLDI}zBE}~KSPBACGfIBa3ezpo?J*$vPrNCA#M174fVc+E7K9K+Mi77gU(!#JQBZT;pqa}2g$DG`pjqIa3d+$f1!QwBrB5{kL zF`4{l#WQbVB7OHlPWhSeCZ^w=8U%6QePA1D2{{#sSi5S|=vIRqq`x91K_ABIyDkrd zsyITtJe*GxhYg~1i;mJ8U+Qu%@652n=DyY!$g zxj4H`hwq{A^h|m)Sv~D9Sk(5j*;Y(hY3M`92ryAS<^fsjifBXa*DfNpILdS-`k z#8AIV3SN$gWq@`<8dBQcGIZ_0Q$BvXzMRb5APnV&%E99_@8e+b=}7&ei7&;?C6u-I(MQz+jD{_=_R>z)(Tfo&qi zRi7I#AFaOS%|=zfbku;_TKtG8-YL(6fo4Q|8iP5hp% z6Tv6l$GgaK2um{^W~lW&C>lkT_vSJ3-=>CGrjixr8i+<`zbkMs*?SwHBc9lIJEP3; zPpHg4wS+%Qa;P3v%AIpj+RTGNRGsNCKu`9&5XH4FXNRw2?^vF`xyvJ-xX4Z z@aY@qqZC31W7|Z+9sfc@;<4gEya0@^+qJc*eo>1pp8 z=^q)&$7OTd3le`Nhn~F#D*AmgrWn~HI2IxS@~7D7XLgzH^FYnP!O$@Injy6X)Iicd z!{#=TH&o=H=$F*yavd)FF*Y8`mtml0{#Tlw4h;~)aQiC`>;6?__P;C4ss!R+5v}_@ zD6{_CM7sF@uRinUVTK!tf-~RT{7{A&XEWJe^wH)YeJByW$e(rWn6SiJvDS|)*0(J9 zlY2&mqkRNE`XYUI%V>l+=Ta9$Vbxopwmg`t%LtJS81aF;V9qFTCwNG z6oXYMewRlCQEKtLdzc%)bf-jlDBl(^hVoDP>POHP?x3)L%rRB{-zzx}(r(0&bzH+$h&k7GrrZ4N6M zPEfQADuOeR)$^4ot4zm5$t2@=*7Oe+>UJjbqEp6=gLznsaaDgNC*0u&`|)Sal<5ah zDSfkEvWZM0@9P~n*UsnanMW?(d*e)ljv1tTe{% z>27hh1{N|oJ&LSfuycaDT9JIC(-T2VrI;gi@8d3T5K16_A9^AdBhw9F=0c^Yz!gDVWf{kzM;9WKoGi7}H#k_;{oUu; z{#M53jdGQ{#?P}yZU?R2k1uBG%v)giZ*em$E>rLNvf7tj_U3C2A8vZ>xt))1`8Ej_ zj5~0KT~gyOIh7=`B{5eX%b*zfd0EXXDjXY_I-00=X+6{T6b*m);rB|rFLdXrcMY=HlD<#Iw#zJUJS~?RwR#Jm z=c~zRG@B*uOy#G4o6`Jcxpl*6$n>m;=pBdMxOce%T0naZh2MftU7duRI+ZGyu9qSY z-@@?YXW0(d>3#9krX^zG9?!CPZ?bk`*XJtuN9IDC%~T~RP`GvcOqY==bqjKpnB?jd z-tfuSIJcDGH}J;NVpGJPRhdmvO%|$;YYh*c?^W4lKc>DW7`SP02%~?JYU!g#k|;%$ zGGy}LUFpuBDMcRr;M#SH_N$rlsgQ+UJ*YfB2eR43MS)t`8e>WLJTxNwIuz8|=!v{C zE6~1O!;EA85MexqZ6}(b-GhRxDHharReiMPC*&2rwMk%CWVmp<8dMnCvrLD>-ltin zmJp5__^EO4*=uyj*PuGfX}GtZA?zfjOy%3LmU^d<7`1BrN10`O zrA1YVD=TWgR-VqWSXZ<{?~ye|Z0+BNG7d&C&7AZ;)D;DD!o*T3Gok+x^r@WvD$^_Jf>Tz0)~>^F;ZwB z8=Sp#f`Z1#vXw|)rGMC)^KliVinZVNn`x4z(xjq^arKnqz7*f=m*WYe%oSGqHTc2S za*5c`xKp{#U2vDJTj}dNL1k)k%Oy1Ln!_;aL@A1tx8K}nv#8=EcXYVEO3$O_t59ZJ zHmsS+aZ7IYWffB31!uE(xx8zRGMjJo?Te8x{Vu`KqfpQ_o3Bhe|GW#$2eVk50yUkX zSAirln*d)T-yx5NRy%$ytcmjcP($HZxSH=vqTK0ITh@Gh)$TC-st?Bs`*T&93Ck`A zXZSFkn2&lc0BuQ3IbTVBPI5=v`f0u#og_OFecs+)%qF|tM8E!6zLHkGg^nPjAE*7M ztL*re@kH$u0c(-A#AzJCNTFXHc)vwMv_w-p(n;|JO%3e*ikz|Zs3uo)m1fEam8J!` zVa6SuToCalpPvHbTTkZd*yXMYH+Pn*)b9oTS(CEDjW_aLc=MFk6PykW7b@fEQ*sdZ z=L_C0+JtyKa}T|xP?g^4n8^5AX_T28>H?4VWfHz7k>9KN3#)lf5jW3QKDtPB3TkYV zOTOhF6JJH7vL$?ji8;DiahWw*Y#XTJ=)2Vmtj>=rbdDutcdKD{Ci!&d_h&Ip=G04nt&x_mAqn%1A4uhL zW$@nvhgJUjhHBt`dhNP*UO5|#a1hC|TZZ4z&d2`@@hMk{n}TD0C#y?X?j){q3lq?BuRC!aSC?1Hv(yHQFq)hwQ_q7-kc z_m5Z`y(AEDS(T|8K6QDp$ge1u_?y2{ZCv&~*b7F50XdSORIhs%=`SpAqhHIa$RujG zUhG%qtAzYKv4^*t&OmS0!iiSNje|>Zb7CSSIf36$xcg#jxrp-3w5gVSppSwZeVo(y zwenXBDf{^AxklcBVH!1^V<8fh@j2<{&0UGjQw+urW3{H(wCecVdu+BTvspwfw#hYJ z#uLRZ3n#QLt^==>8q4dP%r#5*g-SN^CHxZnFC|SUU9~s5G?drpoQX<3GH0r(M{c5- zs8q>wqR8~(2+OMR#+XgvQ8Lsj$Jnzqvh5i0dk`iHggCl7)KmFVVn@jyE5xy5KX}KF znT;|OzgdJ2YrnEy_<<{05&sKkiC*%O_g9t{;c;_U=3SJfpWNE!<(NkK+;k|W5jLgh>P4t6HrcxFGNJf4 z3W^1VJ-^DiGH#m6;^mjylb7S73-Yg|HzrMt>CP-@Ao;Agj~&h9<@}f!n{G_If;!P; zDKi9@q~6Y^&vjHfb=PzjXSt`AVE=hg|2Jx_ z2O@o8e0^v0B&9(r#zFP9SzF?zR^4m&J&MPci|8-IC#kxWn?$A0e=-ZO->%rQ=Hb@p z)xm>ZNBRdM2HQ2^MUY0BYjmm5^3adz8Z6E&H_XEI$51;EtZcfo)xg&zpQ!M76bK8~ zspsQgwWiTPDHniY>%q{;+<8=I{F+9#9HMvau8rQ`9V3zcG|aSM^N;(d zjaj5jCb_^r?dXE~-wvH8&}}W+$8CQKK8VBi$Q#S~>d7wN^U2HGhO_p9=RwVAGL#ET zA9VYIzH~fOWYTEttI05K7c_%aGDFtel<2N>^(pa85kj9$BFrvQ3%U-O{_N84KJPDR z(FGq1kESc1|K`hkR*rtx!*ti4p^U?@kV<_i&j%Yb(f;YHojESkJNG55v?*30h5Yxd zG;Intsa(T~cR04xe_S4Zsk9Q&&oxl0b6-$Gq97eIk(2x2>8Yf_?fk4>S1pe6hUkb- zb0hJ&kNswCNBv+kffsL))qYe?p@|m1l?2}qdLxC2(26{RaoEq0i9#*${IB0|r%KA{ z>UPBH_XGHqcf0)D*@A~@w}vK&g=*zFZS{jvL#GMlpj>$WoRu?1koZyN-eLUsbdze?9Wo!Vb%s#A60>j_!S zF0^TIS<~f0Aot$|7I|o3$@pBiAIu8F>v}d4=D|tzzBy3q!vWu9|86L=TUm+iICFie ztc2r9YUYgOC~7Q*-vWgZ{c(te;3NNX`19FR52kZ$gSqqrN<~i;NtLTFO%tlUG>H;0 zc0uXvC{EGS+7a31*;&}|yttVTVZL`*!nnQ*9kZzzC1*ugQr_5(Ld!C59S1847pj>@ z5_+*&i{eL4sul&zvh%pwe(Ud!?JMxm!#zE0(tKVm zi4ABhY_>mT;rMe*F?`u8;JXb2E$_W+F}j^R95S*lvK+GVV|YqpxeWOU;mxjLzLb|n z*rLa=eEi7v6fK`)cRf6KqDe|78a)MzaJq}EXHl&;H2AA`{c{Stk$Hz_FsMa)%J@>+ zNT{nrPUGEjbk^%`vDTg^aodUx2@P;Gm{1ILS$a??#+eQ)kmxL3k>ckJR`*hz;|^a)l`06C278)3~Gp1#f* z>MSm^@nWSRA#P!6mKqg(sG?}8*iP39O`zMseQCLI>&z!pARK5j`Y0|2HfWjhl*f^S zR5v`__=6FG%St*;O$8S5bMYolN%vSvY{j%8I1iS^is8UrI`KqH1{7Kq0l4m4BAi6b zEe5CZTq^Xe)B0>0{%B2O+6!rE)ncXZmK|Htp?$WNGdr3O9DF`Kv$r1on^lgp+40|0 ziZrf&%6e%O9`ZV${GHGv0}B;hAR1Qldf}}{tL;@5lTLRl^Qu4najh752)szAgEHe0 zUj!GU38i^(TW(-FMaVC+qCwOmCkwDK-pbR$HlOtxry}M&CQTzL5T;)6{Gt-5G*H+o zm%Y!AWzta_!e$9C6*yXJwIE)N4bQX7*`XHlRxbU{L{v1kPv1Ijggb2nX>giVNN`>Z zhD@wZfw3Z)CsTJO+;Y}T8(!s?=*#P7L`oig&A`d^&~HB_x(hiZ88Nx5RYHZBB6>cI z>6dKf>&+&7VTaI?%8#QUN6h9Wpf{C;7WPB>E1Z@lT)!G6(&s_ z8U>rbW%q?{TfIA76gR|s`L4=*S_azOJ7^sY`q{sc{aP^v!7)Y~!d9bbE3yLm z*-mAM1UrDX>2PZZvDlsChwn7fi$CZ(%na`Zy`dysg+Q2G)8$LE!?CHD#A}gqU7vqy z4N8vu^dh*lrYpCo)OBd~tzz&#KZP5j-U|gJ<;Q(5g7RjV1=AVH->k;+St_V~=+t;& z*d5wlu0&Z9k7tGuG%ypWOi8G?OAePr>bIaVxh7%!9KzJ){BnNwJtJTQX+)k2Yl0B{%WeB{!a{DNW z`}z+1tg1WS-Aj|L_bH)`_Nzc9M5BkjKHtqP(yZCB_PR;BKKq&J6)s_Z*et;QUMa#} zzP;S!ks)0$Y|m@%3*Rg(&coZ(w~E6WVM$-g+-J+>#nXAuRf@|Mkd5EQ_QBZ?4DssH z30q1XR-2%})+N3Z*D@v+3AC6!77ZCT8CTKXa)Li)eUumQDGY0jId2jdrvHv6IiZ53 z5-BXKdo74DdaQbXg@CfZf1t@LK4y-Uo~SQ@d#lXWJbfy3U6iEUM!(HM_6DicdW`1t zmdC-*D`#d=&`4ITlpmuj!X1x&|Ly*l&cWt{7InN&<+U1BW;?Db^ph47+DJ?G_Q&dY0X&7^u5iK*{z*7jLG&4|T0v1E5hHj$RZ zTbV>ybJLD-e@2LyL$_n>nIaGMj|_#@Pmu!XVuVC{gWC_Ht7v1E8{J<$GS6@G2&Qu^ zo5_bRq`!WQ)ex@yO42?RNx-kBME;VoqPFpywn7n(U$;PKscllp7$+m?LaDRwfRLd| zj)4h%;Pc1m5N|!b118xf?yQN`kfXOjJ^j=EQ?WCpBFB9<*p^EN^Y&+vJ|_COyNWfu zJ%>fUZ2%dMR_9hF`%Nk@!HDjxQm?5*jV8x41=aOh8)^H3aOo>Qf3<|=7uA89ZW>SL z6mIkaiY*NHgL&Jl zO`RVKqL`QS%Tv@x1Gyhr^x<}IqSos z#ev1|_umRfbOgFp?SKN!DbVY+aA&e0tu@tZJd+sey-4hedCF5d_M}aNRq8eZK3)r?BrT?I6EgnjR=Z^4v%AY1YXk z0>!JPdn{_E=QWIuW{E85I4U|f7tbWf^;OOWhRnHrd9nR8lrSQsVW~c=2lJlqM99q> zxnwQ9kTquyc<#sN=Xx*DsvG_E(R8y*+jY(;Jnasdv5Gk}ucD1W;rVcdbCyn{o=5t^ zLMxgn#VDluePf^MywU21jvz;3N`DfhNfBYl310Zurf4H`e`Oq`33Id{PK-WO@v3km z-ykGciqZbhf~Ucm$6+rVhGr>Asb&v>3uCI#l(fol{!J|XI?b!#S=E<%wo_wc#EQwbiboEz&mx$xSoUosXZpra;Ay!Kd2U68;kJEO5&j z@F#floLuTcJ$n5pMz-H#({kycz}`_!Tfzlae{K|d!)7L^u|F*SgF3oqpGV11j8O82 zPumO4wnr$cv&CkY8nGvhz-y%l;B&ehtj5|RU- zFb1(fn!;#bR?DqLPTMKxYP@Sd!H_TIzuQejQNixqg5CAh-Vm5O{-bGt^IHvU)*rO6);XIK$Id=AeG1j z#!Wm(UwJ$iz1<1;rcsqI5lfZ5*&pw9{>)j-;eV!Eeua~QBgexpG83hL3PMJx1kb93J7k!T$Am0Vj+^|<7`T=Ruj zY%mJ5ql4DIAznyIdX9Ak=V-}8etw8^VR*C4{A{w?rR_ux?OWpvrXwvZ9$RQfPdJlC zGo8|&-l}Bj_R79&8$t4xBYgMAKZH8pRmWH`K9*qn(aux)(tjq%RWn1zb8B3#Op3r* z7iMF+V;1|jcw_hStnn36bOE-H{@2e&-=7-b-))n?uX%;`C{v-2EBldiHC^I%N1&)|>aqOOJMqf$KjseM^s9ji2+BYh4GLU|rH7r93g^ufXAQbC>@oy{;Uhwj-eJ@@qhUqT&po{K@) zj~!6;V&vH=5~1d)I1REOCoUyQz9nK1(a4(0FyZz}B%I6KM6tAw`*3ZNlS^0hGDm|5 z9Pn9PtVF8{x&{@Pp3~rN(O)Vz=?y$@Jd?m>X3R7X0Q#y}WZW8!$TC#y-ixTzquY7f zLr+8ZliIhuE$FE|Ptnp!5>*&jV`h18k3k^K_>9gLftB*v2>joXe*T3 z`%dmCv8$}(O78{HGUTtm3a&2T{+Rvd4c8Z{g&(c5pjAW35M{Ug=;zk5<+sADO4sw9 z=vqB1RcOl9L39P+#6~P@?&N?`faKYGS9XbO=$hnwdUv zpNqbBe2$6AbFHDVDM77$-2agxXgMiCo&@D$B1F6jX#1iczOHrVM72g8$1<8$7-W@@ zJJo(mP#=)X7na(s{*^N)LM9fLOTT$N*u2SMNBan_d!fEcI$xx{7Nh9Q(z_#qK$qYZ z(X0{eg>r*NLV-GwH_0!d{&4SEluI*0jngl8N5uluDQSjYV&At$ycb%vd`yJYyW1eH zO}5G6vj{V#n*zi}^|a@z0w7x>JO)Jv)=#IaE2GRS-q-0cn&1PaD&~cB zzU8nUTH@ONX;rfZd*BxJ*wPh8CN1~lB)8K>Pm7Q$qv(=WBxNFGEBWx+6t~CJ zf;NbkOL^!M_;0$BKO#<}@->K! z?z*)+_%l!4vO%heK{kEw?t_2shj&mBE*L?}g}=y>J3A==^IOZs9z2WLr`*TS9y?m4 z$c^)4V>ZnY4Y*dzYVfms@`DGmD|Li}?L*Awd$5yJqGYsO*h$g|X-@d&PFw+g&RI&0 zjtt9riD}r5;5k=J<8V`wO1z3Kw+hFlK$>S9TK^O_aX1;InE*n~RydgC2d`_!YDuB=BfL!fB*USf50JN}cU?wSo zrpApN-z8@{{2aq!vq;$1cBaz%F_4&(e?M8`NoK>h2=nQPd-*C%8aFn{?J(6!b!2e* zzi*jZd!Y+pboWT8TnO^71Lqd=CTt_RnM|*iXtsuqia9hi+HP6+qE`QN=}lBMW=xm1 zjF9a`MpNGL&$32F15SHg@D)l8X20-w2Sr2L4;ZcVQNPh?;U`sRoRsoW8g5Po{xrY% z9usnY@?Gl%dZ%{kkTOVqoF68mHTla`PUe4NC4C-Pn@0f=XT8QLv&^#6Nj-!AlrF@G zqAuadGhRW$6lVy=SE%M=UVbXeCl*RpXxYR-bQG@g7NjS^nN3@>LkgO5>CUYgMI{|G zo2DuDd>^IKA&UK}4aTj8EA~?DYYkAI>K}gk<%9)8zUw(O)G0bb(A;2iWK!}UJ$||k zJT^5?5pc3=J|(XZ*Bq_Cw4RBnH(8!U_p~KI?|@X}W3agxogUCZ%~v zVW(dN=0vDkmL=0v4lN^yhXWo9bUm=uh~^WF*JWGzV>`Ly|UlU^9>RQCoPtQ310 zf4oj}xi)bXWq%hDn^uCYf4|jCYs4TqU}UREn;AmheR}0nMIXcilM(J|9pRY7ZmrTE zM_Xk%&ju3e{V{CNXyTnodIVr@d7gLIRNBqA^6Wexj?w|u45taWz=OKiFSvK`(G=9x z)pKkL)GKmwq?2-hhYu;oi4>G2W4{Od!~-UGI7p#(i&jbVb^fnq0&kEODgXg&QX=EWG2rJ!-E9pRHVHx4AyM$zP2?#q#5G?2lELv0uQ_%V@r) zT%a<$#Tjf8h6ebG-az#*&SzU9%+=JP4eF#Q)9*%%rB*#~;C{lc{XYwqyrX|7x8c&~ z?+S!I8g1NM>~n9FIUj1>G&mnFi6!?WFzQ!=LZ~v@@U|AQb&>TW`@NYmF}&UCd8={Y zOU~4Ms2xJHRErbwu3v4ZOr_&ykF|axI%+?qH+=m>{zb7ySvnyuMwlvs%qBG@1#-*j zTk$Z2UDOMrPwql@$|Rcg1_BaflD;>Y2OsR+HQpePryqPZTLl~1P@4t^fnS}3V+De< z33-bWadMucScdJ}aM%@1d#*+t4|S3rZ;fhs3fjQoo-8^K4J6)TX%2h3uke`rH zF;THzCg3fCxiKfEJ;*Auz`B(ilM(P=p64I-o_eo+sO1*iS0QGQ{y+ z`FxE1-czRUv z>;0H3`I5Efe4a7Jec!)ff)SzWkzPkG&y0H*`3(E*H!(XV^P5mzU7t^S94Ye2Fs7yV z8Y_T&?nM7=ac#33X$Cf^w(h2{446fruQ@4auVK{n3p0 zOo`mT5sV6;cN?DHh_d~#I{)2v-YWHv5YRP*qPN5csNqc)`Y7f>fspuAA0%VdND6%V-ynh3cxk;ZT7NAnAYS;^x*`jB5QKejzF!KrXF# z)VWde87Lpvi@=~|a-TfR>WaBNT<>^M_{-EeL==s~J)MFTjwMdEg2Pz5N9}c8F4q5E z2sA(U2`aA`9A@|4g@^NoekRg6zf9Lxt=J3{1nQ$Q&X0}-%U*qqVl_)!54lZTOx;EB z=S?oW(&sS`j5Xp?{mcO8L!^V`_Y_2gIm=Q+l7`09%eyi7x@-7C{SOwowuh(r5a>Cu zVRkE=D&lnG@E|$x*@fqelkK zS-`sf2lFZN-`{;Nn7anbr7XDb|Lq0PN3p(b{1c6pf3$H?JQm;N*%qLivP(JJKZZ|k z#q_Bs02+%_AhsqH^iOyPbZDhP7C0ez>AO^$O-O1;mWqCKRM z_A@-j%>J6J;Tf%NJBmASIXlriHzDe|1lfRO3 ze625F*_lGR7a%`BJHG)sN-d;N=}P#(pD=W10G4?SI0MIgVO8*b>vE~!-eWpXYe%G2 zJpNLo90KXzwzAcH0Z=lTX$Yzb+@4M}|4Ka0M zV0kCfs0GwdgH-cV$<)1Ae&umy!2VT&opy7b1YD$^U%Ye54z`ru&8g5;?-GX zH|BML@i>os4eb=7f>nH)kBd-y@^CR)J{=!FQN``Vmwm1x-FyjIc)lsWS`u%^o}` z`K2)F#H)jg&2Tun^n&1~gc6^I%j1i&X;5;)imTxDRAA|+7*pLx0sgludW)5snD^Th z=o`;3geM+^(aOBbeBy1k(av-at-xn{xkuJ*svU+#ar@F0qp*X;JNF*>^DfKlxnnCULeTOSb3vk8+!_=EgG^&yKheu7>ZS1ah6xR!*QvVLYGlJFf6Z-Uq{YUTt>)%a{EC4)-R@E-Dt-0?`)@;MBW{|vl; zG$Hr$#|%6+1CP-*)gsBZd@~BYW_>l0UaYO}+lgLOVZhH9AYxw$1TTbvV0Fixy*1iy*sK7%B&F{8eYX6{4cn`jQ+I%1wka97!ggL~eh#pF9&Yf;23j_gc$oQK zupgTG2Efh>0G`cZVuz3SHVU`^9pocQM!IrX6Lf92-tIn6i##IN z$tX}Nx`CN$2?)Kj_2x=4Y0Ui?;Xym0-D0zB!S?l7hxV99qTvrXsx5`k084$V`3kL)a7@}|HcGV3m^CV+)2q`6Av6kF zoDLv%P3chjVXKAZ>!35l==HEi1f|q@Al~x^0gJqUs~r^p`71U_8~s{jYeCjR@qe1< z{S8ThmcvlUX&XNh?;DL?8YXppeojc8AWynANSuG24{S=hW{oTT3_*{JQZ75Q>M%S} zM-)>RmGcfbmhMYQChPnd3wWqVF87KL3l486J!C0;FlFC}Zrh&vbv9=6-E$UG*59DP z>nMeL<~yvJyNH+Xb@IkYMODJnl7@psmVVv&V}`!C$S&S1w7=O8(DljV2I1Fhf9K*vaP9Q^NYH*gFfr;koh5Nh zemvfw(BH`~bh{p_FPnmLq34kEz|{A4KYOCC6D;6}3Nn&)8HL+YSPKw(?1*VE!_Iyz z^SH$;H+{eJ9rR|B+wBZ63w-3&%p7uybYwNaBaMAn5BpOSdX5h4$~;(UvfX%naJbEm z_7S@{>b)?ABNc}a2cty8(1^zBdK$AuO53QpMsVcC%k#+JrE}z7b_;Ucs@cct@F9J~ zSg@wHct0mDnEfR~jCd3`D|JycCi&2of{9|iYx*odUq*CKUVyyIVq2o}+v0n+C_K^- z=LZqy{=wE;e6lY|HDIW-UdfF&%meek+cAysQu&QBGGS~LAw%I3dmC{w!;}~1$fJZ~ zyF^6AE%lU~$b#jTFNgnUZBkd?C#Qjfe1juq=sm7olT?cBV7kaaEb%H$z~{hyKadMY z=qyid+C)iP-5nZtFjKxOi5~MFiZ?ZRKz1DF+f7;&{PUDd&c%`tP_NdUQl#!Y9%Wmk zXyoRsF|CF>+>UW-W}5MFt9J4d4sdJw(4|wl`x4V{ICN5`6UQWV$=x0FgpFUQKfNf& zsNLT9S=u#ElpS(D3uF!Zo&FmnYwQT}iGFSY*MNW>Jo!h4D!(L-W1(_A{40jq4osPg zVYWZ4LsJ2fihx;kpasC?pK*pJ{ma__n@RsttMU`GP7Z2E5(9z(^^l%f+^&r+{)`iXiGs_5*`sMJgP|Oqb%YGfx|o)DV;C0oY_U(Iv;RFjPTSE4BLq+Qs^V=NRC|tN0bW(ZBTPE05x6Ht)*U z&5qA;L%iZ2{NoXuzU1!{{b4_E-hc_JEd)u=gKD{KPleg4+fLLrpNT%D5xxkS1?!O1 zeZd3euaO*0kMqey5S8RWz8KdM9@>$6_k2|_f)9}*EZIF|9O8aHGno2gA* zaQMWZfC+c1C`H#ood+P0wF1-}E64G)E_i6ej;H+$8m-3i4)(*9p%?Ucg!t$ews3Us zF2Ny}lligph2GEmDEBCTrrc}kkzmZJ7GhLB@hkdgGjhji!W+;kniGyS#<~+#2=t`BCd3@mdR!X! zQM+_zvZLsM9nF0;xaFbOKBr};2Mk_V;;`=n%aJ}N7!C#N9QaL89tI+nm>e%yit3X& zrpvF98Z=^T3FxM#qficRABlLgbR^4Ep|V+O^5jKkuPG%C&NfXy<7`KvRm6*~?dkOn z(z0;jU<208ZKEa*_akaY#jNllcc^jas!$i6>HVRc5HYb!R{-sVX-@0+K$sU8Dn4sy zEr?gRt#J@EYm&OrDLsw^~q`-*+Kn5XBMKKxG0i}zBoGWbqHi&d)C?j`~$3#)gv7{rvg!rYRvTa|{`(j=XFHhnl->^R{o^sG5Cv$k4#L*D)95`jd zQ%YK!CF63dDNbvAKJr^GzvE;cUqy?HdXhejmJtC5kS96(D%qAR&&>Jb96+{YwA6O{2f?$AB&FHu1CohTK7*2ig%emH`DKoHb5k0rYxRSO+ zT7xWXnzx`vWCEd)*%ZOH5hK1kV8M}u62EUS;$cjJq}3nAE|(%&BQ79=z9GH5EXg=8Joyh{Wu~Kd`OR>ko*N&(sz=g3Kafy0fi4^Rp5-DB@|+=T`I^Hj{K zZ-qFZf}j|nFo!PmT-x=xmeft~t-usB=(OT!pi>`__&$CRJ1{?YU1ocpa~{`_#wQxq zn4vPGM1pATA@qIJ>GSBcrSyCC4Peh0u;_xcZH#97Ys z8uwX{+aWub-l)?de)0xLD8A|{*E%?NL>%h?l0*!~@k40Z&btLFwl7N}hZTTqRREme6W0(wc^9Jl>pyPzKMWUcfvE5NoSL5XO%buKf?{IJKbTU^ zIRA$i8N~k-hrFC&yo!wQ^@gguH%Bn|3&f2Aao!4Z?du)uN8VM1> zl??r-jYR~92{&dnjn2?sm^%IKPSj+DY0ByvU0BI6DsMs>ZlJ107d{Kd#IayWH1l=a z&K$V9T|fJ{@tn zfM*5Fev#NTJIe=UMDpGLd`8bslA0hqV~7`#Vc64Aw`_x(ovH)P&o8x(a2`|&e((Q` z6O|f5Z*5;Hs@oiGbKjuBXjR26_!1pZyi0f1-R{gg*tdxgyKXfb_+Hp(otmtZhnnhM z^A{%0Ts_F_n)@*Zk5QjgXjatkdP)kOH^0A%jQph*J%S9T zBq*HEdnKmXtS5+ug@oZvJO3;;AXuM7Psn1qb*JU6Qh~yfg#USJARtn5&oH@5fuWA^ z_;j2e?Rb_ZPovJn{BSbQVxg>xEAu9neq3Pgpci!MEV8N0*v>uWSQv^p6?_Kz3Bw5& z0Au+V;7Wc61Okrj*ik}Exa6Q%I4UEOZ;#m!?g||8@cI(43J_C})B(Tt-004tSwHVb z9z)kc3&r$fdL`#825NV|S5~W#i2Cf0xC=!JYofqo(S0R_xrXdLbZZKv`$V}7Om6?G zA&cGwp={NKxaY@)cPa^nVl>!B4)I=!_hXfPkgVoge)bPGz-B(Od&}WegBj@amNS$d zVz(l$p^0n^8=~TKMRqM8(!?Nz6UE%n0WCM%HouReJLCMpm2B&yJ|(e~-?Z0(gBGDC z$A5sl`Gh@`)hk(DnnN;7_w~!8&xosasPIsF??y>Y00iSX_<%tT^1^4SKi@%^WBxBm zFAxrBnE1bFz)0bQjzz4T}8AEYgZb;qnJ~PjxhhWjP;0q3RfPh!oACYogAjdaT z4~a|G_DcmJB98HZ`U`MP8qe!NB*)7|jW!t2s+uTRE#e(csX59JVwG)EUGnjBS@+MC zvZJgp>YQomHM3;=sjv$-#yM{Jg}IHl&QEQHt-$sbJG_J*Z(s~aDu434+|#Bd6&`*P zG^1C|qr)=mTxN^N$OjW8nm2ht8iE^{x>ER+SDgES(&2r`B4XR1NP#=3bTVL1wjWBl zG5QU^(Q<;n9($dvDi~rF2px$*7+3}eAYNB`fD7%sqcRv1TEiwhSG|Wt%`d zfS;|BHCW-W+L?; zEZ81m$8lAl+o7HpBuU=opkI0(9{EO{NBP2zw1xtU#_iPy6fbdxN4V)qLSpNHui(5i zP@z3ed?Yb^$tRPtg?IP?LwPUDsIuSl%>XQ z0_UO|WAanE#UPST3J{`IqAngmVHtfo4?AlN#}ZGW>oma90E}Xd0C{)IA%BX8y|)S^!R@g%&e@f#{eukR-#bHHt8b zXslUj#$mzWH?g~ci(4&Vi*U67JyeYgM-ce~`moSACMND5+})KaFi+JX1{lzQm?&i4 zqbT?tQG_ZewmFB?nCcJq1BGL6ZN>y`Vj;>zY_ceLFFd{)MjlE()f|Zp2oE0mfpu`} zd1eLangQ-P?}~4SPbg~Wt&ewr7rauN1*U__jko}gym_O0Kcao3IYlA5_G1c*#PQ>% zv%3=Q&H#fBY!lB8u}0PTpThhfW{%4_(#xcH>(?9I+3T+sJa2Q^joj(Nn}1NdeNOiA z8E5_1EN}nZC`C2r#~}IZ;9%)*bdryF+@qA@X9FH|$pBjnMUOmz$Ag=edDny0I^lbm zZ{S8MBVw>*{rS165zmNVXPNER)XR#89X@8m_krYn$yPwn@!Rh|bIfv612_|r(o#%E z$36P~?8k`p)~>nBDmzbXaTspYNCjX=2_-)T<~}Qgye-8NQAedko*7946YI$8f(yI@ z0iGvSO=#rc{vUKy1-VUVE^!;}UyZR+Zv$`It4mNJj4%@R-Be=-rV-nQOtZbzk$Ih} z7+O!Ek2x>JlnfH-_b{+1eHL1I3BXGD!OE1A&)Z*|8%}t>=9H7_@$fQRKOF_EQ&3lc3icuo_7;~$G2m>a)u;~xb7-28RdSxkR9 z+o>UxaNu#g1S*^6JUuB>bT>b+S;{<|3qB)%<2#ZI!J~)UNA^#Sl(J68qzhK>0DaY> z1-gBvfiLklMj(h?e5_P;z37Z!LX*|g=mNL5v!mAi;hxs*U$!S(FVXA#SGQfiS8+KY zIyt~Fo+IuHdL2*<4&6ud* z@(6CTTF#H^ikSy#MqQXwH9d0ImhCtz^RjPvIr4G36*}^)pOGxed+NVH|5alp!B53s z5qK8;315h9K@_lVM^#DUK}w-JdjVgYZ~&TxQC@D27g!YN7U{X`&Azl7CTR@w=(pEh zM8S&Q;l%oBC4!Of74kJ`QLqI3&8eas<;rsb z-{?>NM-lV4qH;tKcmtUc^`B{g+)yq@fE7^+5dp<}cUU)o6I(+iq|lIO6A@t9%lpT@ zKr(PPc$busT@n+ zNZ}a6|Bf9GI{Qr6ee!^>&^|a>&ZFMam#}Y?nQ2zFMJQLt$BbXPUz1}g&d&WF(SYl? z>gm@>UF$9?0y>7G8yw#KGdj*lxX-hpj_av z3P`qbc~rvIqwf$MRBDyTR;b`lnAc}_HRcr?c!%LPit`H&JB_iE1>$(#OV#iL4yo=#kFIYZ`WW6sEizhgj~` z8nYC00QxS;=c^%8aB{<~AVNU!Xg!#s_IU5Sx$EkBFQSIh{&&=di2DWbir(h?%aHK? zj>ararV3YD9qdICMGOBjl-mnIup@w{zZ7yz4X4Z+|Infk~Q7RN!z_S5151s9@P2N1%HI36TpJ2;K@h5Xg48wy#X}!OdYMyP?3NZVk29 zz!(v!;|c`}q>2_|N@&!Dh2);m=J7Tx1Is2U#$VvKZkZ6 z<1h{CFH)XbAX&f(Ss%r^91R}V4&1y;vaJ3CC$W-2!{TQY?+&%^9DM# z)$VuJ_d8J59B+I3vTJNNI>k7IN%HX`0$zw82<;ABzXrIGb;GDoy%_4j<_$YGT;biPQ4$(#&YYhhrUHc*pP7mwZCj+fw)eWHe&_*iWs zWiZ1q|Mj1&GIn7PmvWRCY{YIZfhCBjDVA ztl`iXlCV#^R!CEut7jWudZM#vHupBUGebMjh-igi!D_+yqqT?@G;4_0;`sNYym3)P zLA*L8n@;F)eJwM&gBuUy6uL*!6JrbE*j3&fogE8_qiQ~iv+X$00sA{WWgE#$d zg{;{<@2y@49rog-7;2O>0L2&;S@bXXDF%o?pU)x|u{2+PK))Xc@&E?p1Z+7@Tc!#R zwW^zlqma5*`9T2PfTBKAaBlZ@6@8!%tb}YkYzy`#`-yd2x`SvZn&!lzLTF?zf??l5 zGF<(2mxvT#J$MX&qZ&L7f3 zpH}?7FPqf@+MdTqVai@%I`2N>xe?PR%M+m%fjK!e(Pk6=>96;w57j#f*M`RleH|`^ z{F}W`5O~tA6k~_c1J>vD)tSX`tcGF{cet<+ig|^ym4%Y5fCR zbN(-Gp!fVG;P8Fc^wR7Bp#u*7K`pYe{~Zlq`k(*rKWEi?yufSta?Io^&J%^2OCmrF z)hQsOw~34De4hrZO6e^2dv=AAD>-{iImoy3s-%f=9c!-_XCKG8= zkaG%BpJ%mNXtcPBMHzWsF%kS6NIEltC#kcR;c8?PRPj2GpmwqwEFFTOBvj*r%p}%`?;c%G@#zZ@8EEM~R_Yqp zPx5V=tFw-I0bGVysn`$5){L-?p4S!fPbmp$1{M(E3%4!y|5^_MqR-_^@&$sah=g3T zY^}h4lp9Rde;ZevPb!*-Q2AgG!jDkhf5HWAb3nf3e|CpJ4Q>tbqv{DDwrmU4rnY)= z<5@rmh|V^=?Fx_&z-R>+&d5ti>yEK*rN01&he;uh2x)R;sSg}hR) z!3VH{c7hn0faJdWv*%!Mj}ZI5NEbsRU}!ptT%>MAWDA`7M*WJ{=o9-h2s12}1qBpq zgW?_QRSE;r<{mOGY1)LOI! zTi#TkY=|wki)xz0M}gWv&b^uMh#&{-;o7xcu zmgz^44O<|Ut(?rLx4!-$-kLmh1$rZi2L-j5p!mi9iHkZ8+rDUY{m=FgNJ{{<+LDXI zV@pQ_E?9YEVYB_l&W>>eDsjKE#bn{{CmS!`h$Au%ByVsEwJz z&?GAu4IERvsSzWF*PHD~gZH`H_I6Fnw}?%r>Z_;Jze7pjO4#q65`$aM!!mpR%;JX; zCOP_NPBFdV0q1S8Mogqcn_Ip%cen4ug4s3ApficJ74q3?m(->qVf@C&o5J52%Lr2s-ZNF`r-P$OzyK>ga#84pCP#uOq zcW`HheQ(JUZ)=+?OD~imZbh5pLu>tu8^bOL{7&1f1>bB_L!x2@!|d-d&zxOSYT5HX z-CcuCLX|Rev4gP@Iq0hQuzpDb=`86hSoC5m+2jtH!(6dMuSbC5Y|+L9KM`()jcJy( zBp+Yk|5u@s3V<@Ux#2QOQ91!zDeZu;fxOe8>bbf<8jJziv4l(E8R*-g{x^r+?@B*J zhsnyXI&ZQrc4@t%o!im9vvv&n)zp{zKe}V(OH<^*^tVE;BYjj+mj>*kUE9#JX3Exy zdZBm}P*FF-`oBaRS5xsU`N>BOaa+ME1c+j#FCygs*;fA{^TTv#8|^!7i(hu3>UgXwUyj$oMLH$8dW zHq@hf#Hzq2hq%N7BY=vby5?_xOe~8CK2I57i+#yB-vjpZ8DnKr zK+}H_dmcew&6k;-Po||m+$>s-=PUE_I{FN-o=%8A##Q>3nGnxZN;QmhZKWJw-)Iza zM0auDNEBWgtuXe{XUzbd%r&ou-q3(^-@!&kB9rHeY7~0%PIZ6KpL}G^3nWmUSnMEZ zRC4bRKx5ewg0dp3DKyf9`L>-f!Et2$@e2{>QW5?g$jd5_aM?| ze{4tV?1P8l6JvJc@K)E0ypN|YbBl$0guAoEYRCV|lyg8t2Y-UA>ASO|ZJoHXdB_oY z`RBkk#)U_gRhd+}7{Iq>jlZ`0Of5wG6(gav=g@t>7i5+K6sv`RcyVVY5QX0E-hX5lc9 z04Ir%yZv5n^#ei2B1ep}WIE3vOD%#Zbh0&YsQw1PJ;zB45A>~&*=}TNOBzf{-vzFd z#>3Jezr2`)&>vsJ3Sd|lN%hYbOK1Zir1Fr-;xm8D8tw+^REp6(we?b3+t;M?mz%Nv zQwGP96W|*6`$V%I!=IJM-8rXX>r*Wp=#k8X7S_#GE0mT>?E??ZJ5{|_q-R{jW2t*R z6-@6BFd@!JU{9S77_R6C^*`K95-#1Zr{Yzz_wroEde<#(t@~(Fw6RdFfk64hpdpi% zQ%{ZjsgP9-j)8nkZ1*-nHI&$t^!xb+MV<;i*h~A5p3m^GsSEdO{zG~6c;rsH55g)Q zyO&F&->8+)1`R<9p$_!8F-zM>`+r?fY;hT@gU}#&Mbp47EFHLedJ$%)OQRVH<^4AB zFQgYl4_xOB87rs#h`yCiWd65rg5w)ZE|iF1Tx_nowl(PWlw#{pnZ<75!^xEzo)6PEp{q+{H+4N3G5`)9=9&!CrSidmAXbW z06-G;?jy1-Wu_uftHNlq=TNKtD*xJ%`Q(8{WQH?EUo{a544y2zlO1@!r6+ckw@$`CJv~KlfeuR9tvpaO*vK zv=9PeX8##{fAMUBGTRLf>9&XB8%Hu;dT+hf_W&L&{tFDs(P4FD*2Ioooq*T-C%g?S zGO{8yqbjLNqaLA@dC=7T)o7%K(EWL=JOH1lSUNnpLuM{EYs+<|@tykglF)`kfWTVn zX@(mq8Re6;D3M=3@)zmFz-I9nKTez>sGJO0xK;3a?l)20PA-uT;4h0}Y2V1(1xXsw z`KDR2>V>i?4-*jM>xc4}(^{%Ut(YPV`lkv9y-s`Lp2&qXeC-j6^zF(>K7b6_Y;sDf zI?&Cj(A!woJz+mnYZwl>pmp-tkg>V{Ki~D^g@_-Y6N|}cwU5KN&~?`hUR#~R|E--i zZv_C7N6DbGL)~jSiGb3ndToG-?8~IdPlF%l%z9ClgUB_}yA6Pp-O$OYhKLkVLNzC! zG_`Qr+X?BSd+VU}=>EMvPZH)qMvIuus%sR?+CYq=766^~ZPV;i3o_)Fm(r;|pbqZxmY~OXka4j7LVvx?N zCqJX2gm67|%ROq`%T*|f?@h@gz54h#V$0P`wyIlX#}|8{XDxPR>AWqKqr==UHLA@s zs$g^6i!o$N6wpFkk{3nzZXGIH>t%nr4ubtQ*q5kk!VkuzJN5H3M3Taum?SsKLd6iBm!Mc0Qyq%qZzQ8Y~H5UXB7QErmGJ{pm@I5^Uz;2_vVC|jO?_S`-R8t zQa&K}P;U`v`v?1Nvid%JJL!RCxqWRx2HQkKFvYK3vx7-~VIKBd74a{G^{m*b)BbD? z@%&JdQ6z^Lr<=>z*7&-1GupcxHQg9L3vlOwGn91t^@ z;QrAWl#wSovsAknV4ZCA5y`yEOBrf3jab23g|GWh`*-Nat2Fv=nn9-3L>9J&5$ zG>5<@n)!d*Z5R&>S{44C1Xa9XfMQkxY+KWy*K5(W_042zSYjWSNL!|Z`~GcK55NPC zzQ=9amg9}R*GNrYBf3Q24$~!})3iSy@XB>bVsy}vcTdN4n#HsyTe`$z-_y%qodX!7 zSqQmXAF>$3qtF1{{!4)R!Em0UcsH!C$A-o5<8CAXVc*Xp%J4@Zth-n=xm?$+?lP8g zbv4+_CY5?7lZnHCa(6}()v~54^R)T_S405= zMWWqhq@P*|MXR$)onMlcR0)bh+wHY}utnVB zg_lp^pwB_{Yr)VZ!j}$Ls|Fesl&%)&s5=qyXMz2$f`29-tBvb+IZbPtF-e)P6E+iO zGo|+4L`&ybQ3AkK4Hg78FV`e5PJU8~v0`^VtAyPvpn#Bt|pO2j0LMuclm#sSjuh6%!GNwnWlNwUMC3 zOOFv1vuf2Waj-$TJj^(+;FKdijA5i;?WAf(y-aAG%`+gkt_TI>2Ju2c_Q(1{e5^HNH^;XFI6 zC7aP=WY5=JYjswr@UXrmx~|J!G{Hv`5`amh1G8S%4bXN)(RDNAqDfRS!&iz$ig__|_ zyJLSkA&1@R>YVdXv`>?*-s;D5cYTs&ljmephcLXecywA~w6=O|rj0qY_5l`B$EH8E z*OBDDV@d>RF`WbGzPiZLb^1<~qj}TcP(r3@^Gc}X^FCPc9XAh`vyoZsv=IJacMzI% zkjA?RzSDWd`gYG`#_LU%L;2vqZCuC73#iz|UtR*5=p=l4YVZu5QWm0rY>IzRh#Fqa zH9sFf<=u=MK16iZp&U(?-fp>wi2pKNfU z`laC1`ozqiB^!BlL%`m6VaO#>7lq#2HRgIUnZdU=3j00vo(!t zm2U%|Ypn|Ki-a-LXPMrfy|eM5(c0ZETfwkJAI>|fJ@t69eJzvD13P|1zn$u-n=1ex z27I)DW^oW#gJrJDij@6CLBUj82kI;i#^2~K0CFWyT#HP=e(O_Kvsa_`{NFe|@C10M zd!pN8r>M?@uh^pU?X5LcN()XAQni;iRw-CVhYqv^(vW|bZJlPHGXFY(IfB4-O!%zN}a6nmMDgo?VBvJ;tbtc`hIU{HA<1v zb1fU?L_cd1PgP$Yitp&}8I5NjWP63dg#?}EGfDp0ax;N`;=0?$MkY4;*QbC9pWbI(u2UMWAd7?J(EQkNN;b6}5R|N9Z?g?QvF1tOB6VPQS5PdAX! z|N9r8{~a6xsOZ3%5z6S7+4+c)TRN97A)1K97=r6KZ$TNzNaf=FT!)&=#m0i3j!}hG z{bh5AuNZuwC6}WG+YMwS{V7yTLDmb<*c{&mMF3am8OXt{| zeh&BnHI{~Q-0q9kjMBf~X9z_0X`ldrqP4q= zP5FAf7t$~i4iAP^aYI76AT(_+#eXv_N|iO7e1x9Hd>ziWw{7P~T3UCfXn^dDT1mqjXaq^AyGcOlemx>A#oi%mwe^kJYsZ zI=8*^ZvS48XksCB!ywft7<;f99sj)k!H0d8!F8v{J9KIc`W?N#nCrs8 z{MWPaUzfQgpK?PDXu0LZ+m~?1THrE?^Cd^eF!@cM2l;2jP+ATY>q8j5vCZ$ zRDWu0G+d&Kbc}gidMwyo2(No>l$Ie+(O<(d%*hVQZx0i@`P2wSJ0^U`X5h&@8dxTH z8Du%)4@GaDT}E}!=g(VCwd=m->&tL^)H6+N^`G!AudOt@2+~My81HRIQk<+UpN(K7 zU@RSS=vL2X=~X36Tsd>|z}V5ewdL7aOV76%Ng5Ux-3Bp!yO05Yy(R?8qYzw%tnK)& z2(RAl7#1Czm#)ET<1sn)W;>9@LZuGa1qW{Je;xz>_0HHyp zKMn-KaJwZ(xo)>R2(s@&+@xMTta^g}l5+F4HYk?&c0$Pw6Y6SFJx=k`L1I_GC3IMD zM=+uczwDbcg)<=N$+u^8^cJik(4VYq9YE{oRhScIxG2ag{{}DtTLl<@t|uGAz}Ga> zb~L(7uZ5Z_&3B~?#ZquZxw(W615Vb5qq{!^J(5dtZ*#x;yCOLS zd<#0H$s75J{x|d0#nVr3zZu1KwAZ=g&1nVWaKDNTeQ1}-G)OCkzCF?2cM`GJN8zhc zVWEb4^#F%RKvvq}5{TgC;X-_GI7w=A?TauL7)3_+FO}buo!*|OYi-y?y|e+&Gc+R9 zI%M&86OWnsQkJjEDV5f2htmPGab9dQpi!<^Crr;@P55h0Z`{x3dC?ObT&XQl&f_>v{q{A&D+5SpVg)ebJ?A45?q=E}l_yTj zRL_Kx)%9epePN;~oxewJX0}$^vkAKx?FvjpgJkImG;xgRzj38JwP&?1B9?%cYOcE| zo^geHyll(L_KnnO28UQ>eGoXI@ z+xcX4`>%TVzi<8j{eeRUjI=T9%h#n2BlJ?OCWid{{A#l)(lgzH(0++YkoVj533v^X zyck<-imCZ$s~v`zY@?2|@Br2@Y3utj4D4eSwdTqc>d1ry_^kXIRSNuP7dn*I!~N8{ z>8PC}tYg(C5RbCCtMEv$OZnRI+&NM)8cQ@Q7ohB+w@VZL<(rnUcQ<9>l%9$~|62|| z=pp2k$+U-;<;XAX3Mx;OpchMl2`r=WO*;vgnV4nXE@U{tw!cpQ7Dh$%g}BLyo+AVt zXL>@eWSl(5{Wp(?Ls~+g^x8nZxl_XAR*hp0Hj^_P0g<#Zr)`6HNvaVI8(gTMprG6R zlIfk45Ktn-i~hJhoO0XrhqmJYVUMs2mKYjkHRcK^5yu;Ruq$0}ItaqQf&};T_8M{l zqg~2GOC}F|3q#N-ofJRulk~LnDw4*Kqn4S%L?tA>-esDsE8~o$?$lgfjwu3PU573_ zJF?v|UX4)&2IXf!(?&a?Lijf7h^jt#bDUMsYe&YJ&7WFdR z>1oL+@Mu)^_NW0>x|ERL7#cTwBYTf1(^MUv71_^;m8)Y7`(&$RHAW#x%6}h4;EqUV zwXQHQ+Nmko5qyNeKOkQBjRks=QBXjViAQS|COZDN7eEA{J4nuteJQ{M`~QC)x_!w> z3|4$0s6tZc!0wLcgx%aY_D8kCTWEQgVZ}_*{JeeX-b1hlG}@c?hJ&#e@_Y zkoxy>Cn&H8*ks|_Vp9`CT4?JODgv+@Xxz74SiTiPTQ=_YoZaUegZuSYcIYzl5W%-0 zO$M2O=+yHpAZ5`!6)M_wxQ9r|PN~!eq7$83S^Yx4a%-|)hOF6yEI9l)w+Kq*dN0-M zBlhv}`7Ucp2;!fuAoA0JCOn+vfx#Sx1f%)HqPDzs54BnRIS+=HsbzH5q-%m5C@{*W z;=~kehK1N$>+3CD|E~Axj;g0aR}BWqj)ySqbz1_DmQfq7?yx}N=kj^B|n@b4mNSv&-FlTa`K4y z2=VtDH+dR)*g-+?wA28Tbiu$M>oxzm+VpO@E*(Nl;CE()1a!ZP?^|qWH=0&bXg6W1 zN4ezM@Q~q8aKX$pb&U5=zHh;2J6sFpEZD#KDn{rTlU!!EO8|T1I%`-nQ)TxRZsO;h zle|Kq$$JU~!Nse)wJ!MHS!>EY;U$?5F*viJU$bF%TD6mbOVl-4?P}Yz8Thk|)l~+~ zk^5z*8zbR_$q%DM_MZtwuLZE+TCTTch|ni*?u%QqCh zM~-FazI@1is8;6a$vEk5q$bfO(IJ7d-M!yG|}--;Ju?(>fi zl|+8ks8Q>P{Q(OHepy*?%0+7JRxZBrs2D-jNIQ>m2`JJY^VP7d@NtgEKKvbXO)6k>~V|4;q+wlP3;`c}f0EDAF@7RRNdRMk+mT9s9&{`<( zgWGppf6dK$_v*1~@f(c~h)wR9yPVZ&`Xvvo$N3Zeew5zzpenesYB6Y={Hleu z_G)%w$~$kkPrsu2`mPa!o6_yAu4p+B08jDjXk&eyhKq|xValAQx>ljx!NK8-g^r7? zLYT{Dae5VRY}vVB6*&1<3JOvd79|h94)V;^oLo4R`Z_`&I$By7IeyxH*=*T~YDTrf z2%nyUpuE?CjN`>5pTB>9l~EEG*ZGHUumU+foCx%p6X=NUUB>Dxfy zVSoIbt)IN)va9J)h&Ff82o@X<+~U+tUlQ0}f48@E+iclR_G8=U!x*w%i{5P0uT9Fi zm8v$ExHr1Um@hskWU=%|F5I31e^v0l=jqbGOr$J^#m;zA8A_kwA}%iOQk`6malFxz zXB3qJ!kbY7(s9N+&u{+m=tw2;Si>tR>{?W{8gxQG6WLxd{kX(cdvVk>G@jZ9j6M4P zXj}e&gng-}$B%|zpll6StJ$fTaQLKE0)kJGy8v^S2_iE2ZNb-EFWkg7)RfQj(VvW? zPKmX)h9a23XQ*~R@Yn`k@QpKTHiajkhUDQItzDbjdc;M&c zoW0c=$ePIA?__nVR)M**)eU;J-cbNh+`Z1o71^APNhW9Ku^{$jQM z2b%mvdB6e?HH;k=XD;lpE-n+z-}rThC2YMf2Vc1*DuqbNZvE^3E3WyU4+EEN1fzVK zcFEBv*-rXzDEOaW-hrSZ{;eME0rxdh@c+ij|9sc~euYgPI8Z%^rtHmgc;!HG7Z+Ze zmVI8Y$NS47`}jR(`2YUNzhAl21g>ze8Q{>0&1eh)BKJ2v*M$Phw5q|Bpuc`KK88P_ zKuMg&Yu&Xqk{4bG=kWI5S>Ashn?EfLweh=H!=yM={~D=|-Se)YA*@hL3dwLnRy$zd zT+JA?miDh-^_bc*TA%=Iu&~VrUjl(Kc}!|*7|;m43XJ=HgToT`1BU-I7csxj{B>k7ESeSH0K9D@Kyt#95pgCGZCg-<@ghAQZeKKmsYjkd)Ku>BHh9*iZM%rg@v*X?Ieo==6RyS}d%P?*F%87y>fW#GLPu zDLuAmUc9SE&+@*Qnfa)tgY$F;k^Oy@3;}@i6wA+R+$KsiX02b){P)vvcsgW@-^XcwoPAPe?!gKq794lM~j7j|d=3Et+Ttg@0Rs8F{^7m;{13j#lS!6YX2@bsRrfG+5B%m_16=!6hI9`a0Gjjd_2je zU&vlSLgZn52`nLTJL*Jo9-#i8Yrc8&23%vN)C#x9A^tPTcDF9 zaykj@q~%reYruk$H`SYj08aNk4YC1YNK&!g1<*<##c#cFZ9`28y zEuw#E`V(JTPA*Tk$>CF2SU`LGV=>kjk-vWR|Mqd4{s3#S30s{Gd>ISKN%h(D=iBOE z0Be>~RQ%N>m{7~|*H`&UQSLSjDwS|9R4*kY8v)VF3N)2@ueJi+QzI zH)C~ldK%ch?6+VuwTJ`{-?8nqbMM7&v-;?c9xUiXMNAB7xcauA*u&~=>>9S$z`kO90c%Hc|iUPuolg6~S{X`A7 z`W0(%b_nqkw>5f;{j0~+j|oW_l@5T1-bpgmzGlA&U*w8^*xlSGG@$oUSu-i*Oz%SF$>AXPupZCOW86bK0yV_+(rD%>C;AJC- za~}ON1sv&t{pqZ=wu|~7{;~_eusePzDc44kaLInFKmDbz`~*&a&_Laejpu=7Z;e2= z?FgXt%lbw2)6WRK$g-FcV8d+J(;8kwr@N)M`OlwIcZ-pxq@;ZR`AgizfqZmc*6##G zGdQ@U%Q&vO)o&n^U49)+9lsov=JNB~wn|^ZYGvHI+D3$FD;GM8&VXtX`~CyF)Jm5Y zpW{XtF(L8?f1UB%Mq(TsSR~;iNAr$Kbpd`*K<%*BRgZVwRdO?tj}gHVBS8v#SW`4Z z2Z_FD`=yTGsfXcnt580E+_#2}qLqYOqbo}oVdEI`z}pf zZDi)F)W2@MAIcPYz8Ml9`I}mR^wQuK2{B7iWRBZ^tYTQpu%)-5e-F!Rnmft$#8^ zqmAi}TI8pElUwc%cI=Y3Qvtj#gYYYSQqGi#?U9mzs*l(g^VZKx$v#JK_diWrlVR8S zqU!#nItYP_ZLk!br({2I;9RcwO~6*k8Py;~wiE`z&|J}PTxsZ}SRiLhQ;Tl0uN6DF zcoEX#V%tdG$km*h@fG2U$_t?ut|MMji1g^>_fqGw&d z#Cbi)n~;Q(kb$nz{Dlxc)&sl#%?OwT1&pJSa|fz>38_}gZe`nxTnNoe(XahDf-iQ* z4GR@wKM_(50gUYL1AlT)52?VD>|hS;8c|#VVjQ54%yaaeE;_yTq@Eb-bu-)3yBQPV zx1er^UoW9%1Ec^HTqlTEMgw9(4xV9i7ocQM1Yn}kDBfrL!F5-@;su0Y))1HqE3LT) z7+r1gx18#c%Hlm69f^x|Fhv!l7>u5sFXJvdZ&pztQPD6HY#eh~6(MyyXpqDP7!94x zLrW!6#PL1&iKCMMYfmEB7PDa6350XtYiM9!?vt?2bCf2G_$WGONccj0EtA)1czr%?R@yM;bP3o4PM}ZF^ zIJ4|S*CJz(bap5IRAw@(^c&6cfJ}UX7hG<`GdH?wG(dr%{t<0ruQ{Nth|;g_K@NhL z)F}uBL0iEDX5+kdylN5x&dxkO~#ut%fgWD*p_v z>3BdG$-pZcg<$iG!8bc?=k6-MjC?OI|MOa)I9=wiH$PDM4-bO@(K6KCsY;?Mr30)d z|I#5q4YgSiocGu*p9VZTCmvQust7JJGu*^AM~6fAtRRA5-)d(9IB6a&z8oVT4E7`~ zIR#o6N(1ywpLeJ7fFQ%@B@?V>p8AMoi&U^X=YbYprVnj8{Qb%k2Yliw`*OS;IzpI` z>6Q0Y%&UxL>sc1dDp_0*^fIM(82ll(%y}m*r-kGDR6Q6}E%XcXbt16FT<2^%e#sAN z>edqX!y`d0M}trcPb6?l=u9FMWAZ`_#+(sbo2=w5-#)a4@T+Z*9BO35YG~vUk-iL? zck?6u?8aBv_zLui?GX_D3=?$H8%v`gl5i>}0bZJKber4RclVO(c7r^$?hMZlvoVox zx&KG_(ZnsCssJs_Y9Si}NSPmYs8fVm=|B7kTFO%$_GBWq4aHv#`w3vB^4UgRF8Wdd zTE2Mte<4D!G`yCA`PI?#tUb-gXW?74-#-FDun~e{HQ$H>fKw4nD#4(5_D~o_>*)ud zjtwNs-S-{rFt0@5H5!uao!xb3n?g>MP~mxSfA$B}T}kcxsfifC$16G3n6<*dhh)7F zFQ>4bH~Xp&3g{NLxNz#O>7K;HWH`!_oJ)=cJ4W`-@;BejPnl|S0I8Ex^u|} zhp3pV*}!#r4^{YE>M5zv6+VfH-68~TiawW*s;WGs$lQXV`uNhh(}d{aJ11gaQsfBG;^Iymy@ zFmngj`I4xglEH?yFH6h48>QqbL{&PPIBGe%j*e5!`{?O)fB!A~hBChMpD4!PmzLT% zZJtUpn;6f~{|cDoeImMNm1zD2F=eEvnVC@)_gesgWv|l#KAABTfzqc0YKfK}>b(0F zoqU+64|g?||M(+ad{ecdbxOV(FYW$I^@Muy>t0!Woh!KZiJsoZ0kq-W#tyK^o*=b*3HH5-a-@(;C@bW zexynAK_A}(Y1FZn)veZvLl)f5Z`NF23oY*X$TF6Vj5Pir-bRrXFa3>D(qmY|$_F>u z7ZUdRR?91zSKfb@_GLetw!)$p&5HVc8mWQ`8m#1TQ8eF%nsTYl@ z-3MciEZ$kJZ>#n0(a*wx2izRCTx)^|b@jJz(y+v`>{IkQ9oZ#r5k{yZ%xb34j|ZK!fxppc4dOq-^47>3{j)-|ztN4hBY4sM*+*)2(}M z-^d?68LT+U-h4BOcZ;idA=)RQL{v9DDDR_Snt46kQrr|KEl&HoM-pT&v$QYASSkorM!g5QK6 zTZ5=%K?LEJ>)u3GSiMiKZmH;2>`{reqF5n|PnpM+t!%;|<)OBz8`&Xrs%w^Wrtvyi z`O8XL<)M)I_UiHv1W6W{&qB3Z<`g<;8WlbgS(p_8bw(g?5t4dFCVj%j2?sG1ng58` zkPIL;aRg`|>t?$~J1P8!yY+l>x1PNYh%|PC$UusQ;Z2cZ2i4aZKz(7WDerjT|uEUgxWd>P@5`P#L^L^HQK_c^2pYBAB zd55=N387wzLs9F1 z=+zP&x79TEQx+=Fcru#*Pqv1U@+o1XoO%uPygYmn2UMp!P=1e(t1D)S>M{Q1B*bxW zAR48FPuvRt+QO#aN>7O^1F1kqkCGvvW%^;UO(b2J;o{e|PD`J<)^n{NRR}g!XKq@) z!Uz*~Fyd9)(pqyxuql0tIx?u%_=#F7){D6NUTQ!OAi6g8^bY zVie+?6sq{49=zp(Yhj`G`GrSI-5wrm{>Z&qkL~37cuWd1fg=XC%_s*g7s}*LC9HXj zGEK40qP|^4Q(oJ!ay;^z=&a_l)T* zILhgOb;yKeiby&eH(k0!3KxyZ6vSYSD}K59X#;*_P*9J7m4RY1 z_S|oM!rQdv+B=up+};1v4#SkKi4u?C@J*ZXH}>AyW3zN`eFVwKyhxtSvg-ESNj@9#y$6L=9!D)n`}sl9jxfKl{YR{F zQ~(GJ$9_#6BG6{MI8cloX>qj;3J%^Db9(;Q(Pw`^=C+w9)->?u;U{@v{BA@S?oYTP z;`0aUN!&KmFtG(7sPnn8mQW#CY^fp!9jcNq+iXrttGGMbI4xwjE|wOQ@M* zvSLQ(*Bb(3r+Z%SyR|-lC@MIEk`<=Ok+5vtL@Biw;(;(1z0XOjzWx@WYrgF{^sNud z#Dy~J&+Xa-ryLpe;F1x6M=e%|fyP3v$wa*Nv#WZs4#sXf6^cw%(d=_s{L}~XrJsSMN3uSL&SBs3-em)d z&IvBx@>TQm!g&Ug2S-R80NqXh^HXneH;s%N4l&G%5Z437X+bembuK1aXL2!k%Si?C z35AH7h}c@9$$3qG9TBH}e#?FSj)O|emg2X3L}#B@7MGpHKL|hyrwHZk)?|l%w zhx!BR>_IVU0CWYC6o&;(0&OBf3Ok$${}StghcfSmWd;EL!^YM{x;5@&L6oIVd-xA7 zuKNyG`=ftL1Tvc*ZJQgAZf5_BR{onDWAuM=ul_`OAzsJWpCTfpfh2jP{2kxCkEcX= zXt;S0C+<_c5t3W;(O(yDG7k%)vZ>$-~B`nJAUP+_PwXj*7Q8) z%a?!>jgoBTx-cyspJ(d7CT@ZzN26gUiA{8*CMiw!49vdeZS_JiD_)-W%{M_O9}Ra} zIaq1cp3jm)H(E;;S#;?zepErtT0iWRRXwD!SAQ`a@E~D)hgNbAXU39YjDMTlR&`zu zos0-IZeI$v@PIQEJHGoghitivPfd+qvd-$~3lYeSYAm2i8;>JMBeHsTEudaH=eI*~ zqEe~GzmYF>?u7%L@&86auklRJqwJPY2Xiv>y19)Ufs^$mA*y9}wD`Ok>!;YPI+KtC zrbiC-ufI3Mj?)YWeEaiP%YNhV*CI}aRe@P;5liY>K5($3L41l`+97FzaXIVF0z3tBoK z(869yzI&CAvz&@Csr=G%9tDR z;4NGx2e}M@e}OP7gLeex^HUT>kS*6Y3+c%zTyS{&;8+az)2-7T2W-cSxtyVy3xnF? zcZ6}waf+Db!7{O&xuj*|fxRF5va|WuQL^=I>3Au%K90&w9>Ps*{fs-1L3KEcObaM( zS%9dP^7anT(EkT{Ezbzt1^KPuBqIZI_&paPARi6@*n6FDn}1aVz})axF926L3$48zP`TXLN4gKFX(`;>e$8s z_%cap=|8|(CY*2^KeRcMIGElbwTEukh(ZfiqoqZ;0Ag7yGtoVb9=4!n zA%M)OUCM+THf_tshe6B`Ke$JFvN2$k+}{i86-3q2LCvCIEL?t6)`SFELlb@S7ffc| zH^Z%cx6YDEF8ep!E*v}ObdJ<~V>U zfk6r9Z27dY7HDyM41z6D50W4q`gfu-B$bj7@R9)TOb=dt+ zB>_Oe&0Ti?hXl|skN3U5U&_wzrC_aXbkk$YV!l+_0!4`|LtwDvMCLWv*|Td_gs9|= zLbfrN)#JU0l~x^keY(1p;(1?4p7^dG+y`xDlFG_zqp@Z5`~nI&w-`JXXLQ+Det0IA zi8A7LOhMCn$mcaLA@xY>k`&37mRO#n%JA#Yb2ah=8Kb4G!uP^I1=fHo-e^p7;l`+4 z<2G74g<5Yay3v);3EOa_Yooj>6TsG{(O4Q#4h+-#B>!~Nw+{gb%b%O}b|CFuX zo`-$xs*>P-CLi95Qw( zX}6ExJmrnNtHlsCRb&P_oV?nFu;;4fyQ^RDSI5~|TN9PUS04@bVv`S?b;TvYx!Mrg z*u0mDV+d=llb6+b zT|+`&ldFah*U`KDc5JvAGeS+}{A5D?eil$P=D4s4|NQ*=_6{ndz3n&_tjW$zrpbdY zZBdW3cyJ_TTu7ujRTY!GEIIJfa9IM4Y(?+LO}DYKK#Vl8ZZ)%*1Uf_Rw(R*<_-MA!ZcPU>XpORGd6f;Gkq@yUs;fchPL)lN^sPR9>-Og( z9YSH6^}4H#pYowBLKyD#;%8KLDasf(K-vt0@($K?j7|DqC8xh_1FVaZ?phUD#~gx5 zQ8EUsiP|SDl4<4Eq*036~lAw92B}(3P}9Ha34aJT4v{4hf07LpaH2XaaNS@BOsp z=qVeHV%&Qv#Ymn>lhPT4$>npfs@BVY~?M44Rm_`-)a^ zVbDRTL)cex(S1BTtAk%-`i%O0cM?R09=XIx=9zjs=y%MAAcpclE+aT{f!E^q4@c@2 zaP8LZwBL+6n{3(23i&$P2L_2#y(E#q34Gh#Ku;!*0|UdozIm4Prc$>JBw)M^(2$ znAo_eaU*)I_76NIxparie`q|%QrQ}?ndyl+rSDmu=@s+D`u#$F>h4v1CmZ_C#id^V zBlEvK$WlpZ#c0Lz!fD*9@(&fAGs3&To|O}#Dm!~(qR$zx(MdSrfr8>#DCapBLsEY{ zhjaX(=2M~z>@>iqS4_^ip}qI*enTBtkN?WOL8V1DS@>8eTNjRclfO*LGKc&D8f6o@!aMm!Va6Oj z&+pBa&>D>{$DYXiLW2Onmm^5#W+Qr=?UX*>59Yad*Gp4$rYszL!|9Y!b1nu$@>$j@ zti=2x5aZP>!o-Ogby;pjTwxb;r@YF`TYS1Y@e4pR6>Gu-g#oWFa@ci!t#Y+s+W74B-&uf&|XvQonD5 z(qr-PtTv0mY1Gt6BqYFFp1kZx$eU9uw?+Cn4ijbNS7#Mr7MREpK#9*EmZ@e2PAv@Y5nlJF`RnKg%%|J4W4>M*8f9w;As&UAs0H zy^zQkFfI38n;+6FyveXt7sku;zH2wrs^20xd!`c+^X%~CdwU9DYZrACFp5$A(;l*u z|JcLnkXR+!z$w;kBpDx}n$@T^4fqV6pT>TGXUZ?`Lm3VrBlaf^myug)-mQ?*)lFAk zsIyz{%H92xeqDwQ3{Hpu1HXlHH#fVL&a)d_pKlM_#l#TSfwfStF|A#@ug}@Gi;xW{koClf16^uu*|J8F zzHk7t)L8Xn7 zyeNer*CjuLrBU4EB1a=cFxh+vGYMkR#4=(1tsVc^f1(T_p{euTQ zU;~5ex`q~9CRMcK6k2!Z$QrTT#`krSeD_Bn&%OY)dnF+7?84E%wkrYNvW)om%&6&K z3Dw{6#g6k2=x{y)#Q#fA_qpV5eN(7o!kOUZIsGR}HCKVHA#(EaVVPx5KIEwtCYNrw zRFnNaCFFqDzW)Apw?pTHQ>EQ9vRd!?;i3Wi&l^X1sX`&I`|~Mh>#RJv=pF8#cMpIN z-1Q}cP^3r_yFmrR4GB=DlmwP3DX(q?=xq~wu150J`avBxm06kZ)P_g!NJ(!V$2RY` zv-js3@*Fn@n+eHlWL{1@Z$Lmq%mpY8Vh-aVLKbBL(W*${_S*v?)=zu0bqc<+MKP}Z ztPNXdkO(zBI3G&)yW_rNpiO9d6-^G9YpDWow3@TrZ+4)A=2uFd+V)Ml1%5eGNz;Oj zw?Pe}wybQ3vg3E1?p8UbG7bWdr-@5|T2;e^ylbJ~;rsAu(I*Su^7PP;Wvp??!WrRb z-_~l~u*)6vL4RUDIGJ?)Sc08Fkf#{Mxv$o%h;ZYp4Y^*qEl`Kkd}u1KL`voL6vi6g zN7>6nvs_Q2H#L2w4bf-sP8A^kE{e(WoUq5< zyLbrRMUFN@w5#%#4jAC7riukKC11i%i@9T#B^4^9Npm?G^ay4j+~eM;!A`2Ug8Ez} zF>c5|@eYYt9_-Gp7YT~O*EUa_W#rQJ)!kUm3$wuZ$7;l+B?hr0g;+L_rFViwLeo~a zvFAmwN){QZUkbNZ9{wI9Q}N|M5^%3j(|4VbKu*U10c#-0p6Kka$RR`I{~Oz+TZ1zn1pfR z%sngw=QcV5R%2lkoac3Y@`r=0GLie5VnY;`qLBDvB<@_42%X`OiSYUx^?`!xFEsYn z?1c~bzRA_GOi9=GiwHgb$E$t2!7O^sP7=UmkaHy0q+qF9Q4}HO;4?;MJI=%Jv2=~V zR=p#@(LI`QwLjdH5)SFqe((o&l|JmZt&QZr#>@EfB`)L_sRW<)a`w?J(9*Qjc6)H^ zyZ_$Q)Zf`apkjNhX!S|sZ80-EYYwPf+*MX|N@0oQ-*#v7U;pA7`XKRAP*4ZW6_*GJ zc9JXK?ibM^jhW7pB|y;YJlh!J5=%SDj0g)GmE;Px%*yVq)Qz_*?96)PwVpLxo~k6) zOqzlN$a?T3vh!pJig?%CeOa}04>))2tF2TE2*tMa&=e7V{zu(QiZXXw5Qiqo@s9)Z zsH|&Ts5R>FISM4XG?Ff&nDkx&c6lP(Lm18ST1%~eAOT^*3hLsq{b>=hLPsG&X$03n|$*EWec~Dwm$*$lgBJ!9EM-am9yT^JQ=IfHOk2AKs~D1von) z;Rn~(yFQhr4F$h%g~&cTEpUGyDS#U3R9kh(055F$kf!r;yqJ{T8}_?mNZq=tUoqQ4 z6L`v}c+7f-J4MJ`^4tp+GBtFT(Y0L70hCAJ1w}!YL&E#5YJ+b8LJS?qw9mZu)PfLh zXkM}s8Z4^I6&;Tsh)oWJ>!Dk2f1uM%EH!vJ=w7+njpE4Rw5RLc{z}&i-c7Il?nL39 zym2Lrh3yk8CWWB-lwrZ{T{o6kEA`E@jKAF?H9Q;JlYQoOOg7hPU-uEn0J}jBdt}sQ z>EH+P`~fh$h0^2a2Ln_FmxMDkl_!<>2tGK4f$ zHcn5?22y$Ih9_&xRi2t?b;?-C@w>Fh!YW=GfX`N(_vRWZD{L_Lb*0MnV6N=c7;3G1{HEAP`DZ^R`dd=<2D?`Wq?0%MRIP1w2-EGu^n>k`VFRl zMT)0+besoZJZ;h~>Y61dxV9OXxbH6}9$((O@kCrZ$Ie76*L@rUnOaKdU0`Bc9jIBJ zYj$*9`q#VeaEE)$VKr>cnjkYFO*~>4-f{zB4}n~nanjY&h%k24xp0i#jGU(e`dx6t ztuSi7W_Vo6l9zb0_j9jvk_$1U&zf(IZq#{@unlAU5NX0Oh;YAbhR$E))SuZpM`y8Q z%+wfYJrPG9pUB!Ma(^EtHyJG@t&u$ z6OdE2Uf&e2A^oB$w-o z4J5vtr4xT|4GBqt`*B-)1zB)+W@9DZT){f?G>ICY%nj{Z6_{)ITy{15I^|TP22|LgSiJ$=?4FBa_C5)^zG!m6|rt{8IdQbhKueGj? z(z;;7jUJn*FJ!(Oggk>EOokaV1bSQKRTwiZ>h4Igr2jRg;tv>cClb{-2qzd9XHGdX zVtd2mY%6C>{EZZni;rJXt?_{Z-uxM@8WMEOy42 zGk!**ur<)DK4tj`tGPIp5Jny`-!c{ zTXoIvQO=J^8!g-dfCXc0bwK~)N7b~cNHHm=;siE5o$wF%Mrddj!5IacaQ0v@HDY~P zgslM;>ck3t2g!S7PbZ-lVf=N?5?2Fxf-}Sx@V} z9H)i3!O!9mrflKV&!okFO+>s7liqn9MULX4((imuN<1MEob-qVor#30zwlf&wDfIv z>DG3teSR3&d;*v4W#V|!H~=*hu0#;$rUFf5x6icgV`-uN=9MI)ur{^q)VU6}uT|wH zJ4!6ZYcLTkBkA_vq4>p1g(djN%mZ8kSP$=^g`_DlvGrznNT1y@>D^%k#c*wU*TbH; z*X7PVcpwxek!*6UFR~kIDlwqIdPbp-HsOjdN_L3{DDe(HZrX@aUB;%b&f{e_^7=k_ zdDA{nF0UlyVj5VO`{-Y-CmDn_3px1w8-*^>KC{#R#;Y7QJkt9a^MU4ZBhA(#zO1hk zYdZbYQa;SCy4^`|@OYE)7p9;GUHn_xRz%KcGl zMG7fi&^BirWSlKtnlm?aVe*7oIz>8=5ebZ9{Mc2rPr76z5;~z|O}~g0!}6*paL@0d zVXQ{2IG6-2D!$Vn&H$?VV_9$(PMjd}9@;{sd0vc?akyJB@YdXC<0WQ8g#UNVou=P$&EBp~@V*jI_ zxf0TXBbOYEsfD<+H|pDYS!x@+m|CPyHZaaV?hETl==PDzDl0)|+NX1r^`5G;> zrGv%_=x>woK?YvO2%`6AY9b_0cp)L)sGlEb4FXE8&tNMO%`jM-C4lnP`SV<2DKk-{ z?aC;vR<|mb6x9#8?;^J$Te@f{C?j~Lx~(#Dgn>3CK<}F#nk+sdF+Ry`gKZ^|*Y)-F z_M!Jjf@{&;%r-!COb3!{xQ>44{Vg1F;!hG&dPNL}&THhMK=!-(VBs1){o3PAf?wp*goPXxup^BB){J&bxBD1Q+)i6>6TN^r*;_kq zztrCkG+y2g8wIn}ff~0lDmMPf82@IvZ(2rtwe;jR>X)>ZS{CdZELaRA;-d5c*k^~K z=W~I6>#(z85X&iTA@dX+yFM2wJqN6gv$Xe@3z2^7JlouKyE{HQTn57dSOg!j(qJ^Pfq0;^Q@*M z@1xXEvAsQu+_Zb-s-$thKtkSk$~`2y!C<_<)r4PJSvfo8FwkW*CR^CKyS+Z?XP?Sq zc=lJ@#9sqD^(g6%RgoVL8U-%_@XgU!@&yofG;PMMC|pCLsc1}i;{P7;mgFx1J`e+g zj5j{P-68N#y*n(qW*nS)Tga8!@i`eivpcM`Q75pn#yogvVoHN-tcI9nP7(pGeTp#UjGqkP0 z9RrmJAD*dNb>0|?NZ7+wIBxN~wMi7~FrkZQ&28~zkDzbE*t*Nrk&*X{V4UX2-?DL_%^AvfmOzwJ}^bnZ4nGUZ5sF5rh1g$V`UTG ze*YB^!d7IJ=||hWl=F*u1)&^Dqj|Q^alljPZuQZ}=CJX=fgh8#yKiXTNc!0%Ug`Ke zGWZ<>Y~O|Cxf>#6^7puR+C8T!FXZA9_WY(x=Li$J&PUyvT0BQ+Ay4}r-sTFR+ue&Aj8l1ll}Js24Xql;EL+HQ~P&WPJGHOLPeJLkxaB-@eYEFcYZ!4 z=+iru{uz#nfxW< zw?M^8mF2fvxfT3;ZslkfS^-_*mBMhtLw@=3WBLA=9|f190doWAy~@15iY#U~9o3My z?jB~4W|^w4jl}k5J9S4efu^p#%tIzo*mUpWx~>zK+mWH2QTYGvsu<+;cVw$S#u6?3 zmE+fog6p%s1|!2S3$$0BUHbCaJJchw?4$UHe}>UKb4(Cqz!CWTcZpg?&aZ-{@FFiy z8i?lxNLlZM*34H@qAEFk`18Rj*v31E*0Xjs?2lZ^Qzx{_X|fMU_`bVw(UV z2S+oh6>eRSlts!m5Xm(Dh-2900NA27`!`@i4}~xdbiui;u2Yo2u|lP>KKoWh_iL5q zQltjEWyyWQ7v%VJSK1khT508W>4Q8=_JN2Pqjl%gu59%t`$A#j2bX+YJR`v;pCL4 zVPPXJAh_OY1F)zcY0xlkb_MP4+Ut%)s09cib9g{MwBsE>e1}fKSs!wns6+)n5m$7u zB~uE`K4~tsRfBVFrnCD+6_j~;A9Wh%gs2QRQg6IRuBvbC5A#%vdxO}IZWCM}ZIz%p z&*ak=kNB?y?GJSH2EIpAfh-#$Giy;*yf}zFO)j?=Gl+D&fHH$ePLFAlo{KyYzVD^3 z$993oGAPSlG40)#Rbb!p1e%7Iea0W$zQxazu)^ef8HTL3&i>Q_8mU8 zrK=r{^pC6&j^CtG2|zs|Fbpof57*{T!*OW;?vZeU-;y={DN?9}3pmoLPKJj&HD-oF zn)gbalU_OC4ZMLxC{hgYjL^`zM>30-%Bf+<-=WpDLjU+O8${? zxq#%)*giqj*jL0Fvw_iVQMEU3oVf4<02;mmN!BP*_t&50l&=ZDN80!YjZwG#0c;ufk7?p+&!88ZBJ{J%@}NjY1h% z2o56KjXqoWCyG?~vA77#zrJ01trEmghN>&)y4>^HbEI4e9k+p9YRD(4TdfdN7e#2P z>bl$8ihW)}mESCQ+niv&NxaJp5+Q9RNcriSh0niBk!ccMaTcS8)U{08b>;e4#h~d` z0H&E>e9HE!719{`*zzx+a&s?M*l~WZyxK-ybV%Y>f2&SDyL$Nk0ds8y<_T2H`N6`S zbrCgBF8bVBFNlb`GPNC(^e+atA+YxNpD&)rqP=Tu%!qn0&+@M4$CA1EGl&ELitK3| zM%=1MUqg5Vgl)0rYb07nkEhu%I}i9q<8-|6JL&;~q`&)zoZ$IKWOMx4=m18+n}U%W zES)`p3PeDdR|30rQNlG2RRn<+3c!e1_u;mEbdxgRQh@od=j>aH?GYWu7UHo_w`S%A#PeAQ}bC2>*x@V z@rx4h1aTa{ZcLBI{Ic_S@4l26`{}IK>B_CSi#3Ult>;^g-76WI zeqFY#&K}1i9B?D?s5dGpQZfXb&(KS|FafvS=Km@@y0>79_^Q#Ix|nE}yB3}4h(b zLCDRnB%9aMTdEjcK3GSKzI*YQ}XZ?7O!S&_EjEmz?ce;S%OS`3xQWCS224&h%X`I90y(Pl) zpz^dZKM|y<&^YN7t{hoV@<7lt*bnb^z{Rm{YnN2GaOx}X>Bo}^KI{r2e=`WOeK0(; zQKig!gu)-OejpTWCij?gg3v$0f=W{_acOg=e3_bz7R$s!H|2c78lm33lSf@Reiqdw z3yLmtD>&lcBxn}YZ!i6}z*`AaVGA@|gAXlqQsaf{vmQNJ{;n4Itqz;PDzWll_rPUb zU&@_vSwW6M7wAKgv-3}B^+IpgPz_0!`53y zRk?+0!?5UXkgi27Nk-qBS8vIHVG zg<{i_`(xN5hU1?T%NuIV-;H+6i?P4L`@{0TFLC4R3gXUQ#dgDV#bF}tAnGEVhA7+PEQ-VMgICPzoQh255*;_?9Du+gM6P zMl6Mhi&2=3fNeR#MM#XIKY?2Qf>*v!lQ3y5iGo@f83mHS8eI0RM6QZw=_E~`&rWtG7QRjq`A>-FI@_b8vX#X z6$XH^^3NaPXTY5-uuy379gDI!XHXM9^M`mH=CaIHswmm3^1eiuZu{Z zL#sXP2&%UcWMIOnw*IfMYG-F>B`rQr$|N95198burgutA#4M5y4{pfkh4KTMbyze$ zFv^Zw%a}Q-eV-i^ZrOB4#3VJ-(lQyREbN|~5cnyAM1H#h^Hgd^5c2b&IAmhJQI!{= zMmLDgkHM@P&@#Nyuk6ccTvVtr(`2Poo#h%ll>B!dJDjP({-q(9)CCmhC|opQkA;Fs zglIqgsRlNY0=#+Btc>WYW9Uy*mH{OPi*umldDGkAf?XNxT{pi%UfzP4Ao`>3D)gk* z=LL(w?R6pdrNL=(u)Z*G>MPe@KQx2q+e1`Y3gPY{5*?(vyqF?y-ZhK8AUEloZ?r+E zTh>?$Velw`djemXbkK}!nghqQFY@_dL)YzzR3WBJ<7Yyg!taeT=}t)EQ9cvoDQ{Ue zsCZ&#@vhSmz6l`8@4avMly~z?;zJ84cYY}r1IueQ_33rEMEfgZYxo$E)IG{hr6vNN z@ga#&6D4dG@D$P3oWB)1p&s!mWNJ!G3#?1P1fd+!uyf!+n|;8(YLAN zBGBd^%-P`I4KWZMebXtAkp6GXZ=)QxMyZpRKFW77nEZPfvj_$RYU5i7fIVr(N%e5_ z((C`dB~$BD=zc3C`WOJWX)y37Ndf3DbA;6QD`3!?p~ihtJkN2`kSA^sG|z&08kb5a z7?2)d7AjC`&M&A>%B(e+bcgEAOUzr{FH2qwhY^kYtkuuaiKmRmU%xRSw4FA8T8kho|FG*lY=$d0*D+jkl1eUhq(p;1?e zO@=Z%BOiDP%8YSAm}w}OTHy>B>4MLEe`!X3U|+#Shoe%xMQr{`?x#BcVhbnI_Ljj} zn9A1_t?N6b^%X9DfPR{@?;oHrfG`;D%w*kU*wx_^oL!cr2`+`boH~0%G8C~u@9nXs zC%ZVL4LMjrJd|Me;woV~4>1mjH%78iE;1rAEq|%HbCi!@S|um=^N(5IKtQTD#|#DB z8);|am70ZtEH@Qqk1;aEIT0@r*C8rPJ_6lZ68T=c6XswvmLiYOTULmmbp+WbhyHmy zU-Yt1kY%f}@yHWi3GX{3@D-+xK$dTdtJ!|R7p28c;KERWm((5Fku8yV!@y#yE){gb8r8ysQfkHczzeV7m_$p=468$@DWEZ z;eo~+M=zgNmzuk}|DIN<*D`bq0|LGad{ZHKY=D?GQw(b!XQC+=85FP%$W3q~O%`f_ zUw({{YI0mENlhUVuV=0O?`@uo>PlnNJf7Au9xQ2PTBtn|n-w=ec0@*mk5PqxtkZ}u zp0HU7OlqgHboNEQ*PPgeOZU!9^+GL5fGWwB$XIcE}9726AHCjp7XEVaiyk* zQm_PCgk{9qz$cok-++4EWq*lT>cm&A!7mz96}@_$JwztG(Do{F`HU!On9TW%*W5s( zo^>E85BtGEhmN1dn*Zfbw-}YWEb$Prkl;6nu(gZA-=J^@RIYrK#r6WYB(_?6pI8gg zUffXJ5t=rS+|SD_DFq7XomYrTZm=e0JR}jBJi0T6x&Y!H{&>MJOPA7I#K7Imd>*P3 z7y4xvNv(HJI$sh&UUd}581c7g;jRP*wHM9tG02n=%=l_RBo}e|HS@Hk1*Ap>mVC;4`XZ>6Z(A>S zz2vqA#ZLzqb+(NhQC=@@0is1|-q`$j+-%vc^dAYS+7tB)V~4KI?2{C8gW%g|?VYPdw?wK1Nwz#c%iro4Lw%90dP_HNoa=`FdsaJ%LhR`mW-*4UJyLA< zIT%j*xEsfM$J9NOBf`V)MU$HpEeNiI7`nJU&vVM0TK~S$fA7r!BjjsC*=!zt3jJD6 zzwwBrR{5?F1QvY0@}IagV((K~-l*i^M^jp|pR{6-IOLds-{lFbR(hYTk^ZC#MHnZ4 ztG;}shbMF{3N1@~qO-9UZT8|)YeOU+EMsrDZI5zxemJrg+%O>K)Fbun#|G9o?^_YH zDqHF6ty`Nn4P-4nZk5s;G|HORYPcM#nK&@e`T*{a4_+si+;@=l!Hx&nx8Ez+OM;G{ zfUa8(Yj~?l$ucMJMhM@B*!o_q3%f~Uw2;LEc~Kl^QX9_Z?hCI7JEo4zf%sJ=`T9=x zcU}JBRk$XJI%Nk@cM#2rQz{kQR&>^6wK+fY{f$p2K{!ChpyvkP4VxHoT(~KwpSOO{ zy>r@LxaJ7Q< z`^0OU5d6#f?ZH44K&Pi($jHjp^@M>}&i_np@*P@|bte9>P1PQEn6D=gRUmb66CMMs z=_Wr)l=TiXNPF}BPorO$xEaBK<}UC3n*IskZRNmt*{=W?LwJB*J9T10BtkW?Y}T-6 zB|}!vco2k;+A~uyw71RYtcN+R#AeHOzW(!w-P`h$(C}Y&Rqf328&rIm>-=4I(KpUu zuv3ZB>8Zb_)|HohZTzpT8HiCQ@U?vd@?f4^UqT%<#YcFiJf~W`bK16i9If-mOwXJ$ zx!!&mQ?4$|8@2lTy#LR8?%c%Syf7$m`|fNje_ED~Hs$)bAMJ%c&iz!v$JPt*{CbBd z0ZgYV?J{kUaBZACTv8{7Ca4mQK%CC zBnq|Ne;lvSa7(fSyV`HsBYP*Hid`ZoG@XaQpI+b_e#gEZ@!WBklAmN6;!`}h#x<~t z?+sYTGueD&TwHM{OeahOOd*b??h>LfDh`&wwly_A3F?%U!^4(`Hms1_MjG=K13pO%%J+( z>d1O-a@rC*fOmG9C-H|BCF2xV^23(umIcjZD$XzTK*(HNs|RdM_UW*qQIYk}#>HA*7fV2nHd6`^m8;Og#$URPg`G2}v)6 zc(wTIU#tNDw0%yujgMGeUA^hN$l*r`#;*#!v&?{`if01$v-!(qYkz<@mt^fdqG}zsjNL!m{k+7U?$2bPCui~Fe^WzJJ(NEnqoEDo zQW0fi=z@JaZnh}(t}IFU=9I>qJKfofLgVDu?Du|E%1l(VvAF?Jd9SwusUXtZD|Le* zo{4~(>TcLqj27(8z{vP?Dw@ar6=#4y{qyX^sK?GHXltXol1R+2NRObFP^rEk_}2hW}5NfGtJb5$us3*KBs?^ z*<6E)zs79`_fU;1Z#ksmGx`8uNXmIEcSo}d^O?SxQBG4+(~Hl}_KR<}I;A1(l>D~m zUfE5ZP5;d`U7ds>3$Gh%Vx#bII3z{@o2e{~ly~Z3S$>H^uE@)^02pa zCG_pLf2U|iQqaCH4Un_~Sc1W+CjSi3*|A-K+VM#aHYpN&bmE7?LViOXw!#Szi#?5T zp4#!1vmM(A&h`<5(PlIT*#uqP5zMI^hdF~rs?Eq+n~3UhlaK00xr{#6u=!WDedlp` zT`ldA#MoaD57uxuw=B?G#Ugrn)$KVGDg&1l2!`~v(2&{*t1Bs=jYuOALf9ht z@fcoDIO2^;w#$$ClY5DpYUmfo3Dp&z%hZef5cn)&S!Q7TZGdE7iVns4?~{d^vsA?% zX3h5^3ERhnW^V`gTY9W_ERH_ZXz#|kXa@Gcvi52$mS@4R58ZAPUND5cgMBv_xESxP4}(%(Fwp9^7Og;1P}}2|A4q9n-suDTVNnl zBXY`UWw}=$o+AQwoiL$0H3CJFXBdL>vm#Kd`+d``xrP`_E=VsdM#S&3;>FE`QM&vE zsTOq1oM@0O1q>*hQpc6ytw5Z;r{<%-&AS1YRE`qUM7iQ(7>xnYDyn<D}i0YItI zmyS1-Y6+9l^*o))$m~bL(39y(12X8A$H6>_{a05mIEBwW758?t_o%ah7&n+u)N`Me z>kW7SR^Xmw3Z`jUs(`wWEpP+UeD`5#Om49a=2MusY}^UsfeH*)*(!KteC>%V)Vv>Gjr zTB`jz-3V4o%qi$JyE`#7Uh%o>A~Y_)6Hzkp25;9d@5KfY z-yBzN=%WK`5)~3OunS+NqTYOmJTayen6e38wO*uRT)c8)k$Ji%PD(*|5u`UjH^|%S z-us*L-03}g82^4EF0f3 zx~^eb*d(>dqRbl8>OKvfdvTKMZdNHT?!z9tOhq(9fOn!KSd_ z^Rg4ImcMmph3owSIY0-2DTz_(~ zthJ_hd2BeLGEU%OwbECAAhj`Qj#HVJP?&>|HfN9ScXG=t$3&ql8a?J`=k;x>>6hSA z-TG8>30-Y@ev)i1<*g*xT#FkE&|-)9JStA!G~P2MD;C0%!U?`midm{)wn|Z`C*y#^ zlBh%_6RG7XWoXC-8B-+2JhP#hCjr|GDsgz^Wm7>V!h3jTzf{=2Dk&4H+nl6Q)0WQlr#Uy+E3VH z_h)%00|Af-Y5@#pWQ6dMJvXkDD%>aveTg3r(ejk)miuq3l61tH3(&3oO}9nQ8aR9T zZASXnTd$pAA21L42hI7Q3A0Qvu+ceI3aNNPFTaT_aEVx zcaMtqxB7NajtvZtl_;6Dog40tVl_n`uR7+_e*l|ZIjZ2@YYDa58hj897rE|~;OwlB z_5rH5Hn; zFuBIi=oIUb={QI^Ga~H=8LmtfHCnpp1isa#=(D1Dfq3w|KV}RQSjkj9nXIuFKdXM8 zLzf2bFSrP*ZMX{D$?jZyce7fbhKscdI{TezmbXRtx%eaE;Q z3hQMOv}KZTYKaJL>*a1U1KPWy>R)E7qU-EH*Sb5lPmZ5K}1 zsseRSpQvqL&Nf+*UD61d@~$bFE`KX}wfYw}JEB@G&hbTcnb#xK#|^(1ki`8;cyY`@kF zn=O{7&yQ95CPZuvZinIDA2h9mBQR!+{ff0uqLW^KPgqEppqv;}4KTo_%}}9{OmM(!t$FrY^Z4>a_2o__Yh( z6&105tCp*#geDZ-#zLOcump`?qiq-MCTtI>gTE+!bnUDc={V}R>)B0tN)2apQ8G|_ zd}$~gv{jAW{;S1vRjPOXTLhK`&taGxXY3g|RK&8e+I^YEZC437gH2Q6m=00#BHt}} z2aGtx(4v@Eqc!w1B(UDkQv8wT)AM(B)eo=sH<~mVggxCQgKmy zYW~>Ji|v^)m1t3XOwo<(_XyECuMb@bX&0=dXbmu?-<{JAm7PO(P^#XS7Mp)Xe8yWq zHs)X)jzkmS?O03xT0*uzim_?!t`4hK<+QJ&oBAR)e48C zUcv@B84=~n<>5mR&lfjNstOdt-!Z8RKXzE7cROeXi^&IfK}_k{v@-$H&PltO5PCN$ zv-(GG93k_y)U-eL$DtP}bRyX3D@;%2X#Gf~AH%XNgs#g6W7rMnHh!X{J?{@>d)Zg^ zu^pS57Q;RpwfQUJ=7O}=9xnA)42Ut?4hil1hGO#?YCTAn>qrOVyQtZ-;3i3E?(#%k zKGF}u;97^CHI4TIcuwZ_xuKOjkJlC#o^8L{=?^LS@rEUW5b5;#@7Ys6*25Pcsii~* z)~*opI%Gn3k8`c>u1$!q;(Ghv@JBhd3Vunk@Ey0WX-!)cHtmhUC2>`s-o;3>W502v z)_)-PYI2WdQuZu?y(!IVVfsU**8ht{mb2qUajG z%PT;qM$)qt0<3w4nLi7xHS&_3&0NnZ?I{t`LEox#_8q9TcmRnQkJ0bHqC7WT0#Cn# z1|~b3o_tUy73STpofn+;7Zc!8jUY+aPVfcHalMXHElMvY^OA?pB)S!_$7m-Zv+Ce{ zSb-a{7OZCr0QaW(JJlEF9nZ40^uz>5s@#Y(UCiSmA|eJ|p~Ve$Gp|W+0CBT9?w&%7 ztr%fc&Q%d_oa5`P*o<_4M|UnXy}>K*S88t+$SAkhp24`x&M-(~X3GEOqV)XKL)m`W zm@HHrdg|MI(L$Z4lMr>8R4?s8eDFp%bDM4&*2s8SaeVANF*fxd($+;j6hL+qo!dhA zP}s1)xE6*?`;qx|4!Y;q?BC_8BR1mWQuTT=b;N&&p3aY8RSt*qy-Vg?E{J+F0L#wIwo|sCxNz?6%cVDa(!W@Jp9RA zlR=z4ayu^NbW1UN-7v>oxANhw;$FYc6Cz45mKSi%2ReOYlg>_MSZF zi`~cwOQRQ-+nlZBBJ`_=oBx`P&!`)U+ncbQ)XVL)V);rZOtu|Ig$%9~RIac2{j9MR4K zuYNcgW+0beOOp>xywZU0ebKV#U2W(emxiq7@Xhm9eZOfa%NM{wYr8tVb!A?#4pR~R zX*Uf-Ys;%E+J8V(yti@NwLZ7_jy-6E7BYCCHa&2X`l0yan6TZELKRVK&c7Qa;GDZ7 z*T&m3I{PCq{O&_jBsD(!bj0zea|*F>y$O90f_z;H2WnjiP3(vN1>Ch{i1QE5u8~A8 z8TP}=Ztt!F&j9YUGYsm+JOi^|HD_j#T3TF``wWC7u%%`Lx4f06Z1%_NU)y7Z%U?WkkdhF;Nze7s|M0G;zq(7Il@r`e7*d84Ae(*y53= z*qvRC27acyIzd=WQw;iIf&#Ezg_@=zSWzKiSn}zg4g-i-^(lRO{bYib8x0gbs`mla z*)eEtpUbT??I);_}K z8ZNaD@$SCSj%+7ok;9R{tOgs?Y)D3Wi>vIkFR|~mZmO!{=28HybiOW<_ZFh(6S&0_ zu3?p<$49-M5n;N+h@%b%ayb2eV#7{@=bl1PLPPAcFOwy1UMVbf+?d|9`1e_TsAtDH zN_ZgbZ6_=Yg-bboL{76u)+*hj-8LCC2rg%VVin{607Iq-Cs$7dlgUnW>%Eil{S(}L z%D?zPYCpvHeH3=v%CD`Wx^rg@x#n{l0R;9>nZJ{g3LY3MS#LguBV+JF3S-E)1JnKK z=Y9c>XL_g-aq@?C0ll#-myrioUG5*I$uSiMdiEKgUNO}5@{%cYi&RYrB&+b~>+|x( z$)`+&&<1&qajcK$igEKJasbfP4PMzn{&+hvy7a6CT90~*hsF*Ir}~Z<73YFuRYX%P>h;Yh zNmIUzuZgjKtOUoZL-A> zGKk7+`dU>9eysFG#A-H~!!w6+`@%+3StHT>MYQyP9IFXI^8$YJfjMQX?JU9fC~67V z@o@7xV&rP7H6`G{PC^Hsp^Oz5(4HmA^|AxYEX`3T8KUBjr}i)6r{&^CMazF^T~}sM zj_zs+D$7@nxeUJXWa{Y)f>p4gWdw!jx$qwa4~n`B>I|!#59WdrE8{Uxw^=2<5l)aG zWyoTO#IsXfJ8bgv7f=)yhA{BY!#U=wk(mu%dU{OXxrGnywqz2zL{U1ef^wN+cZl3U zu)c9)rrq;jYzl1JUxlFylBe&_JAAqt|F8;p)7xsJcQ0C%Q)oR?u59a$lq$a>PNKpR zZ^0^dhdp!GkIA9Wnw%w#d>vS!vGfz*7l~Z_w!$+#A*-@+GF^J33$%;LyYw0RHrk7T*Q=G^A;h20z0ku=->@jsD!+3$Fr!Xw{^&x4( zbW$c0c@oILkBBo7)ZEF$tCBT@V`WgjYOzY?{9S1e#J+#ZOI!Xl_ z(_BxCCGbkn{lAMRqIWrJ>&S@!4Si)e+ipyYm|``i2PkJ6D5eSa>$iJYS-5X1q-Ell z9`r0eySR?=NZa!OZxR=>lEiu7IlcdBFKvr7)a_40zBE4tD}?8Bn)i+t?ci`=os5h7 zil_gwMxp?@AD;~Inj{3iqEo>Xv9A)%2Wk2!Ucj9j>gp1)EHs%ELWM-)QkY$6btO1d zB_?h98x|v"du?C{eh@qv!q*7w&G^Y=<~Gd-h$KAiQ!H(!vfMYBF4T>Pw>P>rKz zKFve~2x_gbDCd`d)Qo(RYwpEi9}_927qh6P2K}(v$oz1Rk5PsiP!ql&GzX=mR4PiH z2{oI|TSXq>PiJlMGEuJGWv!Sk4{?6rfeCj!y3pT_#lPqjZCQq4+leu>xIh5G#FldK z*z-BSUO?acfaR>5(0!XlTZ}$7_($!PF@ozR&$i_e)lNXCl*fISu4P|t)G}``er6O| zvH$oaOrrxk@cs=^S0^&mK^A=x?y9?^9-yGHrZ3N*XpU)A24gM?jTyFdgyac21lKW!^*bn;DEgP|Xq0G%c`=cF&Zn`+q&rXTxa3(Ed=)SaHN;9l z-|aB!ih{KN^=Z<^$1oFcx8jeeAwfvJF4!IR!i~B5JpqPrOGX+Y3lo~UT`0`x#86P2RU|%5!6?`TUUD| z(fnKAg)3gQaJa20!8Keyq}!$rA3KGWa1V-(ZO;9L0RN+NT#Eb)jREd1igM`OXT8i= zpmU(kE&8qEd7kA2#Nw!)el<)H$Xj4~HuDd(z+e5&*BSmdxVY`^{{)Z>QlMf$##Vrm z^zXMIo5PrMK$APjzXek-RS@NZa|;y?59-5z@@~VD&Fdiie)5}O z@+hBtV2W{c*$z%(M&Eyd&1swY-75eB@`;V%ESi=4>L)$EgJ>v*yTOfAA{(|*Ek_Tc>o8Jd4HaYu>(ZfHA2XrKv$FC+P0C8~)~`(CgV zxic!4dHLf_gr43T)pUio3aYhnNi|UN{Q^eIhcdTKloT#X-jid7!J;WLJXw-*Jh|Mi zrYS21w(lelBEO_ROG~b=_Q^Z12pS~3n%=>n4j_cD1;~7y{E-??uEHMyzXRXSk(Wv{92m%d&^ZZRu4CZ(C?mCNe&j_W#xX~&E#efs)Gzt;l>GGiviuq31$ z2^&^9kh35rG0YL9SC?m@PkAla+80O8=o1YURi9L-Jw6%|W{kU{N^!!KnQ-CAxnk3T zr!~`_=nb`aQ;i5R{+2(D(@yJ2=yf2l(swv*aO263!&%50gup8Sivr6K2c$lSlfFNm zlBHj+Rn7FkPz-NGKbzk!V^Oe5>)xJ$zHhCCrBE?6x&I|7W=K3?CX2(vKKAMzeB63& zV14Ue2j~cUtY@E>)Hvw30D`SJgzab8$BNZIE11S#1uD95g&?z>m>tqIsiqrE7Rhhw zGxp`A4%_Agd5zupZ7nCOc{bv-CtvLI{@;F8kPcwD8#|HW%_(W+Zt(lL==C|P&EDhL z{)3sr5R`4_-TxZ$JI?>X)c>*(xo!1(@A@15`4AR4{jB=^wK4fS6#1pp)RJB_xzxrH z*??DPR~0~<>>8u+pNARj+hY>11+pjmUh&4Ja>;TBP#&{mdW?6KqcLkoSb54!`)P0x zO8(Iz<3BU%W6QVowm-*XwsTm3rs8c8AxwLSt@>H~5Lv@NzvX_ZO^#9B!%BR}N8(UU z{H;lr7S$SM?l)2wJNNncIE9C9VN43yC*bN9KSH7V&9F+1Fl~)+eiNofA74m@%eT%r z`O}p(=^2iALpgm5L=quq*_POO#I-oUe2oeA%LynRmU)^`{b?eTu*uGJC*6ZM7^b;= z6DJPln@-_IsyBVvh78?vbe)nBMEdg?K`U^?)Q%4X=t9&i^eOBHK1?wmRp?UF(Gv>Q z;3~aAvb_q0Kxu9@Skq!D&o6DHT0EI&Mn$`lBK@{a#zMe@2Ic%a1{c zK(k0E)GQ_68T4usWxhs~x0PAqQ|eotJhc$>dV?6ZyMWO#ErLiQN)cD|y&#AdRqKF$ zHAV}z{1n?A)4!8U33rw!W_wGJOQF>rBuRM}RdhmXwA^;6)Htz~6a&wJg5Sd)iyG+C z^_q&5hOG*GsX57){?`2Wb{V^PuvYOg_p-4V<;;V;Ujb;fJ;dYSsQISfV zUNd->dh=zYi_iS4)8#&@7lbNc=B!?Y+PKbP!{JPBGKT20SM|zUPh;{HfNaZ-Br$BE zK%7C#8N5%Gmw18iEA>7?@QNa6i+39O_KZmfrolIyj5|H_X3iin)e z&^Lr?M}2026om8y@HQ8WQk0`L4Onx=BXpuyi#h@UTsrNuPfLpSiDUm@-VJ+Xp{(xl zP2kOm*`3XhUj>?AF?vQyD1ts|kz(+t;O&QQmr!K?^5#6Xin=J~tLd8oj<^>usNSgf zG<1dFVmA@z#^cDjYl+n-JPl)s!Sg+QL)?_9<@a69hc=x;ne63fV)n$&o`UCMW{7NZ z2HmcxDLYf9iqUm`U@IIh#}qy7=3%A9nQBl+U~K}vJgXsw@2}Jp+DL{5rtO#eggSxf zIg$t)rxCW7m@lR+sgN+FQ!MD?tKz1yd{qQxhm!HdIuz0zKmJNr%%r@~Nk@yTwcnMU zfbOK*Vj^qt$x$lzMO4Y&kVGwuQo;|3q-uO8hL;i77=sC~r1;70D_9s4L83tjDeDC^ zc_1JIt4L+)gbXbB3cbRg=~d(6q(yBG>(NkS2<{5TrVg?1D2r4wg@W{U>YKlu))^-& z;2h{F+IjdPiq55pf~NRW{_xX7A>WTe%#5fGDz2WgZa674cBZG1*JYY)=Va`qn9+B# zS4JNr{ihc`xFu!SyZ#BalbQDU*5+l)=NhXM+M`2Wono*2m6`LiepTDNuJ< zDOHwfP}5lDz4%dU8;!T*7Jv6X|NjBPYV^e=baSk^J~U}hq#CUEG!y%VL-1>+m~hJ2 z$BF%ze}t7%Uw3w{I-FJ{Y7G9Dft`y*5G|j>86EQ~|GOn|VL8lQdcu|Atb1TQ^H+kM z_+(}1j7a^y=vySQ97*yYsL*!P1}#Q=&+-@`e~FM%QbMQTw+;Wy?E!jS5ovo`MBfLn zVEWiObEw1>CcJYD2~y*f?|JM1oeI60p{9|~*1MhjJpLO?C`^5_SLJ=^@^c!YhGp~2 z8)Af0Vn7rrTezG$agld=eWZ9KmF|SrzVj;h4)Y7f22LbI{?rQ9*%u9v7EIeda~~lG zcuvb~D=#EzemNV3Z!e#&vCp#(oC{A6b}(U|!U~oB53*UpX@NC_M=rv{ccQM(I8vXt zWSrHRA{QeH$#62CYP`PGM3jWa%h8@~oMuMa@(`OyP`S;#uHc`uY2TKqqGE(dm4GSk zlyK;~Hc&DF!BCpj;&P3an*5$o%^WtU?H)l#&&|5Wv@EB-Q93pMxH zY($|`oK#0qDNdwH)`3Fq3m;Dhg(2PZ-8TfO<#byCcfx_9g~z>BqRF(Akv-r>=okK= zxtvTuxrE1^e36_1(kiXPcNm&Ry7_o6-TC-%GPZ-~xIl|-_T0M3YA>H&N{c^RiprLA zo=ntr6Gq&so?`oWe`6{*xYpv_7;nmEiowia1{P-@hUd%V**&mnN#*uoy5zwxW;%yDZ=)F)Ln)c zD&=Ag8MkJ)z^RhfnpZrqFR zW5v2BSt?Eh*Bb>$p3Y0%>vtQCf zp8)+xW^V#YE??OF+K}q{|{rBD;`%(MFF9UC&tH|w{zo!r&#*j7vRFkKr;lD@(*Jkp>i|hRw z{(%}^cp!gEdi7Jl^lDIha43ABNAssQmMlVTxI(R%H$Ec?`k99%a=f$6@!Ig^UVp{F z3&<~f&C$1k(sSRRh(JuSB+^OT$14O#$*rD(8)LbY9qMglDz9=8iAASEFHD&A+FNYH%3>Ol*>Sqf=yyq z1AA8B?N1w4!(P@V>l<&=c6>MR&lg&zc|P$B4p07n0c2ffaFIs{j2Rc3T+eI3B zl!kKA>NCIMM^i^C47n7?Jqfg-Wh`Pivews#B;VVnLIh&xljlimeyf0Qw-vw{cf2cM{+3w35CZNmb z)w6C4mg1{#Ta<)5MnNVKX8F({k76K9h@CTg#WS=oEmZR~30J7GAJ!f^svkwI1kqQuiXBSv2Ng!f{tz<9SSb(A$2Nr{bp^ zU%NeKJWLJFp-;T#Wn)uzU7fFQF=f0JlwGd3A=;g(;#PH80?IHq_rvOo|5yAEkg;E6 zR~u6tu!$_KIracUHOX2$m?Ab6kz07=3rsKO?pY&=LE$JTKh7?muDB zTp|Kzt<6>cHi;c$*2DI?c4F`l-!>J$m8kg}M`qEIwy z`FUxV&#dXmNwL6HiH@tUtXQpE=6!sk;^L-tbs--UCw0g2^6_9Un#7ks$1bW-Zwm3R zH%?Mtmi~B@q91optI)t(X0larB!2QS5Bz*dN4X3 z(0Tbj_>MBIzgi#Tc*Gc+raQft>(o{5nZ|d|qF_bBq9M=zKj1fDCKg6E2-Px6MuD6& zJ=L%#S(G|qG)tq&RIRaErvg(wS4wC*TZnGZ7`4P?l=^pW(!k|I-DkTY>oteg`vHWL z17ZGw9kWD#La7Nb*?6{-Q*z?U%3Hfv7)fMxN*URNQ4VAb)fKrK@&nK2-NpFN}fdWm^~+ z`N_#`KJRGHVI8Qnn9dFnQ^md{T)A=L)aTK@-D zWgc}rgsnC1$7?p&Yg)=vx6!glxe+BY)dA||tJSQ2+HYgUR>}2F^oC8Z2u#y+M3e3z-}EHC@0NCHX?&?kAT-{8ghH1k5IzsT6ne|5uW)vGOOrUoK$ae+eH zMI4;_pPw~Z)xYIzi~C&`xNZ&!*5|z)_?PDSpZ&&x%s;2C&H~2SIXm%~K;2P~SU`;s zn~Fb@>&!*y3ON40GbW0K0s(=GT!#CB93Wwe5DX$_ljGIy9|TVSaYnAFASgr}a4i6o z5qa7Ar?}9Vx)DtNNB{e47xf`x4qY{zmKOT;HJ6h7xoG!9F(r=^sUl-NFqMA%_4`yMOx7smD+KjU|EG(7R?5{#pq64l z=TqYb4X>!!$q!+0e?NmT4=x*oOhaQT>0<`hsJ)G|ylonTHvK6=v$ z#dWo6U7NqQw=X9I1MLs&^sHK~UWJKVd_JBdwW96QN#-Ng?7B3*OJsoXR3=5bMT$Rw zARl)xF`v{)%5((9{-%Hu`cM{<#Jn{m(L!^3E>IW0ix=HnD$y{wG0J0?xGO2e;njax z5sk>*W=b5=c*u&pV_Og8&IY|0m;ILMrhK2harN6O&u1$^#8OoE-<|PaoABeq-}kW+ zD90#Z@w(WZt^r#6q`lU9qP=hblx<$5FsVM8GU>sgY*`Zp9{m81e|03uc!>@WCBa>Q zoQKD9OHLH1y3BjxKmp&fKrp+5v zKgQ+2s!D07g0Da_xx=hJMy<9Z;(crP=G>|C!|_SOXVOlD->}O}l+u`dT3~S=Rg%$< zxJy<$a4BKwrGJ>ApHOx=KD$zc8!H<^<`BvKw!g9b;Skd>O@3yGhcTIaEYU>%g~muN z{V?kCSXyJ!uRzI^+JFmFJpcXM_gIeTnMusfD|bY$OgRy!mr=uE>%V3T{QiRJfqK0P>4^XJx`_bYdBJ==DQRfq6d`etE zaCPRU&*1EuYhG2j77!fEUVTnYm&px{!Yhy!luow=Q8!WxeH}?MkAFfUF2`#3fH<30 z!se7dCR=|W#QE^_z3JM;3G{ViIhLy`J__#1lr6r7aNc1&B48_q{nd3h;H&^tE~6Ng zwJFxNIq%};(Y?)mA=P{Zf%ief%rVe@qlzRzS##s%!zfRwJg=Nmv3~!L z5b6IcmBJVy3o4Blf2Pm?FAynKE>LJn%&I0Ok^+S0uzaX6?xew?_>g2r%EI>{=v(l0 zN5Gxw{q>13hNviMtW+cYF)+Pj-B8ic$A(ODF?}1fUsb+9C zkO+V|eb2vhFcqR&*wtGsHL4iGCkmL}LS#sSYb#&=qEfEZq%7p72@$Cy`e<8odQgC= z6C%+>iN1vHO%`k`ia9{Ux-d1of@C@_<3WMdz7_zxwX5)0W9-=%8?J9AN*TkL|YH4WnFP?w8g{4v&nySn^{0kqmm$RG=$7?8K)eC=4NTF zKbSl206-`Nm1o7Z+TtO$V3D_x;Y1m#cHv+7CbY%nM=A2=b9(mQtK)cHXAaCg-Yk#4k1c-AKJcX26Y+ z{~b?!U^I6=c$DQNK1p$V?g`k>;J?gdEa`dl2N)l#;O;kOsaB1$PI+yM#5QsETtF{luX-uF~Guoe)|NnxPf#P z*-9qp9a}yc>|E7BXd6aNoMVKu-2g0un#Vxs1?GOkt zxwyDYXHtd74?Gdfuig<1M5DVOEmH%L#65>gt!`__`yc*imUR^Wat{V zgj+5WjhT5jeU!2#ghpJ-Ozgl?)lgJq^Ci)>)(HkhQe(tW+kH9I)@X?B#qp8y4x)Zv zW%85|Mt`N5Ld2upSuYR%t{M!}l*;?2hI5O-238SFF?wVQ#zHHc@4$H#nl#i*1nr&X zF`tP%VUDVJoSMj#DTGUc3`H;kyfKK=UCJd@nxahHQ7!eM?}q{0s#6x@3zXOuv)e37jM*asNNIzB(Yv zt@~OLr3_k9S{g*8ySuv)M7kRUq`LF(}Elok*Kh6V{K0U0{yJEL#^zW?sM%AI*Q z&)IwJwb$OOR9k=t%#30H#oHE+bu=u3;f3b4#8xkrf48ji$rZY%P`0yAlm*@$ee^&t ziXk%A5amd^3ws;C|8gh0(s9VmZbbSKD`!!3q7gA);; z!m$;lp0N{}$)x1C!yiiYSZ+&GWUB9CeM9l%a*I*_!a-_N&QfwY$4kgvtDhX0Lgru+ z?U@n(m=4(XQryLA?Op%c(kjM5`q0NoKwVmG>-&7R2NR0#{dX9m93GmJ&O=QA5nQ~t zrzn2O{G~Cm@k6fFu+{^>P@vP%4+-h>-i6paoLUz$;VOCB`S=`; zC1Xm!{EYVfI|D<9cuU8f-Di?4L*mmlS3T8A2@#6d2a5X z4Jto=R(Btrob(dgx8F^$Qg1;JIofq^qe)OId{6i0LswQ-{G9aNbRxUEi0-=m9LpobHy@Qg zN)A+J)NA~eHNcUH@|Tx>kNVsB^YC|^suL@eC+=Jiz0F=g+~3npsFxIqnv|42A?n4C z%qIv3n&RT0Rp_tr;<&gTENS6T5xoKM9={oZil@QV_xC znxqhA5UkZ5m^#cKJG==ewBFVPv zTk<41o}s^DsQzU{V+3i?SKMCx1{5}jfqCUS{4Usi{JI!)s`E4EnM)58(iESN5vJuD z&@jHEJx+)Rf(K;TSJ=eHMn+`s1m9s^E@?ga&B(y{;h!h+Z}>PHF5DPr%WoVqdbSIF za@Iek#gf@J5>3es>1a#|`(V&cf` zp8GE>g#FI*4HCAHI^bGj^%>fx1L-|URnG!uIQt}p8+_Gl9-bUd{jQpO5cu)IYQ^g+ zBGIjNFAIj69{0O~7yjsbBW^4tAz%o0B}2)EPNh>oJqbnLjTnvDU8xwzcGb)p!RF(J z&}>&;Tr}o`cVJ>E)v@AQdVpbS0|U%^9Oo?i-|GSrci{#?debQs-+PMX4_5QCcaC4m zw&uD@rvj=tfsi(c7)T(2ozQ(NL3FyOrmQle@jLzx9eWc~Q+Y2zdF~AIidZVhDpt@mwmODdQcmZ*IVN0QDSHdW-2mlfwMt{OvmP z{GGL>t=P`Rc~On6xI|VlKFz8!K3S}? z#=?Vl`T`V|F?VPQd2;wH6VTrNtFQ!ona24eqF2q<#1vqFwyH!G107wo#-z8~=9Rm< z`=BN`T00c=QZvmW9^XcWhF@Kkd#HbUf7Hi>I^T+v$j-eP+9xK-B8KYK(x`DBnduq| z@OQXI%n-gbQedJ1t8lXB?i?huW*e-{edutZasVB!8T^&6oMH?btz(!1mV@6I`@^Dh zF*WWFi*fv7O^zeiNydGo?md91)#isyDWhiDx6p81kkh0P%3w(5b>*_sqf(60g{-<7 z?Zs?Ur&8s5tB?B1=M6n(E1~g{{$3^kjQ8%#-ecMC6P$>RdBLLGc>#WyJxH&mQThIm zm{cfGJG`H%lRU&HX5mcYZbtp+>H1Z1tt%ngzxgPE3PK0BjEk)d@nvFow8>99lkH}IWGG6#s^t^jP8zxvoVmyLbC*n>GvlADAO5vmOqi1@ zZtqRn+WG8ZyVr?Tsz-@x0lIGE^;)D7r|BDad#nLRR0O3ozzJG!jzxWs5tLSQBo=w9 zdYLBk*HrdRqevv?*;>_Nu%3>CLqpxFU21&igM`K)SVcu+9zE1hF3nMC`ZrrIpd(xG z$X%W6pEXGt_wlrIUkhyg&JfNtE{^N-`{e{j8l=I2g(W9!o{536m834FcV+Hh%ES!& ziIrU9&ZzUE&}-ugl{MOiIDx*vKP*s~#i*P4se0LKQR(fG#D||GBgMKX?}Rhx3V8j* zbCTECwmFs4lhWnRoX2p57uAyLvQU}`0kAu@$C&5qyxy3x%sRv~$m{ILakX&@dmvKu zTm-Mu#^#wT1Ew}fh<1 z4l(mO&`(s8N_xgV5iyytaa{VOcL-}p!G`5zx}?^C;qFO7)B6CtY_RjB!+~xE@4b=l z2PCd8awa8)&?i|Il(Hgkx3?#ei+5u6X~vENQlcf+1p+HG854ZM3GIGcpA!gPZ3!F` zA5T;?yZqw($5hb2gDjlDpG2(jW-Wdm4_p;Cr1XJb69pK+S@V3OP z(P!OTf9Wr7Yt8^8d(P*?Vk&LLtg>3x@BPo9_^t$XUn-{u8@^e0y|>~dpi@QRu2ICb zHQDxXB7Haaus1C*tGCQH;%-XM`+5%*-|+_t;wgatcX-o6vCWbdY{fTGJb~Y7C(Q#l zc|K)`_~{VHe}SwyB$Z~H3eU07nL)F?xAbED&6060iUb8QEK{N<3wKo-(sO|JCC{+{j9hFcWMY9UUjO z-+_HF+wZ`{5Emyf#sh7!9Kh~DN1W?6kKr?XPkz%4RX!3*j8hiH(D2e~#l2gC#K4eE zr+sJG{(!VCtK%#t%lC8%k1>}H9W|Gn6gO_k+g|2p*S`CQ>MzLmsM8djMqPVX-ntD5 zupf+OZu*R}AqA)2*?%8O=Bu5e?;%eweE&cY8Zq zYgQR#`ZkwnwBGAvrvyw$F{tDQO-xMGzyAE-@L9U8*$`w( z(s%Xsrq;?s*eTj33H-7&x98Q9w+Sn*L5;2$H0c(LcBrS;;~p20zE2jMdx!7$JX_TY zz7$CvfSu$&iUd2^agM|0TSs|z+SkaEgsxCL3mzxLI)nz=&24@<)GU6A`tDF-W9(9q zK?{oJdaS8dGPHkC-fum_=uTy?Hp*J_$;D=UONBs+|%(!+`T*=oIBm$YGQ+xn9( zp)CRT&KMk~*!C;ysZt#=fwJ=NgZ*TTNtM{g7Gm6QI*$F;JYw2dx7ehZ z(kS5vDify)bINsX3|!mAq0hXewG3R7u7$di+y~fdoMvZz3GPxEBaZ3?68L?Cm4cp> zjbH_qJfL}-cE3x)p4Y7$AS@Os1eo1sB#jS{!7zUz*OG@=QKnaE&P&7(5%?2Od@EL= z`eApqQ&wg2CN^NPJ36`Mm4CLYQIr8_yEHjY0R(N_g9E zRRa$fQ-j%pfJTUb_|cH7y45>J+AvQ)-wIe#Sy|4{oiT=fYlFediS<~GS)&}?f{9l( z+Io0;r=bkb?WPJpE<&131AnGiSH1Ge$73zZf4MPfaXYV@xuEOJNLEqrX{x{6)VSnQ zQTa{RFWKtgviyS4yu_{Km90UWyyGB;`kU{nk8Z5fVytv!dJ~39wdbCXFwZGln@6i< zd}%0E>ny&d8`mk*VoAwLX7HQmO=2|?iEfI4=Dw^raerq(yV6TbQ#O0LtWoNrG9qu0 zi|1Ta=GL1a%b=yRo5DQVXl2`AHR6?ID5PdPT={(`0cAJVB(&}O{@9hoBUSS<-J}Rz z5+gOE=!_D{4FNpM1+{1nsVTRw<54fAEcPy=Bht83A3e6rm%U{eDSa2E*tY)GU@|B7 zfm>>_Yjph=*xuY57ugt*lBJRY;QGi*wiic}dsJk?cE8k{Q_t&NbUT9HB}RXtuJ6 zO|G`eqw&~bVchid>lc0&VCWa?oy*ly4Nk)>R=S)Dy~gG} zEqRubVdCVk6m&E3o0rzhtWtAZSK^{C zJ#}(@^}XY8si)2~t-WSlCNwH#U0R~LiNl>-a#60ecD9aRpAe$D05Pv2!NriZ}VCc6SK z3aPnhEmJin{S8tukJsdyRQc;e`Lq5rjkV0DGE;i@Q)Z9HHk59u=}5z^?X}L=4C5V$ zRt?K$HOsWBG)IUyT#jZtT&&0DV2`i5viXaG{|1tOMwJ0EB$cQ02wReqy?L&iIIvZ( z0AY!9ob6PH#thzv8AgAQo>~$NA$k(cjepLUz<-APTdoAg8Gm4)5SIy9__$3&MkT>< zt59%9{N>wAu*7LjK+Sfxwzp-9Zngt7O9h4mC3&Z{mwHga=PF~OUH|_4r1f;p+7;NA z@qLQUs)vxu5}vhA$ZD$S-t2#}wtmXs+G`|RHmytV+)r0wB34j&t7qoU0&P+9EgM$k zmUo>ynK*8dQ?HY~UodhjBN34@$u`w!Rl;DAzN~J^Xio3A;gaFt(EheaeLDZX^;GLu zuL{;#VVRO$4w}xI=_(x>zpLjon&o+>*I5l}Xha|9H0PmAPDq+v42%$MN*VhK>yr8k z;(E(j8;%k+iH4hAUC76B-QK31l>2<@9X7e8NvzvmADI>C*FI*1{8I~1VBdFBa*DF2 zo)yx|;pwdrt*18^t`?!BoSL7zqvx80zsHn!zy(e0a`Rt}#2wF(DVu6O)>Kfe$eWGS zERh_Kwo>NM@fAg5K4i4GeL!z_LrJP)S1^>qT!CfC$Gj9*>Le3h+P01{RjMkYmr!cG zv%_t<_A**OuJpTiy-MU>m!+<%9`o`C2K#*_SY#5khNEeBY=X3biALto&!fY!o@KccZ1NW8t%N$N6Y z&@@~IZ6G)7bdyJ$V7&%uDNsVREzp1uip>=SCRHxyD*%;~KjZyQd7q;cMf=aEq7&bJPHD5;x!5w-^Si54he5^A5V{+XLZ`xgs{jrCYQ!tS zpek>8V<(cuaUpBtU6|?h>$QQ`MuVLr{ERDq$~oW+Uk2j7UNO;bw61jVrDYMKugCC_ zn)xI7)HdCPFZl!8c{;xn63@^~26&{1Vm3_C_tS~$_uB1=1fhIe;`Nhe16eu>KbRD3 z#{~kvN>dGf@Tt+NAw_8WUWvRblvcs4bm++EA}=d3A}QXgAra~%?7**~H?Z9NH7U|X zV5)VI+;23@m~0!3%e{W`CrJ}oF0I~>TgmFD2@!?#`LV2{#NEV@NLGiB9H=qy8|Oel zTaG6wvXCTBmXy+_WUbE1>w|-okEijCQPlwZBb1Fgan>I6Kr>r{#YTL1a=j~ir zw!ktK;oRcPFj=a>I7~{+WtR@LTLqL7ZX9@HH)7Sye-DqtztMKS^vz&GkIbQc4H)8( zzt15nlFXnng!WdWRNZlx3V`8qJVs4!k!YBX&WY+;$6t(jE&4v<5u_>l?K#d;GIZxy zfGd0eKa1RNiNMo#EFM29vCAX`rw>~eG7Y?~bJ@rE)pXglZYRg6g*0^Al$UDGq9;dIq;MDfvQO@z4Va%g>Y-)<0!1{8FY_ zA<&z*C2u4y5(OddzsF&44=q>(x5_Zz@hf^WJ@Q5x+!O}RIIXh0>qC3w0NvCbSWJiB z?RxA+&7KwWI{dh~-Z`>;pG#)ge8@;p0e9SGm-|}OQ$aCK=ogFSXOohh9MuBJ?c8Me z9_a*Z`;&KBlUw_O<0vW3)B#IXZ^^`jC9KIFd~&?gFw}|+;*t(4xsH~*S5xU`?pg6* zhzy?m%D!H`HbK2Eqg_%U(7D>J6H~YY|EAzM?Jq&r;lc;~MtRZgg|rW<4rYFd4-{B) zmN|CqzFNs&B6>F>qiFm$UH|(&dn%BRec^okZhau07Z{>6yd5Fx?d{#(Z#$S~Zdx$e zpgy`Rt=`*2^9OF@yrH7NJ|G=4LnDfmZ~DXv7-}qw`(>Bs`oV8?Yu|)l9&hJ?G1ZMh zwpqz&1bPh)zvDCRkT&vxjq0Y_d@M7gq`1*-PnWj%2lwYk{Z}8}e__XZ%qiXg@ax+h z)iQ%Nx7R&Z{Z`0P!gr~<31nIpLC100=V|-z9s@IE(;wW>uSr9DPDu882MZ>E|M%L= zaCaG^FKQMz_VXF%8E-#*DwpJ@k^12>mCk)DnnxvAS`tIEO>F4T&rMO1whDW}l z+_<6?`*n6uxq`=Y9Th5Ly!{{D*O^sIhCc182_*@6RZlv-Z*Sb>r5J^8QI@{z;P>Epsk0`{JiP}YZ|sLBUeUWCmRDJ zXFfFAbccYYcSj0P`!7AHxX+OmP@4JaQBIed)lJCul7Hzd- zmIAPAu6o?u^JOMx3XvJ&h>^EKL#XyZTN$4v8rae1W7t4$B-G3FTWS*$627owR&pkc zG^b)M);g60S2ixpnClfGAPKoH@;R^xvc6twbuBBim;esnn{%XKl=uoLTWyG6z6tw4 zusL=s@=-vv)IF&Efz*Sx_TTOx(#AtCh)Tqrloel=ydxJ3E_kA2cS~@wW|#!sZ(l@O z_?hHi?4r!>8EO<4)1AekKUY18=C1gGL1Qqf;aC>poJBRNfv+Wg`{gXZt25F4tAU8+ zpuvw=4|!caEyK;S0`!wWt=VaFt@|d>O6-r|uLUL~BoPlwo(l>56lbei zysxVNu6m)GFJCr#!?pPhu%@ZiylrZd(=e{p3v;z=&n?KG^4m?BGIN>hg_anseZ1T% z(L+?RWht?HDgO!NgefTFXibQ3W>2Tb)(3#fY{i#0c#C8%2Okp&SjBZPUPy;)T-KW5 zp*j|2`L@b!!8U~IxL3od)B&R3gm|1Sx?`YH@I6G7n!-z1-L4_oA!|{~@j_uHkay|q z<}uDYYrff1-@umm1?zT{8s^%Rxk81sS74P|+QWGFqRB=Z@PjPa-8okyz%RC}s&fKt zl$z@F@pj#w=v81cy%u*dogm27JI5g-%Zl`_hnM}lAYrRkJK?^7h~zI5SH3XWVfFE2|xGuwGH z(zSmEai-hS&m*PinIycjZe|?WBC$;(F0vmg>Yptq3vh~=(nYRXL(kJPXyJwN8vHPX znfo#`XA&G+L}HdL_!j~WTM(f8w?h6!jPa?9s zEKK07^rj_MtxEvHW>VOFrG))6*t@sUrS3upB;WlT<0Fcu&&vh(P*GZKN zw7|WxKbJ}g4dFseWwZ$;VWBzex!9&^aTaxU0V2c1Ch6L zeuka_8COyTR+utGgGui*n)WgF=es{;`6YQ9IGIy~jyX#dLAn}~7kUF_v|SYS?06QJ z!0@1Ijr{7yl|Cn^y;VwjN_QyhymGyWr$2RfUNbj+H~1&~T$=oHYPZ!p!`U~X&~Mu- z%hQ$b!1rcXFMXE~6pyiV$##h9}#-Wq-|C?}VcrA@h2 z*2tJ%&23-bzPA75RK{1C!D$@XvtNj)dBwx^*x6q!cFA4_FMpEsF4uyC!2Zk{wnI!a zIMbqMEIdoG!}eE_XM#fpUMDf%&M8=Tj)(13oeD~c*k4=uHb>cz{mE+o#KY)!{fi@Q zUlbst81?iZOb+&2HlJnecztz<@G-YfqTqvFM=!|qQcHej~#?#hJ8}3=m86eFl(zEZ- zsM#?b-qB|(3Ggv&bt!qwkU{Qu;o|AaD`!a$GL+I>O)pRPb?@4}6oP=soFeJop$BCP zEgW>=+JzzQqgsXbeQ3_kAnDAgp$o8^HEHKRb!IShMHx+B<(MroC+5_ z%L^CinnH9;B9woNYSZMC_Jx{YqE*xKp1K(I)vX;p%WHR;0U5FZqgNDLMbkUS-5rwG zCmq+zhG<{voJ8!Eo3iG)WGwkM8b}DZcfz)f9#AgSxXW$qJQ%*ctgYYT@?5k}PO!c| ztZ=J+8&$PV>K(f)u$g>yfd>6Pg67pc)k7eleY=VVyL^galX$ z_Tt>XR+?#zp~-5k_T}YM6;On0)UEsZRRI(vf_WXtHEc5bdGv zD7DC-A;hHZhQ#k158ZZxri|Pd*pU>ZIT+B(XBY|Gz+A_VNp%0!p*m<_CwTkfg#AKI zC=t4wvv{2>pU~x?uN;v8OF~U7U)xPPSu{#>8SBT%*d=)l{c{Wc4R^jE`SWAV_uN~q zHH#eO*(C7+h3Y;{f>yH|!X(WlMUP3i*5?J%IAyoI@!PxXj30ajxfx#e5t^s}NFV_e z%5DELkV9_j=h$~%6v~}X)Q$YTW^rgtMxFeN1<5s8x?l=2!&tJb9zfTjBYxW4AIhj# zRa1a*Ra1Ecz3G(967;FJ7^jRT6Y@1$wi+fz)L16T-C+FSN1sBbP%1>h?O>%t#85zVO?$Pm>V7$X68>)3VkT5P6)SGi8t7=dfPS4 zr60vUN=}2-?6Plq)oT=YdBgj==>4Y8{Li&2G=B5;A8&r^rqC{}Z4c?czk9ynX}O5x z6F*@H)J5%d2Cc78UT!8IwJz7r7o>E2x0WOPM}7JaB8r42&K!8kQHZ_rguNL_=TSQV zKGV-oT(@#Ud28-8r3GWr$s@PA?iQxmcj`P5IF74q-C=;9(VEuV&TBTWfW^~w{aGJm z-htww^}%s7XQualOHa@!0?Ya%$-jRI+UfP*8YC0ATryP(qhuZ&qvobl`Ml71%!Opu zO-X9?qsy^jL;ZMf9txam!NgIzZb3yq0taKdB86E%f*+FFfR4B_5Ig954p@FK1?}lqjUsBt{&uAWKz3Z+W6dsm` zLD^sPpVC%Y9W5fw^%cOnIyIiH66TWod)vZjCq79HVhq3DpTTZEC~bg1-=+-PjiQ2> z9`8P*(+2Ioa$JwT$x?61YsbGA=(^gOcKn(gQ`3&R%J*#LB>Z3)zRS0{?a6VSfL8h{9AhS?r;!8IB+WBN&g5zbnmK zvLgjY+%ftogZrHpmEtn^so{D5q5IaiPc74!3zUI|wqUdObjR6JS(fjm43IH0sQ>z9 z4AS)g99c4&h(`Z_a9C&+AHzHUhq#<9x>w)Iox*qObYFr6SlJ`2;tRT*60z4_TnU1` zH46(-74v`Unl+epMBsya}4QW1}kUIfLEg~UR z!HRkq{^L_NboR&7p{$#{S0^~0%n!J^biKabs#u!dmNg@G1Ee9dR_T%ZP1dP~!(hCI zM5r=fE+t@HEgV}l)8+Q$f+JVGJoWstucj?o4ttD%N2@IxL~ZqRi){Q*ThMMM~3@4OqLW z{=Q6}f^~JMmCorKwqXQ7@!=GHt5>Q?Z$%EcC~b?b?Qhpk$AO4a{(27&QMzKMzW#kl zb|8bddw@@dg$_s%S-NLou5GfqR;wtmB0ntdO9rk9u*{|~E5xseHS}HXto?HK%l>5z z0v0Jg$^YnHfkqT9C{yOYgyWn51~@Wx=U961T#S%k9AyWtN8sNh{$A%x^Bm@^LkqKE z`#D*tQInlP60@Er&U1NbIB}h=67M#(R!EYeeU{Iu&g-7Hf9L^0%J*wbs`8O8%fNpqJ+6(Ve#h=yOwqUDy1j7s-h;596EMqjG)`D!04T5QYV2$m z&n-ha1%)ZFZL|_hQ?$xF{2cN~4ba#V!(xpV&LHF-3ELwMRE>lv+Ia8pPlTf? zB&7|NeP`3QyC#Owv&rxb$%^pE*lJ@pcU`~oY{t83&+;&y^wB)0jZEsd}K_Mmq_ zr~R=9qvqSQPSsAVGg6<4nWE_7^luv~M2W6J`wbPl0kA|G#`}sqCQ~*SYs+y7LnsA| zk;p8Le!nm#CQ4vlOJt}Kl~N1eCq%+ruC8Ox`x|T6OK9Vfyob^OPWGW@eTXd9+DnLMU`Bk!MtPa8-HDOhfS5&ZgFMX#CEBp(vRi1DI^Y=PLwvN=vJKWHz4{DS4VsWwz@Q3cIFoa zrLAN9ephcV)Iz0uhoz6H=Id2&dmLe_s|ES!5oNQ}Cxj<}B;?vK?VaKib!?bxNf>bC zn@iHn%DtZEKk!_8_{IFwk$GoqaB=&L>S^UZ7S?J66H=b{&+pFM#( zS5;+Ixf(5%4ecqZhz_ekRt}WoOUM{I^&f%O=KYOlSU)k5{k)Kdbyw6GRbM%@M0+X9 z&Qnya;Z+t9`5(zcJb!t3ep|Es?sOp}M32YJd@x|Z!iBHTOS9$uG>fjP@Ac@XpCrD= zwQ}+aL({0T8Z4OvVfM$-Vit^TZ7Dlzn{Ly&k4i4hR6a6hEQ56uZR-D3+(Zdy;)IpX zNVZ_TEy7uBbBBW9&2m|%1M*yTT6azVv)#^Dowt{}iZ0#&U7d9&rl*%DGwbaDxq(B~ z8ZONzxr5Ju5}0dY_rl(POT=G1IBxurJtntbSoG=`kG zou9plrWqKl-=X+>D3!}3JFbRzw)z^caW<`=G{ug zYs4?mXpnx4MkdCt^e*Gba{s6oH%Y(J(>0+#a~E{Rx}3ng>u!_WPC2*jt%xf>+w%P; znHG;;Ra_#1x_KE*RwbUBo}#3Dbv3dW1Q}WB*?evxiQXY~P0?`GQH{(GtvccJPzJUZ ztKW&3RbLvMAzSuZsP ztbY#5%`&js7On2MBxq62W{~*pdKHXZOO~L~Ol3pO5H+VOn9_h-M@B5PcG-0N*R_@; zPY_n`UmBLAX)#v(IFvn!pxZc5k=1CQSdCz4u#)4N zjgh`t^C_U#(6l;f{N&wat2&e`Bg?ERPp`*G*Pu#OaDFIj&?R0%Um`o(tXi_MLr=q@ zP*)hEH4^k_AFdWjgA$-CEBo8~LcRcYu-^I!Zm-{9ouvj@?WgDH{NT1flLHwkOKLR@ zb$d=t&!J#P=(ovPQ20x^G$J3v!;+p}R&KH6l~J1!g3tI_CM_D^tjPNsD_*5OFH(%t zx#!Jq`Y;bd+E^(6E#pNel4D#~;f25Shdw7&lHG_%_fZdR5O9Z2p8kJ`{!$a-1~?Wr zpn6EZrqAP{-hXP~5x=&0vYA;1c7Y3{kh&;m?swTP0Qh_@vjV`|X#FwOeyDVcE)X8- z!;%lfIt6o3r2?3K74&^|PiX!DPh20Vs_{H}3l-WQVyv1pjTLBwes2EVgORc80ZMBf z7Wa#dl--J^B`4GNzqg(i6;xjOU>K#rx%x0|vk&BW9(NqO`L`n<3ovYs&FW^9kTCob z;cK5^F)eDP3vbS3@uS^uzUybbq@|{(OFZ9vQR3$98GUM#B@Aajh%*g@g(L48Toko~ z&Pg-nr>oG#?aORHF3fdgG+^Z8v9GhHJwN!CSWwFm*i~^YcWd-{y@7--3H6Jb-;TMR zf#KT2f(@-(dcV_Hm~Iv!klqLS+7l-aSo=C%%2g`qZL18D4c?X5B%Pr0 zu$`UEKARi0MKmC2G#)m>_x4n8>|i^ta!-dQ8Y{JSl3iu>1e8kjpu<&u`8eMuG_MQl zom`VzT}meyNLKZK(`@>d1wHT3pf35vQe=SD8|xum>iD5uEM|5L$HJZ+arr=W{zn#wLL#-|pw(hP@+s7)TzxQUgn?6LPGs9dVc}brI-z z4BSNvK!}oMePzuEsITl5oNilUbQ=h+(68#x5C~-bxyVYlNskz?M_)L&UZIS+VEUz##zQ=y$Bx5uAi5%r?HpC$GLtJ>T1N z6qVM>j8WBHnmAc{u)&?R)8?a#auE4|k(*8YJm;#;dTrU}Ko-xdt?&3PPYAg>O$ya( zSNGQzs?~4nDO7bM0O6cp${gVSm5kAEuU(k`D*<2AAlXrSJEwUS>gL5iwE%LxffI4F zF;P*qHk^Tt0 z?4`q{7qTHVRE0`em3c9|vObr`;~Y@^wIRs_<(z{tEh8T@0ZfeH`zHOC$fu%r{G)0D zoPr7ObKK&iBM_6?U;@ozl=Fh#4_JGsAhS-mRr4IWUfiVJ_rhqLi7lQq4SmO6yh~s+ z!{K*(E&ue#O;Ng2gy6+G4yn)ac<`e<(kgJiDAMoQs9ma5Xpj#(#lx{HjaW^+=&ISk zkd8UpdGjQlk%t3kpWT<|UU3ZqN)cnUH&RWNi|&UQU-Y0dZC(|7A|!WT2(o=oT+#PH zIdJOR_+?q!@IQq-KTX`G39*!r{SrTI`5|8&omyd(qR@@YI7Wd+nSmJ-3^xr#m30>X zEWg{?7N`S6xiiWs9&#_&T^W9RV^Ev2j^6QDCS6T6VRi(83~`?!y;iFKR)T-q=hN5x zp0N>tiylYGDyhqlOnQw>2Jgl6hs#rl4l1{0^!b)YAA@^GoMQsWPD{B-k+@h=VnBpBUee^Xz(zXQs@qKoh^ z%rL^5N7P@>3$L+1Mfs%j>owYp1K;$bPk|_Htm*OoYoUah@6bBHM*>B8#SDeDw+Zs& zxI2`@kMV*y5}Fo>j~%@MTN5;&xI_Y>qH>foSp?t2|60x%kTcsIlav`&&3a-Y%r5CF ztMTFV#~Wx893+ns3P>Z)T4ZV#)X00Z!h?-KrrI*m321z=LPip9hwP9?#6O4bm`BlO zy_@Jb%Wz++!A?N&<+XVvnJtWUemZae{XS>s)rzXvOus(~=sH#!@a58K`BNIBnEp#_ z`OoJCw1J7FOwr24BZFZ)3u-EFVv`R z7NoA5)3mJ?|GfJm(b%G7zUMi&XI=1?*^%ST=4;q+20@C(D%6+=5Jh~*V7+rqXg2<~ z=I08`2rJ<8;+Xq*^|6UwGf7usLc)R*;Z+HT#W!-d$Mts((0rzEZ?Eh}QrT(0=e6&Y zmZg7?3~!1WZ?x6p@(*xEz$D=x&$|gkF6a+6Lx02^(|*KSPA|W ztZak_BIEcuteWG_bHkbZX3bJsCc6JzazP|8xvuD5#a2=+8jQZ*_ITAN-+Sb_5^ksU z=a`xQ4!Q*X0V3p9Cd4$qrImw4x)NVlHUs%Vip|5dyjZ~@H#q(B*@ma#CvIXlTuJhK z{+pTJil0NDKmid*roYy#gu zsMmsp4g&fJw*tkdQArjvGxMe4KUD8*|jypJSJ)+ViW!~)I{V9yP$89p2ise%;^8RfPmCmM$UcS zcqoyGEeqnsNCH8)R3{`8OkiK6mF{(54NxF8Qve}o$w(Sk8Iaz$!uOEI~HpWlh zfC>I7pxGjs`&iKjE}b+I@WzvS+7G_>q5b9a)zZho9N0z3qr26yL|s^4sDkSUV6o+& zDdAYn>j`qlKup}~oWUk?+kNlJ7T`|s6T>JGyIJjY(wYApk3JIW9wc-<)clpKC^n&Y z72ET_GOj9Thhu=|nKq|sR1 z!br_?TR5*?_aJuws;#n0>TG|}z~~7!58|_a3HjI8*ZVTPzX4Q%*+0D@}Nc#u(XIdZKQ=P`mDSrCQK92 zB>p@A6B1Cu${ln_hU>__J4$_D)&hhp+ccCV(tkTL{}b)+r+})dF(7h0<}2#@v%8l# z6N$g`6BQ>ExE9whZ&#W<*W;yJdeCUsmqKhuB(v{eb!(7rb4R+AQUnEoE{hJp$U&(6 zZ{iTY?hCjyNKZfDAZYIRGNC{52NA8+*PD8lYVEL}?5WQ@-P5NJSl;+znt8?|?)61h z_YF+e%;W$3poEg)<5IfzrXcquyAGskayZT+mT)> zt1ygtRqYLz)R|%6++AR!J^?S0xmE%l3(IH{buIWQW3IbH6>pi&2yXpMOiWspL!?8* zAho{l9)lcV9cyR>v3Ueuk!7&@opV958>@TDZJ4=f3n4HDrikB5_-~HC+o_7bA)9;f zh=|*qINTHQKEr7Jftt8&B5t7{L=3EsqEv{;@%|#8fpHzI`{L+&&hSsaO9!}ghJ*b^tz+~09I9?4{F?%GXoK@I}`cALxSuavCvWwW$b>w@_FRgrUUpzX>ukh z5_HSgJpDjr_DkgyD~E}melY1p-zEWba8lf>dv(y0RL{I*q~Fo()Xc3(nF6fc*RTXPP|&4NY%FD{!i?%SEn~L!5z0)*YPG z50l5>5=Wqs@H;ySodLG(#AtUsFVsgV{N}rbx5fWWi#Q0?!ZWWe3N4dxV@dA8Qk6W( z?IrJnsnHC+;Lp*u@`yVG!mc*PJ5>&D(X|xGg%52~=K_(sTj+L1TX> zW%alRNk4J8#h)Z1?ECz^^;7=}z{9lyPb~ehr}{4t%Aoz$U6zy;MmqoMxKz1lC>Yp3 zd7=U$=_eX4Dme+tQZ_;=rYnM3wP-)R?a2VA2Hf|<8|ZS}VY)RY9KZf?8aS3;ZJ z>#3p7>;RT!=0>rRGC_vX`?Hz$HBZEWj*R7{cnUBe<#w!aKUmrU8zuw=Ue4S8e==Jb z9^z3R3mhveWb)6R*qENW_ZcVC3t-76Gi?*aeW(5O7qc#-+RW7thTD+&UDu=;D+7bP zMW3_9$}tSIb1*r+(Ih6jY$UrGtXL=Q_=rn4U+$43LI*SYLkyl4l}p58o;)_AG_rtv zaXLX)&j;jZ)LT z0)#%Gll9H-_K55e3AZw$(gVNIf)?k^)=b6MB@5a@nZUEgd|S0lr|##oj!9sDnSFll zLCYe3sv@S3aaCys*1guwzKPK8Lbrj}%DvxAVlilK_N{}+gAHcRivmis-w_Ql&>S1x zFSX$TU-9w#I~^*i>!O|mLbvB)+{ME{Ap#UXitjXo1@fL4#_jCM%Ptd^*`#8ht zbLj%C9lq=hfO&y!5}VZfqp={)v>2kl`_nJbQ_C20p1?6F)ZIvqqX zR))2RW$T-6$zw>=$|OFw{LvF!netXLMHf)Gi{hwWi`D!zY=AvjoAwW+t(%Db6U~DC zg~8fw{i;uTb}bSBmUiMm9UD{`rVz!#0@Zux=uYcn@4_bxH-dWg7C8Vp?0&&fa-jqr z;Qt$T3H&WE!4pV2HNO5Fvj4&1-HY8p}n)0BnHE?1({~${EHD!1Hlgmm3u-SKh zisE{>+G{YzAs9&i#4<}Jcsrun^uN^Ee;(<}WB=lC+havUseG=0a6)bcLwslCA{R76 zELZ@c;|eVO<=bndjR69xd@>J^?#@u(cpaM5zJ``T0fLiH`Vy^4{?2S#>Jd~Y7#;8S zhpMUSy_hr*!j|QP8l^>;f0gvvUyg!MtWDj$PBO$ziMweA8s1t6rHfZC9I!Z30I(*f zw7r?P8JC-|>C8K5|=6D4kQ;)}+9oLW?)7>!tHwA(q7T1L)u3&Qy z$7fDdN)Jq{8kq1-bhwR6JlzJ$b9KOAWVA&K{v96h*QOFix&h0rv;b&XL`GV$p`Hc` z8z(Kvms|l8+OKE(qc+Is(Nf0?G(qdIi5L?6=0aI}Z8@qUHQcIQ5={m~Un5lVSKkxq z>(Kbe1s{B|8ZvXPauTBKS+he#c;jU>M4~0)_o(i?+UFaW!adV%bK6UAc7qMlZqAwS zbskvSbOesQ^D6#$hfr^`_Y?r<|GMq|Jt7FU?L%1ZI>slaV;n_p5inebBe1w-FP`o) zA#;fcB>jZdd9Q%zbs5v+NAS~Q0HBLO;lPp_Jii-&!QxP`=EOjE@Igf#11^L+DVh@n z?`OLP{k0bHH(SbUXA3RP20cQ*3h=j8^SBBDB0xpnB9;~s(Jw?y0%nq7pAnDo{=yL; zF|9E&%@nI8eSxHd=>arJ$H3xq>ulN2eV=2WYDn~De6XMmK3wsR`3*q8o0m{TN9W(t znZN%c?1Ba|{dRh0hU{RuuDZ1ETc9uFrD;sY>>3VZdVoo`rnQL5*AGP1i33(@C4Mf~wk@br2 zl`S{ym`jrF6*&cwb@RfGq8;Im!n*-cJ6>=e+qk%$c?{me=haqI4DQRmpnaj3UkKF& z)srj0@~v55=%5$2Uvw}ZhDG&-8b7_llT^#lvw+y!Vdno!UVtYvu$gc7tYfsT1sFvj z23M;O@ZDQ)1GsEvUbN!cSpNYq@~A72oG83Ce3@65@|i$uIlJEfEK4NJ zAJax9y-c{ecSj}Dw*jbgV(6&mtBiyIyC`*4$KV}{6()nE#D5sH+OCL+Xx?4Ve);@_G6q&8x~8U}D7mFg;b_uh|am<1xVuRpx&6{Qjp=RLKWg$($GA_Tz}#Fg7a zlcJxCXml|*1CZq+vIn%xF1eyxf@=qa23FTXJ%j}mj*SxN*3{qYoJx`ywMG@A5#N8| z9(8ZaDWWGuVyiz7O2)*4t9|w~>cPV z!hwmlT_*0Qd#Uy#S~Wqw_9CoiM(a7^2(*iqAwN#y7&%mt^U{+su`@jtWfh>-%~YUGcm0P- z$O(pcW;1(N#E2qA9CGV{L(jA=G(^E2^@tW}4+~+f$n<%a557QYzCna&(FM(@5HJz~ zQ=i9=l0AV7rqgV7>FxF6pRM8q9xr^6KrVsH*xL%wd5Rwhwee$yg~JH%`ftP>o)`cL zd6HSyy-SZH7tmHo{}Wge6#==BAeryr8DkF?ANqmVfKY4bP2l75r;k;csTRdi}PR4p9N~MhD94f*sgry2~=bi6kOjx zudpyTx6)^w7+I8^RLw|!9U?D)}*!4Ekn(K0WVU@dWl4WyYiH$X-9xr5z-iKA*wJ&r_)I*(~HD)_1D5leLD zY2h2VwsW~*RaM>%2?sUbqrZd>3J4d+D55}JkMdMaSwr!&8S6LIgC--q%?A;8JpnLk zQnbyT{egKtg+U+u8sL6xrntQUXcB?`2zeueVQP7@)xRdtS!RJ8O|e;@2Jt8b5G?(K z?qOG(Y){VOo$;oztWD(QBewe^?!~S$Kl|W`_Y|}iWbefefGF_#u2{r6Hv)DJ+$QF4 zb$$=H3PWGqg@<*R%w|bOb*)$+{|X|-7x>C$ofk=M!eY7JwJ zKL~pH`l-k8`aaGWT{JuDYm9JETRQ@y;63r4?&!)%AgnkKpc}L`_C9`eB&n)qSWtDL zoiXkC`>@v;tjZ}S|Jz>gKd!P^O9IRb(+%3D>IRsr)O%PhKoSmzyg`-c`Pt#|a8y5m zVV0^Kq6Fp*J}eYDwKrfC(K8fR+)T~Vx1{iFXJ+ihBjykBIwTr$aUZ}TEeG9IMF*1y z5j)@gQ3V1mzmHR8MiVT zrA>UM9`jUDm_z5%f+A*#2+8)mvChINsLXx0nfZt9`xFSINNVXcGJUrv z-lgP5A=ZfeJv2+zUtCQ)C)?bD#t~EC*nG$ zwn3c%0c*b5=T%7p&%hTBeS7_d7x#QU?++)YF~dhx|F^QDy&XOFjKp^ETf+HGP6ykx zpY!{gGs{M&;IAidUG}?J-*s?uJ-Q8D|DqWF;{?9z1C6l|5?dKRBbcpwS%{fxVe`#{ z0vf71IUXCUNcVFf+cboM2Mr;fj(}Qh${;jdukvhJm!hra07SVnz~7^;cs8^7z4jAv zEEDl~LrWnvVac=`C5S6?wj&y%r*4z(HFH|Kh^KFBu1=WL{%PtlwZCoIk}&#)8m8) z+jB=1Q)ECb80xt{Uy3OMUtvZe7hLt;m}Key8X^*Svcc6$w=mL@KoM&#BtTdWkW6b( z&uFItYtU@(jVGT?j3UwTygFyX(R+h__)M2d%x^d*yzYBwP-;&kbms_L?|3yIvzMbP zLcdCs!R4``jGjjq_SAG<1f)wcMaZLlDEMa^*8)t6nIt{bs+jDc?pg@UN;NY|j@ zB~H#MMb!}V3Pj@p1*RD$5)}#m1#3;qYW!7<6^U~jJGaKg(Ce=+Xzb+)M!F{2kAmDEVGvSC`1X9#eQXnHPgkrmOriaLZ-bIcK*h-&ffLI6AbBaR7k?_ zDM8L(JTV&ss5v6&;*TnSdj3G1-W2_XRl zY5~B4hF;plgc=jY1cOCAQwMybyaQz!j$~SqdllVJFULZ}%#7;}AQqZ(qfBuLpOUIQ z(^y_}5F})5-6fBAT!>s6t{bp!^zP>S`~zS;pYtA+4&P}G1Dn$NMkq2{mD<@hZelE= zn!rSYf4DZGDaRk^NKGq*zbU1^vY-b{nRAv4t4A)wJfq_B_dCacd{}!nAP8Sb1*R{* zRWP&(18w__^efO9@ndl(`b606#gxq;*Ka-_fm(d8kbuV6#)Ipm^NkJnDpv?}If9c{ zpkZX4bcLZ11xVfsIlI1ArCPkJphcz{Jid(bhbeFiTyU*6^V@ZR-LX)q5NWac>rRib zkpOLvTAv*Wq=LtU-{7C&y*Y}Vr+tY01m4z}xa|PS%zV4w{X4S}o*RIK(I6l9|Eypp z>t;ZMNqaOp|k`T1GXobpOn~0VqcTmbM=2MVcRq6K`G#F z*cS{kP|mYYT+7L@=l-k1_sE|Q_La{0=6GtHv3$39q%44FSkJT>Peq4oW>+oI&-L*tzU^L#W}_~?RePdIdk+j z9Q61;2h$B2f?=}ky)o)__YGWTu>?K@-21!ZyUibdi(e;;VqSNLXgLb@ASp8UC=EPh z8d$Qx>FrX*&wOVcgt}J?iuVpNYc(kIZ@ym)eG@X+_UN5@GYUjMBf30{F2VP;`@SK31a=8%SeJ75Din1`@gjoARUpLr%oGT#k9kGl4!1p{+fVudrdHBy?Ce~ zXNFG7L%}kUKdoQ=q^#?-0o2D_L5y}W@sJ8lX$Cks$X%?@hXzo20107AIpNWDTsSWWfv{p%j6~&f2qzF??4tAhjs7ULhD^P7E{Ba*7}4sQdb_XY=0y z5gYeTt>eMOZ!G+~m_jXs_f@8KoE=s(vbQkKX!vMkBIOy3`$1qYDG9bJ6jK$NNWtI9 z!lFvJoL8)g-%n<@n~}<(CRZIXZb?bp0ktIhrus2Gk;PNxf1B+n|RG^!vE_5;6lbcjEN5ZCSX;Nf zEFd}9!9h|lP0vjEJ+ExoBP18baHSxyAW#q;&ez>CM3F$4&tHANsmDLrA7MC<-7ZQd zP0Yu*4qB@otEj8WhE!>XEF?cEC9-YW5$Mt?gms=pUL(7%$S6F-G#!Znpe)tD+GBEy zw*5wU)R~@q-MPQ`Qb`9v+EpdEH!%q5#Y%pVzGEpf>>(vusCIBniSS3Z51fwHZYZmZ z6^$iEPh;Wgh`s26ru7~5Svp$zGuzo^E#DR#hpk$N$#@`38hcBD4!obX;je;~29z-@ z%lP$jLtGPAQEtKQLD)nhKNg0VHOVynIse|{0wMpsFhHJ1c_ytH`wvlJ|oG;1AS(HMCO9N24P19ANR!qbRjC80{^N&sj83_AS>vXXBam}pXhV?Lb zDqPkKrR&2KhHzDMtYRL-QQAp*Yt@V_pK3YsMKPyyxQGa-VhiR{iuB&1^W2BrhsF^# zs0yr(TC@?K-aUER#W-Vpl5PXKi~x*`_kmViiq!O~y^d{3S0!QAV=`lUy~4mqXar6# z=@`N4n(dPrbGxxn`e}A7@y~2H%!Gre9>L43I6thGTlL)1u4yfCFtFU>Jxp+;J@fjZ z?>O9Hvb_|DHFO%2w3e7oA13wKVX7Xq`KQZL@_Ll@f;@J4jFu6SdTZ}UXs~=)MfiFI z&pZp$UMJ^arlyHxuu=q3R(K{5($GiH77EQ*z~Z2v7069}M=1NB^*m5-Q^HP9+gF+F zOzT=FwU(io#kY039D4JNhDX?hn?}M~u;ZAx7>iirB6!}o9f__vYec%|k-C0NuTpWr zl8>}s7sPwk7T&fl*+k>outU5=8M{vYnHs@dkU;30J=udwHc0Yu>CJ=$YEbR}Su%yL} zDqM1>b2i5uD38ZNtVG#8?%U#~Q;6cKeoLQ`KZwVTvJL#1ifhR#b`t~tcK(fdO|}Br zPA(%a1}}~r(Hf$rY0Ea&EiK@jhv54xg3^bM|QTUn$% zN**g*gm8hqMI)uuV_cs7mx{PZCj_RsBUjSN--AHOV}_j@VcB9D1Z(rv)!_E~^l0|* zvl8>gEIX}IaHvpX>(4eA$cMsA;P%-LijEQySeT;rgmbEuH3W%iqHw`8{gq2B{Iu{? zm2?@2Vc`XGZS9U5c?g{@x^yNA9&;Xu%uK&*|2m>}_`cZQ=HEM~%gAKUfWe1uVy+91 zW4(!%McdHirKY9iy$!;H=npBLuoG9nALg{HbR)4^^1BfxvcU}CmxPs_?MCtNo)GfJ z!yMUm!)O%REE?kQY*aFIsi#CGUD(L*O{T%TKh*Z9!Ub8`QO|>ft)qAz(KL%r`qx3> zBc5zW%>xLI3y)yTQ`Ane5&g11cay9s?3 zL#YX)Ty+!z6WKjmvSOn@|;`^cOW`V6pI#r`8u1H=H+2OoCm@W)zA5$ z8G*5!iXi9>-+n&ZSERtq)zo*EZQI<2L>=hw5oSB($DGHgb@VdI)0bF(N8cA>y4t zCFAiYs1aDUy(@d!MHxd8+m$?mJ;2ILRr$~e?Pt)XWCuTm0p<*~wTZUqXN7kQr^g!9=4rCOafNKBNCW=lCd6UIZ%X9rgyqSDApNT zpu&n*CB;f+BSJWn0*CU+dp>!O`NY}3MwkC*ij(U77b1q1Ne5LlP&?Ie!LfI~o!dFj>#}f; zM?)U!WNCP=8d(!g*`I?Zn4zYgs@PfXvzjE*Xydl=?DYQgIRBDR^evorfmb zR1(S!_ zH?dI@A13c-f|$Z$hE9grJ&vS@Rlo^UY$)Y*2;`8 z$)nvn9f}Bb!?c7}Qc@G*soDq@tc?1DanTRw#QL&ANJ5c@*t4o=rpKbcs%iQ?6@?=N zGC(mOHh(L~ql_MR4mv-S1xlTI&Tv^KHU%AR$T`_hFXo^;+jWx+zIR`!k(n|eqHc8x zUNmF)ast)QOgCayg%IK|2f5^*;Kv1+h|X>XBv1_0P=pj*_}Oj$sv+&bW1Y3_Tfg6Y zcA97J<1)MabmpaH1pc1q)r&)yq$mDzVIu5^#VhU_?&@#>myR-Yq zrB`izeitze2{NA3^Jw3G4M9M;_q0$0W(2++m~b*K%f;y)$LI^i&e9NSq0k{7d{0=x#<)^UcHNmb-!AVDnhJBG)i&Lh!s+~P}xh< zX=QV86R&1?QvMJ5&VL_6;d#t}(@Q$?2v&o#YxE8stA$$hpMbS1>LIODBU3Q}Q+G&W zCheON2$qQ*YsMo+UAZVJ55ikjm~y-f%oQpvzplI%8Z(S(f)2GlCxdFw;Fm@Wi>>F~ z^rX}zL3Mg;)O$$vI@1*iZTV{cO2;|p$!-os$@5Sm!n`Z&pV_9qwi1?BbKg6zB%#^I z3d(H9_?B5@V|U#uDs@Kvwq*?P1DaSK?t**KA)VEo2!F-&S3!jqD=2$}Jv!=4E5ove z%Yt-=aZy6WNEr}J^`&U-rJlk}5fO@vOnXhXeh}nVdj@weL}ISJhjU1*)5MVcj|1W; z5bk>{1sd>P57xuxa8*;n$aV23_C2Hm&+0AS9zmKF@pG)l*l=_PmF1=PTX2Rp_#CAdaP7W)^ zk&2UCT#c{Q(mm|AFULN>a$kffnXJxcJ0r!?Un8z5>EG9Zn&dFjT7IqTh5g@kEyXkd zOl!Pxf#uf*#;$Ix>@2w#Ot2YQQO44E^d$@^i*fH-a9my7&pN z&&j9p`qAbePMdgtl~Gq-8DQv;uMVdig_e1+DxQ@wjLAj)j8~msg2Oc53vj+NLO`7b zhhM+-BMK3Mza_eGgD=Nu5=5V_!Wm|XH~VH5j}6u88jrE-^87B)@ z*2^CVk~m`%d+6EO%Z>PXcgPI1LN!#s6BhNXr#&g6udi&EPxa|tM?@`6ZYBT78M*_K zWZCB|aovY8$=`2mo%cZ&XYmJ%p_uWl;2G0`JsA|XIoKl_O-xqL%feX0T(|qDU^kSl zl-67SfkbMEjpVy80>u_~9g3(u4tZReg}53V^nJqT>n&+_4P*Nqg?gE5oz#V{b@ooG z5hd8Tru^z<#L+v+dWK5dXWz(6R#PO&3dsGnt&biVbR6S%J{XA-mE831vAJJxOxXWk zxm}QY#M_e3lQYla#cUwamH$fxcvtDjIf)`yfg^iJ2xzA6j)kSK>l9>AY##{_UK38U zmnZ1+3UrRY$T?jY8Vsp({a=_88~ctUfwpl>kqn!SU^B>LnZvpk(2Tfy%eE8o$cWeCG!$m0|X(w+H^pLb8NO6`};w; zw@;wpvU}5W=}B672PTO&vZ$^vLxvm1Ii@L(t`z3%lCX|UQ*gq&c1bo(6;e+)Bf!~`H__|9%^8e(HwOpf*zG%A zN=C=uUitOAddd2HGyDLc67Yg2nfGutHg^Y$bkp~NXw65e(Sh(kL^mWf$Y2Qg#k)F# z;Vn715PclTch(wro>)&vk;XXr`T5@&+@zf9Tv3Dr$8$?6@n`S7jrfUT>3jfc=gCBk zY+R<<+Al;{w<5_V8>e+;uOI)?to3j6spC72LWvIf=Q3zqGulz;3&+p4z}zzv0Gab{ z`$_mGJkN3ZAX_-qBLpfd^CRok=B7Oulw&KJ1C`d+w{K%hoPv3s!`h zDn)Hqv`J5^XU4Qld(`QxzdA}Ce95g390{hH{M`S+6L0mpQ*$+=Ze%5m4hm810w=w! zM%sq|OS(m;-O=6AL~z=g|My3ge#n{WCB4o-H1)VOIH1cG6(4qwN9{8UjSa{R^g+ZX za0p3rFA#Ar@UA!njO!viG*Xi!S0=0#ObOTO5`wtd>T|3m+&S-)pu%rGaaOmQ0Y8e} z%|9mx)GYsJ5PlRPs~PW_ITHW|_EXii8a^ANjJ)s};r~*XABlxG?KNruL@}k&cXvxY z;Q$0eZr4h;YC%>4D915)y~^^Op5YdJOe0YIqP zkU5!`#$e%4V~{I=yzlJquZ=1#!U1r(l;@1;^3B|%jcAVAJFPwAq4Wpm&kf2j1Z?5wajflQ9OE$OCMEg?zfp_eSC7H`@-zzFmUfPsPEjo^=ln05m&(+@GN<#tCcR<{83 z$tym~2mpdWrgp#g{1-O=*ALN2clQL0r}Y3LG2Z|O3*D5k82)o)fVh7L>uvy=$Xf=U z4;BJ`0#a{;7R7zj#%E=FzTtBffRc1>e~$FIWEBi%%X*Ti?YAj_SyevI=drNzM<4V) z>J&xPxy%*|Ie!y9f4X5&6OS`1?}YQ;n!kTOE$ol(u93B{KcEy$|0D3 zI?cm9wc(r39zA^Q?3p>dJ@yFPn~9*pkaAC%uzgk{0PtvY8`n*O3CCts!)UjA89|;b zP*$JbIVj%AMkMx|Gd8hBz&_U4prn4&%y94zzL9uR6qK}RKzGzW5(nbQe?+Q(g)IP+ z5VI+R5xo&}O(&keqt=BSr%kQDW$HS;nryPlx6vEhP4%CTx66-Ze36sv(U>Q*Lb9f{Vq z!RJrDTK+R9`3*wV_eJQK;iD2TcEc*meKgV*G7_RDGtwZWs@D%TTh7hW2?H@(GQ?Yc zPm5(xE?wWAH{D0Y+^P9i)_zd8F4A>vCv`o0y#zd-I_^m44mm|dVxmW8N7we?{_1|v zW1*4sC&Z6kMGjADbG{9^osUFSef~e6c7>(tIM?xGCT?y(JGsE0-vD2!1916nDn_iq zd=h@X5OjKzn27tpVf1++B^Dqj{_!y(LuMLQl`MhXZBbAFZus$noCam>t&u9aHA+vK zpSjZ~#b#EE&tEv!U;(c;tBBFYr-Z-u>KLvQQECU#UONZF+}6EN-mKfK7+wi*6&~h_ zuavZ9pI~e+v`7>HU_MFC=Ee^hKqP#3_}2}suJk(~xY@E_Bb{D*H|3-h94zS~Og%m^ zQ5&0!`G#q<)?sX=tuV^?|9!*%`bE1rgU2kQ4p=O#+)a`-(n-4^rBH2=4~MJ0t1kgw zz2pVn<)q9^l8}e2O;EQuC1j}VKR+x=7QpOT8pGHm$v^6mYofAke(Hwm=1&%6t>9yA zL;CcHX>p}uyPSKArJBPJm`7IR8+1^vrUb1<95L2fKI-1WOFh|fLkqO*_r5Htwi2to zKu$AZQC-ZfaS#i=xRoe-+DNF9Oe5#-FA82HpR{p+C_mNf;;!wL3c32yh@&J)E$YFr z2QZACnW`?fY5<@sUf1SJ1{{TTD*ynsVKOIbO_y2uysK+S;?qZc&CjRn0{ib`h z-@X|DZk4X9)xcf^Iv|{Dc|H!Hai8ewOayY3*v-^6%gT$if81eNM;1QU?+*$i%KAZC@w{nl{x7xnyQxnmhVASR0nB$&+sm z@DX|NIG>gSpa~l|r;R=)$RJlmW|3P0?oICvtE|*$UiCteAPnK#$+69aYC9!e0>ic=mvOUBv{2jA-LdzNF`E7AdoB0N>$BFF#Sza{jAO4C21VU^U!e$R z)3}{f*eljLgbn?$?Wn#nn4fxpi$yBASAQDoGUGdmo1xP;IFi<|@EIG>MIk&F6n$@f#SAiT8j+1?LWI zkkKfyQSEDZyaJRT3V_l)U9?H>pKIIo>RfJ6T~a4X&7J}UI1vz?N1v2X9veAT|BD3> zcA)@9%JG$zm7G3mlDv0ENaj;yhzR@2E11jnbj7s4a-7qAZ4klP&Rj~ALH<~&_Y+Ak z4Ur)kBRUMPe2N7z?Avuma~W5j&fyGvT#bZBDz7HTklEhaydtu=_w+H|HwRVE&q-$b zuw1IBVppGsnrmmozGXvI>Yq$PDnh$uL&BELaK=++iOhNKg+P+777pdvtU7P_tz&1; z&C{L0D-Aw7KxU?`GOGVR~}Hse^CyG($qOQ4v^{5$&m`)qHY(z=Ua zfbCWUj+2?n;1%Zc*?NTN>G}9Z3lRnK6BAkWx)&ifEXUxeCn6>F@yS@5N^6&aRv0!!OsT`RybYw|5Sv~JF(8d~bBDxWTcR7%6u7>L z!1GWascKZt6KdiMXu@Kp=6fAZtgtg&Ai@`M9Cvr+2`2Ls@q)H zm;GBQ*5;CHV%p!Xy; z$HC>zB&fX)fzU-EO87Fr*sO+R*U`XZ9r8wl!zqcI6a)E0fwja!-7%4j{~H~UAncULS&v;Z z;*yY1&>fzY)R)jtZH&F~vFiHiwG1>JqqmO)`WX*pR@M&B+&C#wCGTUIm?48U7$?}X zhI8YF5vx@Eo`Jye$90islbQcuiyoyQ?(ZVyVIgm@o#?qnRHokF!*%qu zG}nxaDkHIMSLk1s&ZEYkt6WaS)4mr;74wnP^%RM^)$9o@OG5E#-hLZ@w$iH&Z@D9s zVx|waL5Pw}D$dXUqpW4=qssMllvG^Cyzdac$mfA`;TtA%tFYYq zxrmzvd_>Ik8}q7Mw^d8(Z*(VlDp!brXz5uqCa#}`K1OG$YCdzvj7@_R%d6h zxO`BkV%cxJ?4V2@1jDtwm`I--oF6pi%{ps{pWHpw@KQRg#`>wZx@hv z2vTys2>BLXF-^Kz;@m_<$JR6M@~IK(R_1=9loRK@eC!!!H8sz1aPG~>%S(@?l|EJ< z#P-{H21h(?;4ZlcAJ?=18G>Q6f7OBvIhP^qz7N&wr_ri3627>s8CXB9J{7z58jrmX zpl;@?B*iZ0`JH3?O$i&rJ0F&CpyG5G@(8t@BZn3G2=mdq9GIE;XKn;#2F^%rf==wK z`rDR|_dr%~QTgP{zEoUhW~-Yp2H*QCqNz71lOKe=)TH-ZAaC(SKeX7>TzEH8dZ}c2 z*Ox(}m~z=z$G4un35}M0#=c;HTFWu__5PP1lkG4nTV%el6f})9qgwL$kgrn67TTXt zh{wNfIW-A!kQ9Z72JX#zN%3sd$eo?OgvHOL%z5f7Fc-%cav!-53Zz&}jib#}xuu`OFi0_SaFrPT71kGAVPO4AC^F1GhgRg7d7mmMecAg4j zX*WG)(~|Jrfnm`3%hHyP=f?b1_B@dY2Tcr%?s(GgexP8m{T@^XtY$<*>+UtxyDJ=j z+vR`#Py~pm(HI}+S}aSCgdCi)79E@)cTMivJxIiBUzpHNa&UP(<-JdgwDX1(&4G?Q zO5#-%lW04^gl~40M+#ckIFl|%=VGkg&ReG4PiNXG4(2wc2)X208$G1*hb@Fq15AWt z298&o_6M>Z@8O?DCKr|pX=lz&RdVk~Hig9|5(VwZtvYcxjCa(Dk=%yBzw%Z1eVYB; zV~nzyU_`PXC zm8Nqe4JUgcHSEwpXN0gkKtUf^!GD7!cN@ADXbkr#+GiPI+cah!DFePMAG@4Ezp@ps zrUp%=^Z4sL`nycp8})Pc%v)j^;oLLpYf!ZK2&mi#6?R3-*{jdhc4@MX%nYAnMWlaY z8+nICo!H>u;Z0#J7mc*R4qkn&+i;dXJ_LxZQbgN4ZXjj@Z1Pq~Bz95x43( z7}njY?gLc>ud(KqT~m>Ngh3tgyIjpJ^~{-X*40`>PYv+(mR+A*vF@f^fHt6G1=VF6 zEAjgj$s-uH`#+9QDoh&bV8d=YPyK$4P;z$T2H}s7<08kF+JTF~piN2d_P$AE7mgmdxA${Ih3VE^$nN8YU9~QrS3oox? zyYk%Dm1s3GhxHoAzA@|eKDUT-&#Re>n3hmxWbIrP7M}ZY5vo8mgiqesO?^6ZX+XYU zZ1DWP?ydhtb9R4WefERfsxTC0s&23_(jo%tUXzNvi;-{sF#HSGKH+ zm{V^&4JVEEQXTY7yp!HaaK?MJxQ3H6{|tMjqN$_--COZc_x_rnyWN)?F9=r>UXc{( zBr|??Y`E2~jr6IC!A#ywFb(Sz+;%`pep}I=jHfZB1-Boe3-N{3Xi3KcpFhWE?-X^_ zD|Co{tg*cztBCod)r|hPbrYxBQhdDY6R#yn5iFDY$e>6_742PSR<^SL=1YMas_Hm_ z4q#rItFv94cQ_Z08fn_N8Z!A*W49Ddvn6{m>S?#CpEl!YGX^Bxdf%6#`M2mpw=&NW z8EW*eWBBDL<{wPf=bvDRXRhl^cU>t6=RbJSD!G!FU0|zRUX4U$%v$U6!=U-N`(dAn zBoUi--l!Q29Qo}4MvOIU31qx~e@m}u$@8vmSdxLvfI1K~O?((5POyLqRB*>v0<=}$ z4%dR}+hs8bQ!J=@RI--Xt*xku$AI0u7Qe*rsb1;5`NhuFGbvlIs}ow z=6XC3efXUCe6|Xiyb*!%DN++r|ImPxp?xWKU5|N^o0X;bdtyErt{dc|n;^&LkGlJS z*}u0T)nxKsrJWdKt^`*QT?9`dLp^-PXCVB;oE9cN-)nn$%&p90Mfm~m<=7-8c&RRX zdV*=7*N9B!`Xm}9(-vjID(}dvAO6<5JWn5E^|-1P*vDX|=+_K+G=#eyGTtW{V8Rdh z@m-XJ)4MEH`dQCClpUV>Y7L&yP}N84Lzfzro~HsnlyowoG$5u;*uxNv>uYNc&F?Th zn3@B+>w&MsK^XGipYJA5P(QKZ@sb zD9Onb<%KqC&eN(@h4&2&AwUAh57ZW<3uOkMv|p*ycZ2o3Z-4}|^iS6^=3SK|iWE0O z`Xuwj^2K%ONjVA9TcF8O)zm~#CLlEt6h<6h;3p+yww5DV+Uk6mK=EJt<<`1jFOa%! z_p?||^L2$^PV@Pk0dX>k=-68)LcNa|)@M98^KKg<`Ut*?QP#O+GyV7Sv-+-Gy3&*P zh!z*j4W3ik{`PeDlzA|#t;oH($eKX*y|5lrp2o(DV4sS<7M#W*yb37R>sJv3b)ZsA;D#w)6e!|T8S$c2?PhSKMKymWP zX7b&3`-R_Esjz2c_mlrxE7sU01Xc2JGvRt(_j1hF<(}v0$_TPvb8(aQPshiTm#p}r z9G?5-QZ|o^v;wE&h@VoZ?Td=NJ*gx2L4i{N$*DDl*B=n}DR$Wu91xQwuVl193Ia0=tq_Iyp7T!n?OBEgV={RzR zOua|n_8Nz@V|(($UVnT0v{E}W-nwf$GXrax;0Xx>dpp5M-Ki+A!A|{_&Bshkci`jD zKE0TtWVzrt<0?E^W<-pFfv*UPKu_QvVMfBcr-=;pxPedrQ-Sz%ppYIz6MCneKMq$V z`Ucd;vG?)xM;|nB)6vdgefaQU#+>5$Ljw}<$lNQ?j5`P?R>__Jv(~y%9?-;{T{Bmv zZAlbUQvdy>wQxJ$gJe9t^_TUEzX!w}dna6(gQaF$jh?`Wm3Sm`sg}26StG{Vo{)d+ zoBq=bH}i~g6?ah>L1ljW1W#{ltLZd)Ca5RQUcW1ZKZVJnrGecXP0Wak(MNRg1Ji4m zmDeKad4A@JHnlh>`L0ux0|+}|#N%395y`yt&+*r>LfpBARJuJP#g9sCxH!+|bS!*| z{33|%`gS55Vs@t+_axI+W_@?%sG;8tFDOxNPG+H23;UaPnOOV|1QKx>nePVAsd2_C z%r?;0>FQ^rh?TS-;U2fO8@2o?LlKG(N*pLo0)`@+&bPlWXX;#Y#Yw@exoBmsmp*I= z?eX-!>vLwd|E7_jC-8$D$mEVqJbCuOWtzCFu#q6Y@w{VAGea9rH{|UEePUR}Ck%SC-Pc^= zYWG8eyu9+aR6+Z3Vi(b}#Jx%M#iN3>OG2eMaqM+VxNqLG_s*T$PgB_|#hSRbER2Kt z-typwp*({;4s26fEao{}zd8$UF21MaD^lQ2UKA2wx5H=MgO6jjOA4!M zVks@IveQ-Yc!J+;UNp%I2NP3g-4T#2qc!#qL)89={Z-8WevAzK$bd;kaQNNsnKu|< z&sypz`Zk`L!xPS-_4~xSPKe^*FQ14&%IQyIq&NOakI7qRgS4C|Q4bVwhboQm-bB@M zx-uTY26OZ)6s`Mcmn+`3#zX^a1(#m2s*jS6=8xC*2dm5dt3ptVt%o($;i2tBaUb7jt=+ZjGS_1W9A>&3M9b5>J7lnGxz;Sy+|0;vl`q z&5CU00!Jbs$8mtL81O{nU&48=pA1eHrZ6js|;x?-L55)WsmObzOr?UfDZaW!hn z?|)R|rSs$oZg{K17PIe4^ozL3#yIYfs_6r!pAeqDO3aCgaoys_DpLpsD@#VlgkSbl zf2BhO+Enq)RQF5Ip=75qBr4`viRHY}v#Ia$yX%5w3MGwHHl0uNa30ph!P`v_w|)md zC?|m#Cj?#hHAojWLt>{#+Aj`QEo){U*$fU3$0&yPYpk$xKK}TOMVhW416B1k2~nzj z+lk%X3`)CDI%zW5w6ecVOkC(w^QCrh?PncwnfXNl+-11}B|=CcJHB{a=E+%G=5^zVNXB_HkI?Md&Ckn&uIN1p z&yV@Fa9z-BmD&p2`^pp(QrAcrElhqp#i+NK;SK7Ak6qeVy=ZYCS&I!{q8*{$x2aNG|GvI={^kk!N=bj*a=w)OVmfFc?jsyqj-zNJbE7v8f8Goc`zoSd z7nv99{<*0|3!q}Ww5)KT4-%W!@IHQ)SZ%0C=A(*do*J9YM}D8y{3DKz-EbaO)Juh* zc0Ue}B2hdP-hBh0NhG!_q(Tx{Bt7%E=aYN;8Bc0~nnyrxE6Lp?$GU5hi`dPLFn(=irixyK%+V(6 zWbA|KnaNI;CP{)IPlHpX_7^uv7yb1pO{hr1u_%W&S;J%!LkLW9@)N~g%8W;t4-ag^ zz~b~|_wc3S6ZJwmPX(BUHjU>v^~Ug9%Z*3dCcJcUY#9`T0#V5LiubMo_(*h<<)A zti>HFS!m;H)8Nf! zcHt%NTW0#aWt-fASJ+b`E5$v&ij$z|WdOCWyKx0_E+VGRX$l)yu3Q1=fd7y6zh?sf z`LwnB$h}4&gyRxO&c6efrds!2@Ib|baQZt;?!^q_CIJ|QN&_U|eI!f?nMJMLM!<3< z(~;u3tA6H-Bk|VMr*{=y_L?(aLh+XD?|gsag50Q|my|$#kb|4$uV$lOn|t!od&c)G zmTLy9)7Ki?$WM)|vS&9;B)a*?fALG1+}~u~&G!yV;G5+2HSD`yp(Dh21^;2TN%BG0 ze4mSBG~}A(Az?x)Yi|bTgSeth)e0)x8-B(gi5JyKTV{k=(bEA3Q{S0N=xdg1LE#_M zR|B7FVE9b1k6M#wY(g%89+ycXH*CywqJ(~J8TOgDWjlspHOiFDi3Qyi2+*^E#$CBp zSxJdek_oxVZ&w~{WqKg2X-2Z3ev~_Cogur)#&FoY@+`~Wtl^J%iEhqZ^zYxn0Nlap zFMqlU<45|?uXTvP<0c{|ej_z3ZD#cT^JliXJny!LBanAFX!|FkY-DQC(i82bq5|D$ z*8P^VgxY8r=Fu^h&%1b&3!hx90J|(6xA%{mF|BsxI9rX~JEgvMjn*~VcSN3i&kPFM z^+JMgHb>*>3rOj9Kxj8xQsR%fZ{P~dET#0uHkIqQ_lMWOL=;}``Cfc?t)bc@g*4RT z%|od-yy<#@y)7*z_v#-AUllOw*X36INhBui#A{^ln%maTo6eCSg|D#AKY3*C2 zP;L>TtolbRK-pE=O8=eDa~TigRX)+zOvla^Hz8Y3-KX6(r??^t? zU`Wu_UMFR&^HiB1FCywp3g0p?X$cg5^eayYeU_oxKPQQnzJ}Zh*}b|FaF98$Tm7USeSoN;6Tl3nb$GI z&_|JCf4=nO7*v>&3q5_NV5ROiKU&1{l>B@>E$-oFG&8@2Kmcd$YKdvLVBF!WOh>&U zt*ei=g+5EiQ_)+Bsj8|UC?&oc)-Jwn@qlWN8{0KeuhwOIh>nC~JdOFJzgy072}^je z)bX;Q-zF9(yLO_?DzATU^P1TzAJf)W9xt`u%2PpAvm{d!J!SS$6vop5XjN|qt2SXy zR7mB|25Hq#_=fyGO=x~XI7*Snh&EC&0~aYP6~jvUD=0>RL z_k_d=vSBD-2Se$38a&F*A#)NnMBE^;xiZ;is#3xS_)z@V;#^i$pELAkNT*4!yNYzZ7>J5@~W zOFcinb_zx4I(3I~f-^A6(ve8DY^kAMW2gjvM5Aq?WBrDg$1u;0lfZDSQJpZ5jY^2j z-z$dF?Ou`zr^QqsJ-fbVEB@_Ms`1%T{D7$uTz)?WDySqyoq+1%KLFrDVTe^{! zMo<)_yM>`8r9($ccnJU(|2UvE&f24eP)?T$~> zoJG7R1PQFX9Q>sOng6am1&@7c2T_!^PhzBs%y4To^xUoNJp|%fzjJkH9Qj=E6LJdZm}qxv@%AP}HE zm~#{&FX_~i`1v@m3;J5pSa4dKRRl*{RXE@EJp9z9oMrdEDc9{G>x|u3(Dwg0#7^4{ zp)T^~guY&W^`I9eOUOk(-*Z*Fp#Q0^ab8kC7)vh@a@*K@G##0~`^cY=EI{f%YdhY* znA7bS^3xCNal3&@h&SGY33!I#NIhXTq5Q=*%L-XtZ>3e1ZqXGyitg?6#TgSWj}5IF z1)Jz)@qlcBv|=!&X+q{-@;!MtR(U%POJOih2-E$gG~ zT?3NlTsq@0p5xk*7G3;cjAoM-$5DSKP3MH6+{mFwTLeX@FIkxlxL^wZabWQ>q#>^r zAq$kK zesAUcrg6{|4vq-P=-ne`U`5EfXsDu;x@X^fyOV5{B7&2X?u#l=Mb2f#XA0yfb?fYj zY6W@+X;l>OsNCX4Y3)(n={g@ry3Cj19J>VGsb4u3N^_jmJUpn|N1 zh?|E83k>rCUi+)H$h6YBm>;H#i;DngTBr=GW(g$b5+L8wAgmT7N6uZHY_5S}Gb5-= zAETgP5fOcgQ;R4ZJtn0BN% z#bu8Lf43~+$L!*V7O&c21BE;0EcSYpN73(v_o-JMWIXvornGTe) z-aXS&3ZOo?mqW=SY!skS9>o1x<-pu?drJ~WCKL!vr5SxPzL7n##sqN1V)djyP5Mh= z2x!!5`<_HOOGrpKH!mWgqlcY_;lY>L-r+}lnMY8?u>aWFDLJV9b%M3JX;1}MaMFSG zj-%Vcva0#i*vHOH^? z6)Z3bm1Bq|lV(i9R#&b|`H^q5`ew4I&N^2+v}KQBSHR9S->DMCAcrXQIl--RFP7H; z&fR72?z%v7E&R|semTP;-Pb^6EF<@aSRw%Pdz@BQ=yS0@*GtF6g{7yb7hpr-C=J8Gqo`yF1c3rpV!=)W zk6BARPSe8B_9gK;gNC~Y9gV`N0}g|&mGf8`f;l75O=_vbf!ArG%@4tTaL*6=2w zpD3{mCZlB7G6WtjAYt$=JLhPU*->lxHMgl{8*?i#->5$0H(Pv3@=FJ4^lpSBl6>A1 z8rFam@cM&;sBQ1T{IiBug<_R)#UhGovmfLr!MAr4Luwxl;rPfZw{JB1wS#LmIC9ov6m_GYGGbvPJR}B(KXPWudMhOY<1HY7w*=J zH90>Bx+@LqeFMvXe_*9duO2KdtNhx1)b47Mnf_9;n=_!b-Y0vAo&WIHtw`)dYq{sk zVCWO}IXem61bDnI)Hydz+Xc&!RV1vrrm3TbaVVM}<#hhOIo48V@a+@wwU zns7D!@8I3ts0^nhClu@htM`#+zR(M(T6EdgQu0pHl=iFY(f&in100K+5cW?qmi6=M z&F&6H7C+w0VSLLGz2PvK1ro%eVA_gHMMafB$RsAFYJ!W8e9M3^J|fo!Tp#KA`0z1A z4&h)Vb6SDgO4a&=>=c`r!emtGOk08$lx{pZNXjaX21O2?p( z$D7{U=Ghfy-?4L8*mml6% zHf`jp3mzKdN-Ebqt?BqHI0xpE&%FmDBN*dQBD<5{3RDM_Gi&jOc?5U`Y^b1Y9Mrw= zOq0*OQI-B`Qip59(LnOcC+qXL=6>jGW3f6PoGN~A9ovH4SIc$WkhoeA94K{Dz~R}5 zF1H)P+{ZeZ3~md`r1kO4zi_i@!GY6f8+SIf`mYm18{S}02!#ZWGET@VN(}Pt9i4v3 zR&Rc;jMom@h8^qfYdMxLnf(tQn~n8)q84 zSPVHSpE5m^a@;Lm)d%J$91q>e{}wt8=4pB99dvl}w4DdQLU&B6+_3T3Nl*vKo`(W) zSiK8T_gzAtH-{|OX-M*76=0zFvM+(E0eH|=vQ&^;?s2Y76>{bVzMlIzes3WG-?5V| zB05IfqfY;nu1v{$!_&xu1Y`bHu=aUly;rfUWYmdx2t8K96nWK4P&W^R5p#kfahJz^ zVS*DE%@jeqA+z;+jAhCj-=h~lBz8sCL3U}Gt<@^ZgsWX}%k|2g9eV(AH7J$*A~V9& zj34i-smFUO$%q_PL%dZ(#+oK=+8wQVnimRw)op4%pZz9EVw|4kmNG)WDCBP$wq&1T zV26i2qf$~8av4xi2*@j+Cinz*VQAa%I9%gqMBs3)(^up>AA<6l$i85ts6nFHB(hls zZR*=QW0Ulc!x&Bhw2@zKZQK%@9<{FK+c&QlqfxMOK+XCkW?JpJd*Ug|5k3iEf*>9X zyaaw1jl3g)VO%;)=n}HC_uSfsjHYZdl8;B#GT$c>i0px9j)9k~^w?fPP^Ud#{ zU#)h-`>=YqqbB$ikPsKKob6px17KpYneM&O{uk^|EC|`B$D|W|#axM1#gaKxtEs8Q z@{LZ46J;VCF+u*tdOap6@u}R7+eVUEOo0_H;agrZ>)Pd!Y6Z1K3>EoNrCip@B)ERJPFT)2p@E z-*+XXjbwj69|Yf###y18FLDP!MOi?*;UdkD1wUFwM$iSi2hiRi+vv4kXgD$CPSseGHWk+pAVZ^D|>IuYpjV3o1rbSQPMhX|Uiqzkp8W=_RmZ1Fpg}HrmTWB?!!-Fm|!TpTPft!d$GIJ3b z^bCzCmm^Ok7jUHJ<*Z+2KvP;pKxZWJ-t)s=qkuft0JN40J<5PM-1bQklBXGeMuL^g zY9wFImJ*XHtp)izt;ffL2UDaKs~?BxUlkL@NOa%DLe;r!71~tj#WZ&h6YjbN8GD=B zWg4nd-d4uzX2sv1Nn(0B(>(>EOsx7feLUMcPU6K}kdLalZDs9Uq$0=(;{7;`%*>r{ z@k%Pa?uhs9=v!%v7&5vo!&?P3D!o457mu~zINE1M5DBw3G-qt)Frnb59@M{-PLszQ zHHQy7ox{Dzgvy^QBB@(dpZXgF^nJK6{{&{Rt?^2?WkcIj4lOA30KAZQAeu>>KbEMm z`po!s$LDB5M_X&H>8gpf+>xHR*uj;BcKy&gd?uTSh%Z6-wJfH4aZ)p_UMc4U%1TO7 z(%F2RT6Up`&HSfq)J$GYA`kv)rO@HbOH<-hDl-cT66NRT!_sKko^B|Vl9lxt4Rz~L zf}o(F(2BOMZt+duIW`3oUF$sHmJO(eL+-JILQx1zuTQrx9XfF#kcTv7zPKRS3d?D| zDNxLWw;r#gCv*UNZ^E^ptxX>i7-Xk6=sd$?fp`C3T zPRX~BmsDZJ#y-Ikmyt|vf&E*TZ-Pg{+aK;Tk^%P^vuGeVY)D_Zl5Yp_efFstvEX!c zOFVb-;S@CN7Wl-K8Fi{lZP;Jy!(dDob=(^(pSm`@OHay%#>gzV{BnZuaj7=SAhA&?fD2W3pP!>jIhj zAi~!=zf7)u8?j)wJ{bZZS?zbwdhzOOPjF~-Vs0*mSOX@x5&52w`&YO4MGS)?{7Z`x zSXocqW(XHYFi?OR-IPJ+sOwMP{m68h2@&9{DcQNjZ)z5GN$Xj#I>L{$+b&-;hDW_e za}8EUSyrZpSsfh5)*>8m1IX7O(JkBnOZ!J5vQI(btIrar{dlV|uy;K)Tb& z`R$s3BGv;W>0hM*rrYv<^TEAvvzr078Ui+xpji(p&@U!Zw;1>eACM*Dja0uVgjA18 z9HOW|1_Nxh;=~kdvfB3hQ>QCL8Yoi}cI(j>%wv!Mt)t~|l3zqvWUC+VzR|j>7qo}f zAbnSuPC+cSCdG*$At9W6xWblkiHn%=Z1`OH3H2up%0h|-C@PQf4(_MUoaEEA8?R^u zBR4;6QLb2nxVg&nJZ-ZsM@QUWZi@2uKZTW0P8ar~ZpIW|t;OM>w@k7P@Vhjdwri8w zQy=%6S42cn``;xxP8mzq^P_+NA@&3G!g`MFS&R=zBe`YY2>mj*kFP_Qb9IRqXiNoo zMqB7?MU8xJF@4-qa6UbirKa}>>Wi0v99HSyMc-hE{=|M14k*fJ%K9n3qK*E8^p-`~ z;XZ$PF*OP6rZ(+&zqUKXkj}o2<-AOV-|`swuA9C$Y6ZQoR@w8IZL{QbbtOwrwU96j z5i49Q72&u=6W?%AHBqY;h$ivc-!u8bHh!PUQFFm+n;`UR3)A@4mALMUaR|+PYJuXq zz+Q>_41QdenbnFw13cozffRrQj#iq<-4Q)63Oa{tW!qkmM2)O1#k3s#@$Mh-XM;-1 zZzCKW>q)dfYmGf+JUTiOQa=MT(+2NRDylhu!Tdne07KUbhCYRYqA`H6l8$(1NkWQSh`Yy~qdFFE?ADK?Kn%U5MJ;hVt zt{2ZpzdFluay;7V7@@ME2+Kzg=D&f71+FQL$R8vvf{)7&Cy+=||~s{bM|{(LArm zb7v$m`Sl*Mi6(@AnZo;vUkI8vmbm)-q$y(kPPQ_r2~%o+hdNAjmY#H;r*8Yi27tHa zN4`lG6{>8dW(BIfc>sI{CK?t8V&_)Q{f%6tMpUfl>%PRd0KGc|tnD2=(Yb1CDaHL0 z*Ac6~#{dJYyFWsjw7o&Qf2*vVVx%?9lj5#7Q6O6dZyEK`gU+szv(&d_wiCthk!ij_ z!1@p<>KghJi>&b~z&K9nJXUck_9_|NGHZ2)>}gS5tv5dc`@$B-TAc#2De=JMVv2S{ ziS#kGIN7kr=QosJWI3luYd(3VydDoStX8@yTU_AOVo)0?U;3^LNFrf3@4`P-vocIa zVnrX10!&HMcp>!l`S^lJ;S^G5o^1Yg&gn&BdatI8%Bxbc)g1-~mxOYTzUi4hpG=?g zG-HVa_MzSfY+bcZS;^3MY;#r;zO&-7P05$?p8&$Xbz}dvtywH*m&e*+$c!Kdv~I_R+Ge|WB;p#nKtB87uez>B`6F$P!M1I&Y5 zfVT27Out^PcUgt@FaRr|fxY~Y5Aa@|ILv9?ASsS~HEzAhvj{g9z?(*%EP*c1)?w); znrSm+MrdJQ=yEMkT;ZoR;j3gAZY2>0{~Fs|C0t*iPK^|}Flg94-mG=F8chNUIJauXBq5`y*iIA1&^EB?JkV=1G;|)r`SUOhF74SnKy(=boi? zN%&m|bXgwlHBSDkZ;(qGd;`*XdPoTZcTPx0GMto3y9$y4O2)Jpk+>1=DnYYKJ_@lehHF!~||XcyharKsnV z!K01t5eExO?(3^XI!hh-wclMXJD$p)mNam|M5+Zo*S~p&bioo=b33@)NCium45c9M z2S2_Tc2bvcAA}u?GnauI{6gEO2-jD^eox1E}&{;)0I+|2rvz z{iNeedAXpw4oI+5(bLPCT>*)mQ1BPGW(9zXiP;8R0`Nq^!0jSFzKaa_Y-7(b^t|#VqI4=#c*rsQ9LMJEHBDEHj1Q|PJ1;5V z^w>(TP47JBy;d(EZ>?RaK+FBY42`zIDmEeQPX`zsMgo%kcq+CoOheLk0ZUCR>5%4+ zdg)Of8lZB1xyghW<`FmXR9u6)SmqH)LekD*WZ=!Pz+v~CWK9Wq49TxtoCB!x#n=GC zOm>GR)5?tN#0Sxxfly*+1C8Qy9a1h+2)%Mfl$jhl6?GG9lgD-;RwWY?Q=5xC>pLWG zSK~inC;_g*ewG{9rg9MSxRc)B^~-shO(0q?H45*yRE9scmtAZ0;qv;&V?Y?fu5S;M z20xrJi)TuNkqdwk$*|Z59^mrQ0wz9l4=e!h{`aN8LL!7cEU*Dr+BvB~!KIZ~X)m~` z_H$R3$K&eFVX5qEg@#Y6{MiD^%F5blGc|jb_JSK7QfysnAAJMQ-7}j9gJ+Nc+G8AJ z*r03UXTD5|xXJDNmfznusLivTm$&OyB9Jdmw=&WlJEKTRa%@TB-dTvKr$nFYbNXiz z(KN|k*BR7|JO&_rgc1}$47X-2H2mJIgrZhrA6PDzb#Y7Gp%C)Z?Gx6F&sdk!Ma*@&<|m>P3WYwTC`nn#dSAqD3t_QOC5l(o$g1hUdT5Rz2X2jKoDWPz$6OAV#y+={mX3 zj6TNN#Xxl+_%yxp1%Os0NNqT3;-PbFdY+rv^@)w%@UE)4qy7@WwWZsxyfrN4X3$a&`7rSCiT_M_t_MV@lXa~Y`H0S&m zO*@ml2AAh9aZ=f{!g;JlsOjslf7NZr9+|{Q7DZWVvtPcmvl*cI(zCJ8;e z3ZY?P1;xc%1&_phFP(uz4T(2uWL0!*Y-{nFixrH71MH1SfWbp?Sv~NS%?Cz)00=GZ z=*Vt4WDxVZ1f=gN%*^uj3d&vFIyk3J&xsayyrsE5aZKYEjBfOt>`|W{WX8>K`j_a{ zZv#r0Mv}La-c#6!N-N5$S(BdNUY;rFn3O6vVhM}SPo1{UN85lKn9B{hG z(fjgy)D>p~X+&IHXCVB8C+D@mp$Hu8GPY-Ko6HK9B7FbQJoq;x#?TJ-g`2^bI|BIw zbJDzHGa8`i?Z_Nq>0~y{7lMfsZ&|zyLQzl$a8T5m!0ydN>c-}u1u`I7#cJ>%KZTZm zCwO@ihG(6iiwMhefKixpog_-BC|M+ik1Q28G!UFlG4uA4^S{1s=^SKHUVNudigM09 zjeL%)LU8EQq;ONCjmn>=TF|>qU$AI*axzNGO>l06!Ug08T0e{eISkh4uYQf=Ikj-3 z0?~A4G}%9UR#D%2La#?6$kMTlZ^6n+Qj z)5yk*uF13MRY3{VuT1x)fehxJt$=yk+domy8k{NwynZ^i}x!!TMJgBk`F+Z~o zI4#+{5=+a-h)iow4~|K3bGE;*gm!nY0F?miI|fkUg0-D&+H?;8e`- zrHchT-T0On-i<=A3f^mLp^;}`(hl+WJEn%p4@7PR9KF!lL1$JwwgH*Yd@!5^SOPR= zI(oC~*>?@~Ug;bgE<+L|_!}7$sC_^Ol+hs7k6Y=+YGCKTLCc2SfN~2*38x?} zq4gg-krJLGs6^2o=i@^A{Tas-ryryqi?0d9=&&0$kap+mF<>rC(+dxy)#{!RLHXom zKE~NJlm{^2;{BAQW=dl<37w7B>3r*hVDrg=GN}fj2@dBxc+?ZP2=q!e{xrz?UE?-Q zCs!Ng_kK`xk+-X^vHWYw2yT8lmzgtQ=-lQqurV@@GA?Chk;ae-#g!RfB4@2 zJ0FC#|D}-(wM`8oV-gVQf=bdmP|TiFZygewJL(1j5rll8#cw@Jk0*m?kf)L~W z=L-xi1e|{ge0Y`}2cF1ES>{d0igU5q{3 z8Bs{SKuHv)COWBp3oAUnu)qxL4}r*20bq;t4Gdfx+_;+k(}FFDhU=~?**RIBG~9#? zryB~SZ%S(viPKf`x-onpjh1Ot3y5jqB8MqO>3}U?=B7dV^iOUn2*a#q@^L2 zOhy4&0AHZ74%{EVEj2N^bSnhtZyzwZd<7`L0J{K3Ksor2VexrVKR*n5vore|G_&b5?XR zp%5T+gac05;4$kKT7uI#yYiPnc8eH?RaEdFr{6{4;5Lq*=p8AROCeyZ*7;t!f|O1e zpklwimJgJohiyR?)fdnRtY0s{xg`UF!r2Ij(d|D8z{G?YY-72HEr4B+4_qeFZb?Nb z|KYIcC}I$=+#Bav$5L2+iv`kbjJZ08XrL;w3Lbr%dGC7AUtj88I0yvD&zWpuVic}o z_zd$vJ?m!4qot1xmbwG)^))~Xhv|iRCCaVs|NUJPN1#&IrbYsde5Cp% ze_?DKoJwFx9W8+ELH8$005n8lz#I&_xwU0o_~FCn(dXdqwYx<`&z*zjT`wRqxqMgV ze+hWc|G$T%Rs~_KOJOx3&xWu^!K5jU0Q<@ecuaX7brW#>(p|{-Q}`{n<0QzS!$^BT z31!)}@eu616<}(1e;}ONfe(|k`r(|ytUcbA9TXHq=ZsJO_W#t2e`TD@rm#h!8@!MU zQA`jAT!so@tOja&QXtu+ikuS}4tr?u*sz^-1SGIPG!W0SSheIKMAadUIgK0~8tV2~ z@(TyY^j?$+2FHJ-vHv3;ElVQc&R@n-!lIB_yy%Cc#PRx%fF_}HlyW={`^~&l>~v8d zzMuYfwfX>(Q|`#Oo$1ay3D$U!Nx&T**m>$>vHyhWaXy1>pEF@JNE2pMc97R;SFNa{IHxTN3PCq0P2<_ z(7v{Hswru8b#)!hmqLt9$r5m(7V_Bltu^5;|I;M+o6qrrea=;U`UBP>2<@;v&;#Vm z96~}xWgU`QsZD~FpxP449YGI(FdgCj?TM;ZsV)}aU}Z!(6nz2ijA)VVzWa@v6N$Zl zPoew{tcQc?#6cHUQ<>`5uLet5U)S)dce$N3xL|g;AP-jg0NaN2_d4)|V z<}U(sLc+zzUtLc#=>d9PdP0aAzLT&F90);00ZA%WgSt|!_)r4XzmGgc86DbGb^VYr zSlDj$xu6yjB0kw1kpoAjB99JWo*O<13JykiZ4}YWCMtSY5a0XYuX0-v*79orwO}QP zBw(}%2hKY*;H+7OOq!9A(Z<*hhDN_G|COWs-|Y(yPCSBn7@4c4=7}yV{!$;X6i4>T z1xhwNLBPjv>vt@WzcB?UT7x3P&7-Q=p;+g83U@o~N*s2^y$P^MG?z9`GA1 z(X!de&7zZ%&XTqTQ1ZVkybA#>|3F~pzLa)zDnbE^zd#5HM)n7;j6pz%$_z++ZGjHo zC$hn%)S3Vmg#5~!~Mxw6ERlu)3DNDA;U9cK|i zW`D)A|J~LM8kEhG?yghLzBO$7&f4=N0(%WhJg_-EepEOB*2M%bnrL$9Ia<3 zCns+jmVo?X!GY640~VlmkBa>sndw&&aU_p}lzAiYFF3!E4ZH)QImC$nlJ;sy2t1`H z0igv%Q&^#=A%Bfwf~mJKQD!6zEK?ycIWJGsfAyLEPZrQoM5DH9Y5?e(6`n_?fL|k- z<=EKR2$MKEJ~@Hy9)OdY=YI!IdsOTPbTAS#aCW>Kk4=`N4$_d#1AC&czyE82Otc!f znHuhAErzf|j!_T+8;41BmJ}TkhTvNO;fl_q%5JL<<`^6G7F+oiDDav{(X6Tc-_Q;F z$?^bf?2+dhVqXZAQ;aE0p0w4*Viy)RSahzs7NVwJ9T;DkRakzE zT?}eS7{0#RYNXTpiSbvKl~1s}z5I)6wAN0CRks2Tc=z@uGijFPps>*C9D|@_TjK%r zpio+2;d_F z4#M&Au^MoY)%vWZAY=2_jH4U^hJbN7qCTcTUx|?090``)?%;LPQPd+jVxO@cy#>>) z-!sA}0wtf_G1?~*q|g@^7vQPr(q|9cbv5=v9A!`I1IDnRkN;4091F%8({R61mi*{RfaRummBp z8U*jC7}&cP$|C?OvYY5E_{DYaMpxD3ZVq_D01&&TDqc!uig;BG7N?EOn{KN$N6Y&D9!{{0J+e??UZa$xL zd%J~($~%yQVx1!qalQy-p^5yWTiFRz0g|nqYs|3ZSrI^w+LeeDcuLoR&BHLoS!05Q z36CX@i-3a9S^*B98K6tK3npK!Cq)etV~)AVn%_-lS8kMn(5bbZpae6^d;4qot9}uwfJcOw3@;^ab$>55O1v z>vbp=1P-+3vkq{TcA&aE{tbui6@OW#i!wINCm=Ah-*)5Z^D|M76_CUuMY#8jA*(8Z zKM{j>qwv{mK}HdI^QiXWpZ5wXLAq47k7mCB55yn{;o-u(-ty_z7)=w!UOX9Dta8QGTO9BP&?3s32q&Owc3%>wh5~3eRDrm?;k@4qw zh~Hu#mdw<-tjU6_%}8NX?*xY}{>n4VdCbNAmJ<~q!KigRtZRT11V=dv5s+yp&KSt2P zycr3&Hv>p=_w!x7BkVsEyctwu{mX3!AK9{@i}x3`-UE*th#2O{6fOkji25 z<2gXejA%}6NK(&^kaNT`~r27Q2nbLIs>mZi@ll+VWTM)-CSeN-4}KZ%3`m`a|Ga2|wjQy;^|Ob^EaG_IObl=?)Ba4VeVpkKs%MlA~@)|uBHLkTK)ZXFJm#q#)mWsiQJJ z4Ga*3cV;Tz-{A2FM56U?Co46&y-ZXnmn0smIxO7qkp|k@bPP|D7B^(Rhh7+yPG$>x zs5Gkax`Ai`VGyTv0@cK7OD@dMS%yO+IGi#QgyIQq^FGYYJ=+KORN^B>)J-l6KvsG8 zojB9)7)c+j{|bq86`&t;aTf!Jn?r>@7R^hKPfBVJqUACtA()O8zX7ik{HmlBZg;xn&4g^)xJRsk$w0%Uz zjfg#}kYTbxaPaIXut&ytUe!U<@rK*)b>zq=vpE@!VW&dn#@W(+)fxOSbTG<0x zxg$1!HC2NB-$ZWKCW*C*n4(!=S2~TUVS3%T6)fOO0FSES^0Dj>hbROjI$O_~&Clu{31H zw`yhs6nAUhSbVky^FSEL(3mrNg+ z%op4VwyMKI33d|fj!NIFosg{cDB?csK}%&MB=y2!P-%&RPn7-<92$}Ddv#1lPqSwL zS&fD16YN#?lDiFC4lsmRC8rmbA_PB~byy+7P(zxWSh1>P#(^DIm$c%lyurqxRM;U; zjqr_$^(yk*>J2*yCNG=VOnb-&Pd*;0$Zpcs5}kT6f)({@m)~*z4A-*V^eu32Q{E}I0rGjnGXUS z794$1wdemF&kzO>@<@2{%xSUqKxnQ3!LMlxAIe8f;XKoFxf-pbdaT$9WeG9{Sn?D8NWddBHtxsuHNZ@ zSm(QVX0Dm7Z)!O8buBj8-K{CTmhgH7f!sbf zru&!-MHUM!$MgYZaKqV)O4Dv^yf+^E*e$A4`MdfEpqt~~DWaVwBJq?x3-jXWEC%7*d`=-r~SREvke)1Cd7)+N$;|C3d+ZTcez zurgt{PCB73+lX`!$<%1lqKD-JLE;@8rX-XtTgWDiV9BEG4fPx_X^K;@*v6gE^#><# zIAF%yYtA{`+BuHzZ+iVJbeZT z;J*Nrk#dshfcukNMzMhto=NQD)K{STsNeTIo7AD%3ngEd9ohwb#pn5w4;hS)dF~l= zrm=XI&`27mc zW<(-h6dSJZ)cl*s$t7`yUW96kz`3)P@Fh3Rz+ub|*Tp%-cRMQfiX&bZGEur|KafsQ@Yn;>Tf9D%!WEB#3fi=q$q5i!V7(Ozuc-1#N!4X9>FWdPuM9_%O4Jr)XhYBcL}AKC?2e;oOgB(EO=G0J_!xm2 z{$z)nbTU~qMrFy6bk3Mb9Djw`wkSA?T#9T5(WiP*L61tpsZ5WyqeOap3hD98 zbvk>{BgfQvhzcL`^aT?33Q_AQR->uiw-Wh8>S^DsDF!5CBun$DWOk!Di$2~?Y~%nyUxf+lAe;(LB_Euks>fCWwc**J z_|oQL)>834D=c4-M|%&+VI8I;b{ZJQdl{BLlr{_|+5Xz+rURz$i##Lu-!y>IuCzVTgHz*{{u6bMWZ)1VsgRJLW8JN}8$72?fE4%AfCNyu* zCFt!<1atiK3_SPnFwY#Dw&~}aJ<1c2!>GuqMuAWp&vV9fijg15WHE7SUAi_mfjksc z=nTT9#1<1=S%>Ch310G#ayAdcnNsXPw|eUp2tCuH@pQ9YDNUQo~Ht0qAr|uy&~+>2%7}c%F@C6Mjngnaa^+>5{X*p7)7HQ zrQS;U65q4+ba?gg{vui(&i44R4Q)8&?n0XTDJ1@kb}MODFXVS$%uY0%dzr+rn6W+L zQJR|w^P2wh)wGWFCh>{Gq+}j~Dg7qzCgrRv;U?m2mhlFMUea#v4wcB7;o9BUmxVgh z?#CXj)vCiT5RH(n;&1$22{+W!jAlRYcZ}ji%H0BBMH?+Ce0#LRvrrBR$6eoy&S;ol&OjMtoU}ho{WcsK|3}-HpETe`q(Q7I@<3GYi*?uns;TyP_0rTKtED(Qi>!{y+H~(@&(z?Qi=PUicq&KR;*wL&*$CkYY&s& zu8aZT&jr!76|A720vDWlZf|brSQWov0DKtRfb+$Saq&_4andc1WnB=mz^(q`VKCI;Wt65 z;@U~sdgy%EMf9WXPxct4YF-J$>mUarAtX115Zpgdb(Zp+t+7eMsb_q@HTY({;Kw`i zBOI|zEBo&xPKKUe!`BECsf4gu7r+`%o5aP&UWMvQvo)-o%E@>1F-l3(yWza=g#X6* znE@u|pHHNTR2D(b8ToC7Z?Xnc8|HL1JXxXVM`{rFW_lj;Hwuxy$XW%u1u94iKFy;Q zX^Ng_jZ_M5RJP(T%UF^X8Dxm9qDVg8B-&)m@N?Vv<}!_x@gmWplG<`yGj=Dfv_FJ1 zj>gV&=IUJkjaVng*LUD+ziQslt=ivgiul%ty=J(26>Qcc+E@RI4ce%IA zHK+elTH{OoQ3rBTEnU%6EtoSEO3jsf^8JwG}ZzC**%o;Hb9O=vAQ`p)%*DC@ApG>X^PG zjYC~jg!7UZg%ck>8U8R<)=c6n^}B8vX`8+N%ydfzkVs=fJP(&wQu_o8QfYQFzWQL! zHpm@ybV*@j4Ak0ADn{hnWWqlSM6~Cob3r(+_$B!0y)YfAk-$&{JKD~2^giF|(|@%M z~&^B2@-s2MFA3j^rMYt{T&$ z%>!H$3WMM*egv1m92OQPrms)xi3fKOC$}W#2VW&7YG%(_d!*n9Ef-!zU7yxm%w|TK z$dyZ3E$LHnQfU5&ZeL#kZN83v60fVLgd`HbGu4CWwQ$v#7*cLePHChn4UwR64{{_4 z^OH)Gs4TFca`1#6wiSPP?dnI%r7Umc^yqVGvns41W6B zIkNvMvbb-v#a{H5>hK84KFl;MQx#Hx9eG&m+)}^|eNyg3l%ZpebM;s|!XK1XE%T@d zDVw<~UQB$wB7D;bv&QDmRAoN!^FO?{hvMDFSeE1;;qB6JOB0kSOYg>!3Co5u4hP8j z%0~1atf%ln>$}f7UW?O{nCfVSd6KFJC_W`|7e-Ra(u|0H8T&#!#6g;;N{fi(>&J)` z!rqKEvK%h-{iG~2q`a0iLD!+9+fH{sw4y5dXVjWrwYcKZIIWvO<) zkR&-UgIA3W*f-Y z^-*`&F1V{%H>DU{P$Z7Q2rF&UNvyNGf_7qRY`WzZV`FmyDoL-f^q`YjmY{x8YyQ7<* zohUx`j@b?;NK}_h5H|{8EN{T_5c4FG}(XKA!zhe9?qbvE!LgbcY37JUT zSZ^ev2cBY>OBGI%kzOjs!R6jX1Y(u{jpr(1Q1Lfvw$l=mG-5#xCD3Ko2f{m!)c zp@F_rablaUuZ@$ zJuD-0x&hNd*VS}}8kaVjQ`dG7jSS_Wmck304Ue|8w=DEm%`On|FefqNu&`FXiYD*8 zU51abx|~V9j*kkz2fatWP z8`BaFJ9Du8;Q!27#4~XVdgrUMfiW}%#9nZ6KQGw2R~#~aP_mA;5}!#As>k(2-$u_P zoo@eB3H+u&e#xj4d;-!n^|5^%$qB}>JOSm^uAtNfJv)C;ErM5nE*v#nQNQ!8W-wMb z_wP6kslJt2oWw_n92+ckiC(2Ss zC3%hW0?J2vVm^O)?k6RF=vif?_-uS2cvk||h1K5D^i}WBjjs=)#(jKD$>r9A-a5f{ zOifXsS(&`EzGL8zfqqHQ^|Tr04Wv@Dl0_5E39_`Ks*M7n-YqHF=h;@3q8nVT_?Ry- zuQ3h#d<#oo>Y@xrx67s0h7rDjliI=NroXc>&vgmtAb2D-tiT}^-RP}7+QM~}D!X2t zWtEr#ma;8%mYk)CiKm`ZmI(ye+ZWk3@6=kW$LG*PE}2bH<}*LwDSaODTxakL=&wEp zit~H^p=#g!XiF)3m-?!tL%NU_81jOCcp8$3xXHXG*L24fGt(VbAJPnfZFr&}2*JGW z6U?A40?ya=3WKmz#gIz;713Gmr<+|O6i!8{iahFH@moCj1FW5m{w?*v%%ak}%*0VzGmUrMS_#mXH?uEc-pk zF;)cH&HsAh;9pBn2%_%o#bQd}%NkZi`ewYf83&y|wfwO5t0;#EpSJb>PI3&z%0wcl z(Ayu+kSHP^8guYh@$|W#TvHx-ab(6UC00sAN>`<{w3SEDi*V4yP*z+Li!S!59kG4F zPjd^25h-7KmnMNfd+zM)6Ri4dkOwp~;(+R)=E@#MC1Otc^!zZ+e9J|3k0hhT&^l28 z$qu`x_ub_s=#Q8;bWiz?NJziET#bhdhWqK0b;Yk8K)oZiG4y8lX^4E)oIwq3WO^l+&LFNT+l8xY2s6-{AVjVEwx9|$v%;eGlI7{Xvs2UP#%sjtzJfZb+P{~< zTGyuG35I8C?C08OVvw4^X(`C0m2?&{O>>G@bjn?r?u(OR{P>OG|MB(KQB|&6`>-Mi zNJxl?B2p4k0#YI+r3eTHUD8N*rywA$gmh!j3s`h_gGhISbW8u{a__Uxd*1ha-xv-D zf9$bkJ?pveIpt(V5Wj1s-hYc>p9Hx0QoEn;;;pq z0!ST*sH$pmjdhlExojY2n+4_L$u|^uPowQ0%cyCUHZ5sKBl8VYWJk6X-Or6Cruo_O zdvwi)M}~Qu^%f=tSZ`{d=!X2^C}*TR&M?C%r#-!wX7Wx89X5pxX{3UmsINc6GM%W` zP&>%&{vwTr9w;aHK7ojL(uKyGhWfECCtk zqpMFi=VKk8g)c?&@!Oo<9V!BV(ObxAhF)T951X?;~V5+8KshU1?b*`tOmQdl4Ki z?46}tP-cy388-_ZH5vVar7TsP6H5f|0fAjBM}vdlHo}^)iLk)+5)r|r)+xO^H$)H` zMjE*t3Ov>lZm3b-r8j}aM8xsPB53iDWXyj%tnFZ{8Fd)ngr<~s>VBpQg-k)`OOiP{F z**;hM<;iodglE+El$V*1Te!2TA}#hN0FdLd$gk^Nk+5Hd7YPO>6{N|sm|Pm8Fn?9 z(DT!|)B7Ru>!!L;gAXH-K}(z(`mFAYRqvFpURn>@^d-KCF~R$F+v~H8+=Y~6d-o6P ze4Z_Tm9_TxxYCnvUCvL1bC%O4U;5Q9rR}n-UbMNg4d% zxwdJ(zW3SQtzq|!IBGTr_0EXNQi?QkUpEj_wygIvlB&dy!%9O(6LZhkqQ;2B#Jzq& z%wdZY$2o3-Sxbq$r(EI>`eS)lSnRgE4|7j70R26kTNZ5sz7Mx|gLV6}2V*W%(P(?*;0lU*= z81W^b{lryhMQ?@j{tRzOM+67u%t@pIdN!1)4t}9PO*5(9#>z z?n@?-lPT$*QF%-J?CxH{BT+n$%f0WZG#ycb%YW7TiAJFBIbyv%p1+u1h@gug>_#rV z+op5E%ZeRegr6=hk>4trvQPpgxt;r?QOYDOU1cX_jW?6N9*b+2^1Y<1msHe(9Zhn4 zU?uLuT84^1A@3MiV);JdU5954xA(1_TY<*-RG*BAQ+*MNGxzDDKND^pf{;;7;d2!B zEfPuZ-4MYehQi%9x*LlX`>?7K(X#oWgJ7>UE8v#{wcy9szf{M45=$|)!?260j=y-{ zqb4PDdK0u0w?f$4!70`X8{o=y-cup0WvE?Xfp=IZAyu$*s==djXJse9l{c!QVz!TV zQRB;R`p_ip*o`PT>6gD|-M^2dVA0B;F+Wsheu0xHPTQ->OLAC8d(%%G<1TSUOGFwU zPLLD|pTX*>_w;$BXYVjn|L9F3CK~kxf2=tHu{-TzlvLPLN;_>B@|S(9m8~@x_hm%7pD*mk+(ZDBwoy681_AOBL;hJ#%0mu^!6dZr_or zvt`+lIuf!`RO8IY)3WsLY6>gbRWy2IgGsA*6L!%;r^gOmvo2!0wcXxSs}_~k0%2V9 z{*CfMjEL&`__K-Kfz)uSMY43IN?f+47B^l4hUu0=GIdeIraMhrk_>@uC2U)2?D`!y z?C~8c+X>eBgb2$S@>{tUjIlg4%&l?@bzdFNY=rNo*)Se3ECl#H%b-^_na182)BM~A zbhEkyA>qj&f3 zGC!8S@xU;&JwxhA{kP|K*u(b|gg=KK?~HTDwFLY7JKjB?`4yXrNs=yBPwbq+Pw{*+ zCE{{7HA8?C8Yk#d!i>4X&}|Zh5nl}OK&F}X@+DZtUmAP?4nmXO6!G<{Pd(j*YC;;r zKfgA~zR#wYqmG*q-R1Ir-BzlNE<-sXX`@DXXN%J+mzHy+{mILZ2cG_pyb~1+GH>7@dnE;n%NnyR=q=)SdN%&kxMDw|&FF`Vx?$ zJ@Cso!J2G+L>_#TtBvb9y~_1BiAzFa++xK`H}xJWef<>CUeA8Obp5PKU@E1YrSdsC z)&r_hBHtVHo)sB;Rk};6Qa{f;FJWr0DiOW;yL%+?tfBUh6Q*m2-OA%Nj6#z7SBx>- zBxpr+Q#&Y(-)#n(FuD6-2l_AzDR*E|I!qrpoNP6xc2EZnN2VT_CHNEE+icXrV*ahs zf)LH(cRMM`{9eN+(OQ!y@%)YV_*d`Br67q7sVnFoXNshACE9YRLqCb%8u-%X_`w3! z9g;ofVDrK!*lV#9yRpwAvfLiK`p_K{ht_8{o83?NF~Jjy{GgjPSeP0|_X(N4&i{;| zL-S?h`eEizk+krD)WHEi())7-k`e8x&jrmGu^nPU@o)HVW2{X>y<{&b{t>)2L|*8^ zq@HHL7Hhb5N}-h2NBZs<2aWu3?BJL{%4yzxNj1?dQMFVttgDf_IkOE(@=m(0#oGLC zuB`I}Nn*J3&j>wyf#m6u*Q{^#{MuemOBVfyRQp5p&&gcCWPgl1zno&cdpk!lHz9LL zSRzguotnl&SfrZO+i&Sc)h$Hh(%wFJ+soQ;044x(^+S62ThOwoM^A0yCh|u||0bzl z*&v-T*ld4!6lIkL%AH8VxKK>_QCWG^VDXMlmhY_t@fKl3W0Ab^tI@$paY{xaVJ*55 z?~yKDgd^9ae^Q;upHt;IL?m1&d2Q#{KJjLqz~$s@&1$#xgEv7(r{NKlG{G1GY`@xx z*Pc$Id+9B4!IoQk0j;L}nkDOLqNGRNph{npiU~D(dXBD1L+Rryb97w}P4Z8tfV%QS zPq2G@{XT~3ewTL(vSmE2VPJZCevgB9iMWZWg2SUVF;xEsDH{4i;udjRAC%(3;4WIS z<36J@B8`}~VkaQ+v|s2xtMw1mn492AFoyDx9D{6>I{wB*MEF?xooQhE;_gt$2Yt-N zs^4wMdYu{REI!b8|FEY|TCCtblGJO35dV)t9GEhlVZg(b54x+vLUU;ff5weYFCw;EdRA1e4V$3uzG z0TjL1CglcBL^}|JmWAZsj&Zlv*g0kV=N|rRjwJA$1CyvfQO@8^Pe1f(6i)e(+f$5i z^dmP>kSzMi^9z{2Vawx`6X^@hXy)gLr+;t>f|qE{`>U4G#=9j+^8CX_V~U1pYM|df zF!xe#aZoML4`K?a+o&Wx&*<=$MxO}~h7nrLU-w8=F7gYA6bCXDa}D>F(y7tzMq>Z` z)tl<;{05>Qf{&x6rQ0N1|G`TC{sB0p%|P$x77%rr^Bt4K=>EN40*5ExUM6hp#ZmZ4 zD(c7rFUpFfqMSl+@RQ3({%Ureajme=rJjf~z#9>!8@;Lbf#&1K?|<>)j2Af?4mAWI zlEv^cUv;KjvR7|4sG+r4Q{$?XuuMRB6ST~_>qh8Da`8wjG+CmW3&ECKedaXSFzWm{ z&@dI0ns${Kt-&0`|b@D|61dro?`<+p|V{+y9Z*nkmkShkgKi}>UjrE1>5R93h z5A+G2SfYOEQ$V9Y3tjBvl_9SKs-(4A%sa1itv>LdK6s;v=5vL_D(u+Bzs>TiwI~!3 zB>LCo2{1^I5;nW~a5#gOJa}zkmygiT9xdFH9BQTUrpyKxM_(DJNzsd10;RdVk<7{n zltg2S;*s@S%XHg|GuW8EB`9ku_qW>WpQlX}Bix-Ju$Emsab+B@Uw5tn=q~nebrXK){VY^2MTTs=6W;BVHGivx4RBEw!$?f$>UA?2tAWN7^q#0?P^)cE z|DfKKtmsQY-#fNS2!0xzig@fzfq%>h9*K?K^JP)t;ntO$a@MhN8ic^v@7bJU%BzXt z85QtUOu_NUYVqu!o9IvfiE2T`Jl-mT{HppCBn;E|<7M&NslcALjc zZ96_q|6l*$G=Z?qF?>1dLTFCJloh?XLBh=nA2q4;-IAoP^ez2?wU9n}khc4BTUL9OM7-ueF9;kf~L^LtT-rUOuq2=R949L7F6P{`jU(->iq# zM=_5VZdTiKt*uwR^=bbeG5-D?L$$F&zdr6S_G?@Da)Th|D;D~Vi`D+7;vcT%gI!n^ zp5_`H$}5dx&!fFCaLU}>!JfL`K>S?H&jM7aIcWsSRzr9^V_LJ#Ay@Cu-OTxyEdf;> zr2h!QKtwn{5cV=dIv@0}ywfZ?O@Vf2CywPVhlqAF8b9NnBoX%UiV3!V|6a~y_yPaY z0{s1J{&_3&>85z{S!ih+{B>e2|DY=@02IMXG2(jqa@XKN4rM!W%7}a%qJ>5`PqZi{`X-; zom$EylxQf`Q(#XYZk07DQIrziGbK$ITGjDxlxghrp!yt-lrpUCfq&X6ZKQy+ouWPw zA(R|XSr3i(upcjvf4zjJ=S#{~z+b-$7)O%Eq`(rj3Ze~}LUzOE|M@}x`I#{U_5OmW zv^5}&D+h*vRag&H1Sw!8lOg1MKS1K0V;o!uYN>3n<#AiiH1zyAvpI^s(&%XZ z7f!JTvy^qf&l$vj4*Qq2;wIruvUSB?=ArrDHR@j__wSN*kr0a<&5d|e*b2(q=vVnv z@_5kjs={Tt3h`-I)5T|xK~%3Mx&iSpg`jqQe^v#z1h-a=M@@28SOzbe@6|g<@Vbw{ zwO|6?&zwRMy;rSG9r5%bF!Og=&9xZ$nL}JO0t9RPy=xuIeJDc$zx#B>3cn~L^Uaiu zr=OaBWXKpSzw=^teUBT-{*n%5DZnsjeLIXWD1Zl|id-F5Z!*{?A7+ujgk=TT@faiX`E5eQaDKPR?~Xyt1xxPbn`n zf2u4s1 z&cT773EUZ|Xn}Hw7RUw>t$eri27#vot?qm7>UDF-W z1xioW#}v{G*_mL6`|F$+{oC_wT8=yO#ZV!|_&!$C$>xFOR^@1yjG4hTH2qmDd{Z(j z93w8d0Zd*a;^h{egN`ViD)3iCz$W#H5}P z*Z*C;Ii|=&4;pisd!iw|yt0W8^)T^Qi&Mb{!r6Sk>=wyW0LVghX!xQ_OgYa@JA|-Pv)?w5<)N^#qzjUt>oHGpE$OM^A*HD zvizYJ7Llto{EMfg>-($BH?K2aUme8$tV?$-h^_li?cYWkNtf2V1vkVn;f*W-8Z>Smg3@}!}MGtPHLV3el9L)${$lV+3Buc z;;IlUVMMWIB4+MIPAqDN(Wt+_y_#?-%p!Tj|g>4MG1Uc;K;-gYtW(j zi)(?Ig4^}T2I!!&PH^AybzQl}yhIK-PO7(d=ZI~VMD_^l%__(C0e`NawB?hXvD_nw z^Z?VXIv=D(t(xKy!YN24lKwfxprqMW9R^H1FN}u~>Ejmlcy0 z`SsI?8UC`HTMA=J*J9JOd)(+xOs^)4Lwy^R--c2B^#RP}27CZQ|8#C7m*kcHQjLTzAG=4!9t! zyvD!xGp$k%PDipJt()x*CPgu?4ng^3R?H!Q;^PJ8%rUTq&s6m(5#R69kqjAuPMg^n zX|n#X?Gt~iD>pW&Fg$z!Cymusv`_5*Xc?$Fw_?zd1DNSN)wK`;bu z`5lX$J`Zuqhv)EqRyJC>b(*AOh1?TP%bg-3a^-s4A~Urb zpGDeG-(MUHtM^rlo-tJ|E(zprf&DQI%s?ynHBS8RZ&}|mQ zxVUE|UZPX38}ovr6252!n5w(6chex3!N!$EH~)ZT}JM6EfFT%2s4IKsR0#> zfvg-J+H@7p&B`j54S?oNl%GQYLq^vOK(}$|nWB*J}2W73GE&+n!1LrjQ*2cMt9I zkDeB^n3|q9^{8x*yr;zHh2Q&)OFDvCRJr+@Z9eIxzqW)zcZOSqhyh+{R6>ZV4P2ID?foOEQGQ^TFL9E({{rr4_xhT%uKrHrN-ok@taw zNu^0TI_qiQg2LA)?9e#3fKhJw=$6??Ni9QBncfZ^Fl!;ibyu$6Gda9+%_3Ttib}Dq z2h~a<^g9>YdK-PIK2wD@Hi9Jkq3$nc<%*t&bX~y?j4cn`haB6`UB981B`-d~xN>uN z#rNuM8|_)o(_^*P8iZcEN<>>kHu&M!ofF;L(LP;BC-ikcsDw%1kl%VCtN)woWPCMN zV|x^ft?MJES6L<`;DoqB!lZn7B_hkDbla0}7@eG}ZojPPe8kJq+H}9eD97OT-Ft*N z%hpFdqF0rnQp~vE(NA3~H2N13{y%36PX?4NsT^#orjSs@DwSa9&J{Z7NT!KyaT!no zdhX2?&n`mpkNhD_%HqRFZl3W~^aSWMng`blm?ixBC>TlB)GqZ8nK{I2h%57@%xc$uaIIiY1m#PA)+lv)C z&26a@^%yOA6Q6x(Uxey#U@ls5LUJ4ZXaManK{cj~YJeh|F0b2ZCnYcq)9V!K`Bq$D zJ#F!C@yica&rzt%c%7{kJZYZpBJ*KO? zXe`F~{=%d@JiP*@FyGX2$M#5t6fvXG632jSrK+C_IZ9@k3R!tmS+$BBj%QJgFLNGa z;NW+~X!H8;^uoC;bqg{cJXvYrIXTng#TxCgJc$?>>Gs1)5t*+Fjt033%l2+6`A9_& zyC=0;pwkE6DbBAxF%$~Axr~rtR2)Ufb57&8}9OYELfUBt2sd^&Z3j5LT9?|{oaCc^dCw^-q`St5%2X5uZ zRmYby?nt)@Se*;)giYcV)oQxy$;5CM(D+_eNq+2O0?}@dY0%tggx#Bg_bJy%3AZw~ zt`CY76qSGw*$fu7``z3oN*f#_-xV_7MBG!fOu00FM^B!3sM7cgF@qwjraqj!e7$E; zDtkA%vYk3r6bcR5`7O88SkxuTb<|>do=S6T|2K zQP2LGkqJ1_wpfwzg03Sl;a@9|KiL>Zp?T06l_1nn>RVxGK}=^eI#F%D^JZ`JYB2wN z65h`O9}0O*m!q)L``i3X>SclI_Uf08!Gmzg_Tf!?4+6@}vge6{5Wyf+4#$E;F=q_v zuuK%orBl|Q=^o#oJz4YXr?J*N(<|Iivb(2mz~b1o=c$U+&0kS1vT3DmE|Ca|f!0q? zwMaWUTcfD$p?y)y#C>FRcRyF!{0d--xC(yl&;M|FW& znaJs2#O)foe&=WXNtZnzz$km~DSS0_KVk2Rh+Z8j%52B|x1aN$z72k$WkEd+_DVoo z$N0>bBt>32XoLsMRo~FFY@u$6WQo=5_}qIc2!v=(+^g$+4j%`|K6*<)aL)~A==4kh zPv4*8>>d^kv3vSBwNGy5u~uhw=4xeCrRh!v9ydhHDmDRDge@`!ib>l8LNC}j&SyM| zG%M0)Q7L0ixQus;zJax<6AMf3$o}Z3yUEc{#z-dG;{L4Hqkt~xyretV3%bc&*i7Nv z0k|ZM+m^IPf_M0u`sqfs-RhEj`BvoUYL4?mm1xBLjN|hWH5Y;#TjO%azfbMgiUysP z&ng_p@-sE7t17{+rcjD#b85oWF$Em_|IB_f?}WYlOsDYzHB3-JChjp1=@8Hr;V&$W z^(s@L6a1}4v*!u=fXw-~O72Zupz2wkz|Zj}OE=JorMgA=vj6_6^b{l>u=c#$NUZn0 zs+zfCo~JrmRQ%9(<=BySMBM?MfrszS$p-hEv5g!yC738phkrCGl)YkvaGF>y^9i-7 znI~ERpYH>?H3IOoxh~11MPK$n>B=W~B(%Y*1I@%SHIbdMt7{XL%;pId`;O<`Ysb^| zSrF2jGsvFuSAxm^{%Dk2e?VEX`>_6kXBMJaMo}KihQvw2(f%G2z-O64_1Dgr6gOYl7eyj^LLRf z&UX1yDd?!?F6buvm}~;C5ed_4gfizB)pDe1yj(b>Y|uXZl_rn{(RM>9?9!ksuBfB7 zc7Hpw5>;`ZI``Zh=Ry~B$jxO@M+^y`pWw7d&}5D~F2>sps@p5W6A^#*U%oIg$}rRi zg9M!ewPn`T2t^G`!csWDteybOD?_f{Dl{BOp5v-p;~~qg7bUb7gV*d<@L007YQ5fD zm%x8?k^2%J&o@E`cvIAdxumYSjkzZ=sWao?qFL*A=CK{Sza6%oD^%mN9)cRUy5e~$ z+fpauIIxv)P6G_Ws!Yn?3*ok}({Rr;mln)9pkqgy#Jg>xBXO>qxPRXEYcR(Qz-?Or zhSstFkF!UC!lQTJP`|{;RLW0b!ek)!Tq1gsC|Jql{JK(~<@jw+-jbX22F`snp}>=_ ziM5>!62=z|bk7r3kKVSTsGQ$jhtth>)~bTzS6dAdL)P?nOU3B!yDVuPe?R{+u)dGa z{p)xdW$-J9;3HzkS5Ms!UJ*k?BC<1%udgjUcj>6Wm3wL>O~@kXz7(1oF-#VP7bkJ^ zEBU#Vs}Clun^peBW&FoQ8tQ<4LWh)z{92DAw%tS=V!TCO~>9*xx-cpbhASk z_R?_=TF>lor$LGHJAp*Rcnro;(`PfTVW6!N4IYEOxFoTI`B1fMks$pl>bxtoj)DPrS$ zyN@l7?ONp8W4L1RP80+vh-1pDk<}KpkDRB6zpVClshcc+^x21j>(F@cVre3(*b3UG zw?`J>tqcr%xAtAKqPS`E8y$aj(wwKr>}`mhV_ntJ3}?B;w3po961)FF4WKT1YShct zL0D@F-;@IEf^7eIHz1a~a%^dOfe*Xt{LHx=5Md%hf4lf=*cMe4a9?uk{d%BUrt3f7 zk}%%(`m5o%^KRkP)KnU;ZV+P(I>XhG`RQZR-Z~6f8+S+`GKI($ZlEdkswF-pPuTg? zJY%cW1#SufnZByM+Kp>IQ5r>Md$+Kjb5L)}Ch{=T(iV5dbB*uKv_~*S;oZ1jqi4~Y zL$?WO1j8%22Z)r$fYAi?a+8zEK5vF-LI*@Ujoreo5x z<^NgH{{C55ivaS9u3yLQ%9n&}z3-Yp5cELXYXZ~=4zAbDbWJSX&X|5wme1mo=fvT) zRcBTn)+x_pcxxXD?0yyaZDG((-P_65Of6`?-91;mx>oEo3{i!vs1zDs3a6G|1KBGL z_T)<_tx)&4k!ow)1$)8jaA)*78K#uft(e zf=GgoW4v6H|EKALm@RB{A{i4@OiTnAp=A>V*T{c`pB=$3OK#oP43<$_M$hVn8!~8o zR&#lf0W}Q3&l+6s#UhjVMLd&BdT?nY0qPh@oOLgD=C?!eZ>$-XTKR}sXF92B6Al$w ze{nr*{DRb)P~&$!3g27nYff;O!RCpBq{&gZ9!&w?1@5S-;bzMIEe~|am^3;+;p~TW zxwjFS3I%7HjH8SfuIi+TCTt(j-Z~gnTKSQlS-Ic#Yyg7xj)6}$1!x#{Tw5k|!)Urx z5(u{Rw*xskRHZvgllDS>J9b5$X`)HPwzgSX({&2DX6mD5dIgymdLVxX9&pe0ON_3&5QvqLRG;)s zLfqrUn{dg;CdX6nn7Zn+N9~m$SI4wv$~?=}w>;3w8=CPqYa{k~$hD0;sfbLmBIVQZ zZqzm%9h&!7^EZ6)@?(nLjxNaD8h1^CNQh(_s(bF>=|{e)zl?(&X7OuL^jc5(N`Gcl zXKZy1-5FnPj`7ft;O$LY;cm%y)sZ!^0?tbLGk4WnD-v=m472V^gix%ZI}P`vLrj?g z3YZ3}W#oH*X4aFi6ovCLiT&R%yRW?VAcoB4EP;tb5!Fg7pUUl#6{PEm{z3d9D#Qvp z8Z;NXn^phi-b1JJL?<2vq*S{xfYkoggxj&{;Irle6hjIm$lA%_&H#i%(>JY+m-9Rb zn1>U`6PLk3xQP{d_`8F%yM*JT*vjz;tdLtxg@d8#b~ZdxR9(-Z-=^#S{HV`rp+k>n z%q2kGy>hzg2FRM?u8#+ zxN92}Khv?2UcOORurgrG&;sKwNePtA{?C`shv6z*OAQgcLfQemp_+S4vgmuI)600- zPx5DwI>&a$#xo9!Y9_tEoQx+BMbN}Q)jAc{8jo>bOJsECpWfM?NAYunFiU5bvxTNF z=^@_VmV$sX(>ibL^#p<}K^HgP7p(@Q&=MtdCU7KH`JTO18etum*y6`%UAU(mtl>ivy{rirWI7%e+)Eg5M?oB zxU@%T|2u>5e_qtT6NSC}O#wwJS{=^SCGHJa-s#TeeZ)xmRACfkDX4ThM3_SQcPa zA$$v}^WL3Iwe%P+ld+#Sxy`g2pM^^!tnmvy0SNecy(`gRXQ2yOT3|2&3>PX$2Q^hd zylmPyte--m%}Nl}P8~l|c`#v&LJDs4Sv7FL^kaDA!GsvCFac(;LBXp?CVLA0JojpN zV|p|kmS1NM3cCsuZ}l`74WcIaW(voem>@j~%r@1nN3(4BY`V*ZzELEveOMf-WEDS_eC zeyZjosUAP9r6}G3qI(3~Q_FEnK)_sKx!pR+$mBbH|8BrjIy8!lVzFKE{N)@UA=t{Y zuvShmAj?mfOr?@?sWKOQ_5zx*!=?{HCqS+Z>*B7wtB|D9CxNU$c$%8FZ=S@=O$( zUv|?&mnt-Lt*rd9BLhtmuSZw#oWM^U(ymb+O}sopKt&~Yqa5M{dMY#Pf-ZPXaq&s?qmcZ&DL%n zJha*L&}&Z^$2NwiZyl`0@v>DIR9G4P*{Xv;j}tRp$J+Lz!*T*9GPs|6HyN%Y= z{6BugvO!h5YfK6q6fT=0^vJNo8;YY4`F9GwZPr;UNHapZojPU0GeXTTFTTj?oSaU& zpBI5(Lp8nabv7W}UD&Nme0wFy4MP(G<)+>1>PY87S^6x~k&<4d`RLo@{dLRv-*BsF zeS7gG-`b$NWL4OktDz8TgbK7(zeh=y;Wr{sg7Z zTu%jco*#>5=?{&S{!#zpUH+Sgr}tMZ4l3=M;}4X8o6$kwR1RK6*OxwS7Cg*L1b@gN zVC8Q^1ol6pTGE{A%v^bn5N95;jx0u9MdBoPQoRUeRg@%aDe6`jRT>Nc;+Md78x_iU%x*9dDSl|gN@;W zW(=CGQ|5nba!^Kj+xus!0DPz0T#Kpxt1r2OP5X7|JxwC+CF)GqAgeH}+X;~W+M?eT zw|w>#I}Wf80#5GV6D>ITJV!j z=BiGQvtiB$N(P%rsl<_Lf)qSWDG_hY`I|^%kX92uxlX_yO2Z!r+8<(!~Q5{Alx)%bl zJBLs-FI^L|tCuNm&{cFYe?%;-o1``oUQIU)Kus3#q&e*#+$NWwM&5R~;e6IUykV@WAcNvE=58k>VX62l&*yuhvcCVf+HwaOO?&)}KGr2zJ0AFI6k=$Xi zE1@7TnxoV-f#1EWD^Ac9jvx}~yb@f25^CK@*nZI#$vDd)Im7vkg}WIW^BD7vixxl$ z0iDfz(buH4PJVCg|9Hk$%G_@b;nMer!h>BGkt-Desz(x)=?X8%cmKEi68;#g3GXpD{b53dO z#ceurG2Tu#lyN*Fk=3`ydbgTU+GWK<{xisEAzT*DLqBO%ehnQKMzZ zqNhKj{M3U5`!WoKPBz~;b5jz8*aF6|W&+LWe$y>;C162`ObcOSWDyEzHjM8B3P$Xf zuian4ms598`T=N(AP-Q~w`X{2mA1ZylUl|i>kh>3!JO$YN7yCQH>iQ;b3qx}ga`yo z2gA@B5W5cTVx?TA3Mhu!R{G*AuZ!arr#pQ)t_Yqdo}_M>4VJMH^Xp%)n$EY*Pr4({ z)ehtP#(p2Vvz>Y8H3u~A!N{W)!spF^UIhv)zUwe2*{0mqV}*Xx7+#;M?Y2+Nd$R3O zJh>z-iIea-K}@3e%*HJY&%^ghTRpo_G**Lm&0Omz^`F`hHXq-|_h?qc1e#D*tTZ4v z1`2_!7)vX?E$Gscamv}4A$!;QmcLEDr4XMm&FmzICZL+ZY zE$1rjfyBChwn{OAp2B@>uQ$;D6s75W8v)t4JB!V8sf;s~aIuyVvrLOdctVzvpzMx* zTga`$RS3hN^X=gdDJlqC6)2@_lz1lTolczE8c;~cX4mtLym)@( zD7Xbpjt8=lwk!Jljx%Y;XLpC@rGz^!wucHVmxgC*VXxVInvjA%-3 zh>}Lvmh!;BPGP~3MNptMT3D5R43-S$^`-BU;@~B~Qx>CPDD5cPSrl~+rz)h1qx!jp z8pWK$*qhNX7Q&k1LawFvGyyc-`@(f&8rSG)mne=}kK|1wc0rtCcZKSksPpzIoFSc6vVy0RtV%%m{29`0|X3D3&$D#AP zN>lzT+91L5IBdv{r5ED13_*EW2qG4YeCmv;kWtSI`Aom%)h-zN1|ciphR%l4%lrRQHU$^?W*%q5m^?~O!EmRZ^$*h zMzEI11YJs|9sX?;K&QL-eR%3PDINoYM_>8P98=I_3UJTY3Q+H6&@-Juj%FUpV^pJw zeEzL|GjK5zMKOV(G+Vpz!&5Y7SAXRKcDoV#nt(RJGt_7|401oUJmQOHZ-S=Yq6t5J zs1*oqS~>pP?i2!Re~2EUU=PhkZUoj7=f(Ehpph%d%*@hX$F}$Q)E;R`F%Djr9D~>f zk<9EI7H||GD1jUwFeS&r{q@DFsgF*ByZ7e;mt9KqQNj+i%%?XEoiy=SKydZlN zIdWF%tbQmg#z|pA$Y|v=hfHbwpbSbAb)-r`hKw5asP_Y>iU~lCSnCH9(`|^XWVcG! zP$A$Q28aTOnhkinT+9RqMpNdxA^P349j;s0LqM7=V?-^*0Sz)-!sCmGX5L(cUbmU@ zy8;cuS`Iy{iz?xI83)7uv(4jspXpw|RdgGEp^j{i%Gu?lYBzx~PN7g~tLH~&^89Bu zH46dhR^~5bH1nTZwP`MeR%NOU>Somf%HCkL*fq;>X=lEDv0)Y@3@X!)ea7|@7$}Tv6J6Pn#>Nr-{ZO{Is_9^7Az}cPoV66fZ8LW zY?(Ff8uTI}JPWt`QGQTd5d^mAbm5*QQ%zBjBVODt1^Zu1Wuey_Wkn2tan6$!=Z-VCEosSDVnQx2?p zTW%%5 zqLmM+-aNK8Q=4>*_Zedu8`eJW~?&fKcm#xl(lvm+}2p(dWcOwvDT5 zEFDkjtGG*Ve*Pr!-dMVgT zl(%?jxBB_S&#c^j!cp?9dxiBKZ;8cpU5uM~MF?;?v1+6JtP@cd`E!S3wu}5ix(`6f zNl2-D?b0Ea&sS^pzO9=d1307g>sG`yXsj+?&&63DqtUz`?|!YaL@CoQ|0+jrF@u|x z+lRV&2$D}XcUYU*V{DW_ics<2#Oq@1GPbNJI;hn-f;CH)a6!dyS7}EE-}#xk(CBbFAr6k{7_TkjSb*)l@t%3BeUS}bfl5eEnKlo;`3+xEy!*!UEYzJ*0t;WovEVgCKK z$?;0&sXu)Jf&F!QPF_zfKZ>_sCvnD~*Apb+*hAPf$6=ix{eDUR3$%Z|B{%PQ!m(CV zle1CYa*AJP6YW(;f(VgNZJ{fkX)=T9gXVLYFm3EgrR6A zz)|T^k@?88!08G1Ai7V1hDVL;jSy>ps90$-ij85G#XS=GJS^pk+`g*ktD4W6|)eIrlK z6}{))LqkewX=#2#jjuB+qEkclEf(ZHbGmFUv~P+m#?^kSKU1(nBEL8*N4Fo`W%ZGF zdL7uCN$@6^C;X)2>O{+nj@h=?SM9GlI62lfkKU?stJ*xcDr_M~@c6M;WAvcXhZBYi zFDzy|a|fc)dHHIv2|t+HSXsUhZ{Nk*+`yrq`h9!`#AcgVY$TD~-!DT?3DyZP0MuVC z)&mK9uYCbx=I*q1bT@*4lypc4($ev}X4l>4c|Pa++rM1T**$Y+?)$pl?^p8h!_`Fn-$CW1 z;@hYpB6(UqU~L#nX!Bg+FwJ!@UJ~mGuYJK)8C)4yi#i_j9%0E z9+rhCLrxhPdMmYox)2_VE$%iByKrlS%|j-TlF^-VF_a>5n(`WzECu_bataFlUTdez zzCVHLh{tw>0mi#;qyP7-VJeF8)5w0A4FRs--f*whmS+#aO%&|wx`;ue^WpkwQ{`UH zD7~#BI#`<%6Gb4?NOYxsH|$;MjN<-pQ`U8B$R>Gt%~!+JTSAg+ZWdf~1DY&+va5Ty ze@oLHAPJKMVOw6E6#8{(Dixkj2R+H=utuM~f60W_rMQ3ms@Ub!d!YlKv_W86)`x0( zEX@u8Cer{hDJySFH`#w*UQD#3yA~bvWtUS4lM5OahYv^xhVNEc_7cK0c-mqe1qNfd zNBjZnXCo`WUzai{Kw@NH74qojU7W{Tce^%k*R>&?qqf;1MIpT`OXVy_nz4D%ZXuDQ zP%FdhsZMHFH}k!$AdB}#4cdkHas+Yr1R|9(XbQ-*O*(xr>W+{nBlz{bJK&o|da8tm zKW^tKU!$15<@_qGHL+`L6n5+(2aAqN*pjf<{p<;7pq>n3DhIdf3YMh(t2fcmx#B%t zI(quf2pt1_F&`7RnJ5~Gs|@?oNmn`DaZ;Vx(!U=}L$X)iq|1-JI{pf3twT; z&nnqB<}v^L;U9+C+Fwz$*%OXfHV&7OC7jvbgbhPfK(#eLI1+rrS1PdBQeqB z3GR_`OVUB6ID@34a@UOuT_cVKZ&!r6T0^>7+BDtrZ&uP91&j-ysF_!j7Pud{L`6?gx$`AMRAVUk&Z^O5b zE20ok>>Mj^j7ggK1U+qSZ7|E;l9rKCb&3_H`}?%jKxa}6tk{PHvS3u;O=JkU4vHvO z8|6|&@x&mA3%<&>hPpaHCRp}026CO-H_^I&eoB61qzUo9^O_=`aB4a2b!IM0&)!^w zS`Anff;P0OX0=gdqpZx6ncM~K`t{ou!3?tw2!TfsD%Xaq5~I*>#gAWB^KiyvQ@a(y zd__5V7q2`eQZh0)+90!@2xm)6#ezph)C?VtL%3V+@C?Gs zcgSxs8Sm^}yVVQ}by5$jMCE;T1K}0s;E*2KOgV7fV(q#zF~oOrcGb*T;>VbER6#6A zQ_Fy|bz=|LIQ3sk6EmS=L=Iv7B;MYHx4P;GeL~ZA>3U z&L@AQuDQ1pc^v0fqluLdJyq{)a)a#Ok(-!B!j=a;?~~qk|M^Xc@+FA&DCu}U64G<4 z%W}LqAYgGGH2J0;^Ez9aqZv6oV%|Od=g_`DJ-!yr_)WtHs^(y291&dE%KM$pu72{a zU6AG>qq6{7ssB<4W~Tt4TvIt;f$kVJ?JgcdjmOxA9B69L13%C{3TK>v(hL#jBkC4A zf>8G-Yr8)_f_VwY^r-YJuNb-^C>3BS^|r<%DSb2kpHT{teS|!$bDZqx|aAy$ca00fc>pcz6 zOjTHvMWJH1-R>uN1G$b;T*49!yi`^g2k1TgyV||PK}|A;%G7bx(Bi3XvwtJjyEQh> zCUXhlI?7tC$B+MBGdO8b=QG)LcZvBp5a+WwN!lcvRoy-an_5?S4R7^OC`QGVJTB_0e<)nzIY@4n|B~qyP(GN*rT{*p zFSrj7KV{)6u{W4wpcP&IoQ4sOy19<8WyWa#d7hEx*no1CH^kL{|Bn_RkCL4K)WffD z@ODR!6pl)jTpBRHEA4POCJa8pQ9-;dNRJZpx2yktXByNtlKJ}ju6QqR3A?O>BP-TZ z@>x3UG{lplMT9qzEYTAzqW4H*LI@)78f`gs&$&Xnq3XTYuA+#1QQwntS-1W62j6*K z!Es-0*0V%b&Ql4&t`WZreBuTbBC{>)L6KWTT0v~Lu^XbOfC2`hMV2C0PI(|93R&uZ z#!Td2gNB9cH(-MtM>vAgycFS=FTPCpwl8-G!`Yvl$mjiaB#b?2$mtmxN+uRV-Eo{c zI0vC)FW-qgq0;liH%+i>>UO@B1CHK>C>Ey`ulx_-TSN`^j$K~uUz!WIL~_CDi)72X z6or0h!@e&sYWvUE2gbh~T5OF6d85&=ur)T}b>Myt1%P&c<9*U#qiFM;N0G1Ah|ZDW zF=FdLlg8S;v9vU@5T?XaUcDJJu@9M=OpcKd(Wy}9i%GZ0zp42@&jnIBG(oINN534~ z9&j0!=;2Oe87vm=kLGtj53l|3;UPQ7A`fVR^9fILG6+tmbmw&+9b=HG@&NgZo>?K- z<&y$GeflJ;$a^Kze~7*7{d$nP^yr&#!VMq2&yKVA5#(B6&NiX)EnA_xd1O0P>&i?v zU;6g%Qx0{bz3P_=#LeIMoM#bzY-Nbifu+nDYq3(r1Ie^UgcY=NiHT9o&COw9Va1-v z8H-SCW-!cpuxppZHhGK6S6V5P(K8EW7ybGFbxGySbqd@Y{i^ZXS7x;+3iRTc0ZV6> zlw3^PI#&4yE-nVB?R)d|;mj;yu?JU;WWsLGRik|~9SsGo_c7MDJwbOR13H+%vCC7T z=WYAedf?CotWJE0s$oGJb&3kfUx!P1*CxI242!U9o8q6Z@~<=dpP3K*%X3fEVgl0z zEoV0$PFKO{49bif3u>bEGbyfC1mB5OPI3AiXxN|sEH1l0n!8zH%z#$Po1d`fOMp#U zE97@VEFXBcjg`0jv<5scraL$S4mR{2ANjDk{Zr}goST;)cL3dKLAc7BK+r1Np;+lb zOV3#Ke?P?ETf?hR=C7va6go2_YJ>LHc6N5h*{Q)c<{_*2O(58uwhle*xT@tPK_wn<8K2{vzd%=n*F|F;!V`JE-VY8Od zEU@U$6C!^k87wR>2W@TH188Aw-cde#0t;)`>GqJIk1+U0;5GfY%p)drghu`+Kpfb1 zx#@ay{FV*e8wCdhG@r{}>{rpS1JEHyAVFZKoc*#i@k<|px~HD4m;-XR!M47%Ur?{} z|NndarmKGY?wvX4G?@g&o?pVrE(NUzQbK8Rw}%<9W&j*pseca0IJWB44sx;U2PiG^ zsX<(9Yz|D9$-i{d!V~g?l^4}t_PfGs_FWvoUuEpxzhBrwO2N)hxdH4c0PWcE1s%bV zxCp|9-PMnt(&($%*Ik zyIcjE&lrx)m+q;OOTmjXwXRV|(BCbbnJqxc*s5gSWp)b#!|?3(OI#fYnu&=CfVay+;04(b$u~F1 zm&gnd3FK-zK?fg_TflK994I^QUz*N?WDQhKLu-on@rlKH&g$+n(QGH5N z6phyc{2x(3L`~1is^pa+7dT-c=6#57lOE~t?s9*jN|4mw($cbJaB%Q5W^w-0e{r7w z`pl*U7(cb_eL$=O43zKnYHTRi{wyli(A1=j69@?j@%;Ir@Da#OG{Nff*EP&_GBcw! zp9U&=hpn*UZVzk>R3-PRK?Ml@`0O-)MOb~fla-c`aPufC-vAI~m%Ls;9^G?rg!3re z;X=hDef$SwOV1Jkz5D_w^=xXu%=2-=M+f$>n`L4wEVh|Sfp9ILU}d!D<>MRsW!uJu zAL#%XyEjM;3JMA$>JRr+|DW&q9^w0^6;byDgr`_at*oqCXJ!oL)rK8_(S=OR`*U49 z;<(E6n*vIFm{U}QbL-YEwrcXHfk2kh(ikCXCUwE=<3QVD3GPR4U&rDn!^D8$qe+qbh!B7thm`V zf0su5f-8Nd$xj3vTSBlSArXJvdQs*mpGe-jh%Mo(Z?m)Fxv4R<(L8*(~a9!AT^zQG(U z^lEU8-U@a2vNgBDl`KUB~tk9v@NJZsn2f_<^;uU3tXvzS1}Mh z07M@ZcyKN^f{@#na+Y}*>*#Afs+yW{1vj4mKTk3Z^tb%f9ZxiWQd5YbbsfM6BWT#F`OijAYi@c1sfp}~7dSU%Jzu={2Bg8> zLJ(fxHY(hOU|3X4?7GtDNNQlm$5U!x*k|Sy| zY$e(hbG=`AyIqPsAkY2o&D(dN%QlEvD~ZREM!-9e+=;cNa|~Y0_OY>N!F9kB6d`G& z`yMdT0*6~uM4X(QsEx($>^sVV=+XcgA)B5C$9kHd=JRwYjE<0y(9##^O%k$)v0K;3 zG3=wG!~zDX+dC5)e22hjhc7q;6uDlf_oRWcea`RxX}x2?iPMDahxFc$&2RF_crD zjn^^iHN>ZeM|%&vFSlkB0sI31npt7gNU-z(4u>bqxok~z`McVXc4ohM19_92obAEl z);GMky1ssr><{xLJde@TFol_HZ33zGcRo7$jrV+qi5B);=Sk#!52`hh`LVrMVI12D zjx~TUZJVB^=uhS%U84wT?$U=ak?ReRJ$X3F+%hk8<#;i>lf3a1dW_YLAP)lmj>uwF z)a*sPTbH);7>p)IUgN|6ynftP=qf&D>9_tYF#uCYMSUsF7yIzT9y6Z7h=d*!lFp$ z!m2V>T2R1%g6OzCODC9?@3XV9tyK9mnSY~TR#wk|z5=qNAk8$$9+^z9e~8=dPwU8h zpOo_=s=~hIJfytB{adzO%G>kJ&F?&7r0kak){QrTfMP3#n1qBW8cSH7Jl~5GbMA*Q z=CyS*L~w)Y8*(Z@rfg3vY|RT4#nbVAb9WVJiu-m+z!#YmR|J_RBqCz0)9!zS{k65H z)84mi_`l9&Nej9b5}XD#;0g;V=-c&Ae8?ytP04!P*l_*t(C}3k>s!R60B}nK7DlFv z^8S6c%OtYoEtSFEjk1eQk_W_?PFMxVX59_S~ky8{H6aLI1|~q4>%eH8IFW!Ni)O2G}Z>>37c;9bv(-ql9)dMToap` z%Gw;JA3$h2^!O;dV6qEK&p6#SmFX2omQ=gCx}H5p*hU*F{AV|Nq>UcabV9f~=uIP5 z_p<$&j!_8EMw$Ve4-)j4zI}#Hy0C8->3BiLV=ik!RYYC$2M?CQ;MBA4@9QJ9kyTS0 zB%g$0t9YPMQEc(Z*4&0W#e!zzn^sh`VbvKP87XN) z0T-K)mX;nAf{2He3S51(yzXyi$1y`pI(vAK3aY!iS6^gr19GoU$&!(tkvz=n4`c== zxer(7*t@nm7bE(k-g$;^HEh*6Vqy_$#28i#$kZmS}2f zdMB{}C!<}~nre@O=V;m2V>w6d=3Yz>9pe#C-(Tzb^8EZf ziiOK#YqCLa`{~q8zxHS&s59Mt`m^8ip#Ns&T*Y{i@+)v4Sc%8M4QpGp{@qu22$7Z* zQH`I;yxeoaIK>!E+?B6qVDNY+2St?|Sb{Yd3GQw#x2I=F_V63G&?1R56!R8xt?DqK zuqKdfN6BA@jYkJqtw=X7UJQt6(+N#+3{=Y;bo8D^_2)ZT^@Iijyf_pl>WFw z6JSxdkD{YbKoe@^1MUBO#RRCYc$hy_N!7{8NzZIh$+k3ab%obr47%u>%MXHl)2pPwLwh`vcgnE*%FN%O1(^XAhR!US5FU#*TJ82@c1}gNUhj{7bkr zQBwB#is}TH<|Oh)C&q_yU6O&N_Dw@z2&8CVT>qPS|1+mzeT_N-3s9tFmjGV|pS_qT z)`j;``^VW*c^&oS`%l;t_$nSAK9h zXXA{Y+_uxAV-?mXG(3SdLCD33|GA3q$G0%TK}_36BG|yxlpfQ2as#b;DB2gnNuUFR zpMc(OaY|r6%PBppPJm=jDRZr=D+WoA zT`&!C3<{urtwbIG>N=ZYmeu|@V&Ytj?d-TaRx>AW?Lr7>eg?5n8e8#KlbM-e0-Vvv zx%vE6?wB9MOZoZC!(*f5?Kd0a4753=4FuImDSICVrYACD?X5rknA2GCwq#@ zOL=>F8Noz3-CMG^Xz0MbJF~e>}a32zuzrW@u6nEl;-9fpfTW8GgDP2N&7y% zaD;F2NJuO%zE4^7yc>oA{-)`@hB zMlW2374++itS@}J5?jIjm*kTt@nmQq36&G}ZCDkcihn<%0QEE^srX2X`Tm^uE_w#H zvuy=a))>!>{e> z)GMO=C{Z++4(53xY7JmvVd0*Y`q}fJoh_e!cYlbzd?nordNj?|GuS6>X~}RFLmfm? z6W#gVl;#3IJ5rnTo^$2r*lUCIbTeOMGj#0E@1tC(Wzw6)DS_N_0lxOUlSfEhy$2QL zWYr`jWYoHi3vL$r_m*%Y7tF09+9GifSchGaD=Kevw9U$!!#dPBjT%Yj{Y7y%bxLJz zVFW0(%Vp-!7*5-|=})vjlbjO1M#UEfwfqgd*Y^s+J*CLS`q!xghAhpqK`0bz3bx62 zV!BjDnM9^4N|KT{KrEWc+VuEcARS@O_9X<9cRP?0aPAd>%O4Pb9uz*NE){)%!Gpv^ z5Slhy%HyxCt);JHz102Z)qT_QgQ=WVOu?LLirN~OmQPOLQ$0P3VYdPo+e2SX{2+Nn ztD77jMXM@S!NGI$qY)LpsZG#OG&9uR{RgC_j*Xm!O*<$X4m1LWX)t+2U*HK*f? zpCRh20r09%5^-`VU~OFo1LI+E;f?=1B28HDC$&v1U)H}=4lE4;ghboGz#Uvt&XQ4w z%`ODI1q?nbqI|rEz{n(68Z?@Jii;+H0?V5i8X96*-(6*8W>zpi0L*iX0*zlV06&*N zcIudanN8+4KtuxlrsES5-ez|Kr>NZ-S2(*{@b%?RfSigIL;Zhb895j~(^_V#D0bWP zI-#GkPPn6|UX1Z{ceyxk(VqT^y*E#i45hh*sJ(?zE9ql)R9qLt+JVXSB8 zb@}=APru|kDE4;^v}1cVIgsf?r!DNdShofSFtdZWJEM={LM%@6NvH_54K6OeO~EMa?B>=1 zym1{Bc~&zREH1zk!wdkNM@%Xyfk66cIDWIbrUsCSVNfp=eLq_UjFUVlWKtAJq5qzc zI#D@Iy#;M?^a`?XzJWX4jr$7VJNM~r(+YL|9XA&$@qs*q%AQZ+0!xmV7|Yp1#!%%> z7ow4`) z&Ci;iBkmn_-jMLf=wa^|DNs2)bZPB-!$I)f*Wn|9jt0VuM~P5`{D)`SXls9iF-{GPiz+6-y?;`~40}+tDqodoi5%$UsR#syRH!X?mOSLVt0p`TK z3qr+AkG7_eS7)1h0oTU?e}B6^lbXSn`JTknoal!1zUyyTpLSqprKeA0#Teu1{fo?T zlb{2LqXp=gS!fzc%MT9aRoouYNy|77Wm^0&zC8FQ3Q5#isujrP4{Kbp6y_L6ThOu< zy7*Nx+}=+NCIQ9~p1vt+LBWsP^<9x)xY`Ej_nt2=g^RRbtO&Vvyma?y+aWJ3xYiHO z&xHP1?OWIrb`PPedkJ#kh?n~zkXwAZ$N^EHqHy(pymtr9uZ zw@9)>c$+=H0Fq7lSk~X`mhZ7scNpJ%XWXoanWOdXxe2u_QN4It&ThYwiiX>V(7zAu zg%R-TyW3tIPTiGV0M_MpknGH8mtI`_Sta?#`5ge1_7;5th0ncim@nw)m+wvip?|sG zx%);j85PxCW@ctfD=RY~yj&9a3I5BVs3bvow;C%O8^KBOmkP--H|8l;hq7b9!tkn5 z3=9$60KEIks!{=bB{C1#!R@&V(jvp#b-@BI+Wa3J;=2cq%CVe|20cl!7Z*z$GBlV{U+%yje#A(>K$ncdR9`Vf(Q-4k`Fb= z196GCV8qfB*x>2(l<84~Y;f|bMe9Xbh54Ypo@5^$r>>pPrG{2D5(keTuSor(&f(7! z&VzcwC$ag*7P;9FZSbn9Dln*c$E(M{Pd5NP(+YBND(t6tiD*7YSK4+TkV#Nrk;dbPl)XF8E%4t5$_fh(5Tc=c$#{*`QvtqCBx=OywFR!Tg z#Uc%lNLI;6S21RW?sY6;e6OOcbA6I7b!oZDakG zQIcb{R$!*f&B365-2h80rq_OJN&BIk%*%=x$vBBgJ)WcsY?oPH8;thzcv zzZK64Fl4Q`Iy;vivOFFf=;-3N`&wdSa%eXOg!Bh@j;`*~2K=K1&`Mc8SdQ?_(ZjFv z*2Z-}nE1LJ!`u$s2+*Pub#*zm`P3=wZagESUXMb)rt|d|RWfv{3H79(oYSUG*U&{c zSv=8r@+=}9z2WOfCt84pXn!wJT$G37#JNw81^K0-PFnNFyImnVJM2u&SQ1PHX0h2} z;g+rUs_6#4z?Dc8G7F;^L$jib1|G6^C)7vh2GtWW1)t~Rw9KqCY%mHkv^?#Q5nmm< ztZ=U6?M}g$h-k9paoO+m8(z108glW)j`3Z3@56=~P<(Z3yXoAKU~baF_}@{#7MxM< z%HZ-+frA1^J4*`KfJC1T7kIoq}aa!v}cif%iZu# zzd+3yUDxweCNxgt6d_k87Z(?m|G`NY^02whg5e`4d!zA;$U0Y>$;rtcOe#aav)@5p zh1=mjMev8*Y#-ixgrjFdsooJlce7yUesM!34%e0lt%RT~+mn(sngnz~J6y;!EoBZ1 zN4*f}#N?2JXRIKYTlxpoVW7hf9_8hN==F>YVe|iustd09x!3B4B^%eB+;AOi*t2lo{vg3Ee z-R27pzdQiaTEU*4U)_Wq2VP586RT*J?ff%$I!Cp`*mQ)o4*|L1+X+$($CI4 zQwMCt)+8r=J^-WAQpY6N&~5RKaHy$#8iCGQhY&3#Esd>BH-=h_fu`AMzG?TM*tpY^ zr!$>KM@MJlS5H;|8X6kuft#N_J;Uy}fs+~VO|r1E#zkO!l)#<=F)Ed3N&hUPz{~~1 zFP^mizNV*8_~$5F$i~{GJ_C$i;^q=?*;4_1s7y%W-DqZp95xXmkRo5&v{tu#uOa&l zd*irxU0sO1`%C2{Xs09I-p7r&uAKny3x<0u5Yw|~i-dQ3(`eGs513So&a*>K!V{ui z>ng)|N`J%3BC=H2*jisBwIJGBktba#-6bO~7x>BO9(L@ol&CxVNx!!Hdr2|%DRdNs zZj+o>_)#Uic4jsJBb78VpQ_FiY9Z&se+94I(H0XB@_~x zoR^_v)eCdQ<#8Co79D2OU1HE<-pARy8pZ&Np@qvz_HpbMm4eye_zWcx{e@h?#q&YK zi@N7=hq(ubyEjG|K!V?OiwMIVPHmEFx5&e_Kfl-#>KEH1vS({Y1xUyS3qhgpC#~6- ze~T`Ihy}bZ0F49o4--&Z{3s?{aiJ%U-c*YtQz?Vvoy->iuS0M>`FeCUyD+Yv5p# zOV59Hr9nXPwg78B@v*+VcO8d52M33&u`zXC z>4nGI{anr}q}Ghx))#FpOSmPRgf(?U@|^usgD}Nw|i7qU84M zzt0gNyGDs?!nEoqJTpE;GX##;VW!_2KTPRoeb`-+k3C`$9hc~*4u6?Vr7jEe656kJ z>~Q)dL9F>HuadA-HE4HfIVN)V-Jbx^%{M{=K|7C>HPq%o?aEvoQq1R1|Fl zc6fB1pHG8_!A`JMrmd;DmmCAM+GZd%anXID<0zX^fGkJMuvt3#|EaRrm9ZW z-`&Wldb4>P-NBf(6#I?W0Ua2Hg+V513=l97PY()@L5i8xGFS!$aN@y^QP+%H*07Ib z4BCjm)_e@v{CDO5hTk(V&;b%63ciHWsx>0>&_ldVk=0H#yZ&`LWya;G+TLbvCV@L(=RAzG&zg zzv9f|eEw@G(1dWbf{vBQ&v)hpwSttBlrLM__bIyS9IX5xzXfXt>_`uVeVkYW{Z|IB z8-~Bl-9|)MUO&R4csxT&jAh`;N6wzbcs{iMnjn%_wOK zpMg$NlEu9wTDom2;b(#BG0e&<44EFGX)vg<_m{>w@h45FovAD@!MU;=qvqCW&XlcG zT;6Okra1#4&=OL)l{k51aijk!nk#6xbcvmqi87H2neVXgV(dCTakDUrt?|qkZi>1OdiNAkhV| z<^w`B1MAQr4H?pJQ`cPV>^VYRiV5?6@x=EKa6cmZc>qvJw&# z8QaEdG0I*2$^<;xhc?9NYnrD+7riJ+Pct{$M@K((Ep|OujGk%<`O?7J#>=+9B@A1_ zKp{>98^}67QPG)>00TEq2pK>cnU>h{Wx&L5ijRs6qeFZy%-m=aKJk%kRkdJHJ*NE6 zBedI?^^4zgsNKit$k+lmRAAQIv@svgdd8jTd%XxY%aN?l^KJsf-QNl8@-mJ74@iDI zOZ2RJ_8wa2FOMiYC<)8vvFpq{hayJxtezKu9dWg!D4(O&TiS;d z{e*nOBc0SW7^jM#&7YWICo#OE|JF+4^XY(xt_N4@s2UoZ5c8uqmGG|6qZV8OHFr99 zvFZaB!xE0Y=kN!xii&EhthV8&A>}oXg2>at%jG;MEa*0z`8s4r%(@l(+Y51=)q^GI zGPhdSmBpjmrUK}%4Sjq4uHVE)ckAEo-76;ns|dG8R3RKWap!m2+;c6hXG?J;t=ar3Hzt?dUasqyh~Q&BUA*8bh= z@95+@<(ZeQ7R$%TyVMD}CyfDYLEJWE*^7Ke2KIfvr7+ie2-;~XI!(GcMN{czZS{@b zOARC)DRtqZO;w02jwR&yg~ZUVYrDc;I}fAy{0GgWc*V^9zjiTq|1PUOZ zxO@s>xAzxRswygw;p-qgab(ZK^>zw?QYrE(Kk2OQa_KsRCZaUq;J&#KL8q5TNCK_K zA;+p1Y^Zg@?C19KQiC^AF4q20?RyNaofs(}K?MZvIbURUT(ld!af}D&)ar-sn^-0~ zQll3Z(#7=DdU}jjCv3vxURFPraF(md&wIpe2ep%St18;6nzrehhR0SN9U^#g{hiCX zF>ig~QfHv+XuXGht8;z($Ll1{dvsfV-dhX<=_!k6H}GH`6>}2tfkt|@;K{~nQ>*X3 zm&Mj{_KhPLfWP$xcWQ2*_gr4FpIc~XX-3#Ow6ExHh#REDGD%Y`bBXvsA2no%D?b zT9mIK3)PkxZ5iv9GlVN4#&cuSZ+NIwTaFMW?`55U@#4^{P!MMQ&`%_~eWyQON40c;tt6PTHHAhrvWo_A1Z;GNBZ4K6M& z0etYvl#*8GY;O(Xb5eWM9pWEDG?ts2%ReM`?iE_b%3(BE^pNBU)S`r1M`YZ+G4mn! zz8#Jcgz+xj@>I;)G02&V&dJG1>&|#6x^LGdzCV+>X;XDeC5t&oQOWp1f`0o>Tgsx1 z{m)AP)+%4~iw6KDI1)q-tk^XP3_hd8O%HsEJO>GmE1sWK(oxBy3<1oO6A9+Uhpp;xma%^BlUp*bu{8tfxzkn_~x8rQgAV7<@w~@LoZqHyX>MKFl z-nL?vP(w|~{C%1#|3!;ti}$sn==ME4eA4eV6(4pht2&ytd|}OAQ@Xtt8;|mzA~<=4 zt0s3insUhIyWX`-4C>Dq%AkwFILh#MXw(A3J~A?eVk){>cs)xL<@`D*pGTjcu_bq7 z`I!7bvDKkGJIp?rpNmNkqik(Dv~_4)zez!1PLPN7`H!E0Z}JFX{Zj@h%cqUZvKYQ^ zje>oR^qw(FPgl%HySaD}?!^Ga-IIZoc=>}iEq~(#A@1Ydqg-8r8#_@&?sZgy4_l5a zJ~)4V)NN#Tp(jPakE5$Hz1= zn((5TdaieH4vM{KD`{r?;Q_C7251>}W|-E(FFOt%D=*$BiQqB`AkWwY%=;g_o7)J5 zS#8(ye~9A|j=y&s$dnU^r#n&^{A!9U?wsn;{T@hDd}2F^@ z=Vkg`5N8|ilaNPYB9c69_Cf`NwHi3+KyqI1PkX+1L{IU$_W{{p_Ve2ZeuDPj<+!s2 zf1mQe9&H+oF==c%R`g7_)`bzI;RO$DQRWm8HhLd*fGdKf_W>fHz@Ruap%KgUQ#4b~ z+fO$K(r(FE!OH$E0d>>-dn8BV4^R3JsxOvK>*)zYMXajBdZFDYd?&_wZW-(eV|@p?N;rFA=QXcc;%GQj|FCW*~tS$v$TEJuiNw=}~q||JM-u)7z z?dBD#WW7fPeUSG(o|wJD@ymUvut@v-tZHd^}+#m2B_^z6*#} zH4jGMu2?n{7BZ7@+1T>~(3Vo%SC^>{q4r)Ef+@$a1z>EqV;m3~-rsExV#QmF`U%G}B7Ls20 zUM|p(h41FP>v_obD!rIORBU!}^Ov9S(N{{6)z$LK&w+{0ezS2~l)Zn?veDmtyDdYu zIrhrBOPoop$(6|T^DL3ND6L?)Zxj#?fAIurOvd{^=C$119s#FvOn5jJ!KSBcKI(9I zy&`=}n4CPhCo5Ec48yT*g9)YQ&NDyFIsi6iw);ekyC8PeY;_>jB`WtH>QoZjuoeS| z64@L*j>^PcR=!2L9-c)}!|}G$FfYMh#AisJmZxerj$0M`zsNa*b4}wI`>R!oo-qB}yiE<2?1epO&G& zN9fgtmPnRKIO&mWepT^JYC8gsn9L@w`#|mcdN%7L)N>*5(A;85NNDkElOGw>Imb6R zBt$7d;Ljm|+((fHKQa&Gg_iKdPi_DJQyf$|aurX?58H#iV}u<+ci%^KIHYT5ShH2} zhS~)QF=_q-36o$xSRuddlhMMy2h@8o-+Rs0^IUwct|{rj{XH;nlJRpPtf)x2m!YiW z{d<@6a>89`YBhpCID_CjN9r7JX?Iwec7y|B>%>mY1?QDQ+J-gLi!A``)$V41CSatni;#mWT*numVkC zX1azh39!CSe5P$C8%z3n%c0ZkcqR*st`->#o4X&-H%3?OZKDUbIoDSf8DI%+1-}p7 z#bj8{>DlpUs2<}>m6oYj^>a2NwY;0FoZVYGumWHU71mT@5gl#jD$Q&St10p7PBmpQ z6!Kvw=0FndxzX=6S!GV1=Izru(T|Ibjm^Tw*2X_!NFTugncDhQJnQ=BYjFNSs8R8& zPwY7A#R|j(8rKo=4Ors^dMZXEsvXYVK}jibz5BN83;p}<+mq4OUko15@e*bk8#}|? zJ4=pj=md|oe)DK*6_-E?(~8sCeSI%vF*YY_abL?wX`=ju4k#aDvywsw22_Lw^V+0^m?B*o$PRZ8rA;-s0Vx<{+~|{Q z6GL5IRrRVW@ohNlE>%Cs>JJ88yg49eNKsXF40pWFGtO|;fhBlh+%Pr%lW|9$sGjgRr2W47#R}5V`uc@)3Yz{2 z{BvnxD72-w7aNFy2@QVz_;CvuPrML36F>{iQEEy&jX34lsQ70TMK)mkBcz_Jun04x zfbIR_mkFgq?mX}eQ^-lEPe9g-3%e6WldQ6oyzkhRcu@U_=jNiCJAadqXrq1A_FCH0(+lVw36r$7S@Iu$DZw_+&nF=AlQw^F zGw3z2PQgh&PKU>m5Z(`c_e$zF%rl@~mBNl4l5FT1@agk6tqhg$_AH63mbi9I0t%w2 zZ{Hv*KUO0WqUkbX%4q2F(sEr3IoBs^6T(f7+^d&ff2yh|6=bFy;^AA{u(#m2$t+fN zp9bw{$Qz^_yoDWufG1hQZ2-4sd99M`UFy)t6|(9OyL2)wGXQP!Q=0l=hx-I*Sxz{N zpc>_kaSTMist8-d9@Rt}*A+r1;&K1&WLE@rSHP18%S$*%MxcAMz&A54?iKKV5yXN( zqK?*{it=uHQuwY@WGgWzT)OB6?qgit6oYVR7nXHh=P8*#sH|xJ@+H&Gyd%UBNFt}X z?YkF**LYxwyvO~QdnZtGxxFaWZM%To#!3LvCtoriqY_T|2|E@3RY0J2YAQ6&;0th* zGMM|$hTh%RU?wG<5w*lZi5LokA1V0j)gr&mlstUfJav%*AsFilxg`GtWfmqU9?Ab0 zftZNpsVRvzw&d}Tdp(6(qnZ>7rwqcDOdYI4VT)G)ACUso-W-&SeK>mP zGj_-=u9+LY#F@b<7v`XIdSR7e-p6bQ+cGK1OupNif7%kl7$Ppb(-jP^kn@MEL;w_C zZjMv6^RCf^Tnr@zEk*Jnceecwc}hgxC@bue>07!LNUZnHKD%c7hp*2O4I!H3=;;1c zxMjqE@#yUPa|i|?W|{rIUR+*o4or3R*8O1DSpsipTTvo=TxMd(xI@|N_2MiYRwLuJYi>KB;V97YXcR2 zmKPsbdn=XHWS<1u4ObEiyKNyr2T~45u5*+jk1& zdj;y*+RG2hJ$8%9;smlOqeK+s|*1Tnr}_YX&4TA=+p=6wfWI32&c*Rl)8C+x_Pz}U*De{-{P@$j;c^+owu=fu(xat zYYe(cNk!Rd+vU2ouztga*#=sV>^a{0^($6K1D(Ra0QR=CyPKA`vTnjg&MHKfOGis9 zuQEQHy&Oatf3_dHZ-~`$^&avoG7?AA&@edfHTufhn$rAO#uf*`8OvucAD@0Y8bLw9 zDdy{Uu-AQGV-m%5iHL}-t_@>HDl8N>hNLQ*Y2l@2M<29+s_nyO(?9C7NB-!LiV=XN zkB7`ZR>l~;$#$c*H1%vH;rT?3ggE?hP-DogD+=NVoPy(Z-|S%_r{DD>SZvy@oo^V= z(gfv2;)u=*P>%7E?_R}WbgYst{=OH-KW`z3(7D%1IlP92wY*eyBFS|i!02i+-splPoAnd7|_GGi*EUg0F2@M|hWr-PLE z`mbf}1HWx{;byFx2k|^MNA|wwm*`r0dN-}!V(spctlcl$Aj7lWV#SQQPJB=Q7hIKc z58(eQYldluiYNyifRl3pp_a^0O-w>eFU)k$b#<_{?bdvLdAX!4e_bJ=dX}u7S1OiW z27*F-U*J??_C=TBnArS)hVfwk(%A>1?ijPRXizF?%?}iA@cl~O6a!sF1}m)tlIIgm z^!infL5{Qa^r8%8?vVK{`>Vtg`QsbuLN&Idx2J2Ik|Wd}E;Gaepg8Zo)4)k;nVgz3 zji{%FIUQ>yf58b4AShrvyH)K?4ljo>2cf+>G>K*-5@KRTGe!+U@32fy7AD*<{J`nQ z-$=hPVz%>mj{6BTXF66o6!@Qgk#%)_LiQZL!6e=b`xMg#N_f74!htTA_22#(zy{n7 zuDR9;%m$WEgDHt`?d?R4+uP1B zUlxDz{82~7e4X`U)skLBgeY}mL-FxrDP_4ofXQiBZOG<2M(XOFMYiL-Vwp zWVs8y^A%^&f>sDNQU=Y2|MsV~G&6!)W0H-JU(QA34_#kX>R9q2sQ7VWRO%E+=rTDt zxowpr*{wi>wD7th2-%~!9xK*Bf-1*>L4ptc$ChLD^J3Im#qJp#(0kso3 zdf8~+{qqh7mHyU_@WA`Ws4HJ>o4 z33fdmX|#SXisKwWiAk5|oos)>n{9|$(%Z;m^)43Iln zjE24($wzEgtv=N6voxupZX_Tcqx8QDu=W!q5pYWwCiohgh))v+dTCA1$LA}&mc1vx zPqlyV!vWb|Sog5y!qjx~A5YkEJDUdBQg4|$`Lu>9EGB5)g{T1TiEGWUABwHfV}>vR ztS~+m70Li=0Sp>+l|1erbK$SiZwaWmeBzn`X?Js9!*a}sI1m#TR|iQq(izp6$uFuX$1rU z5s*|sy1SKbk&qB1R9cV@=@Lol?rxA$;I03tC+@rNjlpnm4nz6(-rrs^*IaYv>OZx( z#D4d#OFk`KM0erGV#%O#*Wez|qfBTM38o{we>Fv>H!)x3AU92rPL%EO1nw{&?pxY= zbWF?`!J3Z8N+LTilGbjR4YsXW_N3m9`w{r1(#E^SoWkr zc{qK!Y3Qzrh9D^I1-z*_k!2{HxnaWR%{3$b<%|5zrdEOB`Dc&Wr0bGA*Dpsm(9s7h zCmC7^?!HHtp^f)SIXW!L5AkxVusBorm+ZzX@yEEsR};N2VUp-C_*xZu9dcFR%*-yq zy`Y*Y3$4%COg4K_%B{!%l+S#@&Lqy9OCMpHam3Gm=|KV-aPRJP(t6nMXGlI!tYxE6 zd{14)!Fn;V&$ef(n^Lq!d!g`YjGD9SA@7Tuqx!Z_N*;u9mElCZY1*@BFh!|&;cF1| zCe*sP@wBt6`j*Qx-QxWyw`!NveI3K8O6)}bw=~>1`JX*Wh2~KoB@I>^8n3r^-AU2x zDy!7{F!pX7T(;d;*ImG5c?sJjXV3-F5^{BC-;ir|r%Jei%jT@)VGTU%zUf+)22>8@&Du^MCUtpYBBd#zvZ=H zc#?zjQFrP4TYF|&%COwe-Q9)wTV9`YofoG}gUx2}qRJC6IZz)*v>A1tKfBJfvp2vVM^ATc zHB>B5`=KvfHTDDl zQPdDa-*$Ba&_n|~%Q-$?<6zp|m?W2a{6k-JjE7H@60JQ@?y_~LS$d7SW?-P%i@ALPUc`UYU?uBDXZhadQvCkGy zvWikXa88ZwX>lufU;8*TY;B0vX-|_v$X%n9d@@(a-R-c|;qYF|ojiKIXGZ8%xi|)u z3cs3za(oQHDj45BjX zq398hNX=Qbv)K#ME|<30SYj4-c7cSM`+8cnj@X0wkAm)?9nfFPz8SmM7Imk5^JMj1 zN2Aqej2pi7k7N{mysspx=V{JFD~Uy+b8*7i6}tz_ld5$zIVaL&Mc4_Aq*4eag1f0# z>UHjk$FW@bgpfX6afCW3zi@Zg_Vv8cJ*r!FD#_G@v@B`8tSv=lPTmCt1(J)8G&JZq zM35~vEfL^Zaoz|2MKxp?c?ieLoiP^K`# z<3n(WqSxY(l+^Ai2*jj5`&O4w5HfEwaBFU9y!ru4Tepm~t3UeOCe<7Ky``BImy>g< z-I9<*a_ywY^DeM-5*RM#L(m#NdsLV%dspE$(4-h7UUgL<*1+cPdGx18h_T&U$)05! z13}-RR!g!4eryABKV2@3nW=AH58T5*)MxWXOLL!4`;b+HPL1{6ww8_Bk1OX!pC0zx zy^O}+S-x{Pr|MWLDA&pPl`;G^Z@XdZWDroDwN?5V+)5`v^|Z+1-|a@f01_A3d|F2V z$xrX5VbTiYXCrBs{DHm2)o4|3#kjaUvkB6AZ%W$Ho(EE{EXvOH`OFbDw}nHYlZ1Ds!P@QS%#2=g(Fw1OwskaGRLf0v;i5VxqT$B zw4Wl;G#l zp+(1*wWA~GUSy-2aZu~tIyyP7@4j_GXHx7#>V-#<0Y~M6=EokN$|r`kla}6r1iyp- zH1HUFYGLsJqk1vHjG;v}C!ub8rtzuGMco1t_R^h0s7y62$+l=i2%~Y+UB^Ock5A!KCs$7X;Pb*O}88bl;F-GdPVasHYexs%D^9es0?meC( zw{4gQ1f^^LVGkmvzSvTFI`^(wUfFMG0Bw&;efP-Pxmmo!p3|QkEOTMbtUqtGX=uib zRrs3Gok*L#6W3(_N?jc>cm?i&DT-i=&}T0Oq`Ah^p5{hBeX66a!RLY2MXimUIa>GJ zi(Tnst;Pdiow`MO=%p~MS{HJqSL09#1+(E;*#};3eMnu+jk%|>MJZR;^X|=!%_@I` zj32*Qr|cn!oH%%FS)2~C9tBO$Z6tmG4)dO|wvFK%q7!_X7#+NxuI~mFEga6=CQ+o8XXx}%pw~t zF0yd2N$V?PTviG(b^jTok;%x2&+fAO#J8!9si~Qkiwm`omzS=mw|;4J#wYvB<)-Eq z`A{miQ)TqZ{&#?<#nufdkZe$xTlU5&)o15JVlhYoQb z6m9LT)$xQcghvfv`^equOK_*4!#oYL%G^hnHFfieO6&v6P4t><$`dD(0N-^EnJoAwfuYiALiPsFFZqs+sb9iTPUP6{7>3&C8ebY0}*y*%@356 zVm^2S+zHlfYUN&E=*!I-M8A8AR6f5;|IxJBN~6e`_H4N30Ugs;wHbwx$M)x@9p2!G zsq{Kt!xpTdv~QpuB;kE}QOEQ6=?Zlj?E1OG2E+Lg-$9Z=`Z}0Uz$jT86ASBAt-$Gz zLzzP=(fiukzG*;3&km|n*QWFRICzUdFQOn_=E_{#+vkGz+ZVv9CgAyx!}^X6av`A+ zx_6(pMim53h0%6l>48Bf{)%L$JT8MK{@@#kMHKw#m$*HA_g%wEEGLM|Cd7!7 z^VLbT?$o^V)H=aj^gI?)q`1YDQ;v4!GnG?l1;Odbfq&wsITxJgv&-Nn$+(sLAE=xd zxwp4hqv%Nx$Gm}NR9qZMk*)3Y%v=aL>b{IS!Cm(or!M7H6*DH5NS`O%NPFRv3g4h= zJ=m;Y<$m3*-suf*Yeh!l1_bY3zIR$n!tiBXp6xki zZeh3(t=JXnZdaU+xKp+zNAM=;GkMMX5)M-xdqK7DCPI8OK(U&V0wOxh0Z z8C5~e0om8DUuR|(I2Go9=m=kM3a=%vr+pjdxE^*|cRVS$l{0d5Hde4;_3g%ln_l0% z8}CmTj~AAnbZiUyIKZ&ti!Tu(p`NAG?0M&BkA6O5sA0}Hlt#wDb=^`UEY$xob^%i! zZ~IZ*ZUzerPRjOuyp#o2-(i+u`DlDinqqMiBz$z*j7S~1Zh^suZ6EA3jQPc`8`7w> zhvOcDXB*MuZr$T%T=+Cdh%b{sl#GaN5M0||1DLG8`?L>JBS~`9VH0^tapL25FX}Xx zs)O<@w}($$Q7_!4Jz5U0=XutLQdGEm+8sCj{^``Je_Ire=!cXa^VcstN0IeLe34xS zy&3GMjPb20_>pX-fPiHy#1CV7)|wJgxy~!Q|D2JSI6mz!$B>rm>2E@bQBy;`b(nEI z$a%F8??U!K>Ama_R4md|fvz8iTUOh9b13;ZKJZ4Z1e6sTwwiX|s6aYhla-aN1I|&N zL{efRxx0tQ&i;!aQjvM$c_PBMAC4Nn%f%X3NTy~N;bTs|4k7CJ+&kq_xK4512unIN zvKGUO3dBY4zx|w=DqfpW6lD`79yT1Cm>7~Q>|}b$SfLFymPFSbFDS?ucT@yRi%#IJ z@z!T&+jT&wb&J>a$DNT?ZvR+mJ#Usd_n;n7Qv+NXQzvPc&a!r6ko~0=XeUv*%;S}O zej0fJ+nx>Euo%1FA5pQbBxSc{y^7_7Io)uV@8%hmaO&!TJL z(-@e#M%;Fq2y^W8#nKP{8uizWtwRTd`FewtKoe^R_O6)6$%p3DssLu4pG)>4qKg=z z&igjw9$V*EXJ4%{7_ZE!^afhyW3P@;9=o`lymRYH5^@Gz7iB{olv-XubUq=c4 zT=lQ(3e}!xV47YkS^Ma6dXr+3$dC9?6l?8;V3Ly1dt|M#Qi~nfG#Ki{5W4*Pe)rSE z5&VuGcV~zasgO8w^6^r4O6y7I7?0hFYOy(`4|A*7V<%?bx~Q+6o%eMnoz2sUxlgD| z%_m<}CvgZ031xA-=npBS{1gO_!RwskPM*2oTOccUrxo+hJML{xLA&igB7? z$;UeRlSTP^0xiX3a<+r&zC+b@SA=*|?xU(JNVn?+%e|=IB>%)my+YMz&0POApC&Ro zIjn5TmB20CiT2hl99SE=0=BYwRW;4hMHv|{)a9#<{=R8gv2{fm_U;}GRE#0Sdh3A! zyx^z$JQyFhQ&U4e9cgEoT4WsA(!J12bOV9gnL%LN~{+}Yj%#a%lg zc31(-3>FeKdO0+U+aljS*f&HvMM)eGu5@{ct>BToe~#QDjp;&`CKmGO|2c z!`vb!nkA;xSy*t<2a%b;)^vM+zX{xYvP6$TuMAd=3+|AKT>2Aa{*E^AO&vGrG3}`E zpd+)gvYNp1j&GtL9b%Fpu*DNt{rbrU`p>Pob2?^r%>4W=jqelkvQ}>)2@hD}QdtzE zd_FzUY8!&~E-%&;gu7(S*J&w)T(R~h(`!;`bxHTmPPr)pM*?(2-KJF~kYTYl=NlfT zXd&sr_V)d%9i) zM|}6=>guX=^UT>ynSTGA((Hy;X9Fykgn=4S{lGwpa%WU@)-7b)&Rp#Z6*u9Q=H?kt zz(R=g1s>jdSE6^STR4TOt^C&rEK z>v9p8g)^Y=;)F*$c^ND>n~Sy{3T)@T*0I2--v0V+7{3C;($bPm0#NyPrFXWS|G6L& zYm}xdk2Qlt_IvJNa|tj>{yv>vlHLa71=!DNBBP?-3iN^u_Z6UHx|f|8KX0oNi{`t* zd;6){+{)0w_qe+spYBN$&M(b%30t;;jN}>tb*tqI6fjrK79Eo``BYTo8@S)$rGfeI zVVSg`)?8Ab*OV8D7QI0I!~}L!qQD#wBxFFM4i6XCTVTo^v9-vf$WwcL4cyfwe~c(V zXZnNC5ws9+#6(`YlU?;uV~^E8>5l{2PzV9DV%^wSRa9r>H`>2q2s|AgX=G?`{o>-{ zG+Szc4%QOrZO8O0z=$IrnssmJjRn|Qp9Lp{mN%kq+oQQ&GlOGWeCmWZt*$?KpsK3Q ze;M`1;b#4VhzRk5NssBZxiH4Nv3$WUHuytvb@9B02w|2Is?*&L?w#>@5}VfTANf}J zB3d;Bq^HY`fugw2i;Avos13Zmg9esh z337%DDqE@!Qe%%>$p!AvWnJAVY5qHk0l&)@**a2;7XbnEEG+0MGfd&cSiheg4+`R1 z^kd@Zh)|m3^(HDEoL2@`%IEUkWLxVXKm<3~5wzBS>mc{|vEOojCKGc1qHZ5~>=}rp z^BQ+i5RD-geP~Rra@EUu$w-j0%-tMko#*GD7=nXTv22AANC%N0@A@6WVJo=0`RtFETF=<3kBb`C7qz_73ZHYL=5UVGA;NT62e@X`5T z#MlP3-AGUD<8S@-WA7;3)L|nA%!&ASujAt0TUGXb`^Kxcja$vLKop~BIV%Lf9s{ikw*v)Ig$uWhIVzHj+u zXS3x^;unH58!616hE|PRq>$m1NSc^@uvBDa;doK?UO>mw0dwkazYd#4qMym;6B>q@>WEi21{Q1IMe9cjGZXZ!~Hm3N>8uu%sJ zAVE+Fy5bWNg*BR%_|VnTK0Dn$m9O3zRekm9)ikJ|4d6@_#~yE}h#ZdFL7z|`NqeU! zXGqFsH`$N+t-{!Wz>^8*zDh=((-iJmNP)$cls;RvT5r{6=tS<`p#M|+LJG!uEv~9c zIA4>i@gzma4i%(8{Y5Aj75{qQ^&}xIU+pyzXvvZ%dOqBI`~xlZLpdpq2p{s{ts|a$ z_wJF+*<*Vu8sjS&F7A85O8!N#c4@?XpwrN_GpA?z<@>dxW{V31M?m);kiP%(6gGvW zavzl;ZBBY((_DXdKMrM+TntNxifXvhfgKhRGDWjlN>7FKvn;EC|3c#^!tSMsBP{%* zQX4uCP{{^@pz~T@*+*!B%iWx3r+rAr9a|rle?JiJEqo9sEp!Ti*oW_FZ<`WQ zJnLGaV5TcQJ~=l0`9jXwcAvo7_T%+etUO8=>1bjKU`peioJ{%o(J=u91tm6%+q)BE zl*u36V;cJ_t%5GQ3>&+qE?Mi7HRQlvm$BKoMPXkS?O78!olbSHI?a=9`VmRT@+?V4U)kMA*q( zMoQ|^sMGQbpmzigWx|@8OBlP-0*sX%bTB92O zx@o=nrG?p!%)2FV7=J!J96B;`q?Haxqq(I!pg{K6Fp+BM&R-|4ClJ~Ov!)QU+y`St zPcac?8WdRQ&jZG>fD(|TC<{xuBCh-EVU`naX?M%0w%={2BuIZ4f|ao5&OiuM8w z9pEKS<3H&ttP_7tDrYz;nPT;45=A7+SzsNg4j{-sB{4PU^)fsMD{?QMdZWxgKUh}A znQPiHbpAe&1%gM}NL^|AnIZ`JxnyCI@bGYFgRjox=XH(P?6-~Vg2ca6JaPXxO_?|9 z{S!i9VUf%r5XvvN$%hWOvm=j%mGW0PkixuRDYz{qe^o~Ri8#|K^ zUJAHc%>QI%UD_Ua*g`?wwZ(MgXnmg7SpA+u7`%zV(?aG9}y0S=Y;?g&Ht>(}-DU$H=g zrzKL$vN_pl0viYCq2TaH5w+Iyp9(x(87dDR?0}}0gsZw-L8Et_UTESG&WjiP0*W-) z?t_RtF1$C{S9`rLz;68LJ@5Av?mMn(*H&Y_HTGm<#vmW!5mSX<Uxh1+IDYwA+Nyxd>7Lud%R!*iNeUBq?Zqk^a$PDddK!mKx9&|9v7 zH~!`6GPop8SPF?Z>5E-3YgngpAEEosv$K#c5)+4oKb3BANp{mT&EEWQE6!cL`9>DX z*N#p5xaDla;6$EJ{c=HJ3H$EoPRR;>Q95@FX3&F_%@et zFP8+Fj>=-{Gi8)ksR2}pu6De_>;(0l62&LYll4>sY6C1>ToQ_o{C#~YHvFs@kk_{N zaJ&m@Z@GJ%?k~pgd++V;R?re_8d3oie_d3@3E0ss=i5lUpl+96%0oKdYNJ(Nb-j0+v8#+5FFCDuCj#ta& zS;>TvWr{NYD}CzSLqQZG2sJK2@OVHd$2(~zk$>B@%SYr`y=_Dx@Dtw_*+GHkIy)vB zPSb2p6n$WXQr+w8L9$ABpR0O;5@NP9NVXIZ=`g)}XL|SHBW0hW-D|I6W7CQ=P!@rP zUs&Q7lNoba!!{Q-dN?>aO4?YK#p6Op9Ac(GP`MM)J?N1Q-CLBa*vee7J zYy}T5hq?Y_E$KDv=R`UdcA+t%KKpUzCVN+|9_SZG?d`oOH!v}z#PuRSQ$18yp9LAk*>K86DCzpSo*iw^u-RVw z8!JK7{fLYROD~NGuaCsQKTlCl7V(VwOnHaMr4cgTF)?Z%aauF;9fy)&LlP;W z<09{hdV2}+$eZ?+E(q`hm;xz$X=y7Bk?AG3%Q-o_6M3Hc`fCPM549rkdn zt*z@K6Rnz;3zgQk@X`-ZGHex^Zr~X?N?8o8QeMAqM|2}Ic&TgVF!j9dwzY$0_9^8; zNG}h@`a~58g55P*V&>qqlY0C4^wL(uZm>K>L`dVCiC$~FL-DEu%HI4tXvOA4KB?12 z*ml<(B4hUN-&In2IJN+^iKO#|9}h4vG#TGOv+&vTOfzd`brIFfObcj_$&VwGlY@?; znfD#49<+voA4|X}!}pbMG6E|+|9U9H7zjZE!0kZ0n*rK2({({XuV3l!B0W@}b;O}7 z{9?$AjErp6K_<)+Z_tf3X&-84517+s2YEBm5*evvF$v^Kc@sn_x@L_G4=;1M6#Bk8 zzYf3SjLZir$UEjtUPm)%HE+3UD4NB#4yrb#B@-RJEbh}g558`aOu9)FF>uR00+4Ao zmbNM8nvy!(ZsdakyAGU(F8iIUwir*nOVohfd^*)Xy zn=L0YA=W=Wp0tGZiq(MTkd~{v{MLg)W6NdQ!7s{VWh-W@BcJ3n@!q_N4F;eggCRxg z z^Lfg*l=d)6YF*vyx$lEx`C9z8Oi6< zuCVOv{HkwYnq~+YKhv`zM-Pp z-nI_N$-(^Uxe{b5lGE|}<=;r7_$fY^fPNDzt*@_-7jdyYeG{4<{p+XVlmw~6>aF5J z6n_=*;&V(ir3K^yma6PEVE6ir8|nDIiJ4iB2Vt{q{)cjnAN=xDOx)aA<2x^M5`~i< zYxnk#T;j^lx3)DM)e{erjf#!+ePy>~4eH($qVDKHLGPntUcWuf%#dj9Y`mx#uE4t< zF3hVpt#@>C($LpHK}@{(v9$QUP3LHEj0m=oGWmTv6>P`ddAX#U4wnH#kd>G3I}6Op z$}$~F0Al{?s_7}^OEU#Y^VFaNmKdF^7@`=q9yN8<)U=}A+1gt7ppqhV>s`2d^BAdI zv(8c{wTI#BnI77028$A}Vro%kwBQXfngX!;OPoihgKjX5>V;9GrPv z>htkhCTbC(#zv;71P>l;kPM*QFO={L`f{1*6gq|ClT8U5-r(Sn;`guIy*s<3S5qkF zpBU3}a$>)JeHG??l9w>A-re2*z9-WWo7bEEP-&cR-pIMUBI|=Xeg5kl(gqrzN_rY1 zQ@Srf@tmi37e-+vlc}f1VejlMcL~3_O?tqlE<`w%j06<;Qt`;yk>(%6MI{ z^XMmZT#Nn;pDryXHs+bv4ZN|YD=OxDnx&>;FtkX&^W$K7JoCEmb6lQ^B~Vsu;?H_QfTxI30g{QbCCP#IeP;66 zYZ-9~fo^~nqF=s@i2g8#ert$GX8-w`mL_lpW0gW!Hpqnu1tkvYZ$L-S1X0V32J`Cv z%Ax7BG_kHXN?lbZ)E^9hFF9S2oc81=c@!bMMG6XvVuSEM{lVXWkiYn}ABoO=w1_;d zGCC%v7F-ph@ZUeQ20?C%YMTFilK*}%OHvLN>vxrV5+24rn ze?J5K`($`NU_wAAB0`}H^sB9|FC!yYp%Mt75YTUUF-375rh765MxVb^CWX%0q4^|j}x zR)oaFvoHsj%2ChD7i8k$!Bsg1_TrgqN5sWH_jd|$f4A`G_kIPM|Mf1PMS81da_WAj zfOuFdBz;oOsZ&h~u&I=h5tF%v1ve#q_m88wh(d^Ml9G~)JJZpgsw2f&?PMb;KXX=lZX| zY0?kJ^~UPL!ud*ei?VP%qjw4K3YD${op5nHD=IhNk}vh@kaxECuwhA ztX|?55HK~2vAzE1uoA(Cb`JWVxA^arb^)WG+z063B>_poPFyWvqSZokoe+I&+Fk`v` zskZuH)a+y!>ljwUb=um*&$9#|&JF1Y{t z)c;(N_Y7V1e}5G|k2dn`@ZM|)_p0a>47`3F@Z}*SB-C5#O18GK@u{g1rK6)m_#`zZvt0i@ z??{tSC@|o4{YetbtMU8 zuVio7)Q7j0mdMh8^-u8PRM9kGl|vdjkS z{>Ck^SXUKYql8*wO$a&sK6l|H{CxiGPA%ogPNpJH3@Lro

IGW)icgmq$h}~CN>)2*UY?MQ8H7unDjeDM z5TaL%&Vbtyk$u?y8p`^7*UT{zgz2Fp@!`;Y|LqC?lrxCabQP^&GstcqZP<$_3Ky}> zAt2Ru`d63E4e1_2uJOn~=2V$AO1NTxR185BJ1Bd&E)RYERlVGn*m)8jEkwB0NXe~_ ztG4~|-0EL{X~w@|>z~qWTj*k29L5ET)$C#kyMf5?!Z#p_pd0hnK$7w{a*}Y%@hWu!uepiXy0QSWqN9%7fb+tfs7Wbw zG4fR7wp*(Wv5vMg_uZ@ZXF&m!ySB&ZJk!iw>)7bCp~Nm75dQM@E_$a0HZd^$MrmOb z3~9u=D(`(uveXR@^Rgd^RQ^7aJag-Cg2wT0ruW|F@M!V)MGTP;UvYd`?C9`s*Z=%+ zasbn(nU?>51h4r|c&?Wdax>8#DU^XyfgizV-*H+t$JJH&Ufucg@5lqKT2Imcp6x#Z zvnV2vNL=|8f45ffn&F1cd%bVk+fMsdp1;yB-mTj&S=X=Ur`Rm+UnL2#$X_Y( z%ML@VZA9@^auJllRq7tS1KH|;hL6r6QVa&S zjL?#g<~tfxw_SI3zivKEIQuI`tnaqjiZ9KQcF@sY`lVFc!tW;3{ob^QTRq)B5jX*&->*f4>1aA%U)IwKyOF<6JN{VZeWsC*4?H^06HSSH%dmT=S zB75l%TWL(%i763G$|C+JP2|R-Ek?gdJ?*lgxoOz;-1`(P)!4e8bg7r>>vaO6ThEP3 zs!<0O=oqk&Agzv1VmLO71800Kwwcu5<#mZu-|bML_X?>AbrN>UUH>lfVD$(Oi;tf= zdedX?PO=(bGWdy91$hwfa1XZD)XwPF`d_psPeubj=!Vtw(oUVI?3uy|4?7CN2G~P! zwC7KSUbUp;PE{Y@8CU|cRKiIG9tL^WQH{x;X51W_- zm~hxzADMyMd$d7|^gO`)1H}RrYv&GPFRr(y5!-vxUvOc2Lg8%>1*a_Laz&sI;`?uJ zZSdekY|0)&7e65E8oXgJ^{F{=%9d~~IY~SmKL&jzgJ5V{DtX$L=w_yBv70mBzwtt{ zo?Kdwk;3fYnbW-}85^VPYKK3ABb`NbaUF87+;r{^ytZf9bT)8%98- zJyjk1@1ypoIoj6F4l`2Bc%YsxDc}m#0#5@aU?pvi*o9sID(s;0dpEH}=&oD1LT+3T z-;{yJaqaQZ(erKBdc!N+T)WZ`_WqJ{LZ1}4ury{#n=jck9n(Y4l!=6I0GbRcWb(M* zshfw#q(8c3@`p2v5RLu~mGHJZN3%OVYtno6F`MJ4!ZTqXA z0VBWYp29`R18zM|lFdYFTlD1>Tt^erdd$04F3`<}{h>b_cZMz|lX-)1TdSk6%;5LEL(c!_m0o1{ zn63cu9NF8cMQeIeQzfUkBQ&SJ6k`Dm^Bt!3OwOF+SMCCo_zXpRgWCI&lwW;ZODKE| zs$u+AH~k8waodR2FrH_1(@uxkXE(t$qah^tA$RESvH-tGn=9Ew(F2h4A8hadQcN#L z@Th^g&x;n~x>i@E4P{bZ89a2oe!bp;ZAgWqA#J8)a9$_y%X^Jh7VG%Q72=6jhh=vb zjr717EKN<9=Q#Q%(=um4`5<-%(F1_I__vAI!hw{ySUg^SdTQ6ZMA=;p;pC`0_ByB zeyxCqJVTnM+9Px{0RV$s@WCdK$tX3>7fQ*>BF#6L$Vnm2*E)A$@a^>gXESR5+eXzI zYa(!g=Xwyqr8fy*E$iSD&yvG2ZY9q6%b~%Gnh%X$7+87R1@__1A(rix6da3RYutgZ zS@U|Ek-xZN=9qL%LuMzQA2C95I}L4^1=Tp!$IWS(8D_YcsK2~@9ZQ#@soi?O8qt3$ zHzJHwVWGm0Wq8PA1k#g@I}_A%jOvWXlH^PctQryTr&a6d(4V3k zXk7M5y`Aq(be%2PbzP2~V&@g6ec>;qrr;2B$_BG~u7!n>S=aCqYlq9kE!ERfgTdnU4Ls+fWbFpy2 zCV@0*KA2ew502dIa8CFoQ0^0iHY2xy2GnOes?_|nb!SYcv5A&Cx+Tc&Vmj0Oe&TTA zlF~#5g4YPL)ys1*U#{QMh0;h|*aIn?&xH#HsoP?m5nXd$eW!(6roFgLdcmGdSxEET z_BD<8Wx^U_TRw`PP0sXMew2XVO*6A!F+~MSr>5oIQmB-j1|NM>ZCk3v=PflANv8j49HVK#N#qyOv7 zechkFe)QIdMPE7^{I73>8sI$PGH}$I1zayi_*l@&fhGLQR7)==pXkb%Hl^gRG^h7K zPS%EZmAmZ*7<-Ry1rT46d3(JOM!__c<$3j5)&MID!071Uo9eRaw)9W4={O^|9dK4v zv@*#`S38D}N;%^PQ<1nFYL)s;0zF*X!ozErf(3ZD`*5K9J`d?-Wo5DU?H6|JWwQ1iEgdS zqLra}1`DkhLN=-;NBmGRZz_u|4Ws^*Bna?SQ$t;~tC$a6muwDrNF5eV+5u09iL37!bTSk7Z~j%M$8fzLe_(^9!VqIDNa;f9 zNw4qHI)dU+wiaYlZL2xv;4}2Ea`_V%s4n%l;_);%v8t%k`)<=-;{5%>ZBD5vSVf_r zQnDuxtoqQp_erxOJvbq^g4QH^qP?WQmaQ%j>s*z9ZAQ(D6W11@<6DoRnS5*1E6r7m z`l;z)Adx9})US`5>JWt3a9%}ol=%_Q3m zUFL_jQByexzJ45daM&Hrrow(lTk1ST=_mUBC#{S>5zK%(-(l|9iXUqFo>4DVjQq%W zvy0_%;UmK(VBF5N_W9tCFK8f$wQtGoh;JO_@+2?r%I>;_qff2p4-uO!|EPJM0izB6ZJWr9g|mIFek;{eQ~N7%)JB>Ha@{L4w# zbKg;h>LS8Vkj>l5b1WZQ1w{-mu_xpBN(RgSVGH<=Cg41B^k3V(cFY2C4GnxC9$cIf z29PZhOUnNK{Sqr4$asaEv0AWH0$9WVY5+U&qYh*)GEHCT-cW%)W8BPSqjqjq#_4!( z_sX19%r^8bCwo@D0itMa_&7NqslU#P&?3?L*j=f^z4p^hK{~%l*7fsOHUj{ zqS8kLOY#CbUDZp|Y$j3yYGAoAlnzu6UjEIaCR{PXiJT$;dpOM1p%<9v!7!A|M)Z$+ zd2>W7eNoIdyHluIHAG4vM}9I{Y4I@LQr3Vk1#thz3&3|$ztysy$QpnfM{Ae4TDM1g z+kw`Q(ivbFN@nLThu~O-8C-uLA&|GsrZr$M!zi&E?I&l>93E{fHL3|d*;N9j8U77rYRourgO#}jLykRZRoM!;SI+P?D-eI^?X4kZ4bpI0G7Hm+hGAbrX4p#|H?(_YZ zXC}fWS_8$sTzF8p+;2k`eZDnrct?b@kZ(#|-s;W0>bi;1Q%FxwZv@Cngu?Uux8A}C zi;v%&)@DKyk8GySk%qRW;|?n}^Wef_H%C1%o)^u*C)41?MD|3hX^Ym7#v|&v&YjbB zYfog%bp`{5+|t32uEOy$0eb4cb)@W~&WPpY4{wb~cw|oXoTYSiNz;Ka_S)ZPpRtr% zY^;aY)+bBe*844tVV!Tj*&A;K4y`OLw+gK%`H}3a?Cszu3NhT>BRPA^r8Ft>)^srW zp)jccbL4i;_cnN-RXuNMJy7uYh5l)w4UH)mL1 zUh5SOmJOF_{if{u$O>G=J7a*=z&ap5#e0N#nSp30RxSbL$icqjYAImP;RixJC>osml5I+fm!gv)Jjo+ZB(yedNlTD84upig|PMK6VyyKD;avk7mG)pELSS zaAi;OHt!thfHwrFT9lP~bw7zn3(5}<6IZQ(BrB6RpC%+a5?jP-#PW@RD>kO+0XTyn zV_`r>4*#JV1rG3-04ESOo_$laKS?;#7S7%F5VT6Ybc!x~Ys$(28=naJ`w4`SO4n)O z=)XaJh~<|=f4p7;56*IrTtfSWn*W=X$F2i4X)KS6L%0j67%Q*$(=GXA? zZN*7)82HA5rgr%G6`PV)?rKrYR>M6q^n$ETlCsDpuGMXZ`UeN!>h8F#K0mza-K0@p zpSqj58~linxxp&`4i2pUMh-dHSu(VH6YH{Jtv#)C3{ad9>Myn(c7sBU1o_5z&OY$= zbkkP5lo{nUo=&UpY7md_#QE@OdS?EKR=9PuweTpgaPi}Ib{h~O09Ihm&40$qtPq{% z>EuH5l>{1hP!?x}vkUv$*`iS>iEkDi1nI zf+Z9YyxkgrTc3>FZsvddWZ?&>`N(r%;We1i3?hfcnRqx2 zl7pSkN^JsX=xyBgiGe;P6#gmR)#YJlF(e$0VEYrEwXSB}TGWo=3Hr}}FEo5NlikX- zE$*SIzOw=}Oi#ZWb{d(!8iFvnc`=8|4(+?$*eO}+B~NmXey`omvlFoyjl# z`<@OER8fGOrKZ^X%~JJWajJS6!K^w~6Ky_XM@tn%_ZW*5@GdL(WScuoa2khI$3zZ) z9a@{WH)F{7F>7|Uish~W?~jb4Q=Xt%w8+kn)cezQ{(t>6KKph@Z94z2J`Kp#er?jg z73=x!pJS!}h}Z4c1uz~Eo3#3-^*|U+Jw1%Z1;U<$xHx#x zJa1sBhu--#+flGUWP1k$g7{R~d{ZB9<4h=$Ej0P&@Qz0OpVtczZ}A<515?l6Z^iNErC9SLT49d|~PDaK=} zIU=(-VWK^8vlBZEJ`2>jMV~`bPs3ku;r@MNW2aP9b#)%fB?NFUxTT19%`eHWHyfn;&WAq(qWceaxz51cEzVdpWnZPL;IZN zLSdR#&o=S{Lh=^r@E^D~YI!yxL6k_3BciMRA~Dt~gKnzsu`5RdF3v2_qy|C1MQ_r_ zZd#elrjB&N&kSxQ6)f~VY`UW!MfuVHKf`?jsg}tPv=fbFyU2E5Sbvih)T4;vY z`X@1LY`1I*o~P;Ln{tyuBZ_xV+r>=~5RlkdsW<2?aNHRTlhqR0L_zIttS{<~p}~}F zdwMaR1y&#jiMSBO1!TiFW+|J91E_&5K{V^RX$XABv^6fWf6J46qeUOyMYrk&@U7j! z0JL!FFYP@7fc%d&GGEyBBBJ`37J{khT=k^UX8sgzYRp&l|2K(Nq8)t!1kpNm#R3tc z|GeZXkI~L#=YjrHd=lx9$xMj=kb3%NFNrPQ$Mwi+V3Wlp8!@4D4#4Ufq(3U?9NmU~eRNgyN`$D_I zqBGlW?Nw^e({uAHV8p^nI!!0cK1g5;2Gt8jR_<-~md{TZr4V#*ikwa3$YMd`klJ+w zDHECDYOl*oEI=T(oNCM?Cj_jjDTI}BT}rhSB?^#X*OBFA-SGuClMD<&+4_}+1EQ%0 z0&k>V3&9CiXq`|ECb)m*5)tqhLqScbY*|<&JV*=PKW46L-dY-A-@4e_C}sD&KfzaD^9`xK>UEamTse9$23+u6f^@H9-Q z=D>nZdu;IoJlduQOb|kH6f`$k?S|@!H2PsQ*=NJ5vWN4*gyqU6Ks=bXhI^p&kOYYrj&#fu@oLxJ5HuvQ#f--oBk=>QBo4mvY>-yBB^Kr ziKk$2G3FSj$JV^w?zq>V>WtAo-J1)RJX_J5U>nIFGW3@>Kw!F*}MtP5w87Z67lr-BOek zJaPZmv!%`?uJy)m5vb-ZAWylU@Cke$Xo6sm0BUo_J|T03`rQKT5yT6dmMcw2d01by zai6-cRiIr+Wn^q7!ViF7^X#J>==+-uMO1H$UtP$%4Gw%|B23K_84)Stp!U9;*bPxh z?BQ*9Uth@maC)#*2yRE`J%Ll#B7bTt0p8p0Q0u%~QORIv=MX;oo^xYq2P{%zWRSW# zLjBE(`%1-d&{)cIksR9JA3ybwX)gm)ye$K6?t9ic+5Zs0S|Rv2@bK)@_7SA1f{-i% zp>R34L|FaC=JhEVFRezzh(x?&GZfw2@XNiu6Zpgje{_WCHjE~#-uXnHnu$etDY@Yl z6y%S;gP^yhreQ&#&`463fVZkR(BfC91H?*XFkHP9raF$B~*ypP0ZLW zV(?+|S~lJi%MiRJ3`Fig*(8}MNT_bh=0?55!|fOSs{&&1=qXGyP-h#@XXEz&(D-Z_ zj&Qfd?Fa`~1ft8fsQ;*(oXAC{l*4^nyWML~IQfUnc6BlZ3Ti;IA#9xxAy|HUKrS={ zy2T$3*1AL_Jy;%X&VQEZQqVI{z|w8!A_XVl`7tDR7}GzrOg~bGojCNbgqCz_lJ43OGi~K>cojV`!;h6><0zZy z{FPyz9#>U;E~*fdE#d*MPv;pn1w%A3Iv35)Zts&Gh)yzM!zdE)vjH^C?a`F-*=bf2 zG`z%C;s!S4Wyf(jcC8=T5M@pu@B@ZMq>jf;j6LFdXIjdM%6T09zsj-Su_|y`tn2$) z1+-4RPHBMI;5JuKDE$@YoZ<-f${h+&j(>IDgny@fzaDqo=A3HRiR!R|JzjY_6h&qL z>-?nWx*n7y<|7JJT5S_=ECbZQ2{8``@a7><8ZtZcnoR+S;K#SrkBa8v1)YVeIqgcq z*Xvb@`@iCvjBYvt6mB#k;3YAW2Es>-3;u|cyrehR!z3#ADeAJohH?1DJ}@&&kf~jT zBwoe0O92)acpXBG{*>ntB<2k)oA-~^bPk)4<1s5j^{=gOyjN4~emUdg8GgytI{UMq z{jYc(fw)NQy)Z&ZHgQ*hP(*ufF^v9diNl}(D;AGo$ayU20-HP!E223uHp+q1sqy^= zDCeP1=ZO(8ZW;|gogTQYRW~=|2@F+gs;MZR8jjH|3fz6P` zf}K!2MIBB`uU6?>4CeFu_uC<9VE-LHd*-@zI>T(dxX#x+D zO)f@2ikV>dUb^vVQX7x*Y2pJ|C9dM4f9XXoqs-dTK()JIXzx=SlBk|M*mwCdoitY0 zd|g6&ugq=8qGwL3CTJoLlhg6z*>I9vnn&xt9?ci)$pumd zWBDTc2S>k{DFvHR3}Cy5{g)3%=3>2N*O|QT-Tt8SmD0 z&XTXt$79}YFcBSx1nfk`W`O27k>afUGF9|!s6ukO+>A+$do1DA)T0w@jx0v{KlWkl zQuwBvy2(Vf7zh!HjNJav=hM*8iH%JU<{A@3mLs)CK*A*bq3Z(L%uB-#qvvsajD@wP zyNHadJtmShU(-6xz)dr0gmH_+uAWWkry@GJxo|r+6)7WczkdpzUS+?7{jsbW^XJTN z8bixyuDt5wF+?am`8lB#s{*otc6p#tkLh!KcM`gEZqDIksKDi6cUN;&*@cN~csNxs z7UlZ+I_ZB}(&Aq&X)05uShN4p!5^>C&UP@GQ~EXS zi{iEHCmE!X1&eR~`jg*O%De`mraR5?=I)DB&CRKQM#SytOSw3>bMKXYL`-g16#FWK zWwwge`wvlawDt^#>44+G0@BheL&|rcZ`sY?Ps*=)eipuil01Mxu*B@}!J&;afx`k@ zCExu`1uHPp@7N1Ed_bo#y_s8dge z=!N%T{uis=R;AGU&Edv*`6pv+AQD21thvh)na{!ixBUl+I`F<8MXDt)2T^+ejD|l} zHgXZoKX5&f^7E*VpLl8NCrq9i;mWR6`N`kgSo)J`q)`Je7sVah`)(b=3>K@@MVizRE| zawK?u%RQnrZ8GRAVTE3cP-vzzHU`)ZuyJROg!$KR*ua}KM*c#N4)F3UTtt!P2~O#7 zD?jW)FHfR0WE_9@+t*H-iA)WoIB}lmI*{}ZXD-EvZRi8W5nU$^OmBsee7u9HmG08C9&1Unb27?$0z>pC1H?V~_kc&F z3xJ^61Lb~3^Ob^>FOIv7Ijy}vb^w)-iTu|}{gDhEg6Tis|79uKZcyMGWbKnS>#Y2r z_nu*hhHn(nYIhdR@q`?O2EjA{U%*qB&irSEY|OA9YP?Hgd;heoU2PI(6MD|{)1O$# zSs0O!MgVBBJ_7sz;plXDM$x0u$hMdt?qgKF0_)zjKL67-a2v2wNPDJ6 zUKVYu{kQzCwfW`|Fu|BPonr{Rd;R}rTPTr<{eJKNb(gha=tRdA+oOh9QmZuL`H9|0 z+qhoT?0-H_Sb?qY3bOkacr8aMi;N?o1p1{*565t%X0@1iVzT9(Vwp`X?;T?t%IO7a zu2ddo*E2Fgl{$BS&d1tnRYTUCI%&*0F%`+WX-b>N-6t%F!(~OG8SnMxh(ZxxT!O^# z7xNO|%tpIUil%yVDp3d^HXWa{f6dli_}E6L z$%$9|p4V;@J#q7G9o>kD%ZwbLq0Ojl_uzK55I|5c$1ts&+*HtyPUFVw)f~%V@zD$x+9M4*T-j7_?6F z|J##(gV7`Tp_9`o^s6~f%JwSUiv0{Cj$C@;Q#|a4G+9|_UfVj^E&kBeC&BBz-2Pqn zuhC4wYAd$jJ`?Hv9XB%)|AuKQxIQIof#ucB5&J|_VltFPG%*4~CnDwIn^B%!^0k$2 ziGRdjFDk!I5f!lHs|Mi2HalmhSb6w$j$?zEpJkT>tC*0KT$*V&E+^glJ@n4={Z?;a z6a|wvH{7AbPpV?s;{!%^YKnGD6 z&%X7|3OV-E_}Y+k4wc!f8G_+w#k8z+?s`$x)@F2-8t>gRceY=3n$w$a3aRiGv&?Py zV!loaf4MM5=pYw7S2OE}i|Espl58$<)5(r;Tb5Ohr0;Q<5s>jf47VZ+`<;)rVktaOF;H9pdiEh8+K%stQg-df*=*So+C(ZLA2ibmuu5PQrb^YLzzJvIe;;S zvk0^SUX!)~#?4TFbUeTQf^!3p$g|!k3YM7Pzk?#!8c_h(3OR8P(s;R^hbQ;L1=3tb z+rHQB3?fqH&ym?kJK4s&I|@ePUr(m~JXH0uFi~1Xdi(hT4`tN_g;gbp{@ z2!GA+(i&_~2OoTlq4vFmwtZDA=wSYdtMx{Q@fstbn&Z; zX`-Lhl4a1(0cCz-Xhp?4OcbNdAug=u zXRBO*BStwPR{VQ$Lns#ELffn09T}p*};c#iy!Q zV-;siwA5t|9~*Z6eCmkuuqR3r_w@zGXtjL7V)WCa2lLUXBYY*_4f43&b!kBH}i3_b~1@%LI=dA(`5r!V^)eydi%=< zum(7Jw*B9Cak&eC+3gDcO#U5^hVb4br-2NozhOA0ZCFPr_@Fe?h@T5yZ>R@D2B#Z( zdCIiAW8~Q(GG};x{IJF{F2UJ^GZuZf!?)yiZ=L<>|JjfsQyIHro1s--6Tu`D{RYF?M06MN5PGjZ&iwVY4yT;quHg=oGfO5lo&_mH8z-1P51asfGbkyG&H zP+95?^Zl_S8WPrC@Qnc>P5pWj8@9B&v&~K^41`& z{YB@QC4$L7HhIBscTerBvZEqx841g|SNs16J98RQaOE=mS^8R{&t0NAnF=xrZY>3i z`hZ5M2T7j~57&tQ%VzE7RU0|g7;jzsuS=#B9(3yXNMv2nVzu<1m5dP}Q_-ip?GVB8 zi3UAwb|@uHPk(Vz#u+~AO(GMjg9C~PXN4z=Ach@quS>n;rF=z21;ReSeH3|cURJad=Q#kAjnHk@Fx6+>6iIfRNb~Wx5#qK!vw?+;5y6Pxk~0LB zZb7;0lgKjvr(a03es#-hpHo-)+;DuKEsT^I@_e0a$FGfDg(d$s5%AUEy{n^4v!c_WW7}Sl}li8@vkMO?RErfx=$@f@P!VnBvy zaV@kT)9jbV7p6C0az5L!`2WdK?9;wcb@=X<8htS0n<{E9EzYnD54z>CVfc}wsz(dS zO6r^L+D=k?q;0y~tNtv~dAW;9*@9SqRn?0aU){EBDd;1^V@6Xs1i!<@!bdpYw<_AB zmFzXpBBMpRqV^pbkLJUI&pWtea-qfSm;=q%&M^cQ~)4Sv-` zo^TW#@8jI0C33;D2~A2S57G^s2)4b$By&0!p)LupO42`xI6&CCb7VcdC8?hhMU8H` z8bgN`>2UCBR9LOh9T&GbmJL{F)DB3S{A)Qfk>^q^28GS1B4Pa>FTnj>&3078Fqy&F zpZr7FDB)3!e!7p9173oWi~FR_+b+qjD*6%YsvX|VEQpwg0c@RoYZbDYePlSu$?ec& zIbXxyz8@;M*v*Ug?KhL(NQgR(;g9{6Vq5M4G+cO}E`^4>PTya?*TJ#on++}r&^@X9 z&1Z(SGNICNG=(%l4f}{Ai?Y&EyyZ3^ae5GfAhg+wEKh{^e?F(auYTz{zyEfna;s$K z<|6cwd%Opn9%lgGP&(E0C~yP!1msycwpaf~7U{#H1&{&{Z+ke0s$YS7X+i-Uto4@c zn(acO*BC`fyCUC8E{RuahQZPVfvErFq#7(=IazC>Y8L1ncB=z6J!e9E#)zm`=e*B> zp)^bF#>gq)N-+j75b&E?4}Z0b*mw)JvwF$m0ZDK3%}SuwPrZMzz#;x2*1w7sL;J^~ zMw;A_Et-&{r+1hVP>6W)TX)acQy-p8*u?s7ZdPW|aw@l5J=fd7)F z)dKhOPz(l{)DFf*ltppk4`>h@+rjnFAkd&u9GmLs1NtereVBpcKt$q%v%wlAZ&K-< zbofs>U%C1(H#%F(Q=VT+c{zSnGEE=K+>qDDQt`lv-p{_Zk<8&K? zX?U*dOM{gAV@2NtW`Y`mc4uhfSsthNq*XrOkbYMOsZ(l`9kfOu`JsMpMNY=}3H?Gd zZwSg;x)@a<&~4aBXe2IB(_LvWTg@(1Y$!?f6`1>nzoWa5CYQFC@a-Zg(;mX5V#r_~ zY|mbMtJS@q@1%Cy;7dT2z3kF4elADHcpcjZ#rJ~s+4i#Yhz^Luh697Xa9+nA)*9CN ziNJ-cBYL5jM2@QZ&-sUfL2nvRYpqXM4v?iyF-dzr2b2CjfY-f1Nln6QfXd~&@Xp2^}F^zK$kq^Q@(HQPplltoWI?(^?|evj@~JfA?{%K74@r!45) z*v;EgV6}GaEJYUj&eaFa!UHV(Sq>4hphG!;lW-HIb@W)hLP0UJmW{!-Sur30;BnuRmetyx!p@ZwDIHZd3depY4b&035CGppWBnBQ?zQFI(uQ@ z*oWL2_3r)8Aao_#&x+R-Udt$0gzZ1i?-?J{*{cZ!NvscOGxY*pmGJJj78v zz5p?=zyK^!LI7pCsyu231f<*>)1UMoU&M^Y>H1PEz@f&`Nn~w`ey<4E7O&TKTWTtS z=sq%+)LEIe)}7#Mc8u$49NP0I(C7Su0`$g-t80Ljx>40fj{6fRWP8**JW7l*_IPxW z&DVDrpmPPxDu@uGn%6ppLlsoyEn+o1lB}@{yX?1q*ri$W!FsB2xS*&q9~{gqi5Xe= zzmh7041tV~%b>{fU_Ou5&jjr{-icv8Z)C=Ut|l<9f8F&my<2+AsE1JTYt#?6h=n#@+-^q?_Ia^R_cLQZUG@F^YT6V~^<*Gv^O;i`)?X*(pFl;*#-^1qT$6-+Y67kulD;2LY7VF=v+7}` zvP3yMQ6x3IgWG{dSDYV6D9)YszrfO4g`?5HbjdZwl2kT7be25zfT}f^_{n_4w&yb! zABnk4>gQNTJtvCCYEv@WzYj!lT)#RBEd?`I3w~dJaOv*H2_iR zu%QC)CFy~AxfORku<4cWK9Ke9q7%3JR(pTbIXQ7WLu5L8eU zj+v(9itFR^cUnMhGGi@E1N_Tac1vkQVcy(Pa}Dm1h98z!`Yu2y2ShaTH&&j?@Pj)B zrKc?QQ*=&q{fp*M+vFheHRLS_5AH{~{Vx{LvepNwiJ%|M%on`y3Q_@$w=x-Bieicv zgu&AYF1BOIkxlgMlA911B?>`oDdjfukzZ4nLZiTAy9#Z5E_n$kf(^6+tG{+P|9t=g zmHimrK=Qo*?tjDaRC=)hH{xp%vFpA6vcedYi*pcusd#5AdQOjJeDs=h=y~Ks;-KnS zzI|d1fJP{fbFqC864LRBUZ%Vd7AUy21v}D!)}gqocXxL$w$r#Bq(WOLHuDTxKCfth zLv}(~q%_I%b8t(|W7(!|B^2{!6M9PKw(tkw8q|V<_Y^ox1O1Y{0XPozHY?vom7iT)8NaD2kxC}I zGA7_e?eA|W{)7vLyXJR#8yJ)FbLd-p#f0K6G7KsJq+*vqD*Lv=+>2i9YDi|ZF zj->mByDH_%i8}Qw0GqFuGW=LUA21) z_fQ8L%gqVD=cB9VV^iJi{qm(`S&sy-s+eD;Gvg$O1VoOMxWcA_w z$$59g4m&G_DD)c-ck)$0JgG=7E;iQlY`eImd=ehB(PmD>b~Nrxq!pR)G;C7lJ}T2n z&9Q@UOtEImHNNS?@?GLhaf3H%-um7x;(SBzp4rdd(SC_VQOGSZzh6FJcGU&C0i8!^ z>^u}>;&`$L?gz9 zdBsWV6~M&yepRJrGg>{fwDiquy-<25P=Ikd<0n;y`cqv^Z5(KgT$w*!ZE3P0okdPV zCMoATR;gHpEruI*Ad&k>te05P%T8QKy=?nY)P@){S!D zxt~1Xu12ODNZ~Lyoj_bz=S-Z++lS&_XdV&kchta~gaL^J%P(a_lQ9L|x*srOrjn~mt@ZvB_50#$J4QR0K| z1q!Lz*d^n6qG3~3gO(22aE%+CL>^1~V4Yp_NKp#z4B?CRJ{K>5#p_AHRv((*G4m1L zsP*?08M6*?KbcmQVef&8><1@~zuq=LL*tq=d^OHOE)f59vI+F4miv6FOw5d%gMUfm z2tGsBKy>Oe(20Up>%cPff_wmXV9cq9W2EZ9tv<%XlBSpmwS>EUJ?o?FL5H`bzhn7%^2k{=IYGO_ zR1{zN2l5BNGr?_vRez!3lp$xQ!;kDpP}ZXlOt60NF27SEeL9tR1($|hMv!tiVf7DU z!!3BAr}B=h`Iwx2qqQIVGpfawd9(N}dKD(^=kK)*t#}`cJx4Nh|KgVncfm$3$rzGR ziOl`TR8I5IdkU7f4V(g!*8@lg7Ct2A#WZgxMuw9O$8=6g04vG|rOA6e@9-B-2}2%- zX3v*d(#*zr&&g<_H}{69S4A zJ`9cFcTbHJUhB)Mf;4AhGoTR33#Y$IZYMi}Z8B~ZOp+TBj#+$WIQ{k|?c4+5r^|{{ zVm3xsK*ee8oW46nA7CzKJu~jhD1pgh{LZ>rxyK9CMdf9FH8suKph@AeqDLl3ivWh~ zxyas`5-l150iq-HEnRanv*LmsV3DL=fSqxRRS_q4v-k3-ynAnU1C-iCmBY)XOny2r z6`|t5+2dR!4b;k+OWhKALz;sdtfyBN(yk-4;>+EbSj_u}WC?+PA#VL5A0)XZ$k-m* zTgtHhP#Zooh4E0WfPl!ZhP~%5h2UvlAD*oP1R)P_9ANiVT&G$N=h9r|@GBRL^oWgVLZ54pr|cMk9x{9`s? zffrUQXpP5MSVF)r`v_4gH3>#eO9RDy#v*+B%Sv?#jUhE!flg86e~if6-_km^R$wQ$q+ z3EgZZ$PZ+V>Mw>}O-2gWXJ*xSiifqtC>5ilH2up>JTolJ7%%1uYC5Htj9A~}hm$l) zE^@{E;e=;PdQ0FQVM|q!V>>M}>3)*b69`HW0$uFnWN(w*1HILZokQ&ay!=C4V zon(9?y6<_RDPp)G({)}*u2-Ore>Y$Zy$NjRffY(i;{0?lsD8vW=KH{O|JKhtdiJWy zTF=UecPfM@^aplwOeKw^*g06LU=O(>hE9+y#xxb&2w9J zdXuB7qP^u6@S*2iBnJhkwY{u-`hH#ybl+G1p$$J9sjl!m6Ff$?X}vbJu-TH}-VvbK zxcm=J*=-z&nuWCw(+v_K43&tB}s%Am>Gt2cduf^qfB~q2S zp#0EYCh-~i&4ceR8kKMzrklQlPWhYf1v#lp460dgLk92ER`Z55P5mBSFkBtnKGh&DFnd416VHJ_CCk^(@ zd^^5mrwe!5b=kNMo z|KE8Nc7r{D*WRD^aXeqgu>;{^*AR>Rq0Mde)-*Mw9P%o09WeTH8){$M4H(+OL)@o? zC*4%pP~q}n(1+?My;cY;q8Nt?)2K^-uim3Byg=hh)=SIug7) z42rmUIN$Ye^TG0oc2Gez*76&`9# z_u0rhkkmLYJ|;6NehH^neKqoYk~sk%`&(d~wq#R%f2(H_oRipgRjbbLfGTA9^lF8i zWsHS2Q9|^_a6ZZ<#fXWYEFy0mq#S_}A20z8bdteZN1UR0Vo~{2L?| zzOY@$Jkdv#qqB5`r!5)RW#>JPuQshg5Rx*9953*Dv!T@_$b*Y0T%+jNo)D2(UiV`~ z9I_6X{<434>9pS;yd0?KJ)Ntj&x@B^lZ@)ic5i$?LB~Zx{jyRB*a93UeP(V&TueIY z=fXX*O6EnBq)qLqxDv_g9Re(iudG7U zS)Eek zohJJH5mzoY9^31KY1jPa4^F8-zI&S`Ma79mHFWLMb=QJwP@N{8%UO96I~M6jXF75( z92^Sm>E~0;&CPGUTo+ILEL4?g^Z58?F~ThJ-rQ5ehY{E|;vkOk=Hpv*GIEV~`s@8c z-=;ZhM-qN8uJ4?ukU#@bsHunfJY~-&@w1Q6V@&Zwrm+SU2j1tvIp@mf<6CW3J409b z4GSM1K79+xnA^JzI|a3#9r$33N33a}H zYO9ozFY$S#`sSNLDy@!GcN+XN!h!W0Ot0&ASg(fH2UQZJ!v5Kfl2*Tcs2!wz%Cv?- zL~`ciKo1yD_CbARP1Is4)~YwLtXBiPhWJKuMZFk@>EW&rzY^^P9tn4GM3?uRnSi$n z1QP5CCYJ(Dfkl#d)NQ^oNe?*0GcyXpULEiiUz@DBMbL+=N>I;#fHdd`8zNWQcJS&ZtH~zVC~- zq<7R1q^yVtVeS3c{2Z0YQx5!qAB2xZaaaAYdsuM$qdL^0ggk`*RZV8Gi4Rm3bAd;# zM!-qTt+lKGN>V;23n+5iTMz-u*Fh+7we+7bH(zzdjF~fi^qHNQr-{7fwb&PTdf~E& zEqSgd98W-8^ANy?DJLL+?_KN@Z_FviaRnk^L|s#sU@r|jTWdJ^{mA*Scg`u2kESkb zN)^UOk(UJ1LfLg66P?ogYlPbGd~ys{c1%&u@X5W%yFyBexu#QYrhVMoGIHIwL>7UJUa?csp7ieJduqBJ%O~1bl*K(0Tp(SZRaKum>Z<%sUQObk zO(n^u!uInbz9Ef(CArQiKeYw8?9xUG1R2<-(Ox?(o59tbQEO=r~KzTTq3pL&8H{t3o>4h7HP+b zVQe*V4`U@Tm4*Tmgz z`Fn(?^f{iG+-=pRxmkoy@ldW}7BwpQHk=S(4t!E4hJp5EccqgEtljqcj2Xxc&NR4B zKP3JD`ye3?qIT;8>W;x7y-8px!#BAuKkdF1lSstnnqoTeUTS*L+pxl(iYy9R1dObn z#37C&9F&|R0NsW)>*8Tkg(k7!7Mbv>{d-W$E`NHslPZ0rsmA9%Va7%ErKM!me4qEp z(lPkGX+abds7i=s>jT7Z1&veIc?)mvWeF8YOR-m5kLb({`rBvg10o&+geAx#O^ z`faz>Zeu6!zf|Fv%x^EUqHu7LD+imLESgN9L0NXk61-7lRQ4iet(?4Sm}2RVOI7ep zc%4-__UgM-n3q6;8gpvR4NWdImFB36=edNpW|2Duyr|Hx7yVS187j$P4;!Pn=$Hvi zv^=1{<=AZ&#BSN&<|s(fsFLcslm)!jMj~{v$@%0)&{T&NuEBp5- z$+?}rJMZ>O6xDC~b-TX8>e~DYv$_$CMl9rz6v*<@ z+zQ{Lm0+x$`RHfKd@Sgl=cpR6L{&t3*-eP~JP zjRtl*Dc^N?M}_pzkcwK2(f2Psf0H#@@4uHwkzJGI1WQ+p0Sn}Gt`&ee{xaW0$Dzeux<(6aBEqZxEw3w3%=t6e!0?5e3cHcmqd~*XZMD+ls%nW zxx?!j^9J7FTFZ`mqP>cwT=~3*bP6N!x|O@}N^5rcpJDu;*<8X!?&8nhK)`bOul||Y zYkKG<7S{#)Cirl@?gu&kZavP>O1}odI(=;|(MpH~rEcZ7QYP4hte7`Rd*n+!2sy6gWhAx#RAcl( zSe{llNb1C^OOn7F-*rZ!qmq*Nu-K?zJ5G#Mtt*eHF8*2~28^-tBbTEtkT8|*i zvNF+nPx5e=Az@TcU*5^4vq;jF_Iu1>RnXHVgs;m2FQd&P^Wh}ROUijo?CxJ)vPA{5 zF9My8x9Spvu8bh6Kq{L@)?x>WE8)dPJH7sJhMFex93jguPsEeMV<6l0BW;8u%M)$7 zf+Jk^zrv=Pf9&Nem;(s_Z~hqb=_GZ3i9Kfbs|awvo0ol9t?3V|>+{vbab8&6M z86OR}By$%Tv|YVty~g0{Ak{v#7i`mbzb&9hPyE9s`?jvd5J*2)ocm|RQ&VfbM&p!e z17FwNt};qRzM&iQi`FL))IS_mNv#aEz*0xKD=bOy=FV_rDXDv?-H%YC@;W82ps{z| zmtS(&exSqZcms2eVbe!$`>S$4s1GYc+@(rl$Uwo2jt0BJ*4IJ3FUN0`eZiy_9aV;A z;6WhmNh79hZt|WX`u+08x7Dh@&vx+GA&m>G-lTZAIGxx$z~U+V%kL2-d~_kCXKO8(vl8&dvwlWdAv5mm*5=yc@cH z#_0{NrYfd-HnB((A@xTVCl-~#*T<7|wOur_xaxvFV+%7gy^xR#=tIwWtDV7d~kI*{6jqA{K>z1J6kWdDnry#2EM)} z?)`|==YL?X{_Ll_V^<{JSseb-TjD9rxHIVh&KI0UBi$c5t1{@Tco^?oYkKDaq34nj zkt(1u*AN(G)AtY3B+}HfB*v(-_D=JS4nR5t)I&@_iCOJ-P(^;Go+TP_=`DLg0B1fx zeQcG-j;-X(9QIa*X4k=Bge9|yx%OUy`%py+Z+t%Q94p?5rDn%ct|Ce)_MqZ2k2Cd9 zR^-;w2lbpnc&^zu*_WOfXmrr^1hH|t*}FFirM;B(YK2C1Xokz=gYC1#I;GM@at z>v$)xIQRJrws8~8sp`SEWnuQ?!~Kx%$;qW-s+V(1Zyu6aEZIBi&h_5JAH6z#28l!x zX9&=K4l)-vRwZu(MStEr1|*v)mvQ(*pBjEu0o^ibkmE5|nNz!AP9$$=_Nmj;K5^Db%aQbF7IaW-CvU$xGiGigF-{6d;geh?;v z&Y5KvhshFwwFpWS{>JI%|ByEXm(EvAF67mi#3oF$5+@4wqiqR%-=1jt+`DgPygi%% zcIcrOD@9CPH7|Bi-0<>qp^u%=RAS}ZmuHB{C?g|Md2Wa^%BDD0VufKl_+xzSbvTBDh& z!XS6Cy@y!R`C>cj&{-B$x*ushM?uJMC(IXjD)Z>yF5g9l0LaOD|UFQssRewEZv_O z;`;wR@BTBTovOvq$k(^YkJlgmGs63~HbAvZT;ziuuKrf6C(do{xrBm(0yA9#W)Ro2 zJGL2TSIYO93cp5i35vfq*>NMQ*l|I1)O&r_ubO#w7Np{J6_oP*B8`amUdPRa`$-=V z`O$&?Y?!_ixP$A79e{UcXS1rj-%Tv?pZO6J?X8%%D*%2!`a1p1msDAHo3gc$Y5XQC z-o4g!aa`w-(_K$_NCh=1Y(eVLz(~04kI`Y^%gVFp?$m>hu^V1_E9%=1;RK3H7gR`# zSBn!?8XemyscD%%m11;d>`V47kb4-;N32X?57pE*dw307kzsLMMV@z$3`)^VT|m_( zA~oWbzuzAuGPzAg2WkOUIys0JzFdy5^lcMkUtZ|*2Y;QK9;Fm*!ulL!t1v?5n#qtD z_%!csqe$=fW*h8s{Q2acQ8%K=OzDSPE`(}MHrz!QztW5i9&_>X1E47AHdVoPjV-7` z1k{n3{d_b2@SFzhTj*L7p=+}l%{mB+4nu|Q3ekB@Uau4;d9FoC><+sRn_4m+wam-02dy;BEFWG`_rnYLq>2-fx-x6^O3p%&<Z1%1(YrdDxfPT+w)TTGP599hi)xG*bEbCu&=?FupcF#O$yZ$)f1ekKW& zWw!(?ypA*_yY?=_G@Sl+AD(_Q&HK|XK2@gEv{L{aRLCX)bp_{ly?^lHnS5ZUu6!eb zz5AK^HF6ut|285+ujEON&=bA9LJncue;-=Z4pxnlBTIkn-}_1nb?JwKma6hwuF8l6Rp*PjPje9EblSi3wl+lTBq}e1ir;aseP4>aD)7W+>@Sth(K#x(d^Xy zz`@*xzrQP+eYf8h|9SU>D`Csy#`%BWMwtn3BJqeCKi+rz=e1jRdbNm2OgipzRaYB4cs;}a&DGs(BI1EXl@$*rt7jw6r;p_p zv$?)_>PxsjoGab>rk@6p;-Qs__}A6v?cz@3)PuNs8}w5ja*MIxwN5vNtiB%Crk8EV#S3)? zm5yaT!k}o^S9Wxqm7$N9XU`V5{okv?IlQj9vp{OZ^Y;-%?)zXufA92n%Diu-p0)N3 zPm}h&lw(hoW~|L_d!7ua3fC?<7fF97#3T5ZRh6M>-Mnr^S_W@;?|rvl+e{M*t&r!* zNvXz;n3PJj0Q{i}-0(dycqr{j$#pnG-aB8HJi{aPW>*CD{Z3wKfrxhoO*lKf4v5H0 z&6N@QXr0tCO~s_lt~SvaN)rTgL$L?mK59jtl8~}A^{2{!ll>EOPl_d0XxIx3Y42B?e9nu5`*}+mcFi{+;42==G=e3{&Si}vjV0U z{%ye8e-2dQu`s69sT>oT6QLPtbm3xirUH%+Eu`@uklhCrarhK8|TO{@~mSyK#4Iu#g%XFFo*4ovvgAf`0WE zeSPWL%N(KDZN8V$n)IoSo#}lLvfRk|6mXQIs^1Py_<@tp2ZbhOdZ8+B)ldPy{Wld` z5Ja#etRTJXmTv%Gk;qOBN0Aj>4enR$I#275EVT1JJjdR3&M8#|I4$!Q`}qxR;y$Lr z6%{QC8@R^HPZ!D@UD-KwLUsOnuC@ERRHdG@*Co*yy9w-77c5erQtS#qewz>sAqzuf zpVFG7(}f%9GTYyC(p&zk*(yjh=Lh~e-D^o~=;mIL0BPX}4A7#aM&VbR7f79y&9 z3;7*M95lU8v&^M#jkVup8@iM0wR~;dkw9tL6$t9sy94styk0TyC{*^MLuUmaQoZ|? zNHlH!+>^JYhe0Ode@Q{O;Y-I2-!)(CPzT%ps}gnIk`OM5;diR><95tAh5qHa@^RL* z>?#Zn54$?A*->ekfV{g1LzyWN(Uc0X+dI_p%!3gwfVo^;Zc}odO?&9M>LKl6PMaC< zh$0-v{l>hm>9|pFdOU9%_=_c$k`HoLxb;eXk`eMXTp!aL{$*4k(fTGvr|7N=O$Bvr zLH}!`k9|czM*78jzrGsGbNYO#kjYV=%}OM;94+|~v^0!pYGsIFMVCD3!S_?*7>x#B zb9=tNYTknOEGw~_0^DI=svw`PT?oi~h!#WuQOWL4vZ;}ONJv-X=vcofx-yF2X?yA2 z7fS<`-}gL&!j&`_phbFjB#F}E zFICu~W9Xl@p8@Vyd$O5S+59C*(wn}7*#`N33*Moz=~-PjJMi}djlD7XV6wl)H^3+s z*OL$pp*D0=5Muc3kvtV??*rO*zzPC**Go&YYGle4NHm~hrxiEk0nlfXtRhn;S|8|9 za^1Lipoi*AoK74@ro_)S#1cH(Y^tysJ$iaolJ;!8cr^XG*VKPVNqBh;(M1xpw8-Y> zBo_0R_+MB2Fjg1uCiTv?shIrk+lZr+KjE%)#nQ8l3Ch9gR)ZO40t)<1Fl8Fb+RX|& z0Lh$_&!&QaB8aO75^v9pzL(ORQ({cN&sdl2O^^3ZOzxLRi~-zd21z)D=B%GB5?==Cm=d`{H;vZ9|y_-LR&3t3OeH|1~{JUq_Up})|9okgH+Df=$_%K0b6nVbfJd0$X`Bg>*vo&F#93w zIUL~#ovv%?Rb1oY+kXYI5=$e69&Xb*IADydMia>mb|0Xx*jc=QQIm6qp zEdv#E=G1PUFM1QUi>h1Ir^5yfR{!bIgjP%h)K38itc4H z(U7MiJ2zwT{SZHw)W}!xHn&>hh!4?6Y&|akqS33d1@~c}wqO5x*>PsFqTigzO^jBp zsjzL9BDiE7D5_Gavqw+x`?RBP1V9sG<`rH5_S_p)}N#4qAAs(I_%iXc~^ki zuXJ%d?kTQ-cPdphF)XT+;YtzLyuos#qkp(3Bmzi)R4r2|$9#5mI(q=1tAJ&W3(7&@ zl}?G%7odCnyQ%@cg&h>sFWxF(mbjzG;Afr(Na1=TF<1-=biU5+PwJ9e4xyTdU|uD0 zNm7eCq$&UIp8C?ZnFe2Gm-pRHo-3`hJHUXS1sEXnl1YLvaRvhASUyhOROy z$T4L8T+Q=N+T%Y1mgo#~aJWFe%QVum&Bo_ou6mTA8*2DiTa*v2mZA`k2k_|lJEy7D zpW$3FO%!;g831kfd$-qi85}SJJk#^a=>7yosZ_MQ8;<0}I32D&itgw0*1)?NQkyJP zect%EB(&HnAO}zWhslSRy+T1Js%+1@M};*v5^|44u5n_(3 zu6V1lr94Mhei>9MGo&%RpBEG`8RmFrK7RLkc(^NR%PjMy!ZPp9&S88D+v!oyoMO!P zfL0_yf1X&DZ9p?F0?u$DWc&U@k4zCp$H%L;H@CMivHJ6je~ga*5o|gIYvn}j?PnKS zFI;y2eH~mqd$(NyEP^dfVO0GgX^7Z-MdsU z*$e(|pvb*vqJ4UJ0a!keQ8Wp1jQNY)+g%1KHII3&Rff69ztdH6?qcoMf$~;8 z0SASfxi)DiD_XSnrJ;V>@dcoZsNc~vp5az|sqhNqP_M6Ss!BXksOm4eC{cKyxgLxt zn$?oONXq~oO0&on-M}ZUdL0qzJ zb%asQbdx(V_vygK4Ba}vZ$2P^7Q~k*2MlQ|umm$BENEOeGgTkOa};YWPvINmhia_7yj^JGrhHFR$Ex_a+`Kk!Khe{I zt6-F-@$wnoL!=wa3nWOXDQlZkE7UQ+`Tg)KyVa54i#)61&owC4&#BUE=$dz=TXSH@ z0mWh=>+rxcxxIUJewkmn-r%R#Anw&Mgj}AotmwBTswVQMP41;s(T%6H97{!{xn=Rz z00A^=;g~CGL1HyVF@sL$IjS?X-=WuU*l(d3CSBv~j<`kA3EY+Wz0}=MJEnv8uy?WW z1gB~Lc;!1Ij0g)Rb}7q7Z26n-!rSB{}}+ZM52;h|i1Ba`1xPeZnPIqTF~ z3PV`$e%%q=#D6s}Sj|u9(x{a`%%A zzm10H=X6B#;-1G}%sLvisIRIVG{Yve%Cb5#FVyF0r$h6)DY10jaPi*z?V3(HVI(o` zDeqK;PX_W~x#v&9I310j?2e!I`9mX$<^RgDoc5 zsthe6gBMCdOB^c_!21u8+rN+HgBsy&s=tFTig03!goR0bnHr0y5Xylvuj}}YGD9mD z!l>XO@@W4VINP@Tl9HPpDt##zleKJLHsjBVpL9v@lLI});9&R=HvJ12ICKvboibES(Pn${QAY}bU9g022GgcpB6gw4S!L+$u*YUwloR6 zzuVTAcGv&r*mo@|f53ED30H}FDDYn=Fz{a!=<$C{;6AP_c7Vk-rwd+%+e-pGf@ z>y!A&QMJ5AJYjxx%C7hh`>_)Bn6-vd9EAynYY0Q%%@5N$FB|8?xA#!~HWWqic(&^~ zkzG&evnv1&~6;oA=ocF=Yu%;Si=^%gtij!e;#-NOP88#_EgV1Bn2r9)XYP^J% zXTgT(w=db90CJDHN)&tXP*2zmTt8Ue5Z683NSXwdds1_;a({k`q7k{AbnGp4w*h%& zSb0wX{sJq^I*MpM!6=U+`KeBSaMGlCew5ZgtgMVEMwCvCS;nQZZC0cYOeEb-JR@MD zU-|pl)+nBO-qKuL_)H7T=g0(9IW>DJU`AbrsVT5V#F+$~B%6~Fm*4i1iHvyZ5}UAa zNap1VqDx}K`TsmP5YFR&Q#dw|RuNtNq-EHJI*}-PGFCbAO#8^yKeM8eCOt&Rbo|Ts z(F^8TVhn*UPm%H$E|rdAecd=GaM}c2UV<-Y1Ru@;Z z(iqG1Vs@Y5s@G{0fI4RG^J#RRNQC#Nj^2vABFH)ivx+$CD0b<|| zhg_P5k4mYR{NCRsqiUC(>U7!Rl%iTj>x)&k-aX+Zdyk9{YxP1u!%`;m^j}i!FjKLp z>CD-+{iL1tO|a^rqK06nm*;$HfCE57qFO;K%fQR`TkPxyPW~$0$8l4-swcFb`U+;b z&i)1$J(c0$gEW@7Vl8e)Et)Eo&baTM4`j{E{R%*2uBNN-+oX z-rTV=6mXofCdpzil8)y>UCLm4Dj}b^twY#)yH6=(%Fs-1T(eW(iWI)_d4B5ixS%OE z;a1Y%VJ|$g58!)zgtxp`^OmhEYd z_CL)>xnJme$ZN%O(VA1|zHhcn0{lH~@Oxc*TC0L!>ba&UE6d$O&v#!etWs z;-bP%t;r8{|Jw*k7~>58q11mwnaI6lJfKZ<^c}+ofTjdGfyex`=}u|N7XQjnXTiDr z9Wsl=T%pEjl}nGVi2PuequMLG5W2&)y?1!H+Bd1-zs|wfx_$!a>3d7iDE)<7AD+I6rIC-B*+>_W2QqA;!izkD+dfj4{ zE9-549!)6>j(@?<=Fh;>UVCAl62b1*2WV^1tomR`3st57;FC-GhS$97Gx4;rOy1*O zpMdwqCN0~17KFmEa~%?J6V_#Y(P1e6bsyb7}@Y^}Y=jvHXgL2#>f5h_n5 z1w>qU{(;YLPow6b7gPxw&$eE%UWDc1hwLs1vjxJZ+Am4>xTZsSIDD^jwL#dhK@N(lKYU{zi_}_lGOW8@!p6*F?Drn;^{#t%D zO@$9~)6y6kc2?6YjKOKTb}zI(EJA6#8|p zQZ`X$g!BLoxPf*~u?d-yD6-5?L%A5WFab{NUv4?>CODx;cWI*FuGj7t+DF;@LFcOk zAFu>Rw$8igKgj;iHO?jcx_0qeWEOsD60}m@h~01yM3i(h?PxDP9_2n&LX$ffZe%k* zyJFGt8Q}BV#Dr8a(LWbheSi-gS|w>{$TxN}nht*Eon!^eL#2_w6&S!(FACa)9yHU3 z9l)G_CicZqTCpE8{qM1~ud&1lGi~AI%)o!^Hl2Y_B60djn| zt&?A4?TO?60M#N7wQL26DU&~~M&4GgGs+vl6|wrJ{mk}z8N_5bJTy)zbUA#dJDvg zJw^NPFj6U4`xIWLY!k!kSD*XOI_Fol`2$|BB%t`W^VGngX{Q;QSY{o>{*|F5f~D;j z4sdAJ;lCT7Wz=LASgo;8;lNS@SvofS+ z_kprX62PyPLy=9}CI)Sud(kRaTSk&Mdx#k!M;@Hzd2e>5JPadTreE|kD(y2%Rd{8; z$F?`<+WT1bLoM~*hPjFXu@FzjqLH55t3ylHMNEG*-di4vba(bnfl!O%{LRGDvEjXW z{Bm7Y{s|?_6Mp`M=A3+OT!>0`5Y)(OyZ&19Wzu#UwlyzX!89wL!w+#!L^7i-7zga6 zqPgP597IF7X3iSKW3zfb?v9l&q@&tX=BZkT;ab;$;$ZH;hDw5 zdv5gNzjy-H50+INa3wx(!E0_E%L03C{KOnS$?VNb62a%gUQ}nh5)qRBH;(!*P3GSY zVB3FK)x}`Kk?BU~LE2zk!@)yKu*dam$MvTi-|s@cLz4YhWe0CBcbf7;aWzy=?d-&( ziSAsW-Cx{0OJe|xT%Qj|h>d+V$iQ`y*f5G(aVyCznxc2B3+xhh-<^T$1jN^utK-Y{ zklMb%62pRheZwRiBRp%z4T-t_c;Zmctjzl>L(g?ddU{--*{y8Bf=0%@VhvvOpgY;0 z4`Q#0F58z@XtXABbpm157u@p!Xb(hDT=^mVuUmk}Gfe!poQ;-LBNGM2%#`~>d1|S; z^pr^7wzl|lNcvPycsWfd#(zt6)bPHR{_{Grbm}aoPme8`xd8=u>eeOJ{J|O>p!?|q znMvaegs!Qob64Xl{yQ#L4iZ!I7KqfgK#M*}oI$nwP+?KbBo`q3>ZYpgML5<4_aCPC zUe(Z*hg~;^ibpMPYj{`TC{(#(-deq%z0+wE|4Ip&bp_Mccnq!BWer#1&}Vj-oD|LN z*gFEgyW>Q7so<}?j7q|qFr_VS|9KEDx`5u66&5#eVxYB5%A#rbxX&vQOr= zOs4nux8-AG@ZNQC?OK6v;iZ13lPoj|unG#B7f@qwvTg&nG-X{>YWl#5NT}}2`%&3G zR)5B%Bbg-Gb+c|Ut@I*mBS&H(=cD=O(5luzz?ZPI{_rkj$4D5-&eY)_s#oK-A~>gm zSwQcn?yW+M-S6hDhdzO-HP<1Le6;&6Ex5aO{s+~m42z4}03?txz?{YylF1h^Lsfho z18#47)bGMf_1u?g-U(g5Ez2k!I_$;bJ{YbOk>~xb*?ibd-}yvn^n6XMv}ZIAX~Sn% zw~O$q<8&&9Tb|?rRq#29Mw0-q_Q_`?SF}&4HI~Dj;`zf#B=q8hSf*ZZ+I%m5BAOemIEgl{oVz~VcmVBC; z-LP;0V#!Zeqo_u0-=A%YYYCP~f8;vOQ|5wJQwkT>ss04m7#494)AGm5cA$q|F30KKHFrQBGfK%v&q680s6Cq%2b62Qq(Z zzNopltC3C(1zSFG?)Ob!I|#S=WH1mJ?bi3Rt1SlKIbuZH@>Fd833l)CAP)fFtSE^f z3B{)}WO9q}*_l-22Q*5__+WEn(7v$*}}6+-i|Yq_veNcMhHh zN_eIcPh*E)J#8W@!e7=C47)Q*x7>`=2%^ zNcn`$rzT^#fTGLWK*{HUg3Jf=53GohW!|>nwP!JMP^BqyaBy$QOa7ISbKbh>P5!5P z3Q@`)^7~DFd4mWwIC1ekyGg_^JtiPcmm1JUBf1v|O)#3A4#2)0X#9ZzeT(NR$ZxFzi zYreggc)W@~mbTPRRO>%F$dGmK1Qk1n&nkovV6{#Neh@AUGsNjA6A~}TcqEIMfY(!g z8#R^Ip=C%J*9F!-)xS3yM%MaLHQyp4m~0aa@{^>U8iIb>UF?(BrG=Li?7=jZgy(bB z1KH=iF`nO?dOrqe9<}#`YX#c=Vy+Val;aI}u5rdc5yJ{N(~A~tMM>Pt7)|;UK@G7u znrTMEHs?ZEa@}Er6cc%B;^8G4cfrR#1`lY3pHJV4JqWpd-XwCSnX`6wShUnmmRegT z6XG*s7G?zfs{G#Z?u)|~oZ+{`ytmoRD*JJ>=VetG?$~01T|XK*ewdqKGwidWmFruY zM*WorZY??2f%S;#@x3tMF8f0=NJrF+%Osoz1KS~K5{!gX}<-@k95 zG@KF783G&q?ec#(5C1vlbRuuan5SG$qW>``u>NHA4Gjmi6di`~PCKrummgcN{{FxR zmseJ1;BXi-Uq8P;|Ek*B3-hwa&VdB7KYEv|nVM$U`2Cdi#l;N$$oq`v)1hqCl6!Tg zUb7bNv%SConYa&52EZP9Wb4fa)Qx;}=}(G}4!CNrNm+st&w{ht`WK`jze-2-`r88% zu&t>zP7&!DsF&(B-fcmd)s)oW&*z$ai=YutK`>kCE}t0XuR||J$jmX=W8|)7Z5;7@ zNK?urzl1SE^M;QAICF%>6LR5l!SiGehzc~&_PAMx#d1DjGlw^JRJ2gJHfd{UWNFS_^E7&>Lm;HNp?s# zwcC_AANuzT@ZY+osx?qu9M_4QTKcsUrr~j(;K|yLV`Xv8x(o zwyt%H=6_BD&kY|kaez!SX}&uE%tFS+g*zV)JBX)gxIJqJl_Fb8ZgkB54 zYnV|>5EdHF&u1HsS;jxobmc*wT>+=o%vJavsCXGQ>(qVbQs}fDlz%G!T`H9aLrzrF z_Dw%$Q|_RB{MxGY^j${hB@K9SCnpRW2$nUva&H8C-*q=C% zkNxY^A-3nRSfs7ny7!wC3Rt4gT$(2B^4G0S092*WwVbf~P(VE=fk`)?Sk8SgjI2Ji zTuXlOsM$($ow}R&l&_|vEMXoxlP+N6RI@_mXw)IOr4(&Af};ZtHng1{SU36A(A5{6 z(xfi*8Qpz~h?^wPaA7pUg^y)b}N1JN!2WtpJCHx^Vr66a`fR3I_lF`SIf7Xww%(qdS04rPS#u*Y7jc zV{U57hcnfqMXeh>YQ_ijue=yA+uIp2F$BAxa0Nsd9{;WOP+qDl8m5nN@Y;wn$e@Ot z#iS?Fi#dXLDsn(aK)V%ZyAax^f6&(Jjt*nTi?`LyQ)obLqdtbDAaS^DgJnfoy3bqh zYC`<@u^0)-=--br!VwA&{(927cb_}*9;8+VR-@*u%4;Gn64Z6x;TmUR8~wswe<9u;4jm% zLn=q?{@nbqd#D3|X8)n0ml+dUB21axH~s28_yr;0*ZuYfLPL-i@$A&(h;dEw#6Xt0xQ{16NC`*<9=R8R+PXM5xK zVscVjcsjoN>bE^FxmI!IIC31_^ zi*=if-5a#({u+NiGxCMd3Ru3$pQwCu<-LPI2Z%Zh0EnNDDx{eHY)@Xt__b%yrcC38>TX9mXy`i z=1dCezRW2~vX+u@XMOlWQ|1k7+@TM=bgb*k=Q|k?q*z-h9IN@U_qrZz2$;kzRHfbC zw+(OAW%&}+(#OTSPP0e@#hv6gfhgxNP5^sO4#qQ@FZDDK7f7483`5@ zEbFzO-5vfI;n@V@!2vM%mv@$pU!Q5^1fOnm_O-nW%y7Pb8GD6U;@eE@*vNx)4@>l~z4hUnw0z!_NQii+q~7{mZNZhfmqmxeF%KFaw8v6eze)iWcOkL0f_5!q+%pDT z5F3=V5bQGr0dU$5BGU&3)4Ov@5^_pCn;>tN3>QwoA$t}j)}k0Lc`C`8cFGecpYV_U zhnt>V&r=L?gVOsRO|%z!qQ6Xz`1Av<%+Bd^(|)_H;RTG+NEJ!rsK32-))Fy;A?a(0 z`TUU55OMCJ->KvdS?FzRJiZ0jN2|TP4Faj=@nwtJ#PRg=C8Qr$)@&HQ6ZDt2ToNkq z^uR3T<$xWM0*GFTvWpJb>{+Z8z#M7`GuA@6Ek9p~Mmrz&&1n4)yi<9eOGH9s#kAW7ASu)Tb4nD6%qUk* z?xCVQwE`Q>^IEW;$>oDc+uFdkj@;hqg3+i7pJq) zI9a*OHWa#gwyeG#(7j%8N?DWUVJ%9V!HxdC-R*C2x?s981U#DD)3Z$zf6=3u@#05g z?q6K0)sG7E$~6?-HS>@C@&9()>s+Y7opzAz@1KJI!+Zh=sRc;Ae~pjd1-vF_6n#@w zB?y!7Wg)x}d6S*}{cIE$d8S(8u8G&G^bb9S*eHR6RNqG7@6ce z;IX4XW@F(EU9x8#>E=py*OUQyRP;6UQZgjq5MalS*-6J29K+k(bAs(pM2}t=w03q( zq>xpw=xQtW;!YtM+!}0QC5f^xc1GiN;!UuFGw13B0Ph}1Fkp!+`Vu>P80I=Uz1%?S zArr=P@~)CWrpv_QfZu_T>JrHC*OR`E4i)M5CEqWNb!nvafR6A3>@{}Vn55GRxwswr zya{G`E=qNMA3;p_ZJfcZKJx!z>@CBhe%tNu89JmJ8DeND0g(Zu8>B=^LP7yWVSu4) zC>gq>ly0O;x?5>b5Exn*x|@M#e*1a$z3+WI_y0KlujlPt-}Ac8wbp0ttgZ9Ygiq~@ zocGg|=gn*QG{YXhsI?F7_bo*0w#F#*(_*z6hp!RD%}V3(h-^^*dc+l4l{`bpQ(wd% zRkda_tEdfT30sSB>?suL74U-}dBPjI5FcJY7a$MB+{9b)Ea}FnwK=)juFZO%bH%XD zE~CD^yiE0C3MAPKmc}X$MNM?R4bv7XQ}rm{K2Qy@)@rRFs!r&4UEEHnBcNSzXaB5_ z&YR19VpM$d$UHD}^hDi4^!gV;G&^X{FB$S`%@|vp!y4?Yh0c|`x<&@9`f&<^HFSQV zcU2!Zn+g?REY)s|H>t&Pv^R9@lggXi3!WiO!ojEYvtuZRpE7Q8MRcIh=q%n@jrrK+ zq_)s_$7n5*Yr;q;ea|P(YfPqL+%!+o|o^!%s2lVzo)Fw??L^5dA7T`1<&ZqY?3q$XjrwKIK5fn(FuClExy;7mxf_Mt`d=o4G(>yS6}q)@I?1l03XmI1!+lV2hbSR zov!;MPPX#k#Y2DE6_c%CBFgd7H-AEC5Tw-HF80F3 z`RQV+TV1J zIm2s&L-alp8`MV_sR3j3+~vmw!v0P>QQD1WObAK^kO6bCMBbNKGv_eb0B@98DFFs; zYDUzOyR**8*VxV5)9=KdO>@@RFwjSx4;}TiaMoP;R(~3uQd8THr^QXxZk23Hf8+Ol z2)h-}%%iFZ5K?sqqkrnRAvKQ2N4?-Co)G|7h6g}3x;SY7(%WRLGpF^&LpEE}Ynz{C z`)BnV2l!8m*d%EjpFyaljbr@mFiQW{*c_yy=~W~Ql;2Hm#&X^qeRChxdF|Pm$mw%5~^x5nqNUb zjyw)Q+M)4nBv)F4PpHxEmrO8;->rr5#Xx=+AB+!a61XpQPW;rjnDmc78Wx{BPr%FY zb~{-0y#~Up`{qg1RifXWI)78O0SbAp*HQ;R@h>HPCVNM^SGC!?1;V zGo}6ojkrgeC%_NYNQ{RJ7C)4V6^?H&B^R!N7qkwb?<4`QDpv`eZ5My+OIu+5&PP41 z7L6%Td}`J)*U-mj#&=%#uy3craK^k_ZBHcRL>Ss@rM10CYxnyab>X;!>D|P-yB=qW z&#~xwgh3)utLS99{w@(=E0>oU`pKnAh%PLn;n$TcWH^C-0%3=sexnZ z4B{ut>0xJ3YM=&g1--wq(!%O|pM%h}v24dZNj9Edl(mU(lx10^BF2JD;6yG@VBA+M z^K+%uD=ylCyU~DXCW1!%xlajs$Ly_`)k84LA7GMmSw-MXk+AoW#Du*R95J#(d)yp4d2Xa?a;LGE^0h8*tb0YOmkl%`D8 zo3>7db!JkRv3?_)h#oJtNx@jY2yBFaY_q($RVlSf9BhiRM2F;#d8`S-DAHip{u;t8 z@;`(;LvvV=|Mmjlq)53s9C>>V7V#1lU^du!J3R1O=LhZI%a0>;Xoxz!o$ z1`cL+YXft7;KKh9o-`QW2p13npYWv9D;Svk9W z#)qftR?^tA|8m4RmSi$Uc(ml!Ik1)xPN%KhPJQ!$6aw=GGTroGgyW+Sgz_*%h22P# znR1M>8e`1tyBs8}yZWNf};y zSyAk5lC^oC+Jzg}=dn9%V5Q1xBhv7?&xh}F0$%5sNWlEnvwfC*+BxDNIM$-^Rr2ET zmuKIv>0CN1Sw2tJ{5WL%Pr2E^HY1En<$SSTTvYsjl)L4QY&log+9-Nq1;M6uN|QEk zG0+oAQR6YImE-Mq)avwo*fuIZ#Qe0OasxUZ=APuF7kH$D&Q%YY!Fl2W7&oe@4l94> zAtM$bzG9z~o7cNB%D6wqI$o_sMUynMq_f1WH0T?K!t~3?SBk8Cf^#MSkSv_wyt|cy z>GIxmqek==l(z8wJeb1@35q17@O>xyCZCOTXTsx?wiq8<;_{;xFun|3SS+V4--8c7 zrv!PPk#1(lf}1q^x&tv`$DVH`MKspC1asbuC?e}k6Q{_|u0p*qMWwFp=xKIX z1jS2#4NtX;ht{B#?H{IHUJ(6cC6F6xGPp1iV*@3(M5*i(D}Av>_h1^=H<`g9ZW{}N zl%h&v^4x|_i4aj5nPcJ80Je7Mj~7AG75H$Ak)653ry6IcbM6F8X&Bojssq>T>$2`T?Z^!-U5p$74j9w@TPbro~PGt2Nwm46vU!JEHOX&^asN&KV zP7&qmBcRoFr2)qQUi36(gWor`Di$#6!`66k>3vGP@1L9nf_$4r8$m_csztXVcwTl( z_Yw5}fkjZZV!iCr5`W`ZmqotJ1o!NXhdI8`fqZ+v^a5#f(?{lAq4e}#yR6H#FEd+L zhu!qx%`LzACw~~PmAOwKMAXQ2j{$9b^+XIDM{H&dH)`3Z_L&6t=O0TSWlWfB(f1oG zL(V+Wn91Wn<8#O#>J$>R)90`xuN#IQ8i2}rB%esBN-?RTZmbX7sUt)E8`SWvtQrSw zZ7Qz4+f~Qs#aAw@Ub6eJMmid??npf=kbb?!H;^&HLImxkNbwO1SOBEFgGhR)zuBR& zj=oDe-vHXiUNZ!|fnN|`2r@YX>=Wt2K}tut^DYA}=RisMbWRz3SsZ$aac9j8iUz6o zf$RRq`DV^*R8bl2%UIFVV*i1}H0wk2xa#A9xsRypzxhy<@~f(Jk55hta&v<+{4zJy zKYS?T;O0J&<>ch_z5Ry~uX_ z(y8bc>?k6gNMkm`_;SgOT{02*{Bx|otAeR@b8JZBKjCO;GSs3BIuvHI#kRSv?cNO- zL&h+6zvl|~DP-4}VauSpHTd&YAZ1}gMVl}Kx{40Q{3|%L@DotY;^4=kZWH2W-)G2F8%jg z7Dy;QNDK^8w$^Fmg1HgGfo6IP1LV)i@e20y{LBD6?`}fBJWiOcfWO%&P8H-Iikh&oLy)_)Q28-;wye!RdA999;xTDIK&NZ9}d1yZk3p?%046g;D2q+o24 zgmVS62E14oX=D)uu^1ov>`Zmh&H}}Wu}kHws8M86?r4*xl1qv%873y{S6pPV&`rW< zPE?GNq6D774i>u|R+;6g1y-4W5w{54{v5~G-9G)t@-&uDI+?n}Td@4ZIdg?BULrW6 zjV@JqE_v9c_tG4CW?y!Xo%_naet3PWATHQmwA>1MV$iCMEQ<0JnCHRxyR9}kmgq*8 zFvN{fAoT^Z71XyqsdW3d(G_+#D}@l%l6|==-0nk?O{+UxGyQf;QMo@ID~d!a&f%dv zj?;B4?!F>2l8BMvrWvE3io`~%q^#B>*z2wP(mq{$@)=7UY}gDJvyW(YMRy@y^q$Q* zALnHWBq(bd4PyE!-y)mcjwdyRNQ`y#-q1 z?Ce#w9{|50rs2lK*|vgCeRvv;Z?4Ha-L^SKbu)@|Bg?_)0T)E-6RKf%m3p#M+FiZF zP!L$El4fg%8h=!oTQAT8U6M^?n_xv!ENt7 ze+fd6za#!~5bcXrsGj}Y1=p>rh8z;brNQYavKq4}{zi;8gJvRx@6S} zuKVMq3b3sTpWbgJyv>TwDosC}PLWJ(PM|{tyRGmM()%Co zZ#ZJnpWp$z`9Ec}INTpEQQZ)eiiJAGklSN@)@qZ4DNd%%-;xBWh}ll!8{g^yB}uDO zUOjasLH;gZWqV5f7$QdGb<_iF*r2KAHlto#n>6Q+2hAJ~08wMacq1D1V&FYi`x3>FmAj5EYvy4Z~Z1;qY|5lsnAQ44QOX_ z8Dy(j>eW_fa;27t!Lk@}P=aU3Hf<;O?!u|L}xnykAFu#NE21>v9` zQcD;k>CM)Ajo}Tea?@+L`-_!MI&)RQTznx+epYdpm$6i}h^%EPBqsB2pSKwSz^vt_ ztgP%Z22*93x~KNA2ek7O0YGWf5D}SiOna|$?T({cAWjvA_O&pns+7QusQWk*d>Nw0 zXhmrEE3U~`5UB!z#;S9fRbgUrV`ZhUF%+}Y^c;+`#|p{#G)!5!#TS%XgX@?jRMO@JeX#m?^0}hq!L0gbOYyQw$Rex?lO4BIdCm@gbaSBE zjhX30@ANPJX5tnncZoLPV~Mk@>Heg~I}v^WmwZO&p>bP0ZqUC|?q|wQU1Xto!Tp*T z6~SW|Q1zNT=F$0`GUz!x7iS>q%Ytea3I59oyk=@3oIn(4@le)qkD>%oth6fyKe>#@ zX%30-`B69EbgFJ`cjwaiR5_M4J#BPXN&&8P8LsxS)7Uo_C#L(BX6^tXs0st_Hyg2W zR)_*VB1DkijmsKWqF(-WAaoRu07QxV6)vd~rZww4l*yj|{AKlqNEWYiF80QJ_UlId z2a3u&Au!;JdSdrLj&fBB5SyzEx=!8Zp~a=T=2@u zuOk0LsMdq`pHOZ6qV4y8Ek|b^Fg&C?BQH=51~J+%0~U1Ab!k04SMGH+zgpa7Vvu+r z5=9TLS&~=%DPz*|W)eu;6Kn?YIb?>e8&@m-L1JJ0raM6E`8PDHy=Q!tUt6o_^7^tq zicv<5zklf}h7(s%q@YYo)R9V@#%0-|l=_l(~8TB34+^mE0Jajr?E z)HWlFdph6E2w_pkQR))KNCLPz>CSE4_F}qUZZBZs4tM)a$>pmXSS}XNbv3jkHMgGB zRA+WQTWM8<70eWYyii*Tr(7FB1QTlFKAl0@a%Hj$dQqCa{(%n8Gi}zJ{4pxmSLoLk zNFSoJ@hn_Vd>$Og10x1&*|ksl`0gp3Y|TetmLGE53*37l9oqzTZlv%6sfwxn+I;6# zS|@ToXQs8sJt%ope8Y>3EPa4Y;@M(}w_f0nK^9d|poNmT7bx^jS}gG~sa_dsORIYn9{PO2 z5K$=wCqZ)-*@ChxgGVV-_`$RSJp(lv#*W>7BI7`&aJi0q8upgH6G= zOJDvDs2{h7xeBmFJ{Ck6lXx#)wj&;C$wuvuVD6ig6=8DJMwp4pZo8F##{TN;l@l#+ zHmHYYNj6ZCx;pD*Kd&Wf#{^yDUleuUHrqoMz379Y-{q&WTWs5I+^f(dh+ugD@E4(c z!4Y)hrBeW@btGZ28Gd`VgNdcG-}*!F2sWQ*YdY5tvfk=;}W@J0rI_j>{V5=W&xu!Jm38!>63^Vv6aD$^oEz>1>-F$5^9ksA)ZK^c z2yLcbKX4Fv9QYrXv|}+Ov0_}T>#!dCjSH6$Y0CithieK#+L^6o7Z6JR3!ghB3GO+T zMLF5IhX7#vKa}5?9tW)#q?@1I$qJ6I?xhe18Xe2_wV_P*kj+>oI77ZdFpFw(IgWh7VLMZIQjRz=H ziWZlxxUI zRCR2RuXHm5{uahpl32?>M`nDJ41lj>azf6-Oz-kBHc#TW(*J7D@L2LQlDR!pygHrIA)KwLt;HQLFRxcFjMhxRCPgZ zOnjLwVH1(ifR#grY(UswSWXpRx!f}*+a3~{tP1J32z{mJYmK$}`EGUp?2+c?=GNEq z3I{RCL-pBEQnAVglcle$V11+{wD0rBY33&Y>7`HwI%QaYciJ1)>n-Kap#7n4Rr zzUJ*^6k)6?)#jJ1<##?Y3QzjXd7l4sYc{Y2JBGxkw?o-n*wYV_+`Zyc0)tl9JGTn_ zOklwP#)yoIExs>+nLo*$zwKFo4h3<$o*BLKi@Ph%(iW-sF*uKU(XTr+ta zZs%xTfC|?PXT2~E}RyOPNev9m0al~>4n>M)QL`2 zGI~(|E+@|dLlKYMy;sF8I&2ZvaMOjwd<=MD1D=@Q9Kf&1tJqaM@3qRnKZ!uUMVSpl z^fDX~+P!mqxBZgDlEf>Bd32~s)3v!Ld_G+qRlMWd+0=CR3fKM)?V5RD%x`F2IICDe z0`erP`t->D6q)Myt6^>|Ok3@KITlEi!qcj8%ub=XL{Yt}v>b=SgH`svblt1>BLyG& zP4$b5wT0y`8Yn((04@VX^YaqS8R{mV~V;OKRwT3Nw%^7r!0-#>a(M9r?49pyXAeI zJ}h!&Cc(N#ekp)4<_o!E!cF@L&1-(f;*2zlvsl=zOD60(5Nkzf)znm@FGB0B(tb`D zGrwPp!H-W18WKChqHoXBi&|pJuTuX1*X+X%!+%)eGo)=z&i|U^xAz5d%F3awch@_| zm3Mb6S3E2FYVYoIaK`|^brN$zE_RbUwE2w4^qm`a9<5P9!ha2D1jB6W6C__ zctS|teVD~g4nZ39IX(Usw_lq*_HF&+dBvZ#aZ}n4Q5^&H;(Gx>nu8vkq5HLll3vTy z!TO<&?3Lyjh5!&WjqNvC1-;6)G8rV8zWnL6g$n(##l>y6y+Rk4$P>9YAWP*g1t8Q$ z=vBjWBsAzI(Yc?0D-oN$_yZZLr_%_I|5I^AT;(D zZ!KKd+fd8ATXsp4Dlf9jq2j^acXJsVijv}XTq}{H?(PI7K?fXAo}0ng!yA&led2#bSymK?$uV=m>b3M3L-a#bR!?C2nW`|x42AzyI!g7|qace$Ulg3lGv+)bXX?@SqW*WUgjakB z>s8Pu`KAGTYYKH7W+v_I{No|nRR!sm;PEt4E`a=L8tz)PTeGKDOE0sOXcXj3W0BV5 zVl!&%LG#bnW1Zd|`|%@Sl-z6jg)kg;s9V3R(P!Sx74OP)Nm|?3h_bFD2F^n(oyLEr1P`L zCF+KCl8Ud{7ctd|%1#KHuVtiqvXgPrHU$SIe8fj#j**R_npaG~d}~>@n?XY&@85up z;28o!im=9+#FfTS8>~%pg@9_+#lkFA)wS;ut3Xz=775q^TCDR0>!q;N0eJxZXxQBA zCz&+xKd~7k%YqvZuL*$9eo3@^J{;7Rijw=>%=Y8yB**A%8qrm@v^SfL4`O=rIwNsg zriTotmDa7?8`?w)xj8sbw*rwbN)Bx&D9Lil-g8>9Fckue*K7Ig3T0e_5HEe1m!F(< zR>0d&v5$!898PVmY2_cb+@0U2|^7pjD&*t!c9{nyEbh4UN&>%ZR2$LZ7J-w{R) zCLjTm#QMt8dN53g0=#>L`M=qEY9TyfCmRvc-mhS_O_y#zIo8hp+Y8VPUk;kue;lp5 zb~R4;+i++E8iZh`;(T$~sG@G&!?+JNv+G*(FT(Jb9(BZ$kUmw&Uyr}eh`w6wQpb$c zpXVTvOqCX~4+ZmhZH_Qh-#K;VJE^q?z@lRPF7Oc`sYcKa2PA%84U8q|d4&61elxzg z;C-Fqj{{w>eW?liXg)V4c{hEgycdbp+i814tHfx_V^7pIQ@I)yzyrhnB*~Usg`!c2 zf6OR|i2GWnyG9BOac|E)N)AuUxgom^VeE99z#7M%9VANQQ+X#W9obq z6#kZocaFSTBFz=BuXg9*Q}206+C1VNY1;Inbs^0x4Z8>a~H}WjzvtDFSVC=dHU@tO?IZ7Ek*D@51 z^G{i}X;###XBoA}rY6i7@5nE;CAG-Czr=e{M)m!Sv1&&1?Td*M_k!m2ngiRH@9H8+ z9m)(r$<~pg$L@<%W_hU{<5Uz0E3-6h-`awqO}Kjb2oEG$B8Sc9wJ-}hcxnEXI$fIS zDWkl|sxZ*H`v!-a&pT~Lw3QoTazcj)TWnyPpC+*Rf&JSOaY{A8GBFR`sX4FDOLB z_H+-CewY`oEXAa2#>|?8aGGw+L=sf%S{lPh&W$ZEf0aIQ5fUGByP`1qA#{adPE~c2 zEBn>GXfG-F1%*({JEE*$9Jjj_!wO0vkL3QIa~UP}4^-DBC;T$fIBtM#$I%BeGeInB zkNdw{<`^TKm4`W*d{BGXtov>REtlP1SlI~rg?x%>iWw=_^Pb~=O&zt?DK1rJG`^ST z-KyZ)`@8>`0D161)b;=E$N8@mNML{|F0G&$k5JnwrTK^O@bDV%v-f}6{aVa?`mS95 z0CzZ?v%=6L?{vUB6*TI@ly>W&iIG3%LXVc;EeO(P7&1#C$})XQTCgb`FMR;d=iVz( z;=jXT1&_PfzGF#SBE~C<`q}~4wOPIs*L1#%g*A^?PrvvE1!Utq{DsIYyM0wg5BVin zU`qP10ZtAzo!yexnTo~2Cs`%hVmc9@Z=%?f+MrfC<$?j8t*eMPCwuRYoa9aoDu$RA zY4iCrYgev{Q~zLlM1gO;PL#zBp#IQ%V;Kg|zE(Xiu{try0L7@8l4NMYKSsSA_JWhL zbmB>B5NLk?HdS)!=hnq>x>f?l)rr7-G(cw6M8B3Fof6T8OJ9%aHBiHqyQZwqBoyR1 zEk}McY<~xiN2vdnDVij!(SAi0q7kLf1^-^Ds+WWwfg9^Syy zV4PiTe+TaMq~sygQ$BhZq3pQ)MSx^$(B{dgP&_LP|dSuO1n0C=)qv z*VfZH3>sJ$W%Www9$q!8HSotLgK?$cF7sKI{CUzFcPR9C?Dsh>+a5UTFp=vjnz*Wy zGYm-ae)LjRqNqd~%vMTnPIy@y4`19DA|z%&(>D%4m~{{&1BrL^sGV!!oIf%c|hWAOP_9mQyX zI15!%Y?RxAyEIQUsjHEZ$BM1Rg0kstD8rD&GZsw>tqZD{xoied55xlkxR0%ZXT&d{z4q z;}bAXFXPD#CdfUMstkLGH`ns9dEY0BY?In&g1nZH+bLQlsTpdsZ&oOrmXT>15{B4P zdj3W{!)Kf^J`OrQ7uD5QOHj4*RMLhaV~b_~_sv!&$IA;F>lIQr3Vf+c4Ox+6Q8!~r zWN&2Mq}T8X;M>QhucWfK^sErFjM8}9JyIprjCpW;eF`G~`7YM!WpMbHth%D|Tm8F$ z-aaF(q=gaLq5YZ+mr(}pp<%B|cLt*qvVUD@|1B~Jz^Fc=C;#hsYT_<>ZGzH7?PAh> z_M2+l_g?*}c^mnU(pu4e`O?{S|A10Qm5yLR5F;$ya ze%R`FpB@0ma=d;x|Hlk`Zmx z1!R(!B1X(Vyb(W`T&%e(QU>ur`-I&m4UJv^dJp|@C|+N&c5@+E<^G(A%9CVBfTCdz zP<_AI=m6YpLs5-b!Ftv&B)ZO2ErE53^ou`EEH?7WWQP*(`L^Owz&!qUJ%=Q$0VhyQiZB<3( z*M?aXZRX6|CPHj%5gcz2#JX@)lkmm2#BWpbkFfwm_zCx3avJ@UB<-)t{EMm*BlblX zVs~2b_l)0f(LO7I3!_60O)10>#p*ttESMLc*tMi3g_#_t(Zw;{=GQc(b0}OC=VKe` zYU)YH#7^iwWSAEsbi>RNN>cuJE-b}6jv)Qx)o8QDH=K3 z%!ML5#c+%0fXM~YK?Y0YGA_|r-|Ie?Xd~{FI;@v$q@sU4T&3yYI=^fJkA$w$c5oUl zElv2*F#&uL$TzBH;|%l=6X*Rpc4{svA{q({QQN6lSpLsS7f$m`D-V+$NB1k??=HPB z-Q0eCEZmu3bPt~OfASGZk{Vz!;c(}D*hTBT_^ti#HPI@InDyyf=YHAFVy$6;?q0P- z$+KH#1z#cW-~3OI(HS{27PW@0$gFU{vDo@{{iJkbgmZRC*sZW!YqwdWuIeIf1#dKh zpQhTzt1@uZf;gKaJ*tDM(k0Xwdz7NGUa3{!f=~D4Mo{7o+H6op?$i5mqKR1TfjE~W zmLHG0E@CHEvp`>9ny|XOeuu_(O(Ha}{yF%e;|5b7Q*Sfwa9-(mD^pme<@0fhuv9e& zx0n1!Z!H8$>kl{1xBX@GX&y3`Wm?)utJFT9ez>de1;OvW$7vEq`_rO>Kh9dGn1~s%tvGZ@+C5Qz#wq>B|5mfJ^Q8{TJ7ofM=#~;J+6gfOrxn? zaSJzM=*{b2JFp@)qFBK)$FydO*T;oRnr>g3x3-^REXM^C?rEDlnPQ!tnf1F(3NP8O zV^I+d&`<8mIX#b{DhB>qc5%|a8Yqk4+Ni4HC6?T(>H;M*$Y09IJ_ec4HtliBCDp3r zC+Q$Z$HibGDzwZuU2a%2TZNz%9D?}dN~z3v!XR%)(wuq`Yp*Ri$FG@Bli`PO-Q}Dn zsKTkvOKYF{G@1&+f@5l-vkjr0TdH(qopNJ-buC2kK~RqC3(`gc`cJ~Ev+gc<%D*l0 z=SW)AJ1DcMD;k*B`sQn6E^G^)bA0Gdx6QYz#O^s!tvR63G{Y=8CSz)2@VdEGVsmx) z>3x*WR<6S=sIXXKoEMowJ?=cLGn+9@iEX|`w4G0`Tgqnlcd2@T^bz|{VXxHp@Q%RI zb3-aTlCN8@ncRMKP2lT>@aUYrb91*(4k#nlJheMpP(>!e?ZOq6UqvXwacAhHhO85* zPrE`1bR#ajDWYx8K+jvB+8Z3E&azXOY}p6UWq z<|CHl2T1%Qn$iKE!58Pt=2$4U*z$q(bsON?eU)L~Kv?>*X3}Vne%HuYf;zsn82_6V zp{6;kzxPtVA;ai8rm-cZR%mj+$TAM;)KuHY=$bWKf*+wYPSFrz8OhozI*k?g=^W63 zc)VA968I*;UCPCe>xs3nf!)LZs!Jp?3StHEY3jJc)=buO@e};gTA?q z>jOU?AI>echrT8$GS8lo9r9O)Sv-^>CI|*~xk zN_T{;hD}v;WIAAaF$m=I&32S|9JbA#K4fhru{n_ zIsv0ZA6DHX4qO`+`7BiP&UGZhgGkKy z3R{{}eqh>Z1V(8Ta4wx+oRfmyE`;dNle?ML&7k@lXTj=RAmRLtS?7 z6mS4MA+HZ8KfAg29o_+}$Qyp|SXPIb(ZmQa2|+h=h_!rD`-$?-06fc{49=#gLiW02 zNf`S>;g2mTk)~$?Kq8+5l!kZwIOUvmunc0ihwEv2SX9&Sc(D>iif|T!vGwJ;xQqU%Ef$rtCj4t>aHNt=X{HQf*24^lN8N*uMihw?!6+@?i0CN!j; zM8(;xTjd*=H8KEMDe+5GaNe)w9umJ^FcgRv2;d5NkRiM)bYt-tpFo~GWk|d1wQ?CG zdiJ?gLiSM2n%5X2`OPkO^Cn8JIq_tI!8O41ifE&zj!ftsgbmL|Fd^hGiNTD!p(}Yql3IU;@Lp#p-}8%+3S`kkK#tVMjFDC+`M3!U2jwT#6h<5N{<3X z0EoAIWl+jLMSZis0_`TuRW9-lQx3J3k&*Og?%eYmB_B}Oh1SxpuY=VL^G6E=cnELJT#^~X1+`V#sjvdHH z-TDG^90>H$)hB=uiZjSz>time7+n?kYSUV!UZ;cm5lLH&W%SIDV3*Uh*PlNlMyEu; zY*FVNHD4A#AvucQx@bc;lkPKIs96<4KI+oK{fV{Ahql*Q59J3;kt)q>6C?fF?W`6? z+4nC(7o%~B67;#Y8Fx-vss<8H$~IJ;8)Wk&&kP!bHvLJx@%b2z7?czy)mt{{+!XO< zV(o-C?hq~X+zwH@!{dPJFWY^#T(# zxpIqm`xWS(C+~T%SCGAVuZc{@o9YGO8!-mq_vwwG^2r+R*Na};B14kL@rPfA|ML*- zDEfy>VDWDQ_Ry)I({)Um`ya+tUELFt`u9nvs0qa?t7xqNIE@NV;l2)Qzm^T0LFv(C zQeC2+7iE@83PX0nOdPkxAD<9>!&r5g(qY>#*>8EJH}=({LN8L7yt~{3avYP^^_t1P zcEpiKJnVrJwaC&{wi)L21D-EibbH{D14o6&rnCZNw@E7K?RNQp{n^o-1*Djpy&_OP z9f>BQTh1(5%5?-omdCrO6U!V~8fzXIq8!@D$jpUrpIJ_Av?w>99ZZ_5&S>mOw&RY1 z-OGW7q3~6_AV^~ZAcp>97)b!6oP3C2^Oc#C+XJ`hDc(<>${!Zi2r?u_T6&eaO|oom z?Zw;vs-2t5d}tQ>2F#E!Wnfc9;Q-=gb=W|i3O>^jae=BxT32y z8NY3I-L#OfLe^*5a;fxkKPKI3cZ$8QqZxmqo~{Q~y0ZKv?R+t%wH}qm?EW5IuTQFr z(DQ@iRG0V%Ow+B_8%*)8C;KUFzFh8faiws6#Xs7My8^)Wuf{P3zYLb}0R_=DFeprf9xR zfEY`K^6bi+)DEO8V##7&>q1wRF}eS`zSCuCZ>daLy9LFlK#op%I=22%k9W@BrliG3 zJbX$NMS?tkHokPGu=(OlIm3`M9@A%SLYfgPKa*y9oa`wk>*|kcK2mJLo0S&2@zQ;aPRo+%GIWOwz1qx5q<~!!A~!#BHwQTMfUX=V`&%dS4t*f zc#u*)hkvb_6L%Tgf2e8pW^$cQ+2J(L17ufxM<4}}i7%^cfgyg} zY_hM9>Km_OBcyHd0_!ZM=B-VVJq7-5C-l{SH<&X2!NUDu`cl2yI7)R4y(KfX6Bv0; zvu759UFO>AIAl6ab3!ev)F~P(SewuGtv-O9(#hwg+i?zUWn^2hgWLX%wrF}JMaiXb z@PWS)9dp*h9!KgH(U-lkOyp;%7XGW&&=XQnd>a`R-`j-6ds4{i?Y;zy|8bMSdAy1S zX?w^ROXV$9x1`T7thzb&;E4`%VojmHOP82vK}m^njMNoVnv_S(^A2yProTUy9G(|T z{I3+}|91pC|EuUxRQcb2yKK-~4knX{!?dNEQ&LgA9r+?A;nyF_Gy|e`80xFr`U!E`IPRmOFGaSA&R9W zH7&S`_33)$up}$}Ma8#}xiig>W>hk8w%Uf~1E#9rCXVbCbEM-g_(!#ycqK1Nhv;r_ zWIgqb^NMls8m=y(pyX*GN!uskWQ!3UZ^sYa_<+9qpuuDx`i!9pQI=fKGt)*20qCkc zHmtns53U@=lzE1he;G5o!EtJ3K;!l`FR-ic*N zvEY>PbQSs0mCzj3eMF|R_Uo4Cd|dWOhJbaM=O-k5l>TUrvu1^>|A8cX?$_&;O^;`# zmX?D_R!h`tWJp>S`qgX)3T>Iw4YwHAkf}nq28hi*iPR0`x4h26Y}$01YQ8#VuQq`g zDf>xHv~mX|#B5zYSAR{VN=y9n8vij9q{__9?#yHLxvv6;ihO+c9AmY$Pxy?5z&X6u zPZjwjzL#wkHaBYrMyE0k3ftr13Mkv~Krt8jZk*8ZW$zoT2;7T^n>k3R z94MxL@Vi%t6BLY97UG(Z5<7Y%Q{*l`#vmuj^ac^Q?s2Ub+9>Egc_yumM3`Z0|Cya^ zj4CIhHfB+<+o3R+6}q)n2rj-$-+hXksY4M0YH!fZ0Yua|1%j%U}=-LvI@ zl6TvbJzb-nZ^0$_h|Z=jUV)N*-CoVANSE@rW-0Lk9+95O5M%Z``Rs`L0l}Z8Qd{># zYpR5EhigFpO*)*jsHszSyMm7mO208h-5rU9xIIU~`sFf4>FTky;t8&*&tyEx98Iit z1)2O)Wj!tMFzh|`ZV3cRLr#0x>Y4Zn^NQ;e#PPJmRlM(QADPS9p3YCY%;)!V>?FkkiXZn|`>EEB2vKV;lQ&GkLeL5{MY^)TLhffFRojL@#Ghmc0 zVW_9UdF7kH?HXs(Y4eF8fo`p-?#;T`2c&e3mg7*|6jO(KH3d9#)rCXcS)e@@HKR8u zgA*qIAwUE9sEusxX1r*fhX2hNxy{dUJ;S@Spzq(3767CzQ&kiTX8Gv%!Gdh`&<0rn z)O!~$8D?C8zgdqXbin1-b~#ed*_S#sktsEdItTtn=#+(P`f(S15uUq`(_j= z?asCG4(i!rQK+d_t&RoM2 zjv~#x9dw;K{bxB2v8>9oiTxG40J)A1qU&bUqDV2{Z?n8qPez5_CY+zU1%WRuB#$Mx zfBC(WcxHqnH>@}?zl&n}Av=2eLB2{<1CeZjrc2n+d zV{a|8t203;kyb?xXI(lqgt087nd5tlN@YYJLV9(pz=M< zJ*fxxDqGLdl5dJaydP zVQoaCVKu`3C}Cf0$**Yi*8mJzm__cd{08}psil)}`0Rl>U;DIR=j=;!ap(jXkGf(d zdyRCWpv7YP;Q38P=EMEDpwQH}jH+Lre0HE3m1w4wax?Ze;QN~%>6Ga7`$J-pZ8}@l zJ!SmJpFhW|b!?LXod7p~ywM54Jcs|RtK9u#RTwV&_fxv05Xh@vI7lqi%IdM`RkQ$` z(qdf08~;7Q;lAx}Wk->&G7H<=t$^PUkK=`@AUPwA?n+oQqs(=q|8zSA{q$o{?1ukarVURYz98Tj~Nk%`_Sv7a} ziC@Aidhw2F*BJ&&B88-mybhfbR;O(D8!Z_GSmfdV4_$BJ)CSjfdk3euySq!V;10#1 z6qh2Ul+vO>gB7PZEl`TPOK{f!#kCFY5*&*Am;0T0pL2h6o-_Fi2(z=VeXX^A>j&_S zRJUSIR1EP8vUQd7#xebyFu3Mwa>zA&=fVv#D64Qyh~NAT0eI>g+bdeO%;I+HvBfeE zUxs#;{%UNoxn9y~Pk9a506SKbF|gmA0UW)b9x<7DFY!hr!=l1-NG7 z&)i33iL2LKW2p{v8A3ZO+bmz(>t?T(x_=tU7RSR|mQ5L;Y)pv~vVK|})qsa)$yk1) z7aJ6R{RENtuyHqehCT@Z!cAPQGnki5_^QAc(v0vm5d$9b3lTm2)u~@#1ywVs*D#uH zNxmTux8b^}DlN0d`b}@MPvObk&jM_#oOB`mLx>PmU0NVRA0Qc|?!xP;J#a}T@l56B ze^jrp`4f?T+~nO~I=lvVi8%^nT2f?pm-ZhZbixA9Z@^9Z7d<%3mN(;P_z?uZxbJnB z@=`?_XE5|pwIs=A^pvlOcV3@SFR1?TY|aw91bE;mQ?b+@`>GTzgj7#nJGH1vRrkfe zYUn%516Nt6j|W!tOLt8UaX>m=WGTMFEw(f(l+Y8Doo{JYYp=b2Yzh3B3htLse#3K& z9h7kVO3CPsFuv6j#y$HbAKPU~II4!WlkE{Vk7=WxR6bHEx zfk^3O7Gwswpj!AX--YcO-8`3G)wEPj?Gk2zZv+8(O!v=v6S%PVM=JjB+1|n30q$ZB zkt!d1&lc`=-v)Md;qoEk|5yu?u`fz6y*>h-*91Z}ND?S=1R|r9+W9k_`kYCqCx}`H zWeBUcJU;5E76mGau5xZbxo7|4H(gkqVP)fcplOC3%*5Rfe;rYjf8>?Uf^)3@^#bVsE3W$Gc^9i4fjV~^6M_BWEY4s;ID&bWjplH|M@Cd) z%Mscgta>@d*Ek1+l`|T$^`m-QEP$3u=qho#&t!oUy_gGf!jm51? zjVskvWb_O!?w*ghpe}-vDWt#c#vSntG{_m)YqX3Cm-sP0AwoC|XuHbUq~JVcBh@1b z1=@s{K&{{^Msw_LY|-zO=M>*G;qt1+s0n9sPP*q*2z?UPYPLD;XM`hzAGCF3%5^no zf@ma+b27Dc@8UD7fR0XnQ6VQ>GIBXBJmiH#6@*zbRU`l8o0V$kZ!a9cv3hTBWhCWM z@niuM-7mYGUo3dHrIwrVMCO#6@gMI<_EQCv&-uq~h)m7bZaVblG2DH#FoM5RIJDFB za>@dWIL>lj7{w96w^G`3ZMVIQTiRra`2!jAndjBeicN+Vnc$!IKb+FWDd@_TpnC83 z@mmthIUa|gN@p&Mq9%FFcj{}mJ;47s{+a0h(cFyQox|BP;hTDu)WR0wj#5V8sp z`l&h1=3kxuECF`Z>PeDEzpFLwz8l5h%%yp(z?a&8CVF#+I$LH)20$Y?tN2EwL0IdY zE7+%t37aS#bW=V?H#FdF%-kBXsWeR3`)Ppz_Lpqm&#w@i1431B@0x(Jv&A}5Sf7jO zEg<0pzssxYN~D(-(foM1P7e~WMij7@oz` zhsq42%pk#2QJ}0|@vlEdGfc4Hgl7(e?zk;xC&b__3&sMhYNQ%Gy{P=BLrFBGQyD^X zi-gFYOBjlU6Tk|P2jPt2xb&1#w9g9@%pK9ihiY!Lqn34oqvH%f1dWC&*{Z)+qs^{l zvQhkwUE^{h{<=d{kzR6>yc7PcndBXGA2~O6y31LGbQTRKSVDAcJ{3p95_nhQb}u9z zWrPDTS}_UnvBlmKw3)VPGqvDH{ywQM`9`5F1Y#MuK4+%9%v$498FftM>l;qq_>9o5 z(n`As9V!XDc8L;?H{oSl(nu@lr^Jt;0pxT@8{>tuQ^)6$M`x%6qn+slS&uR6QbVbG z2)dZ;cKz@@woc0G^Bys!;CucwuXFEKcnw`hkm z{3;xemWx`kq;_@;2nb z7qm$&)gO&6!OpK#&q(xgVJ6k)6SEgxW{%XK?fdqt+UPbh9JW^^Ag4Y}5G&LdE>5e~ zLBjOCla~l)R1u1^oHKCAaOb>rWRovB6ki!yj)lh{dGt%5C@C->+mW>`MGB-p)nBhE z*^#9B@z54PMb#L&{5(3D+&5w5uVZC}-=++neF&K>a9|fphxG6-4`O9%1qq-`Ffz!o zup#;?z5L^l@tutMXS7?6ZQ*#xqi2(0f3A(uG&}HP6_l=)I#e*NFMs6NJC`T?x|5Ci zee#{4qQ((xoIxeNLP)!pcT^yrx!ex}jiqTn~HxD-qH<{YS50A_eC_S$j^7)m9h zd??d(+4g8T8GE~9p%-C`nLLi4i!8g^{*5rwb9GaZWAMO6XZ0QnX=4UWn;Ia{{0M*v z3hmPnG_@rjVA6`?GwPRln-?l(#s#FR0gE8{g0Ct9O{_rP6?6EU(Gj z20OlLaJB`Nk*=y*MD%F(?|SiB(;MeVTbM|u?mJF0MrtW*+CR@I{u(Ng2K0cI+AP^6 zk4=Z?dIrIbHqa|ZnxzP`ooafuy<_Cbd!;rU6n56m{0kIUFX@SELd>S`XTB^xK$zvE z-mff3cqP-(rH!3M#p;w{<;kT4TmUDBWU%1S{|qt5py2rS`bPivXDp z8-QS5BBWClF;$OhH61R5;)pm%UkSU6qJ6?pUY02JdwLXAf-K%nORZQW7(M$jwq4*tfq7*@3v%KIKrsqX8g(=*1 z9}`#b<2Z&2XI!KBg}J_qUrt-aA_F~>RJHD}TFhj4G^ zd{GlV&IF}G{+bKMhWs!nN5sjCE0o?W^@BdN^jAjVjHgb0mZb- zuhDcqWK%OSJ~G$hJ(}b<_e{8cAsZ5b@=x= z$HomE^Lz7wM{$l+`#c&_9K4K&SriDG;H1d7OLZTEF=dW!qbs@$D+&my^oGfQK5z^? z?%5#ADdR9_1Y4dh+Th12P>QVt8&8G)g!FZP&Rs0m*eJ3JeXO;GJ&7L(JMBN;nX6ROdX z_7Yi|jm7&`|Il%fC3{l-qMkA!v#;DJNJsX35Ywv4jC;m2*Pv;OzwDX039oY;O~qVt zhc8om=2zhZ5vH;a^X=L(#9gBFc!RO~^$O1Bsv%0EOWjr7lP{>3WX(N-x+vL|8QquP zBk|O=#^YT^93(s@*q*S1(5w{gnM6*|*d0kq_T8OJfy6seEbngx{5X4ZXYCKpIRvBE zD6|z@XI*q&JpNv8^Cz@6C20A&3kZm@V#I2EU)g#MC_hEP@I=<0-9?tTY_;P8(t%tt zAI0hVoe_=Sj4&kI`O((2?*bl^@*mP#t~TfFRTcTmR$fMeMu-`GDMyGJaD z>yHl{JvSkAF9x^-G9SBKSBO!LAQ+^mHgLLdqd9Y(obsiD~VNO z>zj^DqHUabxSp5gG8v?f?*;Q#`#!w~G5EsY=32&B%qTs2K9xl~SfbK*?*J4gvK5sX z5M$hd4}$?txVol(D1S1xwWX(3(YoD;QE5+Gl}y2gX)06bMZx8(TizUkR;Q8!f;}w= z5r>s8wNc%YTXd9}+PMJC845VOHT}wMWt1#^dRX$+!CLzJTV}PHo`E4BwBll?Bos?)}878H7zA z%xUXaTslsY+6E#>N1*By9F~u4!K%GPfQB)zNo&$xer-A<@CNkvjXK~*&H?s#NiHlZ zwpmq<5&F&4)yCtX+Bex>E|p~iTO?RWFeaO&J%pO4*-LhZ(B+R~Imw7x-nSUNhCG>fN~oh(how{<=dOU=7e(ce&UX4TXftD{(9rP_#+Q9yAmGRYu~rBDGfDxy@WM z=L7d*wx6LRrT~?<;&nx^|2_`tMEO+7Cqp0_FEPfgTEg}{_ixa0M-LBqCeqEC(>@^Qg%R>+?F}M;%gdQ0vFFNb8D~9BkdErcIaz)Ljy)O{pV>il@YGvq?&QBmY1v zn%I%#Rjz03<@5|7 zFz@os6R71aOhX{C{dGkbkmd`ZeS-+cu$f4+K~lL6ClCpoD_^8PJ((K@0lWw`2NetT zbC7v*`Rv}C^Kafqzh{GsF_ZwPax7bu2Yu&tedSk1u2&Xhmh8D~qJyeB`OE~@aHq`v z5``-z5z=Yt^YtCZzY!YriLRRiLkN4v#d#3sZG1GWanXyR6q~Kq^{cU~v|>=GHkZnV z$cfbjEdySrc9%II%CS>ubDnsDp<$?Oucw| zA&&voR3hZ1oA?&4M;?ra3QBAs#_xae$(LO{+l@Udf(ax}xjbZ0vhaTYcs*_bjUnq| z9EbXBD4Xs*5MMlskcRt%{cJ57mHyI^m5XH<80|-xjl7twnDvL!5j|?RZY?0x!Qr#~ zD>wVP>j}zxK>lO><}odVpm5G`Q}IVM9QE(;3bEIqyA6slV69sk4YrYk-Pe+{`Gz|& zR&1~Jyjm?^%xvBZonugZM^YTAbRKQ@THxg!QI)KmJctiDVO@@9uc&mNX#^Oi{pJFD zIttTehu$Rvx$#3mu0$qgJd}A3M~?n2#$1_&YKd|XF>v2h9X^1q}tO1F; zF;O8BjS{kyJm^KC1)H6)*V~b^Y+#+m45xEjf|KW%onSYL06k#;OMc8A@t_2&Wp*6+ ze#u9^4XVKgjD*bN%0E~tedv4|$Q%zpas=NVezA}G?cSw*;7$*TSjoB8?=onw#fNH3 z%b5pxwu?=c9Wr}R4itu;rwi6Zg+S>9S<8g-%$vr@!QFePw>`Pb?$prv`!u*?zcWFZ zkapc}fG=+BBD8;DthGKG;aJjOw+y@qy8iunE1p?k!0^*Y%sNV5^A6L)b+P?;x|@|T zi9ackXaF45x3~5Wf~cD&>z-HmDhEt?v)T#N$L0*oY_$BV*r&VEf7VH#^40wkziD*aI z_pl=2xp}k$2nLU>Jq2Ls|K{oy{ZxfMPT{mcjsWFt(dL@5Z4ehNe%-R|IP7X8?I`q>weIQ3zl`3I zXD7oC@~azFOgV3cpLn{(gFr~cQ`8#rCchdpCL;iS9UiQWng|YHg>Fe3kFTC&BU3o6 z5+kd0lH+9^w(ae^4MoJOG+}p|7r!XL70TFILU`n5Yu< z?%AZGnFONRQ(25*;U*$GHLr;AJ{3l%_v25yMtdfIB-&shX?|QGLrTq^r0Y2yDfosr zgE=N^320ETyRrca4h4ZBwpHub@ z11^GoUWzkuep3J_U9Vi(zc`$d{B^=({EI?Y#X(|OA@f#DiIwR+ag{`1#nO=uflIDr z?6DB&_H!1K;R&<93tpxn0lZEF?>Y{yVW7*@WvTXoMk(;+PMwU|`e%i%8vHnEyC13k z@i2Q5*zO|}q@&&z50Xt|UvV!>WaGMgEv+&EH6Eg+`W8)3j`Nd;5a=gu1Rf9OM&{^~ z(zu2>2~2#oB&)uVc4d>+#-Q(VzjkvSSs`v)by#=*#-|T-or!~Qe0O)Jf2V`6Jaq;% zqQz~qGRa`#L**^A`D$-t3>>;!bLY7)7OIbT+ym{Q zDK|2d_`B~|tB+kcq@u_2FrHPJF5s$@iT zx=^%W;CN`uy2#6bT2E$1JaeA8HQZ`?L6_#8l$RaoBa9hq;jYz+-ncQpLYYSotS(r5 zdwVB#r8Tb+%886@U*8ED%BVbllJpu*tnMgS7wXYZ=_8p-A(`e&KR_oi5@pZAB6=#_ z>US)i9OHPc7Oa%5(f0~k6v{i`lJX<1w`VqkO? zx;)Q0O_=B>CYVJyP7jT=(Jm+|Quh$KgCYKxSLCgfiuQy^)27RFgA}QOT$ZXG$g*-{ zG2c8i*Dp#<3llv9H0cJNukAOEZ=zKu3pI6*2son-zV5MI!NL9<^)Yckhsx;>{X3bn zuV@49K+UelxtmoN7DuzaEq-!AErW3A>?3sX(XJ#r{XWRMavMwVAx=)>QFOTns3v*z ziZ&lAkLpVi((N;3t#ZM-0UR8~VJz+)R7b0psp#0KwDXRLv9#ra< z=UXz;wdm0S$|p)DbQz7v^@m73+*%jlaR_X2^L zpvA5xhBT1A{fbB-#XP*JT(vo&h4IYpTJU_up;f@sJ7RJ3i)bks%0-YLVCdV_A;ZHaA-{LU63$iSBto?s0-dWfWN5 z-SMD5t}la6-4rF{p!deUS!Vs677fjjZ6AWLN6JHjuw+en#J^GnQ85KfKQp3s`@Eas zrFjd6&cZ29yu-EM*-sN=2@tknPm-k$+ZT_2^Zxikm5Es}1pd0&68o9SQZ*!az!Kx` ztinrhqrttNcO6JG5jh)HP6%oCti_>8U{pOy=jphQPraR3qx^vPuhaLRi}Y0XA3ou8 zTENcCe-12S82*N(nJHngGnUSd=(!D}5iCkPMs4+dsABQ6VA(|q!_2fLg0w{c5*dlN zi#d`%g~qOh;vC&aooKL=NkZ0w0X(M&Qgtz=D()sWWo63P#PpX$pR6k>QkiOoqWmwe zV9!afH+4Vt9LPH9Ka{PWgdLNDK52Q!NXtKH19P~#3ZtVk)usJPHl2@&@sxj(c6%J? zOkza#Smw#7Iy}(i5Xn!x>y2M?X%TyM|a3WbNMn9rXGG&j6mv!Utk*Dh(0qswm(U#QO*w`p&In@dy z^^e=%kh2+OS_+BWDmPU-eb0(dT3yN!^RB?1kop{;CG&teC~GfDo0Lh*59)3pi55Dx z?%H)M^$ZX>vWm~b=hk>j8%&Y)kgaA6de>X9V%~kZl+R8CFl@Un9Q-6T1~_! zb$_W{vB^r^g&%&##lsI%SDg@?*w6!K>AQK)1tpHxJ3NV$)^tjVn~zq=mE zlW^_!G4*15An5SyI92j0Y{&t~9}#8W%b}6sN~rAM-n$Yo!MG~KBMTJ#!Rg7?yw7F3 zWt*P(yA==S=f^(B1wN-MVF_R^?MbF1NZP_8k`;$hyxZ_BJ)`c*>LT7m!ZXO?%)fuU z)wsH=P-&@Bh=dCX%S6<=&g?XF4q@M-bz2t!{6YGN*I`LztMc{pBGKFDkYC66P?aub zxqMjFxT2)!;a}#pmCLe7T-84<7J1lvNwze5$Q{K+nG!O_ZE&nnskd)c-H7~rOD40E z&GRuzabB9h*5^q*vA?2_5ct>v?gH+-KMk$_`cnUb7qd-#I<0?7k5>DE-D6_#YH2v| zl@xbDgYe-u6poNS(u7#SigusWp9p`adD`yR8le-u@xAe>Mkgw-iOZ41^SW|xm zgSG#}9nB+G1V)BTd@Hc)xl<9RUxQax71X{V!A~4carvt_f_4Cd5We?11l@RI z(T_ldW*aGx0pqZsM_}y4IXP_2aL6{je&EXG~kF z^{@Mf-}`QVdY)OdH_g#ud43G|gDD57-+)n?sliOgOd)k_#B3iA-$`{%glpjERKlF$ z;roZ64Ue8PWQ>WLbPDl7lt)`?Cj#zAlB*0oqvlwEXHG+xINcEVRY7SaA8XX!_zL@n zZB1WI=rmoMk)@CFKk0b2y+js+Rg3be+%G6E1fwIvL(U<)oeRYn#aI@`KA1Xi{7n;T8HcZ{smnFuMJ0w#eP&5rISXj?{sa7be-TEHU^& z(K?g~fNbsodV{uB7f{}8cC072-}V4_yQWefZprSsYU>e)7(mB`fqQETUr(3~v-2Ck zxPr+pzPPX|J~UrPC=$!j?6)FXcjvM`qRHl#NIcn|1|_OT5j%&~T=FSQS-GRFbyVi# zxc&{FBShGB4lS=o8Qmzq4)!f(3iXq1*H3&9XvKijcuxkg8xlKcSDYX>w;bw&I#$P4 z`2#Vqvx&K@_$9FgpKn(3VDKtt5`}tJCJA?YA*K|BSJ|f$p<<`pL+RW^vWYYCdy9hE z?dHp}{;uWWvxQpz5BJh0{2?=48iO0$MA(Zpg{$aeRyUV z;H@#Z+0cG2Hg_aRIs@0|F5_ZT%mPHeBg1lK{Lrm{#B~oprE#n#!gl(zMQFZgd<`GL z*$^((A4)$oR~jvsUek=t4?7SlijcSqMv-7N1zyVaf<3uh^roT%SM`_ zj-Vs^4f7TQa}WQrJs|z(v_91Rvu2v$dcWfQuQgMk*_ZXc2n{#4+K1K+&a0^T$oLVU z$b*c8Nuz9~t5>H`lvD0s9k_2dI=|?A%N`a6#m4rUm141xM$140BNv`)QUFFmy3k_0 zOPj$&NLY9!SDHpb>q(ikL8msmX7c7c6PCn zwIafFK!TB}yY9toShEN?GY5nd3mf96Poxls`NQcK>kQYm?_Bk?xH^gW@zZC7`uAJj z-bl1T_C?+5=7%#zPwXS~Z+EdUeUBB36UiNN`^LHb5e^~2At~7)<}b?6LZn=*@p+#6 z;gs|$pQo2YO0?w032b~8no^|HKYglFRJ@SbmSQm6@5DmFtS6fZD~hWM@LfT~8HLUu z0^vj_X0`94%zgI?U*8DdMyE>^{LYUfd+p0ou zN#p6fXAI+}fXlG8YX>+FwIQOJOb7=@txUE2it$UulDF>84}kg*MaeZ`)K!4IH|1f%kr6aY>6F#=j&X$P=)0?J&YszgIxFO zj<~?VjJ2h(a=eqz<&!cE_00LJ(Z`SL5pijV*uo{PN%%iRURc9Kc_<1UEme+7xau?KccUM|Fff_bvSzz#K zl(kvrv2Xb?&HFaqagfeDz*5PhS17MiVUWKnUc#qIx{TJ#*z0U><0!oEnAbn$&Pk=$ zY~nZGaGAk+D+VU(D~zDB)v~i>!Kl!B`mJX}8B-Gm`wZ7`8m}eSjUCx4y{e$47bq9% z%wt${;NwC2nDM&Tu7u!d`^8AJzE_$VyHV_9LwhiulmHWA4{4MDCN}q9BB+&qpC?R@ zs?MbpJpIHjG2j|-e!M)fvwXm_c!Grm zrY6pA8tI}%dy1PZlSg~-@aJ>o)_5}ex$LcxYqFuCUCm@c+rg27slD2*ZJnn+i12?e zA$)%I4^$~pVdv!EWA$4-fHpo9jqLNS=Ii0f29O#YzM;Zpt`|hWOrYf%>Qu$#ZP;*K*|T ze+G7EenHrGLe)RIjl>oJ9->1CHOSj7>%y<& zOMvCFG>ZA~L&hJJercX{1_6rAR>kzy*Xxohr*r@ba}?h~0C4@`3pH+>G&dh_jOn{3 zl5Xv;)rhn7=1@$g2cGEeL{-N$MC?*%V208)-P5gS%HW^<+IxpXrtYb$;@RBoHs|1b z6Jf8P=79%1l;S`wx4(_$LPZU4n5dlz)#lSzhxpno;F*#cL0ky`cAwLxmQ5k8GQ-@h z6}?$8rtElW*~vTDPt~oXw-_VSVx5T|2Y)@p;@=R5{~7&%#8MyU6PQ0!(ibH$BmvKKtR4DeGY{OjvIZb{P#jA zRSJ4Zg->}}552qRQBr9)@k!~<8*l@eaIm}CgBoB{ziuf3ojK(=j;az={gjgF$FGn( z9?+;L_fcp*8l_f3a$SZC>FHJO1XV~O0WQB;FUXY~dbPrQa>iToy-gyAmD)=S4*M|s zPA%8FNBrduD@Nw}whjCWVNbBlwCkEzGvlWcNR%(C^smU~>%wc20WUH^kndBdy&CX6Rnck&FrxugKr+bMoPG?y{}RcrkTbTieuI$kp!|yJ?v& zMpV;gm)wVT-{gXRl2`8O?2={r+px>@f{KcY5&P44TRduZ{8i5k&86!|J~@iBdQ@HJ zqR1n!q$xtoR6E2F{&U zH$-r1D23Oopt4fy?s|`nm2}K)N|}6Ux=_PsIpZ58Rr>JlY`uj=I7Hfq z&a-x1hVW#+F$(2$Uy=)=y`UkJ=p4Y8V$|i71Y7aGb{^P)y$ww6xLVL)Hrezo4UoPz zxTXt|%J7U*e6axE;r21JysK)H5&#!nXq@og)4R1x-Q+w!(+3I#@1XtXI{$Z=F8hCC zuZ;gb?<1t%m%oi#_U>jM?(O~tAdE!Mnh0zbv9z;UgVLY36EYRuVSTX|y}Nr)8{rNB znn`(szgcsS&jd~zs-mksv5#@NUhesA`IRPIqEd{GfhPR)4_m)d-}-aY($YS!`>(j) zH@KZ_E6!(=4%8skc5hE7up90N9C~fMcYhW_5XnMva8HM|b&c?IsPTX@0AxVQ&&G4- zR-I9hB#2sMh9TA{Bo7#PyT_Nhv%z_czRD{{14@SQ3al$;xhSjoV!b@`^!CW7j&a;Y}HxrbK8G{rxMuXZIO>{!yAVo6~V znsx!C)eRCRluLLO0%Shf*=aUvQKz-&bAA&+o?|htiG~0#<6fZT_=4_OUtUfn*G^ax zzdNd?Ci!SB!%`3}ePkV;g&ChvaIls{R8eprW6XTD%^#HwIS}$;*{YLDIq}B;ZhF9pG4iMW@`#>JL^O*=I}g)Ax%hnrSZ$ z-jDrc6%-Vt>Ti8QkL|(DGoI3#HyszJA9t7V$$sxwDh)?&<)w@;KK)TOF^Ub!Kw5VU z{C$bzLiZtfg4vC1BId`tk26#EEI!`*Q!npZVyBm;=~nokY%gnx-#zQwe@)JlOS0{_ z3<%b(V$b5~jT~=Xo_Bky7x{ZezUp-ww?^;a;85$h>lronrH@2=F8xBsMTv}0@0zal z98fSdm?CTR4<^8!4KFDrh3s~1%zo?0C_8z=Xj{YMv%7cJi$3u1{O((}FjkL4a0Y_m zKbM?+x&SZHq#1<_2o3a~GssxbWUQ*KwX-ida($1k0_##SbriI>n`=v*DW!>iURdoQ zKhOKWQEdo^)&v5IJU04o?95Xk)~D4@3p|+{t2AMUB)^MEmsN>F?xt9u6P9byoMego2)Z2%6Bke%|^j#34 z>kl8xxjwQLf(+CC>Y)&wKwd9?W7Y^-&mR@$Ew9`9Du9qFk8O_+7qmHV6rEDFg?jm$ zDJlIaq}R3-pOd8nsw(a{RgYT__z)pxgVgJSi&rTl6EeC^ojafQQnLoe0-9GyK4_9v zBTwZ8+1u|eEnby)$1buiwkpUx+q9UwkBKDJ?Ak&ySdKQ!CjN-8H43sN@pW}0Db1Z_ zc_T!9m1h&pq&Vg^U3Un5$BzOS?DfuIE?q&U4RBIF{I~%sA1cv9I4`PGAgGh!8hCg3R6eDjMOV7Ejr=QE7zik?xPmg4% z6PncAD3(@M9=(*s6W=08dujKzd+&qE(+xu0C-`;Z@NST4&F~LtyO@F<5SdVK{`!MN zyQTjQUyJ)d-RoM(J9CLUc>Z=YzYI(y?ijL;l>d=wG;KrwCVpYM!)L&M(D`1~1Wf5(UslWI18$QjuD zDQm7YGtZ;%)gK_l#`gn1tps(G^e~6I{S7bEnLkNhKKVqP!SzW_^wykhd@>5YRKjE8 z^|DuXM&Q$JV1BGm&wq80*dw0y?|1&axO;w$2AD_e<_emdjcv-h;6(Hh#Huw8hmDca zS6|>NpVo=4?g>Qg_KVn=FRis39BevO(+$MD_gG_*BvvMU(~ ze2`i;&CF1HKv}lX=tr?sh_N2v1d$+I^rYqYs7|_l5adYP+|3u0W1m@-+`Odno$$Ly z>icdi_Cc+PD)g>>bg{O^B$8$ZqL*wY8Hr8N05&v#wcB&b88suy^~LDf5xRhA31l;M z(jPe!ID8#fR32G{$TZ5dmJb3MYGD(Ruo^8ozmY@-A-dfg|K_!(fYA+G-?%&&^s5Fb zzBWB{>{2?VTD9iOO0z}A3->l)4uw&REu_8vjG^Ux1uX-diGP7^8Mr;~HUxPw-7x_~ ziJbxAAd zu@RF=dh|0GOgJthSNT1PO7bgRg$f$@huIf4)W{c<3l=I!rmVFdMUgPJarho6|7<&x zzh-1I4QRw7!`w=@FNRoOxT3vij$h8^{t#UsiNQYC?T%Tq{@xv%+;PAexTB2V3W`iND6SZeV2_U=Q$nl|%s^r#$`Ss@O--+Y^hYWu3&&Y2) z<0X=z9S$z}c3OChguAqPMCWwBSK72sN5Afkzk0k;6c6+2A16ItuXc_24oLSUlLFt0Pvr3W zvViHivb(yJI`3VwdgZ#_PJCWsMDwBEdfc9Wy*ft~u$sWA_-;{^JaV@sK$R}bnOb~s z(hFnIQD8!_X{lD{cGT*EjV4@NXs^OuWkueeVtsMkPGVf7{PDQU7PLHl)EEcD{BR35 zi**~UPupnSKZFv})AKVU`qE>Iuah*Yn-kqxfJ+Drm(dm7@(-faqSYvH^^@Ls>*p54 z(;?muu>*}y-gVMWnq?e3jX;T)eQI8E>BD_6yDk87@>Zu5u=u9{+L6u8ysyqwu{c3f zt=rkZvgqu8^^5T(=*T|SM^lT7imGVKKH{|8mg(EA7F=V(0W-mh%qD^FnrL@q+As;( zTR(TIL?EDJ<74~uvz0L(ZT6L|!kH{XCeMYMOV7qWe-lgp7Prx-ULL?kYOMQOD^UJB zcJ4GAQD>5#M*shNACPpvKlq2h;oNbVLKEl*vK0pAwrz5ref+bpQbkXRF&J9^{UDsv zTsh!6E?r22#it(V@&yo-;}}(Ajlh-sd_Yj!L5*k_5*q}tlw?!p!_MHDVGI!$eG^DW ze@LAI3uwHjaKqp+hcEeV3{=wGQiq`HeI*9d4C~ z4BWwo4X3VFBz6qi4K0vHH{e%RCIMn-O(Rc|2(Of^@MAL^1Vh%S8$1b4$ z3XbMvQ_j%y`p84w`t^>Eb=q>UQ!Rxz)uy!RP!qAz1 zha?EgB?e)r3@S2a5XuX=iQ`S@jnqIkry=SE#PC`jhi>gmNAs20$smIKZj2!Q1;^yJ z+{4D}vwe3&QJvm<>Dz`S#24pw7j-}6cpd*ca`?yb()S3*NjD;(q383=EpO=CWNv6$ z4Z}WqsAl90REp9qpXEKBs=b~|#^1&Dm)jH3TazdQUr9Q0&a!wBWjCh7{bAGuiLswa z7celjZ1kbL;J59x6ApgXe_ws!-W%cXnmsdrY`EIOZH>I%BjX`L=<>bMMn$nF20g*Ws@Qbmgo%*|Kve|zNz=bxf?IN6X1#Unm9OeR$ z)@Q3?DjEd}X1TYc%E-{;O=ST1hF+E#`K(B{LZyKd{-Qpz)aPduTW)61Oc4 zlF%a`S-)r4n=PPS_h5Ur1}qG3vB5XUQM@+nPi1le-wGzjd{Ab$?gfMedLn0P3pq)c z7RlC$Na?7oX1R@so4B1-Mi)-xl<0q5#jeyq@?x7=ABL3~zVQ8hc(Dn8{kd#BSq5c5 zO@Abi#(+9wwDGf#`&6OZzO&66eW?+IZKm`=9krY8=iRbDXqXX?=+PtcEA*#AwuxC7 zF#mW;u)7Y`9V;^qkd+u-Vt8o=<$g9sKnx4*U~MliA)kC6_Q-uWZy0+EQ!TmvscDX( zOJd?+Zhk&C1PVx%AW?H8=u(W}mGWc=7{ zWcYl&;{Do-)Z%~}m?m`y7lO*ze}%+8mH#6JyfqkZpZGuH#Fs$zFR5<>Osd-ds<#~0 zj=bv$MhiSQX@aw!eO)E8-|$Ox3c2yR3Kt=pw9Aj9U)GGf!#Lpf;>S5d6xSHx{v3gZROE;#+BZxv5t!e6Qi!1n_)H?-sb9BK0*C8TPH-*tr*(E zRvwUkNyPaGsWhDB@jFXKw`E{G*fx zx+QNHIZjC4B>citLN->eTDEd@baWdqHif&ak5+$}u1|wcF_T+6DGiCLVFatxQqDq| zdrLnUpzkl5>E@+G_WSmEd|Ym)>ze^QV`0ilQoY{$gVT;lRIb*AwVT~g4cy(JAtZ$_ ze7CSzN-rdFz2N8%cufLlhsj8iv~p>vGfBKqD6|YRcfeyYL)wN4@^0S99cSd-?~Iw0 zK+m0gVCbwrYLzw`=}Mn!051kP^Z^pxfhjRYn`+>l*|eKdGk z^{%n%r9aQ?%_Cp}H<;=zm3dxXQ(T^KzmLH#Si>JeQ#StkqH7-g`=`{q!Iw@#-9;;M zp6GXeeakUL@_2l~-^V=?9bnXpjh_|cQ&vjgx%RvqwbO7Z7JD`%!Jz>Rx0Y(0s!rXK zGw^K-662ZJ?)yFQ6&X8AM)X5*HA= z{F*4j>_=QhgwWdb7CHQKSa!@~6Pu2X4!+bPq;icbM|zJD&i|qrq95yFh~+&_*>(@4 z6ZI5Yt)pS(Hj`sy=V^g(jgd%ynblE+r7g9V^-kc$z5BjJl-ZTP`)R6>5+aXFJ-gHs`&?8kmyd#Ej9jV=uGq_IIhidU)J)`#0Ho_~l)& z{9^r${eBTT{*B3yTTaz2igTy1d;G=TFBz)jDfDR4L%=LiTL$ z{gd70XN9^Yd{R0Qdu-CO!6NLmfV}E|P5V6d^Q9e@@AdH`e|6~x#Z5bnE9#FsC|g9P&}5wY4ag>P;J&@cwdfdfYG>G4bV{QDFfw=Q-a=2)R#P z*4KXD%;ZSw=*1U9ErgPPFd0Y-mwR<&KdUt5mg{!b8i$GvG^mk{(~uAHiS&6tUQkq- zON12E<5P&&Q_~EK>=<{5zCG#Nre>_EtFz^G%e!XYxV2C!oM+6RMDgv2ggC;V=_-+l z&^|B=$Qf9Kh0S$(GI<_LM%BPcUP)lrf11aV^)q`RPq&wVvg^J@)AEVM9qjqk(R{|d z^GFj--E_>z%S+oRYFJ`Eh3(%(6I5+R-dEeJJ9#FQ;qCx+5~HFazrLdKnkGb*0Ko}* z`xNwkKhgPu9N*?yYd6Cq^ZQ**=h$k<8i>4~)q$;}pK{_&u(v|x2Or;gW}64PF>_Ih zF`9fAz0``my~3_o^+dG|zaCh~#yiiGkw+Uk6M8aHGG91n67SJ>8}`TH96Bow6dj** zWzP`TNagFl*kgrc@9gYQ2-UN)#UIB{lf76nQ*{aPa48!*W=AKtlOca}@zK-G()sen z5;%Qxx2uxtv542~CyDMUYUq7=h`M5i%6K>~-`*TZK>5BQ_eb=rL6umlyuy3VxT^dt z5jTtNU6!@OLJaRjT^u1vS#KYo+f2m#8Fq(MLhSZ{U5qluP0I zxG;bI+3!M>>E8AK>jjuZs}>oP%);li&lzwcMS(?i7lWWAj?v^C)8KvJ zoL25f?}JiPt$7*s)d;9T$ACz5lE9}DYqAc`Zx?1SSEC#t?rNEM6L_HuL`5JUVLLY? zwJwJo*_CM1XN}?evw_D?aLdYPl$b*4Xv$C5rM4GCLk5`ki3Vk$LlLNEcBr2R(2sEUwV*O89glbX^@C~yV&j!&5@1qwWP%bE^S zrZPy3%BKhMivV@LXE`Mye*J z=AXMaAPlj?vk0^8`83|t|Ez~Dt)Tq7vz3-m?Gm*-@9VvsfW{5<8+9-wXRIepyrYs= zMFm9jus5trQ^|7BS65NPFoD9%8Li?OE~sB*cjRzmGTV)C!gboZIh=jiJEmF`1-APB z%0<}3W@z8u*@mF{>?mg> zePw!9E`K|+*!GSD<4D07L!mYhB>Z*rQiLi!C;~W0ViDwVfds80`8U=r!ny(U?SAC``=xps_{NOB%paK-jK(>(Gt5=722jk%`+ z3YP~;2Z61bjHeFqDfX6xx!B#SkE8S|hMuHlhjd_z6-5nr#5f8G44KXhcpe@ejrWu* zH!>0M0c4O@pg6>GdY}4Sm|ZJ``(!ql)B&Q4*;8#|IcD!3=t9lwW3TR_#4V=Tf;Lq4B|GQscX0C>=Pb^u5Ke;sBDbVw@S%q~<$9ZCR3A zEOu8320qWKb=l3An~ywHU&`HL9h@*Dm(m{wVY*Irt@KlB>`Q#_DZH{MI{?r^S#8GD zfFc*`e7Nt(+hXHbu{G^U=gZ;DW(5U!k3?aUB3*_xDX8R{W_aTGiWqg>WN>~>1i~%_ z!iZFrj>d1%3XLQ^-5%8c68qL!Co}Jx>oQABZGAWjbzYx5oexG^38c=Z>S*N$chUrI z^oFDAwYbs!n!`goJ&9i{xaZ>Bm3l78mP0UuF>-&|s$kQgBU9jQJmvb~i^}rUO!cDZ zd%!%&9xXE-kC*#f1rx93{&T9HiVE*t^+5#p1j@Zwk}@cmTCUT;TS}p}zOHCd*Ar=r z{58bcI;a1Uge%8RuK}f4Eo0*1`yQ&CdXi=QCPy%TNaq|ws-KY{UDi>wV`mYt2kA^K zCG+Uwbz(Y^LqLi;HESIXSWb5#zl>zpMa*OU9;BMVZ~phU?ZKbQWz#j^zp?2s|7}sQ z$QL5;?Tk%=0&=>mcV#TP6VB>qW$5JG|U8i2~bU%|VhshZWT0J^ zjJv|~EjFp2b+`8^z}pp49pA9O-SDG2tX-0HH8?Kw3$`_2&b{Dmeq#gSOAp~aps6efG9W4MHbh~keH zBposjM3}eri5>rX_E#PQux_=wgCXKK&Dj#R9-fB4G9_O7O0YpzDkU%Vk3f!5oc0wm zGtCibA$=l6|BSxsWj({KR6g9PeG*s4RxgJTTx;iC%Zk5({+Hqt*<$u?_k@^h)6>|O z95Ncru55ks#@%Rd@1P2jsj~Ht{Y8E#1aUb9a<1v9YHDiv>THLCwC>g>NQgq0F~f;> zVx4KXq~Hnh-m3u~uG&a^Q_Gm%6w$W2P|#EAjk-eJHAv!ln$iVTKcec$4*Fd1?2J4` zce#0+t>mMOgAqT1eN$F<#&54;^&=1N&VxU2v-?$Y_+T(;hLTy7Z*8`E|ELFvgII}z zfdR2r5-t|@6Ja$j&)fCwF9ZDGH5_s0tr(MSb!y=qlun{Iv8uJ`x2BvT)1=YUjk_Yo zjC$lGYXAZg9{6upVBoCvFxBDS9)g@{Tw&oiJiM5LUWlT%vGF&+#f_Hw+oKf8SC- zX55dGnr)m+BZUl>g&~kJCPo4?PCi*}JsQ{_!W+PGPuR*!uheL z7OsiL(0%qgAkU>FgRetXTdD7cgue(Kh;lH$eBu67Z325#N1LnSBq^Mro^Qltk-SCe z-?_!EfRKZ?bhLz=6d5EAw|<~c>+3-PT$y1bw7BE~jn0pePw876W$NhFKH_z)b;yX3 zig_nf=~SC^G%L(PCW>iQn~43knTGkgO5P1FA#EpSGshapa;f%mQzsu-W5)Qpv9S@< z-YyzPp#fjPf5O#biR^Qj6OU81;3lKGteXUcZsCB6v146UI8a}8|3Kw2x39?6DuW+4=RD;O=DUI`tlO zmtP$8Usg95#im9z1M{B+=$#M#Pqk@+QZjv>bT|qU=KH=}0G$Q-$D*OR;{nvXXay*D z>odcxQV|0We|Lo6k}S`DsKaGdF4daVAb*{Mm%%h(s;_rxAEONw#}h8z$jHc3>b?wA z`IlSD-!`j>3DX0j*ey%Wm&{xw>eAG;b^}@udzFzN0~oVAm=X}si76!{a+;d#zNp7N zmkWh9T6URsL+EJG&!59x1*P&?tW+%feO>c21JlH828|GRq3TqrCe0jLP3Yq&1*P}B zfE@l8YwH5dP+=rw-=5{CtLra|t^2&cP_uIdzwppIJ_mhW240@CpHqYrbPz}?wZ-M! zXAWN2EI)Y8o2V3@Wxxiofuq0;IBKT1B~-Iq|(O(r~weW5lm*n_D*Fd$+He)w`0N*$sB{ zA&(?6&mySC7s6Iw3VYk6(RchAc^xCn5PP!BUk#}wJh+?&eJoekBAZ=hJfd_i>iQaJ za5v}8*k#h)c1NhFq>Mu4PyrT4zirV}6q7S80KbGy`WGs5bD?CO8+-3y0hjD&!UkT% zL6KChFD+-Mc3anjo~4T}aDFfuW~)a7dX088p$jz^fObqO6RsOfL@75Xw3ZMTzR%#7 z7zk3eJmuIIKcua1YUr)JUe;XN&Y}X&r$0k|zo|eQ|6FZ?meKii#Ign~$bt`86~3sk9=kl@+K_Jw%Et{fo2sa2mlV{27WIF^WBI|+mq-(- zsrM0P*Wb^t#(rLs_i7XmXaG*vz{QYOdJPrn9NHazrq7k}PyW(a9Hr_b#(E<}pmG-8 z7bwfcS;*x(V>U*z1f>xx9`B|q%)W(rs8?p1@YJ9GgGVK46qO8_wtl&3#SDAXR)$@E z`dLvD>(gh-r3Y&97txoDNy~eu&k_PFfloO6{QMT;9;FGm`)6-$W#jkKyWJmb2v5R- zjYBm`Ty_6&2n!Rxy}i9G$w*8N08CQ_*@=F;-Y?0{HkQ37`8*yF(F)&<<;qrXsR<#Z z{)uSO&174N+|i;@DAX2DGUiC6T9eEG}I!GypQr zkmPRFw9TjvFh`oK?Si{xvfymo6vShvAycppYLqoM%mMq~pu5oF^ zO^7ufo7q%uRMWYFO1ry0Lu;W_Ee+5ybpudz%KKW0#$Ka{ZOpW7D z9MV=Q++&B4)^f!e@WZa ztIy_zdGc@uzIK^}qyVT0l16X(6A&%AT=VtD!~^Xl3>LnI8h|29EHSnpF1KHvav}%w zl-j6H5DL>IU~uq?iYI~H14!y&3bI?vZ*1n@f5V%qbcI{)XLI|}UP!&y^QK-d=f#)K zH#5M%!X_pr22&ZcSUd+`x2I)4J&~8s+r-R`!8EtEZJ%74VmZkBTZ=)^{-$N_fGtg& z=DF;O@zwKZ)%)P`WA%8h*VInf{!ELobM2>ov>7hV!u?TnPz9?cqSZ{_?S&oZ@^9pj z9^R6_{HR~7@#RRh5CQQ8k$2|Zxjl@Ga|MPA?2sFuf4e&&>ia-kX~+i-kJarD=-rnr z66~YgREzU*b z4j~3flB^1#QEmiKDCWbMKlE^!_?!Sk?UoBh@wmazvbf&^=e^!l0Jjs>&6aeWNopLT zdcot%Z9`<`1Q!`bJ(ohIVN{?_jI zqm9V7!2`U(r0@n*`Rn^n%I4quw@3(&y!?JKu%KqFE&2b}MpqdW+uvl7GRXNZdd(WF zSHzLhiMF+dd8j6^8au?d%$hst6UA1+>40RJJlV>ylNmIvM=I)>gss6CZlS7pBO2c3 zK1L@%D2kipv?7`iux{SxViIoE&CTsXr^_>qzZLamq(zFmiNkebS6)(8d;o}b=6Xfm zEtvpll<@p)wbu7{UYRo>FDm)vTgtUtPv_V5JaaxoMRH?Krx{X*het-@g|e`K4&hwa z=jx4Z^JqTQuD+9o)W?FdvNF>5s9v;#zAs3enwtIl)8B~jBR+hu$kXH4x-Dg>)XsP+ zH4lU?w-h^RyCEM<<=8)CLG%^rkIV{DH5c5ulEKoL258m}mzwrrHnE?W<@QWaoWE%> zmGiuoLqYFhXk6pwfvWb@Y%WjbjJjM$J<#B=63#PYt64Tw%B0I_~0yNv-{dMH26UvSu8rqp90q` z$(v&dD0G&+T z^13^G?=^S?p`I>Uwgg)&)wnZ{xgFNNA`}pqTJD=08Iit;)}p}5S8lH7hg#3aE4cpg>3Ql$iD+W3<}_h`EUu4dKl7b`1d zKygtKorQ(P#?PjVJT|bv%(K>6u2b?PHbGYJJN^A~c@fW)FeS9l{*UIs&l1EQgu!Dl z*1WbKDfIP!3fAu0|30WkO5TzpPlJy=XpXS1hO}XclWTu*V^fhr8U7MP)O{p zPv-C;@CbKU1by}(n7$5Pb$q(lxP1hTM)_utTEQtI^rvQ!KeQuV)D91Lv?Z5MAJJQ+KD^j*DLNoFx?iQPx z{7BlJRj~@yzg!;{^=BI!{+HPG?*hgP|MmeIz0vks`hUuor1r+UVK9VaH9wjwW7IB2 zgAGrJOeaI2^T45fXM$ig_m5f9K>j0G<_UbLC_u8QdsxocS=f0$HR~;HE$U}%v)!KR z=`LGrV|IaVEK~-Qw%27=USev3L8iCKac+zs@Enx-K9V{%wI+i5Ho@P?)T_p!V%o2d zU~g|HHR!UUzVo2DO_WB1p|XSw(8Z!E1jy3^bf>|%0E}^|mKhX^HY?LC^FO}x*njvj zU|_H`;+DfLtZ~;J-xfP=8aE1r>mYsG9w<2#F_kaKXeKTAb z9;DdcY*Q z7p6G>%9>3gS|?Men1rAT-zCwAb2%V0ZWGm+ijop4adQ%MZAO|&;`Eua+2ay3ZB7U` zDI9m_104qYQ9!HD?GeMZ+5Oonp`Z&TxVManA-AHDpvEmS=|zN@?*(JJI?2^-pVouC zUE0XkjiV?tnEe}_8IuKrw_jA~+vB%04{jJjMtJIUc7b#nxja0cG$^)%{SLg`$^)`F z!4P~{LeCj#QK_bs8__ec@1rpMpQ2x&H~vtXn;~RHm;OKZ?&1UnnI%;<1%`w9_g=kA zLkqL3WDg)p;BbY2RW(O*lZqA!(Sy)}N!YIwQeMZuDVr#4XWG@N>`@!F9fGV8Q>87Cqz zoL)OZRe`mHps_sEvlFoo$hI_IHuG}x6e2P@ z^i$cC@_^R{equFlzb`r_MtY!*e=1T9|FV2H@+UCa;u8%!hjdn+K;pf4j_)!%StbJA zu@J4&U_>%Jp@_L~*Ly~`)M9(71j2YA@D&k5`p4(YhtI6SA64-$Qh%37aplxXNU^C{ zrsnNvgl=BAaGq&E{fRPMMAfG|%a^~O#)VVop$zmZQbGquWF_m&@po3DXHlO-*FZgT-= zoIdwl0lcLSwq!LWzAnY*T%E4ncKw<1h*gT0R(6|Upl+W>dmYyyyP`CZlHX4113+D>g4OUT?7DQED>oBc=uvpfA1-Fdn^cq1^1l|0UFBf z592WSQLjaY`Bs#j!Z73y4P03Mv3{F8*3IM~Qpxvoq<{ni5DeZf_k+uo?J(IlG7-=I zVf3tUSf+325UR=ua?<_5x0k`Z^QKs#tAwh{QA@Vk%XYCJRT9j<#XE4mW>(8owPunf z!sUHiBcf!^$x)U2_UUk-#(sFwCyo0HXxLFv@_h1IVaohQ=gg}hO+5ySmQvnn5Zp9L z-RvZSi&sADj!P~E&`iERUzJjA!itFZJ2z-`_p>zl0uxgHQO>CDK*{gs0NZY^w5K+u zKt6tJfjvhgNH@`o%X{KW`#AH175hS&HX(Q}j-?9cJRU z(-oJZynlNE7HkYE(16Y)EE{=McoLX%X?(f#yh{l_ukFw}nErDgHu_jDPimNHp!{zV zAgHPLPTSe#aE8wFbA#<{L=XPg@XJhY(E_9-EeDohBL9|ibH`(}4wHoJ)AEgj7A#N4 zB;R_OdZ=IN(`3O7t!X~92XKeY-_sf(O`O{=nmz^}@*#pPgTdzlS6t}KJGS$n&Mf=d zg1e(}hQ+JxkH0HFhg0D;Mi1!}X$0{$DoaS{{4KR2<_CXL zt`=^hIbEU4aptkzg{?VC`Lj*(j%2_;cgKdN^1ym_J+GOJREOzVo$J*fTjZmlPfCz` z$kIO2uT^*m;o1jD7*E%iY@(9nj9Hvg{FC_p&tCdppo&N8e}O9HVCSw^f+nX`X~?(6 zW;$QJv8Y6i+d#3BH`%lVUN?rH)*3U@PKte%c;~~G8ov&Q#jVGm7je2xPAiEJLcRfh zCIhxOf=MwINep;r*?ZkGH$C4GQd|bLt)@Lu{UEvJ9BL*iavX@i%;+SiHD~Ns9zj>6 z!xvchLop-e&%GrR0ycl!(Y;in!!NT|9KSwjaU&$?k4kP)fEP>~`x;F>WmFeo^`6}* zuwY5E&{O(pBu!XDn-!d4BsGEu(}g-X%QM(^a^+ySi4maF014_;HLD3PBh_7RF=slx zc?88<`aJ+a|1%V~by1ny+&Y^jf6)(s%-|gjR1H+Ai5@Q>j^>YetE!G9y8SH@?2>gJ zxTu$EUj?@k5$>rweEU%+;37-4<8x8bkBWR=c9`zZT;V9#dPM{ze^^p9H#fUDuZC9| zZ?4-3`MtnA^7t?7b_cIJ)v|=;9}VeS#9`dHHGxlOM>*|R9YGcG^815Mi%t91i_adx z>8_Dk`bq9$AAkMD;n-fcfV5VGVl~()7e7HSCDQz|6vSU=B2_|UUjrf@`}bv0A8i3G z)(M$%8L58piS-c2d^y6SJ zTEk)uOU-twBGg0o?99KeiXsFzDCo zqMWS1l}BK+;L>K#8b9N@RU%Z*U_kF8+mP!?FfwX_IT%Rx@|3U1$LTm@6KDF`KZp6v z-H?X&pcXC3on~50ZhvzLKgp3YrN00)=0Mc?@bqna{uu7#)z^WJ$h;aaHyO_xDhGpW z*u8h0(3cmOw2X{xh$AElwJXHjGW9}yxaHeXzvhGf+D<@!zO+5=R-UGnOKpDB&QX`9Vh(0(q+waBH@WXU;eOtc({tjiyQ>*$ zcIlC*A*4WoCo9r?FH>&uX_S2JzP+WM({hj0Qyj;y!2C;b-^J#-IYhrYal^xo;#u^z zjuYHH*#jb>p-Ea<;rTRGM#*#;%ZnRr9&Lr)Q=m*$>A+^TG3FaD<$c~9 zsfo{4PSWMT4j#q=b_!y#W|t)q(Jm#ecJG|UsWi&Ym+Pp4g#tZz#*F--g?TPnX*a3^ z`3X2skIV+nkQ|ojp`NaovN$h~>3hqpP0^q4PvzgCd-WfVgDVfFbtN16L{jtD7kF|R znH4N>+B5kbiJ67_rIp&oG&AuKm8TORKg-;M-d7F9$5(c!v_l|fzscOWzIhDiwY?a7 z9#?95m9*}K>>R^$(^zqVn7~4$1qKq%di6Zj_7_QMAIbvrNN1^OEW=BjmuH&t4uw4IY+TLluK zhAb(%|A9yDLspc_Zx?(r#oCM(>&O+J!5)oD2q@;zS}=4azu_Q2R@Aj6C?3-b!e^0% z+gyv^=GyRuu6jGGsjKT0eZ}T)lj=ziz(^ zj+KzN8NAUACS^;5IY8}u)iU-aHG(z+usfe)Mi3`qYhOG|JvKPMX-dafC*SxfM*{~_ zihVYAKPIV$;y#;O2^#t$3P7*UxJ5c1`D4kWb6b7)M~kGfM)lp z8%udS8V~g*dlk-)c4~hKKS7K6yU}F1NwaS*hlh@CUA>xWTQ^tS?;(yB)Q1-P1$Hzd zYNN48u_?xbqS)q{y$|0 zuj5*sh=jj|1kA8huT_f#A<{c(bd%ZEep>yT`Wit`pPNb5H|$%S$MCJcST@UDff7EA z?{Sm1S85wzw*2Rj@p<_K=&2ZvKaaCwyT$tP3;L-)uV@=S{2oHzegMZN2Y?5 zdGzx2)DkDWCCBqd9!rNVrLD}<7i)m}7Gr_4XJjyhdEtI_m{ zZ^i!tZFrMGlfINCFru@&=JO~4BU1_7iibUeWQMl?Y$_&UuYPM%Ny2L0&X&%>C{lnW zmG%m4H`K-U+t|o=mioS#fnjoIQ&P)?(>^S0KBWS+hC0D6FFc4VfrqtVI^UeKY7H0` z9xgO9Gh^=TJURK*e+VwqcQb)%yNOE7SQOKC4*gEHuXSFt?2|x>C$~)5jj6elZnp1^ zN~k1LuyhOF#-AVO0Iyda72{@9YiC+N4@ZyEgKeN2`6tjA*#j!v&WMUql)RM+44QVb zvF8pi>-R==zn|UCnNXECyTBF3w~Rhea(0n7N1p6h_{kEk*3UCKu`3qa;y_2lLuj z`o%O`a|epg_&sy5f>1IMhexh)Icv?GyyDV1e86w6-%>n-B7JRB8Uqj~QNC^(mSKEI zWskqw9WAhQL{6P}2Z~$XQS%&&1*VvE`8ZLGDCsV0Cf(4JkwrMY!m*#NU(!6G6EWlx zW|g2!<69k0_}iSM!D>|fLNoz0q#mT;QsokU>_2YpW~iZhuhJm{7&SjZB{g6wY$W!@ zG%D0umy4_SKNe+%d~D_8U}P9p-6W&5IAlEIV)FoiWk$%e9{EBX;Fb?w;doS92hRsL zI}EvOWza(5s=2eL6xg9+>fq-Jt11LaTpm&>ro8t2e><;}mk9g@C;I}X%!a)20^khe zmVfCNL7J6Egxw<`E}7Z~sw56{$z=0XY~1@F>+)4i@%Df`Wyf%Hvsfp~=E+}{Zwid^cAlH=_xK6z-a-|;dO*ztxN;6sHXh&G zB3}mw+UOQut1(9A=kZY$7Qi*^;=x9cB*@zsecMo2X^9`|-?aP2?)fJASUt?VU zSNCbJeF=J(efMeHUbBI7Q(!P+ZEfwz2b%kJ0T!H^?J4}Xm7GY1T-$RT$&m$~O{}>c z%ZN4cge1kWI)B;oG;PPtK~_b%w^&t|Umc*lQ^-5IblIQf^wUd7?MH!>Q0pbV*YU84 z80B{-8pzCl`+TPyYoPc@04N7M- ze_nEMSosaFq_mUCOth0tQ>t512W$5LG04W!W%?UZE2DHT)mzsF+`@<=<6~z*Xy1Xi zoMM{2CyNI!q5kUHzUxu5i1wQzShqpV{wgl<#31Tb-4};(pMFlDaeua$;M1I7if)(L z=GCoU4IAFTz`*;JA5){pSS~`E$eH`%4~(##wcvH%_Z5`6Q%VblHjl%Q^}zS6p6_qN zUMFcqPUDc{)*+)b;C=7Uq%IXME}diTzZID38?LnZl9>kZp$EQD`LMiClYlS;-FgkXtuzu&?)>8@cNAc{Royeclfdu?^P+(XyVdW zvwD=2wg|A@v_6?8`MfT~e=7E$M(sblcS97LiN^QJeGAwfTT!!O<%qI}(dT@Fi^rhDLh zm~n=LEut`JEaw2XLhxgpx2KsRzX{2nCS5OiF8*@GJ# zUkP&8&F&puB8miNO4eWwSj|v><0`Cs{ou35?PNK2-*<6+X_F~-kC-k1@RwN;z;8pg zn%r1pYa6jyW+envAxCb!JYIA8Zb`~;ESQ~r*^Uxh*lT}aPd&uoJ7u=de7|GzJBpXG zdlC^`T&6sn{-XDcj`uv=j_SYSG&FJRy}9O*r8OB$u)W*el8+hV;oky%)CSHE9Uswd zMxz0^=FH7F3QTv_Vun<{n>8(*>YkC3cs<`aBKYaRu$Wmn2cjJjpz(BVqQTf;l_q)o zQEViMfHuABnYR}(|NJ?cCPO|~d%@%+mr6q!yKwJ|aC@BpfRoVnM%vJW|CEm5IGKT; z8lEbk0}^Gh&Sw~(dL69d&cbgSSz|EjuC)ZaTzx^v7Hf@;^YR{=7~jukBF4+m|IEVhBEaX#`#iaXH5@*3|_sDOE&Dli_r*Woi?U!l^Q9B7`JS&ayT zYC`3?aly7xoF6kYEcWH#!3h-L4qva!yUL&-yve0}In)T_tn$IW)(1L|`GwfYG z1A7$%@8tD1NQh2_=m2qeOwGgNvly30I{Ma`1Wc{><%jeDcW(*Bjv~Y{m)KM$GOr6?aK>IclD%< zr*Jl%*c9>A#1ba*y~fJXXk#$v_oK6`LO%M5Z{nzBw?#wV&b&r-f)s&$Ih8zXfjsv= zJf3}5_H#cK-GzMk<}#6nz9SE&$~L+sY*w%#`T8I}SN+_4>QL+-7@!hiffZ^wR|P7H zkg)iD8pei=1oneAXBSn${%%W?&`Jz@k&r~d+z^x3L+X;C|1Nx;o5X1mk7|0?Lol_G zvyo^N)GcK|TZK$@B1|3RA(A`V$M`earall+`Jyb4h=i31OqQ|3db!YM82$Ds^WvcyqO zeSv!meZ9Rk*2g$TZJzL#OcbD3lZ4Dn(q#8P6N5N>cJqDR!LVGAKup@Mc*K<(4ek9O z&?rV&`eRU6uH>t|N3}*6-DLEIkj5}F*^TW=?bDff3V=nHw5%IM(YFEd= z6WRXdxWKyERoHaU#F2AW#vT2COSQeT!SgQOb$#tbW(iyEBzZ;5#w1P;jYxY&(=di5Km}q(blto{UtC&p4byW>&;XLrGD_yg6D-eTx__a*sjLwm1o}`# z-^5=`PU%)O?LBOgBXfH&t`Vc4Nsyvzm`BiIsk#`Q_y|Q6I3{^2rK_PwYs8ow!3WNS zuQ0Q>gujJI$YR2Ep|Rbvu1}6(3~9AArxSfhE0{`B-}DG!n44*6MG2B7VC+1RVEG{N z%ewO+9OJUUIN<2Lp=Vp|C*;VmD!6rr*Sx&}Y2=?s!)+2f++ulq^3u_?rv@285w6@^ z8bu~_N+Cv#U1iBC?-oQFAT;9!KEL>~QTCU39?=tt?xlg%xvmsHd**ci7IY?}fs>cK zY#!HFXU?=lmhn9&0rJP;-dd|&Xybuszg4CXP!LMX+mMcYO&m2p@~5f%whs%#ocA|e z$3YO78L)Fa(%}r&Cvod4b)G;0S!^=5wykv$xMm|*4C zZEwYET{h>O1y!#mjzoS@U zurCtxa%+!XaB(EU9veMmid5NVci26IM!LhNZaY5W?GX-0^8GO2$%Ya8I`2p#$~|Ak z=N@!U;g&^jB?P1Pb@NmU>ATN{Xq$@)U5_%5Nkz1GA#B6J!<^@1IqjB40{P$Yv)}|h z9^(x(Mr>3rml^0239_E9`}_NS@mBeH<6O+27Er5Xi~jPtY%;gt*ERu{XDPw#fgMx@ z3n^xLw5n*QjtI)hRw@ytW5lCeT5^I{Q;&E;n?dPlgAsSNI}+z9cg*$kx_7HtBi;|) z`yg*!SwmvdM!P}B%^3G{%3O*8P5O6?rF}zBm&*X|Gf8h>bjG*tRp*_4K1ZG}Vv|yQ zE?iC$QQeQ)h!@}`?QewX7fP9j;DtQ%zK93!SvNSvNJ{-?^WHrHy*4#JTdJ#GJUHH; z-8Gz|9*fQ3hxYjHs>U)Kv?=jbq^GYLqVH z8{ZIY-m%o~5DX^!97j|D+81rUTHH&`fbFWT*qPCmYZ=PEuQ$Di^xr=ztmQa@zwc-wx>cum* zhjV;n0-c0{td~(Bc?O&5X8O0qM>eZ-4poe0^{}b`+EQIeR0bg|zm;%wd0zbkK{=!R zGH5_?-=7%B7V~Yck5_Qzze!YvT2cvuE7D-y`M}NbBb^}bUp_6L3OzHrsXizRCXTA> zeH-jF0x2d7F1I`=q-+ZcG*gpvvr_XDGMd!nbU&CHYU_wtCgH@b_h!IQEymoK$cY|= zF(f3|1*fZJ2Cp1DLVPD^v%NSTdW{Z5@<7t)Ruit3Ww~0{9q|IoLOCxDU7W0B1wbf| zH6W*CM?<^WE$*zM6rUPr)=M^eYUfw<@&qUdaBoa}9a1Z9MQv3rF6d|669f&LyATWS zqJZf8``3Q_jENK|Rtsckm5`QZ1I+V%-^?-Qog@36nu;hSf$;&2tz1M(Au$u7Mke;Y zz4>Gb-EE0upjKihixfzjO%zcYygr6>*$8A&24 z9P=PyviMG9H;8@^BEVTnohAiJq{f=xBAxB!7XqbtC76A(FB!%jTpo`l5VPIw3OS)N z62n6-k!9Nla);E->BwP_!bsT#;7p><=%@7c+P4G)JTy2R8A}?D`v9vWsCVfjuh!$>%^EcWOyp|;`Dcly9 zihyt5=t8y|6oaNhtBJ@zpMxky6|5|-gfl_`QGi#EI(U>M_TF~AOo}V*{&sprWewib zqyr{*d&i^bg69ZHcG!Ug1Q9@ewhW+MUhn}aiJk(Mb)^|oIX`MTM8M+tEK8OlSB8_e z#3%ol*$01;2IAQHmy{svb%v(@Bhk;1*XZPPXW*f*w!_KumHTv~JH<+iJJhjNGB-1u9rz-qe4{q z97!Q%8?s+F$*G;1aS4S~$UKLt+uEYUa`d<0O)#XzttF2d$n6B!sWVLpvN_sTCg94z zkrIApE_SEB$L%qgNiT4LoDxPp?ao@?O`hmHx;?$brb+N9wEswOZ#I#5j^4jzSXOPL^|?AwhDCl}^j4f`3Op(<%od6?gYLOb&J9!9t&$5#Ew=B! zjyqGw+;k~sGzbAx;XXciK<5Y_Au;!r5qNacgc}M5V1{x6S@>n+nCI*XpK*7}SyX_Q zYc`t2QQvnsBGGUo&7d4_L|V@eiOK>L`$3(KHSa34ZtHYCGlQ?m&mW%u>A?QC%pX(? z^?sjk=ODIl^PjBizjhB9D9tS`u7T6r-~`ni%1HQ@POJAaQb1Y%`M#Wy0o)e=#FAny zh7Ux&SA(Q|FRh1NbFEgk(!`%g(M_VzGkR8q=1}{q6PJU0iN}QtBOHpHOrnKk#BXB1Cw7$| zKd(F-=3_0SWQeRtvZGo4T8;eX3G8VlZL3B~OKZ&dmSPUa@zB-)kHQm%I@|(nwZ8e% zXxPQkeNncIjYH*=^)^env8n%;pmO>5z#Nf8t{c2HBbWVv_oF)2WfZTCtB?n=SLbj5 zj9}d}n;nx?OsB^=T+C8Inv|Q$F!Zdfwf>H@q;(%C+=D-Ui&Q-DY=Q=9lbM=Z$)3+i zz+?VXEj#^wT1#@CWn4bNSL$N6M}bX09w@doNmmR=%FMNdPzX`=@3hcvnDku^eH9|K z=#d_cE_yJde9$K#TRt=5tRvfb;F4Rm1&5j%FH{ajQ>Og+CT~PgqM<&M)WChZ3K4pR0>i)i5xj;`6A~W>R z_$eaL3xhO&NU-iQA#*)!&xwPV3c0@9~&~mbw zbf(E+p(hVqwvIube&=c5EvD={P4BjNA3W9fk26g-EJ69Nu3h)QJ zS1Vv}oSZ854Xa4tdvniPVR76?v43VqYF*>31yXP?oQi-dZhRM|qr zJrHs-*d;xBFNcsL-)=n>K@TnP%9vbUGyoYo0Y))(abmO$!nX;W$+#$Q$+D{#ul@&C z`7>0Am@IZF>0JxM&yX>4Wm=$*ehPp4FLLWAXN-DZ~*Wt~)T|RV!l)8TDT51D9o3nn8b- z8d@vTZ&&b-AOkor+AO-cEm6h2J5pnLW4n_BE_59~afxU$qPbH`(jA+8f9c2WT0hcx z1yYFKe-Oo)hEalzK6r{ZrDZVMT>1C5oItHv6toiG_nP%v8^!^jJHl7tYm!mp8Z zf~?^p+1TlFLjdWjFae^vSYIeOPf>L;f>zn(Hw(N(7fR`oee_?|F3C#bdCh@MXx0(W z&yj$(E}I@f3H@f@laPMfvVp0{@x3%b%lEV=aW%n)mA}&L5@GAbm^p|_!~0bXjB-;7 zns~G=82c_N8w^j0^p8qG{x!pzL%A_VwBNsQNz(5BG)SXbu|CvAH`c}QQ=LnrpwjkA zmAO^lqm-%CG7;s73nkN@o81@DGD>(@Z8x=A17vya0bzb?I*YC* zq#-PJLAq;TFd_WW}ZV1yof^dS$*kyJ;J08d0QH;&+B82huw_2r7C z!n&&K8-IX>>HlHuEu-3MxUJD7xVyX4Qrv=D@S-i!;#Me7AQ0R=SSZjIcM23JT8ewH zwzyLq3PBoNiuL0?F9)`UDTKa|e5KxnXnr&Zkz|0}tz zZvVH83KYRku-~xRgC`EToT9wPfsliIc`Jw~`Lr>90p#xxT02H-Rz64Wct#GO za45~qZfE62zV^LtmS;|Md&AFF&JTx%g@*mX$K>zdLOAoyFMzeSUEF#4Fl_NWyVzJ2 z>*Gx9+o3|O)0B;SC;7oD8yb#&$mAb_+@?5$bbMMPK|~e>HdjEBpBgPYx6b8hoBG3c z4L?j-S2#eI6vFLDcYcZW)FDGf_|r7OkXLR@4^w4@>f7Iy1V>J^gr8pPrbPa*2$TDD zJ|@fK1m8*d=y9v|`PH}%TE_GFiMg&k+x6B#1?!_nWR{jiWwq)}2ylHnUT9%hPC`CF zf6sa9#i^H$4%o@#Q=A~L#}9TnZPy@7QgLQSUCwv4yKlb{JuFH3g;;8!Q5X(R2n{eY!*86*^#d z`LDN}qCk3n?Fad6`X;!AVe(29Q~DnDUU61*=zyGZrytrz*M!9>p{$DSK~PBD8Cjj> z;a;E6${Ts;TWpt}@1NXBK6pjqrTnk!39ch^-C*8Ps%!IO--M+v#(CHU4Bw$u0uC-k zR$U?U&Cudq(Z&B|8~vX_LhXUZC>_E24~iqa>$9S#r^tgjX`O5+sWdjo@(U~fHF8wR(*;NTmoEao86AlDR^@>DV7RcLV&FNg z03qhueE7_Of;5EAk{9sO&E}sX-lg63Z^*I)>VjcCE_9SaUpRbC*Idzh*M1&j7H*u1 z`I2D#q={sY*W5YEbWMJN4?_+Gadr_k-t&}(=q0}TLeWuJ4w-@?ySk+jP}#3qUyN1h zMZ6!3Qtow?`mAVLWug*gIBP&|$e3F$K07EaC(%=SP(k_ZVt>bMq;;Uyv4zj2=+r;Z^NWMx^SSHF`Si zF(Id>%YyA0svP%%Ctx6F|5UDT5Vf4&$kT!Mg>>HLim~AX#|3Ol|BJ`Ays)?@*;YX` zW3KZgV(I0&DBDfq8n033C}m?AD%>`ngz*QO)h8tN`*1Ivvu+8ndlcW|kLAMcj53d9c+ibSPCJQ6%O$jg5aQ=NisSY*r>KB-Di zNGZoaehPz`cIlM7;EV`0b@morx_6U7)}J>DAy2~WX;{OE&mYn65cfasyv}S)yHg{u zeoEYfn$bVC#N~bbLQ(H@bkS^yAhpGhWcd=)W2n&?b-U@J>J?vPy8P*8g4953EUlf^ z$MwX8u+#`q%~jj))%!ISfG*9@o=~~E{YH=3L^;;~XH&-JVkw%i8`-${y;=^WNJ*px z&6m}qb&el04zap&Ew5NPd$LaRN%C#38)kOY)X2Q!&=_T#7Nff67zjbjKSX+$f%yk) zX?#YZ6;jL^8f4+stGFDS%D*e81Tj&25_PaZm)b+dOI!iCU3&D5oUBozlW{@wlHV8m zW%D~mh0B!`(ExJy^h>?FoZ}k^vI1Ct^2*B1e!POfa?Cn6zu#YwEpq|yHxb$_BKd&( zc5tP4NEN$mX)pC|>YPu*0sPpXgi>);$af*hnFjta{ClduhD4#*r~iM%BmZC;e+d40 zJymM6GnzZ!=t8sD=rWj4ib?Ai-qPz-&SKd>5lfxf@)K@uMA@a@A^nvK>q**Y@HQgH znep<13}xGdFSS+iBpd>Dh;}bbHpR_&LDyR2iez_PRK*5uWr%>}LhhC^5X&6ts2=1T>VA4&?lVX(g=_3VQPI|>ccy}^&hM{TJSu9J zh+hBne9Sv9PuO{sxiRGYK_N~vk*{nL-0`|6rq(a~F#j-gx zia#i#AuKfgKU040J{(K2FWoGO4v6H?1&f4QXFL^9nUZ+)Elv7wdGT8dpF(~1ZBKV< zRYP*cR|Tlq7$D8=XKKfUJehdP#214d-k8$!z;_gQ!6-bIIj0r)g7X_c+AG#Ozd9A>iZR)!n>jqYX8fTovm1ZL8vRB zD|Mh@LshyKrL~u>m(hDlq7Z$#zQwGTku>zJ;EGIH1c#n>zAvw?wGm%I2gz4x-%oI$ zZBOWus`7i0vw*nigVmWN^p+V4AW$(eAKr`#!L$xm*Q~M@UZ=n5CVV?6Ueb{<{$!Xc zdjCy}F{cZ8xX2)hz{po+gxUNb_0ME!iv=$9r_b#KSc>d~V3VMw4<8OvCvbLiZjKrQ zw=%3Ag&D00Y@7`jdyxdk-%j>soFbV#gMya3{pXF#?;`R#Ip{*0{i{2@a9JmIzCHuV znvC;_SX+4)vrCf|5b4g7bxnbvGMbVw2o?uUtK_))b^nT!@4X;DzOZ0Z@-4m!yGZPr z8Cpe|B(Sh8c?|;Ip$8Go1shaJbUIOvTzVS%T_1$r?NeVaUbS4w92)P~+O`XdefH~z zyd<6*RYuyhzTU7wFz^Vo+KgLX%SnnVG6fJw!kZ3*4SqQ5cf{1y_`SlRw?|A}vaMJ( zcLH~6Ixezu0`@*l&YX`~WNMPQ{k5j5#W%&O0vQ7?PE>@M|oD9sd` z@)dTO{>6Qi;rc_?$O+AK9gsJ2joC#42?!W+QU1wwlVS^%#GQ27MmQdZYoynW#w}hDrG=9P~ssM@I0`WvUzO{Oc1a~ z>!@~=>YIqEv%gEMlxU5DY{W<{Te-(sT?*S#LJw1_XWZddtq;cFIdy>0*HhirNmO|x zAHG}X@}-D$sz(`G<&Kb5saH+11|!z;H_k>u8ooj3Ck(vbv~f;FEa(1tR6KuScxwTS zwfU_Fq*P7EV@uX6DPBP1c9C?YQw(vOXG^+6#_^B`q%aejz!x3dSO`JF8$fwO|9=} zJ2D=nrVc|9VljM`)}NXqlc{(PGWuj4W2ejQ)wx^8r1{u!NejkYD9jB9A5AN{a#&mc z&BTAPuOKStyW>={mWM-`XzhAER^kF*eukpcPW+b8sO>t%7`;@Q7JpoxWJUVa<2P#v z*{S~_Hq?{2%x-L0ArpXqRO{TwA#xaw|LDn^_hoqjS`UU!?Tn=kyG(oLlMJM7=ezcs z&{PFd^V^h$#BM?oh$3jk&TZ?jgl9ak%KDDNb(`@?UBpjX z|9#ySr9!Fog|=zT>POz6FQytc0?c+2n)Cdg(U23@U)DgGR-v?g<9D%MthzM9tzljC zUj8^cuDD#gCoT)$0{`^ug|N;v^4Fu=en$|iIYs${#v=u39{$WU+ND615bBoQvyKL> zh^(f@Hln-0?E)7n=!Yrl-G{L2B5Gj#`lQ$?jYTxv?OoOI27)+n1r+zW3kjYRNOsY> z++U6=4GIy=o%mvY8K+2WXw)%$i65|Yq}!X0DP+uSULQBAjT|-ZtQyt6%~s;YEn8An zJ}S?rn8Svx1UGdy6=-jo?1>Ns^wR~qiQNaKKkuyAmtea#?$SSs+2x;>2Uo{^coW0| zRymFidwt6!{d&Ds;?Ofl33KOZrmTtl-z^KNE03h&#@PP{ZBcoXV{Q)q= zx>tDQu^vsM^@Q3kaaLir^K-Bx>gTSE@{*GJ@z{a?H>u~v=Mk15ctHb1vUm4h*?s<; z-OcKlyw&r|;SMuY{jabm7SOeOVpG?O@f^gYi7J#!X zp4b){sS!B7w^;{&fJ}|03Y_wsYJ%xR{HB_=C*AkSA;Cn{5+;tR3es^(a-*2Ty_XRR z#5&5{h(+aSCdzF&t{YeSWsdjI1%`ZSNh0RvnblEeM>7OFUp*Q9T|$&(?o&77C3fv` za$6#$-fYtRbmS{VtB&Mf6Ix8)M9dwbtd79{`ZrjEcY=%wculu?3$xOzV98X!2zFT+=1fZY~Bq%BQ0@ zkB51XL!pA;sp<=i4C`#jQXsL-U7Vm*bG9W0fw!5d^EW;E&@Tmaw;|G#wK2lnVbSP$ z?akce_HfH+eYkzUC~$9m5MCX9LrUs+1T7^wAV>-#DyCR3sk+f;th$*9g##FOAnS(S z#kP4RXLA}8dri1CN27SF>g~D<&QNTo?=HgI;oukvs-V=$fm+HiE^D0MJ28^1sWaHAtNKY%!B1C50{NwKnp%5`SR>vK2m7 z@UGyLi*+s-&PE6b>8slzMGPVHA~0bbpkMZ0z^SDnSEM;y8DdHdWr=5@+EsXlts}1~ z(6Oc5;y>|Qb8%kbe}VenWu4&X0pq`)9}vFQ^;qB1QXq|6C$+Ms^X2%@i*5{(a*ont z7Z5f8bdfHc_0;ZQeq`$GbHrDn&d*20c#FQ-E45FS8T+}f5!vG0<7ao{AER}ku6q~R z)tBn57a3|JL>a0=-Nj9+l3#ChW{3rmd}#kLh8Y_@vQQGmYLq1&AbOn3amg#uAaYYB zC>qQ$mRjJ@&^mX$1y7r_0@BEQF(#k3wv%iR(LY68iHGt8sR)|D&LkS<;Nz~nREha# z`|_y(eLMmB&9kB@d?@@X5bzWUDmr~L{bG;xR0NlY^b|)j9#qP25X50aXKoKi+Cv;( znwvnpmFH@o5naA)H(UpWS*qi4$RA(AFEDTmZ@71GF!SSZ*vWv8F$QOvTgHQN%e226 zd#6N=*5#;RA^2TwW3#LuGP<9wBsvW5L~lBtif<&gL6+D`4=dZ0m{QzWT6=N1$L~JK zD@Ww1MHiA)fn!q&08hfyasp9*oXyMT@a=w)lHlVIZsXTiQnxCVtW-4}1{(H#N@<&O{)G6Tg1> ztb2vD5)|-~W`$czjx1`xJLS6R#{D|Q)q*ivyD4>{^oIaH zm*VCeF!g;xb=OO$z>Z(q@z{<&JnmiyV!IKn-)>kkruZApa7$s~JCfbM>eY2|oYbP& zMn)H%{fN1>ue~$z38wv)x$~+Dn68!lZm&so+(P1DZo{O5PIW=rExs8xwF{)(IW(=h zOcs=kvP&MC&+6Kj)i$}LmbnI5-S98)q-$I2@jM9-8@UPZ`rguD)*gKSpkp#nZ*S(!(7#{ zN#xFqp=ge1lwcK&$rB%)1+q zH62pUubx4vx8^mWRo z8`QLjCuS6eXF;m0xBcRNw_B8~YkU8pV@!w*9{R|iIHceJzVcUL47QY_MVERgdWI6 zRgxXiJ>-fllx~i%dh;kXHjTwM=h;+Qi`!xyu1OaghdGDUn#*)?PmenpdgDr2e(-_U zaX{I2QY2qjfAK#1w&0l0FW7NrI4Rz%VD5MC@41RW!gI$|0B@zKk@UKK)dR?WAlWOl1095V=C-SzY-ru-NPX7Yp8vP1ZiV4{Q&yM!SBS` zJpDc0+`BYOR>dBQJJBBpy}g%0o8f=((`hZ7QL>DF*07fY_#fo@!G_A)d1~I1(!sR; z-$oT$g%j{xa65U-bxj%hnxl#3-Mt_$Cb= zuOylkH}>r;z_4UE`+Nmztj_6kYj0DPX&=mN@fyCx)#^dR05@Zl$=wGq9jfY9W$Dkb z#y8(aQDL$IV3=E%!P0tJoan72ePn{XD9j}WZEem{>vE%muQlN~H^mO<^&8|GczSZ* z^n;4$uRu?`HZ|<=mI4*8876 zh;4t{{D0?eQ!MiTf;GY7Fo>B$cksPe>5^RH?G0SgA-t8gcA*n0VS%OTMgXDtO0_rO1DeLK zSbYWK+I$kJXh#5yDizcX;085r>nw6PZhULr#n+!V_(5czH>V?xt~WU7HEfD2&TfN~ zo=tM#;1>(>S#oQ1s|jopjR1Kx$FGu_QGkfFqBk^>LAixI<8j4vA!!J9iiQUJvAkm_@$!corVZvf1}G2IwN6#l7*cjPLRE_kQ7%XJ=) zEUciIpFde2j%I+F#0fIDrjtu7E>K{P!&jsszA?~u;)hzUa~7~iAnsoOUjXXTUv=7# z;+>uZ#wdz9tWvR;QfTue*JPtIH<9AGw$lz{k$V|AepELLvW_$%Zp$4lbn7xFo_k%(F5DrZQcyq4oDF`ni+1+$!`;^sO% z8k_k+EF2wFJt31K>Su&@T@9$YV4MH=dmhhKBKyH&0_WG=$56FAe)Q3}|0N(-GjUg& zf&RSoSSlzCPn^uUiFJhq=iqje1ZA#LsQUVP<*m>O{m0R zo{icPAxlVH-hab2&Q*IH$Z1@g!HVecE)3LzSO%WqOC46+e60Q?nzgyR`?}tyPfgmg z*x$;7FT7B4)2CzXn~8Y6xK~60U(zLs36(TxB5syPJv8P#pZQ_IlIzLT!if8gGJx#)skJ;G=G$Cxo#i zbCdB|l^3KAA*H~aOrUe;Mkbg*JdFze>O}{nw?M#jR?1w$(JOV+8D2DYk=1Hv5lOOU z59tyE@x+xMm^Ri9eh_UMwV9}e~A8RNl45QF{6miPSj z*0hIH=P$DVn8Vy+-ZC9#9X^<0CzVUPzq|D61Z9i`)?K^! zczq4CpMSMc2s{Tsnw59VrOfg|?)g8JyFJYcyde1xR_%W(c&~&E&VLYYS2g%sTU#1N zM(?+N{Tk-U4`P3~WiiQ|S9CM|8e2blWPPzqbdro=$VQKxqg%p)3!R}Lf)JLy&2949 zT*~%uQ-j&qQ5|7*sFk+G{VZQnR4>_7#4r-^4tx)d|2Z8VH zZw<%c>6blepFbK@6Jy_8i?@5a1)$i3H+P$RcY42@pxZ*kV0=_Kl;O1+A+mS#+q@qE zh7AgEH>Z*wZu@Gx_Hkj#8t_oHM!Y1TESl*QRB}ntsLH%k3;LSnyBktm2}*hO=nj?^ zf~xtFjf>7$zSJ6TGr5`X)p!c?>!HIOq?BlW) zDsxFua*a12Z<0rKF+o*GpimU)`a@<@kcMX2HY{%(+WnMb!VjUbXnlO_+J>IYD*-O{ zQ!@*n3C0P!DP!8||Ea%1?J}J5fK-*`T~U(m&rM$ByJ8u@*P*m|Zb5ReAr z&vEB?!vQ7Q6ws*i{P@jM`m0ix*g5r}jVLLDQ2HC9eR(opw6%o-tn^?iN)*5pHJ=)W@3|X*FIG0C^4XCbdtw#Zi$w3&D>czqZ6JE z9!f#q#>hJ3VU8L0UJ^o-k`dc0~ z-mNYaz^9#0?JK=hN-^?zfb*+^fQ_H+AzL6uljV1G{gC(1W`v@No$n<_`>%j771~pv zA{Z88oA)MH5MAPu%KqpN`hg`g#2)95_2srTkvi<_d|J$I$8~(`F;kdb?}Sw-waKc& zBf27<4gD*%=s=4A)LJ_zG~lJA==CNt{^Ud>9fkObSjbbg^&&E zo`96F)V0=`6C6kr7~?r(N*1JNO|5sVK_2~z8c%x7*pZPCcA2`EKPub){jS%{VK~0& z7lNlrGdNfU7JS3s+OKdf}KW;_BK zwSK-<;W45!w$|72ZXfr2k#v4xn1PaU&AU;?i3-{PD_9_7;mK?P(5 zLQ9M|q$S@S&sq|JNV${HcfXj`6vIY&2X4e~Bf|!FC8a=5VcLIk={I9co7Z`G)NRoq zp%-Pu%C03CAlGa?ecJN~GG|vx_t{kxZ$w)Hn&FHtb7<@+9B>1$$9=AzQ>YMgLrUi3 zudtB;%Cl}@nRqupL<|H~_YmDi<&~h?+de!~mu?`W<%)U~tYu?#p|rt)7Iulp)Uk@| z?oYeqh|t3kC_lj9;(b5z5UfqJ zgYWg{aC6`oB(w897y+aU)<)d1{G=Lv?Pne*+-Er8{Dlm(ppHXP~mf2 zra})3Efanqg*fmnW_dy*1u6OS`cQ#%v{El|_XkW5ZS}`t?kmatBUzAT`AYfc?MmuH z>#;73rOqJrBajBs2a#jMYnmg!q#HH0qv$DYt7G&NB z1;LP+{SDnt`Ibw!@3ORS`y)^1p(9yW8N@ceLr~3>0*{RNY+LCEqy({yIv^3YD_LW9 zA%)4~xnG5@o1&^0bWBK3CLCfhtn|b9h@lmq?q=2rx;uYK--o*IB~2$>EAgF0_Qo;n zugUxnG?d)??Y{ku-&dBGh=}^DsvzK>;0*XzL#~VYpAC62D9jaA{@U4j<8R051%2Eb zGe!01v{JZD;axY(KDI3!p)|S444IZ$1CT6?<>&J`+#W+K`3mQB3MQYbGmb6+?D59q z>+9Ve4ie)NV^ow10wO5hlvX{6$e@TN8Sx{<2GlA4Lyb4l5r4X7Fz=VsbB1LCE0Go5;U86AOz{G9Y$D4LiR zsSqIgVegH%5#Md3HaF@$K#>!^I}rGHw-t}1j5f#obsw!`nQqyHfh3v4@x)F{o~ZPW za>rxK3Eshpop_fb5xeN7xG(8@+Y(Dqhq0rDju+omCu5*#i#6fJTUDW+6#&a3mmiu8 z6$oWl!yAjmx>>8`SokFdCB%^`TnSBvuRTuJwCgE-+&ll{Nd9h;2d_#DWFWIm`vxbS zvq1O$)Af(tzL^xTHH)YkwlSgf%fHM=QxL8J3A;1}W^tS*_S`T)My?S(2`-IxRjz=7 z=GdGiuNNyf=|u2KmCSDKw7h^x$zi#@q|0c$Cov(+_ZO%`ctU7hs zGSZjVSP3n|dbGALxGgHT#x zrH*P;rYcx=E`xxQT+-dZ@zuS1e~o|aQM8-#84<`yCJ)>fLe?Ad8yf&iCx`|#YZ zIrn&AegU1XaFX2bO1)_L&d+mkB}9`NI%?Y?6}ok%v8_6Nmt<>vonXL`j^svDkb4i- zUI9yTKGxfn5y$l{SE&7Ws79B!k5Rv%BwtZFj4;deQ4W2O3{AJ zuFS*iG&}#Zj%J{cHpeHd-Gk)Spv(m+9^I92@Zq{OC7R-T)r}?T9mM|VVL)*HqZ&0H zJgD0|c!l8M9aAuBJE~aOppmCuIVV4VWbbGHX;t1~kJIlDnQSwX+P?L^jv;{Yn-`y{HoNz8d>6HHjR_KF~XGGSI)Q=U&IL1);rF` zfv;6eC?WdOB44)p zsE#N-F0I3At9}x1S>=+7QR-laqx{-1;vvb=3pndR)cv%Z9YSchYVK_`5p3F=X#Hg3 zGg7=G5NjER0wla?d70B$Tnz5gcS1H@XV=&qz85|D!CA)4`XMb!Ysk-1lp+Geo(M; zG}DXpgQtoKM`8wuAIA3yw0RZ#Vh)lNP#%STOWKoIZVLyHoob4Uqh-QAK8Pp&NZpU^ z-ADA19hZQ4luocgyww!V*M<$71z*>-NBKIv9$Wp&x*%|$8_S+ISkdbzAb%&%fv%GI zgT^HZ*=3u3wDP6PkO#uE%x}}IPM*6h&O~j_-;y1P-T*N-+?}0u)+$Sr*@)f&gRv9w630GEi zTD9ctajPos+&dDp2r*Cg%U{29DBxB|o*%!ky;BCVvqjPt3d172GP6qTcj+trtt`6X zhhjLPO$~;a(0UHtIm^lVvSuHli=ABGyef+}V|)UFw3RpimlJd{KBM=v!O5V)y#ce| z8t~87?kR1G+8|IunE~j}{A-{@(m~!8qtzzfEjQI@W29Z1(=N)b+UFe^nAAAx&L~Ws zi|;%rjv+Z9lV@EnWZ!h}nc{uC-n^#t}5*B@t!J0Dej$`lL=pB!I zBOe18!9$d1g_Q{mzatUkETggjCf7-lo5nwF_w3BJTlkIN`FnODccdfOJKAU)&0tm7 zQ=2Kemzm5QtpTKVvG@ECVNeAtISG@?JZ0>&k}I|c-HgU8LjQFF=C82|U5~O1%QDlR z&+G%i<;&`DCs`wzckPM1AmSq~jbRN>(Ubf2Fy)%*s7l(8gT?+0$P3wSKYbFp*Qkm8 zg-T)$B#ih*+y=9d&89zlDTl<4$u9ak%IDIW%|lRB$; z%m5wos;}sVmPymmrk{YFiE437z3f@*s6}@@Qv&2#&&1CKF+b~qH;76UR-BCWp%Sdn zrVxf%G$x)P!3ARpUcl0rjg=PG&@jvp_Onf_|?DbiuN0dcWbwHguC0Ry+| zd2}@|8)N<^!Hs@P{p`ONQj~8lSLUr4!_)||_e46YOYKk;oAH=G5)LCq9ux-h9fuXW z?VsoLtwdF(<^)W>;8xOW7@hK1{v)@+p6UhP<-&A*)UJASpq5Y5!GSycOr$^CYI+OX=sFlDu7k6u`Qvv5?fskEiA8cvB z^twz$0Sv2H8_#|!_QV2X#}#lSd(_>0L+tirL!FL6lQoI=8D!s9hhv-k6g!_kR1N4d z6%c*fr#IPUM~VKqK`do?TVc`?fMb2;HYG~DJy>wJv(xB)GxtX1GKvF~=ZJ zL4(d_Q%H6%G42rHy$zr08KLb*NnA*AK`xnD6HhYtfph4z0RcR*Q0K1nShcd7?cM@OlYXiY)f9Y z%0B6yYi_P6OJk4K6hUZyb<*z-Hg;#Qu`#$+*Vb)21{Ps}EANHLKMcVVm^Wo@l* z-8Y(w;)tihl#S{WOC-S@AIY-5-gTZ9mO8-1%W5K5p6r=jR@BEX02WX)g?&Sv?N?Cv z-g1#2_(LM{N;!Qi;^!*s5<|*%L9jFFJIB|u&ZtQYz+($l9M&Qkyj70^N7ox2^MsX> z78nz`6Go!Ud53p=Idm&bfolLGCBmr?IOQ}0%hD*gBbWo0)cN2?ByBl2~?$x`dO zodSErcIm5mIQ|S7$rCaWiZ@b8c{!dKn}R^CUCKze7ZjU{Ef*>vqcEpxHW&x`iZ8z} zJE&un?DU1SUt7iQSHswkah*|?eR`<{7aXnjQi1_yRL;F0)h_3Jls&tL%5f+Z1cQ~w zc{F3426!z;NyqIsZ=T0nSEQ zRXK!?;X`>WDU?(r#=hnT`s>cO_<@_Y@!MNbhoVni{bHrV^21;z~Rn`6W$bcWf< ze^jtwIsnjnDPigzRVE%(1Vv|mo*0l_W^&;|@lxaHv&{;?1-){fM<9@!t1S;nxjT?W z?PlPJ&dv(8clt-?61u~gv!G}p)x*^ZToG8)dV@_6tNirTlR67u&&b7GSd!b;#(mv_ z!nPh+SZ@b*lGW55c>#;dVL)rvQi`J}Ik~R1qfo#Xif{XnT#+TS{4=lxJ4JOw=JnGl|vsM~&)EsL<^3hH4X_I;!z-QNkzf?F` z0EUj4ZV?=vLW$KS+GL~e4DgNWxhbxCR7>vEcU!}){VRN*R;u z3R}8th{F_Y8h$E0aZ>+;z}Y-xHu0sgRy7fq?2*!2TjG21QJ>Ex{=RCR!1)*9{U=P6 zuX3#MEF?nL6>Vq7EWmmH5N0K5(T|q(5m|3I&v#;(D{t7ywQj~iP6AnN^J2+--q(n1 zP!N}n)2XOau~V(%ilF&hW2O1LSIS;hdex|$&zMdcVcLvSyu~S4)MOzGCxbk}*4d}@ zq|X=;4*#BFy-pm>Zxd9It(uQym1!GM45GsBC*xArnQRyxnFbFdre|Omh5H_Rgf76oY>XA8^#h7D?CpotE*a@KvJFx~D*$h0s3+q- zT36(^)(eH=W{j}|`u^0Z772ha{9j9Q zTGaF>YmYDUx~)tM*M3B+B;_HmGEurol)tw{2b64*x(#;E#Nr8XXL$7Hlje;}6sgwI zy?j58N>))ox;8o@%OTQt#?D<94-%WkkW)1>Btm)r$O8YUaTmXnVGTw^(kiT-N9rqL z{-OaUk1@Y_H~Wl-m|aP9kV53zPcLRM_HFZt1D;l%zIt{zQuZQ?Iy`)3;%V16ZuLDE6CZ8%9mJBL&Ub z(HW=3a2;gsi5#4E{pj=n-KJ7f{Iu;EyVd-I{(z1lG;U{HJo= z{kI-O{q;^Txii@3zxl9R8mw{c(_)r8jfYIZlEy|O5s~SRZJL$R>=Tc$l58g-0Jzy` z4Ipx7aBIvB(z7^Si`2UjL5m!=mYG7TIX9El7;VLFbk9vGc(9yX3Z97wAO zh*%Mnz1H!j)nM6TB%{@739JF%GSj4u9r_|vnsQWYExbvvE+5qi@3n{6{l3AMHpHpN zdBPj~a*6IcN<&MQv=IeBY#L(x zJ#n$35(B$t1BpKF&#RBE-bXp8cglJu3!?a{*A;Y%-q=34=u5 zX6%oAe`kHzgzvf}CM`_Yf!AaJf&k~Y?tbN0S-{#3H|e#fBj*G|Q=oZajUMW~Hqn1D z3K*=JF+P0UseDFYN7L&{u^gvcU+`J+W}L9Sq_Pgmv}^_tN_@54Xx_t-Q^`G=HWwS{ zap=n)Qd2$kg>5=bU>M;EVz~fNFYA5oYx!}EY$&d)^jf-Q~+Rav5U3}3n zksD-(Wsp;%TeC->gF$4&yEL137n#jzYkAq-J4ANM>)RN#+dI&FSaeY~6dzcbU?zY-^oG^7Gsx;5Cik5=kknNC)s|?rdT6;sLzO50r@li+2E%D_2XP1xbD*V+YCnw%x znULg3T?*w5D#5q_*kK&zieb;;Kuo}JmbJw8&Th<~`BQ)0hf_ijRG-lWOCyhLaa~f5h-U8{hczNP?pyw z&z+j3!_@Accl7P1cMde)GEf-{G*}G-{k6y81H62Y1V%A<**@I_VoZOuO6qXRcltK z6~yjG?JxXz9g^3P?GT0ueVl^ zd0wTFpSQLVqDQ1qPZCE-cIipnRkfT#Xa;!EXLm_*UrJiII`&HUtZSq-`?%74)g$ga zJL}2?KcAs%8ou;}Z86yR?nASk042}QW;Nj0(F#CTd%6X!qV@{J8guB>EPprRg=gBw ztmh-MVIhyoC>g7Cp{v$}T{0D3&b;7$P(X+vDC(M~OZ-LifyXIV5dd~+sWPE~JMK_H ziAGJ^pKAjQybnAE9naGpyGv-_&mx;f4@#{~xMeiNC|t$8|q7gJnjP|Unetu$@B=;1w0YVAZrFKMzc z^+mtU8?g8E3G*_Qhg8opkawUnFZoM40jc$sCcUWXDFX9BQs|43!N%A#_R{x+ne;cU zkLU3VKFe2--0VVE}tQ|>*0;~z};R}uCns0&( z%GAdUNM5Ic|BxGd_`aayo&KX^|A!sZw5BL}gI7)9t9PdcVX&Sys?*Nh;;w9Q_+4s` zP6&VzRFCOLk+f`yTu4f2twFgepK&Z%6_1G@$9{xmjYDH_!LE>|f@gd5Ztr+}w2s@u zOubp+o5og)nzb&5sT7K8A75%vG&Uvo7AXyuV|?5W#tYcP3&HgmcZ?~AxMpC0ddTp0 z?7-@@!{sP>pU;1ut7hIZIzrMs8M zm8-M39c&8e^8??dI^&qEf~q?h`UPM1gvfL=H-C?~4u#7cm|Q>gSJ(=&7jwQVlfGQ1 zBxKThKoDZIFK~bMEGZ2B`M0M&s2$GHp^zvmQ1QC1P*Nf`_BE?O7pBG+`Gr8 z;MfJumwf^6o#RSgd9>C_SuQK98hsUYZP-wM4R2H!3G-0ioj8bKZlL=sAjhasi0Q-t z!D66@83`>$?nTK`{-|!<732jM-9f6r!F6W zPJ(UA5KDMhEuN?Ft=8#9s0ZJ9U^8WUyvysHZO45#rsVbBRLdGdv-fD1W3wi%qq|?E zY8P_XD!cs)dZsOyhC9$2r(cu!imty>inr#rE@NsFF{t=vJdQyN_gk453iI02kmY@d>N@i zNOZdd02Mu7kIN+QT{SinU?B6Uz(_=n?7TC=u3mFbW)bShrQJvR*hgS05=iMYGog=W zh|{Tg>A%3M6hP}8hHZ-CiM>%r?8){z1-Qtu{>W~TJrU)CnMGuYo_Za>)tebHoI?u1 zqAMLOY1TZJwEqfFzvEwnEh^>?kNXUz&^o%@y@*k*@Q7~!4R5^vtwRm}P(stLk{jC5g^<54gEznq_=cI@oue?hRG zGZy}74vLM?@eR$Kr~wh3{nVnjDkk7(x*-x0jdtOZxwmb*d*90w$LGAnUph>mX;$Sa|uHCqz|IPm-VJv+SFoLZkiycGL>BVMX44K36NBu zX5rF%y5ogh!KLyoGqDtuV&4u1kq+cLx*z*zGmRkpYX!%WS3ki|4a11@RHmitfs0ch z*=&X8He2T@bMCTtGz*tm=`j7^8G>hu4XYl$`)f_~q=bxT)avto?WdY z8M5ZASi*c#$bE+O#EJ89{3h&LNhX^p+})sx1nxA*7BIFQyiayU;pcJf^~gpHH6Ys4 zy$izXW#pSin`TRk@l6Gr(}a;-tU}pYOnQ zT4~oFVY8oS!oCclDI!17Jt6pW`aJ|$b1~|^3)ef|Fsov7(9w=G3qZmN zjwQJ#f~HXDse^Q#qXfns4pIbOcFgBPd{~MI#@+&QxL;^v2Q}1>VtoCoSSV|Y%QTB3 zu@bx^{?!E|_39ubn1=f0<@naMCFrXWJ}ihO?6~XHnJ;dAXIxiw zXR|`MW~LdgT;d&4#@D`O2GPY*T2%1w<(S+2;-iHQ3hycLlm%aFgg(EJH=H|Hgl(Py zEbA5vk6ttsp@{C>i?I;EyY3Vuj5ldQ@t~>aRC%cr^W@=Z^^4~()<8=YyW*&T4#xA^ zN_n<}Z@(#7mwG%4NTo*X*J$InBJ7VCG+*@a5R+>(sAij~O5=E^l4apZ4J&Z94eyAc zDDF9DIXT=7p&LRKa<4O6%JtLA6KTW&}qy=z6!!Hmj7@YqmZy7taHlV1?4 z4JHJpD;ZH~vyL<&7PZfBcz!o`-R`GH{|2Wn(M{)815{Rhr08{~4E53nNsa1Xy}o>u z0LxGeYZXv$30kwyM`2iY(VdDP+C1xH7%{%D5D64m=eA1AzNTB#XcxNiLn*}7Ao_`& z<#79uX2nW$wCe_6bt84TTic)=zQ{`5iFIEF0Rqpj|AbBjzJXd0h)Pl6 zlH`yDc-5j|ZSU<;tDM>UY??C1dZfLWVl=DOvl~y+%GzD1Y`bEG zN|7BOKx{+PE{$xuIy1uDxL8FzZcH7;qt%4e77b7wui1lAiz>{DqeAUb)>9?H3+14bv$^=Hpv^bGm+q>&H;@ z(V10xAzL!dpTfpSen&HrS&X89OFdHKU7pj#%&k|TFbwhwbYKZvdwJ?iL)-4BtomxT zp8v7X{$2aSWqf(tExVWBtn{1ziAMOZ208biW|{qeHp?Q9*Xuv>^Of2jZ|$9q?4Oe8 zVFz;U_&(9Yf^_OZ?khDEQT5C6dzU}G(}%$bMwTXhly~l2tPQB7QNvM`5;Ap0VZlJe zpvKhDbST*gr_}^K5u&G}6UcPN3a;#H3(nOjVo=y!=B98ifr#P6-+7>M$ zN!Iq74O#~<4;^Or{mxy(*iK~I4>{cjTM1m_w6;McewyqOK>955>69GBUwkRc(+SAx zjs?avA8%oLP*#&mYVEuKqiSLi=5USOk-LE%kv*>@F?4E_ku?NPR8SvHbBRke#^%&z*QF!3tnA{TR;baE;YRM-(xG z+l(?sLgv&3`pPq{dObiOjeLXnMfrB~Z} z^u))6;P{$OQ>{t*bklHYTF=-98=mQXkC_QFvhzASQwD1j;8{2_awW}zg!_jFkNkM6 zU@2(FGxo@Tz7gGOBu+Z-ZvU+y83YXgG_6abvf2g9>#shA;qD6bxgLnJzz6sUK!QLG zT;32$y&mH#S7Mo|f#;(LPokHAB7%n(W~QTp#v_=2(coCM#}kL)ZiD?Z$sWCZXvbZ_ zy?aOaHfo|*$3ONHC6jjL-Bf;OymxqsLcSfJn4N`lhI8fAG;rNKiXoYDC!K#qO%F?G zQR#Gf`~}ruat9uN%|pw&XqFz^>|2r#Hbmhb5s9cKBS!P*qxZC`aU%UKHB6}wJ}!W; z9u14vGQbIJKYyh5x#R2mdVJt=bkG#F!r;{Ga$oU%_4dVv@Ua;~Y)!K)x;=HpDCEz@ zFR4bhXJ6M;vIDdq@)6L>A=^HhKFYC#Rcb)Xyjuabsi&;ElTVW_IhQdrhhf>bc+JQs zlLXH15ApvRzy5!Rv`^a0kY)R{FVG>?{-4vw5Y#?;pOSxE=XfotFicSX?Q>IR7Y?uZvKU&$ zg>VcRHj`2_E0!W|C@~NZPwe+Xd_?ZSpMhw7G5WzR5n3YR^H1Hz>&Ht>FHvf676n?! zX=TEtL-m~R9S}9y&A0^IO@WjGm=)UGvZpp{memw|DLCAMU7>JdgmlgTEM31+_%NTJ zFZDn`zQ%f3XALgGHJ&>BH3w_ww^dABY7sY6gRW9q844azV7mH_s#wg`+Obj2X)H-u z(hbR zgvE=9>Q%7P`RN(&9Gg`dnCZsf$KGM3W!QY#u}RM&J5TSnPl?hCm+9!+Rl}C?QdJ1Z z>2@9k!pPs18+c2ZnH`{~dfT+HmU!jLC09~ZcC*`E%LO;JlaP=4yhJbx%o){xg-!=DH!mPu^hvp%eeP52u^QgWt=vn4 zn~-bl>n}X`B8W|_KF(W!n&E^OVvQq4(jT^?jrMRftQ65%3I*kbSJ3!SEaA~pk8pcD@E}l`%~5gh95BlV^d!EX zz1>ve&bdc*FhUbVZwrn*F`jkdY52?+1Lgd29wW|z;>ORqrsNWZ3>URQy{}y$Z;unvX|bo=m&znnTrRWs2`a26Xd#&NM2(-V zX>9dC6mU)%c-K4X=J45eqL-^uK9}rv0BpH?`lgv#whQ`2x}zQrs>+|IVe98!`>V9# zVNa!T?w_e!!yu|6*2|9>54PrA`I2|n)a|4mYeKk()KUdB9J5ijeWG{3LSEF#V_oI7 z@Z~*8Kp~R>AmG8nJQvg`r*e2zqujSfH5=+HR4k&q(!1vwOFRnAkd6!UL2v^x8 z61Ex*@!WIY&1$hSky$Z+4ZOQl{G?!dPD5}* z#i&mN6=2t9Jbfc!LmTdqy(oZh0 zSP31N@csyb-x(2^q}4FQ-#11VVI=GETTbl$`rzG2c>MPQq&$7Yi7^ihoev|>3$zv^ zwqGLmxErP67wcT{= zIsqX_?TPUDd{zTVS2Cu~g6$af0K`Efk++j4Z|+S3;jKOWrk6jd?C>1V(~j+=8L}1v z{myNLj$PAU?C2<)+-i&2@D)&MI5a(YKJ6%THYi&84rfcWTV&}}(_0n+K12W1QUG1n zmcs;wMdZzWPZfjfe(K@lsq6lJ{8ym@{$m~&BK_n)$^Qb7b+o@;x!KN4qp@e)qfW=UZGh zCD-DQ?3&AC2L&J*1w+aO*KntUk_{z3;ao4S4l3l3XD&(+!h$}=M;`C(pNRt@q{=bO z-H=c>XtVrgHagV51_kIek$B>dl8)`eVwy~vsgy{ywk4*Djgc-f@e{+danl&WV%}*k z(Z2Q$vI3WFVX#C^vk50ee;Ley?&pGI(`A2T#3bucD%aqW$6-D#CDntK|5jD|CenR# z%FkiJuA370`$;x(xO$qdC?dWqNu=EpXBESmQZxgm<*S-@uZOc%fkTL`{JinP7~&d3 z&I${XS6@OJYL;1aHsk=c7Ig@Keek{HaqIc3&#E@umJ-&UTmfhLYcLUkNQHaDSdy`v zq`QRVU%{5xK(+-5ULUw!4wm=lUuT{^<9s zh}nsOm%XetS_cBUGQ`cBTBUm=UFTOyj&^ae z63X%lCul<>RLBLhj)J70CN;QYzrjLFkn!PHInX2x5{0vSF407GI*;oeSxQe zvLs?q+$&3H>XF;!=6)|(Wv#VMw8wra_!kfIO8t_xx(O6 z%&1q)6^#tZ+&N0R1}SG?TKgt71Yv=&)sP@6{#91xcNA|lh8wq4N*$8QtL#}GXl9nM z_Z;Nu7h(^0eK*Q?E1-al)}z%TcBInXimGea-h0Kk7=FzUko}X3JFGzn@}UKr$=V$z zdDIP$B~on{n|<--3RD*v#lFQXmQT@ceiRTRPDj!Sc|2Fr8y{FCZP#YOeq5EM;BaxP zx{s*X@V1d9gVq2vEiWx`^1T)YkDn4}2d!X!lc~AJCGX8fToJsl*N$lTe zk3Ue)wua&jE&TK~r5~VgDlCJXT&7!CWwk(0G@2w`BnlA6^pPET+t~`s_O*sS7tPW#YSYJKNz>=ljy@ z`5u!qXtRc=ExRY8QG6lcMDm8jx5=Z~=wN&^QI~?+Lw*$InFYQ29(S&pOs$qL!{iR` z>y0ZfRrg$N0*u7zWGd;v0#3uJB8kKNd%OqC(TiwrQ3u%rGPH3jW6?#dGO46Uy84V# zFNJE4m`dzO0t1xij+1OVctRTH^3U5Ly4?0WW3aY8% z8U~~bPOQ_nX0n<3Z zT?F{(>rR<?*4BrKnFF>S zBb0Hbxs%M>`^&Oz+8?Q|2cM}!A|L~zv5@dk)xca&iW&-?WrgSD-=w(wr|g%b9`6-a z>OPHsi4?!R!mV*%F84DxdKMu=XIbu*t5Om_Lf%P*TJFIFE@}76Mwn1XU&BU8wJ9L6 z;W+r2bTTv};+6qO{i4XbL-fyi?J$Y^(Bo7_IX!U)?UIb-$tJfE_vGepXVyA&vR+DZ&L0@nqn zPD�I4rVl2*fIL%TLvC_MT*wDVE@^idFlH^*-0PdIF+ufZF6 zAMY!LBOF`gXnjRd%{*5{rdrAarH=b_&Y5LttbN1iM|eJI7i?*Sri+|tu24|&B2vp4 zVQSQOb1^;3eu*L=z_6AM1^-g2qYq_ngy+gh_^KLKV5kkxSChmgwtuLg$Tbj})YQ~@ zNR;?1u|NO&qx%@?B&Vj6{44c*uZ97ae)QZuI{Gesze@ONE7_Kx9IuSL;ED5a%}U)o zy+?Q?+IkdaiHMaA9*(v`g5dmA{lsJ(C!!f$=#%44FTXx8^@%Gi>#3E#gfZ(C}A5Sw1f0Z0T!5sYFyXJ5^_jJANR zOV5RdRlo24F*k#LWw>rg9S>fEM)%QFFvw!%J`(kb+_o?m5NQ(H>S_PYYqF8E9a_SN zK5q;OP_-j!YF??4b9k{})1ayS)OcB@Ukb$Pz$km!f!cxDLEQ8HFVmto98k?HW05eg zY_oGuAK}<)01_U{eg~(tro&-Q{Mb&vhGN6D7pmrRi(=H#xj`V!N3TqMJ?Hwxs2jaL zuH7aMeeq(hcJoqa+wfG?Y0uSn)GFKuoq0z(tHVLxRfOgFW8?thlz6z_~ zbGi0X-P=HvKZJ4xV8sTEXF`*-nzbh~Z#mBM zNG756y9=4+i?KUzdTzuxaCC82SBbYI9p6i5i~Kwgg^6*5m>(Jqr1V@FtO0nHE22af z%?3QE+D}FUjK`0GISFOt&Re=oP=M`2)ppM>YKGo+~4{^4ZFNy_v7v0D5K)&7i_RhOb(Vt`B=G-Q*^9LtJ6w%bfRnFE)LC*hM^dU z_$~tlb#MA3>jILTgxk@IaTO}Qb-%Mz$;RU|L((v+gYaxl-D%P_oOs#wqNjlI6ThAD z$Q@3)*lZ7BN_`4;7vJgG_x9)GE#~5R$p6`*&+PNeHihOCqU!p6Zb-p$PFWG6d}qme zWYiPw=OEP|+b^@dHH?(m1E-X~!mfo((aJzwn1S#X2DbY-C>i*IB$F;PER}b* zG92<<=ZfYzv9;istk4!xi5aB|D{O=DLD|RI^dC91LC8eymg=;EoX|NK@TGzkH$Bjb z#~@R7;U!@cO^!KNK%aHf31#N)hjUi(HEJmmXH2opPn6X!xlE{1!$qm+(FK~1rtbu` zqf$EFVOy$d7dU~Gj;)SXZC#*NQU|8_XJ}cRJNlzv?3AFgou^Y~T6Q=cG~Xn+gFM#= z#B@Q7tM5aLu{5*!0wT*L1KTvJar7mi$Ey*57sbmgMx^)tOEI&@VQwMeQayKotno#I zo0Q=RNMfY;hV7~CXFYsKy1AH;jI`CLKI75`4R%{j?lYn-ARLV@+T|_d99di{z6G`N z8ncvi7V(VUR%juGTJGb@ikMcK1j$8MB|==5R%|UNgHNRI69EO*s9;(_;8&N-92+8g zG3dqW<0>pTS8$uR&6Tn-lA!mbI(;(~21i;#y)7VK;n4CVR+K8gPd}3d{km@yC>aoP zIX1+VX=s;IaA^{)NL!qAU7c@ih`KGmvi(9myhT{F#Y=TXi-l2(Dlp+m0>Sc$8Ro;7 z0C=AOGOj`KWBEm;BRE0?KJ14wx@0iSQHHh?WWN@Lj=X+HS6c#~R)Hn~aVnWXYe+ec zK@Y9KSD~m&2WK?m>I8kT9Te|Rc)VoUXMJcvbB*T5YAKa>ZV0yAlR50?F#7@N#M#H; zRuD67w#Q+-@e&&<+oBA6@Hf@^tajr86A`jc+lTZq9X2W)AfcH-!-^tadbbf+qg(lW z-rsZ@G$TbG3fLvW)~PJ6RYtbk<;Q|ND40C%MquZCiIjQO>e&qu)f;1zh#$AaJJpSk>;4)xn92SBQPOz@1p`)LbSz=CpTarKTavVc{ERQ`smjUc zCBiB_VY1x_-lp;ui(P3+s={n6tkjH<0Oq5@P&;sg<4RqmTR5JCOx`8vFjua`C^Ky* z0~H#OAkA?!`T3N9A0?_wdgZvA|o6XleTk>QkDTWOI(ggq-HLeZ)?bXUTg~(~nHZS*y01 zqvx=W>MRe=)2w3b4xAAJZAy35!361D!06+EJJD; zAV+c0M;DAnd9Bg*{h}A9vFD6PJF2*h!C~~O4vEjP=QQ<5N9c92Ou?XU2b-EuAPVU6Y*W=h*3 zIzs;w2}{uY{K7@=GV=v;G?VgGpUpz|L1XzNZDEl5{X5rkS>%AtpSuYm1Voi# z_@;LsR75yj|95-xGZKiY6P1<=du&PJZXHbsz3I51gcc6~uPoXPf{W@EFb82wfEzF9 z-l`N8*E3#^{0s_jXQcTp{Z}2;8kPt?e2>8|a&q*?tsdQM%GZNvAq5nAe)J6^v~`kD z`{nfSeKh|wF&9ffq%BO>(6q!1IV(IP$`65O;7eM(h#?ua|Mn9B`FjTQ_I;C<)=l+p z7cQc2Eq*-i)cX^RG^*mhx%g>jWRjjybC|M^n=>y>wksV=$^9^*p>s%UhiaASR=`9Z zO#^79&SPp1iIHoSn4D%~*l^u}5h<=OXK0tZbBE!5{a-w$TlB3>%7M0nifaUt=ItVY zE3?tubkAg)DY^)u^wuY7<=1NinAdBlY1KO)LgZa1HW8;K1h1A@y_fcuD4a9N#;Di5 zIgN7cgO3Y4sUMo`8_^ft_LR^iqHFih&?K(tkbF;1XOz+PWyTh8m(wURgM`y{>OfnB z{?T)r=|}`1NKEY9u2bgAX?EmuK=!5y`g1Tt(u7|?VDr0)Q``V|GiOu4|HdLgvU{c3 z{&|4^8#{C#k2*Cq#gAN9cr)be4?@zzs?yx&KD=rViT=<)+mbw>{&p?C>DT@{C7R79 zlDRUPjSnHcY6oxRNlh+Hm^5WvfRp~rLF=RMzL84)8rt>gN#|u<`bl=()VNfq2cEgp z7>i!`ZOCkQZ*1zo334jURRGOa<)27ZSRE!xacLUUgSP++X%dT^eL%M=Mta8+t<9f( z-v@TFNF(yzMq4@b8azBTP<*jr#uZ`=%M$et+iG+?zDC*)kq={ zZSkZFBzH*aPDV9U+cKK^`IhYNJIY)|oR%SZ&aC&PZuRKrVEmou)7fJYjI$j$-t)|L ztJY1Eqxc2u3qSc%T1#PL=VPoGVXswL#2<;e`)1WUaG^D|4xj@ZpK;2#H+R`$SGRD% z_0tGI&Yb{Ld?}njkz3r|L%$rz>KLnpuM`&?P0Y~{9S2+L z@q(i2kOVgs^rU+!qdQTNHQIlMI}Ug5)%@|#RhpyHM$iiermhy)#k@Ym%73!Y3a&d6 zi)+j69U|=PK!x0MFZry=R*V{3TE1d0Cu>Tf+{VGubB>}a2JN0iG~c5}qYWwCTLu?b z7DwtN!ZOkGKA#oU2Cfj6b?5CTmToN&qY(ClqZWzKgEaq;e0L2;pnosZ9mQ1616U)~ zLvq07msYPGtBo@-`zU63D5%1r*4-L_o9Zv!z`84fs=fQCcpWbby%nIogO{o?LrbUQ zQvYS0s)-8t3|mI(xs_xB64rAuk4rv=V|)J0JEdnxl6^1ab=UzNE&?&aauDfg((zJ* zFg?JiJKPui)6hk*A6RWFcxOa@x7SZF4Q zi9cdX5ti`!E7J+za<^&O4~9f0 zXow2B#Q@XFTL>d^Oafb0g*B9XiK|HXCx%{)vLr5qT>3J2y5HNNH+^oD@;t=P@BwC7 zIMRLBsp0Lcw3hE4%_>POy*5W~xIN-epW0KVjftXn=!rV!?7BW(hs;`qS@nbilcfB_ zxEr1ysyMiHH%O3gsG7)1jqd&k+J5j|O4BU)!uUe0DB+n>>VUq*p3wsKv)_~X$N7W*j>CgcY&83 z`Ka)xjIvKivQShkrJsHA{;ypYoTpFTb)VkvKENiK(=4Hu57eCuZlF`~Kk0~_{sTEh zPH#uC-qfW?={z5Y>pYQRJ)O@`?QuZ;e$kNIM|SvgJbP6l&Ng*P9q6B5 zL;XUpl!?q#sp5h=|IP!Qc!H!4A_ZR;-w&pRKv}A6v{r=N(&9_Ouo-K%^&8avnStK| ziAF=Z%}W=>M8q4!*Bh5)C31_nXdtlwhF|02fhpE#yb&k zgisE#`i}U_I8gPv&@R#c`aC!=Q&?aSW*_x=z6^FD@7i#`M%VB+L8me23p2WsH(orB z)L$_Fc!dsd+Dt@6>NGB!75S|JuF40O%k3v5+TGFZv2(_6-{D z?N5F$OOapxs>(dBxc|3dOXhHdeWrc0XXLcCb&vj5?EGKI#@mC8X{T-u%nww%$-4$j z4cvFFOgLnVRI=|UOTWx~uysZIL_}KFpf(z&QxiFCD%IV=n)xQKdz!ZBhWwrCbIQe$ z;2itRiPoMAE$lKWNx7M^`W>k;yV<~v;HVsx-gtqGALT_PLkx(9?!v2RzC0ndpzo!- z7YfRT1Mbhq4D6++swpa@OQ9^piW8450nvQL&@cGyxW9ViIj!CMdrLDf&}|N|-wv{; zHwQ;n2Y#s?*BJieh#ahurF@l@xyZNPc#-h@`;1eCtVG$_XOe@a(>Jl+ZH^BFojI>g zzE}X-2bM0`#X*V7n3LSqJ{IXQ*^U0i6jYS!F(}YOdL#fBjR)@_4EqlqQ2E)JzpAW~ zQsrMM$GPyH{{2ZeWukrt@aX4$x3AQZom@|7+^w#BOLtB*$Z!Tdr*RE-qM3X0^ce;k46~p?x>l^pxitfyzD!1sZ7N z*w-|I{qdvf${3JbG5J#8r~M#H0*TrqsPi??f9G+M7^Kud${~T(TQZ>s%)i zc97Hb*Mf;}Vl{lYZRIN*zcJ)_4FiFIn|I|HT4|HHaNC5K8oSJ zvXECo{Vs{8WO%z6kdH0e?Cx5#dqmhg*>siDpe8oh<;Fv~)4$6&W8Sc;@9MRA{;^g# z_{Gd~_300;(n9C{T^R0mFZPkal)pAShV2$_X5wir=Sw}{4yLNJO7I11-J`=a|m=6hvg~LE^|M_NJyv;FR5b03-X#zSARQgwR zqBBbN8a^{(VF@=9lg#9`hBwqW=A*LA;P;j&UnH$B_77%( zv-%EpX9iJid1$cXr4zBPxfdtIba)*|#pRCg&9n05-AWSu;}>@8DOu}f$FEX~%`Lpq z*GfcDhZA>f};R4f*Tf;{J?%eMs#L>f4ZTGzMrz4oqy)wK&t5;Iff-7z|dCWGp ze_kZF=WcZ~kX+|F8*JxHuOsS#!iEEJeT7PGU^x8`?je2hi(e|LO*p%;Z#`_-^W(GJ zS6}SmtB;CO;JZJoBGewK;+Bp$3gf{LzX?Gx(aI$%@zK;&LgyNLc>7@GeRS$27v`Yk zkvh#`qq|!|BdH;yW)ES`1)IcdCCQ*PAxRy5`^o3~l@xbrYty`wVuN4uLg?w-wrhnqV+VKdHxkc@ zW--Gq+mIu_1OK^`ln2V^;JsG8v@JX|w}bR`zu!;oy0%+or881y@f=48FGK05FAH># z5kJD8xsKB%Z#@70VyztDCg<7v@;`6s|GKoV?MYoK4$E~0T4(-qg+D3c64e~GoDE+e zH6ObmUAUDBGfVYMx7GD4Ipru1lOpJ=+6V0?rqZYuPBvR&V?*E_U(8e(t<5 zg?dZV7_AmRR&WgL*A#c)-tDqaCuHY)bHJ}}9zAzXXU3~_v8Ik!FC=u21e;*96a{T8 zO(wT#Su>tXpGL>Hv}=$AVoP8muxEHU+zD`1n0Z zpkfC2iR5F&MD$7|Jmq)!hAEO8(vc<`#e%Jxm&>ugnUf7d*p`#~oX>6|uEj6;70q0X zy`eS+Eo`0%pTyYKzCn`GuY#qYM|L}u)y?X;pW`=A*1Yf8{I0c1DBuxmjWsjKuWXZA zCcGSMCaBfeB^>ZFc73chX}*u?`u9`7vk_sP{9Mprq2{(M!}S1g|} zd`SIf-(`AgMt(Yb3wx{52k*QUtRG$YkAl zFN-!(a^!C!pzsb@n!W}l!T zOX!n~oflN4O(&k2xNDU$F8#e*Tg(zR$F$;TlFRU~CE#t-PVOy!jUmXx01^R9=h)O< zdubtSBx%thjq93u?|b05d)6LvBP3Av-~ouL<`N{_{mG$@7OmlrXN?u+P3 z6GUwHNHqw>tbcoUtSs5I`q^-~d$%C7j3Ghdd{my2H;_Oa{I9F`U)OSZV2EQt>-!^; z|Gdes%eAu_8jSoL`xpWwzLWCwFmQ9g))B!MJg6tR`0CQ`jM_=FL6gTd25=I`IoEZg z%FlB&e2PyXOn9)~*C};A&IcANAbKp}DvU2?(dxcB zeSGdEnj!Yk@14AZ&2-PixziL-e7t$rb-`m?U_?65AMGpZN;#zDhZEdH2Xh$+`v9f} zpe}y6_h=0E{~^UKHX7V(*zeZ2{wD%VMfrOf7|KF!)ljY|!riujGOLYdY0P?XCWn$a zBdg;Y(=WN>A&qXnnz373y{Wg~zwb}kadx8D4!Yo8E=+5G1^)aTAw;q%hLy9Hd7z zot#Srf2-9h4vW<(XnCA8Zw+O{W9LizT2b&JYFa=jdR=AUsnGSJZ!jD{G(%|S2elk) zd}T|~S)Zm9s=P>mT!MS`;o0$ov80~vky>l!qA{qfGW>UMd@Q94W($2-f%JgqpC5W; z>qYR|*8!A4oSmW;F-HQTsne2JaN zFZ49jNia1G9zsu?-$Ip@Y{_VkC7C7nZx!v;9gCszC`!xW*d=`^b;CCBv*~*;@c|Lb zQMc#ggBU&JP`OQ*=4O21gN;)ocTGN)3@G!;tWeg2%XINK9UCkaj6AD(cfPyZm}rI) zaGkKbudKXWq&$>O?e_k!GM?2s`3E+YR6B;o&lY2T{TH%{)fw`>1Tkr-(vetGtvnuw z4z4#!#G?XZ?ye+bU8}}WN-~xP z{ej^km|aso7x}=adnJZ6zZGX z0Th6 z8#L`n*r$Ds4AaP!N1miolwRnt(uZv z+s|V#Hve4t(fm8IT!~WfO_%|?!PkKTlM;|Ds`%Dy9KvQ|C=oD8T5EV2*cQ` zJx}31G5zt-Q5de2QTy!a-a`}?UCKe|z%Dlh*^^loS4t{WwI&2beNioOg&owU9zu=fIue=j%c0Fjz)6*NCVq9NAVp=!S zmS?qbS~S-ZyF-&t!$-sB}qe<5I!;Z4;R1rH?jT zMYK=1B5EXkt@xsT>u00kzLb?#28!tj2+Hu;M2h&Dx&K{{V~a1j8GLj^i3vph>F>s` zB>qRw#dzZwt7mdb4u2bgy;g`!3XuRn#FzkSt=jWfhnt84KUfg=a39(|^5wqny*_d# z=6qv%BrtsSM7mqZpfOo0V_mS^=W2WZQCs3o)4tK({Jn~%Qq5#H&56At0g1{N)A%=Z z7>*e5;x;~tPvth_ny>x+AJi;k3ZxG{E$V#&PT-{V%LdcA>KxO@Tr5o_uGd5z^}S&| zh#>I0>?AWZjX)Y`B(~{p2Hw$)PoMr<4Ewh>7LbcdQ1=ks)fglCUrHIW!rNto_VzyB z-^aGbIeb9S-_gQS5e%>jNG?ib8rQ=}|Lh=MoM}J)34Awt<24Tj9h0M2vi;95!!#Wn z*Rr&Xi1v`$k)OpJe0$!{okM@6F2Fa&X*R@&No|;OnHJ1t8cxj`*BUyDR9d()ako0P zI_&55%5N44v^SW*En&~uXuily#+Qv4j};M^c)QbCxvZH4LQxrN_>R^SWZx&| z#g3R*u|UE&1jOPC*_5iCIZJ&-uCU6F3rPhgQ4m=FGERi-_QS6$ZXy-SnQt?PGO4WT zqP=?P0XyGv#kHKi9&$|D|9E_Ql;6Wm**Ta6q|e2p z;jrGx)~rgg4zZ2{#VxC0B|KM}KQhKb5B_wfNOPUS3c~(VJME86ERXoHj%K(}S{pZe z+6GgR-G5D@?Euz>xXbR+oix)C%LB@^5aQm8g#5hI4SzbA7oCdf(YjdN0pd_5GqWYol@ha4?bG-X{Rko(jr z;s$vqM)fMAMBMp@qU3vbx61X>%B~!0st>QS`e!FL+#6dcpMI_r%~sfQ29%h-L0>n% zqf-WTQW_Ju51yLui5LvamEFB2Eo)p{8h>D@n=G19Cm1;Jy#|#<9@B+v1XLW=V>j79 z{)CGU3tVunUm%ATQIrGUJ;XqvZwO8E)dY-&GphNS5rY-jYAE1EC<%*3N~oDK*;?*) znUbcz1%S6VgZyCO*8h=2`=d-EBncXG==9)kpFyOs;hHvA>pjf-)%DO7seGd!en|xK zb6;zL=%}_;3uv~f06M+pRbf^8{t(iV2btAp%^svAqU_^Oe6AV-)=7&Hs4!2PIz4uMZ>g$f4|uU`}>g6A|IhRKKW}ejA(tEeb5)xQT;crRtJ;vw1-VCSy{!32ZiHLZ(`WkT#E15M;E%rS z4}pqbYJ8i#$2s}Al&SgTGonx5#`GwO%zNVI%*lpA%J)D}- z`km_}xy(`e)_Xp4UsxAe@OTOlv>QFl6{YbiGwv)NT94OE*Y|G=qS2BMl-Y9nu}r zLw89C(j_4q(^sdd;k0EKyFQTS=Fp zGh7-G;}i{32Irw5J0cgA&2WoeJB!lIb&=!CzpXFjsf?I^ZOImhL*4eVS_cZannT5+gqy}ID6lO4Ac9;ha-`mA22ej(W;UJ97~e~TH-$z@ zX9HKBt{jjbFJuPm_+}f&yI;U*c3!Y-v1#Rn)~oZTfE`40{H496f92tq-|B{xe=+TX zrp*7Q5(F5yv-Uzf0@W3hJy#*#Z|}cl$O>;#$0|% zafF=xdy#jCmZ1cmb&_Q8n8yf&dJ303{2UUt69}j~Kx`{~|K|$Pmc4Kuwd*e0{+w(z zb1!orwuCnkFKY9vppYCBRM}ZQi#0Ei=r)77r+L7;(wRmOZQHyM#UnCySh2~{n2$d( zn%ba5{U|kurpf#Y7a1nUHsxP?ADf6Hmf%MhP8f_YQ)h=Whr8i@Ti%*o2<>f}$hl{qK|~L0s#FW- zlXWQ*5(&QiWiPEt6rd#UPJW6V+Wif~_28h~y!y0gR-uF79VXSPxmx;B&3pOPDSWaF zN&Ah4K3+`~(NW45E%1BFKXn@MAQKeol=~s|g5eGGH6>VotU|3@VNanJl+j{PCo6K0 zt~L8J-`7oyH?p|mt^Bff22Vf0G2@I!`AjC(^chHaMJ(83l$$LVTybtRQpyAX)BBMc zd~&=eki7XkX@sdpxKYHo+Vv9=5?6j$ZAf}+;1aw+$12EjSnWIxs5@KSnZ}nk>e^JM zBsPe3m|?;ZO~;60&0*a#|19W9JEc{wq$)Ty~3*mO;B!Y^(=tH`On_q@9Yf1$UY1LZbxQr8}aYRqLvQg>0+-=&O z$m-0a2Gj^gpYW7rqppE(m#xyY#6YfR3_^B(pXb<-Fq@N-MXTrp8cxuYT))ExSk%CN zaU=W#gM5n~Fx6lv&SsQ8SxUB%&hfrAJynnE5mm9$M7pmIt)Ca9iN30d8Omy!Hk;&R z^Eg^UCFBU@Il+mC+$~Ulx%2RTfdR{tZ`l>N52_fg{#aK?aC>HkmM8SwG{Imn;cu31 zb|0^ytC8(W7$ai>WzY&smbQDdQh6EPk~lbE)N zFHt%tkTaDTe+|bpcfP0-yCsK{5QABxOo2R8zlZ5LZG7w5mf7xv44zYSyCYptTx3zh z<hD+KJq$16yUma;5%D=Owju7i+o_>T0hoi7Nt>3y zL*d$YnSbAM3V~_r>P8AoNRY@9S|A{To1KnKyw1rc!GoYMF|jc1Q;sYAVadT$eg%Gy znN04o>E)uthIsI+tJtZj`C9dJzEdybpmYW#S{@WvT&hs=p1iMeE~6CZ(IzV+Q$^N9 zbn{=k?VwtUuw9(WPI=8B^!82tGK~==6u?sylMOSY$knpX=M9ZH5sc6LApVE7u1VZ} zb^bga0OwrY*K{7!UKdZiJZ6vCM^a&315T5V6VQ@HEInz<0!~Pt7e$m)9%;eOetVu` zAM6cr4`iRT^59woGjY5e_$n25VD$G;nG*0Rdh{cM4JnIwCsS#%pdp;0xB3!qj5jr< z=p%M&97{KK76!!Q1-VAp`lC$$44*>EsqB-h)y)T^qHq*4*QJ3_m8OD9(e67cqzW@U zo=Qo?t<4^ilE~7#_DlUmlLtM9Q~sQ2|GAKJU6XMaxrHw!t$VKcHtOQc99Vkc8_Q${ zD9K{6+CvZfZLxymV<27fLBWthuLK$wiuvp4qQA?3%2!f7_-uT zw`%)GZ;=iPGaTk62EG7D*r+#8mS>!M?bL?o6q!OxUM&CIHR6#176?6%=lf*+%ewZ^ zw5RB1&2{10%a{oGVgis+9fY2qu9hFa%?u~T6ukDWB2M^?H^>+oRm?n;HULb zqVfqlufGl*gzCVYn)}#*orG^+s*Y)9fi-;yq$_;YcilAQ$ME#>hWA~HsL&yJ_m;4# zY4)JCa=fpEOH2`VEO28_(SbG=LZsjI%bpt$mUUOz*e z3mrSpm0B3azCd?)@EyV+bVn~=(jnGjF6qhF29Vj5Q#lPo@1^x6H+da}0BouH}#mh~}}4N|OsS z#VMVfa?iKyS9@50mvN4V$>dAF?c@HPJm~>e&ng5Tsm!_S-JT~G8^rBAxSt2m zyJ|YlV8+JjBxT+$sn>Rj_R=f3CxWnRRzK;%paE<12J6~8@i@?~c#D(E^Jr7{A?5Bp z@0F><2~WET{W%ej47TxF7YfhVVM$>{bpe;&(%d*ShWKpG8^+!Wp6xPaxENP1QrD>2 zxps$_*BxItcWE>Hwob>X-xP~lnH{ z%HT?5EGG!r$K!SV*Nuu0uD9bUT9JnS^S;nW!nhinzqY&k9;w9}TAEKkTEz!>3;FTm zP~Os9rPRTAVg=Hv>w!&3UL%ak$;zvuu09kmXJG(D3sFoR8v49p*)8}s;uD!Ap*RXv z@t0aEvm}6#?Bastf+bpS6h6Z1w@iup9jT#6g>drdX$vg@?R!V_yata^bJe*4*-kDo zi5f`14)lWarV>CQ%++f5zY!lff=LHmcsf!7G~~gUb$w-UEJ%O^7C)KY_H=F|(AvooIu`8%Ec{w;4^-ih$vn zB4hIh^JlyKfE|Lqf)6~D-?J$MBb;Vnp1B2G8|wuKZep!3aYoxo^K~;1al2xnEYh$0 z7$CnqEsk({`97zr4M$W)vdKk4Yz97H!M3g;S6|bEqkO_Z0?CTArn0fAJ+Cx{;{=;)+SM;P(!`pmjO8>rrbhIo zHrX3KRy*DS;1NdsmQ3%pVnjmQUv9t8fy+s+3mNU#Tj*jy^i7uz2CNqNQl8oV5()!9 ziTrjtO7kGx_Xv@xuiH0=d;XcKzYpgx*`*?__&}oZBd_)L@;T(V^2_!zunnJtRsC0z z4{d_$K4))wrspL0=X1|Bt2yf5}34BaFsY+kBvY2;?q^3(pb3XMm++UBWN7yKp+@#4n( z=w{GIBlzXeVxQET-i%`uPp8knWwt*eV*?#YcD{dqJB}T{rN!Ov;QEEdiYV(wZX8&##;jv&^!m>z&vv5tX?I2XXr1$w{5ciNa}UHv7!Pc?xMr1sV;-Rd!tInZka1W$^&OE zIZm2@HktDgv6zslk?*ByFtsxkR3P`@k^tD{RFoVpa!wP$=XLp4>^+zf7%5<;25cu2 zn0w4vOez(fx@fqS^VGDZu+NKpAC8L}g4=k40*P)1-ZPp!v$uD$zSjP_@p+%=nh7M@ z5n`FI)6M@vcDYT%|!7~A2681N3$mBNI@9$Q-1f95bT!rdWX9b>uf z_=@6^F7@)M_esN~dgFGk?d%id&u5f?iuE@}sLg#?K=C2lfe8{850JLV?>aXgzt%^! zu5b{V1p0`h3P68)I9$=Pf7t_PVUT3Y??GeeAYHS2iW}%vWt_nW{u%mxV&$ zS093!Ov+=lL`sl_Xieg3$m74xBp3Nto;nNwaGvU(wWeZVWP{~!LT~O0diTv&vESA3 zTvtnn>~KJzy=HJM=D2mtce3(xOX;zL5TeqXyRst9x)Agc={Ne1Uvt~WOIwNUE2eK7 zbW$!FEmb-aX??5Y9S0{On6Btq{zhQ721VQa1qnIJq)r^Kgj9>+%rzT!berTw+Qx6~ z2*haR-Z3hm9o}w#f!7%QWjm`Q0B{OJFRK;#(X+6BmwN(T?|aYzW``AP=h-w>4HUk8 zbNc4h^zW<8ns7>KS^?Z`A6f2pz>%3CiowPK+~LRQ|4Ex*a%@;EOB-ljd;fcH-?(Zb zI`Ig)@6WoD=uahRc)Fe#A>J@@7xgY%!Y?@+vVga}wN(<4SX){0oqPqxh3>S()~8CN z{l9fxz%QbtZGs*kGD8E+n@OGW?qNr&+ZcY7-PT^4SNsuL5@vJyM=DG*n^_@`Z&W+r zW;C8V$ZrBy;8}du7-oSdLb)CzuVdqAB*kK>##-!EGB|r=F=-aVE~tmx$39 z@n1feYOP4T$WV)b`fK!bjxk=ZR48tEFLESW%;fGpjx*t7^P+W`g|9!_Vf_rDkQY03 zkhJU9{{~>oRyk^;vZTzAbQZJoaw(69MJl$SIP4zkbGyV$DRPJ}L?ctlTsAf*6HhL~ zHf0N~7v*v5|1xmreS}RyQ|Re_!7NHHAU?eH)KFMH&-e35J~Bsojs`2)3RI5Wwk7ub z1|Qxs{*TGUl0xY%7a-|=!k3zY6#nQIAMwMV-8X_NeJ3 zWf0QFq+<+%mH2WIrxG-dOQ*JBi>{L04&~MR34T!McJp=EyMTP)sik{2E_r0~6~?bR zVmS8_K&Jkb*uMYDw68!hz1by)BHxx=}6@JcF=ANXPX3f$*&PNzFJzm<_~E2m?CM{!hOJR)65L=M4OmO z7H=+_aJZlsjXPJO;r)64YF`5Rk1^AcTkCD+H1^$bBGAiEd>8mvd7V_c5OEb!YfwAMV zPrOVX3-$M@r&`DZ!k^E@Poj2W+tBDpj#bAF9cC=Q?xt22ZJXGpG_~(TrQ&WmjNJN) zif?viCZ;AQKgqq2B;6iN8dL|ganxr*HCH2m|MU8N9{gv(em1bUhX3dIYJAw3QluxV z-P+jO>$iJA``k2PjAoppenf25o@=7?SZy3FH_m)zl+Yo*wm|N@872DGQ{ZN=kTD4B zkn*KBYI}R3moB6DK(B)5&#BjY`wT1RjyrgViG~dyHwR#)1!Wo}J}cfl?Kh{|k(OVq z6Gatr>cmg&339pw7oZs+P{KQ=uoB4oT37->fi9GCRm_WjIBejz#nlYoRw}m$kWf)> z%Ko@vECA2R-{rq~xn?0ESMKt`M$Q`&Z=MoHhi!#8X682~)HQBA1)VE* zV*#{WpVCEBJtq=O49RuMvwo|cp$sedJ!+&cCqj(~AbHWabcBvr3x|O`)39m?9!Hlh zCu{Fy!1fU$UsFtFVnMPvq6v%19i&O`C|AH@Fk&#qeZQ286>(U>gm={*zt+=Cl(B~0 zW$(CxUUu`&%t--h`g1?lTL=%wTGT%4#c-*&`%R2#cENqdkUwPX1JnhQcSHM`U& z801ra&RRfe6$=~NFrW|cp6fH_axoEP)v z^EFz5m`1L&-&yYunKo+V_{^GHN5Hh9?OHnV{q*DA#mym~uDiGu_4G3GV1Z{HBIA zxW4EsFMY=7yUZN9{WT;T_VcHy3&3X;H?~P7zLU&enm1!a7@Db&1DmH}W^3{ze(2dI z-5dZ40G%5y6V79S*zl|=E?MZie3Fa#K^1p|U|3XAf`RT*alQ$){YSBOzuAl)t)4H} z*^#(?ZdH<(muKVPkQP>zeV^ zrzbU#b~%0^B!9jGD|F1uuQeF|=&SU1*vU1YefgQQ0Z(%(jk}ke0zwI`{D+kJwIDt{ zX#SiEU&n9#B{u%!(c9w&p=xWT9y@Ns87OgDFL)l0@CBzH=jw(m=mQ}ONeGqE#Qhtf z^#;ce&4*~glKw->ssG&*+GL~8&XvFJ?k~b%k&mxImwL4C_as)rgIVP6;=;ZnM`@~MH5C5TgPtnljHd? z`@K5);NeiFf30XfOkB%bfYg_|Fl<|?ypy>1y9E5js$ zchTaLkTv1hq>Eyy!N;oaYZLe3-2Deq)^>#!QUI+YZY9WZxY}9oW&M_2lGt-S?sPKu zv=YeoSa9b-r-0kz>|$-(ml(w`b4!?6!HmAG*^jG*r7;i3Cw4*xDYhW_EFmutbY{vtpz4YqGfULEW!Boa!&D65O0jm2_gV)Bzm%4G zc!?0X29HI|XFcP#i(Y)(PR^%|vh0i6y3ZW^=9xkCqaE1VVSWBGU#2f0Ou2?Na4Ruf z7FK+&sG3p4ClsFU3t0InLzG{>_RH-t&LZ;%gXtjc@l)cbCMFF)bD5d>WlI5HzK+>- zgjZLzGMGY4VMaPii8~*;7McqCp=LLB`$x@0YBMP4e2!hP8STw^H!7dYlA!4A%6}Qj z|5>GaRz;_AzPGD5OFa3{YZnRq<3wJ9IgLA8;{7eStZK9`$F(auF4Px;Tg)8^LI5ye zt5<{mrf1~*h7r0$8eua=6epIcibpK2rj*eiAA3vbJcgj2NL%_ zYQCyjLG*j$AP&W)GR9T+n;z)_#zC`4KI?&pIre-OXjX7%h*0V! zkD%g90A(L@>Zq&Eb21Hu7L*vIxVe|q6gJKVX^DP~=S(8rTtj0M&yPoL%AI_0-Jo7- z(JzVB9D*OZgLtBMJ%s_k6^r2fVqP#Mr{yqs#_63)dUx(oE0)s8s9tIgf4ltUx`BwZ zb7YG&4(MDh?@5+=xoF-Ljof3;*2{BLpq)=4a+3zfics+Gb^ueGp&mh(K8oJw6(;zh zFe~8R0l-A0TeUa6*XSCY;TP`^6{)o~!6V?{d&VoXxAFCuTvBxwtg;IQULSW%#HD?} zEhHMvxJY)0YI^FOh_Rkxgy1knB>pbB>yjlUi+V1WLIJ_Sp5*|k&oKU^ z|BWPp{&gnQdws!@{d(<+jNHS)K^YBo4jP5xEVrn_W0 ztWMxP$O0`jKF~HP>+dDH8TvTY^Wy$HG>+EseWtn@)apiDkYfGTrar%a@JIj^Kr zCas3co$&3TnMV&D>zh~m%YP2f?+H`sY4RiHp9Zcu>4clEn#W~puU@yE-RGA#v7TqA zmnMonu`zaz#{+*{CXJ7_ryN3C%~Z}93Na{cM?2r6oC2k11X38%_B+pFd@UaDy!|$d zp}YZ@aqo6pp6+09Kx9mpI9=xRM_*5V;@L;#uT=#SwCS{iL#8DyE07N;r{6B??5>Q@ zqX~tV-M!W`q+da+O!AM~1F0=Xh2zNK%VlYrPWiB;8feX2Aep=J!pbhIgPqGr4CEGI z+O|kP(HFx?ljJhDJXu^o%Pnyxm;+!VF)zX6c^;(u;f&fQga(Elw(p40#6 zj;H4CPH2{n$zi353?`7Gnz366i_Qz8r!eQIv&(hI!0>iz>?}g=N)R#G#$TFCG77RR z$qAEGKZ$k}$mB~PH;2-fz0Qb>ID?CwMZL)HASNm4NNmN;`0lws{^tI;p>85e_-6RD zh;;6%IguLJBKru;5Bh_xkVr$-n!p{YUki8$+p|gqR31_11U4E@oU))`dcr$i*CsD{ zm5L9j0HY^>r6uoS0!~o#{7iC~S#pu0dn!?Z>Th#G;Ic158BSUIW&X?_PUhP~Eh=9Z zh2~1*@#GRTD5ID&e)PzsnUceMRNp{iucyik3j21EiC7c-A+S#G9H8+joclEjeVE+1 z_vN6+K4JINYqET4OWE6*J)dEbQfy0|A-3`MsXbDdZ>;b-Hd6$dz~&1?eZ>CiNyqXY zqd`DKLGlW>7dU;z2K)RxIE(0g9B zENdK{n9w)>prb<~F(FqP9<<`$ANJ<3eL2M>KyngDNG!(_16D*k{iDE8WokLdqH3$U zz2b=$^B6{jXyx(3O_iBpX*&~3{P{*LPa1&LMr*O&c8D4g>V*%y{neTIlDG5gbx6;P zomt#@$++4FLuQ{A)2YgG#!vkrGG=W+%>EjVpXnFht$`%J8ukao=iQYGGK|FAqk`0H z)2ar)A9c=18r`BJI68+<${TPMhw(Bsa0+P|N&D`;=#o!LA3J9#Mo&~GUP!UU`PJ^x z!6k&w!b$X{Wj_pSB%iO;80U$IAKWrCKVEQ&UU}HN+kqLrhGpVp=0*Bv5bKyA-bxBS zr)FHxw5E&Be&ZFrxcNKoaB_}8lYnNICTIogbjuLVx}o?lA_um>OT8rJVA zGeMy=+_xuTM_A56H`~Z!FR?cjR8z#YrO69m8S0Dn=T!1;>R(hoIYFO-!$P{Qr|!bg zFp)*;#SOe(S$h?2JAlqHNn|@rpR+$>^&y4ae?R9hG&r2K&fGMmPw0*RmSl-=jwq#m z@SL_dhtl}n@ND0=!IfM7_ZFkH#y^aQ|56)3{9hVVS}}~b+m6R9W`;!~Q9$E~iY2#+ zXCyo1%i&C^_38EchN7IK`%|l()kYFe;)Egh*J6>SDE}z&NWxeKS$B6!i@Gn1pl&v) znh9(J<2(F3zXm*!RX^IFdai>A2n?r}Zy?5DI}?aKQ1wW=4S|7m1Q!uuh3E?Jr_uYt z-1745;irbEIPv*$CIN`>c~65!V=s4vx#j|)=FExsMc`}SAkn%b#^C(c>X(p5R++f- zt|Z1cCVeHxy9ae2=xqeIQUtPSGB*a!^;@g+&$Tyw9{r}oZ~THoUnSeQ$JPM&_Jm=X zqm<;^&9pIvIota9&h6j*M-t{+w1^_nNHOxo4AAJd@vt`+^2uejd%~y1|8|ldSWToN zDG*qnv2!}|w-tF$keQd?D7-~&13Gbzr%KueFUZR8FL_0(S;>pkbH}CVD)HNco zKCJg%Hq4xR^sCCsnvSxK=s1NF79L;EH^^i49`x5^daE|l+YqJ1&ODHAUTMoR4V+@X zZ#tp#rV5GrMS$i_dz_7ij{yr;K+W+&My$-*FV8@B0_gJGrMY_hDe|Jl>Fz1-kA>f@ znzGV1u>!OF!r^QlwE*rT2Hi1-34paGJJfdh)PoeUA8w?NvSN6G-V^?Z_4kyL=F0sW z@zRkA*-6!Co0cbA&vc?6ypEn1>9IPhcwH!)h&JD^CU)urLqzG>nM|T>kB#14+fgEf zV1K-)T8_=#9JzE**syIg)6_Motk8Ws{IGm9u`b^93S*nIZ!U@YM;xRN%WjdCg+aeX z5#ycJ!x3i~2g=*4X2Y(PzN54_P)oeQPQVB?V*w4U5l-Fn%J}|%aTEdQ=>M3Rpbx-q zuL=@ZC`_OQc=#UW9_axL{9<}4)~E>MJM80Qs?AYrOwAdlWSzo^ipukZuC}qe=eYaN z<>=pr$TH@I+&SRUNwmNp>Co6-N+EIRRrk$INF)G~;^*J}1v76Oz@Q9TUU9kQJ$o-U ze8t(K%ClKlov7DS?m5X&E3fO*s*PW6T)e!9#(K@`OtAdZs4d7;(%-V+GQ%l@mp_(z z*L|cj=QDPuS=9CZyyg8B2E0xZG5WG`9&?H~v;1)OPCb3`FEjaH!qd5iE&{0xExb9M z{Eys}q+Q@oo$(bs=o;cp8S8V0ecJJSee`5>G5)P(|Fd`2?alt1yMrS4hl|1Oeaz!{ zi%m4Y)fX|V+Oy&6T)-vPfi;(lV5j_u{agET(};a(r(bd!;{(_0p-I)o&08D!r}4`l zpOE{aUpz;TwglaJh%UWXQJG@BEwRY@^uec7mMr`y_`aWjChj)y?f7d*VGC8h$I4_F zNyhWk%KO&fYSfq%zoLc1Sz1OAe?OFc`Zvt_oJr__0T+vU!E??(>0p;|ShTASTpMO* z9BDGAPwqbm=7xBSB`O$P5QNv?bB^<&kd>bafhZ8zK)~@0Hv3xYOuVUKeA#vvF4l*3 zjINs2{9Z^pmi{xcr5Ubl^O?nhoIvPrq5)B>;gWemoW?`XII#FWamq>5EYTKFba)$DwnrVmj zs<<97-Q4vR)M)iKP5vUxtrkEvWl>fxwlv z0Z;s9u1KguCZ*pLzWt+vNO4!1R<^NJN=_`8z4yCCA56xjuBR}h^czPqQt=<$Fe(&o zPm_y)&r0MPTILp3v)(eRM$8nZC|=K0i1LUuvmGx3Zc5BXdgs&rSy3Vx9+2bs;Ma&) z7$Rdi%o&*`amUAViAceR5`6#V+Cmet)PBx^zWyV)Pb{*M9=%=oT~qocik%GU~<6c9L}|sLF}k03LX5D2ElhM2tpl=j398!Yit^ z_-%Nt^{I5R)irC)HYzOAXP9X{wPN{$Md9p!59*nK~iD@rmUb&q$oUX02u{;Y(z?CKa)LQZ|v zF5gLX>>Na$=6WZoS8bS7v!7m60YP?IYV0X5&5MoWhA7pgJo8|-tsgiqw_fTm9bf-Z zC@*EJdcYgvct2Q(s82I>SlOvDD;tuBNHkJat3BszB(-SgOnCkIyqsxBPqnI{mXl&4 zI1K5-Fkd+G+PdkZQ|yM>g4qawV}Xmb8!x)v7bIpA?V(YxPtzQ3X)~{OJ-ogdxa6h zRzk1#RJ5R&uFM=gi9p!LVE9lFd*~$L*JO(6Kg_%3Q1bVd>$SkT^)EQ^Bfji3f}gtjSsSjuiJ5)c zMWlrH$GIhWrD{LcB`TIt>xLjFrG!3e=({pN+0Jyt63Vz}Eh4)iQO7ge*Y&KtDG>-g zYtws8Ae{fO7%U|VQNRu;{GQlhJF^LYw2k6$Et0SCp`0K2!185I7kkGmO(*pe#|Hc$ zG+^mAvx-C)KdtFp{B&|edDN_~AV5l@tNIl1R$P9~=TUjdcCfSGwiyX-s}!G=dJ1$h zsFZqFV`10-uB2v_1gkm7;n5bot!z}4)|SYiFil0y=^Ez{km)p#9O$_$0rM%UOjcKM z%#Y3cq2k`Jit4sXFzGJleH3ZKAM@vrhu85!-Q0EYlLf`@Fl#>N;3+13kVQvKJjm7# zFwwDv-=BjM8HnAM;nhBExKdJSWqhc*{gV6uAVK3+vf>ba5k&5@YJU1=jpS(4xr(rC zFA4v~`42JevDC6z>~-e#;I@VYD`A$>4dq%hSBXgoU82OJ#@(yut;H6Kf8U`09n8~} z_zyn;Gr<)h`S-$oP<-7z-k`$ML8?=#-j4|S+4eNPUbJo5ztC`>5QwXTngwot0VH-O zDcF3s+rDI5ICNaO{FQ0YaIDh)JY{Fl8X)jT+-LIR<*Q7SLiL}|2|cHkxw$l)rVvuO zXo5M$R)1B@7e$zlAzi|DaaOl^y!&c>MlQ3ZO!qs9hHiCC@B=X&X3JZWbIe>9BDZ5D zD;pMz8dYqdNaB@O6rl zCWja%aIG@4yx7R|z4s(vcoLtV2i$)#`$H$44%%OC9Q0W zRp$-EpMZ9ukCrf2aVT2-Rg)>~=+Fx&6^1{iK!?1oLE1^9*jY#U(=?0X3W^M}#)C|# z>2N2a`oP~4;p?{afe8i5ynmQPUvA6#y?OQ}*P$|HmHl#r9-c~0v`A)O@wWCzwqOIZ z`={vnpIE$)K*XReQ?vg1Jg`I@W6r7HOgLQlZOjs}pGzAx9yq-T*(`2pHWKzZy?R9HKA;6w z11s2KGrU_UEjJXo-6vv0;7G^S`K`yWjc94(c#voMh1iVjVAtbQ>5`<-wM3jKGg2(Z z*;MkRh^vhgi0ZKQ)*2a*M3ijkkCEsZ&`a-Qcy}4^@gvan;9cwNmOuA56dN)&CIR#k z(#n?8&-Ra}Tof$%`1Vze);m|qp=b3uMDu(QcgIR8dD+Jb9-=$Z^9SK<--TL6kFWkd zr-bR`4C^J6 zOu~zc!W+(U1_=SAeN7Vwo$zRb!J<>pQn3*QU6GfLj0=1lBMRHL?Me4B7%kr9#n1_Q zxY|J6c+EjXWr=EE`1DIbLE)nMdSdZ2gGj1i^bsKmKE(y8862c>Q}q z&(D5zH?N1p83SA83Fv3Bq!G5kd53zqiu;xNh*NCQo?P-)p?iGm%^JZFIpO6~AFIKh$x4MasYo;daMh zb=i(?<&TY~9(0UjNobY`>x7R)%G42&4W6@$_V%A7Dj}h@g?VXS;uH(k=tsk}txNLm zg}))Bb>D0Y5+=d!X`(&>v_(piVFF@jg>MFCDI$&c*}{`9V^)D)^J9|BOs@lJt%J>0 zUuZd(KttTxV>C*sitwg@3=dLTRc3f2^G2DbKl|qm8&{jH7(B$>6U7iE1l&K%dWH9n ziGnIQ892Fnrdj!MoAzB1aZ4uD|4RN@y{OiWw)po{>h z8uB?@hm7NpcC(>Oe=l!ThWCZpPH2ZpuwripCk1+YZS>_xY>1#@9UQp)%ayxRMgCq zJ6N{F!+BrJRxF36me%+CBk-Zl?8z%a1x&_*rcrLW0WjTl9ID%$vf@Vb;0fX2Pkv%v zW$RI7TC*ND_3deK zN`!%pl=Jb7NIy4)m64=%(7wE~A&UR|nuZl0U76~o5 zG5MWvUJ>wJ(pY*{$iX6U_No%K%WWv=KTw<)9ydeDM;hs!W*`^&5(QT~WhN025=X^;uRd3CWP<*SO8q^FvZffd3si@#+i z0Vy+a;Cc&q-#KPd!#JA)PQ6O?sK$(>PLm1|ICM^CIb&kyioev%zM$G&U@|1RAKv6F zlKG+3bgpCWPRtayrdY4`-Q}XN27z2i5_DVB6MkSMjmeV(V+yNvXy{KO_Ncc8yHGIm zf2!ExSd2?wrvARJ=bjXg13zGKEGPLSaa2}xKCQxIxzQcwx09k+ix?%BFR&wuLh5HF zuyy^;OhX^x3!X`mwHBxPl>Amz^7`Bz^INI_{jYoPEmDW%7ff7`Qi;AqwqtGjQd(Zb zdFk@AYjTgkv2xh0vC0=_(|vOS5iG99&{T2ef+b5bC{f6U)xaR2Evz%K$l5SsYZxnn z#WO&@&&1q znNs#3c}0)MNg?IQ*OV(ia~$5EH(k9BJJ3BY@!ATk+{cY7-TQTh=WkAa3cgX6-LI(O z@<~swx7h#NF5J$If4HI&p&mZ(T1~-ak4Ef*d^O?%!mNRA?$K={bRG6YJ)%m9Qv0er z0o*Pz*OwL4OIv*}))b?&MRvd{qNhe&YX?aW4?&~}lwb5uCD1@(yFF(-b37%&VE;o~BU$`|%B6SBzYxKH>`DZlYx|F3soyCE1pa#@Z(itHTx|8* zuXjrR(^h)ix}SHjI76fS{63;XadB}mwma9&S=?p1@XMgy_$ez7BQI+y+5OZ(>?oyI*Hlb8rMGoJU+A)gte|&*_bN13 z%u*X%?lu#hvi-_@yrwqKJ$7a$CFtR_giDD#8*3*?-fHq}s|a1CMscB?EM}_trG#a> z=_K1^>XEWFjfv((Nx%x)U#6^~@#caD(VIS=r2C1t1NPv6G2xZNSU}b$i&zndz9c7G z>QN3cnxT7Uz|n$f$$`qz*JaF@@7vp7l#y%GuNkTgE5K#CG`0qC^ziAx%@-1=X5Z!2 zaCE<})_i%_%MB^tA+$MW?n*0ikASot6d(r^^Ql>Hy!EVUE-t_gZ?&_(4#D!QGerzkQ!HVcM z)!i)d-K8mvfb}$|WxcuZs2&wRZ9(mN*yVXvY#vKdq${VZk-S(n{ z=r`j-=f45Wn$@O`>dm?|4*k>~ORot8j}~#8f2UZhjt>K+ z)1`W11CImZJH*ZI#RyjOMLVilxjDjv@u>;FuG*bvII)LTg{?PD^wN8S=HtM$rj91f z@jq1Is5r#ZaTbp-jF2MHL#a{~fl69Lz=*N*`hf2&;9xf?Oy6Ec(5$q$`I`=SwDS;J zNl{*ze7eXhPA^64V!Ur;Zi5E&pB`@AM>)Y@NZDN57Bg&yP)gt2>N>Bq5W$O+m^hj% zk*A(V50;7v2kH6oYC5|}p{5tc4@V{ux$)hv>}MXtA7v&bs{C}OzI#YUak-PePO26< z-DkpkskK(mVdlR}CUHF&&1?6hecTL9H0TTx9T91^$fqahJuIb(=+k(w==GFDUKg15 zKgn1TL}u#G(1^I5l7s&<`;kk#pF2_f{r%q@*Y=(7oRl))EqHoIZcx#>EPWb3u>;nc zDfUaM_l#gR4(BcG61nK&ei8z{4PVtk^Ni0k@vEbG?e)!;;EJgn=0FNEBS%-tOc(13 zc%TVcH)mRTND-aDxfY~Q>(W6VHp0?Cy8*h=DpLtoM8qa@@`Z!+h4IwKWUU4R@z?wf zJ$?}SWT|j7CKH20OdicWvYSr86Vr7hU&rGuWYLI>908c5q%jn7O$leX&b87UyFQyy z!L>IypcnUh-D}OM)L}LN;y%{bKY8<5S3S|hym*R{^nP2Ry%Om;b8iXQ6D-Ysg_W`D zXz}COL8g9<30d+*%%;9Ev&Af}ZC!pSN~~~ag2i%BD@0pUD_>(CC(niea&hop_1I0k z`$oDqN2q~`!I)IhYhqB_+%3&&D4nOJvl7}*MS&W@X_w>am+D~f(PymK8trDf2}Zg| z?MrS#yx(57#ckWCv%!ThD_fGn50t$~D^tt^0_i8PpHvhG)iT?Q{MH z{J#UX7S0COBnZGW^_XLpN^HxA(FiGmE^Hll+NsJ1JAArr!mTsocaeNJDqP13L36o&>+?$Kq zDTX)Z^-`EMomawUkABq_>!;E*pWo@A&+B8L02S$inYhJl@uCQhTp3pI|CJ^F+XtX8 z`^Q-iqnHDw{@2+F?LsM1NO)WCARfD(!+FyU6b$=0-uC)Vt)Tky^c=@$x(ADOHcV(w z&+jDq?S=~?f-c&$M9s5t)?iToN&OrXwCorjZ;hw7W+!QlKJOk?OEr8;F+%nZGmA#j zSSosbd%plTF6SnDFiM{=gHJ5Q0&qe)otv@tslXDop0YZ#mVmYY&xiJyUkA$t>Ln#y z^Q;3(NoqC1qA6j9$T4`J{m|Qb=xyo}b2$J+ z9anu;m^lorZ5s+^)YoCd2}%}92d%dNPryftAqPZg@voKeM-39rAF#x3rHg1ZhVo0! zP!!AC2J+`YgKk7L6zfZtu$4~t2@`hpG$_E}ITmi(1gn9OSKR{sVO_U*sRYVyMyd#q zKytx}atK>Ovle$;%2k3c=t$?Elvkdv2Y!Wk$_o*$V+!*iA_`M4BA|pFCCl2AkY3dq zb9C1@xe`Ar3yK-HmZK@{V>PfAoQUnvp=5rfq0K!SgunZh3XSm0t_UP62`s1o zhpqRHXT$y4$L$$ZQF~LXC5jSTm9({smfEytZ4zSdQDRk%+M%(x5?fV6?b>QfZ9!|# zU*6C2eV)F*-{0r2c;)^}Ue|q}bFOop>-b)E-wF2`DXok9-1c_+ScBbep0ZgRXML>M zVlLA0o-lzHNqI-~i&%mjKqIY6P2aQs#y-3CXwoEI(&6@z%mIYg$Cu5fWzJQ8rN8gMR&42ulR)8Hz{}6qwX7{8G|8=9TPv~Wi z#`O;{DLMtE1@!E*>qcI$*A>PWi+q}o!&cfw)P@*zXm|85EEive7l0>>t9Qv$@8~?* zcw@q7f-_MoJ9Q*{pg^tRS}E2}k9oi=iUuGF_HZ9y@O${p#y$t66-+ zs(4_{mdSQUtf>P&f*@_4B50cEsFe0Ixoq`I&HchgOX6&&aJRU{cZlCCldShFk+YqW z_W+X}fN@bqOWbI@wdezaG@R4Lc9ZHX^J7xhRl-ewR$FloBD&=dG|M#1i_I;u+15_E zAN-=Z9*4Q7Yg7`>-gz~gk%pJq$YcD#@TMYts9av{CgP(~u+r~lNxs)a=C$U{ed6jB zo-mQwq^W0l6~hMvAH7^_q_u_`o=9_;SHJ0djU&OtU%r?Q%{$ZvwVx_x`vD?27%sUx z`P^*8jGKPhpo;3%Wp_Wc4~<{n*zC1-w-o>03$nf2_6SB3uKdKFBD#`Fg_hxs;ajJd zJ7%;Zf)M3<#l_|9t%sD zuap!+o$cU{q++!ycVu?yl{%UwdAlld$wnGLP-La*1?GY_v6oIMhArINMYIthLXw8O9*a zkJ07JVC(NSAv?Vj`-vRNFZX9y^Y1AznwOXS6bigkyT?4fh|qi_@P!%KS*|OvE{hYxkM3 z6>UH=^}wu0@FYac3co@atfzPPPJsM?LF|@iK-#A(*2Zp@gH{qMwW{vuT!7(?1mH$c zQH(H{eaw>20^@Gr3l}RY?oP0@RVX*}CIS?9J5DJ8FBUu9{Q!k4ce{j4N|j=x9H-vb zxpJd_xDOyG=?hxax(OED*kd+Xyr>OZX!B0hLbU>EMXP~y)sHOSzz`z4Sr^PhZk*Z6 zglePl{2EMmn(`xa*0mX@nTptzBTI%eS?F~emi-4}QR>Ev@sYdbf9Se^$_1GdU*D4kgTxj?zN z@ID+IMr2bj-&v)?ja!+jL%lV*yNaW`LGV#B_AhRa2`4f{^I(}Lt*K>s&5koi_TikN zhOAEFSD&|ajZ9+>1I3Nb`_cg40xR67tq|_VGwanKg_MHClE>HO!4sLo`?Mi3x<#D8 zR_v#T49t*-Y1O)USTS zxx?Pr{iMB-`QN z74sSUOqWyk$~{nK{ac|)=jjmATpvcIy?c*rY*wopUSC(O*HAJppSRq}n}7CbHjkDG z^jkOUL2IO0TPc59SA&4#AL-n9(|_ao|DZfX$s7mopi7BFW*n~n2j2L_c;05&GRs6+ ze$VdYWZX!iNp5pJH={CdWbiJawf981G1>p>$YVDan%%^_7j^gcSJ;jT8td2y*1{7I zj95a^Ecg)WwDKG8?8Df2|xm)-umG|VG))Q_)a=G zIe6*VWCn!Zk6Hne16~9SX)6$D$rsh0FyF}}*N1_A?ab5E)oH3SP|-H&msx`VA_gsA zdKu88sR_SLneR0nH=GQSvOvES$Y^Yx7TEmB9Xkl95%B$_6W|3GAs8u;>E@!!oOYOM zQj^OIJ>PUb8}7<4Mjx7-b+_|Kn_d&_etRdYJ)E?1OGL>gUDf>~K(m`>^dO;fLvd@j z;Hr;!tEu#M(5Z`;@`VkKe5`h69UhRUVqCm@Bxd=?sg-9j@Z$bI9%z(e zt+%p|T59%V8~~JtP`ZKQ8aE1ysVRWwxO)-il>v1X2+~OWwJVxq4|bO*9BS{g#W@K3 z0jrHv_!Wc2tQ%b3HRySkrVmtKRELV}I8UKukG--4_dZ_8phJJpLS047z5tW+=+If` zX#0^nStJry5BFwEjQaLi&^gWA;uZLzix`S{*$-1vRFR?1Mb(i*W7?WI>n6=ra4u9! zLVT8A{jp&q_WBR`xPf{t5e);sx?S!pHYO^K zWaoOx4Mv%7ZRU*W?z=z+QS*6oCTOhY?&FE!-fyu-C?Cfz9vQ_^qsGGnbG(ZwBe|PD zY==Qyd7=s)RWK5@-AQK*ljbX>2mjBb{113fMTEF;2IhzLzs%MC_sL#j6iR%bju+wb z{sVSu-ke?s%z0^6$qqs5Ha=e?)UAkc+u#k{$D21(yM?eN%u-1BGLZEP{}q-z(IhhX zqtxrqcG`z}3w=Q(wj|!6?4<~iE1RZ{m@Fd~Mw$=_0{IUj@X-K*iR#o4VrVG}afEw_ z(Oz7VlrMhxHam%8C5z8Ft%&aWb{R}Cx_wt#5xoZmvvd=9To_bDfG(Mt@DKcjCXz{! zONLC5_`JWKGjS98%I%q4w81OAb~QwmkiR{RjA^!_Th~lwS~w$VJwpXxNi^8fFx9J< z>Ma0if>^w ziqV6t9m-mU#4c@CIGfvtS$@Z44-bG>zS#!>x5G8$qx{xSUmm0!NeKC8ch!MjQ+Hdx z;+zA?oZ>TC882LvoAy-4$YEhJ#He?PvZExvh`T6_>Mcqm-aBq=f=*fr(d-NW`4Zm5 zdfTX?TB|n-D{b0W32wi0Y<1PPOQ$Y$t98*z%~=TTE)llPKCB$9aU<}1+QEfZVuQeC z5&R*f@cE&Wa+czK=#TlYv>%Dajgg;u@(__zG%Cu&M;d9J`RPx$oW10Jox(Wr@1Mzr zj&Dv@Hy)e7X}#q%ea*-1rkfdGOumiGtH;uVoK^)qAK&eAnPx9PtB|&c(gN-F&ZmUM zYIz>Y9>2S+_%M2N(0YiaTf%PsxH1#qVx;?jGYWjYB^m=|%b%Qr?WV8(W*XFQiERhc zLn0^{R~G#CdIr-j<3p(YrO1|PmhbyD*dSZThjP&Q25BUghe<6JOvj&1nmFUQbcag86rLGJub2r+wF%$AiAn2Krfj z(mo1iR?NZUQmJ?{zo`~rO%yrs)pN!eG26fN78SY7gAd|>e8lpslA#rVm*+4%9<)+2 zLXgWG{aY`M?I1bjJuRoBVv#+T)Sk%>7B zT4BMZ^VCkg-2hGB>-!gZ&lbKRs<2;q|15&NG}#CU2!O5&@|rp2R+Fv@>Ioe2_2W#E zFZX)B8*VR;`}ddy%$7iz4d(Agni4IO1(!h1U0;S<&4AsWEsi?zNa}}8WuE{o5Ae*f zO}zVZthV2O%`UIA0Q$2;sa~6Pml%;7?iWBZQL^o1)ybalDPu{806`gegKs1Jr8iaN zoZP2w%`5`m1JgYSjPrOPk`{{%lt8^pVDqt^L@Y?XYWQHq{e|X0_6163pK3lv-D!$A z1=Q^IQ%f{2kddPtKWZ@RnfDR%l>_NB}7Ea9rrZ`tz0=4OG! zVorEZ^5BvVRZGy(XxYrF?!1aOxsmy5Rm(?Wfb#3N#FIgdgufW1WCIN&|2vWWKLP)L z;{M*J;LgvBT8^m1zsGEEDRfWt9z7y}%WeslziRj{5di;IR%qY4?lo?6UAHq`U?jKe z^}IT$J#*fH0}n$&S+u8`Dmt1d%BAA^rg6X-X#LG$TCWCB+7P`=yxWhh3HE_&8&LhU zjQ4@Bk}8B1P}JFN6o0{&_FG5UJ?vdp!-BRR@G?E-JQG?Q)jTKNLu=Q~q1fc6=pKrX z&oN#rA1tU>zZvEQ>VNtYo!vVc z4;dnCK8c6b!Uh(Fd>KoxNl5~*WJ&~7)IDa;bF(R5lt$+IYO9aSkhtxz^?#yn6f|N! zf$g||oM~5PcqoWzz87$Gp_#>ia;pPFRtzseBk-IGK&(I+)i!7gi z`^N1rb7(Qa>hJui`{biJKgB>{w3nGD{ut3D_HY{CR)sfYlx!W z&%gb2cF{!}`-Fe#1E^TZb^#(fB+G5sf+j0Y@{0|&`c2GM_9XLT-z$;8A&ezrT~oAc zNOr#Ej{D;(8GYFt*)68vyA~S(qP{w+4Lc!Hm&OTl1{1E4DGHQpojaU3})#+Yh&hP;?W5kQ8*l<`0mu7m@4ipV-U>VbOJ z)~Wmry;w#n&BzQ`1X2gi<_*Xgt6V30dE!>P7v_Q5@_{!jCM_nMY|iMYALMBdbpuxg zj!1$hEV@b-i4ZNsdFi7gUAku8bL>-NTF-RTT1d$SQ|@BU3tf6f?oDPQc(X;*50LJ! z@2L#i%(ZA z$xV&-_KVlHwvTpivQ%P07}`h^qHsaQK~2;M9fQMOW>+PHi06x{pSHpE7gp?zQzyVH zxve*|?xeSvhV{aT?0dCdf77&-`I_X;%`B^3bIEJez2?WeyK#!WI}NJC6QRGO#@@^$ znjISCcrrJFa&v5tup+^`P6BK*Nz(4{Aeq@&(4H^=WRgZ^v+1pJbFFV@%DdRnnXE6{>p{BuMQCqf(F385P~{t=hRA2T@Bv2;$OX z&d-UqC@ewDk9V zSxfzy2e|7L*ga2%TN%qx(tv)>wIz=2tWH;I5Vr)(yR=6C z?TsVVtm%HzRNwJWOFqrf{qgxaaHV#v&*8fzyI(sNgbgPSyK6hlVARz^M*B8&8Cx68 z3=Q4~&T4hL*Ba8`gw;uU1C4}GP@%U^>(y7Dd6JJ@kSm2-A8ly=eVo>r<2$Cw}Y zEGW7yPJYo673N4|M@4+o5E|sKhXb5ND9Ql-$D0&8{qt+?OV@CXXQKZBn4K1{{;8ri z(PGkKe{p#OCmJ?=J zrNGhiboYfOuil&hya+pVR9>$Mw%UeLq^#+RON#xeKAQt;CQnL!4^%!>{M0XR$H#c ze#?a(iSmnY0XvZh6|9n)XkUK9pSoogZFDx2aJk#EXudet!bIGHaoq1y=V5>5osQfj~Jy;ia&ADBS?Ww7Sq(s&0=V-C;^u3cG^68*5g$p+Vrx z#{aaka1N!c5dv9EdKXR7MG$I-?+)x^z_9G}id+Tr@|5t#lB!u?Gs$^nvb&Pon+QL$ zJtiXRbyolQ{e$CvnmY}l3~rrfqk|Dc)~9T>$fAp>-#L-^#1a9Skp0mtk9GX{quCM+ zXBj($fwODCf12SZIY}1gYXK-(F1mTImsu_E2}3(YPp9<*IrtMh55Z&&^7eo>&(bpz z?E`CeJ7x*|SY>`tj6bgznmy!M;E36|afIzgdqumAWIDpHp!f^sU3Hys4EImmKvzVc z004dRu`cO0XJnM!d{{YetM(_Q-@kq~I5CPgZ}7ZbY^U;@TzD54wb*&`R8%r{kI#-< z_86Ci3#eL1+w<*z-FDb8P4G;}X{$SV8d-ElWg*yS3U*1Ny&xZ$L*#h0!aE{I3A}0$ zNt)8X52f&^ceJa3c#qa0;q}$|Z8nd{?WK3zdP=rL zG(Gvkq$D5$IVbV!@qT9AJ1-w{i*;ZUOka37|6lCte;4l8$rNO7FEkV(D2KnNr$rQ5 z0mm~&b+%H%!otgoYD4L1-c$OsR6Fv^r>Grc-~-O!8K`E}*iShuk-uvF&mO5>g`#H* zkr`5?>TW@ijOvyYXi1)8&VkcD7B5{S6Iy&r=JD_W<}){PR@o!Tq45@4Y)yeUrGT?9 z$>5LFK(Tvu+2ofv#gKVP$*jOW^vyQ`PD6GUPxll5_Fe(6z5<}&*Z zWV(giJiVG^ac|SYCV1)qOMQEGq7aVY?$ z13#weTSCqH<>F^THnppkydak#VlfAwF=+hqzjs1u%d|AaqbSe~`zgAv2wpN3G8 zf#TIByExfqgRU~Rs+dF-4h584T#m!I5tWV*b+wQ9t@qy!QUH5owIi1rixJDLsiY&i)dyTULt)&ZxLB9ZemrZd~Y8u6s#ib?{S{q2G!ssFZO8 zLUHF2?thd>ne>V^mm^wyb?lI?CE9|$+fIE{n306`Y5qy}Vy1=jM?ff%msU_fm1K~I z!zVKu30NqJ^Y^tyaUbtp>Ln43U_}?saKS)p6^5Z?KzbL&$e62++`0Rm*2&F}ZPm)SpYT!pmL_=B~ zA^OQV=|H%IU-n>MK$aY$eEE;U&zX$c69;P>Wrz!%N~szqS&=Ld{N`qho5Xs-BG zdOtvHBEnoC)Z)-RM`=fT-5q%+dpKGVZuLPWDLAzZb<-Cu^JwqowfITCdtol6{U~HN z`GR>eaYl9vZgcf)X(uq;%yTW+q_w(_N`vgXo&hJcpzV87&}6mmjv_(^Y=oxcIt{<|Dn18F3;63KU;>K8!aT$PxxiI{13+qhkwSHw~)YK};d9U+)t4Ksd zPdHoRckax`+nsJa860wyyz~)RYc~v^5ep1S(Uv904cno!K%PL)HO)EbA3i4bUv0dk zH-l0qemf7nw+#N7BzVUt_N_1lK4lrp^Q_K};91vqn9-sunBnb~5h7i!u|L7Lv#DQ@6J4Y}mU77tHT%mz9Pg zaby6E9clLZInitub)ai;NBqS+ndv99e)z-gDzUifr-1FO`Gl@KQMuR*zBB=EiP`&1 z&}9*_&5r~@K|z5Z{uB3ki#X7NnCSCWJFI`=Zx~9QSDJ1qe0(=ofZgi*-PSJs&}Xnu^N zIh*w??oe$t&q{o>3YXfarU~ZNMPE75n=y~;x!_i{lF-5S}CT8>(N$OU-- z^}KAIaK$?+gdF|oq01DuE}iZWR6i?z&w0nx(j(e68jaBrZif zJAr|A)BVRoEI$pm5g2TG3iAUOMZP)IzE$Y0R2Hiv!!oLgiVjJ+ zU=P_^@wPowi0cyUp8mwf{$~@bP#9Ig4nCUlvyjqHd>APbV>Jh4kuymhL3@ z#akIW;fIMyoITphYPxW(Q(S1saS%N|b^wOzTYXxrTxI)M2l_PnB`wYXRb+ijQd zKf2gp#79~akL=Se(M3uiD^yKQ*#O;3M*^yUiO7Ee>ckMoQ|^^Dnri($ruHEIq?r*q z_)&VbwqLaS>SU2d?jn$>^rMRYaQ<;sNZp<f({@l(#KR3#vc-tSdsb~%vQj)PHkR}l8i zscr{#_hh#ICwP^6Ryiiso+UTJq_*DxI^%`H78EU~T3C!n*aO28g)CD7AL-5FUab0b zi>=!_8!8P(?Ey@>h_e=*rxrw7gQ|n>kJ!-@Qnu@CMT-QQw8Zv?JYJvkWxSPzC9pm( zCwgG`lGn1a8qqgTu1+l%#dAfVZ#r)gU)?xQ#_K3QFy7y)Ur_vnK<))sdAN}B^eq`R z*Dw22q?^As@7wH}4GX^&B-Dq!NHc{nw4Zqpt5adac(t2|_F5a+mY0{Ca1*RRmn_C& zHh@MT&Iej%w|)$W^8k|fQsXh$y~|Fz<#FhqW#nG0_9rq&UJDFx<*!uDaoZuldeza>;Ak#=qP0TP_xq zGs$t4Gha?mUecWRY|##yTYB{h{QkC)fmwAOr)4xc^ZeLK`3d#~Z+LwN6P=E1#FV** z$AcW7m6$NAzCO19P@QhzEM8UA`+fy8v+I~91XA%uwz6?2PpOM*X+W@H&9=~% z)t%i&$0>_*$-Uosij4uosttQl84z~4-@R9_jqr7MQ`->ILX)mH!odYHteZ_(n%W`T9g@LJw60nI&){d#{8TJ?cJf%Ev!sjT%F8_Ct1 zA@M@(YyWg2Ph5xeq)!w?Fpf9{`A)jb--Xjl{iuv!MoiK5_|BTRTBomLW%w9BmEOcm z6()HIsVbbz@>YZQC#V8_D}E_aO1@o}F_?IcXLu%bQb5~;5Y!(&68e#(?01?k`(sp! zEO=CY$?L!ug^q7#a(;PP=%f^SfWhN`$qJIDx{;CjrC}AGFb@BoXQ0!OTemI_Ppq%y zn8=G?F*}i8+lp(=SqwQ?Mf370FBc|JSNi!}xqep1Rek?2qxg@WwnHZc`Y=##^&VwwaqW2Mj)-S|Ao;vr$UqM=rcx902bYCs#a z0Doux>OVQ-OOiI9pu3a0Y+I&GPZ>Kky}GoJ?^caVNQX2#CK|p4jsT~y7$d_*k@(8J zoeP1JU1luyVW!h&i>$VHFB_~SPmpAiBK+wdvg{pPXCMiF$U)l0;#EZ|tiK~P`^?Mq z^;`+d_^98Uwn7}E-x8|yEYe2HH=E?h1}Pk()Gd9xw9TkR5&<9ElYt7kOa{?i7BI2Q zws2alXGv{nC;Zt@?}#UurF7r+W?UK&X}d}!T@X%Tj`LRwE zUdG7dy}=PFzc|>Ad}!i24IovhsM19uyS+NVmP|XG_mZMMOyKtu8iQ4B5d!b)QMNAh z1w%()b)qsc%AIV_hnq~KFFut31g zD_{5m@B5}#52bKo|96F~f~kY$OnLc>VJMyi9+(5l(r$htG|TeLZR|+jy^o5>_L{A` zFX&DS3)lveUh~fg-~Qz#+S0WnqbuPjf8Tcgk|BgUSzG;-tZj(GBBJyvM^zK)8zBcBt~w_3|-~c@WurW>rT6ju{(&bwz|KEQ0^#rGMgKB_kl`PL_$PD(U=G0 z?0&H6nUgNv&jLfKRr;vpDc0_tcALYjOz8HJ=E?XC%bqMhm+5|OWNep1>#@&5fI4h} zIDmi^B05+`Ke=?FU>3<1Gn3UHL>O1-nOEBJ!>yvBlYxe{jHYUFK(X9b0>C z+7$c6VOa85ywrVIc1SkpkemeFdjAF_bEM>+Rfv?RX#=1+C2oTvoKM|mgxTD`(gwka zg`nL3|7x~Vr4ZNb3)`^jGx!^Umpb+G%F6*bdG*S4=db>>)vJ?*p8WhWNqR(+YT8Ad zVzV5->x1$oBh+`CHeQgp`*7^a(79+fHN{JGAe=)y{Lo72hB;3t%FKZX|1?bDsB08_ znoc*vk9>7p%i1gS>MNY3F)V6{9QC7I_0I9_p`n~Uaj3kGs;b}?w+rxbJmI)3@&>5Wu1?d1vFS(7Aw0JO1ke&9|9P| z5l+7ZMXT#Lju((jO&MXIy2-YNugpvL8PYRoy2$!}{CW9Mj$qSB7kMXy<%coIjpDk` zGI8ZziQkZre+0WXVD^$n;Y24o)LFng)ua@)QH0)Y?q51Sxs99i@~^MGZ2d&`h_#09ETFK1kceW_ z?8j&&5hql|upD!eZo{7Y^dEwY_~4F7n*v>EH6;fAY2eteeCy?fg-s;og70acR=TwWo}7eYxc&ntNkp zhT~<1<0UN20Tc?eDg(+~Cd$)7Bjvw2=`cMa~^c?&OYYGoNd@S|@^i!ns)>nshIB3_r z>^0YuuslG|`hh8>xz&axbrZGvv>Tk+3!+5g*+~T~_~yHYeIL;2$-iO!$+X<&Xv1gL zyfFqL7r$|UQvq3z*idKV_H?kKpNev~_=7Y(HfcAO7OLlgp`l>b#_(muXevY6+ z4gB z0(BBW39Di#%9H2zb;37BEz%bjXxeIjd2iB5mk}q~6+tW*6aAzm zzAMA;ngy?&JUo7+CRr%W&5ji%EQC#+6gv8X8s5Yx4KJllO1dI^rV7-6W3FMNCvFO2 z=Ra$%9zxKz+A#8vkdW!xH~qN*4&Sf-hQ$BP;M|mjaaBUF|Gfu7e+fAmBl>iZ=iSfF zKO?X8b_J^>4zN+VTM4W8{g!b(m?sLMT5Kg?_BYF_pxH#5Gfj)#t(s7)+ap`z#Wyi* zZ|{O^coHVp%YPCfcLLz=B?{nCrA9-+y(Q+P-AZs3y1m9AsZ*PPwJ-%sVl{n2I2P7UmfqA{oAvvbuA!6V-5&uQ#1k(7QFxD5W^w|h z+mWSWd6|$0>Eg*^#{;=+hT6XzC!feow1=c9<;pDzp8rtj?OHPkDnf|S4O?*)uzcQJ zXw7srMjLJi5}}$QcCO$jc`+Y8yO_eT?&+{U@`Z)EVlx&_xWSIN?B_iZzM|ldD>>rx z2X-5x?Y|SwP)UeH@XIz5=k@?*l8h4is#n8k{SHq2f0j(Y^`^K}t1&PJR(C4LI0xrw z?DTOkX&{TzpPoiK?QgSOYL<*u+;{_RbgaoVs5Okb<^=3MnST582K;(qjHTFiAe^=^ zx5!Z}?4P{dwt-}U*zQIcf*QT(S(^BT!2~!-yOkp{d~!|{oTk4yF-CI6DMkTJ3)>&V zD+~R+PFfq%y1Aq57DtsmE8&V#_uvbd09A?8z%yAY6R#b|Lz#4(y-VLvi$a3Kx*3f$ zf6ow0WXwq;P6(c67U{nFZ8=3p@yzM|bCk(n-289yPkjql46t+2Jo@ynph;RqfSYVr zv9-NBS#I22lRW0y9PT#>=%%^r-8fU>7Hdv;9!J6gwI5ky%$F;f?f1GgOHi?l;fj3P z8Nx*n;O08_t&+7H5S#>zP+`Rv3iY8!Y+#vfr020<%j1@BtM8Xr%M_!`_9^bg<{>uJ zr`#Y)fC)oV5vR|{skoLOr5)onAH*xSUj~rdEqVe;7l_01RP64n%7y}LXQ|rOhT)%& zJ<{D%Bn1o8!J%C9g8IL$7xhaCEp z>f$5|0yx>j6;8jT>J-NAjDaH>z{(>GoZq)mF1&K!_-d>3ddzsg-?OT#IKRA1kEM~! zlZ1#2kEF5=oxSgS3ph@?wuUv8_xXu(YCthUSd|2ydsDaPX5}Q)nm0b3{M>TuHXqo; zEht#o{I?q5LXg_f+ra&CDKj*){pLG^?yuZVGS(Geq=5-{n7YLEfH^HvyB$7M9@omc ze!WT{DJ8W$>r~Z)Yo1*7bJnx{3ow7{%zX0Y`~?@&!r$++gA4vWVahQkQ{rW{m%%1u!Pj#dh67 zV9elQUMEopQ2|>?1v!m{o?eTF81WsZ=g$1iG=pO;kc2Ym;I^cVQT2r$?@ zZ1H4yz81Jm%0XU_9m^XO*b!@M_4M>~pWaYS%wY@BVx1CaTD5z5Ne#Gp0yijJe;UT- z9BDzX#EPTmRC(d@{C0(7)M^TTSwz@gD#uf_&m^Ji;vYc+zas2{ zgyz+~;uhy^Z{>aE;aBd@7Q^zJ^IyHQY zL|IuL+?Y)GeyLFZGrj%XC%I))y;PXOaFo?~9WTR+Tisf*F#}Fa(E-^}?uB+{X6((5 zpShakSZjDIzhp~rsOsw8soX0OHO4hW|?6bv||Cyg1y~GVE5aP42Gt z^f(>##J)^??^_EoH}1L|&B+o!9L))Nu`B|N8i=6%fzU6vVpWyl+~p#l$NaLkoMP@efq%e2Ec$ zeJW4}_*^uCHhm*@+TRrX_PI{>CR*IL)JCa1|Bu%pXFOJ&ihuWEIxQGxLrh-2>&x~X zXj!yK%|xk|I zQ32c#?Xnbko750|cOjA8p7C}#WG-r)@*n(DVCPzWp+y4|5UNaKe{)@<8zHtS1NLYq zzU#*c>Z+ViuqTt=5T)uYh9+h?5=~z6fSTW{eDuJ}wcL|i^8(sS^b$Y0m82|wMEm@3 z?RsLiZ4DmtRwT5}?;&9E789XM*XMb|)8lQi{lVDvYAZkXSF-nBzj>2y(GeI;u^a9> zHf&!VX*u}P1clsP`{??O;bqfYM6KmtK%>0t#IFEoJoZ6;a&y}(@YH@H!RYds)ZU?ymXSS@wMw3~anN+XaAZ!WX*V#y8AmXZI2woAVQ~Uy19}yj@F?TRx7HE{*MMn zvJj|;Yi1Ci9ILM!YLCqHgn8np5ZEjl1KgEhxqI{t$E5!9`mzHpf`=Wp^08NYuH$DL zR`*`r$t*z#^o^?Pr7Cjo2>ybjhDb=-=l-bgA-5v+FLAH|?$*DPk6l+`57{}Se73yZ zb+`^~wO!XZ-FcT=1M`&oyd$h)2PTCN)VhsZG0VWdzXh-quB)M|Oua;~rioGey?j}L z=bPz@0nPXH|3f16ID}C7pV7Ha)kLAcQqIm?aLAap996*iVb^hP2fAJay&Nhe(78_; zMJthH<~Y~!*v!82ROUK*!JIaVL)t`IKi<{cTW7ugM_+x#22}RR$ zak$d85B7%*8Ji-3pxNNVXNpwzP?vL*oS zW3A!5^}OtgdU6EChT1ZwyTBd)L{} zAMms-LVFN;8JH2X-Z^tZU;yS&?~vMY zpM6lTWXG02&BBDaMR#*ynmZFm3OsdMa`p+DfnnCMUy%I^U!h97-Cx}i?!PzFELXnL zCH=U}f{eXU46dvcy=PoYwAX&5Q*PFXpCS;~89r9k{VUDlT28?m757(j{x7(6AaE>I z5SV`k{4b6n#3HW~^Az1}VQ4e|?a|w-OLsmF7(evGjND#0ZJ;1D&R6{<`}F-uV+8(f zP$?{M8d>>Z%g^b4cm_}{>~l1)WV$YORsf}p<1m{{{6+m-_ritbNd3nUViek^KDBef zoi*-M!s>1|vT?N4!xsPAy_#fxce96QDddBxkt*kUue{+AvE1H3oS%G~gO?0wm>w1O zeq}?$<8H|}c7WB=6k}u%8z9z=?``@Go59tc0Qiz*@vLh@f5NpEa7fBvmqg~D)N*jd zg`4S;wc$oW=2t%`Fk$Wy;IX4L@O`l}Wlt=FNHxwO_1eON@a*i&YV})u(^hGj+u+1Y ztHdTsfG$8Q%}nH$YKA97?v2u@B9n)!m~%*nUNByDtv zyNnwVqkVq2>9-x;C=6uV6RjD5BjB!w36ZoFHX7t>wJpF+c`Sz)O>Cxt+B>hp*Se*4 z@p^0sN0Al9M+Ck^m4?_9-A|n5A!G7aPW9To#H6SDerxXoh$gZI_vqVn6iAO6OTG_V zeKH@ac~4Z%T%V+oI_dj8BF{z515r>$t^S*sg+SpgWagg710;lKaPtVI5 zb{Di($-?3wV%VKz`+T+43jRKC%#qD7@;W#^$0J>4eVrv$NAfQB1=Bh}p3 zCmD`&mD#RWm7&ajwa{x!JA$mY5=LUQHiiTsQ-wY@IFwi&2&YL z4fi%N+J0SZb-MRkaMuH;oA&Vf0h_yEYxy;R-v}J})i73_YIDCY6&Ha@9Rv&prGYjA z?v19yUD=LB8iyB53g#U&MH<0y$IeYl@JGj?WzR30;j|JVQxahCZF>pw`rRm8PwldP zm-Hy0S7XK(tseY}7oUl8aoIF3 zRbk%T+-%yeY)MQ^#6vFeUz_CJsQ3PFk^es|JeBnI1x>EZ$&ueoG}nZ`p*70slYMfwj((I=^`gc3&- zyN8=DAy;EX@~(A@h6|oQ8jc4dtj@bNaVo8eqD*LaUqEO=z%kF5bgGqtW?*zjfH4m+9^(6v3SrAwZ(*}40UWxH+nGY?fvifDL-8_9?{ zKGKI)jc8^rkv(d9{hMg1hbELF*?!YQr14 ztS>K{G)8v=y}qa6JNNA1yoFeg`e{##RF*QQ>c+YftE*d?ojNY>a8};$9RB%-mYF%m z9n2Z(o}nePQ%wGZBY=<-yeEp1nhH1~ob-Pg9dzF;p>mQXOA-Rq2JN|!ICpiiJm~+= zAdvY71a9&I8AX3Vz!pxSJH9Y&lc3@Umiz2{PGB6xv5unjDA5aIksDkQXi@wkMz~{} zhagd(wk)32Vnaqv2JC#`M&F{sHskceWSm3u`q+HjTykFydEMd!+afY5G&?|v5dZ(< z>np>m+`6u%K}tkG8U;Z>x*J78NlBGd3F!uD5XGcK>6Y$B8bqbLLBgU->3-)8c#fX$ zxxPPM=WJl_eXliR%rVEB%e;|lX$@E3H)H?0te_uWuiI&h!Id;j`ZIH~9yTM^zC@8t z0Y$OC2?pN?;+M{bqqCJqO`KsvMJ4u`6r(8GYOqNS4LQJTeN=eoL(KbBx(83ZA3!s3 zHMV=^Vax`)BzuidPt~3!^_$J_$>}$%6DZ=cIAqMsSQr(;YlE_LatL0Kvb}#o>-iL> z|GxGL5KMGtn7I7Tf}osP0_O9>Oe9iN)ZL%&qbY5EzNR0{PNDGhD|AL_eN64OwX>l=zgQ-#e0*_Yyt@2er$l5Ot?oGS+gVTfNc+pnL)dK(g(hSM+)AFm8OgAOp%%V_jL`Vx4ESo}B?_{fvso(axc*$+|lwW&6!$^>sPyckKqp z(J#(B`PmsaUl_iixpyIYc8wGDt3iqmL3p+6dQ2ZQ`CIeeE3B87&sQ>$d)|^VMyvK% zEpC=T6>`F6QF%gTZO-Z=Wx!T9GG)9}`C=!vwXV>W`RWQ&YRnh1gmY9CpGoVM15rOp zQv5XGtW{%kffK)vi0n~Y4QKVx4x$p#vJ>={8%A)Ul{tdWvw%4WO)3})pKme5eP(eK zCDlWMG0vWkG0v0VQp>X}iOkI7Sa zS3)fq7?UTGr)WOpbD0lqNmHAyS~L-T+Os?&&wR8(a>l&g@r}1=RQ_lL|aMaJL zLS2s?n*YRuHP%l3WH*-2;_;o*x~CE`1A@{w`in5X_-cs!{2165@9g$r;$}4O7ePGv zV%y;|f&91awp1`?8%?{^>f>Z_$Gw$BCyURO#E7UA?j`ky3qvvQ5z1dtzmmdKu0b05 zuUr3tF6*+99_Y?i)BGQ`E}e?Om-mn312QSCD~aqUTe~I_JPVz?ZlTPFCTxmqK0@h|uWD6d?+68r$F($*|l1N7uy zPIe!Jq;t7;e|=bUVeNGXF8$5O3Yw|0O3rN$8z)6 z`bsDC%+Ro2JG(N-1te8YW}%ek@wD+=?0)UdDt1&(?eW5jE#*G%gV?xFfbZBIvU2R+=9cx8a#ZU zoIA^P7o9+CRpliG-^qKQUJ}u{Ph8T7c=kUww5A*tb;Q&0+K)$npgt1~gQ@nIv)qP_ zUJh>>^{gO}PPqOK+j#8@9)4TR5$}b|Wu44HEC*N@3ZNk+a{dyW#QO zU~`q@DjtTjrfJ}X&|aopvVSmUdk3k{2L;vb#0Pi~D5xjld9%*B14YG0E{cwX{# z=o{PA+aU34i0oFreVgI{WVQETp6l|@pF+#DuUb|92(bSmb}cnP?757|(Q^O(c)#`J z?wtO?dlD_3x-Q!@Xd7`aVeAfNrdF;@-6CzL0J~H@>+?r=v?zV=aIv2nJvET=_mzrg zn-x6!r9L6j>GB&66M=9dmoqP`^<={eH^}#^_~uRhw0YHmiZr6FRS@kM(eJCoB9Q(e zO@jlFs$-Q|oN$3exIb9O%t$`rIaxiGaG+cIXDPm^Dv(JW!P;?dkic%{nZ?Zv~#OxO9S5E9ZOMt5>I~UJd&+rN%t$;qan@ zxu=^TD;tu&8XVQzA`|3)f=XJZS#a}=;8e7I-prGw8)Y?-73(yfn|JUO7rzdw&|kte zKA3YIeXW*{nlTXvLpi>MYiO(qsmqeQ6sO;!HV{bGPx9+9l3&~DKA@~)plv(MHt$EDM(iPyDd-^M4ADk>_bWn{eH-`gSj-t#c7To7?;!hO}{IvDEkuRXPC5Z-hBGJat)4y7+ei_oS}#N$nXSu)~oh` zcs9dRzI=VJ2sM83H^@<3-V^L^7i-Ly+B2V*q*s{=cy`Gn@LUv=S#FiOb1DX*w~&pn z^UW?8lJYQP9}_|-Uv1hJ#hGzg>RY{DKWNa~vAr+;sD%!>Tl&TBS2{A~QI@o=z4BX? zN_AJ#BJnw&h+_IuStt%|-u+3k>0lx%t{YdM?}pwr%_J^PL&Pw;=jZ1O zetATEwiy{1$*H&^5qvX1sP@eC^mHwpIW98^q|#47Az&A?Ms2U0Ccq<45kYfV=s!Sb z+7S=;UGye$$ujq^M&&U+AN{oydJGuo$q5+-^eiP+0ncZR)dlNs+_vO-#|G6#ivM_G zVIgje?x@`Jz$-Tm1iE<$r7XUl`0()~*CoaY%^S^o>`G|o35f&3`ebOP%p{xOCTs%eP=KHh>( zF%j3MT-c=>gvyj_TV2Dykl`=Zvgp2#4-gr>@6cC2ws+BByOzQsa4|p8qebGO+Es#* zGQ2P9JywDuOWs{eC!y)&Vx3+QypyZ0B?d!p;7F>E+2E%Y{_OytjM22qQP#9hJ?Y&S zrWGm(S^JH?!4%uernchS7o^YIf8fHqNJ&YV4@VdK1MhAk6AGM8s$ywgzuwI8Ms;1{ z=^46#uiI;L`BtOCnC+|kr#8=rkqb4PKY#ud%~#xUZyUe6y1Kget*vFv?b#wDM>+?4 zPd1bgbv4j-1%G+!FMn01n~s5jA#dbm0Mo#5{L|A#7zg;t$;lU4E=k_Gk)NuZtsEzQ zHu%j$DOC}!U(5Mtd47_L>;2E!3UGFP967sQb3-|R z+!RlD_WvxDTlwqcr$E{R+VPzLY7v`XO-%F*4Co>Qp9zb?-YZn!l>J83dtDT3It!Rw z#tB+NhrZcZ7H0)RHU4O7=UTZ60^|86QNq4FlFu4HMzEko?5%upqhhGc9_6`1Yj{Ee z^@4T1E_8vYU zfBF5zJ8P?{(ih7!ZpwH2NF}m%CYb`p>B&NJb2*%r#{8kRURPrIl1k|GTiHNNq20v9 z#Ql5tKWcroePE1tV8UD|oA$ekTo9OnvI83*|HsC|{V2N9xH#He`j!F%3jR(#KLV2H z>+HDM+1c~284R@a^*g!(ic%WSU1R4*wOQ2TbB?<`Zh+;q_>B`a#2`g?$<~GDvsv>W z=yIXTA5lY#`+F9HH!idLo5!tH?T z44f)KDzR;KcG7*>^KHRBSi0hxLZoB}MqND*! z#jLs;xq8?6v-!j~VoE8jb8Uv%qPuJVae-6%;J)v>ENFen6)XTG6 z54}9`+xE@7exPF&p~6WODd-!b@uHf>#KcT5E>3Q=bK(V%tldytCVY19l&q2(sy+I2 zW4?^R#(~6{XgK^5`Dz>8=wAr@kD3PQPS^M$_z9l`W=Q|@`f{p@xA=UVWB)BYxDjT7 z506SPh<*vXCNnz|n*lqC_E7jsGH z-6=w`)l>B)<9ES3G0F5EoNM?ItY*>}bPl_OV#3$yC6yTFIj(x!{jI5$!)f%a92^6>|VSCy1 zcbZvq(H%2KJvK(eMO}Vb3&-hJIDG-J9tKKHZ?F2Q@;q9Dm6Xkg6Yg-?)6>8}+1n0S z?BS=L*{X%1`e8Vz&nOA>pg!z2_@MN4i$ZZ^d8?s=-f|k}J~viERh6#u?H?Rc9v<{& z8jKkknKy1!;*z}4NXs&ng&_&fhF$k9O)q;$iD5Eqd_4Cypmq}{)wW@{@wIF1vX2?7 zUxfP?>x?JSI{)Isf0Nmv+xz%lLmEy!J+I3CnAMn@bE2+@P<+$7VlY5x@+=&|j=4vl zZv9fnvS#W52>kQR`dD@Wl;PE2KlG;-A$4*i(ahdG>zM6%ku9^nkNmM73c$=Yq<4m%8v8G0Gn%NGUr9Zp2M&CsZ zH)|NItxZb)%PYEv<6wQ{+_RS=W%^S;-_f2c%dwAH5iMoN=~&x}e#N3)lI0Om)!!V( zoM<;x(n3Ejbx^Ylchgr?YyS)rU{?Dw#h|s+Yxw0hHl6ng)W81mmcl({VL$!jj?MAQ zL8_=^9C}4V7CAXN+b}2A0;d76PdDZ;a7OXf9y=+$$xB1l_wT2%&AoG7GyBmPEJZ0` z)%)Yc6bW1x_x0O15?T*wuF}NsUr(p9ep!6(Uy=RvDYmqn6K}l_pPQ9V8 z2()vPW!0}MXx2}AbdvN^+7&YkRMHK}*W_JiccIwjTe&2@xX~TL`=7T*{@2HljBYS=KFQ(f@#+O! z;`G>dpuEXrL@(kvn*mD{bPv=i1q#7vt>nZU?PHFU0a`Haw3nS)m<-@O^KfB;~APEf(yMRDx5!LAd znZI{WP6pn;>FG7CGi3)Xn7f3yMtv(C#nRZ_lahlJ&KtDH-n+q3oj^|^x+&?8r zSAQ~mZxs5buIlCUNfup=G~FX)~(?0 z`&9e*Z_`6pRe}#`9_r>f-j5`oT7Whd5se@?G3`#e=7bIi?zv{Jp5=ylCd5yQm(S(T zdwcWX!kq(Mp;;c=GpRqOrc8VfCKJ3#*={$YfBo-c%xJ?WZ0kwL1pf0F2%FcJsl`F5 z4u$`e!)z~eRNLg-9P3IkbX6xFBfDE(+Z4j2k)lAZeci4rQQB;{C|xt_W`bfAziDUu zDz z?~bV;H@p7{)OL28xqdbmH@A@!BcIFcHLBfRs^A8z^#pIzo|GXmhMm=!Z}SSRw0{|v zTq!F0NcTcFm(B5KaiU(ne7VHJaIDe==6QsLg?;j;Ooc0#M=VMjXuYT|!mrV_VTg)~ z&Tk|)zhTXUo9bqVi!(>HxEUE`VD@o=kD%Iq=zkq;9~>>YhVR~o-;Sp5^M+L`1BgP` zndGolknAywihedOjFIpfGY&*YM>jxLPw1S|$s;8b7_Ec$|H2n2rqMyGF|@E3-QA9frR#GHE)9``y7$NqbNexgvz8Gs~#urF^~TRPWUmE#eIm~gipQ?0bzEi zFx7QqF`HgCV6Y(g!u8jAKKvmhSES%=s5IhX&^mv3iQC3vg;bV$`m46y7=Fu0rD(xB z64urGwP=KcttIdcnj0D#Zlr#A_aC3%XO+zJFFk&8Y=)aq7=8#9!^Oi>PgRQH_1fP> zk8_j2s$L&9zg=`ccr-5ivM=U(2V*C{>*{^{vz!y2<&!#yIcuPoJ1v+M+62-{cy74T zdaMSAa_FH^#Oo%?~R0)X9d|L6cCo{BeWyXSGh+kOg9m#$=cv}3{H-g z&BqNuS4An2>{0ZXSVctT`Q&6?17Nyu;qfwM{3-aqwV{97-dM^_S4(AV%C;V_=6`B} zY6(tKuopg0p!P8PUPHjO*^m%q1B*yECwvL#H@cz(toMyBDogAY*-f^ZqFE9%DXFha zwjDUMe*O5OFGZ0OLdt@2GGmXclFr{TS#rV|yt!Z+1x9--&hLfH>n`Sjyo^JEyJ%6Y1x+%|) zg=`>Au@QaCYvz#fML|-qTAPbLfG1TM(@4q4Ho|Pd&)SVKO07nRjd7wTh7U?0vt@#R zuDg9%9IY6tZ%fb&tNSy^dKV&qb=~>IitOQHPUxez(p;JRWEDswwh9q+IO>Df=gfun zbSy%eq8&VHA-fNuB2ln2i8^8Ix@KsWzF36)nYq;&Vs7h$l2JHg5p&&#F2J4#mwgwT z^v!##e$Bp4^p<5qrQ5;Je8 zq4D)|-XV(=hha#hmho-wi;v0lY;ICwB6*ELrgq+X=$-Ao7%?|ZNFjSI=}!o#RjXXr zcnxY##0WcH2cO#-S!|O!wKg{p_Tc>E3~4R55j|tQC%3L7IwE*V1Jus zBU|}Hy6$j`v?`h$g5DeGh$n570@9|n%$r}jJ&>cTQDSk+pHe^xi-1Z6pvb}cyT(R2 zjdazIp+m{%{z#sPM^!;AE7C`kIeOV!3R|bfbJRG=sOi-YDS7(Bh5vnny^pXn<_CK26Ot^%LO(?`85jPSdIa&H*sblgRA z0R&PWmm`Ege47U-%F4j_gT*$K}H?iO%M{$(}Z29!&` zbIzu2yz|l+-18i&AZfAn>xZ4A1DsAsp z{sui-nJ0()B3XbNzSplnUTD-3Iz$%CDuxO{@#0Z2(q=p0x*tpon4|NYdIX6T8 zsI4eiTTl)LEVD7K!C|;p6 ziNYIF=d?1ZSoQN0T^&{P0g>Vj<_V- z@c3aU-XQiE!hOOcT98djd@2CLT}W3}x&j~AT_{=OxmRTK<;vqnk3QQ|Fwzei(nkHN zfD7U)Jx)FR4*;JChgG7o*F8&eiWOlyPT}>R{A) zeCP?ue~YI8X34xZu7Srljsi_k7Aknwjo|ar4m<0XYjWalmU9gPiyjyp5p}JU%55CUl{cyZ&nD$5mK$I(lq4^gAM{4TP#1z9mx{w}qtZUaJ-;vn=rvYLr~^x;zL+$)byraR8{BP9;xdS3JX zOKbB(Jx)|qR9R38u=4Wu8Y`cXXI>SWzxc)=V(Nu-E#4$XrJ6e!mJS_3Kt_!how*!4#smQDx*sjRZ~YxE@cRzu#` z*CPN0DJ4$_c6D{}FO!m!n-*SZ&3|Tz@Blvi_BlgXR#27jclMXrn?P1!KlT&v+5_zH zU)78imLHxH@$+nf&(>o9wfd5^bx;fn4L;huL7V$(VuqX?J7*eGXpmF z`Lk#FVd*#C`nAdM7%>o!C_IxZlo)}Dc)f3B2`Mdj$j>rHWkdD(Y$4#=A*S2EY*6cs zVrOsPgH*}zoiYHf-~3Nz6AXgTjddH(?Pd$(5ZDlVACxxN>PkrT;Y5i7N+%Hc~b6Ak4B&rF+i!iMGi;An`ASBSacz( z6u9L^{0d@hB&DUbmKTcT4g}sg9=XVyTF^#vscIwk`5ER!DD;>Cv|Xc6f#6}p&7u&` z2K(&f#UFAFfIefzQ8*0}Mg@XZePeNSK`+9XP+(;O4gb%*hG?_^96<6O1t;p!w>y2o zU|nzSbJ@zMJ=AM&;by~W2uWt$NzasGMO`+`87~^IYh*$pkeG9aK-_6Q9F{8O)dl(C z)_362Ba%_f*M!_Rm#=O#9D$7u4uk90BA>7#x6A|3oZS~m!uroT?2yt8I*mVaII;%G zUpV0&0}7P~GqY7n-4V$r&p6U*SiH5lIVgEET{RgHF=WOb_*jenD4e3=`(5uB zhafA$95MI?I2rg!X3kh^+$>eS{2rGZ zB_9g6xhU)EE;VJb3)a@wFm%*pgBlMOKU#cuSqg9*;7b0e1sGhw->Lb|=F6FIKHbBb zlR+jPC>Gc0geOISz;?Vb|GPn!$OdIgXdRa<%BUes%Kfsm9!ZFi0wb3J;A79#z#yJQ zBO^&6f;$}mK0o#b0x3M)6nMNA!B7FsF0_n~OLP^a;3YR1S3LNc*cQsO7cQyYlcH$! zN&4_nZSYW5cnly3m3!=zK*KwgDfYp8F3e9C{|*pLusoAVxE)1@A*?&D7U01e=*C8= zH{l!gLOiVS3iS8y11zwFIdPh&o?65y4fqn*+agiufu-3%Il>(TjL0P=+N7<9EO^wy>QK)2L6L8`K3!(}&96C(!&+Xn8aPWQ z+R`1<6tBXKC}zqC5oasdaY)NZCF}q8FrW+ONJ0tImmq}&>!E-U5Z?%12mc-G$#sCR z#nd-QIRvW{?@*r*()Pn;9{cJi?KYRcxI(G53h~t($`$^531CsrU%o6t$!m`h8By@# zw;WD0>4{J0>_sf?%XqHeNbilePi}po;a8NTs8w4|>RRWiS zpcyEgm46V~2>pLn!jz#i|DG)6cka64JN161K&w!28$M-GPm2elp4GfR%L&;AM8I)jSOa0|8-wYM3T70x8tZ(qn93!Tj<7MM z4|FSjbj0Z+G{heHPhisjRGS|4=J-Jh}$ zHd`^fxQ&{VU|r`xZL&2&6d9FU?;P#_IyXXwa;L%ZkVZ=9hswFV`cp+(%>dU z#}JK~56_58ERsx99({*E$OjZ+R9inNvka?9V-D>usk*k-2aSY?=nid)px1t-9h4c{ zz7x;?iih6B2xs#3qsi>y%ciUb1uanc#RGxV)X*RcQwdpa7o`EOmOa~R(mZ=wQk^SYv<0@@{pg(J|U;ps<@!2jE{YLT+zf$iBt zeqQnu5vN;$KRJ&K0{(%cb(rov7i9{bb*_+1JA&J2-cbL+7Yw9=-iI}B0i)AOBm$#{ zv;*M`wzfMMssZTtK{jLU+Os{xyWZb$=SNDFp&y_kDd`Pvk_vJ(m@552~*JFNxl?Y9H21vIB za~SIKG*HbbBb_ABBw#^Q0*>w3>45{G4^Z!D$bcl&; zj?rZ&k+0|-f*XE182XL~?d||oqPBPebf{ovY_5fqwp|fqpdrbTou!)G1{$|k+mhmaKQkiVTMN1Xib~DJLF612Ss>)C%;BP*bMIW+uX#5hK_Y+ z6Ic(88cz?vR_ewQG0z_TGm9P%ib>?>MZWg(iD@1g| zb=(CAN9D1EzDJ1WcVwf|g}UQy?x}A_VB~?%=H>(K5}>G_3EO>d9R2AEwVi4(p~P+; z0(8>SE>N`zg5o*0W})_r;D$;RMsi>(dN)g+>qxetg9_{RXS9aaem35_(B zb6vvnVeG{|{A$A8uM>EIF$YwFW_ipo?<>UvIkG+#G9=LZnmXlgz=56g5@^%cw#izD z4#srqslUGl_>kp#WD^Z%gt3=}Gpa4n@ER}@I^Z`iZ> z6hNH`Jk6-_Td{-Pm1mfPRc@thlv~ixn}(71g+msxv9bJ}p+LU>Q`xjL8(gQh9w||j zIM^+0kCVuS+FghV{kX&U+p1Mn;6qP-?dC&v7SahaIp2_b$lcw&H&vPT&BYs61gyu@ zfsHSMMgin5br@Ez>EKYLaB306ab0i-F&#t&EM3@ee(^DWkz$N+U+6Bk5C01E)YTJr zz8+Vua5tIZx-}2^Amipyvilb>$mszK=N5p{NhyPoLk} zIvo@Pi9j#S_|Ep543JuFp-C11P1wk{I;99_sy4>tgXg*qbu->8IDj@}fKuv&jNq`l zX*-2x@acCbISeY2Cr?5=xvW!ulxq%90q^%FO1~(&Q;*4azjMNxsmNo;sX}W6jZidb z8m9vkBoI&v*eJon&NlQR0NGHSX98rp4J>dXr@rgVg^?xTobLKs zGS;`D51aL-1*drm9zxg?K98`GN(X8WO%;_Or^U|^L282kkq*$IKw>)W{o&l=&op%IX z#_4X1hTnXQ52A;PyJfso0$FYD8B?sB1MN}1{0$7T>6W#?qAOXC)?ky&2e^=-SK5dl zEq*R@1bSAt%C#@{%hXhAFuiOBl%CDNm_4LYB#jmtdmR8mm!#3Sws~9un~7TVn2yUw zyZ1TpkexPaA^q6UU=FBYqlaYb5PKfy3E13Sz}=MA7)MsUcn&#RAN7p%^y#}i9SRW* zu%6wR&6Smvr{@I#ASgx)c7^)$3J4yo#2tW;Krxc{nhh=0kqoSM51UHDQwIp!8*og_ zGIafIpt!5Vs4e@ECI2=kZw7c!{G$hz^Y(5!!AWi@E-wCzeeEn6nMonCu|NdRKxes# z+%Cn!u-bj=w2kAjqJwS=y|w;40K5W;7l6uNxl{KL|Mm70NwA604qqbpk(VWBvE6rI z5|4yuC7Uz^Vz(&#fn<3yN}Hqll`e=B)p+TK8m15=J6tk$bzpP4@1E89ySLE@Jkkhv zt136q;UdWrheJDrO#)DYyc@ke;C0aHGz3snTc*9lI;@wP&|}>eg&zC)v1sM!L<)df zuPycn^8d5!O(2?Ja}k?4#)e>PnAN+# z^BqXbCQU0>kM1Y~PuQR$O=hEjw{PD%ZZ6*;VOIU{-d zhc72#`9Seu4pFc3JBnTK7q!{eA~J zsgpWYPKv2YSTh#Ox1cX}arJH*yMB2Rt5)vpr?t70goIk^>aU@|J3~v$A>i!n;_Au* zR7-ONPuQ>R?F>m2FKOBFZhA!;wRMZ3@3XTRzyLP&@rCFdtakP7kJi^K(kw2LcZXd2 z#|X%80V0vn)15qG$uFUTGZ#T?rmh5rJuD!)7XK`jgewKB%1r+ncskQ5YT`_QnRAe| zg^Ar|K+mH#@r?MSu)WN=sgAj?A60bA?4*NK zL$$Xmb-tf(8Xc8lXP;F5`5YL+>}l=i&tIw!`g2N4OV55DD*UzDnkDMu@nbGZc^?Cp zCEkXJg#ESxbm3o@mX^A^7c?`OYwnm3+u#z`u-CYIHqS3I5)wM^2rXTTJ*ZNeb+Nth zdtFcbEt3(<_hu&9+M0?phaOKP3>yN5C>heBP(HTWKUD~o_5^U2!tUSG2xx>2TfE{# zT~r}f^+fbsE_awoeE;66k?LL~PEbS`j>C`EH}dfJY9;fNZ=`M&80Mq!31|tFm6tX5 zx$}BC@9e$ZtBA_HdpFUt5y`)y+7=cs@7}vha!>rzr`gr?kmj%sJi7iVGT+_ zTs+Z)cY`;h_y*>GhL+p?D9F((O9ZZvW&RVsw2aKOlY1q1*}IGE?Cr*15C$;$`WscK zz53!WqiJN_yy5ce`?Xe^Lmki&xTV8^2Yu!Ji5WY|*(d%U(YD=o6S^{e{JYZfRo3mm zhMw^nwXv3t=sl-{y`~bHXA2!v6022S98bc-e;Pg7XRIl?73C?OC0b@b;>cy>cRh?b z8_J7=1z{LXw3kwyG2SXrw}e6Pk6bNUx_17V6TPVFoiD-VL^KK7U0hDHJL`*G9TM67 z074vAr|;<2I4RQ5(C|yO&PN$H^MG`k~2rk-J=TcP#D1$9`I+CqF^4XzOYI#1N)>$?)`kwnWH%dRg{SR@n_W`(;7AfTz=#rav3u~@-#Xo-r4v#&+Tgfthyj;rvsB> z1}yotX=dA0sxik=V}VG#GH#6naW{5bF+1L)gvCsc%(T&lDN>gd!1gpi8A}2!YlzN# zM(lFAN*7L%IgeoWFQO0dXX)n}$ybLk7d`+sr$QjINt0Ex@)3)AMwSern1#G#_i6T^ zk1c8yCDwf#;~v?H05%{`HUmpiY(7xpQwsSX0W~;FLG=HYzgdV52OC>3-5bAz`h|+* zwTG5U+oT4=UZ*$(aN71Q%Iv2!fmh^(87fG^f~KAV2@~}G-?vI1y6V1cEoR&MdJ-Da z9LQiq$^gitrmJPiowT3g1}rm(hX&W}{oW9KG zSu`WI^UeDa?z{4G7IFUD))!li-iC2(U2vH6)pEPwVP%ihz&3V( z43zdU4YrSx;x6X!`^xV6ebFkh>&Nv?T{=Q}o(a>|5Ww)-0D2DNJ^#7%BB; zPM`X=kc^anS~f6&56bMkxd9T_pNWtp4O`CXm)YH#YL9)RleD~S6BSHKLY@p%|1ch3 z?xkC2Jmx~J#;e&1nL=@b%G@3_O0)CdH{M^ITx7Rb8~gb2d<-vjdR{6E0 zfy%%a)!iaV==+c{Us78^Ug|PpvOVmSs)a zHt@OQ;4R>q5!DccM*`YCWQ@z?pJGGlJRr1dq>-kFuk30VKqnSKXGpsl>>~ptj*)`4 zcOzBA25drxMfK$@W0kAu^dl8szc~C0hMN;W{h!3cD;i?Fe7VLX8ibNcb#*2ru3WvC zy}!1$nC*>~qcn_BO=xH$NJz*NE{Tit(~*)BrN1DPA|awl2_L+<^cuO!%~p#iY|Gc->1 zn6skXCZQEWSbZO*OTT?x#9UHPY47Mz23?E_bW$`Dui{jgWH75m0I9;@|00A+xfcBy zhi;V>n&sTUURExdXEOCjN%Kg9c217f?BEu2u|*$$(~@*Ul?3>$F;*gh! z#YzwOfy<0D)1d86rFi?B^)W%J;`yZ$lgRN0!E{OJf;)mpMw?~|n3a4Z`Z!PF1-Psh znwi#DQ{XfLwj4i?CG5p8a6h0$%tUkzKpU6nyu}fXpd4l7X+_U$9lM_5{>`8A4!=2~ zWEA!izZx7&)EqLP1?fqnWPS+vIPYB$d0ytV*IN)Kyl6tPRDXtuC}G9KmxRh4+6d^( z)4j|w;v{JyK0f6hB1#crVjB5ZFtnd{w|2eGcP?hV~K4tmL#Q^3f1%7mj_lVq;YvIXAM6#jQ0Z>n}yU0 zK*c$Bc5xr$a-`jl9BYUX(>fw2&`|7q+F!rq6_^M0b{24|&>&DvzHZ7HdEf<+$+WrS zBZMv=brK*({2*k!gD7EamSEyVocM9M&!J+A*<4CT{tl?*Cd{ zRn0$rVZq;u_-4gQ*K0S{Gb(}_Q$e4}c*t!U0%`iX-ac{s-g%B%%uc1wq&>~<4GW82 z);pkit3>#gVdvsqN)k_9h%a}kgcqS+=0R zDIx7S`V}b)Bc4403);o@JNyt!G~uCwa`nZf{hpXNtXr~Zlo!Q%Op-^u>MANC5p-g? zX&hIAq2JykV(ot;vt)tKb$+vm`ixeMwa*tROtZe{Mz<>;aP7PsEoG$s&B4nM0X=v_ z^OXzof)bq#1lTv1ZgJn>!H08KLl(wy;ouaefChm-fLfKfPx(5nLLOW4;~7MVE+>X` z6ldOOVVZq>{^R+J7iz(YAA1uuDEaQM20bTbjdew{eaH$C->ip&a&0cff4E?aOj@=f z&F0H_?qp#pDXHFy4O|L-ho_@h0ioNdx^W(Q$=Ab*lj%o_=Hi2`DGD!e(+>GVNvmB^R49iY6dnKs}JuD1rfeyZs%{WkyNhAF?IJc7s zvJG_zD@vUo6(ZP)@ALQ+gFvqZX&m|Ux+x0YrXQTV&-(rWg4;jfX6?ZZLHHC&qZLjp zT8pA)JtbAbPBR&mE*`|=lxO?}#yWvqe#4ZTYu1~UeXd?#xkns4`>W;T$&=l=HTxOB z&op&(EX9gNTzUt@aT=&95Xza2awtP1(=9eE6bqoWyhjd_AQ<#&=c}B=S6{`tY@255 z7T*3S|GHP3I3?|Blj^gjmprOE6VT>$-Dwk8?Fp1fGe|h$Ps5`G*)fG`$m9E$#n`U{ z&Ly~y9z9Bf`m-e3B>(G$)vKxOW_>9!gn|LF7Vqm2uWHeCXRO=5u+g!wGNBJs?!HxU65~w2JO`nA6{i2IRC`z3vp(l? zSNkGqyWHL~yo=7RSC+yaBVyh*t6{eOK;GZY z#O|)r)J-g+@QpZS-h10_c8>Nv)A`@l9=r%;375!qFL)r-h~2ZniL~xu27(-5a-boH zdY9g4^=)>g%($t(+*d};0p--kdxWekEUKU%WR&j@j{b8kU6Gbf5T!5bjX451hC!%i@2(gODLnGKxP!{GD+SL$3TBrLdt2Oo`dw= z#TYo}iqyp7lA5${<}A>q4MSUaiDpW!vGXXk9+vwY)TUx}{U8xD;#DTb#6J$K(h zk2VE?w?BJJJg}R&(^#*E10$0Pm}?d=AJ9UnZ}9?c|KU#4*@4pP zYbv8d{YPWDTUZP8k20ZmJ`LwHkwYN+KWk~=I{Jw^UtklaCLP}X)&d0WDeuFpeIWa} ze^kQAY5Giu-xO2HjU}KJP2;2VG_p=K>{G00C!iLMw^Eic<*~2ccFm_T-wm=eT)9?( zVid$mE9}sSdFTE6_aM@cN8*OL*AR=f^%5Sp)Ksir<_q;2B}b$cg%HYc?&)o^d>F!F zwd2>%AJ4?#<{o-*kA$?G_WA&pc?AG2G%Umm7>*!?+5GJO>0VVVAjlul)7M3E* zp51Gf+1}Y1Dd2;yN9j)C@tFWQC6q!hq(yasOsh)7=v8Re?lqv97^f~wv-)Fr?l=~1 zugyT>70-fhvdoY>HumV!t3yg0PB(xQbrp_mr33apc~bbT@)R53V(*=W*p(g?gYm_f z*`CY_h(W2K!Wx{HR?~RDYZVJe40)-@!+zsY^lnXXf#Gd?NAzxmi88;a-pQ>iW-Fox zLyCpc6Ymc$E`y40$f6kV@a_VazSmwcei2euO?D77DyRUdO>}gok!}dXX<^vNYjS?H zLd$WkZ=i_KglUIwnsWgzRdv_a=h#&35Zm}Hw((4Hr%QMr8%V8G7|qbUIAr|La`MZ? z6@y;ri}1RCzhCT<81hZ|A<_OdI&N;&d$Lt2v|cxjd{dzZ*$>ua8KDwV_5e~bpaR@u ztr4)brpRV2bo1E$2G|WTHD&3yvTyUT-7b=Y!6+Z}F@@pSPgI>|sg-Vo1=`9Kq%pz8 zM`y_o`wZozpn-Y<2N*LzKW#Ssw97T|TgZ&7INtse2C6cZ))W4~On5*jdO~0|(fd5G z51CXln5!~8Y7jM(11+L;=Spa)nQl%CXp~*=+j(~~HN~&Csh0Tm@()qA=!3@v#%*S_ zc5|Z@rB$n-HgMctHAN(u+;5m6!30xCeLd3DAT_BJ@b>nNMt4-OlPZyBxWMGeYs42p zy!&}{wAkxlU&7JgdT)5^L(tSgXQ*XhU;q$r9x_^qyS`~$wEgwj3da@h+3!Ma-~98RwGI72Dq*~0CxGlIg_Fn5ndd|5*&plBL|oxi>-b0Wg>S!V0(aKyWC|Z zrC_QNeCGUYZ*nG!X3ly2ad&e6j|o%u%e9`C`$oQ=LwS0sN*D;I9fL3a7SZv6I=&?1 z!$Ap9twYy|K~vBJ^{>Sw<^Rl!1WV;PqFZ8b8aN7jv`5>qnQv^Yz8)-#a0a3c78cB6 zopEOFUB5evHGep3@cZ8Pc~0!J_dZ7p1g_z`KxB_*Tr}p<`(;Sg!!GImD;4<1zr6R5 z43qX65#g@~EkSK_6DT%v?izAesOVA#i+YiUlx_K^6!fc6A&yt$6HDPda*^FXt}A1# zh7%zd`=a(IiGDt)}OzLwK>Q08<)&5jjO8}cP0tlJ$o8zZ~IFa`X(tLBMc;I3lh2u z>I^>_D!bdw40u8y!bQ#noTS{y=m2d@F^;SKVIndi0xAs)8yj2JKj2y>!b~oc>upB< z#zEsMNZ>j1#s9hKu+XTRkP93}54;Tec)N`3+uP-tHK7aadw;@FS@4 z&7}vDJHpo%A!54h^-B?vlfUTgDtE7n80WQt!KdaYBR}iuPw8viPnc|kMOvEgSwJC- z0H>#Cm%x5y{mH4ugGd%NO#q7N&oQj7yloP2TS)@#)mw85koddn}<% zUG386;e(U7b}=rs8?q77Q$M&K98cf5Zi<>VTBy5B{rcq8v1#>pr3bII7Q#UK*1q8$ z{DElIHH{J@KWLkv0zwpfx2BCyt$+wt##<#n-Ubro8_r48)YLn>yK*~6+0SVj%7?<1 z1d#vpE1EpkyiSeFI16s6u9@4Xq5ai@i#4`)d+*F@pP^$bYWa{QE}`P#eQ|O8I(clj z&jf(AS>3bai&45AjCsx)`xU8LHPfWzBqN?%e3$HtWewocpb7jZ6)i2T612&md?LFism@fLMzyi^@!9^$ z>9Ry8OzO4oOH4ZHcU`}L4%~iw4uxFEO#^qxCIwo`iINEVZIh0MtBfkDC~pGzNT!op z4G)wA9S`!vdXfiYQ^0gVAU`GL0Z)p<(EuH(G-n}=L_oGi@mJ{KBD6R^t_-z{#-CL@ zTgD+5wr`8_0q6L46fP^&C(8>@=-YUJG!HDzp}6X*Dth(Nz2B_!-w{?2X_l?quOpvA>SKzO0~AO=0FQpo-kN50r!5xZnDBAno-^8mAwfY?qH@f(+eQt zVa?Dk=#0v)77|#>nd=`TDKCs(k7D8WxFXPgkr9^_eOsc#!kArrWX+S7r=+S*DKEwM zNpH8!9A@wbd$b=Ot%r|mezf${&K!Y)jVd0i=d>6x%LoxVhyeoy=I~;1xvF{y^_I?J z;V&Y|Vy%Ah;O|zaw#!2Om?@8*z^Gt7GJw{o@TeJpiE+J;M?(E*Yhr9VLm&bqvKNpx z?M2U37S$(WA=Hyoxw`!W_EVsCEJy3s)*eNS@)l}oPlw@r$2Il#d9w;$B*16#FVL#Z z1N?M#pe3N{I!R97CF&|$>Spw@E}gd*)+;Y|)TKBRmCSc1GDlBMYi$(MpS$Mt^);A{ zST5^;Od3QE!@{-?!Se%}DTeXS?wR5QMx3~xW95~@TVfa=WDr(gCI5%z84PVfcfY#t zcVsF|tU3ppx4zd{1eSV;y9OdCc_H8o{ny%b^$(GriK$bLU)j98)?Myv-Q7yhXwv}& zJdn|T_KaGH2i~RK@O4mC@p*ZCG@k`|1K{~tP&V<~t8vz+7+wRtnHMMY02ena~snB-m0AF9#q3)X}`zw$8c~Y*DTIdw> zXk2p?RIHe{{^oGHF?(K_9(~ckz?t=FcSO5MN9?wwzj|V#YZxB8{Oy|sB?mL0$U-A{6Ht?Iy*zay)#6GQ`iI#LR!UI$FAbY+Dl*$T zjXtlLcCYiURljSTW8$cZL7(J+F!xk}#((Qq46PK7T7lL$`e0Z0h26bUtb*84aF^b2 zpXm<5+5Hf$<06BoYpOjn=OVGn)2Udsk*oHDTCXHgKD65?cyDdE6WsuRLcaWSqlB&n z%oBMY(3uuYRXyL0dZn1G3YA8V@<)FN@cCis9a%=J109KiN@CSZd@`<%TLYWgf@Q|L z^czzjwRL{oBGAZpdA`3n+!^(n1zZMS<;#1ttrMLfE(9ztngbWQwwwV8D}AIJOlB!} zjLkoY856b%>8pyzX2#N8H}fARNDm7{fmrqAkN@-tF^52$*&rxdGS}Wk`-qhSdmb!6 zEb35q`N74{r_*<|EvZbrdIH@qMCNmIg=fVVXDVhwTp%dk}gociu19(mOo%nH! zGT@5KR5bzz%Rnwa|7ewAW2{yiGR@f8JFNwS(kF_^!r5IEvBseayEyl~UBK+9$O zORG$kj-)`l5eB)4gDS!xfNAju_gM<^sX8V8r%PN|;yeWH>*>ESbZqWeQ5o}$yp-|T zZ?UFH{TP&Pp%JWn%XO$Q2Q9*V5UUaZWQfg41GwdgviOW#SU^BvUNIu}pQnI@kde>b zz`0A63PkWuGv>ILOPt_v<9fa`0-xi=BqopN!PU{N8XKS==tPyWKyZm?VE#Fr!~t1!pkM>mXQ( zBhwbJq698yd2q%MNM6p zGhJLe8X{^BrVz?$k_uj1XcvWog7(?jrE6jf^YgUoyu75Z0|En8qgZARJPD)aV^s^p zB;x5zOTLsCeJ}Rp+ub>~5oMJMPH=F~5-gN=KP){;7#Cs6?p{wnKG9JtuChC;x>NJX zvZ!x8+Tq*7i1#$9>rS*ZjYL1ZEBk^Rj zl9rZceVNnMB}a0fB!1r&g$=4ZbTC#G^}*6WwT~^0#{;__^)|f=SLAz;D3H z&cY$Pld@fOJD%EF_IA9Gq7a8`$8@&>YpfRGHByF;l`97ueAT!)&eK1)J`1|D>Ari9 z;BnP=E9HNQ+%|n>S9`R>$i<~vT+GgsWzza=*_cJ%q$)nX`YRDMP;4KoU2g*M=)m~r zW@O7`aOY=+Tqqkex+o|d$arn0%M9yW7E8p$dYHGHv4CkQ32@K@Am^xlctZ5pzX%ds zwwU#jQR^N16=g(MuT~IypABf2M2|_P(EOmq98!w{49b*JjL&%|?bfpFKhuW)95>HI zQZ|c;?Ise=On+2a7s34V1WGSZ!W)*p8aSxG`|GlX3Zuy@cN*!&M>A57LzUx0UfF(= zbKDMylmkD{pFCfgqXve}NaKawNSHIfLA}^HTNm2>s#7<`!)1-Z&%V5SAo5+gA1%9|hP0g|FlOU4&%(R#3P(Ji>Js2o< zp&QCZv!(-eehkB=OA35`xc0esi9x3E;5j;86YFz z0>vNy@qd5oEs@IuKa!LBO-a}I^HB)?;iBQgtf>{KbQW^t72uj=Vwd*W0m$aDqhJ?d z|MgyqBE|*KLg>9o5V0v8(bD2z1btdZlu&iIqeeUMOumv~|s3m=Jjs(uwA@5IGiTCFK@f<#g!x=vZE8c0gfb)QX9zD$8Uw3^osu@s+sFT0CMO3 z*y=$9Ser;U{rY!{6)KJLprjaROyvps(7AA|>*&xkw?zEp+~xMfe}K=*#MH4_XVSqO zCmIhCJq>hvmz!g;xd{?D#^va<)#`O06Ub|g0A!Y3POVq%v>3qv@N99{y-j!PLa&Y&}q zgI*p1K5k8#SqH|(OF3Hm-|y)6yc+cGHwn=2(fx&s{xlo@@UNS%2=9i{*HrX{gH_s$>#yVw zcw1X)XkA@KCOs8k9{rl%bnC+mw@gohP2mSnT38Pq3{-aDg<8-t(((hNrb8j^d#OZT zEHfW$OT$Mq z+Ktw-aB+6NZ<&iGWhkO1+vg24vR+aZ*)T6OX^+2u%H%x^;-gZPX1V1&3?2`^7G`9` zif$~NTl*lbRd>}dX3Xpc109V3ldN%ypKVlfa;Ag#)oVlz0|P2pCQ%O`7L^q#OUjkU z8KwI<(c?M`-K@kb(DD-r<{W7f9rmdBTSCF;Xs5CV4BKzpS`D-x5T1dQkBRllKm|=HURrhKn zV|1d|`Z|k|sWCORFpg9RnaZ=aQ1aZ04y6t^2M34H=~-4*)+dU1OAFWWE<;H11W|4Z z%iipeS%ZZE&NwANyNDgFupR_HmbP!Bx3&^o(;@O3EZ?#IhWhEQ1D8|f$1 zCJ6mTf;Wj6h-)k5XCEyrOBOtbsjozFcOWfW^0S2>@7*Lre_H)HesDEXj8O?E%gA0F z7d%4E_LGoSxOvZ3PQ9BWD{#Jrjt*>=8X%w+nQl>#3EN}pN3+qgv){#1R=HS`mUcZ! zM`t^K-Ut0Ms@g$pNFk-$x1qcF6BTUT8pPH1FA990EsH6o8h|1c~Dv4p9Fh@B|? zXx6U_#i<|}XCWgaf14bg`MT|5IHW^Si!IcPAn$_X!&@k(f( zRl6eCReoZMX6Yqck`tAs^dCC_qwtu*aP)2Y0*rKgFQalqwxqpm7end!_9rzS}o7NR{muj2AxSYV1x|XjCi-LeQcS5+b*n_#oa*!u>WW$=?eD=jN9U zV8pfpe$)?!YUKHrr-RCtoIkJki{VqDY~Y9Pw1@T66OV%yc8C@8vJjEDE?NO!NM;XQ1B1WqcGxrJotS1 z@aYjpf`BvXqu5y6XV0Wcuiom<-D+~0YoI9l@PUq-98Y8WyN{E@I+~?s9Zo!NzWUkB z@!q$>j?3Crwm2d-t(}$VXb+D=qe#kvzezTm|#?p)!xkzoZ?s${0UcI^vgl|-K zLsS(U8{oeqk{n`J^XsE&7hmsfr;nLjpHchZfBJH7e_yR8hyR~=D=pfIcC*}^|9CGE zO`Oioaeud=HHc-<-~k1JZ5fHj1+ql4@6B`Xt{?Al+By_GJi5TwBtoyO zsq)H3y`cUFc-q)WVk1)&^y-O&)`aCFe+6A1m@Q&m|-# zvV+Gq*#0WNX5ZySYvGx`Pzja@z5xMEpr>IhU&YDo?_T)&^}P$L)V+Jx6Ro)ZbBM8J zAd^v>Kf48=k%^%Q7!WL)W!HUhC{@b2aLixf{UY%?AQw>kXvHO*u^W$Y5-ABSsd#sG>rh2Agw|32UHDDLOe8hD!+6-F7S8`JZCqMv5-P&Ao)1 zlQe(00G-jL;pecfV|jx86CzI@t9s6CX}NJlxlTBVoD-${aXAP6CG*xWZDb!83-Dk9%Dy#MPb8M*7Yr6E|<@nqjz<`JNjBf7B7f+xdXzmjwav1x6tdgla z=WDwSvYjum&X+z!7B1W?usS51WNPmpQv#{&t&)`$QKvz#+|K0NNzDUi;OV)j*2d(l5zN zN{L!~$o2C0TSek30B?s%j2m7;??8-c=(+^Zv$ZO18&iq;bIq z8JU^6N&PhCXTOjt3gJ&X-l>E2C!mOG1ow4}qvH{?u_9`il1Mge1fw}a{`~zDa}}aZ z1`(U49I60yN_zY;lyGL#RoopN9e90IS8QrHNlQATn(VRGXHS?-YR>4r?{JgBq*AJz zWJcrQ$TZ_ypG}m->hi#XmjnhpE-tAQzz#y=b7`s+l7+RO{j_M;S65g61avUMN&74I z`ix)GeXW3^H`oxE(0Cn7y0x&nS`Zp78`UL0QF>C=#of7IIB?DHNlA&zCeBz(Kk9;c z8?su*-MI}E+r!IUlR;BcT6caP&I3c>vn~Y(z%WsCJUNcoLDYW*QMl46V-KCZ6Fkx6 zgFT|G*FZi?5O9^P)kBK%b6T_x_eYKXxjOpLw@Ja*7=pz!S+$GJ&jd~n=LPIHr^eIl z@^zecy-tg+IdOKRotK%1gU}V`0p_$!Cv?1K=`7qnpr5p}yKg}1x~nGKsuIS@$9Em* z1}(73Lj4Gf-|Iep=xhl7H{kh`1ib)5*6klZq|D7%t*kVw>=|LS<2X4A?GYkZo;`Tp z?(h@T-qt5oMm>mId&LXwdmX^BRh#&?s;$3+2{$5yZ{QMsB?GD^{^XOB!xVjG+AdxD z6I_8GXdxJ+mpu~_EbknR(V7jGlfOd!miqp^SfsiB()<3`zTUr2+xQhsOiWQ=;;9{Y zW3vC1TV(@vUqr^2F_9WGQZwH9u|D3^Ro>Ca^>uggN2>48yLay-4Hsd;ej7BfbswZk zdSKB1Bl@#6Zk6D^YWNSwCr{R_M$N`c7Rh6!dGx?JC(xVp^55?O1wV@nj#lLS=Te#a z8(3x%mfuK(>0dzHT`-K8{EF_u*WL z@YoxLPA#PB=(tC4YsKDL(L@U^At51RjK{cxaGs+JE&r{&@;F)xM;QI}Pp-E=!ca(7 zRR~}Tp@j5Vak_aJH|e`q`x-sooUGVl3z3nQ_K3i(9lXiLR%f-f0rm_55nuI_D9#U~3OcoCBw~8M246R*bpWs^!@ZS$cZ=RjY@a-Ug>|~f4pz5b zzP|3f08<}Sq;q~7%Jv!?8=F+rH#GDDH;Um)5&yJ7|IdNJn_M^_?BjFIz##Q|kL$C( zj}{fKUz@#xl0~7`xqjSyZdhE(g;SbQ=1&>Y$Bptan`s=Qr>6%o!YasIc9XnUKQeL^ zl2;3`sDj#<2D#E9Y+XqroRsH$>1gr6(l#Pb*N}f7ShMf){Zj}LP(g=^_Uayi zjOZv@8BMuNqT5l*P?+#418N36|C~aK;2A5hd%3`oSv<~7vadIU7~QKbOT5amGi}S=C-hWL-nn!6pFjM zW^clLS&cZ5K{4Sri4xEN`rtj)5uQsQoq*0Y5Zbg~`8A;S{0v3+L8f6|US1Jcssxhq zZrb$f0?-9`!#m;3)2+~7(d&Y-0sp=aH{=2*YYY(JS6U!vX(3|_V%c}S5S=WY>C5fy zGKX{XZydY*Fi3bkm6mEkB6SPvzcl%@js3RqF-?%+hpXeX6ybXpX#wrltYteBstpF>=!NeR6)L)*%ErJnIe% z3rjN)5~$xtyD5eyZ@{7raB2~*{5B92M$M4!MJIksL_`-0EPJ}e0h!wB7==D8+j0fI zc&2^fEMx*jO8W#(u2&t?NG-ZHM+$I%p0U21sH6bfk_VI;>GGX`bPoII z{pUQkheA>btND(I(Og`5OmCn;L#{?Y0=2cO1F6&^=HbanF4oO*wcym^g9{|d2o#=R5(^+MM#3knLl z0ktE;NWiKF12rs>_QetSA~(*s{ixq?@EOTf$EDFzu^dE^r%iefZ5@->cSvB8mUIfV ze`?fzGE zwm-ISR~`1H)X^6moD3hXc)912CL-?8kE|+?#T5}_QBhGQC5L4yGs8yb?=?X-<5EZg zzXLO{Yh)5XXlF<^i+jT_m>eE{9`f!d;ArOzk--C%ZohLBdjzzxe(1M46k1DzqC6rV zmK7D%gCkWy61D-j>jv91$)7I8k>&NIgCjdPGRiM8&Iz2nRQ6X;j$ zhOMHqvP(-ub3T-#^SO(j%FEfBvmHT;Q>aD6rPh+%pSb=XpR5F_ZggSv>Shm(XBp zo7#-pKBEEThO1g5X(0p4ynK8bY7z(whn{$4z+9j4%pO!pa>WYKdO?;2P^soxJRe_Z z`=6b=-Vz!zMz9<&xc~sD0B}imFl?Uf?7WS78}z}#z5djhk_zyb#A=782fcy!e1a3Z z69Vn3D~t!xPm4^r@#p97>*$~}DsaLOqVAT+5SaO-Wn|C*>ROq2Li!D^xaEo&AN13~`sjfx0AZ)CXvtNT~oyCL-tkz~=Du0KHC?}rYoqOlv#kX%`y(9vzeHFPR z;`t%yR$hL7U#X+6{kNYf9WL{IBA%u1GK)8syARFhT`DUpsQ^)FN^&6~XbtgNkE$62 zBc#g!y1)W==S=aSgXjKMlB6Vmom{5sDxjm~s7`?0rcARumR7aHWf+`^0f+WGIMv2x zM|(d*DCTWlUCmH}#*CZrt)uJ89Dj#+{J@fv&zV*F&jrx4gl|Fo_?{UqJeeNMt|TE+ z$ZFi+iRvRo;k;n892sxQn7w*v3ya=PzH+J-ZhDuLa2T1IzI(7-urii?X~JduS`^ zKGX?LPks?hIg;1lH~$&hJ5MH3cq{==hPC=~o|BfD4U=>~E_>xZ?cyj@ID*?Ht@Jr6 znMbd;u=ia5vb5m2Fp2TXswor!e;GWa)`SUWRewa%5`1p8eyitY#oU`1jV#guXZ+uHgt48`xHdh|2s*mw{M^eNG^hjd zN*&TbMjQ6cd(Zf9%c|!D-weAqfjJD?RNs17{Sye))f_Nhn;``rhCB>{-hAoS<06Yu zYcgRs>5YlK?QK=#G!9MGb{1R|CV))mZ3Amv(k`V#Xg5 zZGWWyCwSdQf|u%-6n6NwRW}gb8Yo0CAfq;=$%NZ;d{`g5W4A-tlU(P)tWIMKlQ&4l zw9K3jjO*G%18=!vVDJg5dj;e&=cpIrIN~ZfIlDA8^y6LYO%|`lA=fuHzQD}Cdlko|#Q&kXu?>>RVXk>fmCTqdpcC+;8h_m$1^+9h;nNnwZdJ z5T5D0f%hfa?&tcW*Kf2N_bt?^zor=#!HLPv<6gBHEHmfaTlsz^$!$jlcqj8Z8Fk+b z5kuzJeeh4U@aHic7;5|-mv3zAq9Rrs?xu3GvPEFB@}B(xVu9KUqBgawBXGLAhvb?* z(izth;N!zRs;KM$>52#S0M+#;u&STUD8-~J$PtiS*K zwrzy7sUGms2$66E{Tj7FpsH$zterYHB$C;=Krb>h9JN^zqn7S({72J3Aa|IZWK#5`GSJbS*jGD~$r%3tWz> zDzh+qSXq4o{2mfJK$tB=5hu6(x?Rz% zZtamI@VrCMV$Frx->QnfyVDQ7XHZV7zVU++W47thorr%?*RBvKv33I(g=jnl@%#M8%3<9} zkGkl9q9+1&-; zBVvrcF~KYKJlQ&VM2d-pMP@TUUYlj!;`nhzyoZFx6gCUv-FBN-TKDh2#b3I2_zd>` zJTl#Jr|Fa7iMi7$E_`G5SjY1D)2AxNloDLwyPs-mzI^)@0Hob)mCym7rq0gXoZ_5) zaWOGfNouGGB>2qGkV6qe?ukV}pH-JgQ~m;cj!c#*$*W|Lt?o|Hq^PN(wzq!GBJ%_o zCC5S=kA1u}|0zq*ktHK;a%3b7Ukq{Zt^O(IZev3Ki{8*BAzfdIiS)mT%WO4!VG|0> zHN5irZ3Rd1k8D@l+S-&UwMyx7Ss0}D#jf0yf6e~!3j=QZQkXn1-sgLkVY|lCt+98~ z2-A&<{cbXhKMRv!Ahrn$qBn@UbFc3ipb+oln4Ghlg`14iS7}%g;zund|I>Td2G!ch zx-4_?*R_7Xp-q1eBSD#1uA|9ncm~x1J{YkSNYR9pwQF%QXf(kvO?-k;(hz9yc#)6f zWE_TycTG$(WORK<1q~;p{>gMCNSQC0U`B@p-8mGfU>}5obVN`S_GjIvP2^ZGYLyUe z=2?w(1Q*nKj=+(>fu;+d(DCHro9|Re@_dINhr%c%44S|XZ~aCIBSY+vMJ^H;02r8> zcX3J|IgQ`Enf#FtIir95`~x#MaUuGB|qas!w5iRn|p<$_x}t0F$pmD;Jx?WDA4y z^Lb`7a>xV{PPhsUcBgclO&$ac8_U(-KsoY7>T4j%I@o-2;=+o#gH*aPyI(KJKkh?o zxTTP*I#<&;TV_rG;GBO^(e1;dqv>y}cV=E=`i{}lCC6CliQP>F*G$wKdSYm4A<=tD zVKW&eu|7?AumIV`e0mlW73B>fCnhr-eGhRY^Y!&rj(-Q0#g$K<(P2-|RCaVquKzFS z9iB3{QpD$IOZWWArsEFVFVX^DE^={sJ_yrRSi36EiYO>#%%W z_p$~>eM!ap1`;5#}h^{9U?jjuuj?Ja}LS97{vq$b+|k(8Ev< z9RSh?nROHuQCp=%81Sqcz;LUDH^I# z0!Bv0?eE_kyf2etw#s9Rm-1f{e`FqfoYV4`4iZ(Y&<(T8ITc40u3wYl2Jb`v+dP zSJVTmSy>N85b_zL^TV$OFDq4t9xy&eOulyQ2Rw8Dq`U;_`ZHMgX-E!uOkJ*t@Uj+$^)+a zn!S$VpVmzbX^?WZBD5Yr$M8nEE5gG!ho(eESC~<}2kLj4Mj)rQTDFu|9y)!R8%Ia4UA=m>sI2UDWI7#G>3UX3#n9P-!V^|X z*W>5s*J1OgorhZgpDEYq4U}7cRwdOX&=F^4W%U{5eOQ3yDI03KwaM;3$Bm|E4%N7L zUJ`i`oe0L~;c_J)0ii3h9#E*rHhP2jy}hrul~7e!RXak!TT^9!Wgg-)3|De38M1ls zu3S<6*uLjh?xu5-iS@3G45Msk#}xvCfW1BY=H_N99v*bN;$q)`5OF7s>rfrwT_ZgY zTiyYKZ0hO3AFw8ru6&%6UfIxB2Q)FnLBE79A!38s&_tj?-x_fBhN_Bw5D= z!OM65ImI{h3Xr&JVYGS;Ysy>nr8OhOxCXX9tl%E^(<`RdeMXJ|n=dXdE-jIq9Z{mf zRLcw8u4GI}l`ZEd$#aEb7#7WQM?|Ul*Fy~mj}m;+31eepBjW-X z&LfLd*XE5`3?9oqVJgrK5tFyX{yI+pYyEr%Gkd8-L@2Wv8QXe#RK+xPtXcD{EIlq} zv2fw_v9R2@l#kleGaQ;29yAb!Ph^0%YVk(9hVd9|S{#kU{_yw_VA?Q`4G_U=bGMDuPY73olU9UbZYElr-pn!c%pVP8eEuT>Zp)1$!uHKQzMf>XSWZf@~T08l+p?#j);-vobXN(IM zj#m1LF997w-@-68$?A6@>~(sIs_OX6EB(y{7M5EP%JgDdcK7kt$;pZ=KC@N+^7NTo zapLUUu$I)+WEvP82#tyHXtPkqdp8CqtNdprTB#J#fOw2u_qqzI$ z&krA`N`|~iO&z3+IxgI%M@_g)w!SeWPML~yscmhdmX>+K8OsU(IaM6Usgln!yNP5j z{42!@9hW+M9^mZK_kr3aa{K`HKg8vw0v^F*kK8QL^S=dNh@6#4+!(4Jl5LH6R%Fv; z4$D4*w0R>)+nw$D^L(xcc^%e+ro`>ZJT|RL@fCBRNRr)dZ|`}C`bfsk55FeOJ~gC} z4V4rYWAAXgb zObKhT%t~fpcst(LZwtTa@)g8`7um}fDNzDV#AT;Eg=>f->QMgUYrt0K=nVZfcWWEy z)Os*d$=Q1XB?vQjY6K%TE-s96(I5|oG)dvm?r7d9DJ@l1;sUY(QMLEopuPfesl&sO z`;Bo%OmQdD3JMKx#W8Gm4%uI8*QOn}%Zwyh;)GiA>57POS&iU#$5n_b+unZliZplD zn5L3nP#}xKj;8+X7k>1Q#yr>!kW(=Yy8X>XE(*CnC7LLzi4;tZ$KOQY5TEz-e>&$C z%{I5Q%Ch%z07IO9gd4q-%d{y;$h8ml4f$J<_`1RBli|xzmq~eI@+$s&6w+QVK$KN` z#P^`*{fKB5(;~QR6X4e$({=~Ar;%e0V3{+NtG-nb9ad`&c6eTv z34$+Hp>Wj}kE92E{BxpRJzJ2`gRPU(@O)xqCD_`H+ z3Bu1&->Mg8S&AHyN5zK+)Yp5PO}xKA!fpL3lwRQq{(!2V&2R$Q@NUiC3q-YxKM4*3 zSwx@q!v9o>9%w(qpb4pEmlnlED|aIec%CfPFwu&-m2c)3_nd66=wjw40u;tagCJJ- zw{k=^MlS?cA~HamKSQUjFhc5YMNcX~q6nNI6x1;OvwkVchK`PlP>T(Fe4enk16ffH z+R<*WNJv+gJQS;Lq}aq9XeCDq$K}&9mM{=*1z&MrPz2n(TMe@)unlJ4xpdJEm@dNv zd5?c!*y&bmt_&^K%4%kG4>mQi((dgs0<8v{A(;G4et!O%dNiQ08t3vgDm zT4vTyHcUo7xA+B}Ik#?0XcYD`!)$aO_6SU%$rb)p(;+4994{anhYzVx$O)vC7+WK# z8SZPUc+Mz|w1)aQ)&)HHoMHA-0Bl*ri7?g6GV;D6U_ZF7BQ>ukZ``4JrPZ zh)Nil_^bt#!R2BqdDXcVFAAkO9vK-nBWYYj1R4Va!wWX^kzHI;9&Fg0^zhLW70$nM+O7h-mgWlyv)Noa0$$>lKts6MkfaOp_@9dZG+ z=MUF!UrcZEhn>c5GU<~|{@HRd1rd`Es~7gP<(VkgVfEd;;tf`f*kfvy9U1H3_F%II zx=kV(KO1GuYj$fIP+$1+ zD?$|VDA@G{oIg3x>)lMmd@`hQR$FZ+gs<2$yuFedq7UYa+lBMHnRJLWj;>ojY&2g`Ck6mKwWBV?B|Q|%i@z14)m9v>3s4qzZ6GR zneIF*8XCq;Qyv)|O-jN!%6S_FZ{nUUVCH(;TEE9gTwjY()VNY@4B zSp<8xs<(vc${%THs74ZNjh;h;_WnLB2zl-8O`^wr#?sKmb7s4kha|$3OQ%Z+1$LkzK?nte2b`qQ&@_}g8%SgU~t62hk- zPRK!@gAqNl^a(-K;bWPWDE>D)g<2j#(UM#81X=!rgr@tl(onG6Z#f1dgRBa+SME<1}$)_3%)hFahme)~d3w8V!limF# zgW{=#Ss(5{J?VA-=>_ZBjB(;Ka|?!HSJNPc9ZW zTI?-W8x#>*0@UZ^q428T!A4K0D!swV_^)|#3aBFho)3}{DK&;OH0CDWu~ng3dWHILa)m4Nbn&sy;p`uZOJ;hG&u zZ%epA)Ft}l2@~nmw+(U|GJ?+TA{Z3Igdo_q@8a(%v>+;kjV4g#6}Xfm>w;I#I=i}5 zg(4P6Zqm}8->KQ_fLW&a%a^ssRVgXwz=29yM&|ygj0%OKtW9TXo;*eR+9yEPl~J;O zIojKMBqj>cG&bp*8)KO+KUo9d)|HTmsA;rU$@n(Zw5e6O%u`d-uyO|l#(8{#{0)EYje$c&Ii315XXNvv-k0{4v$y4i# zpJ=)^&z@h4axc}43|YAS0`)#`!F=R3dMaUI9Y3obLEnG)0sl>9vcMw4o*ZqSZ?5HP z*6o#g6g&0ga2E!uK0~i*xVBcj+n?>%xjqD}1sRlearS~MOPCPv-m!Jzar@$uDPULI zaMT{wu$TRvJJvD597Jv3bN~Byc0i&YnZrLhxC6i3VStoHI=Q;KnsculQK$q{;xFUYPp-9+JZ0d8W^CO)cm3r zuwU2uDC1myAFmS6nJ5HzB@@l$f(|TTZh$5tvY9e`5}$kVT4;$5Xg*TM9=>6p z8$eQtUJqIt<$c(~p^94wkmt9d;azACSv~;I>njw>Kr56I6MKb!kai;#^9f8wnJg60 z_*TP*(quUISr#cKpRc2_ZNdgZ^mG22()P|2kzsuj;+oQj5)7x=jX>t-qbaU|_D9NQWl@ za8L}pAvzT9YS}b6yL43FFXP}mG%ovt#KJT%?OK2SQy+L|xkGy@Ppp7YIaX_kaj-dl z9taRzn3h?xEHEjjVPwCLr3hBfPFbc(kwI6LMXrDgkx!xSaI|J##O|*-qt#p2oHv*h zDMbAHI5X`w)`%jB)gS8M=u%QbvydVae&xfeB*)whMQdy8UwU*m?G%`3v7zXkb=$p+ zP@qDVzR2)mec~}@@(#Kllr$}vs=np(y<=$9uN521S6bMzFLkWivGACG;ZI)(u{a+e zV6x%1L7P-++f6fc;iRWB3yaqAG_GmzY@G4=-qkt&0kxv)I z;&{ZR!!wF1c)R7fwM*-!-543$`ChKy45s9N{mfLH`YiQcOvFz;kcc?Jf=5*msJE+U zo{Ln1p&r7P68FaT)RHQHGTfoO8M7!baAG}Fxy){+C$gWERA-j`=s5d%CUn#O-+P1W z0j?V|W&1DA#sotM&7eDHBZ8iN;;Ul^FUVBz{go@;5uk?@2E@H|uQd;Qs}k5AlB|4i z=S<}sg-c$Pj@4k;SUC`z5Z2~csVOe3I?n+|%1=+QV4#-@c^t_f!z7Kve{~;SVWN_f z-uRaHY^$#U!f@B$tkm-8>9A7wVCN+m%vL&ZgGSm>;v_h+zu*PNP`_SlT*2&3W8-u_ z=TkoLh4z2p`ch5kYg%u7#7xAD1$d3Aj9&Z!tCg&xkw^>3&KnpV7=j&aD1|j()3z6XNw*oO;xfY)L||`GME{`w#sQG7|vdoNUW0m;Yp_b68X7;W7L;7vK|C zKaI~<#e6|-Ht4YFOZh>R(_+&VH11!|mL_&_DaEZF7^_572xxVNZ9X=6O@>j2fZk*( zUy$B)3Sik+uuGZ}g4BGEQFzeTX=2thODOStpv0P<#*%c3h(Od!7vOU<8Orfz~&gMQg6l1~BK&q1S;TEx0R zPtWvD^JVbMiSkPyUbr!fFq&qmpaRGjv$Jz)&G9Q|lh1rUZ{|E`dW*_V=a;9LxWfYe zQ`&eUI)U4^tfoKFSd)7ir=nSJN?`==G{Ka3JGr9C&P^%jxUT&fX{z&Q5gsdD4q9x6 zgA4?>h1ysDXDIfu*kd)Bw^jh{U|YGZo+K{zAEaXMvFm+hA71&?O{6z>W;m9-nJ# z>fT#K7x)H#LT@*7oe$zwZJh*Ll0I1997rib0jy1K`ay+>C@=v03TmmlgG;x*kM9&J zOkB$+bzj)N+x5~EonmdA6xNe{9y66PfV}6l@K&lU*!$-{55p`0fR&&+l$F{|to1WS zJmw|s2L!!m1>8>#$7q){DP~BBa(>$M@wxB0u7y%3K$|W!l zSTw?{(T8O=IX>rLDBsrJem}j&=*JNp8HEvhM#nw?o;rKP7+Q&F9RlCcFMy?hj5 z3s3lE^2`Toj?b3NpU2PuL8egb()*o{YSwi^|tX&s5{fXT|cxtT8xR*7(_?f=ztD zm5`XLzu!TWGqnbbh_Q5aqb>~7+4M;K0T1=65X9Je-R$FA{5JVpt;}AE@AL9dZwXe< zxBMXeBI(3x63m~=lCf+G;`pOP!bO%nDRJVQHV{c)v>cKi#q17T zFv^Us?;qF1vcpk)`1_m6{vX2L0;=jfY8O^SX#qh|x=|DmY3VKj>2B$klmDH8Xn7KlX3$XFp+@v>}iNf|a%UX2Vp} zBE9+-Xyk4{$AV1nlFtvN{|4WoH((br0H+{EWdRlzg?)^Q4`nT*`QE;1KDrTjUxJce z3QU25Uhd0#XW0tFFwsU-C<4Ji63rP%iAZ8^1D%HOqzl6L&H|yC0wvqGi z{FxL_0l{P(6aY47g#{}uZNSifKpD_Mg?x=jW~ph@8Bi=j#bu;qLs?YrYD{X(RVN6W z4-boavG7mCotYHiOmr=frdQyC?>hMpXXVRcl^_d3pvwzlQ`~6g^08 zg|@RGuV?<)o!TBuS&+O(@c3$sKNuKPa!-jfO48eT`V@I#zuda`O>qyBpWVK_z5{(P zMM_UsX!!Fc^)7*rU&Dx^V^aLU%#t`*k+{SNB-CzDsPVw{-Q9nGZ4&n-F0ua=KOBxZ zui>hj&o=WI@t2S+m4n>|DuU{w?TN2tEzoty>uQ(@R!B znWe0MnrRnVUd{$SsTO7Omfy?y^IOIb&cD}fTH2d^fGk| zGZjDoG&VoGZTwP=ZSPd|*zs;3h-LjVVhA3(Y`sXWKTemcy=wpT=>uD|@UfbAEg~D! z@qbO*O)mIu?PL^MW>-)dJN@_t z>5O(eZ$8N+JpWAdt^Z&zY~R4f_(B#AD126BIkf>f?zDsr1JEguscCJaFuG5lvJbz_ zN|RHLz~7`tKKxXGMl!UDNg8)W@}n9Bp&o3^gzm3rAW?^q{ z0U6n@x%qE?afOrbRP#B}ZAtyp=Oju>OLL86BOHH%TqT*`3scZ;CIl9RRIIEQuV258 zlvyE3y{HK2^=@YGe0cF2#Pvme6a^)Q0Gn8j(;J0#^oPBX9WnW{_Cr_v!Vj4!Z-WQ% zZzKl7l&^C5LA)~9DqXLnT$pTN0Ot%c)=?ZB999Riu#n+~&z~+E1<}}lpA$yrm*2i~ zrx})D*p(%b>H`CP?<4yi9wMK{`?rD0#4sP$C(u@OCONmJHJxtd$a+SgQ8#A{8(V6I zyO)Bmk`JBx>&zFSe+tS2f%!a;5Jb#ZZ-I0Gx$Ca3uC{k|lzf~ak)zFZ!P|Mgxv!6c z!)jQ>hC;vMmMm6AT^%XNmlx({^nn_pC>9V*#m1%_YBexgrI;b{hTGxYwej2Xmh)u7 z6#woi^87)Xi=;mGq&5w>(%3txW75|Xg^wIgZ}gkq6P4xKL&Ls#;pZGUq$xZC7e@^d zAzr6PbAV{HBja0a>2>Pd6g$ct|N4r^Vn*YahOq6J>$qYtXZ-yBe#EU-V^h>Cdz*g5 z999dT8PFDompEq9@ z0$-%YdHo8=g6I(S-AQy3`vOB*tT1JKgkdf!HZ}m5M7r`l)&JDEIR(^fF@xV>PXjst zCO4BcCv3Mp45bGID1t?2gP|r_4TZ#?&*rr1Fy#DLUm_%~=pKc9Jvzjw{gTSs~aw z66uCxYt;K&mf|B!$yIzo-L}LT@n8WMHv@$9>CL0^@%VWZ|($Lh-4S%57fpR0~ z!ed3n$g;Ahbg{4F2s3hWZnm|F%kaenKhL1q*xVQ*6}k-J1jEOlKYz|q&b|sz&zB-y zUUsXINBde>=;#Jf|GrD`NQ8$#MWMmGBF9RBlGgL0DLUoOcO3gxkGopLuZ`i++G;%a z1|C{2{Tyh6-Yd5GwXa4t0dQCfd@pAjaa69~Siu)AzfY^Ew6yf|Z7Ut%di3BJ*Au%~ zKBZ{egr_kTy|rD;_$Z1&g?RUxcG(6O;qOsH&ZYk1 z&@1179PAFj7KVGS{dXNMewGgnkCe3azPMa>coktigRY>!((UHTsa0W{zx|b`@hZkW zO>PQ(rW0%??TU6-!vNJO>*F}j>pKKdKO%E;b7k<9O~}g2Vx``fO+U3Wsh;k(B`}nK zFp{aCjEz4_`%x}{8-FQtBlAgLs0PD$4D1lqMN+e~8DNVwZ1u-_LI~x(98?aAj+Y}O zclIAHv);PV>#&qmhBnQl;qaL4CWhED*MkREZ?Ut_N_`dWg3z$^{^KOAdRhijcR! zlDC2TTeNlag^=bmo>xi((Xokd<8eD*ED`#M(9{lJC$S5f*Exk@ZP zzSKDU=vq>WP`o9Mb%R-?j~tgc*zbe?e@8F!6E{6Uj`K){hcWeI#ft$cpIain>kwX# zTkQO4t;3g)`^agxO#I_}=8_L($lhteZb`J2^OpQ@0qcGY{dX*OopC(4)4&MwbwFkVK2j!Z`q0l z;dGSD(?m?5IdU*Faa^x;`Sq?0m!?=Z5ZKSdZ0j%~f3zxYzy`6TqjQS)e>}s#HZC`j zEySgkWu;|VgXtB;u^%=Yf=|C^@kD%i7jq)`@GvbI0ByO%P{E%|F?k&=;D7!LgE`e0 z1r^yifbYy@$Uu@bH_yS)I31<`v(1~kiu$13F*^G7rDt&t^QSzXrB1B!q`Z0V4>>R` z{5w+Ju3e(=@V~b&MY-{Vj#QwE0_|{He5t}JXO=)4$u$^4!fEl?KYqYCJrXD6EUv2`Gqo}Hd$^x6w`z?exsoI| zn&pZJvDQ6tPeSZUSbGTxT|f#fWw5+KCoN%pM=l@x;M zMn3G{%}@fx+dg%73QM>!FInA6;w<>sPBA)qgZtw-|NWtRfECKjs&!kAxuM+-aszY@PHt45CIDxx-3*jGW+4Jm@IH+aU!L17RC;o9@fsrC2g6hWm=Harwds8i!cPQ;Sj(YCt+ z{&AXaZf@@s7x6&kA?hwc+?&i7`q0~Z^~Bx>o8%#(95LD~&tty->wWP>WfTSzCAzsC zm1{@5DA@MX&pHybgUo z(s>}brA0)PDSy`)us_l}3`jeViy1VJb(H**r%;}JU826rpfDZ}o;cg;KF`58fpV?g z8P=I^-JQ8mhwOYuY$g}|$ocUIhd!-(_E!q?@QjDlxnG9_kmicJ_*GBoNY$xZ=I7=@ z($fu&Ms}V@MBo753Ll?52sD}L$}7xntIfe*w&%UgM`*=n>7RQ4(`Wp9=Z;d~eQLNR zC8U6ARJ^j-gKsvp(afa7YGX)%1QF~cn?8QL1cLHraPlm*9A@-?po{|UhQX~`M)%I_ zA34K4`iDt^>|HQ1!=m&bqwecyg!ubY>ohu$!MLOJ70*sgf*R;gH{#;#oeQs0=lRhFYf40E^B_N0XT7J z(A#NfYEd<&U)XUdMQ`-P$TGBbH(`?coXLD}JsPNFay2%Fv)N=-av5%FRV0jtMV{Ket!KS?cCg4dZ(l}q!4HTEB>n4 zvP!#{0T6SL3c~i66vYoFp;APAuB$}^8cphEEchX>Z#oZ3S%Mgb9d+y3fsU5e3&_M+ z*hpZ@3pcWU#Qy44OjJU&U)>ZOG1mb_!v=oz8kK)ih^EVdi+zc!Z~b!`utu6&okNEu z{wL=cbl0ch5rsGfg*GZK3CUrY=>>JzuqL^Ehgs24_S^F5sprzlO78v=RlTy~Q^#0? z<&CAxR(8!UfCoo_W9%E4m{F30Ud_V;w8B8^$UK;CAVTOeg+7H;-*nBdUr1??kdUwY zo3&Y&o_fV^_y3!<0SGgH18)KjT3XS1T2n9!ka}YhEZ@e@*ah53nI0a6MSc}|92SF_ z)zF^fmqsdLU13{BX~ANEj2FMpe zG)BK{V`~{M4~JU!eJ-OSd;(XQUEakPIKPJd@5GVo4us95=?mMb@}31(SpC*qq;F9+ zH@^*;0|jL#Ptae}{~ipTIbA9p14F)&EQl-Dcwj)sko|G5wj(P%HpGfL2UY@Cfr8}=v>e9f z*woY_mz1t8r=NNG7!GB-Ixfjww!2^il%u-}==|$FvBqq{omK+y zl)4S)vR`HH*~P>XQ7t6&Gs`@(uA!9rdcxM-mVvp~55idE%L*uY!YvuT3K1RcLjy^B(=2bas6D z6p6^u#eH^kbK?dRX@6+w@4l5qWw}$~l;5iK=n*#D{cQIbz-6fD=)^HBlxn19ctS(y z7u6E7!DsL`>tkDALhFw5$a>1wO_9|1C8zC7HcMo34za&ayNY&*1=l@yfOd zjUAKyHx+t;H=c@Kyw*zmPFtH1D-;4(`UNt>Gh<`eTR}%hPaTIY2nHCl2;i#UMA_Ca zqT*M65hiOtzaUzG zsi?>W0tYmctbwpB!W9uweBc4Wjs>5B2n|Tt_NQZto@J|oV1B6XNK4mB!%2UIl~pk$ z@`Z?#6T6Y|yWv-0%>-UKsv^nV4C5}~s*bl)2-pxh28J-uj{D!sk*H26k0N)VVq+5*WvC8 zz5_E!D#zx1GgT`j@cXqxK}bdx(a?7(W(I2{m#H%8(OsjYSPdXPHC&q%3yH>N@j}ky zOVaG=Ba_V~< zW!DJci@pGsAmG8|FxD10X*?;q#RmM$x#m2&|NOO+bc@$`_H$y&y*pGQopG@*IH|?OJ`FI- zM61ajzy9)mSSk9$i;eLzyYbG3?XEXJ_0u}eQ`+CKnmt|aPEOeN(q%SY_DcJ@TW8k0 zq}@WeF*@j`v2q@h?H8$Dx)8d0cQI}*x*j4|Bbo}GaNlrW@7{m57*w|?4(Uk-fM*Wu*~6Mp6cCu3aNE!Whxf&K|jFV8UbOt4B^x6 z>xCoWRFJQ=KRfzJWLYg+o{R*dOQyX~Hf>Yzj|r(MQU2E-yxS<(CQ-z-&N4AkiUk@v z)tDbX;n{s($pzjbiMwAq)Fu+Mf|=}^!#SUbIAdW*FM5ycBE((H&u@j=gnoB|L3nU2P*?~2+#3r+6br6&6)ZxFnPCm`)y$Sz(%_e@+V!k>kl>Jds8wj zhkmw{fNcS%`<^+&m|QB~ILNiPx@>ABP{oA)*Pl}nA)_Sy`+4~I+zN9UT))cF7`C1C z8_%XI;&`lb2*&XyL*x~7Eu52<9QD(%UViB1>Z|--j$hO;NptB`zl#y!EX~x502K{M zBtt8w<+DN!hZ)T`Q9=gVHP+QI4u5V);v9w;E^{C*yX+4kbG1f?t7k0rl)KJ8C)`yR zNgSy#by;Y2(Eec1!1P9N^p{3e6F~1Kn&l?t^&STrlZZo&!>yNEJ&EcaU2A#yMP-mn z`w1cz4Waceg(0@x<0UiRMg0H$#|J-ZY?a}2PW6`b#18LuISqIE6XCjX-y>}m`L%0a>cDDA>)d}dhAQ6O(s!hTzqlq z`J5pM+p-sg7ARPQ5I>IJi$**_D;^q1xsjM7QUx9BX`JKM$B5M93u<#x28Vs^)qXwX%@Z`x~V;uw|0>6t=dSsjDv3ZxqmL$R8=F1ZwkG2SsG1wV@%xZd)&3 zd;5yjKtUi!L$yWS;X2AWKKLkonDsszZ?EH>820Q+Vo7t}_|gyFwZ_#(Rp@qA^H)f? z?bG>>=ALsx0{LClyx8+A@8bSHoCegMy4=%30>D&Gn=jIz6U0l;poIAXp~aC?#rP9oW{rOvN7}#5PLa7?ywdG6b3BLnY$>`}^Y`?c0hl(BX*S(In6i*iNtS_u0Q} zY`rVxVC8<;XhH2UyAw<{<~T86CS}DES#0_f(+qnoSt;x^j4RZG?i+`%qx0P3wd8^^*O^k_`kP`Yn0Of>UOf{iqCN1rubhQ-&bM zb}(9lsL=vIK{*{3#ApgLMgG^f9c~1QxR8-{x>SCeil{H)tCt83b1-stbY#}!%;!Im zzmA2$qo2?8pk*Uln%8Es-B>L{&o*C+S6yg46D~Q}e7FGpinkPpfl$73b>}Rcq5u$k zcz-eA`4=3?P^Yl+*gm_ci{*SUi*7wp)f|LPZZ3J1qAY@hQv)n#tii5EUgd0WP@xcV zqR$KGX&bCMt86$;rd&t_^X0F-6q^qF``K@5L^bVAwgBQ!#f$5AF~k0!7ea^>l@{@> zAJ0#W^83o^bIxDGa&c^S?j9RiKW@aa=~C~#qh}FsBAE0(-H`+Wb0VZ)u&w=|5)B^r zT}g%HR2j{AYVnOwvnv`ibw5$rKcR<{YCgb)y3d}#>yAdY~w`9~6k}?k3N$keJK&t@!?|XlW@$D=r z_nr}6do=<|CnMe3L+rgTLunKk-KSp})~7S_LA=~`X4vzbgBEdlz0(4u(Fg@b~kKi|u1G4LR_X67T5nf&T` z<;Tm(;;L0%QN6X;n2>n3S;X9SPjYK7H%?kf&U1UFo>`anJi*KANXlb>*f8#nF}+p) z8fR^zS4Sps-0?n*A6?iDtYD(%(LexzT%UGZWcY)@|Y}V^A1zzButK@vEiMN&ChjZ9Yjb-Jiga}zue*w|! z3AmANRIj9#gIxTWeIFdts?QEB3c zJJxOIqAeztxt=SXBr<}~4i#Hd?P(V`7#LOF?3a9~J>&V)=9D1~O9hjrN&2K#dv;+! z2eSJ3ZksWbGTn}sdwv7HgdgEw&R(2g8^_fF#5hg3^j@jkfR7s5Onj(b+K2pk2czc< zt3Rizm4hxSTCGTJGh_GY-1k-ux#fVRw$}Cae{>U+7{~`2Wp(YrKK@wEd{qQl zO+TVYaG8E~72)sVQ~W9ySL|BvQDD4Y>AYQose%Ao6EVAGasZ9uXsGyHg!+}q4b8@C z71idO*CzcPhS^P#r?}KO>Q=*vEu5`=_kiFURNg)+^yXu=#6!DxVxiamYUVUb-&uh^ z^;a2euG()f>#TZ5rc2`74Mfj$LdjGWe?sqmu<^~Yety!nwg=c=#_%@FeG6e1{I5ws z*Md{W-{l+ocLN$cZ_a@klDe52r)7OO^(SFp3nc2NnmHrV$+n}^XG>O6y<1FAA522< z=dBOBUz!%_zdOx#Y=8tkx?B!7^q;zP6eH%Ej0*L7mz)4*g2s@(4@9O#} zU>6g9@s6n}XY$!DTes4;g34UUO5yox=%eDfH~oFVLw&cG50K4*LiXYIN$AGkQKsjG?2(g0UDs(7 zAoz1NtX!tP;IFQs@urqbGLlsOeiDaOoZOrH&0v)Y){DK>!&UL#;4Qmo`5s(SN6w~5 z(uxSD}(BxOLxaIXp`D8v53b~_}8YfG; zl!vYrMbjR~1o;y(A}IRa0?k2<0^QdWR`$G0f}qpU<%!TnK8q-l2JkR3XVLf*HG5Dn zKS;J2643@bGy7A639XS1Q*@DQq~)=b^Al*Pzc zSETbR(bYw&lo?=cY{JR*^3YwE66F`YudiB$1Wrb()?U*%4;aB7CKEuT>*zlP^?SMz zGQw9=ee}5CMJt0(Gh|!y?R^ft{W~vs-?sK0KjLszjnWyB7gJdh!*VeU$g$&K??1I3 zEYx~%{B;(x0bRWwSISZScluCA@nJv-OkIDcm}P*{s4#16tT!`fItG2JT9%r@dT^Fn zDb7so0eK1df{wID{7kE8C@GmZ&&CH6-#UbPSM6Yn#l5+ci6Z^UI@~Q2df~8~1 zm38-j4s$}(96uL8o@xaY=F=$FZ9u` zbN%bt;jHUs^bG{;Yvjc`>f2{O1bPtqky72)Mp$p{cfa``llf<4bFx&;OspD3pi#b8 zLN?|#!Kj}!*z)v!SYnT0lET>fnfl}}-E1b&c03Qqr*d7fbhPQ^ay>>Q>4)UpPcES~ z7;J~TcNN`Mn@WB5OFUn#UP5S8C`DJmuOeSvb5?KKrZ07x)1c{stBn5d+j}RaDMmry z8|OYo>jkAzeCuwadm3MOI;D0(y0H5Rt|%9oYj8+n9KH)=;OEpQaa0JoM;rZpo1dYR!Y`-oeaPU3ZoyQ$-r85r=s^nq&2Jq15(mxVDNZGmmyz%>4})RojN zx{dd|SU;-|kCJ3JjYtBnk_%W8Fvuaiz1%gA#_jVJcZq9@y(IJX7EgWBuYZnj^8e|$ zS@D)-yd7t~P_1y?8q7s)9Tk6H==%8@hwS@RurfN$cp2CxRcLFQ{O@P$?yhJm6Jwg#-Vq+j;4bKDH7Cf0uINau40 zTdGzeG2uXT+RGQ$ZYp_1jpA4kKU=PUXp?G8XqYtg68;Dm3(Z`7Qf-w2jeaTVP+vJr z8L>K~w@gvh0;Veu3R|&qQLc7nkz*k~x(9MBEaDnyTA-c=IKVOC2M0B0WF>5vCPC&G zMfheY`_B;t7lxCc7o_7ughbFuVg7TU-RYpS%bjFTb9VpH8Nj|?X+?)P3wyeG9C2{E zd&zpDa#ky@K=Yn!YMqP6dhNym;Bz;BOj75(Ma9>sGk<&bV-S~fKk2i?X4s3@jMaMr zIKKp6EBF%+w*AvT;xJ8^f~FM#FpSn9LoTOg8+ah!Ks zxaj;At_}}U3$OvxhSRro`|Nr1kK+<^kr@=0vi-kcM>AzAm~`II3R$HoQB60cD}s4@ z^Z5gBN=@G#*rK{m*-mxjP$+`>%Kt(nn^M9x@bcW^$rhjz#^Dq*)MG;*x=xa}H*rY0zoh#7`kB)p)lbadW~H;c;w(!novYX>{+kPs?2s9#`X}~5 zs{n1xfcWu6>@0m&q;`*h%#Al&)&-tL{$qvQAx`N=?OTkC3&UYhU zb!NLelRyH-aIrdzvq*oz-xI$Q|3%Kp+;WKt@c_o;2YOLYBMIFy%Zv#q@Nt5PMNqzd z$E&oj+2QiPxJBX}DFm!_uz#oo*-g!!3-_SZj+7;Ds~>nh+@hXv9@SkLBu3^Qj8xjq z=6ra??Rk|#z#6!Gj)#ymIELgBLk943h0Sl&_)_;!_}2N(hp_m(Ppce@6&rEpaSnmu zDZ2?ql=fF2>6miQA-i5ZeZ4u%Hm1(qh@0{eTzU#s%iug5`>O7}+d=MrkoM5%UTz(; z!*q1}Q`kv`28wLH_ybnbp-19K7>_H%ggjAuQ$y`@X$G?tjNg8HC1n`KDHUqtP&_u2 zJ7~R>qlkF2W0e0mw=6}#s}nFRtNk^ulA3*zvHgA963?Hz9hFPRKYb{hOy}!(%57$n zXF>SqjPMv`?s`M@oQ%67ldx7aQ-}|4f{e90WpyO~N3LBv+TjJjC99o9QW5 zWkfTJC5Ez=_$qG;;*gnD$*D;y`g8Eb?b9z4^}(dY(j-%Vr%aT z+Fu%@zqLgkgn_}2{Z@m`{Yd%6%o)oUXfPxjYW0%|}hqxG6mb!y#juMnDt z5LY#5;T^u+l~xWNEBX_0luCv7ovzQhKSzwRABLhZ3MK}fjtB}n3h(cJ9M(=5x{3u= zljeznc9ACCZ$1}QBjSeZ4j$$|RaNbZZxlMkS$m;@aZevHq*iz|7;5df(v5gv(xqAB z^v!0leP1oxR@Z8_C;kb&gY4zV+5lm$-rgmqVR~qBH|+Xd7imoH zCA?L81hf&Hp`p1RzdKLl%vl9jVz3;JG%1kMIL5ImO9@F*miD+#Wf~mAc&9^rbD+h( ztimnymCNdP{3I^O*WBq4fzQ!hl&3X~VfQ*u$Hb)TEJ>;TFjcAV_3p|qTYXQ?W`ymy z^{od_oEVn-Qm`s<)<-oXpv{@onl^=QL{gPBVFLr2-m^x~6i+jaG$4DE{F=hR=& zN-Od%@1W$IEXh(*QxDACs+!>}<~7Y$NM#Esjf!HAyCa)p(s|7&;X$y*`+~PC=gYI2 zB22P~$6ePB&V9{O#n;a~v*t>NpUQfwr z(#K<2YWx8pC+b*%Dzbp1JKTZYOQ%i#XhG@aU7|541{>RrmD+(XUKe-js0Rpf?gvKHNGX&r@9UGyJ}Q>Nf3ysg5YJ(a)vh}D=90+V zb`Qfw!htM7MRC*@c20M9CfPJ$Bkgzo2ab!$x3e2iILM{gluHbR!LT$=*RhAA=>6xN zMTU)+bh6N|njoHj#hgSq=j&>|4=0$GiK`iCc^aVkwNq6&6W#U45=PR49a-&i_t#9i zWn(UzQ-%o-%n-mS(F%Q)rE}3;WBPGpgKC~if8Sk#sUEpol|ApnCv6wahB3B6&G-I3 z8bc-0A%H=iklTGP$K>-Xy8dIukJM3G=uzYqV1L2v^#qU&rLGSg3V%MjVw9AnO!(<} zpJPQ07CSRsez;X7eX0H~wPMTb#Yo-x&1G37ATTYW#Bqgsv-=x5@3W2M+FwVCzmm%r zC3`16gI=&NL(;s4U9I4eTJ6^cYPAaaqS#CKUiUtMPp}pH6BZXb`~)vDL4Y>W`TDMz zy56&_>$e|&kxvz{IXR>#0a>gT#Aj@#R0dDZiv}lOxIUS-0>u*igtB_YhXDb=zaO=r7_qcrO>D((Y(+pz|(T7t8?DmhW;h56F|T|{(wB*x-6&AvwY z7sk??%EQu`C1({!r5qwFohy}qN=;2|4WK8&`^;Tk9Xye{J% z&mfSXHt;SfuwISO^4DCQ9Kv&CeeNkRX98F`ujta;R07O21z?N^|7fHjy>h1Z|1`>n2E^ zum&q~m${~kV1*pz7)Dh_Cg?pCNhWBEc0gf&l}D_?>*V~r<~dr5l9zZxCNwK&>{BC7P_Lu+)~4$_Wez(#SdC9SSw{ zWYMj8+!cE#rj}?$3sG_(3#)6HlMGC7jD6rCOFBR9;Pj{8$nT-VaxviI)UMC@StV~~2A*H6nD6l?u$)9_mYa_6kLpvD zou+z=jzwks`NZ&)*4 z-_sm`<~69-O2cSYvz0j-wVrQf&Zz8)@`8N8L|SF_!;ANie?vEPzRfUY|B>h8dng z1oyv30-l4HFcQycvqf-U#&av&w$v`2OW!9|n#mhc>p>8fWPY~9gk0MXpe^N|$6g_& zCN?5|ml)6aN+llM7Tqqy60Um%DbG_4pvK(u`E^?4a@m$kd=4IDEK;j|+TRyA>JWe0@>m(nqeAU>KH>^m!!A1YuqzZQPWa*dqSlhxSE$bXq@$ z#db}=;4Y;~{CK*2Gu*f#`KS9K>}F={Q5hpL1c9W}lDO}LIVfW*K-4rbY`%-)s)^i;0L4)aEC!Z1 zR=Fhz!i^8m#*m8~2(a}~_eewev)WZhlVg&bN@nt{ zhbe*!+PuU!B7?)|e!8(=fX|T00H5L8s1VJ+O{0`23Yfx_fhj>1`!X=aKY+7Mtvpy* z$gL0O$0PX?5Z{}0bZG}X@VQL|A59L2>y#My^JFZlE7gZ5AU4&B0+)GyWFLjfb#G6=c&ED=d+0WljTFzJ4&I0URz6I^Vq0RU9X zl#1?Jqj}7Qw)p0(c{xtowiM9QF_?;V9jmTG6K&Rb{;T#x4-UPv{)mZxzdfE`IyGF8 zoH*I%a(7Vi>PDD(tqUI#R;ZD|rHpkR<`bKH3eRFKse6q&vV2^NZ;HL6yn&dsbxi7l zEu`#J@9B{H{m5|zWp%btF#`ys;WTVPg)=b`ib^jS1zW-Hz~K(n89p3V6`+!9L8cr4 z8nq`G6FMUS|0l&`Si7$cNI>nH%3cPFl&)9hmoJagibnrf$>4>KzSPUn0?EK(f7sr@4!Tm8hdY*%-Onet=QLt}&tbpqD?NZu`b6!TIeqBwyZ zI4}Z(gC_xLV5z=RFn+hVZb2zvYo>mZQN73-kapa}L{AL&Ca2Qh;EGTl@>LoZL_wLu zMbRE1bL1Giy2WdJi!D%N5D~~Et1*>*aK%~A)~Ariw2*?)v~r4KjPTC2$uI^wc zzT~u51SQVMS+ctk`d3JFuQ%@=x|J~t{*c_2Hx}l=7xJbvna!)X*H!C9waFT-haqo_ z4{YBx!YX-uxjVtwVj$CDfy%fK>OrAS^#FiVo-PDy!&Hg7jF9R?LSsYYXO1K`K{N2> za^-y_72*sfp2}Es>Bm@fLyS;}WJx%a)S|Ry_v*zyNUJ*apY?(h$jBfPSMTLEgaf>(c;HPa5^@7d)o?r)(LG`A8b~aj zKKw?cc-NiMovMBQi#x1qrYpjkOx>u7i#WFuSdArONqTGJ*b}<(qtVdD!So>|s&$>; zeUn=(2bTH|OVJIBVV6l1K5q5$=f{gX7;j&K4>qC)AoCTmS_pS(@*-n1KSyqq;nP(M zR-uMk6;-!acjDnWB+Td0)SK+FCx$2?DO&3>#kT~h=|o5z-!Y*wk$PrE(^i;t&uIlb zL|LKsr?3ep3F8b;EYmV6%CLU;|=^>^WcbfMFOEGm=>O&UtMs1Jws8` zOij~^`*J9Hc>OB^bGTePE0O1(Wa=)*m4z3nl`LhVC%t3*)3Rf7t~W4ON$LJMr~Y{X z(Wy}~8`dW(@*PX&etkz_kb~|g`9c%D(|s|hgEy<*FSAk;B7>rtQy5#msH*FJwXCLU zaq}}+$ck$14@!y7E7!6S^}`ueo@CKxl)`d#Y7-ruk=LlY)?6P zFrAs~DwmR)`yKB3uv-L?tf<9r{LDj*ql7LPThAoh$5D+%=fyyIa)hN#NARLN)gy~w8o!1v zNMe{`muk1tLc?^=?BV)}yZBwLwb^&W@x*RG*5IFVO9g!3!__kx?Xw#{FFjNUwYk`q z@+8H5I>0oO57CtkXm~WX$*YSwPHT~=1#`5=`gxp8J zjsBk7nJ`}R!r^t1TX$&i;zHvWK6_zumgos9bd;I+8FhKwW)D$vF5ML?o_yqfvDt2el$H zQuor!{x>=Lr9K|P-&MPpgs$IBLgh5qVzmJFO6o2TCy6#0%cqL6HaL&;{l~$hyn|f1 z${erpnq!bo__VvvAU;K$r;;Q0Ql5e)l_zP+Bj;OfTY3X>dd|92-{=W0MFF)bJ@Isc zwgQVmUYa%I-+TH!;sP_~z+x%S)H`vkm8(=a>j7bbJ+`C|n(^RBYF${%YQ}n>=n>ks z(1?&-6HQjF8n0{elHg}aw#{@pg_ZO@%)();!}S;~ z;@B+HuQeJ_>#7y|9mVHYcfx8cPVXJRu^`w9{Se#_t)9T~%UdDp=;o8@s(m}r6=xvmy%WcP16x}dx2>2kM+niHyTOxKR`XdMDS%`!6DHaa$ldm8>?vPkxA zo-iKM;;maK*ZOh5@3^9$VOMJa@4YeJD}1v%J~GZC^)~PCZazF}RFEMA`BU zIBd?!h z!?@)rLLq3@aGDk*fQRC+LKX08nMhIn4}|)21aeP~pe9y$jF+ya-B?RBIg^rIFn2v+ zBhW#WiG8|_Vc#gJ`}dIg==FUM%O_ub`_zrJo1)R`MAs~$-UXj8m5tw=l%6=cB5p~( z#c(;|H=p#?-HST+tK^U)OFT8RJ7r|)I0?VK9Jt=hR@3F6i5=H;kb87zxz(zC>A+&C zo8$Bt%ofy7ijT0?;juC)7xQP?--IGZqyb)PRM z*(Ku%bqeVC_mf2w;LDRKVNeTE&6IdoQSfZUUc1!6&=-67a5vdVPfJ3NDAQ3aN901r z@-1xqG(1Z@jhvmY{45_XX}9tw8=)4rfh{R-f02C3!$`rtF@O{-kxQacN?#fCoyt2}$-bpo+ z?e%5-a=EsHo(Bf!cbVk)+$Z&|BFEEvSFA8;+0QZ?$E2-yi9Rn)_5$-ML;lC&(?#0c z;%1-I@j6OXJQ6#YMd~KIjTU`-@CT|N$BXXEd(y3)I4if?l;2rIOde@Edp1g|Bc~34 z(tK~XNig~4`$yQ#G#U};xV>Dwj|o$mK}J%ny)|dY^~}dK{G`;^g?RA;*IG5m|Ib3-roMbIP z;5zLb@0jUEGRxYrF)iv-B{o)aV9CPjlFV48W#5ArLSH4veBIi2aU=9rcl3}?m4^Lo z{LELz%4rdMtHQG(+@1%JC7Mpah|hWC+ATC?IbBn6!Z|vhXgEEdekk^Mn_TRgjyfMj z0a0Y`7)n1#NafTDG+0+&(9%(*rQPJR3XXcg{Ty%3>(|>G*Ew>HBp`)CXY+x+c)Dzh zT##S@UByRaXwygGHY?2Lcr63&NW38pG)o_ zJ0t@<{R6shDmjmvZBCzAJW;ye=_@a1q%Gu-&xT~ZiufifSid7cXCb||)0*-|nYM>E zX$IAZXaSd`yDD?c+LZ04BGw9_JP&6^{TwAGHKq~Xou0Zwp;n?bx%uz~1I+EW+zxo= zc;)t@(Z8je7;?7AqXxaqgS`vswAt> z4sqn~!M{HXiU|x6FtsBOM`nh~>xkKl-ZxrIjPMWx&ZPu*$BpKS)T-6#mm0pcS2$B? zx*eZVVK49re|bX6on*g1x7Az^+diUEY>^9Fv?y=!O2ctHl7mKd93u$O@Fwu?;xoOY z_->d%Zip6*c8KTIx~$}{sq+qVvAy|uQz&Kg1d_Fc;dn~oN_v$em2TuBD?S>X2tjZp zH0&!|k}5{JRcP`B*ZWXJNEUQ0vBj>7EuAwWvS>|D^%mHkS#LdF{_OZ^w{<;ml*N|A z=Sriao7@i%CWO$F_R{uw)Z>oA&TWoln}!gZcwRN{##XhDn@=Z)&;Fe)siR`y5ul^< z+9%sXjU>aURCh1@U_;WICUZ~eL3zcdS_|XMg#=`5foHPN$C!0>F$z7Qo#Q`p5u96&-+{7x7IAzxE3?>JkQ?y&g;7F z{)$ZrB{{;>B7syL(Z)Rv$RkR2a2z;`Zb8wqN5cHgm4$Yt$`#k{mo_1FktRq_yG`Wxuq-k$WaUsHOXfwk z?b2rj;7??vzfaTlR=L>-Zs^I-F0^=DN$-MmbNrkbLCn%R0ksY%(6ksCl2*w$v>cTM zY}@a8w9^rk4kC}vQxWXJq%)x&f|xJ}v)1As5l|5NAj=UAyq33#5Mdd4(pl-}NC-#` z%u5H585La@GiDpgWNOTz&#>3R6eW0ya2ad9&alw&*;Sd}_9h}y@c2NqZa3I97_-lL z8_mVgD%IB<_bfO6^=G6(r3FYPBAUbF`UgxYgaLv_{#0yRNHbmil%geHd0+I%BKe%! zk=kw&uiG403msQm9umKL<`{u!t58M;yG39y&R?qLI1sD()H*_cj%i?LB+<;7|HRIy^DPJu9Q(l=5t%78G}g!`D2Q&Eu+?2su9)F>1D*HWaPH@$m#d7fZtM-c zI$8Wm&~F7Oky{wJJ!%f(Z{5^tYlPKss z6w^Gd?+!Lf151sT2$jAbY+xWm?|SWOqH4WLCvsv8GPKFIcc+54)#V>O#*icN6>Mx0 zs3Ww%!NRAUTj6s{CmRGc@v*Xa`Lm06x&ry?Zd>^dJrR@^W3i-T7@$ALaBjxx^#D=& zgA|sv(J#1zCI?NVS>q zuzoGDwB-SsrBb6~ncK-%Zc|g*U~Gi0)euRkv%C+sh-G(yY&a3c`5DF;Y9Hi9QLLoJ z&$FkA%T$%((~82<4a!aOlqxbZDvtF#x`Ym{wK#5X9zRAW3pJh@h!H!*4{N02PBOr_sf&0JmNQ}GnSb(Wr{^uRdQv-~!f zzN5z5lto-pHgkBCGQ!F0@2F+kwf7Dpft=~emqz^NL(VD~LdG(xmj z33spLeiOr@wc$v%;q{lk;@K(5BO@68cgh+mtR&cqB(sm-(hYccxOyE&2~Cag?3()7 zuT-`bJt229jAg8L)EB0$UWj|S_pJq61zV3vuLzHA_02IhUPNQ5Ku|mM`J}%gO{e*J zx>-+%Kd5VUR$$>~n;Zro2HHTE62;1R{pU}oDcuyGIFl^}2as2EqiX%JM2yK5Vc|}Ab~$7ygG2XY z#t(9;>sBi#fNN2ED(@eV6mH~i4m6*p{R;X@(#DDUy1KE-*{rE1f%Kt^JhZc^;%FcXsOuOz=9@;@WaWE0v;PT$}0 z+h=W{My0+PEay$IH+(aK=DbLVu%@wIJ^Aj7Vz~kX>NBOC;EbSW9RsOfVOz!G%=V1e zg#=0y+TCco_inyvPnMo^Z=#9>`*TW|)~X`dZ=WZ;;LZI;^jN++*dJFeVV}0sm)5|u zvSHKE;een~{ zb*kW`jKWzYv1espykMU^cE}g}Kt$jNX;4cZ-!Dy=%I|_g?}p{Dnm|O2Q&0=P-ISyY z60s;&v|aI74e(QAdO3G1kJ1m_rC6|DhEi~^dq;ldDZb|8`KG>~<$?aH%Deq!_3~&w zEeyi5+?k?S`{`lxwO7$2erL0A>|&om5|%NaH*uTCw*~Quq`98cbOFg)$_71f8Sae$ zz9fFDK>J=AEGo7SLs`-XGvCFWy}Q zk}(#a@>zfEX~uRsoeh5&(ZPxn^C`P2kk-(F@@2;*ETkV~{wgs@$cs3n0rl}6lisK47C^tfM-=>`jz*4vkF(Y6g{=*eP8A%baxd8nOh zPrgGlK+qU#k-~-*trp=MBy56kE?qnj+L)|}Vw{`mo|(ciyiLVftP#aaac;WvJ#Mq0 z+hi1nYuCMGP!%h=a<5GXX7sK?t7zRTrP_XETK@IVp{DS(U}?Mutpw`$`$42?RxNJ3 z#>jp5{dKRNQlZ_>`{Xu){B*E0>x;dfrPBK|5SVeHmmF`Bswk2r~feHQ$jWVxaUsr`8 z(}?{PhB4MFEaMROw-{*WO_15CL~EYdBh%eDCesi)zLefjujNB{QNy^LGmNC^vQS7^ z0|bBN)Ig%CZ1iicVHo+wb(qrSOPy?u&?w#X64Q|WiSXrN6trr1#V9`L>YIkJO@2U4 zoIXJ{bH39(ch*6@{b|ex&WGW4a%%!t&cY{0`GAUmgJFc>UDLe$oOgrbfk)Anr|x2(Gyb}23QZoDiDi-xPZtFiw;$V2fj@u) z7sKHs1#I>9qr%B(BQ&cCSGJ48t2_L($PTErt@d72Gkh{~DOvsj2~Y9YO)L2r#n=v- z+}F4GQ-|=r*S)`5Up?Fc6#SS`*SaUNq0GvW4kL*X{Gq8kyRft?%JZ}5YOGzd(`a#6 zTZncN!?aDAYgHZVZ$QMl{0e2IeSaz&0YIrUr^ACX7Y~DpAZL^I=E@BWT8s{ z_?AV}=INZtTBmpc*bNc{H3wHssIJb?j_yYZ#N2 zA@zv~uDZ^)P-p?e-R)t-;tY74J9NeGEDa3by zC^)qS$Wr4sRPcW8Kea!0Sq}ZeP_yi;Y)sPgEo)lzbuKnR5wy-Y+m>a8FD0Au`X4c) zI?zArvu%|JJl-~!J^MPB(KJ$Gn_5mLK$ZPs23NW1sw{W{_if9T#)_z0Ed2TMJJ*0+ zm{6xMEdkAn<7UtxL}l(b(NYlK4I7J}%8=R<%86xDo8pdbm$n~xN>M(f-ppn|_S^;C zYM1EN_X$)E6B$i&X&TVjUIP)8kfMn@mPVOE|7kn^!Y>;@6+S5ihe-Y&su>SN*6j zhAR}vDEv@rK<&v=jE@hx)+Z(_M&5>u>Tlc7uu^4b+gnW^w53W^|G<-^RBE`ml9@U1 z7|Y2)Av0vD$u|QUQ9>>kvx&(5lOw@5!3#+R%jM1`Lmv-t81f;^#i!S(wEDy(jimcR zX@l=eoF2R$Z83HT@|jhAy{-gCHvEVkijYY!E@wo|EkX~|ZoLF5Dr1=fD=2!QtqeYi zx*Dp5$`A9~62xOk^|ml)=L#j(h@h?gD9I+-{jG%GEf#f!#)Z#ed0r=?-mq*l4>C|> zW)%D)`sjQDxdrk6Tqj)W6bhGrtZ>^pL=uTaUF?nYH;7Jt%lv6PX+(WTU_Jg;sAIG) z@WCkizyev{1yMzy`}1o3DUnyWG4-}^pQC~iW{rw9qs{7M0o zNt-5oetM;{qB<8M%`}WOj0}EM78&w%3MT@;yJ8p|zNnrPptTHy&3XlYT{T?zf4^!B zVTT-sWsDH{(xdv-`M5XSn*ow1GI*XbsT!?>YMB+f+%p&jD1H;b{~WH5yoJoZ7r9ZP z5qG+xzxFqZNs@+|9!=Oy$2j76xXeuNud+@)}Ws6`OY4X z))cYwn3*joT;qB!uBiRKQ~J;v@b@`@L(|GsBvbnbLG@o3Sd9|E5O``SkKyMJ2DK*d zHlY7Fp)+0#BJ4FTxwQZ3t^Dt`fOm|C56LvXZ@NK$h}xM4d=?-Y(UHhokJ7k;&>iRa zOyBGO^eX=w`t|z(rlSyE=H~tRh+mWrv}!J(NfbXwaccrm025HnBmDC_4&^9O1vX}BKHvO(!~eNNuma#U|F!}sxB^Hf6r%_^u*B#RXLez}+uoY6^$+KlA0KIc z!fJHolmE2_zE2gVfC5thFwGIrc~)P*;N$q81;v0JPJH&=i<{#ZRqihD*G?e-v(l_- z(^m3MT7X{c!XZkYAIJf1?=-Swuc52vwWicOwI8)=D_!IF)_Q3sp@pyd>ze83iCh5< zum(l{A`~VjgqV31+>fGLTwnMf+ZFDNJ_faUF6LYEI}r;O{tVx(uvY7vdk?9!bvbzn znzotZOUY{w;bK!EJm0wx&qexcAIf8(=01M{5aI|ZMgwNH9oQwwh{~-0g5&|hb3QUk^{MLG3VwuDE z%)4ve+=DhH?fS>K19WvQvE_4#ji(==?iLcZyHDm?-d43u6cQ+f4_@ z6Sdh`{H=}o?h?sgL@hA(hZlbl1%eZ50EV3dEuaCIfBXXkD)6V*VAckt1Xnl!DvFXn z`JX+qhI*EJtdcu~h_f;!`$1OuvvPIZD|s4?R$3(%;_v$}<@Me?a^nNkq7A4y6#KLVZD9MF z@n*t_^Fht?EVc#Cr#-p0&$>cv4#Ug}L3`(Iz%rI6!GFPg(I)nN)_Bf09~&;d`@seG zcuOVVmx+;7N78GW!EVn1;Sp5&hKr3|09#PVWfY6f(+I#=#Q^zWY3%-!kB9m!20m=B z>z_0z%@rBCPo6z-{=TXEMO*N7<+cUBKu&P5ygvVa+WU>b1M&;@(^gmS;}qcVebnEt zou7AAs`2jz)-9FLqm1`Lu>d2)7<=xRaBem+E>f#)Mb|{XO;4rcm(Y^m6Un^0l~#ZZ zL!o+5tZfwb%cg9e+~atw8E~D>&K-P%Lm<3BbopzK;J-~Ty#mSr5U;LWVGxQV}9 z&^!LKVN)@oUzum@7>XF9Ah-Gegjfr~Tb`?r<~-j_O={$C6b-m6qnK+lsEsY}ivgmu zA6&O)_5gtYc^F9OO)yODJ~SvkfuW?7Fa-3e689hFzrS`%@Jm!Ybb)b74sDd5y5#zu zXZnXLLLZFZqiGdtkbjBh&2e}Ku%WsMx{p13>!S1SR*w4D%d53p1w9sN4Qx=P>FMbh ztyk{O4;?*fzc?3XvJ~Wa{n*>vYNo@|q--^!{zPXemHn9w%2@7)d~Uz4mKcrB`{0$> z{+PoajLFplIO=6aM$nU$UIQ4-Cs-g5j5RfSL_C#}vQ%v4nYAw4?KGT+Y42r|2DCIz32J` z-3DUw6<_HeT|=?MSgT8~K50#b-qhL&90PCJo7td5U@{7bVhfby9+2wjD0<#DQ6>rZ zpg$NfjS^dsDf?S+Vntb}IYBS~Zl!x*%~jA{&svZVk+Bvy>I9a<9jAWH+#CWLVVEzhtH1SU(}dYT^QRW9(iA)3^()tNzzU<~u5KIHcD_s09EQ%roV>r18J zl`Fk>J-2R9UA-Sp0`2_lci=(S)pbw}{nxaHFxu_Tt>@5b83G-MJE0s0nJPZwG#oSI z(9~WG5RuUzc=|mZ2!g?us&@_u8prFAB!$S&hQ1Nit}{h0s7UP(&-{UV4j)S{ag~vfx%2-<$(u z_lkXfp5l7T>RmUm>uG$NC=f@^W;;=l+k`;t00{yM)PM$tQzU}lik%3-U8tQdgS~H? zD*@ae5?|!B;PNfQl$sLvUCZ~k-$s_RxZ8CA;M@iwgHcDYQ(q{@pgjOsk)W|HfVy*U0hX1oiBh|U`dPAG)P;=5^o8-F z!MjwRFTH*5k;_nn?vs8-{T5)Lueyj@tXE#D5&|7Q@Ns0yQ3dC?pC-y)ma!v zSFtQY!Koo8e-NBkfVpa<2fc3Tirrp)hBWt`#bHoqEE!t^-T4xfHU&U^-PIiW(l78? z_A#esUUVyXcl+;{3j$ovVs_A0LWwR#(u%E2pjQeyuO@m-0z;|LgPY|b*>Ce#n~WlP z|3v2eix+9|*n$)I>S1ffb6t zC-Pj(ZQE?|vIp3K@uG~&$$&9eN-YHae^ys7D7$LC(_mvYuzyqzTsIVZixt7RN`GmK z&xFRQ!ETb_8p9DjkoCb08(5d3awBr?yQ?crsC3wiZaOlJAIIT3Qi{GVvng;802Mg^ zC|Q58WV)9|FV66I`b>R%lc-&b+N3pPcQ#+c+=vAy4oJk1`JlbBmV2WerRtOJ9o`)< ziYrGmdjhb_u{R?SU9b84{9)Kviu^H@$dvd1G`+`Nb(8`WWv(SZg#S2jH7uQt0$h40 zubKCEqOT{-yFgKJMANsb{iF_a)mam3{ltl^eP9%P!U)dUZBTo0PN&apJcmyySTYSd zV+Ib8QGX;kMBWCTvKNP5VU}s|)#$=!-#hkyR~Djsv0VvV2nwSuYLpmSD}-+PJEU~& z^9SI2LPEa^;rt7RA2 z9vFWSB2*n)`5Lr}tCM-O${yYO3#TlUh`b*R+N45itDxEQC>KJ^79@auOJWo0BfvoM}}Ij=Zx-`kivJ zx8M&Qih!OaUVtl*2TlV4J$U%BGiw9sZM3hKqnH!Q?;S(81#H;)wDhlt7% zjcbi5YwIc+;6USm4R1&Fqz9ddIEcw4E(`2;cNa1iH>ye~;EfVo^#_+HcP( zuv#)0#dH!S#FjA3_3zZp%YK$zaR97x4Ym@hHNPMCjQADKzW(I8I zZVu9Y(xOuB#`V~>gvQ|-N}dsRPof9@ZbWmH1)xOIEU3y-3vcqihGs-(OrwQ;KnV)5 z+`xDwB`H%Fb^)P=62tnzpUp}15=mV~XpHO3i(Y_R5Z)IR3~|6!E0}w?u5Mr>;=F3d z$ePUnf33OLR)1FZ2O>_eF~xNzB%vKu!uK>dS7)Bv#ye8^S z-u%ae{w{e5s-Oz67(`%{gDordQn}eBfb|V1oyjAs4T1?HhX<<#-HsM(K36Su<;{&8 z6j&$5Ek-Z)~4E@%#x$+d3%@Bu9YO;Axi1W16P_w;7yMXo)GnDpk0_%=MkAbKh_v4RoWRHpBk2Ifd!)V& zTt^B{GW-qv0m{8M3dEtR`bdgKS=2TTe;Uu1#GaHlOo;w?x+aV>OCYFQQTXOUL`}<) znhU2p{ek_*36dj|VkaP_2-`<429O*f+zNvpMt#1LLBEUU{QX7>#!$xvbM3Ns0QaAh%=6XahE+b*b z)cf~eLy6$`3-nU)0Dy}Wgs6Jgrm{uqYi0qjgkj=~^(4aL(Tx3)GH=6^8OtxX>7~=d z8H5Bv>80D`8psSZ@1iDSSx-Qq+!kVX>njUW$y3C0oUhYEAt*rJ{j# zmEdR=ZqSd22u^hcci&3^@553VJY&>(7ZMKz{3tv50-69YR{n4*NQCpgC*T(cUP)s~ z2h0%GozE7UMrym7P%%>PY+L5560So4Y+qu({n9Z$*ec^$M(#K;HOAji)zKYid&zZN zJz#LRC;Zmh>gtCt)~h@%BOT6!q$e6iDhVkN;#$_a`H~O}Qoda#+AujM#nL7Fy`yeu zD8EVcD$i>hgGcxw)~!zWu5tZJBQIdM8on)NcQTm8Nj|OEcqISCpYQIbZm7R9wT;?; zQnA0x(_;j*9yp4MSDkFy?(ZtmX_iPe zUSY07#Ja>GSOs6%J+c>mA#Bw*!YDd;9ty&wQXkS&sHE^rRKJ(56_2id;jVU+!pXiw z`Qhl+|zk6bg_<`z9WUNCviwCn_E zQ#{8~Cgflx!mqTllu&@eMx4I_*^5j~muu&prIyZ+PBa+{4k*B^BlROa zHlJ0uDPG#2gjhEB4YiGgNnHRo7rrT|+YXS(q@|}&`?2nRM+q<`&Usf)BL>ooS|b7C4s;|1VBG}3tp3uSEhL>4_*~Wc0|PVp!Y=XF6c6lv8`7O zpYgHmoI3O9KGJR_M{&Pswgqb&9E%FW;hO9{kL&Zv-$d{g~M2)>A(;IRSjadE}H@ zxl`v3q6-$<2G1MMRi=sxoS+wt1@mSQH9QYM%F<^r-c>}2v z1$K%kMX-w?n(4XSLBV^~o`;VT(4H36@U{vLU%oibY=vZ~l%(D@cv`iIJ;w)Jgnwvj z4YH5X*{SoV#ggg;|399Z;ygwqwdli2J?=-0y5AAE5(YAZGu9~?4F*fEz$3WRc4h?7rWwDj*agc9obv}aL!&QE@098H$`h3uzuhME%J9|g z35Bs42^t13&anN=Dw92uAG?z+X6L4+Z38tcWk(K!(tL)C_Gm9PLbHYBuO0$_n39cK zT3VWh%JMTx#|2dUtI{NyKx6iqf#;4=vksdSv_l>n zzYubs`#>W?Z6imxwXZAUE;vT8AptYaQ$MYL6BAJ`OKm34cxmNrYg~SqTW9|!m2|}i z8|WXx25@Pg;6`yG5E6o zIVlu>OBXwW=?0Nt>3%qbM{S+Sk@>+6i?>G^3duO*8>OUXFKs_dM;tu3l7X2)t_@@p z7*WuFYN3%6EO`Z$LO$mT5B#M+oE=;Pm1YARTgwu)KSd$`a@13(er}b0uO{P^0NR5t znW5N8LY;Nb3m+Uat-K!&4hFK|O|EiPzLmo^v{mT+Mk!!p1x&WMj#P=s+{(R?i!@=v z6uu3p@;>kKJL|tz)evDdQ87#jq_#JG}X1aCk*0*K_&h!yLPicKhODrxV_!pI^n3PR1cDy*P$L_Qj0b zOJh7};>4pBrLpP{8mn`Yfh5^Qz*QbXl?SP=%>5iHCtP~>-qCtWMfpps0&{e9H^WoO zrON$CM;RF+H8%3q@O#u>jIQ?-_%U8e`_6txIAB5=+ZTgao++c`>uAZI_{F96F)Z&d$Mj#P5Qqg~rYmq5X7S;eW$N2ayRtAw-e=$()-b0)erOD%dy?)f-9@~A+15D8 zoLLA;zo3C}7NQ_ddu}dy-&o>HwwiC5=j)t){Z`r>2J}(_8KeX0#l>me+}yg&uQ8Vh zYN!9#wZL1|NX+xa@v}kAISJvCihe|NH$@0vPn_1^YUR!QiZl6lyJ#Sj_GG2wyk&_xy&MlmrO;qjm2MboDb7DfKKZ2(--=> zUfM8~901Z!Kzm~g%jDiJgT3Kj&(!x3BWjo@FkR~Jf0~A=P6z@5E>@96ezxC_`>!8= zntP_cxOaDPWnM>eZ2VdL{p-$wn2h09saAQCsCim{S~TY*$x@HR;3W0;`+T%9m*ZhY4TFl=!nk9&q zZsH9R3Gzxx^&lbXYG~8wRw#=7pN)rF>~=J~(a;NtU$w>0mXC*|iigw|h*>0ls-Sr{ z*f-NK;duGz;yJA_^oc6ViXCaC!;{|gaQo7gOM}!`@8iMnThP^e1a-SMy5RZw`GR<} zp4$$xe{E~YOrlcH!z;Aj2gWc@x25e61cbC-eA-F-WAmTEq?LJWJnFx+RY`iljaEK5 zUbk&)^2%P@=yXq6HJfZ(Q|*DnW`B~}$3`<%8p7k9w*%XJ%wn)Z3tfBi5)u=Gfy-+G zLO-t4Pbj)!;PTS0d&PDehZVE#_p#)dzaF%z1%}Lz7CBls3|(E_ZRH^_OB(-wS7KO6 z_9U3;xp{XV?gp3n?pxs1x#CuPU*M<~uvDZ8f1zI9fPbMlUeGOuyA&##p3q_0ri}*y ziQ4h;JAt<8FXtc?2=f$EmjC;hOU@&aE_LEZORqP-e@6`(jDQWS8R@_C^xqFe6&?sW z+V&dhfs3!o{kskQJ7rme4Y(WJ#|}eK+}IE!@jW;%_U>HSY#85~5_Pn(en=d}p&ztq zMA6XEL2JM;4PrJyhXp*;z)aL2!^bbD&y+;L$w z2PsYo@eQ*ft@N?Rk2*R!=b>24tS|b!cAIp%VTY+S@;W-15Zy^LozWJT|71z#zS3;+r9ee#*;? zMi4|oo1$33FMx-iQk-}`hlqwUtFk}5j?XN!hq z{|nnc&erO+-x<$kq)mOmE_=2|SiR6m5zZ#4WNnsIpWuVMW(_Dx6hN695P0YX1qBEk z9QLL=b}KVrAGE07CCB~R&g*hUpz-7i>pZ=%5C~!zrgJSZ`#a86&;B$kl9^=iF$+2| z!n3u>!6yQLSYECYVoFiH(5j)3Ac8_rI=Z|6)!4T1w&pTz$mjjulBO4e?yrxpar0 zmWZrZoG(IzlroV+We#sYn1jSAI)LmhcO64@RoZj1xfu(po(<5Zi~RoG5&BEG(7(?m z-sdKjM|+V8fVpRRD*__n=KXAzYoz^IqX-D zk{J#*I~BB60X!YiePI)>h11l$jWLNcB!*dpbUwRc9jz2;L?Y z5o7>aHZ<%7sfWs4xIux=546U=h>U^DM`%Z4fLVSW)R?1=@@oI=f`8tE4i4Vvfqj}J z5zfWN3p^f8n2F(gb-L(&MF{I}92=JJbyb3*7bZ6M?zLzItj6De2X&luSn(@H9Ou*D z-O?z(fOn1>VrFlBjgpXjYtX4)0@O+77TX5ay& zSj}k%o6gaplTb+#)#libTivfzicECBo*Y2r&u(ou)p*nNVp60nI)~)8Os@`~K6c z5Ij~KR9(L1S@dXXi;p&!l@jJG#z<>$f8yOJPtp9+w+s(3yV<@gX}HouAdr9t=Y9Xt z+x_6jYe2sBD>Y^2;>z9J06~^;==(|M4N7OKkM9V38ZTqR_lkMs-uyprAIvxrm~nb{ zPj1Z9u$tVWRvjU30%fc;jWXkfpjYPO{nhO{yjC1nqh_l3ub4C-4640pVq#)^+$X^t zfpQn##ful|IXQDSH=x<=4-IS+=-B(b3nT6C|;M?qDj zv3|ie`RJUuxVYc!CDx}@5ipn ziy?F4SYXoAA5Z}^fW0J}RTcf0EAb^p*)yqbgE!mQ4b4ki%U=@Gx<0kmyDZ%yQ>h?F zvo&g-}LtPc z>|e|S=pLqhUGBd3T|-ptkhgD-$8`Samd>@GR2M#mSfUuWreFuIxB}Z;{Bs9TSJ`t=d+)_F)2vy)v5w9c$Ml?;2Q!46t}bJY{6v2k(! zaK7tcG%@tn0}uxvipAq=|u5v+JpPQZFgI>q`Eq>Ov@hR_33w-xy~OBX=r3q z1u}UQGwjW6pl1D!>{15yC22<|ygaL-qJnV_lS$gvpFxbs*Vh-h-x4~N14d+?E`ar4 zHc3?mc48v~G>`l|?hI5JY1D*3T zSk!PjiPP#%<08Fz!aA9$VueaVm~0Tk+Yh!2&Wa23N(Mma-volD&XGdkf77r+Iryql@}VZ#PZ@n&t(!#oOn^XQ4f32l`+O3k_ za=--Whln2$>)w3blLn0qosVYNdN3M=@!=z)f1Ysh5fu2S8rWgzjA*32)~|{JKSG(Indw!{t~A$o^^UW)(^3ffN#(F=Mt&SZ%g|#MBJJD zY1Bw`K9{CGa&6RZcT6qVCn)&GIJ6fz{VM<^cs++}TqN|%UflHhADhQF^b*_vqqt^D zb~~<>eoAoX37RTj%dUPsvk*M@4CmGl%p)>^fq~Eou95@=AjSJ?xl+H+*K?#G@?oGN zwqxG6BbRxIhUU4apg};&M^)l?=Nn;3`1zsB_d&CX8efixHBg)RsMlzt_9lTygj@MSjDetU!#ACDC`2FekB<&%y=TN@WdCT`FBV_O7r@JLz z%m+WQ-}gSRHzNy&Ho0Pm0;OJHMR&Mg{gVbzQS5uWrIK3+uu1a`PLu^}54I2vTBwyG zQ<*aOh}pBuk7v6SIR7PK8c;LWVLGhkU~a^>tya~gcSdzXM2X)w<=yrln!O|Q$V)Q* zgifO8o)#EIycnA!7;o0Z#Ob){tZnp#jH!YTCt0PS#@uP#?# z)7+vdX&zc!T#7U&*}E$+y~E%g92M0JpwoHr^z`&8h&hYbM5cz9zw!VwOINq~`X7c1 z6&vzcy$@E2f583;O(1u{v{Na;=*p0q1SC=?L+={)+wyZ7TEo1q6BUL?0BETmWc)+D z#JE&N(H!npTx9j$Nm^+9bml=O%8C2pFe8&J>MJxwh{|2qiftx>YicAj-oAYe<=M?F zJrd0ym5(Xp3U4&)<%SBV#T3s}oMnGP70B1q7FslV==`C~ydn4vZPxPZhG&+uVBVaZ2kmJq0TN&!|^N_dLdK|615#x#C6Y@LfOlA&xTs_;77j-MQb}YmKgDiUl z8Z;GHM#OUk^Ah4M#lDq&5gEeMVjZ}9ElrvepHfTHSm`OY?_C@>pV8P)?iFMGRJYJH zwomMJ`P(H@qJpHvy5CoERFF7y3Pp#>_G=`L4QR%5E{r)2BZ%rEtQLGfS1Ep|GOu~6 z#jd+mY&R_h$Z}$WAaqdNtRYbm^@ZuLHDBX2$9)d#YX$xc0`EyZHr9XciZ>>6gS>lj zWOFpub@c})rgPG5+ME;|3`Bc-(N8uooMwt_ z*W0h}tgwqt9NApj`)VOZ?6BsVnUr9~`TXG{qi>odu?}lXdfNWEPGbc^ZI?CzNLN-kkcv_asE|rd8&99m43hul@*7*(<w(t%8du#PP=W3*?#}dHFk8Wb9La(l_^`XO0_|iCswbeCw89!w*H#r6K@7R-Es05 zt6C;bCqZ)C7*38xr0?A@dZ%dvI-g_S0h@7uj>X8DS4`#?i^?Fq**ks8=a#R%z$PA%S9c|T$f$FFE*hSJ)>p;_{_qNj^d)`At z)_%haGM{UGxty*ca2=m$F}57->4^JPJav8J&sxZ?-PNj8lJhagbk@>RbgWLZ$N5T+ zUjVg;Y(QvQN5IMi4tSu*vS~L}_Q%@gllcAoGL!`p0|8~y$2g0m8z*)?Tq)+?LTsF_ zT`?c^Y(&^=>!-byDt2jZcBA-3GT=O|;_tvLx9yY1WgfqM`L73#0AB>cru+oW*LyL#>-(~YY{tJl-yoP|74K5gv9zRH`fPZ;VbWzj&FdCEOJN_w z;A*sSN?iv&B{NTPneqfD1?_`p(@CYcHXemU^+~Qqi+m)MNAFRNBS1^|!o+yX$JV{r zq7#jLq2!N>>xwTlC-?+>w%Av7Ptc)ydoHOX5C}qT&ciTj>)IF&f2#`VABJ7h_X;CK zDB+?Ao7Xdlio*k4vrN72j3Pqfyl*kj@3V;-B-xz08arP?FA3c#{nDU-3VD1G0H)?> zPZG$xxf+sG>VrxRZJS(BvEe0KG$;h?l-e->sxk<2np!LV`f5FT%93v(#!7Ui|#tb8H-BJc12=#_Dv(e=mHQ`%*Bk(9ML=F#h+4C^W+E+rSN zqbDg|>8a<^eB-`w@5yK@2JW#a>5}FoO;URJl}-1&JZ`i~gX=F56AeEaCn+@Eg=}<* zJ;f?Qb1h2}05{=XZWW+-Ht~bi;T@p z3H!Egx24(aV!k%GT~Tc)&8a1Go~J64;74xxy30eix8oUE2R}A6(aB-l-FI0y)zzX>QJYlc7Yd6Aw7wWUT)Xl5gGrcjiPT%QG+T5R>N1O_cev+n zew)&CjnY$uJxT$tE+O%9AO7k$V!x(=+YZ_AEB4Xu$Xz851SUT?rLcd&25h<;Sp| zTps_Rc?y|{$I+O&v)Cm)n|ov9t{Csbfyv0;`nG;fHJ)~e#1UCK!cA(**`Lh{?F3V`KfGMraDSOF_G-Gc&WZ_n5VIg&DHQ2XG}iys@qI`QvfY zPi7aV-MZh6Zz|JiKW3Jlv*xH0f5ELU!H=)eYI%)FCq(h%gAot%Dw}a;^vFhivu`a; z{p;tfJ#wFKdK6--6jhI&+OOU4$6M*$CAdUdQxm9XAYU+(@W~y|dd9bkA=ilG>ryl) z^Uh6zitc`@e#hK1x5mgygxZ3AzVBYgx6M4Z4I=rNGvD$iSCNl5l2s@Ao?{C;zr8?G zR-L}v#2LCAD$gPuJj=PyZ>@bkTu>0=H=}&%S{P3Nk@M7S?&InpsR;>zkI%nN2=LlpwY7GOE|v3 zMwo(87%+r6+9#4)$Zu@SK&S}LJ_MRiSW`WTujd@KDi-L9RS;25S^iA4u``+|@Emql zD>Ng`Io}wpSTr+gdU(F4PrBHUwBPiQ9BVAz#V8u(hwRO@CARl1IDv>2 z@|0dvPbEUfoLQW1nN1e74h;EP4jYMKgmA=L+9vH(EibB}qmG!L%E1~QX;TU7P7R53a7e8?|Hx{YVGcVD+X%#O_h2-6kH+PYo z*m|RJhN$F&;7fDgQZr?Z!H{o*dv^rB7_C2gK{l(uWOgsNm+llHw5v<9 z&lis;-F9>Dm!tCv;Ly%(q2mgfOkcc|3hUpt=rEfL2d9mh2l7ROD$ffr$;I#OTUKb- z7#HXuDJX(o;7bt=JiIpXlK-5w9)BDU<7IbS6|QsqkDBvYaxFP~-fVr^6m`=cZs(e~ zVn-sa@q$z6eZtv+rTerWcW2_SQ6-;g64-wnH4xtt@?)Uko6Nj^ke-Xa;+uuht=cLB zJoBu^ERlHQAlnx)7Irm;Z=*G>XMU)`h%v!ea$ykZ_D?7U-Z2SQUDt@k^ry1GU960; zsArvadnCblEaF&y*YK_ti8*cjK#*e=)%M`bU$my*h zm$IWZE%oyf=bq~icGDNWP^?lJ>tJ~|(-i0iS?GJk&V@7NDQyY0clST9FB#}FMKp-n zRp|+cPK?~vnh5Du(O3^T3GrBJ7qa;H*?M=xNxu9gV`>jjUpGN;^~=wNe;+PtRAL|< z@1SMrYdcxycXj&0=H6}7y{?>T?X`8n>o%;ETTKn0dLOi3b>f)`8DX|*{5$uwr)Zz3 z^oU7JyyWvK<+Ud&O;7ELD1UPXlRcpZy17JDR2=oi5a<{~WQm%n(4F3- zd6psx6MKJP7vbVeftK)heET9ZQfZ9abgLcL??{lPEKgbTv9Y(vVCtW+D>ghB<$Y6z zWn(scPhXBmOKi+Q`@9uRm2p#*&Q~-SrmjUh8~1N$EuVPGa_{qG6lhKCkSAHh?N@}@sd{owa0eo=cPiD~VSeSg5DX?UVY z10iK*(uW#7T4Tw`Vc%x`dfvqf`{U?> zq9vsq%H2;3-NOO*y?MtH)ZO|YKTQ`WndpZHK$aFEqFrFXUAw zDJQs@>Mt&y)f1C5mG?J|==cm#lk70?_s_mb(ECW=&-i_1orjHlowM!SdUk^33|g{p zB-P2JTeIRTl_aGy8LS#@803a()qU=-({$K+C^3=DI^vlcsx~8L)37EQSV+b8qIqU$ zxv!oc0}v#6;7$7%3bx#EyKYnagdQl*a@3RIn}P%BZ;^>51C+xu$D{9;zg(}h6(yRg zbHk;IoRj!!F_b~5dpzhE^YS%WfYl>*-Lq9Nz)J_1;R1Gy-mI8`YH90v7sKw$Bq28>cev$*Z&n3*CXGMpD|& zCJFS1E9D)kThLEL7lh6Uyr8vX9CLKmlDb=USxw;fwufd^F*M0p^JU0L$mcg}zZ2`l zZ8dwY-&<~e)P4OHW!MceiGT=?@$L|T{*B$yZE}3r6m_qJFa_{Le+F=cV}gG^i^q!J zF~KJ)G7{>9l|C6xr?yMPI;?c_`)n?EC3a#OYU1dV4Kfy)fiMHD z*XA)lHNi+m8!0`^Fpp5yhmkn=+Ze8ebI(kZl;&NfaHD>z zvUYCEqvu&Ta>o~8r4}~o+V;USM0vE{j#9NiJ2+hIL~2umTKAWzwn!bBr>7^6I_uHf zqN*lYhmpd0iF?|AvU~m{c4c*f=fSdrTo{wl*5976cv86_FnLrh6o;rn-B(hnFg8`6 zzOzgkUH$Gd*&S?>i{i%zxQFYgKw#O64-~-y(bj?SNBs7re{B<)=XVuUoP>IgK*O%c za9>zEG~8D@^_jXluc(0CeT){+T6p;nTbS!hPCf!#Tq}(ic)F;hu#pL-MMaO;-Ii72?v$O@tmCuy=RnhdTu%e!Y0Py)CO2mXNF>5_D$|pK2OZG=;_vu)` zRMjU)?Lqg0_gv};F4^&BV!v~I$!8LeHl6tSc_>7Z^qcavYb<@3>o4A(wv!W3v3=_6 zPRW1%nRa%y6Qw$@l~_V=Vx^anCyb(e>szSmgTe4lS`+TOmx|oow90i@lACp-UD^)=(PdE0H>ai+bIXI7O9%m)eydRCP`bx+?GS z=mS#{gKSYR4w-*I{wngI{*Ko9%_LD%5tLZ>S&_?5!(YFM|drjQz{`4fb8L4No?Oyc1T0V;$HU&34aYcUQ z^LvUgp=h%KA$`;h{h{=S>XF;yt!H$JwJ#wbpJ(n>x}ZYH@Q#{MXp(H#z&s^#cDCpH zv1YvQ#gG9w#U?7-ik)4hNnYxDWnpJ#UL1^w;Z$c2XKlISSr=aoC%fyXmb6&`m2WpX zlwIHD-LcAe3Yta!nL7)@BJ|m_Hd+}Qx_!VKCcQY=Fs{Wvl{x^;eVCwj?Pf2*3q&|g{ zA=0f8a(i_2A72;pJq~}w;36MjMJA`d$avi<7trCZ+1S|pOqRIQ6Bqry2;Y=wXL zg7W)0C7EE5+Z^KAE#pe2NzRRD5llCE0{8q{BZc zO`bAtgpofj3)Xf%Vv*)OH0*|l1%-r2>-SW@s!qJ7K*rAC)G870TP%i^7~ON2B;myn$)wN@Xla1R z>;roIF+E}SYL|LGtT%+RsJ=fhPpKIqdYF#+sXif0cXRrxhaJzSPQdW_Om1O)bHEhh zE-Nd!d<|tPJ1V^+sH4C3^!gr5mpm-DYFv)d2*rJ%f=QOL2e1?boaW5Si;>@WaB zT)lcVWyyOcYYK+1OE*+#s-{$B^@9~krmz@S+VKYhk~EQ_7H*fw)G^rO#=@@VGp zm*UyX&C`++g#at#h1V2labRgy4E&8TcDL7|)=f>urT(z~#7e_odw)YXxXC(||3fZm zyk!1tWk0RA8h)R1<;rk@Zt%SNh?ort?O+F>Wb{>!JRP+$XTixQVAqj~&tnPY{xKYu zw(z!NoR)7(ZRT|DSdrC-3FUf!S^hcdAoqo3oniCO0$q?cx0e&sA@ulvh#)OVncC4+ zhY*#zL|H`R##ILEf~P|wQi*0aj{AMIkId0rrJ)tX2LRz;sIv*n$>uteS5&jLNf_vX z{9y&cCwf&McmUEPD5hwsG#qP&B z-|`kCFvlPR+0S^#hXy)XN{%~rBN9wIK8EXNt*@HXGh++g@C_Cj9v|wPw$GuQu`}Vz zgG(!g>s+%+6Nad?vVY=|%((J`e^;`Or65F@yY(Cf(4vb;)JxUM;1&K{G=W~ebiFEZ zKY|B6EjgFN@?I^WG81w(IX1B+YNpaCt5Zj`$fQlOB> zv*l&HQ%>h9{eDS4q|3a`0g!{dsPsqM6|Vj#qdu$gu9UDkKen6@-W3U7SAVq0(cRhk zOx1~M9I^Q^KqnyeGDXV8!nE%L$B(sC zZ&lMPaHX4iuuJ1;7XqB?rA!hI4jpXa+yZxQiPtq~IA$s=3p>Ay1c30Jd{3rRogxog zg58Gif5`_KK^IjF-mDzWaO=}a@`~j$hJKi;yJ#mh_5EK4pf|2wYe>&CfP7P0EpgL& zQWA`%aRcSTJ%+vOq$VnnQ2Toq&VJ&aEPpEzFfXI>EO*z`&nY7mwmhLPmfPOKA-lUe;Hci|A40H$_g z-T| zxD|fe>cy(xkuUccft288sICX}_W-tnb4!mUo#!WkA1pPY%&LPs&@>$g3E*aIjkT9% zC6&>=bh7>&ZNZ`-@tv_fwb%PzESJ-F;xTL-orVOHVvz;3^}2<5*Mh?={9mLQcCcOS zCA^X@uyDiAl0q`{v;*7LDmTBu&+Beflb$#%pga_s1u?vf7hs|JzmF2d#~kp6)ImRr z?(HBbHw4K#&5Mn5hnqW)I1HN3>TFblgjRvV>(}%Lw+vj$wA{>`FaF{Jh-u6)=FA81 z4IAp^-O{BMd_NLT^Sa7ujIl&ZhvbDoom=Mk44ny^SY}6Fl6VRK{O@N(;f%4V%B)<~ z-vVoU*{9Z8b3(7}!KTA4({TudoXQS=G*-wC&yD~Jh|>=LRn^>(@82KZ%KP1X&0yz2 zdVirtxuM|fADPvk=`lzpWR$=sd2Q~HPsmi{vcZ3Q!v54NNjBTw?-z22v}49L%~8-x z&XfHrt#jL@JEtCaqY5b|M$QTS!BG#9k(@|eEU*W1)iaZ4uN?iQ%g0FD)ie>kki z(={CNsccX3b4>1l_1o`nR7-HPATG|6<`crjSa=8PKBm!EmK}Jw&r)Y%w>n!uVL0UU zZ(c16rb`@hV)VY?=8evt5-{LiK-_oRX(TxkkLSAGL*;@g2X#l$10|12i%rA@b3~XO zpgyHcU`UhBpFYxI>N-V6Tv9aG+;o#)XV4LSyukUjuopOY?%Xk2SbWfH);1c^V$_E6 z#=-T2^+v&i_46I++LqfssH>`a8{|+;96cCeez8a)x%w|32AXfyZ=1%AS^x0{=ED zF#xwTeIdrLeWWp+bC8J?vj{Jl0gu`1<~Hts-f8;@TeXJVJcCtK#C7knfaSO^bXBm$ zr1OQJt7AlTwEw&obJk>z#5G64wwOB&c+p)${!(R8d}Fs=(lbAuxn2ajYmf-mrBkn* zFxwrhX}BNm66Wb06R8w+S82YZ3(`Q|=>W|$zzv)~wIr2nV zWVS|R2^|=|k76@6^Rd(Ssi0<{aC3v4+%GhyFhwezv1rBNGZ^&%)YCN2Pj;0HC|9Qod3qZKLH_C10+Lt>FLBU{G7!+}OLd?ozJ310U6Z6`FNuF%%=Z6UGr3^R zbe+t<-!LQx{Aa)au2CxEMhkzg}rh#Yu+5pV4O$cHNNd^VkQu?!C+&t~Q2 zT zEM=s_6D=X`{~X-`$tEi}!)C>`lJcUiCdf~JtAA-wA+kXF%s4*FmX+1dKuhEO&BF$= z(y@<1dY+$>8!j#@mq{ELsF$}!)Rj<_9#Bmi%EiUH&%Wr6lr&7h>TU>mt_Vsl z*K#b1kP?SL+!n;n?rT@zDqV!(+!uzJk&VHopUAoI+}hh5rxJN92dGr!OEwxezL~d8 zztFU-GJF<#m*4y`Pq7P{^z00If8wL4KTs8uEF9@JuLQK~xnQOwNxIV^YUb*!F!zkvxR+{MhaoW%Xm!xtd@`PRuPHi_qJ3z zld{}Ysa$=2rCs__8kl6@IPHg%v7&d!ES8f+RCq09sbG92D6GaO)emo`xPzeuRCy~I#aIbwB7(XeUVb=^ZzHO_pGwaVL)Qv~2d2yKWjQ%+nQDXeIg=_Qj zIBC`ss|7JjReyPGGf|24rVRm(5+h^ddu(jPK;ESh2(NN7h02u9s-)psS(p`VR^Mug zH-7`kY^|2P_5rFkXBa|BZ(-4wR!v*Qh)3G`t-n#ve{pCC^b?!nTa!VJp+qIQ~; zGG}c~%Fw0SgA-|vdiOb=kFGWDiScw?J3q!j$5uX@g$KhdZ*C)xRNnf9vY9>9ar+$i zc9Rdk!JU(U%9UmsLmTQ3bKK4^RP8VP(sTvVK89_-tCmw4q-)`~nBC zN`C}-dtcF5XFRw;ORFi6Qc3i$h4iw}he7{j-3-;Mxfx5#{mrM0b1YBlvKYO-a! zQ0Pv|+t7A@I&vwLKSgeyJg;VA-(xi3pLV9?=g9lC2==_{vXiv2>kL8ora*u-1{ITU zEhXTQNfm;2)h0rhd!~|eg(6^$ov2oz(T-V<|(!XAIVa_?83-8Lhl1G@iQ_R4to)<~II*Ng%9OY%%XW}@E z&ir-n82;u0J^j)=#iP(%osa1n=!KYxMI9OHxVgCI$}HG|5B*h4LgEfkB+4cpEzoP= z&=M3_9mS(>Z}4pr6ZvsEJlIu}oUX&osQtS1-l*0+Ki+tpz8vp2J)+a=1&z^stfZ}R zsc))M0n17AT2*uDZbcrb5Sn|RVx_kQ*WcU7noVtth^NYXQrHoNNHWutRdzYbSOQ2^ zcL*S~KIqgU$ZZK$Lztu+tdOpamC@EXZ%U(u%k6d-A;7}z#W!I1_c=69- z^G^^Cv%FJ2VnX>GSrCd<3pBsCDhNzJJ9U3BoizWj@${l}c4>MTrG7TXU$y<~9EWR8 zN3Bh%wO`JC-z4g6^G9J|N^ek#lF)6iT}^wtyCc_W9z1venyEy3dU|^sXga{a6Tay& zK5HIF;vqfJUz&89Ebzb=Vew7isFKwPii)5!8Qgkf_%1fk67O4qv%ba$UDayG>~yJk z+JY@Mj_v?ROc>8cqHVQQXPlARH!y*pB)x6l1(KE>mfU7f8=l`PW*z_Fd~(!z+;+q# zO07$an`FHi+>`b^V_TqrK|2uerXPrj=>&XWj3hwUIKONuXxYAS_(UP`Q0?z0)E@}Z zU%OtB6f@Z57ud@FmE8v-d$VYJ$pvh(;4z7c3*=$~>2&^U(}*wn&m>9M#kB`$H#F5^ z4R}C?w`F%c%`~o7js9)V zyx|jnyu;czKiSwuK7(&QicMQ{qau>rDD?HV>DF}9b3E-}li5pM)wIB#^nYa+f37^0 zC!pA(+=$qbd~}aXw(N_XKc8>re|Y?$izdAG=jm!ZU1NZtI_~+(l1u091DuJ^dL91GKi*UY7E*KM?=|Rg&6cdiI&e`xh~2hJd~>tbM2f6*Npy)Sfd6b^V=2jm{ZGQ5&7|gzq#1O|0A?YAFRpSCw zA3-ZdLu1hv`$7DB{YyvkG^Eu|IaE}eIVN^B5~s>cRVR#T+=Z&%x##%<(kZ^VXlq@6 ztfbf_K(*@Ax(x`J6}3wc;(n!_beyWJ&CFI=bS=@+bt}fz(_F<~1%gNto)6@1)e**# zlvW$g<7y!}HJEnChp)oKAV=H2N~E;AULv+IW<-QooA@MqI&9co9lhLN$9k5d`ch7g zk4MR-snFzU?eT%c)@V@u!h&5)mpHhigd)`czdIU%xuZ8<$SaLq^=iyH*O=XM_@C6tmfzPynp+UHdaE@ zN~`J2`>jJ;K&seOEz7Sn^RD!=QUNjD`PkNh?7eBi(F4twRC$jyUwgXWrLLo^e5>t{ zXc>@H#`YYeQy43CD z5iY)L#LA*)l)EjV8ejut2U)0-Kg~`)#1spQXa9*$id0uy8!Fu0-RC*E?f&&FvR+{# z)c|r{LqCX4xm*iuhD`qI@$6~`l7jCxKy!n6-Udfe)cHh zq)xQ3;Hqy!n|JP3u<<;0t4LNbIDK7wk%{N9t*A^!40Au+5CCIOfOCR)Z>fi*Y#h7& zIsXp5Jd?@(me0;Fw8i~ety>0-SzPQl9d7fO)kIjuV8}%Am2c%gs?=o#bw5YZ(uz*L zzyq11BXyV=ZPp8E$~7qoWzrX@JZt5CUi=>Cg`3TMeA9@Nla?U#VdOCN51ld zekF-{0AI^87Fxel(`X^3&9LbM0lCrEFaR)QPPxHQHb#9w9+Ooz6B(J9%z}}%rZ10V zZ7sve&A1`xrAZO>k zWBpGMB9Qub<5P60t0|55CrC1@seMAJIi<@2MeXPD!Ep8!2?>d^b@3}$NrWE)1G|}l zBSHA*`pBa7kQ?m2L$U5lNo8PcgDUA~p!7^YHofSAhvn?AvMA9&%gx7&%Wrz=M|doc zE}n^34-V^3+-c_JuBaqh&qTVKMOnuV_mf`;<6cw=QV-QdxukZZy;aLR-MthzPW;Wd zYqMEZ$f3r7cGbtG_=t6yFd!=n%pvKi#*@I--W;7PkS^ZqU286LED0Wvo#Tz^7-df@%w4df6S&CAmO>(8t|4cAhD4!8qvk1%qiIR`IACnT^P zF3;E^c?hZT(y7MJ;=FW1sGj-Vqy`h@?b3#?&VorJq zcUx!x`G)4B`teaLhNJ?K@+To`(_mo21!xunfdJqkbL>}Qjjq1;_%Z$}nQrU&quu_; zWw%TkxRob=FtQAR^4UP~K|b-<*6UDRfD`JM2^)uY`h=A|p|xobV*`Vrf&i7*WtE15 zw5~5DL^A*eM!O%)osk#hR5GvE8a+AO_7SCFBh{#5#D2n^=fqSm`h}^1%DE)?-thTB z?5cRqzc8G?E7MUUd{=#H$xy88>Y@!JGXF$R0G~&sb4j5Im>07q z%+rmJY>bf!Q_28>_1G*AM|Zferwp;hEFsyP^^$&9V1u-mCvSm{sQ>m{#{)h-3K502 z>A^{e*+h!G!ImnJOFe!n+&^BLMhY^_{U1Bppqumj#?jesFJ^YY#ntsb6BC|H#?2S$ zHvl90`sMKTp>g1Q!x1qTlCyMu?#em{885l+AT zOJr03Xlv$B7*bp>zs7p^?iJTM1~5%iHo^ICYBe4ZB{@j(b@$4kdZobq_B7<#6Z*!j zALTQ5wvwgduek0Om%e&!d)}DZ+j&>cGmQPA?2aF=&7|nZ+d@i+U_Kh+qn&Qf?Y0=b zra~BJ?f$rItZ+`?#4mpe)@O>3q%*KcbW%T$Edu1{g~s|&HsZyF5Eu`}uLdpM38*y< z`Qp7zOVo9kcMk!W@W?HqPmg_CdjLu=&}5(2u^1Ai9HlUBowP6!9T_7}#d3c7@Wx7_ zOv!YjxVXOq#Rt<;SC}0a z!;N~(TH*p(Xgyg#2t2B$0!Z${q9G#^i0l{ZuL zy&-X*)55bc&-i4MM7Kwy(8G1*bg0$-HoSiL8f>g2qlOz^E?p+BzKlgZ=5y{k{$6WD zB_1K#%gFHqaqt>2wSys=ac@Hf=TY^LW=s85{L`Jo+eD;<_7)M@!d8m$_}!hX2xlsz z@jLN^Xj&`6^jf$KH?#<*1+8g&d9~)sO8&71hl{fMLirC_2W>_E?@zzyN-Ce8Rop&> z87IYu7hCKgtn?BRVO~}&MY-2ntG47Jlt#(tU#-%?U(9*S%d~MlZc*L1X{7U|g-x_o z`%!`~qGkN$*FC63ZCK(r15N6UIG18#)JrH846Q#t;-|p#ZT9G-N2m6gdd|T{`1QvX8e%QU1-&*Ag86s@9&WqqnR3q zPV@J@KqlJCdBQFbRRJHb1%mB43r`l-RVckM79$poq<}#Fvj=mw$jZt(kQuD!QBqN9 z+S(T3!ZfjH|MuMR?x}&|+Wz616`?0xb*d8V7Sb+BF7>?ysTaYF>-uc%dZ| zuGSt}N50hwzJ_ilFW@ke$|kD^mTy&4=_l_E>{m?b&TzQzj<4=gcrT1SI)97Sw6Yi= zfE44l4ZEKrO!t=~rwjf-m}%ndu%RC)o$WrmIL;{#u|P(p$d zr`M^El@U7YvUb)4nAy`o56 zxAk%vGVRsGi_@t~CnODv{;o}be?P*ojgWdC6uqPHDZR3TZzl*K9HnNL>AgJR(2FtV zs;Xcnu3h-5wFXo^oUkwh@z!J#XGOUOhjG3ctOb?L`Gs(19%!5MFQKJT(Ar_8ylYNO zSOwm>AkA`%`JDy2dY{9Jg2!A7_aoqsIJMt0jcu+83bJarG-F$d$r%`2!nm?Hj_54G z6IIdE4$o-N{Ps#$m?* zq(M#mXxi@p0NUdq{-HM`Pn_RNO>x{9y@GZ1y5wJ9R?4mzXeEk!ykqd`A{rctKZ%=! z&9(+7cy9&%#RaftX_mM=L%Qy*!QMnfY%E5>zpi=pU8i#$RTPA_Vp)|R1pS&Of(kO_t;^t0?L1Ve zhDQ#HQ)a=|`3oU>5vHzUD2Un%GfP4`9-45%mx`}I114JXY{ zX4D7S*$RYPDq`@FA--&?7}b_RQ6#YIi|TT;JEXwbJ?xZ#qlo^mk6Ig`nPI{yv^Z4i!{_B@^O4W z#E;#-c<>tA|LBoIZw*AL_>6w#>?spWKI@imD9JBhsXTY}@_C}${sBaTK}w&~&4wxK z`ePFjV1I;;>LARc*m_*F6W2_ID{Ed=RmH*Q3dDrun!=>xfHrHjSBy1bMQzF})bA%J zUPVANkBXuGd~p6m=ea?oFaNptZY#M$lx$YtYlCP65CASB`dYS;jm-kt&f+o$x$z^Y z7PZd1h+KSD^ud&u7xiNVNQIj-yNM1d6o1GJc@j2K9Kk$lO5$>M2la5{fus`0yb>M1 zd6d>E1hSe-|E^EUIKWKP8KNA(sb16wSzzvtii)Bq&6pA?>jh1%DP16HNMB!gc-wV3 z^#FXs=1^>1x#8uB;Qq^o7z|zR*PA8TCEe&C!l!jbrOmdML4Oa7j=J7O`M;8-uX#%@ zTwSpkT`b%K>fMzr%;>xmkVBz0?*q}94R|b^dIg#JiT+)u;^~8#D{jlaPxBr9t_#+s z!K2nt_kX(bAEK`yHO&bkq!29mow$Z1=^jX-9I+#-xSC`8mf;l0YN{B&SPQfnw(`;@ zj2}XGtR>^4s#Rw!&P=!9@S4e;OvJG=Iop&g&wDB1@WpoKa=_evhr;jB+e2o#tT`N z!pB9rqB42=EuTMc!d$ zI|T+>6PuKTFbXbaGv()F0J<(mmwf;g6#|F!Nu$3U#-PL?$+u_#0(CG=_Hg>vLPkJf zhh$_?QGNoy`7VZf@yvp`s$(j1(#nWthehvqPTx;|e{B&^2y)PggLUSUN| zrU&ZsWh`L42z(J32iAc-*Iti9v?&;l3Lr`yRLGw?NWMJlz%|#690FLTHq8;R4y<8m z0-8WoQ$0Y~MMj#Jb5F%M?j-vb5g&tp(n>CM3ccgtn*u`JXp_knW#O+MMI2Y=wfS14 z3=PwWTHnd<)dT<|7vM{7nTQjn<59oWOtcOfJoeWntE~yyhD^^nvJY9Z1}`Y)>-Gj@ zHw^AW2?zdnR9=@%@g%s87%JTDF}uOlj9BNyM!(yJKUV2Wk2luzdoHL1zu-faAeGm> zo1U_JCi|`LAU`1{*J-Qh&Di-NB2|wnIw9=88xF?jl#%T0nWNSse3MqWETw9TM&kof z*fs8#>L}JG>ZBgMNSoc^d5zo;in6_|C36SC_R(*1D3G)kI+0IheZ}eyeKrRyjVjFO z#`~QkWh@XdF#%|HV!wmdemvYprO3&t(CpO0!H54NVX@xGQVo{^(!doW1yJNttIMx9 z1T1Z)#Q6*Of8(lZ-!^)=LI3;YN>MC`aNBzSN@pFEZ{N@cSJnQ@t zBXU< zwD2vbS_Pd^Bp(gSxw*?SR?E$thb;T#b$MQay7(f?VF%R*Cm;JQ#D&e^=HrZbY$kt` zaj)NSa~^2r?>a&+ij!=@RUi)Obh)(RLKtsdvo0VQy#NtJ&o^^lV%%=jus}A~v3|e| zh6DUCMt2-WNQ&s3+sZU>TLr8b&!vHnIKmgem}yly7XUzGLT2tKNpU&j?>OcQrUx2+ zJChu<@TpwS^p>-jW_H<)qaH@P%!R=lShJ86iGmZhe(@*$ZprYG!HZhh1OB#5mRpnv zpv^e3{tKn*nbU=_e~7=P;*(W?=byXapI$@qgYL$E12R};!$e?}ohU!UyOE^{62=`d zN90Qd_1tlY0xO^s^plEiNm}%!_?(x!M|; z{(CJ~MJrO(?9elEao9>EHI!2Y>1rt`_o{rp@_Nm}v{h3SJT{@U%Q??C%}_0Z`pzw* zWB0e`GG^q29wp7}q64yuBjw=_GI`E0B?!Qr?e~z6JfsdUc@X{>i|l9pThq?F!R* z-QHfLB-{9*UW_&l zYmKOD#n>QK=g0aEA_R^PZ_^EkH>WlGTArm@s%0SbAxVk-yzPc9eiNE@&hlf1b@BeF zZ*>SLc*LQEBqbK-!=7$lZ@%_uM%ysK&=sTCo!1UXp0d}Zj6yD#%Nvx5*2;x6He#vw zLW{3OG+D27yq0yyhe!t{qM62%E=&fLS=rOS;%S!!tt*eRF&DhHT)zFg{#b4 zbjdELMX%G~HvjpWc(ok;Q}&ADo+FdA{5ak*G2?>Fen12=J1nabq*re1#n<}dW+|Ry z=j06t>l0e3jAhuqfHZ}$pfDp8H;0$+-NrS~iB=KAis$g`w;QA~uzeGoUtwHe(rNND&;-x!F;-vGH<%KdH;)|>mwX9(bMX(YW`rb@u z*;%m2LGjGlg$iM}clx8EYRq%KDsyF?o3qG33bXeo_PBMML6b&-ExdlZg%C574%6|tUre0ou+Z83DWb)28L{E*rhC1rKr^4E&!Dq8fXuplXK z{;pH68f@eAa;0LO;P*=}Ip?X}LbF|81bNM~s>Tg~Fo3K)&9Icb`k97$&?dp@{|6PGBaGaG@Uw_HX?|S$r{ky6oX)c|M__;8EQ6{(K<=ljpms-w zdcaJqbZ3~P=jYFNSfMOEDnA5Fe$oi$e;4#mVHxD3Osmgi#@)Z(MZlCW%`&qGteVbm z`*;TSQAFfy4Ge~u=ai{+I*sT!SMkeJ-eex!wTY`%l$r)@*NS+1CDda@<%KwGu6*81 zcnBBNFxq;u?AK*k$+{07gRQbuPKoR}gYdhoc=#KLYR+TiA>|3PO3g}6?nIn9A%*dD zdC5t(IZu8v{DQS;0*bM*T?Z+clf?im1OiFU)|azrJ{I(c6V41u_$LVK2}=+%%euL3 zBKK2sgrEzs69s;M@0B|w|0w7xz)5i~e)+MUf$%y4+VgrO>P1xGq?d989zc+z4AIn(ap@|s&LpA^S6Qv<^#~rV=`uzQ4itj76 zov%EgNXOK!XV9xThRFecua)p!us>L76X6f?f(&6rXa?gNCs>DJ`>LJ8@3K5$8yGIs zfZ<#tE>czy8wwZMcoHgSY2A=9U-O%$oMZg7!APs)yP|A3NpY~O{aT)Ge*x|Kh0Q?T zYs(;=_ue)uEaidqvJTNOXOh=Nv9@~N2ko_N)U&Xjilm-q%dtglH&ZEuR_6M=b^q|= z@tDp9%5NTJkGttB93r_y%5*-|mEP^HKY1`hpT0W$`!3QxWUZiZz;g42fs>t=9)BXs z@x~pI$wjZjZ~W%hO_@GEnGAa_c!M830JKOf2x~zf zz5taYjna^XK+0QNpQ#B6N?&i!V!wK>=FgaBhB6Xcgl|S!=IFoGU49Jq@P50f-=5;x2&l86$77Axtm#> zr`Is)p9zx2r{8q?D`VI+W>50AflH^En$F_s|1e{a40M2GVC9J5`zgMPDyWl73~MUt z;IcdL#KApkJf|_ShP?a1m#Rau!4P+SA@TCoB>%Vs@|3I8zGO%hXV~ zL!KPc4Upnxhat+)A%G!l4vQi;$uvVD7PT-dWoQE|L8cCreEg85I;4 zq9ipUw3ILhe?E9(lp~7drWk@bJO+N<-AeiZFW@20jKO(QB%ILeV-KG z7*unb!+k_7GLGFAe{^pTI>+k)|i>~rVbVPF92-tYnj|NlJ$;bOODGDFUr%oor zvSChREpwJX%7PK^POh#_H8i*l=<3Clyq<|mQW?=?n>?`qJ_-f5eqIq}N$$QPnoUdq z!bp}BmcJRs9Wysg*+oSa$rlT(h)aP~W`90Rm3%dvSyY*3AAkncTRC~~DKQEbbN0Hp zxFpS?GeWF7_*CfN9Vj|_h;KdPBoFr523WI$LPd&1m*h6>-(%Fl2aug4DK6501un^q z1)%Z;5)(n^kKu@@r0rG0;bSU5LzSX@2xnPsvmt;V z%bpt*yj%V0zO|is(4(fDw~O_1lz)l10yxaX!c&wg;hU^3NIvD0apx%yp_rk&lB235 zS!*{jm);0k$kRE4{28gz6@d0$iDz227!McBV~0>00IfKGx5k%9xtwH8CP;%HA8g<| z;+;c{XV#Z>uz;PTeeJ;@g%c8L9w+M`JNfQgdCBfO!oS~;9Br_pyIzmo1MLDuAuL?; zPZF2#^OFTSg#IEKyj-)Sq$CegLrOJX-9RsXbmeqp8&A`)``5vI8L?~#DT3&h<0Yrh7nmrCoCt$>c)IA?-^gSib-qtP6Bd9=uR6Qi$bhvftTR&wB8w z2!n!h=8H+A96jHDKuv|Yf&?v|-qSWZ0B{%?Tn`1}Y zbLi?j0L&^oJ99rhK-oZ$)Q&4|MgC>^z_24Houq^Y}jOu`3jEp!%stwPqpwp1|7^zEfbIi;iO=90dydYxU-Ml z``^YR{?!n01t|%y+kLKJ@4t{#mIO- zYx4R2?mJoQiBVDpzP@XudmfH}Susg7RPN(>G+=+al|gy1Qd!YJBcIbKacQoa%MhCWncM<#6 zFXji*`kB~yB$SoZot&$??3pQ}u7YG_6H?cfJMmH2d#yZdV7I4`5{hoOk|T_M+&ta% zy41pRQ;7M;5A}(=0q*lH=kO58Fig!wBZl`PMH{PGfP?gpXm)q2oz7--%rx1`&d%;D z(6gbdes<^YSmG5Q;d@{QG0BrBK1*4-ijT_q^+oKSA{1LfPTl6DdONrPS}uFw_n(|k zG6OeU>_Gh#0HEko9v&VxMn?F+=`_hd?vtbk#h;IU!~!fsSC5lhqJKRU#R5s-TxTx6J;vcT^<7)53mb=_`4cz{r?l)c9pj5wg(IA~1PE^m9PF`u{Hf z!bj4~EUVnC9~0#BoHr*0jJu+|iUcP^{*j{&-T@bK;!tpl8T%3T66r~QK=Q}{_(MpE zxk-N>fR1E$vNk3^Iu;;ar8qw|a!%f@#m!eD@c#SHbL! zGSF{&1b8j#@S4BkcCfHcT5~$(!gGX687JZqyBnFe?*HDRbcUdwG_ktQ%9j$pWAm>9 z9EuwE(Ya&lLkP-i8gA4Mzc8_Xx2jdJ5bW#*C3r}ee>a)pX)~a)<%1Rvf({eH_!KFA z{PkKMm@`0g{=9_{)-4tbz<2Zi@qjCL0y_nH$?>bQFp>~oJU0J8FKA!S;njtdTkz+1!@ z0et)rd9I@R{zFD`)bABqPG-Y}z&!?X2GcqTF zE(DqRe}DhzEd7IC30x|p^SkqGMKR1$lV84kS=o3?uGpo?0um+?-A91UF%H0BTTqVZ zu$lm9Tm~{SK@Fh!iNIXe#yZHml^^bn_kwZfL3GQC?u<#r1-17_e_F-n9zul}CIny$ zRDe;O)!kswloehDfw#jvoT*IN)!A8^lynj2+X4`v<~#pN>CeNvt{7MxrF!ZlA%X66 zl4W#BujIi;qMs)}lX}ZKdUqMq3&Iiv$gj}Q)7L4s;Ya8|KX`)OGt$!f#3b@Sp8Pev z9hkILF9Aa61Q3Q!4mSk^)a?!%t%Bii1q%*n_d3v%t_OdGL}!H8mGy|`8OBs)duohS zX1^C8m!JG!f6cShQ<;y)!N?S0I#Ub^IVstsk4Yh+QNIX}rSjG^U9Ura-UQDxf7wWqJ8+(3Q`v$d*wEQ3u+K zVnxoZ%lx(BOQAs08-%;hTnO5~=G4KKID!WSPp^d{0ROP;6uI~(BGq6}D6zN#)R zrO1QTDz>_zhcb8rGqKZWY6LFSRj@vMQUeY~QNcWdkbi6N$txf*xsm-m2pr!1w%HDMy%|1t0JWS<|~mQ#1&5k5So47%Ff5VJsF#vY=@rM2Xh*+QCyDL>i&Z>q@*_sQc!`* zAi?Sij|CkJdRkjsjh~`a#6zKwmsDX^4<3xk5HsEfJ-V?O0T7B;pq?1O$vGR|yR8yL zIuP9Lw9Mf<&}mE~@_-))M$KY?5s;W`%(g{Qx#V^zLhqtELgq!2BGAX3Z z6@g1t2qB~(xgG>2XV!4$0D&Am#7#9}2Ok1yZb~6g97ThaGhtU#W)>S6oq;` z91>huyM$i#1C`haVg{!`vRP4&6&qmZ99|62zXZp?YtEII`&>O<2l7z|k!>jmAO1+> zx?i==@3Ui5x7{O#73FMa^34ZBl6r|$KwDM!8SM|*#ab|nWdFP4x%=uvW}=m*MfL+{ ztAKv@;_n_>Sq{R%eT2)#FrrTqHtIQ!sH$PnfJ*NH6WtcgxLmnQ-c4oC(X8{$8{L*|^t%)=C>vF|`R5kREJ8wyo~CBia^^>lJI{)f9X0=)Pr zE`lJR>rQLw%!qH^Jm5z_uLH`slBZYX(GnkA3@EqNOT3}_b`Wc-PJoGj$m*;Dmz{nL zlZ5dG%9aRRTlQ6}AYWn<_6Ci7;uO`6kiAZj;s6VuYToAM-zUoN(v3 zmUEHv!UaIG$V%~V>N~DO``{eWC_VQSe<|VB@&WW~t`%_E|KYEYTn9dMplI5e(?Q{+ zg!e{JClp%-;y9rA)Thv+2R*5aFZ*>ry;k9XnM(i;k;j0#&?va3ra9m5gj8Xu7#0Nt zV3pnjZ2QZ0oNNt+DH`ZI(AmUbWus;N4FKKaFkFkvc1~CAxhnd4>cT+7eg)W43E_l43F0Um!}F(y@P#TTm~TxAMRqU z;2>=%e93Z=LPv-Z2BW2>QTERw-K2HoP7CxagQK_^|ox(t%StHbyW*<&rv)ddwM)95*)Z~S4*~rc>zXO z)UcPFvUN8onIHuaV1|k0kJMqFV4RE-)YuN)-rt>)e0M;b(+SF)t@&z=Ko(Ys1p>|n z6O=lVh_EO9;4ZeFf!`NFL7C2Tl z90vV_<|w0TL~m&vLWy3XzT|i8HRpt`Z3s#BRj}8!l>7{23G}6#h!=5B0h3}B&;tvA z1s03cZBkssksn5D)$T41wCfxe78mlv-=4m~A@{dMPyyj;^+k%D$N1e`Yl&&+?v02A*neu+P_G-r>q&uMWH>gZvh#l5?*9aK-b(pe2vj5K-t~cz5Cf@5hbl2Z&F(`gi@!+ zAwrSpcUz7SnhM?*rkI_m6;BjETy#8pJNoY#{kzOH&Aov9>`uHYb>?ir|svVa<74c zyt;9{I9VQ8Cda!mgMl&ISAJVhQfjrS&7ZgMaQB2f`-xQI##0V5tg4^x|=^YX4U5X*l`{gn1 zs!PXD?;py`Br`cIf|~Xhg!@uD$5YFmulv=>3v4*&Jyyr!)i#Z%7%u&dDrvpMhuWtJ zcC}cZwhE?6hFU|aK%T#3pe#2<6`c;6QN)oA-fqZz*)tq<1s3TIWnI!N;+W7LdLc{a zcz8o%wE!Y=3=x1;bEFRZlMV7#KLB;1vO=(aj@w&~Xr=*Y>akoq4b6naA#EtsWIF3G zCtR~-K*|=NC$7sqc&C4j!=vCW>I8Ja&Mxhi`b}@(LdEHfaCi-rKZUTuymcpUYmdFE zd=e8Qv@3G`x=kXMLChnc>}Ewz+L7sSV|(PKnH9ImL8_j`AMkKIdvZ0puiO?YbciAl ziIRst@PFHM>4}1ufnT(NUq#`LVE_j_S-SV9CB~`Tq80}1rOc~dJ z*Vtl(_-^+{u@~r}1bW@x+JZ*RG&D4GFZXoyGqsecJ&)9vQ?J)UEA$N1p_lFK?hu(I zqwfw}=5zvVt>~M8)#th0MYf}pn_aTffwB|6WLFQ#yqyI^D)b3t8C+7f&}+nTv?2Nd8duC7Uuotx1=L75G>L#x80YQiqwegbbfHm4X z$R37^J^UxIDb6M;ugpik-#soz51c+rr{h2y4JaW4>#F!vsw;f~95nf|#NCK~K%D+} zC`9rNZ?!Jqt=6u9eco!eA?q0MBpm@{{fcb1)6Vc!KIn4nAo@|2ufgqcMgQKri(tU< zfdRnUbn@#Z0A4ynoq!8{B4@fEj`t17h`Y#+Z}XsrGY8a_cI$v473f1m zKlul^i-+VuOoxsQp&kI`62n#vYpCOk1ir-$-G){AlYf#^1fGX`{9bI@--~rO08kw7 zD~bK+W(RP5tLzg@J|JOTu8@qnt+O!vTAmE>$198m)Q0X?WaU0r6Pf`Hh;0Fnyzmw9 zM~jxN)zs7wi*p`_T5RJ0Gg{*$noUhjUE!n=Xh@L#`03Lp!%;`kf2vZN*GO#a?9?0b z?J9taJqmD+rU5Kyzwu7}n#rV2EIzq`1(p%DSVY|!|B}YnbN_?ypO)-IkTd|)$U@3K zfEytbM9*w3BMiS?n4C8Rz2e)EWS?}f4S6U5OIXIyke*q5T`tOQ*sU+Qv-UE)bZsf= z9|!5*K%9gS_&n0W5s@c@RKU~2E<|?!Mqh(`SZ)9YXGBy~g>o$W(8E6dJAX<0A5f|_ zIPay8xv$^s-kA=%ONLc)^m`4LxODUFPlzQaionnsW--1pS_ZeoEl~aiNNK_G%&XPh z49lPB;I2G4jmA{lHvFj$Is{1~_w|8>tiC0e%EvNhdHx3>f>+W!0D8-_x)!qb6L=q# z!egMGJ(q5RZ_uJA0tU+hpLxI&iUd7kB>z753VdpJ_$8g_Pdr7-4F3pSFCL`gKX;X8 zp+74E)Gu~}%9hnJ>|LM7|G|#Kzk0w$(@>XE=1`+}Zf=*{aZf2A`iNW{TkH;ZLQk_4IHwe@+}S&ET2 zU>|(rQXc+1;J7PbW{P#9sZXxN&L_Exe;+!Djx0F!3;Ooq(&;1=l|-E{?lS~k2EfgA z>*vn+LvlA_;pgu!Y}hlzLv>4g(hCxQo*ql_V<4`3H-sO{r3-=&7wqBO!|4Oh1OZ0N17L;j4`~G_!Ui6gleBG)gfJK^fhASG z4Q$gtf1;DtyfxeP6LE~y5bXeM&u7u&H+^OQePBGUx)cQ7CL!E$=KTMa1vnxAxOzDm zU;QWK&8Vxu-T3pHL2|aWveUanfZ!y6qiHCzpUM2IMKJaoo@>T0xJbLQij}B~V2C*^X>xX9^pjeo)(%JUkPyPAre+Doih&oRi+TL;PTp^FO z7sUgsQhv7bS$jf1>Jwm$`kpHV~ z!qVm_QB;(vWOY6=SX%69&EDEe9K791Cltl)nl47>>Gv613ppiVM_1FOQj`W`i+4}& zY49)|)&1P_j}5>}msU3cKuTO$$3X><+K6uV@CwtPU-w^zFPiO;T1m`ZBdh zh$gkDfc?Lsu;Ls9z|tWm@2}_SCH;gg#+fklfqw}Z22b_bzm$3W=l=#aOHttmjIIJa zuY_274{v`|*AcTR5-6Y0;2HK3}Rzve^OF5qFIr}m1 z&;POmpbXQY>CnS>xu7Z^+#-U28PQD7a^-JjMBfEwht7Fa0Kdm+1oW6$Lv5E3gnpXk zsDb?6AfjpPq2q3Dk6fwauZJbk3Ieq(lQi7=<^bFS3A2+v(U)lc)h2LOIzYjUc1KIC z!8_6oi6*^$ULX(q4C#HeZykKk)*(AbS~~t}(X^ob{`{{!rbr^tGjAT*tz>SW^ z@AnI+scFXj9}AX7IE>egMR!F1@_|^Kh%}@Q-MV^vs{oS;+!t%M5AOddDO&2lq7M;v zIe<3=Em2*R+H=$!+M6%Bv^W0ac0@R77-nmv zfkgQ3)myxOEmaUK)$WZ|JASFVq>&=X=rkV#df%hQESeMwyBD@ZG#6;@Yx<2e?Vsy> z`Fo}2MA70~N?-!}lL|N{x|qkjm;cnCcSz9E1z=Bj4fgOk8=ctZFWyEg-PevwC5hP# z_7nULwiC}7ZF?rn&e4ooGVhdqJ8d!KcHiS?f(O|9AkG_J8|H9OG7c*J!;5*eH2G)# zl&l~-ybB~~=n;wk^q6LN%#rPrsJE++NtaiXZsAHfnDa{ny2^NXSwD`sMOSpKFX>1S zKPdVw5XA%_qOAhC8BlT54Yp*D`__%WwuBddde5zz@LM84YV19s=`l8oKy3Nut@e~i z*neCs?CRhx&wJEc4lb_z`M^hOyFVr_*DZE9!2H}i ze*3pU+@Fdq{60=t07DPluG#Y=rW*-y%{e5`EM~8sQIspeJgWWXqnBV9Al;G~&yB=6~ zjDfy>64tq$@uY~t>Ax!YNpuo{d2#P3H3RaH9!^v!X@QCoF3M8>?o@z1l+~=l#RohX zJzm@I##eYB`&PwBi0{)zJRz?@`dk|+e#V2YCSE!BTN3W3=j42lm6KZqXt%osNE={j z!rtP`mj88Wep&}`Ui3U-KaG#Zbo-NKM4KK54K?LG#=5p~i)(_eVtVo8zP*PCRa(?2 znc3Om-bsC{F8{7#o|LVX;p{p+6XKEIK@MTQ^n(W|wrkhM1u(`l!}ibxg+H|tEbL#; z4*&XQ&_Usk(R0H2B@Np<{Jvk_9MLdcTG{7!Tw+hzYB8j(l@PN%ey&ng-u>NUc>v4p zU6ez&e2W;rRAA?VYFR;Zw0ppxh%xcoe2-eZ3V4ADwSCS$|HR1nn-%}0gBO508M<6v zzyAB_Z(Z+zY(=b;bB!g~?{~fp7zv-c#bbpil-QGz!tKMZJTRPR0BpSb$MTp=xb|FBS+mw{Q~wtcA^; zQe`W9+w(Pcl~c4?qgkGAV-Dk*W@fhVMC)<1m4QdQtHcBjRo~Q>z|&DKB4uA*XSQIg z+;N`dUjoc51PoA_0@+mtv8@kRcy8Y8Ym2<}@UJ@&ND&)|Cpm-Oqri?gzohxLku{s} z($GBOLH3yUn{lJ;m!#wF787LwMm~0pS^3>1^h~QujoE=~eK(u*=S@k6{ilS*+`I-P zu!!j2s_Cv|4p{RZ00q<6)_x9n$oQgB*kd0}yKEQM{sx}_?40Le|zq4;0&s(kOI54u&-*~wq+WzZK2@<`!0er7adH2FqJ*bZ4jCCT9&2!|f(Ui| zogu;Yv{Vq|Am&(Du}B4f7_Cux>X9Jt_9ya4Lz@m?c|+rSNohMp!{Z3o`KCFelm<&4 zZt)L2)pkY{RoQLL&cC(8fu#Oue;N2}SA|0F1H1#fkKo?W8~M!z_#echpm4l9I1$eC za|P@hZ{O|iy{{Gkgfn|w{b1R|%p|gdK-K1=17q#G+FbU-@x=9_5|Js#^1-O@9#eS@ zd-_u%6u zY?)e(G3#Ib05uwsQ54{yU3@g~z~}%ulW&OuqI;)owUX-tU*)o5PbvPI+5Y}=ILDT7 z7*i>xvw zW*-fF>zQh|6|o6lDZ}WEDbrR|o_ZY5Jhqza?9A}%BXf`h^Qo@UvtPgJ1Aa*X@?cNU zZZKyIqwMoI%*AKud5TJ$>pAFYD^=32t{pi1}a6EwW@(5{C z@W79+%5>%S{HO$9LX&RFe#RmzpP0`h_k|1A?P=rvdyDZ}4{#5VRedjOktGc~-OUn1 z$moCQiU6=sG6((T6XN3T<7>x3>R|+!M5F(QRlr_Q8c%hsdMSpX6MRzas#sMen0BuV z+nX~JP%75G=QRFbEq1_3b^6@!O8E#HgZ0UV)z=Zc1SJ29X%{5TH+q4Hnb?SsZx0!J z0uG|^kZusQ*7vJ$h$80-0R#imVAcSpt^m~_;~<4>r0Am|^tTZk!`l?Go>L7!qb8o> zm8vto-;a-qyE(i)FxwLOGTL5QxEA`~mMBQXbxz;Iqw;577Nicw0gQDH^4CT1mr0gJ zTu3^^DjfXihE;C$a)XQ39|)FqO0A;VZ?1NT^c4{N<_ zKUv3>6+mkx-Mjfu_woLv?wgH}t~StS)0ZBnqkwPEvN1}}g}`<)93b!3VsUmw(ibmS{ues^{c0%{ z_+x2eR&My`*dUE$KB%(Fq#oJIdf_pC3v2N=#p{p7=A6VosPz0({9<9i+X8K#_r5JMNPcm#O=9~{ zv|tv>=1K-*bQZwALxL=Q~$E7>Flb{vm4V zdQrr^d}bv|ODz%|{VF~U-goyrH62vfBU)ZjUe{*4pWasd3F)|a7k__^&M_sI$b)0c zEUWv5cc1a)M3Rt~<$OR`LbZ&v^mSZTZRpc5>PG!(iIsX?JssFJ{_y3 zN^nyxoStvFwOhsGs5Wb2o>e5ZsyahLzMWRJl^p%1SIg%?f>`gq+7hJW!^hIH1?Pg*4`EM2E6c>M;@QR`rzepqZSl*XDejOR}sU)JSsbY!+P{m~wS-<>9~F z>bbIFT!nZ|kW@e|ZF{M4=6d~Rzh^3fdWh+L5H|*62@c;^H5Lh8M0^!Fxsd z#U~89pohGSVVn!eVgZf2%LC(SO@ljUFyto8V@&}AM-^sin#U+4@#E7*cl49UVnOK% z_vPCkx5&u2B?TYUA*kPlhK3jp4m;hom_v2j7MhuUN_dfmkPm&oCVK8HsYknx*8V|1 z+i)S0m^SSD<4$^^Y=aW&x)WXWbw?ldDkg@dxrF$7Uv%_LnOVfNS|c9;(cuy0oapB- z;}PWS50(p7jebL+gna7g;P0re@`ig#jkp_ReXSX?NdG?~yY)QpICaB|NxEiv-r z86jA8eygnC`dyMN%jKlWgPqC9d~V~IM0T{-swuoT$hi_**M8G`r}uM`*pZmDpI%|Pa@yRUakhzt4pE* zVEiE4I&7GW9>IKp&W?yko`#NQ>CCR-$S^WYTvW8PL9=L4^&rkst-RL=6Dny5rft1( z6#@kHgYWa9l{eiZ#JVpJ%9;ptyJvmj8#-~qKIc;v%JQwO{YL24snS|G#VyqVk8-O2;cghH%sgu1} z!>Xi;zI-H;93L3&h8gRuM9)OEFBerc5JW$36tkY%Ym|D3bq)`Qwan&y+q`rTy(~qP zO~4vTwUmPSfm=>Hh3rkS>$A?neUm-8UQcK%WyD2~oUa+#NnX~pu;}=Pg&V!qT;&pE zZ-afy;cGGC<32uG7+Ldi(P-4lDOQq6=!MSgMCaX*w+OA|LrM-_OOrYd=Pi20n!(D3 zDYVX6&t0F~O^!`N^z}iyDv_=CU$0Up_?(;c+?Dw5eY9lRA&}u(zpENl+Qve;zK?6m ze`rhovgSbi1!6uM(ZE~NJ2rf{AE2PxG?zNDIdQGz>-OC(18lA4ac?2+(pEq>{lnw= z4zjjGsfbCqsL*(@%9Q(jK$>HnAQF>^Gr$oA$pF^D(t+AS{ldJmf1|ttXFkOFmCX7W z21os})_-wcR%>Hex(gQ=*jBK7L;NG{7#H`RzrWYX(sP&2pP$>$E7qF7<^qq;6yM&0 zOyJV8zVBtA%+*k)PDQw-AB!hX9{1L?DlkG1ciIZ>&VBt_dqS7|(t^(?xih9yRC`7G zC9U{Aekr2JNA4@1`VfkLpd#tDAa+KpCj4yBF~se(ZQz=3lIR5==I@V|v@Eidq^tXh zpO3@`23O)KmdHia<|Fl78pcZ&%1=nw3VaZ&kJQ%`TwSzZ*U1z@Zm9ERcM2RYU2V{# z7UD@3soZ!+`))y@fzx5oaBZknx5(6_)C@W0In(&5RSuaETZtpO))=T2zoHwuqI#XW@vU}b6;pK0Y^n;D(#BxJmmV0BW)p~ zzzZr>->h)pflQnq<`6o3cxQdzjWwR2qo5__sF2YoBZcq&X2ERs;q}1{f-|(7xiRk> z(QEG;;=Fs9YCBo}jGBhZz{8=iZq1#? z=9S(4$6PwC0cA8AN}XY9mDHo!iPEut8Fz8rbgKB~P^pUxc5`=h=$o5ggJ%4T+RAzk zqajaZpVZac~;A z;3HJE>(j|T)jiwb<|;C%C!duvmxn>P1!y|NEtu-vDEGHS=k46}%BJszgbQY)>T-{% zhHcN@!am)D2~gKfcSsFxV>fzSgF=Q$pjlzT=MPPUwgs}@I7Bad)h-s%?{&x0*97pH zHB7?t7DnurG&|>b*q~?ji$p3P94cTQ&vht|tDnu`_~thNKRsT1HYMNrEg@}4j#;vu zmZkQmN;^b2+q-tdw*|{dw)T+q+gRf_r#Htze=9!7X~2eRmN~=>`a^jA2Gs1x%&by+ zqO6OSE@7K7B3ou}5v%)Ay81dR!M*+JL7_bbvJj%1M*Ra=Z_m%bnW29;%sg<>PbMvX z#b{oCUC$=d+{)FkmQ0x1EjF3NT12erM8IE2{`YGH@1 zx@S})_D{x&c-s;xpq;O+PCI6cd$=g(|6Z}vIRDTJ~;JC82jLJHQy^;lu|{8GuH-BSb;1)ktU|xzR*I&a{)F4XpV5nqgBhX*cdwKTzE9fO#ql)Lf-NVGhI~0ME z+ddQ-%GwW~$GjluI7?rpAMqdxYyB23nh-2)u)IGcl8_ecg{v7g)b?I=$Bf5niaUVr zo0TKl1DRV>e`2jL&<{X&==BOYJ|LDONIaL~fW&XeztIZwaVQN+vnU=$ST><+-_< z3B4DxpS<^d?SX%A9yb(v1FCQQl@sQ6#W0?&mbLJN>X6zI>vLHuz3x&0n}MTUN9+Oy zN7ZA?D2T1reIFi7OoV`N8E??)*mtMclcl>^Vqp=~)HN(h)a-W4LJl4$qgv&TZ7;c> ziIO|nwzNx+b%uSO3BBWEGSFKvTHJ@#!a1}qBO&~^v1zR$d{1}IV5?nROr((1@&;>j zjRYrPYyi@#kyiNpl$QHCiGG=XtXN|!q=-Z7Q}vMNF$eW7l86a%>NfkXN&DT6O!_0A z7}MC?ea-;Txr|Zsa69<2fj2zgc1fb4&>t>U`f0>H&E`z)IV$@F@%^G(bDT+n*@(QS zD(ap=&(LVyLW6bAWUBn@XExE0Srn!LRm_f97U|oZExL);R%wTl(Z%j<| zHBFJcB_1(FR@Zv6ksW^`o7(6>>nLT*iDKJ`)giT4C5Tt_j!3C>23xiLl_k!EELs)x z7L>ILUA1G_VJ*VhPbV;;`<@fNE1^2zKS!Go5+I79dxNs#=H?!uYqP4AR

El||0! z7LJ;eI+w7mZ}-Hup|!*~FHg`R8gzF}s~XH$c$kq#)NSxpc}Ivmqveq3R4-Fnz@^19 zRz1F){vN^g>YDzkGhn#&L+zJEOqnZm|l8v+(o?$A8K6b7{251{*TP(}oT&Ud}-WqUrGv0+tDIOL<)1+W*ZFdxfnRn%7Q2uMT!y+m+oTlia zYBl&8zFwfm^%$|iS2mky|G=^S>;;gc0O(Z5HVx$VHpEP$?|hi!bfOgRVSbu^>hoa{ z5oG};E-y@6yWJBeax&@`^XT8!KFd#Utvg>tKMjWp);P%stsFXi<28uygmBQCNaziTPsOQ_9tAl!>)R11P%ekB%;1C+O0po*B&i;wBj=K_;@Tn@@#YOj` zF)nuYPS}XelJF(O*lUh?D*JD;6j*zVG13(FqLqn=v9_czW#cGbd(2emaP~F*ce)}J zksI}`jEurDMFj{Gm4;VKuoyGFV2!!-e7BL9+B$Uilc_rO4Tn5*pL2b&+Dph8Ds6PB zLPn^)Rk~y?%N72K&{w8mukEhEK5@Lz(Z7CcwO7dBObO-8VKeXwJ*Q1Qh8CbVu72aX zY$k#_S`dZ^>>yuRlpEcGXXQzr@s!CDVoMKh>(59+)4PlBDi1uflY*pBt-Wu*Q-kI` z<~S)(@EWx-=pmX&S5@?!FXMiiU>Y$|7TRxTy=X5FuJVeUC=umK{Uw#l0kQCy#({gr z;&5+RmFDdGu_D|h=#+cIQd#~so1)2^j2716a%DNs?xnGQ_aaUqqDQ8I1@QNJ)JcoM z{ETu)*u})O88Ti0?CbrSd%Hfknq6}L3ddd7LzE5~`Ps5I<+CbckR{7YI7Wr zxqEL+MzTR)kNN@0`re1ZI#E-hTP=-nGu9WV78o<#weRS_A-4iswav|&Y(x2z)gTDx zzRW#nLcGR6OC@YwWY>o)`$W5C>hf7+570dUoa-U)A5wPq59vQv3ItS~&CLhE8A^XZ z=u!$5+FW;F*5*&7xnoypQ{_=GtvJe`A+)T_O67=nDNIb4xi?tTs@IS6Fp7J?-T8uL^ zcpGq`=eJ&aN6<`fJ*J^bLy$yypu#c3Iq zx4wO>+R>d6odQ2z(9r5hl2~WnJFQ@OseJQ9;~7DQFQU&xbq^Rd>xnH$ETRHaopdJ6 zNRLR$Nd`&YlfZ5hKV|UJyz@!l=+-5!_W6PHu}WEAy~V#-Bf0k;(tRq(lE+37$`R-+ zpkiyS-@j%^r`;lQCe>|YPFPpXHqgG!a^j8H;}ad`=2u@ErKhB>ExW~<2d!Vxx_lyU zT#DH*D&6M3pKMz}5Dg;Kbxl7{#r#s5h_)H;HOp*C1~Lm~nbQ@6%}i#NukN3bZx!lB zQd3uVHp1@))RWH~dTBO3E@bo96w+q(kS2q_XeXtjog0qS9YS z3QRtf`;m;BzC{u&ER;N;nRh6!rOebS@->;gK2!3G!gha@UbF<8&gbol1r>F%$wX&# zh(vycJl3195ZXf?bk#&B-$a3;Ys_awY(Clfxd36#Vx+Xrq><~+`P}7d(6!~`@1Rquyt*Swc#*Bn49;VkP^dRbycikz z#8p$tm{c?unGDCS$GgjssPk~i#Lp?can*^C9om=i%o=<`nf=CO?!`u$ukJvI?_&qU zYZyym&gJ04Y02qrgL&TRZ6!qg=f0g(rAvs6D#O~tfieC`=~UmQbfjA5}x(883=kF}IKlQQj zsDqgkH9g2=`W$!~<)yFDES-}+BsH+sF6S9c$Kw#a5iv8(DoQ>;`j7-d+H`yV5|%53 zOgt(3q3T07ztiyIp?&1~Lzvt6M)6Cn5X;z8yLK_cwVaH?RzhZ5NcZItO;KO{SIZAi zs9iq3?A#aD=Wzjf38LD4OEfTtjr=>QI0@(Od3&Evvkk=Nu(Z^m^X1w;D2kQ{ zGcBpIWYz?X@GYr*YfpYai0~7v=ws1>HkjSJ^4)m&?Q4yAtw{4HbT{liIaENBKEvhS z$OM@ZO8UK=OMSx?{q&K>JEi(FW0|P2qcDZNCxLF}UT0v1xO5t`dBY5@XNAH%7vkWr zwv_`_Y3VH)N12BSuBLwdMzXu$W!bc7u5wk|N{i!W%HSx=a;v=J%lF*VF#^pa)wI>9 zQ9}LXNPf(i=@)&aCkBo&=5gtYul32~`&7qcV$W4VNZ8N6>7b(Hvf~QZPuM~LYpk}c;;?|1H=Q$fN9xh)($M8{r%@Q_*xDwj0dWS2CXY3%C# zw0G4Xc`#Qo>B8D%L?;{k3K*hNK^Vdy*GD{9deC&SK+K(* zxr!Zq&otE|gw7~17Jgr|AFN6sw}3(a(S;y#&6AJc2GKiIpFDP-!MtDHZ22bs#9uAz zZiUq8r;o?PPs|71Bu5K)p@L2@-KV@~LwqDYLN5R1#-%eZ(>-!dJZsVB!NeotB$#vK zq!M1$`SKs);fW>w`oK^h#ui^riGO>ftY>(e*D2~8KLIsCLrWhEK_gC)VCyjCnqTy= zX7|t?B|YyEO)Ytn1&_KLOyOZhQEQSxs?-7*UP@9-;o&KodAV7(?MV`QWu><6@4nGl z6cu)0esck6XGpn%E(&~M6(`>}7?N?p#5Tclhhh>>riCdMIJUu4M?iISZdosM#;+m#imS}50VmQRVu zQ|OR(lncn}QC(rmy)++)V{5jI%1?OIeD(UkV%Pb|$fJSVSsgLRbjgtHR4Ja}7_IC{ zGal&(9Q)Ve(R>#POLohSIGY3&tFHkC{tN<3^mGTz73;z@vIfBlPrN7A>+c8D6bi5y zo082|s8Pu0C?ZB3_oAF4*Mx$v=5pzCLoN<5Jd3o96ZQE*Ri;*mTw^oMe*etn;|#$Q z_fqTf@AKQ=ZxNJr?pwGSJaDxlrk|pWeorKRUL1ZG<;VM(>vQ@X)sQr$4E=oaL6~py z?zeArI2IhQ*-SE9{+D`~;AHG{(&6RlSqUqN&^KTfvz&;j?kS%My}|4Y6Rv4a_7lDJ z(Q7w?yHu{_(%kB-nNCfd5$>XhTu$&g|8CpePI>;?ZuY}SA0XJ=ex|bxHSF)ZtBzyN z9BDpH>HRERgCe1?pnI||@3Zvv`wDbcwC{w}!MpvC6{#Smlu>&H)n%%DlPdpG(=U&4 z^s*8Lz#{))pr@tvfN?&Bd^Ge880Vl<+ChbB&f;5dktD=r)!!4zrgsy`O?Cc|;*{=3 zd4-Lq6HUCMsuj!S%=eTc{0HxyAPr-G?N$+I!Z&mr{9?suv8#D4#7khW@{)~lK z3zZBZdEKRDg+$t-0XhrnW%Wxtk2(zX31GjKCUYWelPfJOg!6Tu;TK0(wZgx{Wl{MF~HIwgi*1!^O5uC?F6 zMGY@p9$=X!*1J%YO&Lb3S=xtSIkv;yWoJwbfx@X^)#J6al^J$P_ zIY-ZWGYEF-ih#Pp$>(k+E|R25nF@NM(CGFF@~m?A!uQALIc?W&s>_|dtxdUhvU^s( zI#Z^zQ&=rnq%c5MZtumj>=4J~htZY~7%tDlTnR6HJ@0?PyiW%8{@R5L?Zw0f(z4ZIkR1Z6F!cQE_89T)0U=}A)gP4<`-qJ5a`4ZX2L|W$PMGJVdMO0 zaP%g|E0puBx6hyV`fzf?VDRzzC*AHRPwK@-85Wy`)Y4ILgT#gF3%k3;Zr4j=;tzCA zS%qj3`nrWWt0!H&S<{q2-{K@pCl^2PWrq->7A1=nyH3 zI5(DRvm9A*eKep)qT;Jp&0P8B2lu|}YPpKBr+!h(3GmSEHfQHEZni2W^0lgYQxPv# z=;`9;?cjV2(dii(CvRDIze7dQOVyj5S?T~j#6OjkqTi-!^>R-gJEOpg*yoUHS$iwJ zFx_k6k1^sT*Qma*-lQilzEsq9%3gG4#Onsr8Tn^ji6%112a%6IiIdDDNo`122sK6u zT&7J79`eb?N%(q2BWIfFg=Ga|T9zVs zV7jN%opi=!XP98F*#myk92P2zde#|Y@1(+@ld=*D&9FA1euUhJjC4lSJQ9}{cG-;k zL&NA4izS_FP3gmK`wK$Uz}{>>5{^9^G(vHH&UK<&sk|$J_$KNrcOpcAIw|3 zv#^E^xj+|~{x)Viz2*za#$?3z@(;C^sh3*_Pbz1*YBy_(UU4rgw^3=+Dy2z3EjHpf z{O;bXQD)hjwUk}yrw9<7pXaBQv=*M}im9bslZ+VCa_o6FcmL!3Ho*>wO*IV(GM8vR zviFPWbmLR~owQ-ZG0a+T7TPG9+U{FEZk* z|5pZOleStMM)1B_*HeC;WmVT4m(?3~&(>7BXLF;TIA zxMcdak;0sGLJgIr`|@;FmoHm+fTk~d+VlG8S4Q>LqpX*n!E?CU^G@3LP)ccDJj@9a zZd2CAsbIL$oKh{<6tf!`4swS`Gq?3pXieW@L>_fIAH5o$9ES^i8I09SZ)jjFYSQbf03_>!E`MhjfuI_i^jp5;5t z!%8XEuRy!)4@S17L9sAiYVCdoSv zeQohspUx4E(l>e%r4>3IT3C{77sF7jCm0>Eq3Ali>ANJYb=^6f@_f*Z$g+WS!UM_M zbQiLz;PSQzaRy4om^qR>zjFtgLg%oL2hH-#+TaWShq1Q`i?iF7MsW%5)_4L0cN%vI z8VJGN-QC?axLbk*g1c)2!GpUru8lkS`&(=8v;V!$xj4^z*YDF;J?H3AHEL8n(Zn&H z6oe_4Wyr~3oQ@Jp;J}KxUDftfu?&l-!uJzKUsVMmBS?ku%7m&?QR|!Q_+r^@GDO3d1UXSg7-zN|vkbYop_)t);5PEyquYW{SdA)l~twzRwTSa81Bu z44AmpLcM4tbw-f6sQj2Ahe`|d$oKdYz&NQg@+cop}zE~LO{xnkNtQs?58wHFlU+{mG!aZPiP1^%J5U?+b2^1IZ$)V-5GaHEv~Uf44o zT49bUF=T7Uk2q(6`@7MbEJidxCa#Y zO>uD0)r5Tm+}JjiH0IzXU_aF22tcUv@86@(+7vifg`uRKGm(6u?&+b>UM5iX>LqpY zYu+cF%O7~I=Ot=lT&!o5B)N;MW)sPoPBjW5RrSP3a8}kni3OYNQVq*PESj=^uo8%q z<=J;)Z6gQk@;h)-8_uq4p^h%mS4VQ1O)Is-p>q7@C}3^NXAe&f@MB$f2{R#uDUr#= zO9*~)&AV($Hqxp#-Q;i4Lj$bM(Ik(X-hRu;GSWbV|q@9{0r6w#< zn?TSNY0(=9oKp@BLlBxv72y1q>Q6tH{Nxj7hCV!)404yrlz}kw>@)9wvP{`O-qGUj z0y==+*(B+%@`c60@pgT^64kc`Fd>h+mrRHjh3E3A17Lf38b04c4CvLW>sP5%A2{ZZqb zR=$q;%NbM`$xzF;G3PuB=Ot@p(EUP+L6G?mS|ywM`0(#F2(7uAHnJmA!lkzv&$XW! z;I-QKxnV|lxP=5nv_<3ZQ@V3DDLH;WTE71=Z#YEH<=I03(Bz{cB!(GWMlPajj~47J?bF6hBnsx;tb*$c}Ox|D($^h7AdX2Q7llxaVt`RD~(YKcWH$YwTYPA?Y2u$n!{^u?K6{; zxGe(Ok)li?ddV&d&N2!O3K9zS(r=h5s%lPNiu%MjrNYn5Bd|iS8s<#HPNnpDasXSQ z{M2%Ve!^q1AjwuO5=(80FQm5t1%Ri9OMzoeB&Caa2k;b1EC@uuRpHX7zlXgfhCxzS z)`r}atv46fZj(JdG1zYKD4}XZ9Vmk$Hu8x`jUGog;?)m00sd|X0#zv-IbpJTM0%QOMj-;wZh83#dIqD!Woe%EJ#@N|9Z+#lWxVx6 zFqty+Z4fj-`8GzOJR96f-hs#X=;1}PCqJr*4l;D`azFVMm&xZ(+<)F2&dlEFIxJHkzSi=x~}=gBMCoSl252TgXQW~p_`Qhn08BwX_B zQY|EliAYVqxA@s=wPK0qpJe5}qGmTfDADF+3f#?q^xuWrV=aEX|9BA3Iw&7W^*qcY zpGQLdlm8Hf8np%}cK)fFPeLsr`Z?;k?a$*UxlseBr;j zI$0%H3U0jSb?-%eVzNXzKRws3KvApMw30PWZT_moEfTB_9?OK5-~8G+v0}03md*wi{g_Y=g{p{Jlk)MSg z_NjRblss4$h56HAh9A-<@L&npZE&-9{*BM%SN*_zVWQCscY zlj2Y3OX{O&Eg@gLWhzfb?>jJcz+n{k5dP9Sk|^Pdo{sXTs$vv$m*>RlM1KC!HvId% zK2A?^+I0Hl^A{yPL+vU5XzXJ*H+u6y*v1b+aUX8K3~qzu#BD0XKZLxC{>c1?WFJ?n zPbL!2{lkm1;8}gOh*o0aNfTh%3QueJV1`7756{*?^*(-?Blq(r-TiK7y+#(HaD}z3 zK7T-nm;}{b%IA3h1Fi~v?hF}Tnc@An6`RYjn@_GnE&x5uTvOimS0F3PaWYk0HKT#b z$V9m;i$>Bl$i1kId5DM98S)>DTX1^`lSZ)*rDx{W;@aBha`rVf4?^%b3 zfm-e8O{9)~@wZ#82d%>tfiKQ4hq9vm(9LG2O_cyf)$cz){%;bF0r4&GpXz*}uDP!J zFJ|K<2mfIVUKmPW99}5yq$3l)$H>}M>}T{ zjTN|$&v3C29qR>E&nrqf5)CVAxlzD24(I!9yb?UpkeVtLG1df+CC1bUDu( zxh8^wP!-*-F+#p_4IV>@w8Ry5-Z>LU@u{+FXE%9i@D(Q{jpsW`Gz#_a+uh&D=R#8S zZdO&55%VTlsq&SmGEve&Qc2=T{U{UiFj5cF_J>v_rV;Ur*Vvn2mwr30fvP~PE$hn& zH{_W*N#L+AXLkxKdq4SIay(2Fc;*IxCOz2pV?L{x4xZLclti^<^YuH%R;c@3;{h%@ z4xF|79iuJCd=@qOXU9$qlT?ovCkxL8&_Q755N5}Z%Oxo&B7spLA2^IaKNsT9I4sz$ ziR@@=hwztpCES#p#p)iRmW?wZkggq)Rf@nm!A)mA9v#WC3WcmEo2=R?iD3EQrJ;W5UnwR37p366h{QSvsW;I&;`;Bz2_9nHmgf_ha zU;8^8vsgrJS2TThxU$0--22?+xcN~1Oz;&NjEk1ih($xI?JZZP?raoh0})e)CsqC@ z6PX#<(1cr*5;)V(j8uSlqs7mnahK@Db;N&;@NqAJJC{J*==Xn)xRpg7-wQ#5oNKk1 z&DrVPVaMu15TX$L-Dc}FGBvJ~3gd0tIeos73O&wk;$&y`N)s7SvB$AHJ9!Z+=OBjr zVOFTWD{>-EwnDF>B2K`jaY-Z?tdob4n^qqo8oV5OOnvea=<*Z!?E5Q%`gX=<@p8G! zv)N{zaa`LEWn&`JNkX5jQzNohNZ@a)?5~9@k=Vg+SIZ{uTI@^fv0|?9RRXj-(ce!C zzEzKqNgxYssj=sv?_if0%-P@Pca>v2;HdmC;viLJ*d1yce!GZ{Gx@nrYa8nTG$+W&pxC#vtZCMtl%05DOm3N>n0R3yw zn+AoHFO*wvhOc&-i~sNI_VpjKMKew9->-uIQXx{b+p=A?;r92TJ9K)U2Y#MX*ZlW? z$M6?e3g&B=$3JX_5sm@Rf)MXDxTNq}3M^cHCxo;VZy3?V5hmV*_Kb(kWxEcz9Y27d*)-;6h&&05&Vkaa} zN+`jRgpe#O#Wc|6PSBasq#6O<7mPg_7F{|+x8IPV=`-k1xn=xVO)G%6k|4*lgdIPa zgqM@x4$r!h&@+s>@KIX`dF@8@HZfF_))lrHjr%#TR z)!C0n08Ku;c${OHWXh5wH>S^lo!ZD5#Y}0BlbFfED2PIdcY%QmWo@J(LF*H}>!8-G z5j#rDR3!N*m4!fM(Nx;+EuOr7FWzXwl4EClT7QT=Lv zsMD4E)nKl)LvnKCf^E}l!zVw8BqlX~Uo84Q+vm_+_#Hk5Mo~c67TbYdr%{O9Zqi&O&!!65i!~{)dto%$hAJ_siv(q z@2=tOPxd{2Gkk&T@UR<(M#t9$y!r&&hDlg-5U@t#DR+=tG$49(kdx@6@;6}4t_8Gd z)*6TSC@gI6c(X4-oi#Z zf7c9=XirOB+?X~8O1mNruU+XgCTJYz229wM;hZrDE#0Jv-2}AUkRI2J|6UEz{9!2k zZH04~!?G_;PixA9LDzP0bHwEYz6eM_gkr_kO6(0o%DO+^r+r^5YJ`KO)xTd{U*{BL9(C>xA?u>_y9 z9&|awT>a?q{C84mUjM)OqrW<^?W4!GudSz7OpvaNb*PSjRV?y0%gacKPsV(XTwgp( zkHSJ`%UkbCucCsN5VtIS3Yv&}&y2dB%$UeF9ks9b8gZNvK@<6?<8%oCRlj^U5w&!Fh(Kj4%7xF`})~WmlZ| z_$TIo5fuvofkwP!P@S7_>je4Nq*t5!mxO0 zlsm;mjG;D_U}}zTq*{{_8d%nL>h$R(=E^m3p0%2&9|sN&V+s9xiHp-z!8N;R+LSRj zdF1A4(5EuI*4|L>yXbt1ehgS(xDd?|KC`qq&fy7k6UU}+CD~!z6^Wf47r%_Mhz=I! zAjzkcQ(H@s4mQF~c}95f&FP$&h4@`KHq`U{B8<+jd&W#+v@mqY3U^xiN_<30%$z3^ zK@R1m#I;UrTord|5Ts-6-L1hauE{7Expcf*YGS35;ir^Ii!D_HA6YVcgqZ21gwqe< z2efKCu5BWB1^sdo!6@56ubg2prGiA4z@+rSWb~;arMYPly4D`Xj}%3zR|M%)Dx7CA z5vbbNNjh><>5ccBH&aIo`B@+X=oV-jip@CW|02bF9Itv;Sl5n(*+Jfsg13~HxOJ1> z#&CSIq#d{Q*$~a=_M@9RUt!lzOr_f)ZfZ~_9=1>|b4 zvpNUYgmql@o_UHpw(R7_vVrJxRuLTvD6)&7Zu=du5K zYAY>`X=Z&kRDrDnw=Cb~U4;85q1Ib>ZgZtf3O<@N#X{Fw${MV*zMF3t%W?qP64T76 zuo)iSFgu}W6Pj=*AoEGz-p`PMGl?~XB>*O1ja?dK*}RI)y^BaQ>4u1K9DdeYPK{i) z9!%@^#XN8B$=0Z%9$Z&cBT+DNp>RSq zerk~w`@mu@X*gv-(8=zW+iTY6(N2qRA(s5e8DPjoFok&*)??N8yGYoouEROP5$PvD zxq@TZzw-lM0{@1<&3Bqidq}St?{=CI5zP-~03q7s&x<76qnLHuIJ7}sjrsSJnxI6& zaUUV!db}||mU0$tj-(i4^Y3Kj-2z7!^p%!WfLL%yENkzYN&%9s3byFuhH(A<5$l|h z$i4huNM?}wFG5;TWYpKT1!CU{9=NU?$2J1CZU{gb_~x|+^@aPpSa5b6{~r}VJ~;3z zlt6y_`eV-h-z7j)F1R4TmtJVz;yW&)-%-xq%E$lioqHooV8nH-XyQJyD78XE6fOrS zH)wHu!wB6(W?+Zf4OjX^uu6}bsr4%EQf+l!k2QO~$$_9&7Rz|5levBrEG!-f5)hZ-(cnDJ-u}^ z@_Gl7+7Y542Lm4SR9-wZ{h zi(vK_MQ{{=33fB#kio_X)Sm2saBQRcgRg{*GgzPj{tGz;tdnnl32|K5w8d4g)tsrJ z8G5vvd|{fQqqIQsw}-YWsX6l*GR)tLcOlBTvZH2dg@vOw;gLoI4v~{K5o7ye{RrHe z&of)Ug4efu&;9UG;hkPmF(3Rz)^6r&jE_`hcR~u*BR)za3)lrjo4=1K`I6v1i*g1` zj7#3t`x7i{e^fQI7FuE|u9zc$xY_p9np^(%@p&$q~(7E*u#T|DuXFbO0K0B*JRt+U*m^xKXutQPp4UQcNe z%&Dljh4?6#?1@Px`c2X(89a`fDe@ROhZ`)tKpBk<%Z`fasxcR_ zvx5?HTyzNVNY6nqV7>t}XBgSJs0KdvSa&8bai{6h!Ar>lcq79+x0pl@t}fznA{lS( zcCU!zH$@5%g|O1Jp;!Mhd|z`Gn+P7V;+wS3kIaAq9P7)ig3yj+Ui{1WI~A1LQsoz+ z75#Un_||jUxrgr(!Kdw*Re8j{`@0PCGI@hpD=q>Rs>}ADXtWfKyBi3bt#Fhh$?q7+ z&NuGqVi>D&h-anWxC&}xxJp5cRu|-`8(h( zcTbu_nDe_PQdV;3v$;R;>^iN6i{iezqdT!hqbE8=MF15V5i=WkJti*v^*;mxrayv0 z>?tdcG(!K;9b)1m1~?7y&hcc}KI^%iHNu-~*#Q1sa%d}iArSU-R{a=g3IsCr9uPY7 z;>^9Db!%*+JNbSIlT+t{01V@C%ZHpmQ{nNU-hbh%`bh9_$T9pg1J?C1tIX3Yap#kh zqmt(R&1M{IPD9mveZ_nuusme?+D8%wxk|hOm2N(RXxh%omq7)Wjvl7`jI7C1G=_|y zDDz|*^c3#DlxhWS?qV~*mr%@@V9FdKLgocgQEx|%`qJt!!5gqS#!`h%w3Upy1De{p64HA(TR9uXOdNF+1sp z;*`4VDP+MqVM-8-tBTmKDdfDqAWDy}_>lD|-F*OB%S;UUCC5vNXUANdz)x#|^sg>jI zl=rhZ7z750G*iU>)OAw57S|GX2mg)+U< zQmo;!?R+%3$|k`Lm}Si#BP$OBy@Yo8#Qi3|!cZtgPp-GQ`N)W5V(b@q{?1!;UGLtb zJG_=P){&c;W$~UVt=6Rz4-wvIJQXtKMK7}G$L@!O_BP!`V)-mMt*V(wG%PrzI&_lE zw1b^uxZtwk&}-^{L6a)KW97+|j-<1vEhsV2>OIBOT7GK;II;p!Mzk3HoQOQ1Ls(gd z9q09u{$e9B{qS6^RC^Ntpa<@=x_n~Kjsd5WGi}f7I@_;uy6qJ&5t0p^TWV}?cbpHT z8MfkYzFClz*LtiY6lE(%+7ytDR<+Ny)31iDF&hW2lW)4g(PN7NJ8{e|a3Z;x=stFFLij%1t_ zjlDI(Lnr|&&zR}iCzs(QvET2AR|_JWsq?4gxR0FgG!WLldKsCej2|7GV202lIuCJ{ zrq!OF)rsVu=0?zxmQh?FS_L3B@@j##uDDu{@dR3a9sHJNvGXA@JW@blKghLJXg zw9~$PL{ceD z?@XhKC(Es64Fy8VhVOfgIZVTQFuugGI;Ugk)-PpY^qJOElPK+k9Um1RbmdjfWF+il zKS?|FN72iA_Jf@Y1@S=P@z=%*@Fn5+6yz&$W!+E0fI24}4bf*B1Fs`H=r0{1XW zw>MrM1cb1L8U75#GF8g*WcLytnMjwO*B#6I?Iumbp^s(x5jA?gET_-+C$e`LamTIy zG4>AZgnQTgI;@g>T==gy4+Zbm0+uZrC*69_xpz(##v-Hsr)wz_V@vcYAz6r9a_szx4;*murDe@;$Hw%1Ok_X&sVn>^RKyf_%toh{ zb7^!(In&t5lmSOvG7Y(mdxDVa;m!!lAnQ;fejuFW>pZa3f;69WIhr1+Wjp&g+^X5z z=uYdhX8&VMU_EV2Av`MSAJ>(6Lr0r@gZ?UWIZiJ=KxtZ+Pv zOffF_VOcwN_q78IelT66%iYNjE}o4l19=p$*vWRyBPPzo{Afms-ZWDO1kZE53PK>$ zbZa8l#N##Asc(USCyb|K5zfzR;29%#)bgjj4Ns%r@R4)8;IQ`k=jRJa!ONFkRzJ?^ zh%Y(<2p$OIn7JJgr*ZvGl%DSU0B$~nU8e%%TZCda-w(4S2tS44xI(I~)5jFJ^GkW2 z*@ZL`j&m0|@HG9dWvW)X5QjINK6Y;z2Bl})Xe0NK`w^MSZ4+;*kMpRbtrg84BfmQK z9XRb|A1PekN%iy+`gbGf7lqMQ>eTWIhva2Z*?pkJ)pPf?SNR+l11v+IJb6a2O*N{CEj<(_F&izub(kZpWa4S_Z<4k!i=_%VnvRb z;4s|E)RGOz6}20IM2nxDd|vICoe&^f-A%v+VZ>OnG9yVlV&4?%#+9|h=y2!8Ce@&n zKJz}q$QDxuiXrhy0RT}ZYh4aF8ra^!Ri*9xxE&Ca)1-imxs9l__>KaNhW)M(u8_0uiFDb@79xtcidqBE9Z+P7<6Z5T4 z$CV%Ml31B=NqREe@s#Qhv|18pva90D#fc`@M;;Kaycf7`qD#&%ri3C_9lEqZuGhkw z!ztA$$~Tj}b@?CRwD~|05T9boFLnmGIz0L&Vr>bOX!gT?p5&7lZd5o5@0>QiVeAmS zo;ZbcAs#d{XI70pBqr;lSfr>WUI29~TRZa;PWwQI*dm&N*nS^FZ*oq~vu4)4tj*XG z>Up+XBY2%-aT;-I@t7$s9dXKEwhm>`Ep{FZRyjuY&}!T49tO5Fg0qIbOHC3gNI$?s zGG@rN09Jk7QgS9nu65{dxXbZOtC#~t=Q0&TBP;H-kNEitZYJ#wI=NM4UKUhkY}<83 zEbO{U!1A9I3c2xWn@;_?!zZHw+&Y!mv2;!$zNDllHV~r=pDGNjt*8AD<-r6s5xz;& zMA&S5km%zID~4r+V^N&7Xm!>|p3T1mrB60>Pw{-U&=9m`T01Zvj|=`i}(C-D18K zN|6M^>jXOnc+i$kxfeCh#H5I>yNM-!#ZdlNSLBC}KW()7!>}9Bvf+^vM1>`u(~Z7@ zqM>yki+mSGvl7fkEykJF*H$0^@6mw)4S~abTk1dzBwLDL(jfvzMOg1pJe_U0N1a?b zov;9ZS(LU0b+_G0AEJF&1A-x=&zH5{XP?=4Q$v7Ds|>8v0!BCdR0X5`77aV6>E3_J zx=d*w*0(IY&OF9%7q(lokFd&ENvYsaFxF9Kk=t_a~0sHzG z$%w-EndH!DhRO0QRKYl>ALWR^2t$Na0h10jU7*&*^5>0AF1ty%QXPdc=_&Fb;YSnLH zzg#8;Vp|fe8$g+CTQ^CZE9Xg|N*J-*DUOG=eON+{BH=s*QKtwNVZb(E_Zm^i?L^q> zdI5z&kCL=Er$Rh9s;t8z3Y-jcnDKhTUOl$6dM0b^Z-!lD_wE)+usKrJIlYJ6_RGLh z_PhS7S+JacmzXAqxn6jqhopl}X2zzv|M3@cp9K$i&xrnwzf)Ii73Zt@G^l*EMbiJ_ z@_hb5?OFmbzeFc?oIHu)#WNF`qmwB1tB($>&$kdc(dImNa;8O2CKgE_bZv966Wrxk zvaS&|3)@GZw4Q2vEIniv(NAJeROa5yyrFo>}fvp zc9sbEVUGI7U#k$LAdoSH--3mP&f^5MgX={hz1Y0Rr2B)KyIl~!7yre)%G1H{gVW(E zSE39^tN+V@VC~3kREw-nQh)~w#q({AGt{PU>yJWDT-|3a?y{Uz(;dC(BPj+M$iW)c zcW*-p2T;gUr7)bGR>N6d(}~mUqC-%hnbg=V(N3$hf^)zvV8&V24xMe-H;Hlo^0C)X z%f%;df9Js+qTeXs-`aSD{l54UtUH++^b{Tv0!-~eM;Hy}xQP+~#AGEjm4#cQdo0f=={gsNoRZ zx(^&Nf-4E(tcD@0XZ?7zg0(EXA*NklIEJ^vhs@F+tuG(mtNr=>r-<4XBus1TY;g_9 z)fbh8VP5p^Od5ziqG%o0z`vIE`@>J*5la$CvdmiDPNz$tjL}#UXoc@A2|M+nW7^I& z*3~K8^AZ{QD zc@9r~_t>30TfJP+t(1nr0Wa?H4bmYtM`P+DqOAiWRs>Fb5{r%qLLM(7C=EpY{WZ!r?C$ z;~~`HK8F_YPUJbr@Tqz03M36Ia!>F4+w#0%s_6sZvV%6CdBHk0$f6q!q-69+&X&)Y zDC%uvk<+92yy!P3`qtIHesnyWMe&ZoN^HRw$NFmVK7B zpD^0j!`XQwBWFJEfzGAm3dLCLw1ijU$1KQ^TILh1L_#C)ehmRiuwWC3R1z;FU_Xqz zA|^5JI$J0S?!>*!9u2}~vg|sa-5u19FNP>sGN-@&t#HN5M?LRzZ@?TDKfk)@gF=}? zdR&RvvDhgkK`Db(Y!dvkRlOWaO;6?w0Zqv)ARHmGPe1jV^OYsL=Oh3c0c?oCTboaT z1>rYDc<&}S^HI)9it3BEmM8$7I&ZRQ@`4};R6h2jK#(k!tQ6er=%;Y(K9lz#W!6#^ z&Lzx+C|@hQ((=g!xo6c`$O9BgJPN9tR*e+BwFh~>@n=q(UnWL(;YxeOG!cBM{JDMa zz1!s`V^2&b)@wB&&4L^~c7Ep61Jf$02n)-+@ye0Wd9S~!x$UHa#+ucpwKcox)4W|b z&FlBeo!=r;2(G9(4rAws6jo{G+EOk+Ja#rTmJAaF*3a4cljEcYwi~?l77~E4rhDtw zGA>1{tdvn1xfTBW9c)Q|S1ibhS<@Hedb}T{9Jly#e)m<5EVi}ljJS$)DYokoMNzu2 z?sx1^y3S1MD?P)j(mm2?eB8F2Ba;Sq3tIWxP=3p$Ou6h@oJEdyu_)Fpx5@mp2*A;C z2y|dDm>B3s`Hzav?>P6`JpZSjN3c!UTv2p+D=x{X+(Q}9q$_iNQrF0VE2Vs1eYCUR zaTOO1A3Xi03wgKi%_E$a_TTemFCv?94@I7~x?a+)tHDI%#bQy);Y2}YDNc=ufs=0o z4+>7m3AMaC?*qx;cnrLmOX%7tjui7MT){@3@crpR0ks!Ox!QcX$JKS|v^ATlfWCsK zmO}l5KJ9o{GnQ_=bD|7J+n?lnb`h0UgPR~v@Vp$BLskIy&ls8Dx~Yge6Qy!2CYw^z z=1H_rgUCgiF7_40T2e*1-FmI>m3_hk7lrd@1MaiwpYlZdWfP9V)!F_q^uoa>G&B|d zwn#a+OW#o=y5fpQV;3O(W~*WGgwP#{>kKe;=x>Y==oFe^!5mf~!xDELP1rV(zdKg< zNt?8mGu%tD`tMB2a|M-_Z zSZp5bg5ozvde3UxK0~d2_6wK1=JB^??f$R%tG`E`_)bBlPmK4hPppib&Z3LdupY7U_l>4>*{{{K?_C>mQ7=ngv|Bw z)l>oY5xBLq+%XMsXXSkSA3~CDM+_*IBR}TbxcwMV>#@(DNbI?D==(5>DP7q?8m@-g zbEFfQ9fCO974Twj!WSnXyaEtGX!w{*ZY9f`p43PB{^B=9s@@l8E0|tFPu>R_sck)I zRzU}&ND8w8uelo=8@Je0gn}=l8FDe1cA^`BCB%EOzkr%(<*nRhzqq%dVh9HW?XnlI`B9~7PADk5f&zIu@LG6&UyAqtqAJcj03Z1(p z`eaUF9RvnB7>Nn?1p0@79OHXv*KrO#tR`mw|J}tKYru zQaa{Nw>SOv`ZjDQ?nzp^{=`=Uz_!&lo`L&@*S5e|n8C+Mom^q}d?~}Xx&!?eXD22H zdqL88hngVX1tnd)(;RHt>jISxQ>esA|m}eb3LEFAtX>ea}XRclYjfVl!5`)T(?!#t>3BNmqe`pvn&fd=g@Yp4#Z?$#x=-29Q;xw{IHU` zXT{MvIB_deg*BCUJ`^3CmFWoiB#L?@b8FqWBuuP4Y`!jvyB6Z?kL1}%iRJ%^Ub)Vs zD2i`3^pI(Ai&x93^A{p(B?cKCQk@7_NN06eYz~&p$8X|ATD_Iw#udRtiXWW57hyIW zlhu9{j)?T-+v-moKl#Ha8iLFY?ikbNqNw}I*omS4=jL8@%D%&LUd1R5rqQvSe97275!11TU}Vg?)+XJ+kFj#>p9eq4fX?QA`%w=Xh*V@eD(5hBI+a*bjb7%(;L?6z4aG{4o~1~ z)`KfNYFYyWRvTdRJ5i-+A&(@nM)PEQ`BPYDAhOGVDyz(oJmd94aNQjTUx|x`5u14* zx+s?Nc-MOF?N2chcLtxdfpr!7`L~19yk=;Kr&+`U(cMQ@O+N1voYf2Z44+zz()ohA z71fVFk?&a#(LVET<(9T-VOyo%|E=@niWvp$Ka)bHw8~&Ufc|jkT&%V8*M5&-IC87a zR8(s?lI_?W6cdRkU=h;a=<#U865Y5bRJOJo_GY=>Yia>KLlieXfan%}-V4T%Ec-(h zS?8(h#b7^F!#sBclacMVI$^ED>fzF8YCuuV0CymZGVq+(tIObJe`Xj;w8Rai3j`Co z>8Ua;EYm)0uUSXlhH+Kh?zy*p4!dd>TL<;fuhW(`AM*^P?wnMi&a(^yYFV5JD=!*S;3g zN42uLs;a9QY54bv8lhCN?>d^5xzc***a(X-({}p&2Ha%DMnS$Fv$~Qf{be-4jW-gb zj#`N5qF>JyL!-4+4uPqNV?CTL?o2wZ4_#M=-gPK*7m40DtVeOqzMo3<-N?egzmsW zL1Uh5C$&P1wU`qFDkae&JGsj}TMctO?YtCo$ z7TWN7)25_x6I*Avi@5tf6ZUQN3q_zCk`@c0(B*;XPTz1qV$cOD(7ucr*KmE`0)K@Cp=)&e^Lc`~kMJH%%O$=Hn@-qS{0K>_UL#ZV zS?Z78H?6^=Y8&(eyy=r0APYNM;Q3nEUN<3NhFfyu@98uvx>A^PW(_La4LC4&%>I3!>dq0mYb4x3EJ5Xo0tS7S{x{J&kM1elc$eQEZQqr&&^L6_6{ z?juBKtv(B!jX)u_=M#G0MI#_ZU)n5Y=&&&n-v*TMt3tzwLbH@{7u))cBMX4->Wlgk z;7!(!hu7-8uUhV|@u{mh+DdPS@~GH2_kB*#pE_(kWn@2!j7}M3f;>H_NF&#MUmJK; zUczx5XIcw2$_YJ%GPcX_VrOH9@9cZe-Q;@BQzlqo_8XLtx0m0@*$Lf&-4+6G6swWz z(S?J@6P#({WMF)4-;1eN-Q=--XJaE)0h<#XCX36>4#RL{3%|Y|BhLMAkeguPi$W%f ze(x_tM#(jwww8bC*Ay>r_oeY5$LcaNcB7_u z^|HBDgnd7k9}jbR)NKzr{`IiaD7jrHSR3F6v$@=cTi_^!QjpC_8A!2$)U%w>twr2i zg;`LZMT1#tMjbh|?n1i}aQact6 z-7jStSw2wYrC+CRZV~-TSz%GLj2PwpJSB2-ao@27N7{gD0%5SU=!s#h5pQq}4{J)R z6vzOii1_l|7DN6BjumrUQr5J4pH}U^wE%5&Y=)~D=-XHO(oUA>OUdtcWJ52ucG?lO z&TZ0lES_UjUrHTm6Yf+7>Lv+EPQ(~vEKTx&gNsr5X2ioQ*)t?)+i%l!e)b?7tsVPs zQw#!*o(KJ29~W3If0k)X1W7tdG3WG&nmi{h!bW)5?kcCom>dY@fM7B=J~pG7ld?TBa|Nz|J4YwssRU=ia9c3u!!2*{@k{Q@ zU5z0la-4eT20U#yj|GJsTC;~3Pan6 ztpNxGkdhu>o^nm0Oh5Wh(RC7^facvJO^n@XLD$wotrz6c_}nWKz#eCcm?2!=8&7a_ zv_8}pK?^4{_(MM3OVzdgd)$xC7kZqgIBOP_Yp%0Sl;@PoQKrYGU{@Aog+L0jCFa{q z)=GmQI@dIS1scrJa*RMMT1E>S!TmCn$0YtV%#j{dGO#<;moJklFO5I$^V0oS|G{24 zj%aoy;~lUpfHnVUIpL!QpK_LfX8yHKt!{px-K~0+JXXIAcW)^lb^4S^MT_>Dc(|W< zV};AgrP^PI7dtOu2WwFoqnXc)F*z-rVH%M{eC~h`*mgoWgL#4m(!2%3Ke*Bh2gtEA zHsIH7>$tdM&`1xAm6f`raFjhCYDI*C%p(LEJK*8)mlIwF>Dj-<7>`c%1UssTzOy&z z35wa0k_tG#Z~s8?EnYP_jLV}E@$I!RRrOnE{?L3WH{VXCY2rXp8AamKwmpQ*uJyZx zNJuHbfOsihj`zbWf2YMe3d$vB%zDpIc$Cte9VDF;^8OQW*FPkupDPb^WWV$CL|%ET zFW+76M?J8m6h4x-5#^=B^x_2N1Gu&G&d7`T?i2E&qu^rfHDtP-Fn4Atc*Y8$#j`u& z==YLx99|93}Agq6C&X4IVG0o>rF{c=z~6u|(# zAlSH_?0C^Gg?e$u53EmUZ2*<@WOIg*oP8Ha z%_aXR*Y~?EynHxrw~#qUsh8hjTs-(h*RVI{_~@a3e8|hftfh4eDpEC>0VcRL7glVr zT?96I;%U6?+cu%4xbSDAD@VWWPBBq`EV{9BlDI?HH;*F)pec_~yf2^0 zRWk+SSyj~xH6G1g8s7iK?$-W=8`2qMnL#NT3wie4 z=rdxDo6?vX__knOzYqO=LR#p2E`RJxs^;*Oh~f?By*losY#edQp8l<64S#N5Fv@Bn zvHaRXo3BV?Fur|}+-2oDFIT|t_$WnqFSP(36eF9o&SdhZSEm>mx_lVKtdR8#a7We? zXlUKZk3W?pu6jP~qd%Rfr}v5bWH%aD6orc^!6T3xfM2Nyx3b@M?3CbJ+S|}V_f*Tq z6YUPEx(ipm{k{9sq_~Zn#*dr2MaORD6qDI?V6+n&Y zvPf&mu)XbnGHr2NB%9HDbbDsi+;+LPVXsAUlI){%z%xGZ@>54hw3)~e!>!Ml8;oWj zyVbC?2jg=9p$cHX;l-@9?jx8@kNS*YRTb)};Fc|H+nZRc}Pv+!5oa_Sg z7s@x5;Fh|% z?zLLu{6wAZCl}c>qqag8LTg(~0hABJ!7N%_P*I=o7pJR6Oj!GQI5{C?J9A!T} z{&3*M)7PC2nzG|&^@v^(`Z=Bm~1MVaVEZrcy&VBkt?Qi zTf?S$>@)y0x}gHfC?c+vMbYS&%idY}`Zx)?J>ve5#v9wB6LO!@w+R{vFSzr28Io^y17HgD_UQSdL%Q&N0FV;@60VA%I-k?#Dplu4Pp* zWEk0Os`^zJ(=jt*LlHlaY$JGRgU&hUS8dC?R&p-$OA{x`s|tXHZe?BZ5B;EBF*CT$ zjpuhFs)7WT&lU^)c5`QU0o+{QWcZ7iVZXs-8R20N z+kNY!%K<2H&t#gx!X-B(iAkV@C57RZ0O$_x5hz`VjYIcf)MnwrmOc5(o};W4TMdC)~zD zS`fCl!qrAartc5#C2Nao1Tx+r5?Q?fY=9qzsLQl+(I%N)GLu@G%z~ZwGO2 zQkCY{DAHdTJrh~zEo{B2F(+LtwLhBx#%&UY@AUQJH2q{B4B9HpcqNQuUEY7!sxK19fVV_`d|+D{f|}-n=f*2c;kZ6&>y~DZ zD|*3#L)7fuUZ`2YA#N$le?vSeb$Rp|UiFJdhsNjxw&dWwB*6MD4ysPAyJvGOEbc@% z40!x7mDqlZl8?JahrF{?2o5&N3^XiJt5-4lDDJLV7M|qsy^mltPUin{*{- zfy}G1?6bOAJb*!x;E` zq2IoE0bKYZZ?)ikzN&p`;%l>CIi?-k(Z1qN+Si{Xnv?Xj)&{CsFP6^8V`mz=^~x31hJUFS$#)RvJGi=Y}C zqzCVBYyQ^w)eViknAW;XTD zD@j8&>S6lDOiZlOh6T;|*_U_;iuC>cqHzSOD6dyEaI}qRBSNS_^GgtMJDw>|gJ%We ztu&kf7V6AY{gK@*P|~}=k_pM3)*zpcWFz-tmGmT7rf9$tHYE`yL2&)#3pvKU-m56T zlxR+-R+P_L={c0|6ZlwciJ~ySDZ`nczYSVDh^%&IZF;80uQLsD9Zd(H&@eQkcxHHu zO*mdE-)VqL^$phTh~@CUfV4XhjteJEer~>E@`wH>2!R(bSB3Fk8vOR! zhsVbU%Wdy)=>jqm){tNX@h$3q;|iAGqK7X-$%c227%tU()gfFN^f+7)EZBv(zIBcA~5| zy!L){=_#kL?Z(L5SX{9lPI`cN0|hxxxeJo^Dj#y$6|_K&Hh6Dfn7i76?;uZCV#h!> ztD{9f0^#IG=LnPuxj-WMC^bbF|>h9*#CgVe?jBj90$W>BXCLs z85p&Nr814N)!pme^qe4YB1(^WFH?N&_TS=&0Hw@ zH7$og3WD?;9Q){^Y|F4qkVaAK<{{dEZi}K(P?d{GQ%jDl2xIa3W!TjZIBRCDc-c>9RVV%QzWomsRpWF=bjRvNPVU{iaK7OFI5DrCl~ z#bc?$M6S7~nT`G+YH9@+X_GvSgd{bHEm1zl3gK{}6}TeEuKAt>b4(!`3IGmcy!4dT z*{V;QwDFM3_d|{13>DWXk*#)Yjdqm4I-`k|7s#MfxMmsQ~vbNbJ#z;{`{#hqB1 z2aUI)0$_rJm&bl}9^By=_mlZg1A8~G{@h6NyibzY*yhm_{RJs-{Xn=jY}LmriN#zt zs)L7vzF=q+>Eat0p%feJK>OGZ^y0BbeFE*I&x!6dx{r2}= z!k)xt7|H|Ea~RI~?V@G%t=}57x83RTew#M;0sb-r^~2o-Tdg-IoL7a!)FSv%Fy#Fn zqmi3s5@4QGiYeeDaJife9Fb|cGr<_Yai~B}j4Z%VaIWit%Ed$%CA8g@pI*d)t>9c$ zm5vDzbk2d)w4RF#j~3R54NNQg;~%J=ey8yjd268Z`gP$^s79{D?t72IL8?b5 zlvt-;GDO3B_`acZo$erLF8r0Bed1CTiAUcKBoljoqGDuZ{QnCp{0)O~B*sE6-fi&J zZ8HL2%4nvf)WpHHyQ3EzEJ3Fj9pk*?|F{Pf+a8+1j+Txc+CL#925y;!d=f%(rhktu znwf$V5z4SxT3#+HXuz-&^(H&jjV5u?|CUYQotw`GDQ8Yqv80EIEOR3{N?Ng`xM{!( zzlKDT(@0M0@B}%LiomCx7h+JGB$l#C=GEqh9;y*VH;9)-{*qmJg(o!#W_(c*(a(KCpeJN!(78k3G5+ON>Q6OG6 zdbCZOqr*;mFopJ%%<#S(bUdere;%oBV4p4__bDPAa%p4+HYZlbf@d}ZZ zLMkkFgF-*7(Y4`%p%2t^@n5r7ZaTcVzal*y5h8(CtKrytI8*<=9bhabdMOBni~}k> z!y!gTJ@7U`fXjrcl!sdU^#x^}zK5MuwP2bj)*3!gvSlKK?8=8UZhCsP-HnejiGJxT zja*PXY69CBYi1*99mgY4uyUyeTBCfdE@l7Rlw%uJeFa90(=$stizx1`8q$Vj_mjh# zyY3-l-VTmGZ_kC@h_In3t)T0Tq$&y4DK)E_7P*|$p+~udOM%_%S)*ERgk?OE=S^bg z)q{C9|EC*GbB_F!T@=WQxZL|WdEB>5w?96*JBG8lhAYRaa#2ft>`fjP2v;2m6v@i_ zI<1ULGVqOkx{+wJ2*4CfgH*93s^=b8-bSVvGh`11L|Uec#46m?T?ZL%JSoQXA>855 zK3nuzH%=^_b6MB>vuR4Zf4pyA#!SP^T{g8u(a*vFg1K;2f#OFkfA62h*QO1Pds(wX z$Q=u#K-(|XwO&M0W%$|&;nvID{E`x}A^o#BWPVTD}eurTdWnnP_~+zru?_8F;5`r49MsW}-A z72PuuIe!TK3EewD3HZ?TMsjCB9ISg3LZe`XhOTdlwM;Atx*FIzx$!kgWPAqXW{VSS z2`%A1o6ch96tl=EGXU*OFxv85M9ZdgLw~aMyzg{i+Nzs3^5?L!NvXTS#QSFk<=8Jt z$X^TPDh&QcrW`dL&h>gjv|Fy}2cINetCYtqHaCp>SPrxgi@^u?C;z*ijH`nnHr`iQ z21#Mk#yr5Xc; zOOzS0=INsNS1n|H!#*t)t(IKMzSVAGu`Ucy$<#H1B5>xA6*QdhK4-nrFI2>%{70;M zEBXodJ=(%NrUXLwy!U)An)}1gEp%p-uZb5J0yI zy&Q^@CNVlxqmCP)q#6G!`&$u>&-E*2AH@>|UW}>RYN4m@P2ZwwpSpHPcW{7#5ie>f zAy*IS*CX$K&qo9=LzW4q(X-%Cl+SF7I#fE9$E;$6ogyMnHCOz!@+`WhOxB|~GOF92 ze%KZan2tYGLWgUtpVshP54Ikl$I`y8F|;gbM~Oh^dOMtu2c&6*5=1j+#0sML`0H}H zrgkpWHLvXJPQ+=HA||Qf-0zthAZby=-kYY*^W>Qa94FqcpxB-UTr!c$S^)ATG8UpW$LeTCX-8|k15tLHi z70!@C=Lj7EoJ&Du1|;4fSHwRI3|{b_Nkr`Hvm|j}G*B5+1f_ow2yeF$#ncXin4PeG zOWaZN9njUu8sv)M4ik`wMKo$+WlNGEl&8>ZXR;$&#V2j1pC7EQ*jK`RUa$gcWryaj z?%I#!%9Qt|rZzefy%5f2qfpYo&9rKYXD6Gfnqjc?m_{!238X91Q&?71u2cz(Jp799 z-gNt3%U5KKBWu4JlZ{t5v@l%0U#jyl1CjEwXM>?kMC>ZXm@;6>A;FAy^Bqy%>+Gc1 zCu!1o>{s~o*AVT}p=rLI*q_xp2j2c7nqOIPT%V)Z_};n#S;Cr-kr4hrl~aR+&~Ra4 z_{G%dU!qGb6j|?j)MYvbTnCl+5q9DA-x5O`Q?q~JjW!a7eKBDC^{SVwZPS&BbIfG* z;((Q9ox4neM4F0JOh{;IL>(U6A$8OYkvS(2Nl|g~8hmKGaQbSbBlufR8V{OCCPt@X zSw+*e`yHCRg~nT@E2+?gjqt`TReT4$i_V&#rtUeNsGF8A$(cE;uuf+Pi2yitpxW|* z;3N#3MES`sk8`eeLstYkzUE`6UjlW0qwr$zpYy`mM@ftvaY?Tv7Ho+&6suKWYi$=S z(6LRdUoNKFA3jaz^jmU4V5<3*k)%I#Os9tP5J8cwWb*TLj^IX;?Xw9#-oW!0p2RSX zbDQF6mrBiOkF`y7&8Xg%%MqV1JWpCT((lUAVO_tk`5GpdKuuPd z+!_SmC`G$gTj2|BaC+JD5W8jnVdzEj4$|br1Y2Wm`fG@TtbhcvdO>s(daMDepGpNf zRqPIHX53`eqstxL8Od6GmjFT-yvp)8aj5nI8#NeHZbEbXIbC7;VBVLHII2)x_X&oY zdqY)G;IaV?-mi4?%5I#veoUZli8XEJ$S<1DH_N-Go;!vG^%f^GeZU!&QEiV4sqocE zH)dSt)v5EbSGPO#^Yiv?jA}JolGP-7*#{U0IBB zx)YiFofzZepG5;_CL+COj(z2Ia0Z`+Nwmzcw_Qjj{GDM|aFwNcG)D{UrrO(gHWFq# z;w4cL1B^L{tkU4@*3owhI?Qyl{gE&uYv}cQ|M%R!_?0L3eu$ z{xo@-$Wbi6T$ljcExW2}OtgsWC;TglvpE;l4YsLY07RJU+P0Lda*a&L&lbEE8_b(R zt&@v!>50fm(oix4EF;wpV?VNP9{Nm|e0*J+%;8YX1cWg4H3DjgY=USh2Apoo$tStb zr2WhM>7<6?74d?I)0W&?r+4CnZjYReNevJ10`VXFF>NOCss*r8_7!@g-22Ox2I!N_ zT}Ps0HN%c09$k*Nc)}L}%1OwLOnaj-p|8&Bm(o)j7QsPO5{uxW(OBw1lzsfkI{^LF zG)f^I-mbMH+PUBCg%aA$>t0z%=9nVG^Ybd#W}JUNq&V$}nFFNjFd^|Y$Yn_=(nJ_| zXMa$j6t|812R8Jffk?IWvJWN-cM(-!HQU)0#eo}RNGk>3bnTYwdOm{e+D#_ZOfLc` zB5%1+!$4AQV|3<29rRfM4Wb(%kMj9 z2{F!%{~*Tyt`FXSfM&<4G7R}2=>-D^pk*@X+wJ@yJHL(5N9Ck~XMy>jhUdw+pbud0 zd)*K3vf@lbeTc2eq-Bdh3NN9^qj z(Xh|TGS~9viBU9w`XBpg%iWRD3!v(+kglo41&%5!koM5hcJ?joCBNv$2w>e*iT{$;x{#4peF-5hjptjSs)V6T0%!vi zO&8Braiz2W7z+@Y&Y!8(@VH%VXxK6Ex-XkYvl1!AFh1Y)5Pg*z9b+FTqK;P99ve0T z?PH>hfKSX+6TW0*NKo5KIv0D4fk#HY%q@*jXJ+gADP|#z?AO9ifd&Sj1%3}rY zhurXA67>Q+OVjr@>bE@KOgz=L(X9<(ZoD3BsMJLE;8a#IZ@T$ z9uH1w@CQsdO)qm-_8+*|`uK};x9sb^uFMO}r|Dfh15nQ>p`S`V_yY>r2Kh4y+T+!& zaoT4{?bhNY8}aZns*)VIxDu(KJjb>od|g$8wM)&OjuoUEIvmJqY+-?xE(Y4{nN+g* zmusO=wUFOWstJTKgkS6{AbmsqpIR#6T*49REMN{wt^7n6#=8Aenr@#rmlk=qGGffP zCEz^pTGC0JQQs;4_;%}_0wQb$iaFxIQqIl8E@n=)PN+Z?u^-1~q+t)KK@t~Ynj#am z8KxZa0y)C{%8_BBwKS;Eo>%b?p5LZF%q!5BV^BM&>2_rA;QgS2d<_3V98bl(j28Y2 z?}AKhf^P!U9<^j9%nub>Wljrotyk3=+m$S~M)M0D@7Ieqj=*<6Pkz*PkvC&k>V>T2 zPEAJ4yW{zn`2riY2_@=^61A}>Q3`oE_0YScseuMxm|p!?FNU35i8#4mjY{l5@GL$)!*dVd>CgIMht8Gqt=hG7Z^wp}5xPFKs+j({~w_<3cd zY^tsEna@$B8FbS?1j?ERGIQsEz-LL|5X?Q*K2F}rPMpj#R!$|GWgB4n!^9Op!a-&{ z%Bh0cDV-4GZxanFNFHm@%tG##4EMA}sP}viW4fg+Rvn>Pe|t6L$Non& z{$JzkV@VLqf5h2;OSF&@!6}-Mkgd5T$V|#g|L7}+TDyx{h6r%UCl1Ja+u!ASZ`l49 zEec*<*4{p+;~P%v?F3T5OK`Vn1UplYWWO%`v}gjzZUJs!I#q0-vv>!Nr?P_2qOjQy zwtp8SGh(EabBt;R-4GIyioHV{WYS*(+8`{m)W2`aT61mN??mV$TbpZt;vg%YeFGgJ zh`RCZ&;C?m?`Q8wQjITE8Dx2qTIP=ToSOXMg0g>hl;5zm5w}r&R4i*fiXVy!hz?(6 zXf@CYA?^+?lnG=GO31x$sN$)Z+u&!c*Uk^`4mBLZs&4$@E<8Ax-{rr$xC2?Oi5mW? zKl|kTL4ihqFp=_3;}k~-G`6)Oc0STHBbZ85`(!?D-wY>vEq&mbTBZE-`5wyCecrpu z1s=Kp-Y&gB*IJf2cSuMiV*b$6EUFSvjvGHH_g5|njvWipESQ^U7;huWMl=MKNLKIq zmoxTY1Y-)c10?=scz3#MtJ!g&Zu!V$eNVI-e(prCQc`-Sq2bu1%{AD6JeduOYt`9Av;0@Ap2lq#4s$baCxYdh;4R{XuLF2b7Mc6~lN(_P=TASnvy|>L&hb^R;XOclm6kT+@OFHMr;9eJ?hMUsaEp5el$X%Ua|khG%QqFX;6(3N)hJXbq2F=gexh+nFH9}AR6=rd z`IV-#f4^dE?SK#fq{NK>MdC)faHs5?x}E*!y~JBhZ`A(PChT$*barNUb9A->?g1&j zC)9QSkDvTsiwFkxJ?)=P#kjKWdhOp0%m)1`E3lPnW5(Bi5NJrAdlBwa;RXiN3Za!TX={ z6#kY5tYCLy!A7Wh-V0glsfZXwTGJ|u2(}o?=Zf-+=ySasE$+O8dFz0mS_ynP5zcbY z!c#4AvVs9Z3&2O;gx!vuN_IJF;_3=@hHG;tBBS3zL3c=WCYHVx4&A_Q00uQX0?hBPF)dUdZh@3OZ59pze+bFEq3MaNIiXTvhFazt}JP-UUa_=M8eVsM6P_aP!(QZh34J;XU$!{ z8NAIeqGZ_2Ykub`@88VJ0M)!bxO0b0ft? zcZ{VX$Q98EZXnqIIy&yR=Q#N5wLP55@OcjRW`8X`_gu8I&;kz&36s=*JNj!ltaVik z!BL#YvdC5s_FoCH06W42%9u3YZnPKpYqddb1BdEi<K}wCsaM zk&2m!)xK!^<<}6GlOnL~deEU0PVoAT%Nt>7Wk{N*_N%+L!$|&!p5J_T6TdbH2NDa# zi@roqHOz~}>P5lgGI9w|;Pkk)Q4z!!8`zF~p*eWooOYzkyI2qXS`6JgnHSr!=h(&L z2V^Mtx^ZnS+VJniT%{(c${uc%jdsDdr+mGe zO1Fs7L|3Nc{QhA-s0p6%>}8?ida{eK>!$~X3MpNg?T^MEHp1^{fXkD5)*$wM?e;J)6(a4;LHg zDcfQ7a(eB^>?~NOm6T*_-=IH8+j(}bgrG`wC+4GYm=ye-KS^I({7&GP$&RoDVu`9s zvW#r}jo3p3{$tEP%lz^a*-3CW(eff0?K>)6I*IhYR=y%-=K8+J?T`5z91w8%eWQ}$ zz}WZeIYs>DJIq~Rv=B#X{56+nRR=~@2>@P(wT9%Y#bcRDiJ1=3)=Kqe1VwtJi`_Q! za@gffo-&3Ibocpnc9knRU#Vm-a|p@15p~iCK^9g9r)#7N(&pL|OZo;=gTd#ab@jL3 zL?j+#V>HAYg8E5uNy;d{H&rm`I4jgQ*kvwu1+XH^yoO>(d|jqQf~p--b~T6Q$`zbVjv&;P?WQEYVE=b=n`)h6ZP#-`2?8hwy_RIYcH6uB-OpMcxU?!+ zv5z~x2|@)niFY+3;7tQ-!(H<#p$3+L!hdkJGZbB9exZoUb}Mas)4w*jGBPmaCb2$> zJveyv|Lfi3&liyLIc7g2%a9zdJKeYY(rGK&QUDj!WrFnR{?q3bGg`)_qZPsO8&hYr za1y&HKoqW0Hv}pd5w9KT(i>)h8qjKJ#EYvK4%MwLX5>^Y#cGOKOWefryvn~c;8<#w zhParD283SfwwS{VLfK8(gCkb$xYnHJ>{+HGYV9XW*nhb7L3y#UC$%}MtE)z~HZk=? zB-`L$iZz`1gro2WWVVU&c5*7~<_yy#jl(tMsY!5gZdN3*u93fG{2DmW{=oS5!x?I@ z=QZB($dSeML*&mcwD;Ww56JHL7^G`F%-UiQk#>NLm^^nR+OO8F9cLrl@8Z9NE^|kb zD3OI_az;cv5?k5Qux5`Eh|s#V(*S!TPt1oq6SiGqu9H9yYfxBi>Rh%M4>io^i)I40 zlvlF8s+;}bETM_ma~SO%fl?721?VZm^^}2gDHQt&(oX-6}@$^!~9ItgfxO4Fs-3wJIDC z$QeuYs($pc@Y5xmaJjRv5b{;$m1x)d2$AmFdF~jIPW*4Avf?rOa6XKE>x4^KSI~UH zjF4cR##o|!nF%kUIIjO;UJ*~(i4s(}QaW>gqxooq&!l#91vWr298(-C=CGZ9{tN$1 zqWHeAzZ}M=VRibS=2h-X3IyLnCG^uZeC(+T)2O$H05%|5!}?|%)V+@qYt_7=PfBI3glmi zE=W+?)f{cC`kwdwG^56oJ+&AS{3fJT2NFgV@$arukf*`{%f5Ohsf_CMi{=I+LC~j| z+|q`B^agCRD0NdfD{AzFms_D}2)S6>Vxe?Grmwt+!}M`N4NDKaWZ853zV~QTnrNCPSv}=Z8d|yvFwW|QiCpd_m{p(7hhULh`0e&86DHOcNve_JdX0ozhCcMS3OXR#? zByMM~DCZ_4gR_w+tZ0?~;X3}sDY2Sw?{Wf`IM_>=JxrN%dM0OlgcvUL5JCUS(KI60 zuzJut7y7l!pFWw$*&RftgQIF$PAXCN^WwEYIT9ibXYvPA!}c0Ns3u~JLELbMBn%ZM z5Dg~jt)HD2y~V`8f+RtUMW%5*Ozy0$i0*uFdbmm5oW!G^fM?oDGdZe;;Z2HH|EEbBL8~Op8e<#OiVuIa-_3aIQ zwwDXv^1;EUP{Z=U+{f+E-kw`5qs=>(^om=8=J}U7axZ1UG@xFsiM(P%%%%g*XL&oh zzSx4pzQh<)h<@JcpV53fe%Qvpp=|C^4tfTB=w0})zv_^*#<}l1ZlD|zOwnCNu;r?p z3%`Y0a!%Z91EpYGFKc#ChSzabuiQWq8h;g@n!)iLCZe8UD1{_fV_ z*$Hw8b;z(}+`X^9d)VvDv${MIEcb>gI;BQ?h67PK#x$WSr9F@FW%wAu>}q?RN1!Uw zr@0zvbHnNbH*Qn{vA_QG%=s>smmAJ~*>OzT^|W5C`@Zsey}N&>elaFNPi)CY**-NA zbd=AG19 zsH#CzAHj<|w%pf~N~v)0XS_l&nnVhzYV(D-aN0mQ}`XFY?pgq^_TAs+}hNslBW&oUxf6sZH+~+COZpkRz7mYC)Gq5vFB`xxmfI%Ce z^pqNT{V4E_x&fk{w-?azQ8w_1tg!=Mx3d>-ll?od!r(2AN%U>?P;$sOB&;Q_xc zVajlX!L-}lWg5_k!|BCkk$i3@P`P9}tOXC)lxKjSF8(hiSoKMd-y5G140bunJVV{S}0>)|JIt;9jgj>;8L zU2+Q5CmzK}K>CO!fUcn#23#5a?rT{E`z0@xjZ5v-_UIPC> zlDnel8+4NbiU)-0lBV)r-0BO{nvIMp`ZB z&`v|4gxKRrNeS2W9B(W8&UM5+?oo?=B z50R+}_tK0KAr%VZugBx9MT_e2rb5Jpq18{xAn=+6IYy zH@%Q4zgCQc$wUoeev$>o-r=+SYAHu!3$ZVONA;xt_P2(ry>O(C2Lpu)Bc9!aDpGLy z)bl9Q+H|xo7xdh;qHdrEC8aUI5rZSAFy|K0iL5Ut$Zq5?=-!BmR7&jGgz5C>d{p&P z)W9h_);>I5Vvp8XXy*9aF9!YqyDccvnej*Z8--LzzY=5UbL&p8Q{QArSwHEp8u zpd*K=Tp*F>lP`I4v`d~3ag7;G53-k=ju~gtz}k_bJXdi+T;Fmz^k}1L0b)5Lasz#R z&MP(cDUKE3PAzB7*_vqUai@L#MtfGGCOZ%|olOjyt@oDEcyQG>B~MIGKht31vbCol z^Rl~aG@AtpXTgek;(Fk1? z=VjT46&Ga191yVO3r+-I2wZbjG7Vy$?~V^{<*p>QoYAXlz9h=i=vt9ocot<0w+11z zseBnHdgp%eC*9m=hUHDeQ~d;pGmkC70@c0~rP?Ti}!`} zQ#jG%wW@~Zf_>@Z1+7<6xu}J(&Jni#JGByWNa7GG9bA>rFXe*~f8_`iS3fIGsaaBN zn40R~PN$e18UZS#nA3)e4^%^yDNR+z75qg2%E|`7QlhHeY0!KxRG+M%9)@!8n9oKtP~j-GRRQekDW52hOo7E8^vv-NzX}ad6A>iy05p z1Pbd!cI01ZLvQOtz=*9qXw_i&aV=;iCZeO&JXw^ems_(y^gzaVHNmwKa_2?R*>u@E zHOqlmM$yi=w56DDx1ZAiQ<;z@Aofq;^l=CsU`Qq$Tiq1A-(&aKP_9vADP{-flZ1s_BsT1wBpN64;=3Hdb|`{nDWFYQ)4KCdz*x9UOcV3(E5l zGkI-wsHh{nv->GmKGjQ3UjYL3_u1Bpv2a~AbnIWUew&p4(5>s4lVBZ zA>|H-JXZ;?vdF%Rbd2)xXR}M?hgKG;s zYl}bG>0W$Wp*(^Tt`9rC<|@`jFcU#POdJWW3rqd+@w-)gVRzFmPQ_=lp5FLQ_?xIE zQpRxN$=K2{Y`bg25~3b&m^?MW@%l(Vz50(&&7Tgl{6IIpnhMIGRtq@Z7oj=)`$Fyf zIK4*LdEHpZ$N4$-ysOplox@KfYo#d-f0h<+#z)C+?vt|47buGbI`a`<4h44?hC ztNet|;5<4EuclWkqmYKCG)wf?s^@<+-T))moKJMoF)=PXf6@07_}*_|q>E8Kqf@9& zq==TAeUy_z595<>*Xwuq_Z8p>rIhS?@*Z`#4U zW?wvzk8**#jyV?y>|^(8gH7hhxdNcuQV;vV8VV4|F4uPMQfLim^U;?br6( z^hHrZ9e$E=nMMrqe%*`1D3|f>@+5GI_mcDT?zR&f*h2{TRUzx3EgIS?F9@k(5c^zZ zaL8{@XP^Z$n0LMgYfKq3`t?(&Ol7}3eUKRGM&$63S&z1BmAa~4ZYMgh1^)WhZMOgT z?zYQjF^|~6EGeVYoEBg4zgd8u3bN>&sOB|Qbx)-5*gg98cDRVGHz#xT@U^wI3K2^@ z9P6n8-|w3+#)2bgWeS`^+JuvUW_4;RTX(rZt5_Y~MguK(sO2R6v)npp&=M!<;svZ9 zX>`}7=qk|fgq=QSe+4!L11apLtw*qAxDU0p(m5j*L5>pCb^(uLs{$HQ3lDIC@puJS5Wn^;^+asR!F;J>%!hR11kz=<6{{5u^1v>t z^8d8SrBA+Vc@PZSd4q%JbLijs=t264O?TJL3&;H|a0BmSc=}xYy97Z}Pw33Cm{TTY zuUDL+Jzw}8qOR+~YxQoBx9iWnUQRM!{7ObZy(oK_PRAMhie^yz59WB;Qr+=`4~VK` zLMHZ&!~{*j@L~e=<@=RlJ+l#-@RTP~#L|855=)^@4yTFiC};MNmO9#VWYk??jqO;n z?UP{x-D(8B(8GS=#BjyJ>MpU%V1DG@t2a8e8DfnFE^YZVO0(!SZvoHYNcS&f^0?Bq z{$C2C*U}%PIln4~uv8$022Mz>Kbs};kVL{bM++nSxQckv!p~!qT})>ux3bG`kX{V} z3-LCM_~DOAVt4dI3n8{og%F|)fS*x(=&=c$xi4h>O#qk95KJt%V`&K;M7C&40#D+L zcsQ2sPlo-P=c;HNT}0(Aroo)(!6pc3*#!QTQPwJ(l<0qr)@4P97^PSZ}+za|!L_)sT2JCuN_{PK$}t;vIlZPZ&u|AmFe;V0$M9 zzSC=Wp=g}{r}jhvOJZbyo;G-X>ronqNQUR~&x3fS=59_7<7LS$bEfBJG#^l}&T%zm|8D>5Q9BOkqjPm#wsdz$H-+MHao2>Vy&XK*bkOixO+&~|g zV>ZKhe8C5D64iZ1C6+wQGzusKhh8( zqStIS+T~&%?7q_Ql!DlphAJnqZ`hCqcDb=wF>w+Dp)6v3B^qEZoeU#Xj8}ErFt`%O z@IuTy#$s-A@Xp(GfPpdapnc7JUa(dR5np8U7nEwaZk`CnrB!ey^_ zZ&>&zb>ko3Uno5Tu3>ro?^s;#Zr2!imWkxhWJ2t@PXLf?9b{v1{Ob&5rUi>P3Z0{Zce03o!AxB<12W=VFjD6r-$bKVrJxTlYrQ;hOax*MS&O#=) z-tH){pLTTt32qs0(B^7&(LwC?cvG7j{!|T0-!J!RVKGFEzpOE3vEXwt)#)how7!e~ z9t!ghJU4gh;d;loM+(*nXZK^UHQ4#sI`XZ(DUsM=IW}bLn|jH0O%vKrK_b@Hr3T;F zg8iFjc<3l{A>$ZMz0^r%egu3yO2bx4RDLB|2 zZ63P2AGtBvux!P1Lp(Cl{kR>2Z}dbI&)0J8`;gzc6|milH?a19jv{{228%47Kk91O z*}x)8aEf;D4(oP!6@zTfL$$qfj==iC|BmwqzrP2hrlsvA$&n4>>b$3Kk-%8e56o07 zUek&S6eIK{Z2%4QVF(eR5<@Er8X$1Bw4MU)j_DT9e@AaK!wsCT_)TT!(xz6Vg zcFd`9`xjf73R2~hW-PAVUq68kOgEE-uqB)#w~Q3SWBGnxIlkseY_-TNuHhcV!kQ|M zH_GSECV6Bk6Gj+7KP_*dXKM1CX4^%D>LRO@u zL^bKG#+Ft%%0y1e*QQERbaAH&2CS2RaIVI`fQS-wSY!fM6OQ#fi}Wmq!J2aJ$&TL7 z4*U(E5IEC3xoV56LU&{JD1q|Qol)`QVDm@R)Ar9`^I!_{q=CvRD4Z`2C{;FzL~$?s zq6Af=-M{YC?Pe_hk_o(v2v^eQ4PQB(&L&4g(pLe_EL?0`H#&z&tyQ&m6Ex#BdxHZR64r!Su_ z>pP9mfh8FSAy2dYr}^W{vI8RZy2f2eb6lwc=1u~$b{Ib5zes6FTy^fsW(_NJQS5U| zbTlAtFRa^Iy!XE=osUgTzw45H)MSJGg9B;^1f#)nxO%m#c)2Hhd7Yy9PoXOn&Pg;Vl zbQ}n(jBshX6NY}DaXps?BL+?BrZ{=h8Sf}QK{|S1v3dRtyKAaaXKLkCrUktljY^GC5-nd#fQL zQhz>1=_jR)1olbIr&Ya?7T|A=s8}U;FckJ*5_NsY$j;9nxv-!ie82P=^w9D9$*e*sHirwRR6hId5K#QY zy$;cq6-U?C7ql2&nL@Wa#Wg!rrB(gp^-3AA>FG^{ml@;YF{~^qt9H2tyx=D#?00Kn zGSQ432>CAfP(;BUyK!qx^(&5I(rSO&GUQu=x_z!}RKy6&$Tw@MM|xWgtei4dVh@L; zJBJdod{n;vs(G{4mLMv)afz~UoT?0jV_a1W9RwM{_ZARU0Bx=fMC1O<)k&WUaEeEEP~M>zWXCxS|& zr<$&WoBmFy?MCP06zQ*Z$ItY7d;U`|lWj|GK~LRcw(#{0^<0 zAviVDg zwzzY?`+?1&N|_*4jS}Y=EM*nUDbIS4IU!FgH`?`iw~xPeklr$s0l`_56SMp3t*K!* zI<&?#L^e&$a*1MsvQNKUroZdmXrO}F7;+ho0Z3uQLPG&cBedQhhSNfVMgJnC!xf8 zTAnX|lPP&^n7M}Ndo#JBi&aJ3+KaKZzm=c0(4j_K8)73BA~Wq?dG6|s&)U`PrsRF#do>fFl(Xp#@r)oG z9xnZXBp$F^B_BXXSh5VSqE~Wtqf+_xON6bh;2v zkH!dE8~`5+#c7>v7uRJQ)U@+%=+sR(7a37nDtxa8s$b;h{U1~5UlTh-BK#lhi?m9E zTgpG$Z;v1X=H*4$23rL4XM2z?)BnCXB)H}sIXU^6lmF9k>2;~Z3rg3Ei?>m7T7tkp zxw-)rwlx&Zc@axK%S+@5c;eijkpg_&OCxS>>wgRRY|gHND|Eq{`T4frv}Psb(vP&R z=mEZHBr0bD-pw1@vP*Jmudt4js0=bygD)_xSWhe|Bjx-)0me#{OJu>9|6+JuUR%M3 zPT1_lXPMSP5M#+4+hiNB%edRs{DPiW2xpO#X7md)eIprbPyCb=@VAH>OoEmcScNdqy`;v#xLtljo_cP0&h+gP7#L43;A>K^w9AK>$!j`Ql) z(-CMZbKpBQmC?hc?vsk4`#f7FL~tv0Fs@O*3nF2P8@x81BGU}Ldks6w$&%;DObFO{ z%`LGQU|0P8>NH`Q)%Dkp&ko-4sJ)C6|5!;vnLA1ce|vdxW+01bZ@>3zN@!~3RNvXPwtH4n zvPpuDna_fGSJzuc`XZ2m%NKBgmHbZX>cl@s!>7G#CN3j$EZ7|SX1I|E)P)dJl?Gc7 zmub|w$%Mg-KY+2kflE6b{N$H(h1JbVPwz5y>U^=ev?0(1SVhOV&*2c z%R&W~xB^6Hy#gT~$fq4AZl)g0B}VmE7N?T%#zDUHVE|W^FjuZ|_o7Fb7P0z)`0L4@ zhd5&2RQgFvA0_hkpRYXed~{Zy=hU1|9k%VkmJN4I$t9oG^&iCsAP7S=f3_UKhEGxu zk0tU4Kd{o-dJooTVncKQO5V&10eS?0X|7AYAmKf`>~{(@)Z>k!r6?=24l*@}`&#wG zNnDTlTKD?HSOLa`8@5l08f+eRgi_uJbl|kwzjk;g(qD@~Iv(gN6K1f{jTH75n}g?j z0TXkkiNz*#{*%*bKJO4UJBxhjW|(}Iphdd?IU{Ss0|gyBlUmBfGb5TUFWcG4u`Iu_ zeS9l689GQ*FNv@2U6>L#h3j(oZsRd<*my9Jv%L^m1!KP{LSC<#nrwNto~CEfVT&f z&5UqUfj7Hbpa!5zQxI6>2_%8U9M01=Se=oC?(e6bW&mm56PG?f3x^zs_eVj=N5uw) zIJd$W-}x`4no5!MGvy`82_@MQ2R0-{1nvZOi$~G8tWG>TJI{3UMB00ht<%!nmSYsQ zzFxqV&f6+t?-oo6CWnM%koStRa$Ug6n7v~Q_d_Yu5^Ph$Sf%kl&e8w-@fGzyMz)47 zu;l-Pn)*`EOQ{rMvD|Duiksup7lDo)FU$|wc)Nkm07Y^x)XrymC#2Ijqpn6=X^54p z7uVk}<+RdFfJJFEHN78GF9cI?o~o{?Hk!|er4bsqq_PKV0%zR;9Z##=>iI})ga2$&*0f?Oqmt0^<}axuaCKthP6-YgqjR z=)ITM+;3(ob&+KeuRp%1@!df!y|p+>GI*~gpWjBpjq(tv>Dc`C(!$B08CJ~Rj5s9s zukAGoO^a{ytvF#!RL-ek?ZsC{>_`+wG;Xzh)dvpfd_ilF`#pu1T$ty=-7T4uriq!U z66&*d{|SC88Ynh&luFE`HB;gzn}osXdDd_3#qx86Vyx!!#AyhWknLu(YM=A<>l7yC z2{i#`qczZ0MZtBaV`21C6uTYXW%HLn7eM(@YKD76;WLH>FSF-G6W~5TK01z-Y5Ga4-E(|APEe zFhMSJ^eK+w1D3Ay`?wA)AA0qEqhLISa;@pTTC2yKDXY`iDYvUFNrP&=$d8T9$mVlK zbo!J+qQ@}4k}n%mk=V4mp}TXtjuP z=2&v{7u>$mgAj31BQCVCZRAxE|LJ#}m{H`>aWS&CRqs}~S?KKded`@~t9S?qi<5S* zwUSy*{&ED^*7PSFKN{}h$zIu*v7_!Gl6iQ0cvXSdoyH5x{%9D=`hTI zCG8}SOq`{J+=!SohFuVrgp89hn|$+Z7U=Oup?@aX0KC?*kP@XWuz6%u!7Y)-%FOI9 zPhQ}?;n}L$+XN5fE7X5da#zn1IL1F!)nXO&s1&x&P2xVErP&au|jo zC~b}b-ROTW{eyM+587p^=*^we*yv~0z ztwp^6fl!~!21K3bc!j;`#}Cx}`=zb^cpQyE#T8~k^eDGNHVh#axLYHFGc_XdW8rvn z6?nh4cZVO8ubtJ&PsA`aJa-I-q^z?dpJ)T;y^%%Qv$zbi6M1=*mTeA@pIXfz$L*rN z>*5=TjIEMyKx3!zEtnJ%a|3nkD#=6nPrhOJ~pTK=_1gEE(g6 zD;G{+^?L?iF>&wmV$ygBsA}L)3R5SeP<3}hMF(ECJ@>`8)r%M35(Vl9`gr_lYx3kS zqkO8j*wE^sDLu=(u)T*gomn~AO5jHHPw@@%Q>ST_$qWBqk2R}WksLB11=8Pq`tXtt zY5AV?$O|*)<_ex?2jY@A9<7-e(i*CuC2CeiBqGybjK7^*Qjx&<%6Q`l*q|*&h9X+9 z!E8{y%e`nlz3Bt7mX|k&Aw}O zV;iUo1pxiSghOAkcj;!$I+kfQ7>A7M)VvkFH{>EXcnE~_39`3`kMvgxO=B~WrH+Ez z_<tWUij!G+rUDNjr31aBsbJF@*4=@<{OK6vTQXX|c<(NY`ulUe(3g08?U6vOKd_ zAr(bJd>I||EBiBopB#`6O%(c>a^_XDv2J8n-CghaCtY%C*!v9}g=;Q{-ZD~@J-)pe zN608v*yX=7xFNJJx^7jGpUHCnUdz}J_*po^ew&Iji^ljZqHz8lFCnx zu7O~X9SUh@ql5`!df2?%Xrz>aWeHAIK*<1Q4iTXffB&oR&Xh8iIBIkI)ck58u1Hzf z=vakg>q)$xpUIRJr`ThIAuCIhjut_dm};Z2E{Zv)ME(II0q6}&P+k!o@wJI7T*s1a zC2ukj$QI+oQuLwmhO3_LBil4#0ha`lcBvY87^Y_FeGuH)E#|Uckf*%6eT$IA7<)jD z=r;V9qTu-}jWxR*JB9xrcec-dC>5t94`EnR{}@Jp{^GYDmt+Gf%>2)r4B9=9FE8Ik zu;TvPB${A`xl5anuPAVY|9k0Q0L()Z^Yq3>A`IvvdNobkYB7^9AhM&?*_> zjjaTMGhvI@DzTp?8~L|B^WLhogZKXH#6Uq1S=!z4AHN6?98}M>S7;HJO@Y* zN$0a4z&<{N0yhP~Fn}E{JnJ$beV*+m=Yccipmn8pYC0%q#bxLu_oE*kWIFLCIdO&v zh0C{{&ITT{y}h=rh4>}!ZO+e8Swkfj%FDG2qkuUhuvk5z^>^OnwF9RJ&D3aK3VMZ($3o}N(4@FH-XHF; z(6@g6@={RK*`5q_2rzyZX6CK44XeZ{sp@&(zUAJeD{kAv5*C*OnJ6h5^)MHN$Nih!G+pT5l#r+@&?VxCBiw2#P|bo{L(7hPuktpr0f z`^W-`J8-WTntPWV5fVWRvd~}xtUB4(fc0PQfZ)_9ZcAFz%hja&FQh&Us(V4^(j0X^ zj>bQxldJ2w$~Y9_ED@>LF4s>iM#uZ|cg`}|9`O>BsV&%=E4%W>S~ZQNmH~&7E~2JZ zv37?i{m5!Yt%1_Xci>28puEYK!)JaCA(3>BjV4P>E1DRd#rro4V5J2qUpwFT z8aWlfwr(K|(K*?_AE8b9LRKLd_$a_Jn%R57L*lRQ`^HBvC`vI#FUNvy%lDW1Q^j&_ ztHy^=cg5p?jf*ctKKj8)idi~2#hhP!dXtKqO|E1I46Ml}u)n{VRZGX1;&&Mk6Vlc! z;rldC4OaP!?Gz@-BJTtYMRB)9lx6i}aT{5W@%Jc^+==_`cN*_YJgAl5goit~a!&s} z8Ls-RUM8;Y%lcXLnA)^?R}aR0G7;W9vjTLOmYSNP62cGMI_XbXteYOfY&$zM#oBOL zL34#r**fy15LEVm|AGNywBM`K4&;t&n)v`;|R zE+4}AZv8vaK`pIoS7BW^yI&On#E#||PBaqYCl&7#Rie`Nh?57wKgy`EcuUlWd8at{ z{j(r$1F~;&F6fP<`4)SA=52_37tJC8M#?`!1EIU$(!UojZ39n+{xrR8U=PtUUA-!Y zm~%z*J1&}!XKyf)+R)MFZO(w#u4UMeWs%8J{>+9rdiliK;z9Tt~VWt#cmX5k^U z@o{BLGs#Mp;VT`9IgkD4h9}k!1j5o3KbaCQ_(^Eiz`0 z2SfvCZgggZrhL%y!%h0;8FV{>bwv|jI;Lj1toXBDFw4g($dUt!MO(UR;Ul={wb)05 zY~BcYx75wT26PnnzU2d{SR8x?%cSN=~+J8C%1J>u?ti*abToe0sy1{O8F@PRC9 zn%3L97fMb(P1}h@C>leGNulz>E>W9dNG#n^vZ>J`Xr-XZrMQ|bENOC9X{SQKr-?^{ zIo&XiZ<$R3^43h)4l4u-Sz0*z<&1g0?{1s!pD`4alp?_e`(AhU6LL&6G{)(| zgwb-aB4;#)2?kJKJKB9nCz1aa6VY9L!OxoEqkG$Dc!(+I#<_6#(-*v!tYPU;gT+Gj zvjd7yG;O&pcxlXG*Lz>l2Zw3IZ!(DKn@Pkwbg+bZz@>xbNZ^M>>x{zdtA{^s@}a&k zOgJ$LiE$O@L6h`F7v+nyMvV2*@3s`|=xzRPZfj}B-Uu8ev{*!4?3mpv?dhS!vC1?# zYG#IzZGyWF{~8C%RqkG>V0GO~nD=dXAQ@ChF)tnh&%R>`8xt=(m5_}U95xFi=TKpe zQs4l?Es!Rsl!2wJn9C@OO+kY{3c`l-iE6-n4m^&B$&uII!*7d}z5-QKCN6Y5ux#yY z%ppaJzQ@Hj((W}9A8=5za2x5C#mUb7&Rh{o#}fVqVX@)gz~B~QPvgAGS10x9+it)h z(jJ+=5ul>O+jwH$Esr+yIs> ztQG9{_sjU3cY43(I-I619y6ydL0+j>{Ul~q;pY6WT+@T|k5ogN$z+l7@1=jAdrT4T z&sM_l)J1WU#O?u;LePMXc}67CmF{>W{toW4RbhWXH-v}0MZFy!EEsc`hq z3smeb`VA~x5XsFZ3!ANAX(<|3owpGbKx7Z5a7CgR&5x05!K>lF=tox-hB|@7HFP+2 zocn_GktjHY-j?_x2>)^ETnc&fFE{5h z2oFN1EX}U3eLAgi|J#NJL`;H6!WScd(e%1Smz~wjZ{9vB7We1CN@Yik3~r23EuaIS z|EOwx1CmSVC)PJc-z-D7m*G+9!WkDhtmTbAI8ULK=q1aJmNmpH_|CW!H_0L+9Txr2 z?>25ah|>g63bVF!@mETOB=}T{Dubzu9)DE3UIL7r+$S*rv8$A-ii9n3z2x8B9QuA$OI`N*6_ZonV*UWVo16eP3cJglnveD zMqfhSYlXliD~vD~3zKjwa>O7}1|VMsT~XVrWuv->;&Td~zYpwz^C?o7c{jKWupicM zrft>Gj^>WkL2T^oCT$tO!`%7ah*Hq>Ti1*n1~lBA+*8zeKW-E*)9WnXJwT_5D(!AG z$G0(YWcdIQo=!S0q${etD~kr!7M@f6R3i-&jftcsehCZnEq%q_j0;vHNl4}#|2bf? z?Gg^c8I%JZ`Cyq>&>)EPua{jzY0Q2Ow;IqweJkTXZR4$37=L7zmx~V_mT{R;&&Dbg zhD33*uy&9pp$pHDo!GUev+IG3T@L(n&Gq>ZTZ=u|(ZjTiEy%&pC95V0ca{B#a@aVs zBRk(V_X3RPCuc4NTsI{t!olbeWs<(^<`<2aqKumH2sN^)DS0DDrCdVU-C^^&bjJrF zMOpWJ&vnll&7PrcyW=adW8y1rV-R<8Wy*Vga8&o}WcHaa^|IWQLA%}GGTEt**Hh3w zGsglArdBhiK%6hM*zzeOQZL^41;~3355C^_6d&N%)Wq&|#`DBT@fcJdT$g%X50zeT z6o}Z3!ih|MLu#(qTzg*F&0@L!^`b&HWBzfWa#f1li2q%8{(XKGM-UbfN%{6IqBNbY zkJyf5i$Wj%BU2)mslZ-pVFqv&zu_;Lm_iN4F#dOf{q4@1$$@&Cu`Lv&tE(Rv$mZBd z>3Pv^p2=2}syqlMU0ncb9x5S$kmq3cfWo!7;=8qJoYR5S9U%i(*IqR{QYCu7_|yzQ z;|#-N)_2;n`#UKXJgY43G|(xs!F-#a24U%Ni~|B$*b?8?;I1xd3T|nyGk9(q$8&e! z9(81A2S~z+hYBBNaZxIR9skpD$y~78V{zv%01H*tGNQje$3 zPF>+{n~!nuNYb}vX*a(vg*e@@RvPYzQA3%;KtRE*j;jALO?9# zxPr3m^@vANt`M5tWOkH*Jm(}Iq1zw&wl8^Q8_^nE@uWNqRI(uO>V=W-yP)V7%|>~o zS}sTY!am8)nNFe(B;i^OvtNnjiv^@uI?la{2gW%?<9ir$mrtbFMA0^Sor*FqH7+?0R8>)iWn zpXW)pF6;bqjQZ&SbDN$iYyPD8Kdq-rGl)-%A)+uMfa`zNKd75_JnZ7b-eqc#dSDex(gspm$q_^Xa@E;}c9X-T4)p<1S;*4JCDBCsBVHoIIP%f3%@1a>qZ@T6R>FsY}Xd-#aFpSn2bh{85AC@KDUYF=+W^ zPuVVQYEZ01=I*&;txJfBE?m307SRaLZwoH%y+kYI6%Y4WgYN2BQ7Bn9o@u}~)q>UX zwSfh2HXnIQ#8r_@c zSHeMU)WGj+(oL!`6{CMhs17}5NikW`T`3m&)^xn+pE3j>+oXfW>U;R^9>NO5{%wCQuaef z!lli-iUG&7O=$Mcrq6*gXBYlu7nCAcVdqlU)|P>zb?e&99{@GAk6+W>~( z*5hIRtE)Us`WUzKDqqg$gUqvc+T}Vup_;EZ(yb>Il~*T~H9L)SY6FM)!GEVHHGrZ2 z*SzYL_(y&ItN1p8^1u1O*!V{m7jk}n!lnV&vN;~B#DI7ipg^)fZuv;t4l&^U$JAq@ zm#{L*3>_9pTO%G<7xCBM%Dz8;Gx9w=yZOv`hJ!T!n7h9J(H{+y^gP3G_rMh?P6ASN zvEVP8NrE4}x!?G<`(mCTkr~XHgh|CC3}?$ z`G{{Ncvfb`H0yi_R1B3LxI8_DzAsj>CQ8yPjVNZLJY?^L58F4)sF{%d!JoM+k^gf& zUakdBuqaXf2R91oSpsa^Uw}OLgJXWUE_$aAaCYVbKD~bdnr8OBFOuL0e`e_!Hc&;K zkI>Yq*F}HwKtq@5VEWt5GGSC|ev1VPxdLIt(X8Wd?pmBolTN5ae^_-JYtT=qu z7?8+Y+#NW!<_)4h8@d~{ni1ssLe$}H)yxCOOK?n92SlehHZrSTi&4F2fTRc#5G%27 zlx;?QwWn#PmyBOD+9pTuJMiga(f%=MGpypkb1Kt5eF`Oeq*$r@Xq?WUXeD1G)=1JG zjjx~itBrEiZ5-sv6)!>lfXhrH6qX|vCRU5lI&B$d(E*Du8mLgfEi;6frAe$)gZmkfAQ^Oq9kpCJ(!w_|t(ao4X-!LT}RZ1w4 z2#17eUgPwJf0Dv_WIm%r0o+|n?%H?`md&D3nJ#%>quD^~^7)1XhPK$6=_J1Dcv(I@ z9-O^5X%+V(^C)CFnn#UHS3YU!#reHm_s0+&@q>uJ2F|(cd-zuAw{#IUO!`?kdeLsd zcZe!WHd{{GI7#UTf1g8aoqSq$Sx%XbLZuVCY-iq7Af7SlWz{w{pfIxX&XC+$R<@A) zDblFvS<~!c^wJ}lI&X$CpXjKh6<@umEp?Z7Q8+%Gq?FHF{1nAo$DtHxH7ZE@Mne*B z-YgtO-p-c}&zTP^@@H3KwHORti%CtTV69?a@q_Q8K64-u(6BjffD_vq=09_R>8aQf z!NXnOsFrrXPq+rM9+75yPmtyH=ob^`R%mOR0;P7Ot9~#>U`{-(greDJss0(ezN;Ku zd)Lz=TOW3zjU!@77pW{C;RD_&(Xc2AKjdeY#H$;R*2ArylNJKWs{Ej*7=IrIMl2!u z*Khkbk5{mh0jQ)T@`piC(BIbwPl0R>vQfCJTNnKrnX6LoYkdP42Fg2C5 zFm4&%$x`*3z=EwY&K?(z+UrVE$;}9ZM#x88Q=2awGep#1`h? z8#PnsNB4prV*km<=3KnWV__$y->{7G$Ob!wT5ns%kYu=G26*q&yu-P!%f-NyTqO|t{!1pw-WnTvjyFhZd-}16c^?ezw{q$LzJ_vb=q`9?9WFILr?~ zEV090j_qT`5UCy8h?SsE8pdmkw zZuR}TgQTCTFV3LHofLAMHMCI6SAZ#ekT}N3FcR28YI5I`w$Y>|N7rfLb%6CmK0dB8 zvZiu#g2F+Z4w$2vUQFi0^osk@H}bWOcAT*`e&b7f*EyPIb=yV@Mj<`REbb;xz?u9Un8cN%*60#y?F}=gO{FpnvTwmjqVOT;A#X>Ce#9w$4_kAgb3pV zNF}XB$PUeLoi-)^A;VER^e=jnzx7>j9pXXlkX}c}UWbDN&U}rlJK$@4da5hDdt)hu zqq10H0ahNnTIA&{D6T`{r^ey|vy>6UkTC*{_dHP}rJ@sJ_PvzfFp?x(Wr-}d!O9gU zy!R`^tk=CSP>azgUF*_NF}^RK?jp)ipjHm2YQ}}kLo!~dW|Y;OCq4>dqZCv52z8vH zMy{NMEiT~4McjD2OsZF6;qJM1thF3x17kPQuQbd?!tL&Kkhr|%H&D~~>qu@%7uiC+ zkf_rj8KX|rr4rZ*P0?lzMeMk$Zg`6}E~`MQB7rcDEc~I-niGlX{O6MP%etAY;)VU_*hsTP!^l`BR{CDG+(cS7>tafEJKo?iEa) zDuW8_qR1&3-M*trXQbjb9eU$j(}mHJJw!Y3aVXtn!N5Tznv|D;N-QVZ%@^Avm&~>4 zT;6OX3hyiN2!1&>ZQ=mYOA~PCxbN}Fl8!oWiDq2{pS9y>6A6eTnXwlrN=O|U=#S&K^ku8C%SYNF)OgUwAeK=llJSlJVg+EuE{zX=~E4d2UVW;$ceq)KMHOp#mi^m+Dt|v=E_8^hUFL-F3;%ZD8~Vk0fc^P()>tW+{>c zuxzZ90Y3>MT2rD+m6HT~U;s_(w~sj=f7wA-f0vj{hl365nZp)<^J!-*?bSg0mhFq2 z@{vX2D?@pD;dd^Ho!OO>wnwrYLJ3AOi5y7KGozE`xEU5!4khMcJYCY6W1roRa|(M$ zs_6SCqL>ivI?M*o%vMo5w*15F3zB(Xva@EP0gW#z4Jc;by4$4}aw@6{eeRK--ZL60 z&vrg`Xn7wuBR{bh16ixX&0LSp`@+0PxnVH_$A;k>N7v>vu%162haQ22u@ZxHPT&jq zL)9Jje4SQeqmJS(u@U<*WK^yGc3r^uGuYb1QnuvcgfO6`k zr;I6ea{u0=ldemS^jW8II~O(qaA%gMv;LMI(fzW8qW39L*w~t*@9Un^nY-6kK_%Xz zg6M0Y#UdbMXaC3zcx^o_P^6@Lh8#aQyggGvnSxE*uX>Ef4L}{B#@u%NKp;1meWh)N z#-T>qCw5TAWZSs1gal`uL8BZAQ(pDdMY@}GK{B?zE5|lxN`-f=%^*b5u4?q#@h~2? zmP>gRcmr#aiJt`)=f|yljPs}G0gl86MSEerq}6cmj2gL>yL!8iD*CORSp-(HS7vO# zvzX6U;^2|SMO6lU8Q2|~AjV6nm}`J@ssWMvWHWN1lO=vG`o!)GNqkAfgk+1ypN{N zm-V2bOBsTvWE$?P<^Vs~~|!Y9I_p z^mYW~>oThC)>fvz+0%`V2zMk+G)&kwktlKjOuVFJOfh<)#C{IurSZ_%^gFFS+Y@U3 zszjfob~s}Jnk6KVR@%{Zd&Az>5<<<1HTKqzgBc_nN7v z<$mI@oDA?k#sa3q=~kY`|;T5A?RN+R0Wl>Ep>(hfyUYqMLPi`nCyR z9TVX^$K}Mo1z2{BdsNz$QF_HPQ5}@5Sx`xi6scUM;kHHGM8kmgAl||dWR6)PS_$Tu z-Qx|%m0^vegGLl4Zbn`iDk3GR96;&mZy-sMzyu1JhB})WQ}q4|67k@ADnBvjg3ExZ zK9Wc(YLjgwI(d-bh0p`}Z``}qYI~L>e-J1pm(2zGW|C_UG;Ik!*s!ho2%A8|!<#5* zO}P+jxpQ(Z=G(f@s4`Qu9eYM?U zyUX)04?>c`*}^@^{#GVMh2QUZ4|jz2;8y7+x>#)zrp(!a@73f5h{xgmL#hgKZM);&+^qbR@`!%0 ze#bAIB#|8FR<1yJ9vxwC?XKlEH0#~A8DUJ^HhlF3r%+Zv@A*m^Hfb~!8D(6{2dIT{ zCxQq>;37q@LU~6!?USon55@_A*;YxrK)+Ih-~=2sfz&z-$c-{qgf**4o$nr2*L%6_ z8wbBEbzr$AHCEp3PY*JBsQ=u4YHlFB6gUURL^>H~@S|?}EdCgpiBS258CXX>K5gS6 z(QxwGy8dq#pzb59fJhd%p`~oS%w%0M{t7f;K!D8KbLU|1dZfYNs_vYtyHt{ z^GfUM_QC?=DB#GQ2lt|1e#y6@M(M}42VF9Ye+{{w;`)}T%vIUYk#1$^aJF~Sb}+SM zG4|bAL0wrsVk%Kk1iA|^D_k<7-8MyBo!0dtAZprz62~+2DM`eYD*72VRx4#Td=9UH znjjA7efy;n4femgb<)V4QKTxHz8-^E1NWidglxiSX)OT{cQ)lC&ldlhsrQgrvIzNA z#OKK@=@;L%AY?0@(TvIwYc4?YJ2PR&mE!jD4xCJPCGY;99!oqq5_1N~5h5_d0G>7H z+Zj*CEHLy^KlIAGg|w#WB2x2b|A-78x+bx}gHZ~A@jysKH!ZHnS161%)C6DkS?|Ww zxSXZ0=qp>~H~7_hyhfn@2jp07e$pm`)2iXO`iye)8QgNVmV(>O8jJ!!68A{)Bw9CS zqtv{p#AwQ~$`)Ed(>PN_0PQ!zag|Z|6_h#mM?5HZG-LH^dzmYZq}55O0;Mq3A!XmN zRc>rGxtQ-|uHf~cRnp)5J^>Gg*J^->-w>Xs$Z1tWt$E+ONb*;O6ugA@q%S{(nnxz? ze~)p#N>e$FeP_=Q{me6zL^(B+@@j(f6}B%UlUKuKa zL1FP~7S8*X7Shj-O!nY?y? zKOx}-{ZBx13#;NE&8z^A)42Knw98GaUDQ{HE{rXp?>&vL@^Y7|OVPtrI%F zetQ2Cz0z>%eQYe%n>Hto?gv~*!>vOA?61x+Qsrn6#kau$OAdUbY1>HQ_2Jy!p;!BS z&E4CGcv6j}zmnZwV`ej^0ryjK4NrZ?z%F#RI&vL-)*3}CC*62x>nvi(xy;jhSIZei zuB3?j?76v=&~^TD2xD}Yi(3e9Ew_64z^NEsa9og3i5TYvzhnf-&LPV*;VG~i!4XZ29EOYbOZW>x= z!gh3zto!noFd#p=6*|we_c6Zekd*v+n#^rm?G!!|8@~L+1QU`MNH!cW#EM&6 z2!s8|ek8jV;?l|uA3Z>)n9Byv*Ap=lkT=^&G&yEINr=2thtC?3_4QeRlI064XHK5}t~+3# z9>sMp{|T5u25!`{?tx8oHyc2=Fx4LV)zSUVQo9H*LW%bNzR|)vfaLdt;sZo8!t35o z^s6A=uUcgBd`gorINvdMWJE=p$vMLJxFyE2INcXD0{|}`mgjaVv?24a{qdZgcSjh{ z?y2HPwy)Sg)BW;Y|1;D;Vy)c;bmC+2f^NsZt;jHcg%%DqV7pvC)FAZxUsIJ7Mq1sf zoiatEB+p>64?PHnOuxBBgwp*l#57w_LKFZkbV>5+t`gD@=iQi27tT_3<6m}n(r{$) z<)@>1JasMd#EB5e2Av{2iq8T()2eX^+0=V z63vkkW#;^(fFmu=ZjFqbSbYA>5m$2WuCDGsHlnch7&`u6P8I;V$7Y$j#LMl`F><&j z-F&j=|Kv&kp+x_&`Uk2;Ij8K~@?WbP ztmP7{RZw1xA!JBQ1$_%v@LtO;@a6R|Ltk>c}8brqjlLB@$Z+F)2 zI=17&xtS8c2+TH}hOdh92(TEGMF4$w&d z6iH?ZzlwOaG$@599wZmSx z>RO4++vn3)3Rx~K$a;Ld@p34Is9qP&;xarN-?s*k)AF%=_DK?-i)HCV7`VYVEzf*3 zPWOWw@Thku70==;Lw(#q_4h8Oc+Fit?LN)g$AEQ#`F6kB_h!)^xdh@plxA(GbzLDOpSLZijG!i$A!JFLbtKupQCnmgvGa!J+Iv`6;E86Z&JJZM2wa@dI$@YIhRV z$*X01vIu2ln)KRejN;6_aXWGxg-a&#>2Khw35u=){Br>oLnawxF}hF}%sBo>_7BTV zvpF8oIl)$oAJ{p`r%D*~;OzFS&LjWc=wf=l5!EQeE<&<&Ctwb$Bt9%h1u~=@r2E-x z-bKNtlJvH4#q7G9jj|=aGnuKyO5-mFiY||JL6LjQz}K~pCR}nhBQcuw&q;576Xoij zm33}#q3&w)Y6_VjnmXCB-f)fOyrAh5hpaD=_;3dlQ#eUHuGKYF5`%UUfb7*7Avhs( z{wzz>x`l%TinKHMN5dRuj6-8-ZcCry>MllYU`Xxh>+HdgdFq5?boA{*JiMn`cD5~TR~ zf&63!bIQOVF(VW8l!fU;IW zQ!0jjwPoWvl6@t#XxZ6k)-LbP^?IUWZju~JcI&NISho7?__8+@vT}6PEM2TqU|Wk- z*-@E7yI`&-#ggB{Jro8N!P>T#=8zl2eCs%?PQuxDdMASOI|3&$sgLXvhrlu^4hq)J zP=C2W?Ur%p-2rKjTf*!VX0S$|ee*7s2rRzpke+2NWy@g@mQ1;*yE_Ub#|48av~2E6Db2ay&l{ z(7mpGLX&`A>VBcAL_w!STeFvG)dEWOY1S z+zR`^;03D}c667Ug9)-gVS+$VJr@)vwO2Yzxd+vKpw0+Ke%NdI6Q_#dsj?#Ad^RfI)J9N<-qOJe$GsjK+)t71I-96*%>Lbbc6H_8JmP!?Zt& z$|s!rFGMXRs+P--hGzV7qZ7-^GjEn!M6S=opkW2@;5zX-)LRn8(OrJ&pPZIxQ z#2+FTnNWUMS`m@GM;usd#x}b=PkLBEkXbnB;jO&=>!`s$fEya(OG`T zoQ5HTXZj6Mp7V0-n3kMHm1vBW(m)Eb_{}^FT-r}x!ecLOIXJ9*ozj5X=0QF<1Nc9D zy=7ZlZJ@M`ySuv;cXy{0hvMEAcXtc!6l-x06xZOcE$$E?NO3LhZ|-OB{q_9`Sx1g_ zt!w6-nWTW+ipk+A3t}t`Ps`@DIIRY``flE-NC61}VL9_qxQ3_#+J#X^wG&*&s}}2W zNql+*Bnk`>NX!ES8&H@K!>y7x;4Brczp)Jq_WD9OqVfm4l^l911P*YPqMG~>yYfaO zHQg~RgA@r(Dps&H(#GNz(JmzWP)2i`<|G|meJE638A2c(Y{mYG-M4fhuS*u)r2Vme zb~dIbl=`1Cdo6o*3{*0~sOx?@<~rvI2P}xCj!4(%IUa$|2dw1?JVAiu@iC>q5j_`mI8fU@&N*$8*twK4u*{^_@5-u1N?u*b5f8} z^elFkOQp{csA(H5Pbc!E@E)PCHD54jPVgujQX!T6T0ANFl>bKYRt#c%5_|PKB@Tis z=&6v3?p40uUMaRd=-YOsNVF})#Cj@zI_Z7auY0`>NTv{hvma4^zkYasj(-wlA&Bes zGqDG&yWtNWlwZB-hg2*dZrOUs{7)A{*j5BwbCGI}!S772(Hlb#3m&^vn&;$v-R4!* zUU*x2Uy$u}c1X1R;20a{(9P`53M;KC zd0kLPeb-m)i86nRf^4K;D$}D+9giqjot-#rgxdtFRR8`W@nha>_y5GqW@xYoc1Km( z_Ct;UZTFFV8Lf2HnI(9@9eF-O-bO{{*2;;huFrioZm?!xnvkfm-!buRJHc_D6vb-H z78W;cLWL$evMYZ7oeb>g(fElC4cZC3r3+wL-&P)C`8LdX<805+ut)Ib-o38`U|#|! zZeHIde~g#fr2Y6Z7eJ4SS`;6H5`&4W*&p6Pw6NIX+|bGDQ0H*b@%c1S51~}`qBQzu zGa!$_<8AqsLW82I@bv!q-X~}y@1O&u%w#j=H+|i6@k8X|0DO-x_F30*I@58qzg5(P zGCeY(ehe37fXtXH^Fmh3g^h|jfb37c**}qgUsVJp9Xa|YRYt9=7}0LwH>}!#^>?|6 zEeHGxWO~86=ecAg$%g$`aHwtfOOVM9co3~8FOzWz`S+GD zMu_-d3SVp%+mll&SZ;8|H$5Q)?C&b>2NcaSB^=a*k)s8|@bdHBxf7`%V(c@|s zwh5h5-a>k`t3oIT%wJ+VY>ll+^J~{5zV`2O3yG6DK#R{hT9s*R%Ju65u|A!kkq zhmk+#T%{0~h3G}@v-8uT7PdEJ=5uoWCof>J*{&aM`BgTB2SCCfr@Xbt}w|Czj{CFE6%wZ>v7g$q|@SxH?n8oH@^0l5gBur z7FhKbGx!IfByoiyKKCXGyGORjih)joi|9wUT!D|R9L;S-Gn;7Ui!ou6V*5tBh)c*n zgJvUX!8#x+o;WVfhOZwbT2H5t*rmF(s#SJRCFiDOq~P;8)=``K;SZje2d)0e(mz2r z;i*3HD_Nw56#tRALmVBOScCf_v;;$iWP&Fo!|w1aK-7W^VXBdVR+L@->3Y^CA1~ej zNyb!fBe{jkFA0#P3vM(w&`+YtT{d`AXPKa3=y-cN(K1}8f#ID{`TF7zw14QePDe+l zX=?h~6kW9AG@iyt1Gbn6{+H`9m0D04Dh8FyXEruCdgsqC8b+ze8~k3kS__EM035m; zXKO7dF`c{jH_Qcr+64a10|{|)vWA8~O!O6zYdCu!_sIze3CFkm-l~z6#0K8=ub|(` z;rB8>e*DlDy0+CaFhF6&(XTOztgYp2ccJy%kwDcm?@>BwvQh=YG0IKLM@xC8c+q~4|DFMNZ7`MiI(ZlT_IO|ba^3FjigdCr zxB>?S1~S}N>pyth#eg%HX*chi>geg|-@o?e;zOb5qJ)6+i<7!u$sMq8rpH5>le@e7 z7f$VdC6tdb(|4vm8(yOb@gnCa^r+c&-FIsRXZSWNBYp#-sf>(_7UV#hWhXy7rH_#u z)Vv;%Z1A-hAYRBsp9UW!+^H6Kdm@w#{#}3+LOj>iB`&f>FT>R%NoMxCci%6G(a&;0 z7|Oqb9c`64EsjZ(1^tcN7#{Gr@`)t&*tQgjxy0sl@?1Bz{k=y>}8j;ig%DnDQ`=fqx zauc(x1TQLu$Cc0T{Y%c&lzO}0?0u!25^VCIH|R&#o{LB{7Ojv}NU307$Rv$t9`L_k z@-q4@v{lt4;c4S**6(7JMIwLx)@F2zZL7Ms$H5L+-MN`>Y3NlZt|b{OdDBv1*s9n& z;WG`&Xs8*cd)|VOu&%^x5}YKd#2;a;{gMq0*JOfLhJ6O|jlrp%xW>mH8`>)|^7E;z zmW8II2sY%r!TG&NxK%=#3hESEi?58&#w4d$;O~-3L<9N@6tZ3+#3PT<7L6a|iO$-U zHNI+k1@Vz>AYXcJaB6MgKsIvPu&e{aUEO93UGh-kTdsN(x?dT z(aNk&G?%@*TO1=Sb9Tv|3F`u8Mb=HW9kjbmY0>Rsqvh@NNKed)tz8SF_W`MPIq;=TIyLFxRnZ%2StlzIA!(i6VD zm*@CSMRccDXdxAaO*1N^?p!}cSXD77*6Xm9Bgb`CLG@~5pF=rGBYn-1;`hZe1s=Le zRhMj(czsn3iNo4QH=CW<%k!{Ms;XMa%1J`K)u8dFVcEux9#rI0wL>9TvpB87P105d z-|TKzn>U{^-sqjim@E21*MJa8Z_nj1UuExyehYp&eJ8nRSE-TbXiJ4hSXigL37RF( zOxi#5*yxighbPB!Ic@MRi_}8|?a<| zj$>oyQ5smNMfFd@m{Yr8bh3foFBKbVx_E_EYqfN>H`98_Y!H~^ZyC2UvWAhT^-pCl zW@s{zN@@QL5K z-{f#(y@|=^#6y&`=ZiYZ@$qpha=%#x;_W=2&Bx)v8nbRA0tbxlV6rem89ig;oUW;9 zeAXLac1caa zWn&*8KWqkENWjvTM6~)g|7D<{poFrP1QxP19;FZoqB?;;zf*U5p3m`P4k#SOoE}!D zmC%CE=YL+mXXsI%T&rbs%l!PeYExhd)ti+jpPz0|6o!?7@+aB@oAihoKPfH%MkNt5 z851CDJ?qATEG~1-%wH59O*Ydb&`?WXhochu0ACV)IE!-~GA%QQX@UQ7Wyq0>8Tz-$ zpNup=1k+38+6@1mDCxjBgF#w%Mab*H)d%DEuHsrpRfE$ddDPcaJRzlP$sPw&gegFH z*VeFQ#C^c463u5VE!t&#Sy80zy{SC0{-2H|5ho0hS<3P$wytkdJV}Cfu4Q8=mQ~z`y598j6VO*6k?E3@;b{VMA>b zg1C=tmaZjmQ!qlTWFRFXiXXiY*OD=I}In9>=ZgHCm_@|F|Gp(N7H= zFj!(wQ9)YYreTz~xhD(um} z`cIzAWys|J9lX#JER(M<5{I>{Am|+NW!q@dJzj=ner_rD&~;rIv$&(2XayMwm-HJx zOn=30duWI4?5gkHRmB%Eqjetdwq?lTpx7|eC6^uF-xg-owxaS*_^hW2+7Ijm`BYSJ zo{}|Fkp&gZ-=df-K8BXx(50UopAS>hC*gac?e5EM^=-P04%&HaADl$Ekj1BLtfDk^ zB6BvE$CAaMak5^cpglnyN$pO`iuE+tkFIo?2k0D&|HlFx)I`_yQ~q`|L_tNt>Fn`g z!?y4U`k;=bK9mOfqy?!4jw|MLV;@ou_-&J-pxqF6wqxEh8d#A=90_6u^MF|1RfcH; zH?;ws-@jSq3YeHpI{-7*oB+{Tqw49)wqETO?PTaLc-(xVq)gv&%erfPOD*{3x+7e* zY|GhCGjEz9S8I8~d^EV%;b{n<8Kh}RaloGuu#%+=8C^J1;4o+0hBUO#;ZU zNZ2Na1#vVX?*%KC!MbiZZ1(Q1=Koa2Fyi|3|BBcip2^m)vJ(YW@7~Hd364BP7J`0} zGPf)B9si}bZ;>3Oah@`a^j)OZA4=f zeMkrHJ#jFhk}ym+{MN)JDjLvyp&Cbr^9g$`b4jo@hLYu3x1glCVvpkP`l`Qet9Brg z*d&z0xQ&*L4<3G^ex=YG1CwTOIqGVJ#?ij64_X`j5CkRw-2Gt40?co<>Fxaew*e9f zCS%GTEXy=^4#BU&42rqq-*xqXCow{C}efgAM{AMlD60O>!WTU|qOwEtg zU|B?>O#%94L0Q~AX|R(;rzzgIlYH>&zHoIj0v&!H=fqC2P885)9tp2Y0owe39YmGOZv8!O6z(y9KmvhC&Vn8+#NwI5U!%3}#yCa#q!%|uO)J5p2d_)1^^H~{osX$D2hmlS1MUV zGoj@*mF!j24)u2w{QwWmGg1^qb(W<=&>noghkMzmnbobr?_vaaI`xFT3&AANKU+a-*EU~ZUl?X;& z9aPs5y-U8=Tm}8x4JisCEum2U#e0BUMf$$gL z9>`*-UrWj1f)Gc6jBwlLigR(*&oAKY#h z8pht_?&aovp|_Bn#G8>?}Fcgsv zMMk@SFOsJG``9$!^e0m_Ij-FKhKCXDbf)x4sK@};f5dC@Bx*=FHvyKiBbQes&f*D{ z5vb5gci)}7I)&l>9JNi)&Vm2^g5agjA>75VPmVbOX9_P+LpY5Ktmp(NT;0>Wn2m-%M;5ENFr&0EKhAOCI?$kl)`3S@f4G%)b1ywfu_=6v$=NRblyLWvD`*U_ zMgz_s59{xDtvtwzU*JRF3YV5BN1DEhxq6)P|3T9j+F!#vb*@yb5+K+h86s`{5@?Xa zfmZc0rTFJ6CVX-)KE+~^=-<-I;$;JQsXB!d=~nexvKl5MNYGY%BvL8%IC)k`oFy}Y z(6k05`US4|mBY8xp3Oht+0A;7U?>_e=G{Er<^8U&e2+HpC}YR%-?6*91eYBZ+5;;V zOw4DAFt_=>qzXESJ8_-9%Mhj)$Fy70yt&03gI9bxmnkggvS`XDKPkt>;`70P(_<|` zaT3l%J?MEkE&!eS5N|_@9OapZ&}?_JXoQ9_ZJvg$l>!7Lou)?9#xN2{@3MH57xmmU z*r+%+6PgC+7;JK%BR4o!E^HOcMF^hBtGKsv0U%BCH-F#m1rOtVUh=#=iF$0To%(Ld z_fr;`@HRvnztPxNtHes`_HH#{Hct#m7O5CdqbUEU-Q7Dl4B@D<&qX$E@DXOtzD-Ak z23%$o!bC(aE|o?4i%W4?4)<0oFeZe&R7sY|J!hP9x%xDB0(#cL(sTpeuL_HbEGiC5 z14)V|q{3L%^mK@6lGa0^(T zJ0oM2+&3Z?{2l#dwu){pY?~~r^jr9KAM@j;XvIDY5e{vYxxb+knA}4yGlWOs-*(Z zz)@%-tbw+Ke*B9VdgcI)Q9J}JKsbVe|BPA6lK&L?^=THg9zv4_xd?uH9NLM(p_z(H zP0c0c(U5<8yhWHNLIsCsEiXyNP| zaG69!xgO$J+q=-`L*r7Ec;+!J!7O!Rub+f$6w`=_u^*oje{exjeWZ*t2I*p=Sj_O$ zomS5i)mx~`MHam*Z%nk8MapmjN7vSu9cNmn$!LzW9<`7pFTFdG!!m^l*E!8sXh;>8 z?Z119cWfT~CtgDDP!-?$5ZB*y0+zhY{!=%((_l9b|3xJWAv8N^`n&6R*NH%n1N`(b8$e&NqPlMe6k(X8nZPx=o!_tVY2LU;5R%{rwKv z$p7lUE+61cvS*0@00)2$j-*;$m*W0UPqQmvrb{(_&#T*{By~Vk6lAZx0`iNiS9-ag z@)udWE2srE-*xa}^ixFLEPT6ZqVM75H}*S~fj{-W4ONau7yal8VqY2p6b;1p*Z4JW zz2tB=t5%|=5H&`I4h;-TgG=nGKY#waoe%mkh8Cap8uU4f`8fMJVxAi;L1T{#$IzMw zvoj2(?8zE;k8b!bZ?>I;7Yvhgip%Mymg(Xj5o|RG9c4L>N;U*HAwcv$n6z5+~tqYL)g@>=;l zjdQa)8*G`x>D?-fc{W*SIYvf`CKzRcg;C8pX?xL*c}9aczl#vV8^GNU5pHi z;vwg&$cVYFp{gmFl14zC=-~oF+vx7j_P*6#!Pqpw%lUv49hYlk?_J_nNPa&$Et_b} zmCv#n#s3}gcQLcYm?TC>6W3pz0kwv*qDyoI9GWc+=H zPzXBdZ7{*xQ6~e;sHNW|I@ivzns09{?BtoO*k#i2t0bud?**#cuTiVlTPb((@X9Vw z)>bqSoB#^{f|T{jwn#eQ;;bDlEHsIr1B;l&!S;41eXTWHyE|pX5KU(+@1GsGkn_lw zb4B*jS1S!RNe9*;@QnN?K0GY@DmJ*SUZR>&d5>&^8tq9V@FrkV&MVt0mLPta?`4Na zkR8rtnHo>?7Lxrv?2R_8L?B|rx0iz&H$T|mv;fX)CX8twrqyQ=@^Xn$T|K3gvJ;C@ zU=g@#%4&W8Nfpl#o{5D@v1_H#9J130VF&m-HPaLYHD^aJw= z8;Kd&yTtJfQt=?R8SV7A)zdqFc0L=6K;QJ0PHlybrfxEy;oQ*vq+kR?xcZ(>IP#yr zT|Z0X@pgqnG{d+$2=`I4ew;?L7;aW%Wm&h`%@KF))kH#ER_89>W2Pfbq@+W&4W-OJ<#Bn@y4k99 zB1L9wqG7;Infn*=qIdOyA|uel^^;a!`9;od>ajM`s7OYtu88)Tuqe=A{sgztjC`3C zg{0DG?*J8+Vf1{Z!CFZ-1kfD_O5go&$nb5PQ-msSx&XV_ox-C)Tq;~@Nq-^YlJ~<0 zRNdcrp-0j=UW>jx7_z)egkN0E8o7KOg95cT{P|OW)X}>UyF-Nk^J@Rk&@)3asXyEL zo#o*0|3p{JZ_->QlgyacJwf=#wL*ott`az+ccV>5z_6!VC(RKKq`7IhDC8pNAoX7* znZo2Txd=C{vJwvseINclIKWUlH7M;|iUI?+qTIoH(x;xl^?c-?@kP0w=n!hb#veu? z#ZcmoWBIdkK_c7d-{Sh~Z7$sDfbY#`$j^nfZ)`X?H7)wdQ6|83GdMgOneiF4&|d*t z@DqY^B4@#vaaS!t7jTT^MY&ff5NkpUra}&J{FrPQ-Loj8W+d19S!we}LCj`9J0gDj zRieVXLUu=%Rs!IL8rcvZNKQrMwt}Aa039e|F8$*^B~%z=z34oM*tQOk7EH8oXWzGp z?zv<;dWnu$P~Ul)tD^;Ska)WqIe`f)XZufp?#l?c(z6s2etj zKqQ_?o3$E&Ko^)<*OK=8B|kMPSOx>{zTEUr`Gddrs#ot#sgDtSoo#LX_`=~L!HGRN zumfP%z4zHE`)69bS9C_V$YWx15KL~oh9Da_V;s|31RDl-p?jJ|X0$A?j+Y}EPRY7V zHL*_P1#~7CLnrd#(waECsFu^wCsw~t9Nfzm2BNvzoEw3#*+3%x$75C^>l063u|EeU zuV(tPt1b|I2T}xdS)ki5IE$wTSHX0do-ICf31Vwi330~!N%J>O=)CTc3vwZLM#%wy z`eTHq&bet)*n;!E8 zHe6Su+pLLNA*co<;7)c*g=SW_hCXc>aq~51*0^x)An}nK6KG3*tdw0xS`qDlUPlSd zVpaRH{TkV!s6&v?<_OJ_Io{_v&~IWgN>Ly%+FH)s8Eg&wk}X$1B#z`D_Pf5Kld8gh z*rK}bEZQ629HtC<(_=g^{?o*?-MBXnJhsJ`Cf_@Y2##R2II&QYSc`g^vAU3}GN!TG zcp2JBKd$eEQYa1@LT?1Th_jZ0r{}=_0|CG6Z)n50mkbT|3la8hK{WyfBq@NaeeUX$ zUjr3-tDtc~Qha)A1TfE=Su>%61cw2Y51#nOMyl9KcunG8+l!N7M9Uw_Eq1KExwS~* z<{kJGwN_p3lN>-7TYw4ymK^#_#Jh~{3?FPSiJdFTE7z|mXwQxmhU?a0 zk6$}noQC&O_wOX{CIk(gR1crV6S8eWCkDZxr(JME@(OQX#9zp$`Gml(5evBU5zEL% zD;`?8$G;X9bRU1ju`C^nY9D6$0*rO*C_Vspiu`S$1dH~4tjt#|;saDE+lq=`hoA=y zZU{y+NTNTcmWf`OHk3B7&?=E8F^pJ(k+_x68KAsSTsl+L{U{B<&QQ za||VpEE6_8JqtAlV{)eC>7j=o^9?t{$T-LQiKm%HCS*dmvj}W3RKi`T zWoMJV+|G9~h?~a40h$Yo;aB~*!a~{sp`}js*h1ez679|{LBf{defc~2MBub;vUs@f zZf+9{T2&Hc`d!9l&6g(?F`+mwW#io=oHZLti#xeyU&3> z7ksvybwf4Klm;R?@%Vt``e>jq*Um|h!Z5)}X@hxX`0?@4e7d%x(1Wkv_Nml6?1n(1 z!4;UTn6udbc<`sA<8jph^GWYwLj96+t&vK14tN56PIAKeql=|xneXq#d+T& zkJQqLUzI00Yk*k&0vFqVr$IP#Ja545Kehj4o;P{M<<7 z+%uW`)8&^SpqTBi@{WjEG z^}!Z;5-C*(V-IE87n4;%D|4DrVC<8lV4N|ZZ%@{9Lg47jH!#fY5 zwiIN}{qbd;u8{)pQ2SKTkx5^dYAEX#5hN8X=t#9`%ZIgd7Wc}VMYN(V|6!En(3vRm zz|E`(W!z<8$76Z~S)r3>Jx#gjN2qjCfW!@t8Y0=E0|M_Er75@HjC0Ph0(EnX4=TdH zfB&SXiXw#V?%2CpUd0T>IWHU)>UUyYjN)+5$XTwH|MY)xlf$g60uovZ!8j4)NO zb|4pAalqN7j>%o}FC~NHkzl6^csjWXTvZ3B^5cIfmnLM%2&pyME?1c5XN12CGF94-?Edh@}MI+M>6e3*k}bF+?!a zTD_g_kaaxy)t@%w$sqhvzhQ=r?L&XuA!g~7&AoxgXVui+&<$6xxCh~{Gx&oN`@M%g zH@MN0?LQVYe2{gwd({DlOYgkj@tx$)O=2eEuEZTuAUHcZ$Euvxb4v!A(JE9ZiyG3Hm$A@|*Wc|)j>#NgwZkIPW@po_& zTp?!%pklEa zO~kR_+c`z`wLD4RgNT@ntk`*0W&p<8+R>$Iv{Om{SSne!_t_fmBDsgW!635E9^bE; zvzV5BluLt?#AP#^VbUTxhpi(XvKY-+xs;*kClKYuJCNjyrjz1xmo+A9F;M)=r6t-x zY6;>tv!0&+<=gk$J)kzj%tJ4ZuJJ5UTK~J9*rCaRt0#Hh;?HKaQnX~5U)pQR$7oGZ zibaOrQvchx{pozk^?}x-{;T5U%v}crbAWm-{*=HQ5T+#WLqw3A<{rknu(ya%Fei#? zCCOJRlw4GNKeLxdWmaV)Nj!F)N|nnAo?_|(*Blw{Niy3h8AcXDyIdVdJ{b-fsv4_G z+4++!K5T&LpChhBkzh$&*^^_*aF2s_b)beq#pZz{cJgG`RYzaLq$C+;0qO=YtwcMk z5d8mlgA3JA_!s|0=T0%ueO;X8y5Ql89qJI)nqozIEisu_b(?ij+4r;3cx}c^do1nKLG zHqa{?#)vJ+RCtcF!Ms`o=I_EYjEhr|9y=rIT3~olt}nd>H|%1-_QN?WX!7Yd3c=9! z1AFM`cqGU}_K>s&S4E^i z35=M0_bBil=>J^68k#}qK(L|dFa!Lz0QwH&;)o44c!St_UrV%@h|e70cULF)cm#KTRK=OjiD^7}GdZi?!Q6aAsB6#_opJ2sa>~*xXnI(;r6+hlH7p zpFu|yexXa%SJfd4Ch`;4mK#tc_=E{H(14!kF9}p2uAn5~5HRhrbdRRTQ7oZz?i=mK zbW~iAcb9Gt`62blTuK88c`ZkEkg0x*hP}HyD*p$Tk1!JlP6{ithKa_5<#eJmadG{e zfqLel!~K!D08L#P)ix5fE;ckKKk&s4dAIwg+zMDbNpa$TfKOeiss-{ zN!5REGOIEmQi7WfG1o3eBuPadN0(pWLB1q1&iN!224xqh4ZuH7g-HL7P?mhF@%=Mch$)x2Pl%BVo_ zRjk9o;`(xURWg>OeqGD)v|ngaL3lAOhW(yK%Wdr}BJH<}sA}2Nx^M(x05VzG9TgHK z@X&GiVqnzLc7=8s2rq%@+x7%g{$qe!NLYs5Sv6q&(sukl+b*_0Hp*4&?e?)_dwYi? zd!IOrZ}KOl@tXyk*)NGu2Ww_}ty?9RklsG7spk;gQ^NR}Cb@ZT4t6Z4$1W3WgFl6> ztk9T62>T|<>(b-EnYxyyi@^FTYbDv}uCsi_yJTdKxTSCOl8Dz%^Fk-E$m z^*OnYG@t4F0_K;*R(EE@|Mb;AIEQpCu}2%)O}0{D4idYW1belA=`|J;qd3Y=>ROQ@CG8<==Z zVRUrmgG8vMt4QhK%C23c@ReX)S^(dJ?N%T!{ERuh&|ew5D$igvxo%eR>iwf;Wd$23 z`(U<8sr7qdWZZ{*F-ztH@*>7F6jYy^E7@g|@J~tDJq}(&DWe@3OKo*Wy6-yJC8sAd4gH>(}sFVjH_PDrp%p%Xy3OqP)7V5 z4+?7-WJLP&shS5$6@F3WX5|)46)l%^-Yyv{vGXe(M53uz@gGd)Ft;F?{x97K;=R21 z`kyz~9oXLL{r|K!3#5l3>$|&K6g~2*e{o!4DV}HSRrk^xcW^0zaY(5oEyyjR*0D_? z0P~E_!q#+YRO*Vb58bUK8TQ8cv(k$aIKd>BMlQhE~Q8n%hi&&=Lmi_8)N3+k{l_AkxMhQ?cWHvHfmVD(%JlIO4iOD@YQq z{(B^_Q~=8heaeB`B7C<%-!7mwiJH@yZMW}@vky2dWKY+&VNSqVs*I1 zSj==i>U7U#)NI5!>&q?u6IvfyD{7Fq(ocsBml*z_#bw>sI&ck+sy#kQHvzK{=|Kf zZ<`Hc{@6OOQM*%Bj~atHYf&%hEmb?Nn42R74Pwb%@oe7}w`AL+4bQExX9o$ec!Qcq zefH{xcTzn6@QlXo99?&apE_3eec9q>kD+8s$%Nc)4q&GBB8I8B>VPlzTU}g#zo1|B zg|!Ha2;+Vq3^ws)pm4IalhX_}WaluPuWqT)Iri3)8Q+~rN-4}-N;}m+S|m3a3h~_~ zzHC4!UfX1}A1mAbae3&DV!_B@x@8jOrGr(_q{zFW`XKCQH{oq<&!h0D?~Jt=wgHd1vZNQ$o+-?L`79$qTHo3RhyTaYRVPe9KtWi{ z7UOuqEO6N{jD(AC|8c0O*9E49$~$l^J9_tzR<c+Pt`&;^5 zF_ronxdmHP^|ej(bBY@_={Rau1Volpv}v$Yz6N%cUMwvUSej|_?hlu5-%bQ+uV3+R z8I<}Vs+^-Z2{osyQ1(gi)n33bc6k zZIfjByFT3lUMIl#Bmex@W>*uTMA2rz0SDOULPC4@kDtub@sdw&D&#$R|5$$p05uzw{!6Awxs0n%uyBV$Dsa^|7SF38 z9aa;53CHzGySBH-i@a2oX!N1QL8W|w;%kpOe#0gr$j@V>WLGZDqI-cfmy2gQhWcuC z^vY$Fgq{etr!LNeJRdC-#>7NXZfqmlq2w#64Pa`DD968sH{3p4iFFc~z2NYUYvi7A z%tO|StLTw?U1Yu=)z35fa_sT*E(2lq<$TPW&%C-!aglViC77Lc$DktRnO-wj)T>Im zch)*v0PfZe8i=ZF(`nmdaAejL)1J{zD@-EFqquM^_+JMJnrujtfahHCK5ch*I{Ht0 zgvx7|s?08U8e|1A$OR$BV)1d5#c#3`A2jEn)>Mn0tm!)rjsWRI`tO!ysF6k#{OiLE zRxkqNUxGfoidA-Je+&|jj&Hpyh85k*nN^D(9%ushY=ymYI(3P?((q0_f_-8*_Cd8s zYv2%Hyfh1!=XlL>Q+Roe**jT-w%xBitGBz1tYBGCK{U1w(2=W?bskxI>|#+#SDp-4cii7m?x*BNJm@^1NU}33xZ(O)XsqJ7lT_4K0C>7^QjQ) z{*87A-0h=e2Pb@a#6gb-5HI;U*>4^(cHC=v?cvt6k0P?<7s1LK!m{X`wOg4@Wb;Av zSJ6YeiVNmJuioN1U4)TAH3s>j$p0Tv?-X8H7j27HR7u6D*tTuk$%<`MY*w6#ZQHhO z+qP}zu797i&wW@A<6*swIln%8Z@nS5OEYukvoR6Se(9z~VFPPQZxU2{3SKf%w2t?Z)veY5KWViN2n|K@xd!_-A%hw1E*I{#Z%K(j zYJzVqy1Mo{s`o-D?i+(-vq@D}T90sep7B5(%M92Km*AUu9}WJ*QR|!)8GT@l??BHE{$oXb%KMsYQGRf8d?5afaPaqXk8Dg1e1Iyrs|6%zg(S^ z_v^>3jNN|j`@h^WU8&>8ue`8XggoI(g^@!?LOIU`>`wn`g8PkC7?k))SD;d&mfW> zn+)7%#bn?j1`=bDjb#oVSo4==Aj@ZwWpRN%^3p+zhC}%GU7~5W3{_!SBy5$+vxjg^ z`qtu@QS9P`(v3a(Oj4KiZul@(S6gn~A3xC4;~9;8DcB~(d(h#WnraBocdVEb%#e*EuG+K*2li3zjU^?Zjt7Al ztBYfr>!(2zffMZW{$SYV2^LmQqTv3{(zP$?N(7{IbC1e*;51Sr3GOY$`Dl(_Mg{z(#s!@gO%d0X9P~V3#o(klMODAb z9bEU4!s&-sk2Au>llV!bg#8#z`CqeJ^CPJVD|Mw~tC^p{Z90;WfI%%!Fg%|(0nc{F zxUM4@29?!%%o&##QO3Nk6~kl=`5qy@0ti%!P(F`umdY5MS_?W}kBh*7yJWGZ<2-&s zB=qajgnXuBALNE{qjU~>4mI;b@j`;jYr0NlM+ic5;us=u0;^nysAGY4i9Y(Dw5KI& zv%6W$7&Fg;D>>ysZZvC5=fgW}8}!kGJdu31$Bqlu@Xq0qR5egC4>_bngu{P|GszV8 z&3j_APa6)6BioO&Jg>H;h%FyZ8IF$6=03H!d4nQfTjhL%BFb0_oFQ3%m0WEW}vU3~ZS|eNR~`@qpv%3Z@dvVt)TJw|~EE z{vJp*VX@j@G2vbz!)-z$_)T)AsHz(D;#RoP&#|lVCL1?N(!Q(dFIv-iE-0&|wwKO>bM={_>>r3lD5%_l{?=_b#@D$D@00Xl{ zaWl$9St9eeyoBy+e{9hd7I7s8BETRnkXnkWLq?$V+yO`GjJL1^sMOWCqr_@hyJ0|d z1x*M|4q21gd`amLRW`$u)6@Jzt&vUs4q%%0)AU4Sa2LLX`KS5R`E#wqn_^9-aP6#H zXjB^(7!WlleAe|WVo$D`Xn?8NzQhal5-gl*pWlRg3G}Fd9+>oHQY_yl80+jmXC>9O zWnKuwRj3G6Xg|wc84|D_nI}lGfzfDE0SnWR&fyC38`B|bQev7~{E@9xLj4V%@|X1y zH=j?f{PyUGu8gbE4HqSzej&=@661|?PR_BPSU{cZ>a7mNvitMrxLibEqM6*kIA;+3 zMH`O)C(uJMRMfv+qH$A8w!|;>yw?^Jnu1qe+MPwe z8R|8;=*Dnr90;CPc(tmv2#I)%>I>vqq(vG^#FPY-ony~h0K;$IbU5h$(#X{f?PhIU z?-ZxlSr3q;cy$z3mRrEw>TDWBG>h!ywcKLZ!G`BW>cjE|s2fWsw%G;-`mB7Tl6eI0 z&8b(~-~?FYEr+=pS*o#K$)$xd`mG7$yq}lfJcr?OOrZr>?lnNlmYl53Ye!Pz5vn6* z6IzgfCVSp36Rr{?CjT=DHPQZ~27_0*?l=F53~ocAEB042IT-dEG4Lt9rl=Fvzz^hx z1!HhY4iiwz;(*f!%2MLIc3Pa@RSlzblo5#8&5e#OZWeivJtn+lLcBl}u`9$60zWOT z1`iJhmtR{b+q2O3T0H(7vCQv=kNK<@VD9Rpjme5s&LyolI}ab;j|*dk#p2|Kd1Lyj z8^Nk+U($?JELK3kEKpuJ&oC3Y4!vYpq{kx15=2PEg7GhN(yhe`1`WJ$2UaVtM@i^k zCGDYGH7P=|<{slHF$%sjO3va|@Zgpth7sJW zoc1JmIW*O~0JYdx!q9YeQKHHM1bF3Zep^|5ACw+z5|h1txq=&s?Pz>r2p?N)x(?cZ zt_PIJ-LV#Q_M}ErS3<-B3h&@;eWQB=*MFNyU_)6_ z>V38q+v`3u_jC*^X1d%h4%(qmL7U!eWV?jA+HZay*?+3hLJ1c zjYVKkjIVn_j#L%i-T?I4f>xwz7Chg{akU@nvU-q9PRT}bdK^ux2}ZG`m~8ac>#F+% z>X0)=61|Ip8X>=GCAT|uThY1K%Iv&WP+s6DK&$bpH=@3B1+}elA@SoOxq>g;y6oxm zIwx9qNMNipb@HA_5REQdT*sdb;?$t%B-0|U*Q9QBlqlt%q6L%08F+Ykz{ikM=8Nu> zwTRGyi^el7u&xF(w|)F0)waSbBJ5v|))({2h+B^H_#DM0#`x@-el1qI^$S%n@&C$} z{HpW=al2ep8{qV_NY}$|;w*PGmy|Af;D;+yZ@H1jxN#KZ=31LkwBE;&=X))1=>{9= z<3oS>g$>nZe#ve&g|p!_FZ-Pbh3!=V^d12rbltEI?tu)d7bWvU2Z1`b{FGmlyb*+Y z>H$LtA)4V_k>jIrC&892^K+9^enz6gqcpD`e*W!**v9}Bl2Sc8Ok|Z)7nbA~_%D!| ztAGHC5HS}}RZsxlP{Xs7!%A@Z3$vT7CWk6UAsD)2_SmJuda-$5J71J!QM+PVoO^J# z*CVj)lDv2Qdnr>Sn4bsU5)9y8m<_3-9Vm1G_H$jFS`s%x(OL{@^4oclq?s78=q`M$ zIU5F)wU2TM7Y8?=7E}D&t066cAf|Diqlz;K>UC9DQ|Qh^Ag0Ot^UX(b_0%Q*uR?H6 zmiuyI4woO`T4pRU@oVCOMZ%QTmP7R$)dpOFYv#<_ukDpmfbjz=+O53H2Li2|67Y_Nk3nLDczmGjkX{7;ovVxq+Yb!7dSTREt}&Id0d=lk0WXW{O;uq)O6Ja3tA8w&B=~serjZ1pZV3sKy1AtYWl)xxQTGyZ)!*TQBH_VY5Jg@FdSlXl3hhRg>!&4t|b!VX3mJbX%ZzqnLm`sd^MuqjU zshWKitqKkQ=S0_w_#mGMXNTh2c2sSzf#L9I4R?^eRxg%F>-#QEv>ON;?GxXIQU{XK z#BK|NaYe})l(j?J`HgbFO$KsY*yt6Z`l^n9nIHTFqo|R#zzLN3p7%DzW;p%X{f+t< zr9{(;!s=WE!nvct{bM}&JAK5+z*3WxBF-cHI^!@gG@C0v(5h>eVI>kWjE(R&Uki&9 zj!*z?p8DF7x?g+;k-esN*@A%4!lUc%Dsa2ieZH3iO9BDDe_13v@~3imGnO2xXZC!S zH=+6V+vu7nXT5B$@l)ce$qIE`^lCFIo}=?Q5D^p{iq$EoXC}x440hQty7Paz0G+}o z&gQ~qC$OD`NWbG^22KaD5o<%ih#Bx|W;av?KtNY6wfNrdcrw0fKqj(hh*CgrFPjseviq8BOoj@zKEv zPhO{=d9Gi_Y@1r|1xy98WCR0M&WicFMJKLFl))+xsMGLed#t`}PS*uh`8En1%T z;%mAh=FH#Xk1EArLl(G#5;VIm-AfI!iGhnc8a#;7&Nv54Ez;DlM6>B7an?Gg-XomUV~Gm4iiK3QsIRHHeN z*F0Q!q;VvBp$wU5$3&+gf}V2!$WARGJB{PtA&mWk`b{JtZ?#$=qC)x<^ldJ=v<5@p zk)zCO7)st0XLwGgFQFgOQ=nG{5cx-zBegJR_)T6TSOw!J+D!w3Stq+1^aIX4D}$J6 zIxp~d58?8nerTWE(8Ik~*qt~E55CVw?HCu%L}J|M=)FAWAquPN>a>qaUI#J;p|8kJ z5|181R>U_xr5`;go-yA~>70|h!*L}2=k3xYa5<@KNan_aM;J631vFou$+(;9NF`;1 zIeJP?F%X9D2&{?ucdcgH9}-hYiXoCyIUkg-=6N5#sR6b8AJ_c{<5%T-8^!7Xqv7tZKBcsQ|JvZ=d@X-X3U+-&zUeV~zH*_}Yd{FcKE3~uNRcOPIeO{*b_F!idkCQ>0fo2IGriZS>O1q_~-UY;I= zz1SC|(Lg8gZng>2!s=ynh|*5@EGf%`@WuQRto=KyAmgSCUT-w#-g4mC(B&-`g*Ox! zsLaY+*Q^D? ztFe*%Z@75NEm2(0EohqmBf&$5opg!~hb|eE?5IcKaRUMS8Tn%Abp0O-JLu;WU_3AOhDC0^8-dT-ROvFI3Pz zf_)`OARcG>T!mX>)-I-eo{X82PgWz{d!_}!OX0_-9~Z=0nGSA+6{XS-d(DtYl1>ms z7*0qJ_nHgOhb$1fNL=YUS}{t>WtJo=&+)-iVjqC{Q=5CMDYtW!oGE5pDr)>xd8Oy- zmTLd?K6$*EG!CGi0z6o$-Mb1b9$ovLIjz-X7j4m}{k*UWe6y}t`k^0auC{$Ys6zTy$Ey1CwxFtK~ z1ho}SK08DN8(p{gj;7LjVYrtZ{`T1cCF+uOuRqS26Se-L#kJm_Tt3YloESH@i}_W@ zJ(;3m%#DQ0?{wWgHaxbuS-l8(Z0h2Gu0MQstafGVTKhW6yaql)Dxl1SrhChUjPEbr z3H=6zjR<#1wLCYEzh;<{UCWS&3WNsI)CVI8PFuU@3}QMh_h| z--(|RwyIq&gB>{8v4cq1KBLL&-9M#lbR_`*%Wiw@I&7f-)#X!B6J&16}~yIW`v7S9cJ}jVKNN zU>uOZ;SyicBVs}_MhJoc`sb%(Pk|&TAD;UG(qqaG$kFGxxNUNXJ~*IXR$Z>4BE_+P z59=uZdY4x_XzE?Dalt3E)D!uuIVb{z#Ax&i0O5<$2c3iErC8l}%xsY#5NF0#QiAeA z#f}-ec2S*~KPP;8t1o}RlTZtt^1J21afPf#eTeKS2-F?|ZTyvdc|npq^FeKx_MT!# z#Fqup`=(U~_nme5mVJKh+YR;w|H19l zgRuUEYr5(a0X(I<3iM?U!k^reSDU%i!EeqszjG1ANYh}YhtEv5pb_~@ki>5^ECHO1 zrUBp8;iG;jI1SFxlrTgP142Z(AD#`Dwl4zee%DbQy@2&cCqZSRwihIG(d<_gqsH}F zlSvh-hTLQ59!KXqExTe+x3Yn=x)bS;adCFjP!EaqN4$gz0oRGnF(JUJr1y6Mg4JXz zo{-lP+1gfgf^|M!Uzlo_uAk85jD^!}|Ez^;j&#-yGiqwk)1fRPW;xS*1Fr4+k6g;d zWclZND!uBb;x{vD1z*7Wb13~htykJvsJaS~JM->r*ZxOsL*6v1c|0F8Qf$I9y+FJa=?azzF zj*uQe$fH~;X0z2C_Zs_0JL9@N{3|j|2gs;9gc-tgJ`1+3R4(bgA_VK#OSOI)A4C+f zB|s;ECmgZ*-1=lFN|>%$j$)2&+BiGEzxUYWmGIPHZ+mBz_}$plDP$r$87^@cjrxxy zMJ6MaNwc%5Apf6#hP=X^ zCQ46&k7tK&;ZGF1oVRPcGR{PvhLI2kE-u^wb=r;hq42*CAJjZ=w^|p@EVEnllZ-{M zh04eZO=X=Ka%Sv;pF7Y&jsPEk4v$)6V&xN;O=wo&s=mUj>NtXzJc1XH@=@?c$WYOk zA1{vG^c&rDhi4wDsReuG%1S?FYvSMr9PryPc&d|Cwy;CDH9)e6q-rFyY=M891vgK{ zarf5@2Xn1JuS_il+&m-`8%GigEXjaqKEz4FrhY(u__kOy5j|t9EH3NcO$8w=BpOLv z-U96h*oWKVwaox)YpGV+DIp1soA*Mfc_UNEQj4U#b)h@UTa?b=-*bQ<7E-q0Cw=S(y#FZ5NS+Zit9ADy`G&vq9Ztry&O zF*P+%fZuy48Sgm;-2k1fY*&<^tslb_q7i{mC#NKTDJ$|24XvuTEllhc;|`ZS60@0_{^AL~~QvPo+h-PSrth{x7b~Q&itiLZm6*DHs-35(Q`aba;l@t8M(3qb-k6WLtF^G)zrWd!SrI$~Phj}(F4qAt9|YjjM+ zzD3OVTc_4V=Norh)I#1%ZCw7Wp*ROQPIpFTd2YXXo3=R7SxzS>&VPY?ig3|SvNOU_7v#rCnx}BmYyqrVgoFu3MXp~c zvBTf=$rIT>U=;6vrTkXe(nm!_W%PWqg77T>=NS8j;fe1K|CcM!q|@#~xj3&!t;qMK zR&$`&SRL>b60ncnn7)DzeG&Jpb(;JE5A?eK>MkQfDNHUY7eVE8>x4Be!42h&wG_KM zJv>$h-T+X~Ot2r>Gc||fQ^kq<7`bC1*u<~)(yQeb3_)IV(Q;_CJFfG&_=Se?nbiY*KtGx+m z4gM^Ioi~|yDBT`tv!7yO7!cH=TuXzTiRAj^U#CIyfvYU!Yt(r9+t7)hC!H0tR0FAF zzM$vJX);RTZUu9V>u-YM<)>P&aw*PkPYM1{Xw;$*R>|Grwm*0Ivfj8sEjPUrHiFOd zSK-L{b6$M`wMjI;yqAbJs#MLrfY(1;cuwNmyI|2?Huxp)5IiUxMU@@M zn+MVao>{hfT3=8)a2DHb8LwkafYNs-E+=BN?*;_L6IN8p#hkj4got)q{Dag)NcWR` zAzhzOIA`&q?*FE;o^3y$M|dc!_OGFB$6@PcI7&BTLG8M3k;$j~3PL|wxJhfNPL`SF za;sDl>cf`f>)hJv;9xOaVRZpS`2;J&+KBYN?W@|jWwyzdYo*-Lrq9GQoih^V3}aT~2JOY+Ar4IKn_6&EQ_WYo58+7=kIuQfepdgZYBi>2lnFrSrn^=-S~h_9CHhYq{6EpnE+3 zI9`Vy3A(24L$TO~ zie(vgl~Kw;gj2tNs4!rOobpl?4DSF9kWRAi*rab>w?Sb5c5SJt1*hL&IpHg}q6+c6(Dw z-yCsx@Q7ku*3Wp#o8BD+E$B4jOubNTTGhrx+!I$?mKZZTu1H{6{83cI157=Q3$6UL z_=5=KUnDO)w10l;AE|uphc@>z2j>G)I|Qn^D$|seQT$MVoPaP((V3mt;sN+*nuhX z@81JKPZqi*R0nSpe~f>oC2(Cq%7b+ZD(qhTVHRJ4-#S%C4;jjeemN``=iN3W9h$Yl!`s=~8SgHw^8K{aY-$Af3qKr?$)%8LTm?dOGCv3{E`jx!M! z@#;GDM4axTFT6sm$P0v>xrATlNPax(ZGIn6)ZxT+3An9`hIzvGH60J0LJuHgnrs7L zmzg?Nb7?C_=0MYGyShG_i}m(oWVGy_jC%Rq%-}3V-lfwQ7>K-$9D-_AaZxG>>giMb z87OF2j4^VY3B|wf^0s#){RmViB&v?z7r)oJYSWP=oLi)NW^Q)-RgzNGw?6Zsi~5~$ z#MitbtR&F3b7&8?nfSs7^AZ%)blnR91D|>{qw>hqec#YbUw0`GD8EKf6VndjQzuOq zm1;MXx2=Evj8Z2}EEy+hm#L5=?Y_>|i34#3(h8`X!9m)5q$N!=4rW5(iq&(!T+vc} z#-Bc>|8MR7zlRsmrfmE74A;v9@!r&ZT5p0f03Y48JWSf7YcdoQ_CyRibPe^21!7$cTP> z2h5H#p1_1?t;j5K>~e4n{7U;PMH4)gjcocEOA+=f?*Lv%qr-fq+3%K*FtQwh3mbw7iBV8q*_x5VEOtr%~v;)J7`)- z-zdH_jXq5(1tqm-#5-OyBnclg9>D&%MBzL`{0c*HW)8TydT9AuBu+|=8fq;1c{_u~ z#cLBKk2j=jN%f91Sz$w&-(4NeBQqg0p?sF}nJK%`;6r*m6rtSYpGOr}c4EHoH}^>8 zb2jv$F$OKC%yyk{u_OF0m=vuokku;{S+1m^P+RW&THRWFXlrvNq*DMH>(9c$ee$D?bN4S#IBvl&v)bc6kA z$!Jp`2uqFE_Ml!FeXPvHd3`t@z>MH-wd0sIGJ`$Wu+=v;LMdq?Fqc?{FswV&y5%4F znO7#vC&sSm=Ma0jA3a|xs#3O7#2AP<a5BI$g~)&U7{(}jJvNL`*R%RMXhc=wbOC&tEEBL znNIo9sb+yItRP0Bj8~`PqD6m* z>`viGR$2ZN?6d2>bv~DAeL(2pm@|xtj`KQ$F8R%y)RA5)A{2Q@NkvH-;4ix>w&h_5 zcT5&(sA?Prn$^BB$#S;I(TOK^G8xfKHecdu|K4lcN@0TIIC)AnU9E?$Uu}A<DL4v%V6C_!hI60u|GVz46m-cj!zoVFqMIJhwATF;O64O$+Z_C&3Ev)Cfc1Wg6q z?ZR`FRog5)1oWXuvKfp#0v#EBj^mV|e00_{Mu3J{muh>shbwyVY7jHgN?F~Os;4ET zeY0Z%Wu*P?Ep%8{-7dBbpC={x*O3+8_b0CfpOA3CA|vcn`~$H4aF89%2sP zkTs_kUlk9+#0-%dT33Lra&f|CWih5X8jN1Pzo?&fI7Cghms!z;=BB+c7uv+7{JOWU zoNQRL)G8pcT%5SwC%5F?)E|Nh{Aow&8#`af6t^3Fx@Bx$(mc-go-w{@yO zMFpbAuQnUvoKLYv!Y#Ex2yjSlC|Rp>ba*`oZZr-|e-$1?>$0TTPkO+N0G4a4k>*NN z;Er)bqn4P1jaQHNOXQR+Yz@1G+LDefJoPimbgN`j)TSk|737z&LiPA+<&Pe*G>vt| z(YubMc)T;MSNF+-}N;l&0?>P3Q*;z?BRjzKY z_dk5y=fAFAcbtz>=N7xLAw6CSx z?U$_yf7JdVPacXk(Ud7$>+-WL)O;s+b;rpxHYfhihnRzK75bgQ;{bFhep27;7cRuv7%nSJJ}zB&W>GToJOi_#U0*pWg4a z32RXjFX%^Ekb9_FZ7i3qjdy1qRh}a0h!U_7DvawWLrLw47akpF=BK{=TprS*8bH9} z^W$eQaS+J-{+A2j-@4l)6;)^IAl3&}7=xlt2Hicq2G6vfF?7R6T-mG3_P9VTp?YL) zq$N#DE_ytZ_k1I{E-2$iNW)}|v{qeA+Yk-hWg+4lD`48@na4Zc=&Qj@H(waHaV8=Y z)vsJyGRqyU*?W2_O6EJkH9ifNiP9((6_lQG0{v-bVb5ImqEe0*P2sZ zy&Z|wv^_YNwcqi5a#fK@s{~kD?FEmNx5mLsM)x`%OlG!J>M%rs@k+AS-Oo{gppZ8) zn)3s)06i+K?=Dxj@A}XGmbc4SzQ#GBpbB0?ZJR!>TdUQ)S=~*%Ai56yRUO+Q9uIb9 zYTxWp%iRe7r*QuVD^&cq?u~w&{6rQ7rK!s@QH zYemO@HO%*?FadUyzN9x24#O7Bx!9c|y(RScopI69Zzs~}|BhU0D6H6H#gJ_qk7n3? z+uCVpXd)hQDd6ZT7+wIXxN+pUd4iZ=;=ak9bj?( za^Ce1hCExq_P{@=V)7u>aCl{9u1FUJAGdM31*+j>VwQR&*NkI@!h@Ig8kWKl9-$O6-7#!Z~o+T z@BPWw@#!`2{EvyBO{rAtz;SnNQ_pbjpPkXw1Yx-V2x#2)(%$9n)JN+;(7W=umL;`& zjVbYQarA!R3r`2w=NK2D0=4)bX$=9;R###)%kyN%Lo9IE&gDa*akFfSEos@-)JB=A zc;I+G_xaAd-y3ZhA-|wBc5{l14wEhQ$HRt3Lz+HqOgVu3x9we|QqBtdr3tvD$D7at zNszd~FZlfy%KDKs1}q~540`^DgBZ=qs!~*GWfHP{`9#vPmLj%xz^nxS;FkPd!GNKH zvXOPCoS2!h<1%^m{RL}tt@9=VH0%R7{Ef$>+z{kI-S+(0LZH(SCh`K?Al>a-9;ysf z^zliJa@=0!g|)T5`y`o3sJoR_698U{ZR4#;O83v!q_bpv2psG&*t?*|8xIa@#sr1s z-i*8I*8EidAofCrb~1OSofpBSj?PExBLUs#k9-=osL)AO)g0<{gjmGigRovaXu^R2 z4IA@nfVRkJ>_TZzq==iT1r=@xorWZQP@9_|tkT|Saa7i6N@(zSkgb)v5XRxEIEX4k%U zPDRpNp#ii!RufTtz3voYq4Zwt?L~H#?<%Fn?!+^E!qJ!Shi$dmWMM|Q!_|1C%$+S~ zH>9OPB84Ls9#)yYma`W!Yhof0n^rm0juH-f#LfUlXwn;9Rwl1iL4eQr@#@saRjbLOG- zM1|#9XTlNI`%$IfR^`q8tM{iI`%&4w`kVT$`SECxwG1n{V;yDi-j%A0TN)-3+IUbA zv|Q-u;P2nLRrx?g+TRT~Usx}cVKEa#d`m?IsUPZ%=E_>JNple!lQ(VQQsK?-LkDB2Sj<+BPLeT0JAcfQhQei_YID|cP9}s z5kwTUfo$w><)eLj`_>5fXeD+`mJ{v2`xQMHh<&56t3X@0+O}Op?oL|p#dY_K2I-cI zJL;CVHOt_hk_iJxt>CfM_-98_>P}U1%dA>=- z`(x^nH%Dr{@8kbj5#oHoE32qbnshI0b$TvFii>M*0%N^5yq-6KhO0ri+%7nO`tPHM z|6h9!^&cLCYk<6mKlMEcACl?+(!~F~;i&CVDvx_m{oM%<+s7uWzZ}L*;F2=!eJfrr zlIX}b6Fil1G|q|b#nq5hyO#^&Zs?4PjSZSWbd?qh&lJg(MJL~fZ3WnoLc5_M{G?&R z>A^yk`lHXrD~JX#AJ|&`w}>SiYry28#_gB6h_OGPGXF`MSR|Q{UX*9MA#SIJ$vI0H z7$iP}l_VI?9+(4`&A`RzA2;6P%ZMQMbGQv$%x zE~y`ZUz2hOh&ut$o^Dw3GJ{8;OUUg8eu+%93E@tAUUv@LChPVTVUxx2d(gB&xFW1d zShR0Ln!&7yl~k}Ro+6*z{^6o^oE{lleow_k;V7#U;)ME>y6fhZJCLY$WN(G91et2 z1IMSv4cETS&W18Y+E;fvZOI{vZgn705;-+FEb^a~!>!z=Zigm6 zk_|4*M4kDylF6oCLy}l>0AvtuWlvm!KqllHZ9OeB2+`ymPEe9_MJUN6X{-{>g4gH4Pna7Zk!F1Hf%s&fIdG_n}7dL>y1 zXU~+*s*awJ{7#Dw_D>>)LU26F9a&*lE>+U0LM{>007-$~p(-p*-c6C9z~4w%V>ct} zg;(xRf#J1Sv?4#jZ$F>URf>{nR$$y})4WoR4ZSF-nzy#R_lzbiY)HptrG$xTNWiUg zqElWts=S}w`MSQmn`}146xZC%tIW)aTgdKS$E276t= z>1gs-@zICx-7R!E_#c=_13@tkS-3b1fyA8H~iP;M{&oWAZo zSL~5#@09LUs@*~<7i#5uD~bu+mlaN` z43jAnYR-c|98qyO9mmxuMoqAhgEnEX$m(~+EVQ{)w#gSV7_1C*h)ECPh!N1w1E$^~ zh*E9yhGZ#y{M?6tbYeubi4oc<)Y(Wm!Ef2pwQhN@pQ--V(r3%-sg&|po$#Cx%CLcB)+|WQ$$$;gLwTRC*Ka^1JyBC#+4{~)UZA05?`L^YZfZ=9zF}q8`R<{b?%~tA z&Ec*%{(j4T)N?>n|84Hcb}+_`Du|m#81+V{c)96EsJB|J zBmU#|Rr_BX`9J&`96#Kf{w88Oz=!agGZ#VD(r;L=hgp7&=T9!JulHsmHkR4h*|VsU zb({9f>uuj})NcjIgQh z@67AhGO`3d&CnOfm z3enCu8n%l#`&JYgRMbwABJRbX`TM&sfj1sr*oQVwYjs4%T@Q~AFdm%~KR?~86jZq! zJR7a^Jl7vZ9E(9hnk;_j{`DQPCao+!$kLnoTt7yQNBrSrFVbW6X8f+mSKG$tjcT?|ihsLr+f z>WU37tO;HAH^H^60%=@no>dcKMw!zvnqlg@sAia-g8ycopc^gqLl8+Hj9^#i=|)^;WdVO z?$qPV;2le1V3Clwv4K~-Wz>EiEf+EO(@tSl6#PYLmYp0NzC3^EV()zAG)FXvW7l9O zK5J1VLS}FM1^d`xM%`TY??H*kQ38Q64#P}EHMlX7c44+ZKJwoOnR^6)ZBwly2~CZC zJdPaMIz;)b))q?|=-?_A~%47QYMhD&im8jC&yAsX%pq}qr~m=#hJxU?#3$# zh8Hy1tBm{Q4BL{6s)O(mEOegr*=>{@Bs-4wAcHi{vZ<_6wAbd}kjYAw0`s3!64-?V zVHBNsd@XpcdidRu_`+qJnzR-C^Bp?9L6*qm1`$jYAu-k=McwP+D;RW>X##P*1Lue9 zsJ~6n46%ktX+~whVNu#*h70upmJ)yG@Sm} zv|%3>H;J_kJuW=bQFdx@2(YdLwGL?Vu1>Z#O6<(giYtn>iBkOZEO$W9X2a2U|C%+% z$X~<>ld@5EqhThFL+#Pt)+x_P5X2V|ZO$ZZLbA9T~_)n3sL<^n7pdZsXnrY9pT^FDlakreS~WGg}+e->g9awV@W9 zIPfdWxYC^ZWPb4Ohetr-y)4Mni5%40!qRAIscUqd2{82}Izj-@T5)#j#7ssP2DAK~ zqZy#Ix+Nr}=j;C4?k6X!&$X3J`ESPJB1k-v zz$?hzDq$l9m)1e@UuqbbZlFiB+i6}1;tJEJ^^2yiu7ZI9DL7uXIOmm%;8UJAr-odf z{}1UofGv&LhuH;T$m`x@`L$yG!MBYDFih9jC!fX%U!U{|*l!TPyJBCZVQ4JK0w zuuu9CcjP2=jPJ<(FeeSjm&pij{-XoU`sSnkm<^;=euvpaCZKXyJ{e4yV20P0Tddl9qW;i|hfR|>>p{zOZZ+oRnG+TzIQ?#x_2 z>BchOVL#OZn&k-?{YXy5GeetiR2++q8vFQKccGnDo+MthdhaN+{FimALGbmvgLHh} z{)56H4d4_FX|xPI{vOv|rjl_K2PmtHOIra;R=COrl7K`yBig4$p`1UN;o(j8Z#laA z_nXwxSFCD`$^SAtQjFw{WF1LA)htWUe%S9W>{#XXS5w$zpKtb2p}YDA5T*|n5v%ha}z4T!j2{kwFqmr z{|TlDK03O+a=GCCSWIwpJivl){R4fvR2%YWr!$_K=OA)FDTNOD_<^7Tio z+#F(kh5Omg0LNq$0-N2#SI_wVeAm44SdCv;BX}&5Frlj9?b0rkjH;NZr%POv?py=$ z396Xe(a9S5-~3zu-2`dQ8UCtu$Zg3f+Y5_q_2raRcHkJ`-S4mi6zv$Co&`Dsa=)`uA#Cau!7y1nqDp(W!{kBnDa zPP#4$l|W8kBCB3%XbZ{<*VH7&*xCW-0nOpfrIcShShotQ!BQt;e{=k^c;z44O+qBgEOyl=Udd&=R9y5L^n#lv^t*tr zbPxNOekSD7W;|TXNuqS;(zd@*=HG=sYO8Dfl!{&F2c|j!by-q-Fz^wz^@k(gkLQgN&HFhG2w8$^RD8M(QJ>oa(@e*4hEXcEy6!D?UE$f?0?~-_q%@?Nf=yUu4TAxwWJr~e zzh8ojeiDRkff+XKVH1o!nL%(^EM9*c;e&I$=eXxJ$oraxfMGkUwSMLco-Rs6)wqHD zzziI+i4;abt%28;t>5bEIBV+a7W&4e8Zt@#(kXm||H!laRR)nztUnB;Ns2Pu=v7Rn z3(Ad%j#`e;P;nw`&Q1ioyx%j-f@(xcVYVVGgnhF*o4dfY2$+s#9Sh_oHa)Zr}R!$e@y1kHjEQzT$P0O%xpy;9Iy0Ls$0N8aj>>re82dLm)q<$3ZK%HQ|#lnr9~n%-jER^liC#XSWV0xT*ars@tWHnh}zH)2A6IDGUq2 zW0h{@g=b+ z=1S!`Sy@l8i7=jA^j?YM=6{--GP|_pEatp*%vc6cQ89k3EdfTl$S=4?IN}rL#Sx<% zytK=}XiZneYpJ+(`LCTlT5oz?xE{Uun9sSMQt9e_=G0_!J$xO|#1l{tz%?0@e;b!{ z`860yFfFGS`YxALL8&h&jTzS&rlrP?w^(C5K6;G7v)nm9*-Xn5Oxeoty|3f8wpbBR zYS^o1@@bWoWb!kV6F3W42uhli;=IJ<%IPJn(i0j=mJrhgud$XmY`dM*yX<%qdLG&( zWl;2rNrOa|u0oWCxy#wN{>Y+=_%Mi}S`JhOqb@JENaHid=XJR@xSurV>;9D4|0c3p zvc@gwRH+Oj?X{^kh!sVyWri$1U*Rm3 zfZ6KIb2LvhM`1!9+AOB1j4sm5FO2!6g#zF0*F-iB$Hm&ZHWBKkY2dgal%iTNGS=bb zarfwsgK4Jk?nra7>C~y)5Hdk$m=>GLlDN^*Q54-Vv0~@EDS13%RQ_-&B*EIls4>>o z3??tfs7O;0gNI0Eyc8Cvj_ohPwQFwZ;=*buV)!<#G)1a{IbdL|+RB3t;@G~W_AdG= zcMXyL$D`GLgVK}d0j&9IQ_dMs4CW4|dq|tJn*YgCAk&*9MBOF;(a7B)roCQvtlgWO z`1dn5Cnp9(wQa7nw#4r$+XS-71d;VoW%Wtz|vw{>I@~8_x)1 z1dfc6E=Wcr#XRX`WOV(Vy*kN&Xg+mlq~VTlRusXKQIJ|Hp^?hTqW7ir9rsS^FodYT zzmYv%4wv-W*47|B4lB&%*@@;agfJ9wnk#E;Xd`*qVC6y-ltIfUUh^|M`+DY(I7~dH zxPSP+ujOyCV3vm9mEQx!@8TK~F%>wmh9XIs=}=orQkwXeXjX3fI^WUJEt*#?!(hU8Cj-ISFAemww6P=d8`gj)o^Ku)b12{L2dc)KfoU(f&24^Th9mWt)tYZxZz=ODD-xSCR+>XL z@~YM>zXQmVvkV;4u(9OKLT8kftJReTy<1E$TDJE!2L&%G8y0Us?GFZKbOjqpzI6X3 z$kaLseN<;>Rc#l1H@ySKa`IalzP&{c4N(XP=y*wF2>J-n{;K999X=uo{Er3*cnpl& z0u{94tv@*l#obDzRslK%J?KVKGcv}I%Ou+ZwO6reX>Nm&zpQ=?zH46Ub$>i-l79$h zlApInM#A?*9qOkCe!Rc1+8;_Dra1o8elp|T5yKM1uWD0MCxES0I%oDEZ(rjq^^t+T zhlEMeY9hwk64MoK9eVi#uA6V|jq>e}{v*!maMp!KRWFGPW&7PpU+9qZT(PR|lSi%< zeLH!Luuv_St?F+j3$`LhJO|A6XHKv1YF}a?{~a_LzV_EUNOl+OR_0ED6X$y~`FP7U zU^KHzFU4ln4%WiFWBUn|PX)LAfao!*sSE6C!4h2(nCW&ojy~}}E`TnDh6KB+wJ_(o zYGbqpky%8f==&`TxaVVI-qOvJ zUtg!8=ec0++l7Rrz#YY>=gS-Z*G7hy*Zu#hexM^U4UM$d$?M~S!cyoD$fdcXUSyQK z-tZ}jXPVNZ-Am5_QA+moWIlK9kZ z@sDqzA)^Q5ZPjXn#87a)4~kHF^(0IQA*G6^pIdyhpY+Sr=G5Nn!agzpv z$_^H}JhTYMc{ShK3nEu@p*Q`&rRQ4KbS6&ZhBB?j^^MHn_4P?22|3dbX1Jo(^Od)K zUb6D&`29+*vJm0-@?guQ7$Fx!e`e!n0GIf+IF0RSKc~rM7o?n{ZFTWkAoX&UtEwkZN(x~+} z#4?c+f?Ia@*v2qI=|ea;Za52}mrdu=@f+^`6aWJU`%*D6C>aP_**KZlV3(Mqo$Y9o zqi*r8_I^LFsNwp2Jt!;toxe)kW|m|Wl9q;*=}$i1+r7;JWQ7&csgHr`qBQ0vFWBUI4VL(*+ZDtAyhW@RIeO(({>;k-62 zCy85&D}zN1=lSi_g5pprFUK-9IvjVK9`7midJ z$52WY(9H5X|3@S|G=({FcSz^0S*MrR<#Xe$)|Ru)Oj=YGr>gO8 znA3rYM3-{D#vgw!aue+5T5m$eok6?0HtD^Sdso%3W*5$TFHMX+P`}JHHIQkYBu+Kd zT|EUCR+9AWp=A8?_YUdZxH|cy()@Ieu8Wp3IEhulsRJY+q*o(f0sb{peA| zSN#nXZhDF1+-UA&Y*;ur3}2>&;e=v0H@dh zz7+`fyKAYVh)YMqRrXD?kuiTW;?Tt(4wg_kgR<2)8+@~v(TcC^8^yR%9|(lAav`$;bSr>5BCW#4(nnopB|aLp9PQqYOm4|2Zx5rAaUAg zJ2s90w&IJJfCh!Ixs@FSQHPFkZB#s-CpBHcpH5MS#zk@I>ETAx(n&9?ZjEMLV8#={ zQ{N?tdVf7Zxpf=RzMM=N|Bxl+|LB5kJadQQ;z^ZRwz*2lGt zhl|snLn)dV(@%$Q-#>gFx_`SH@UhX5^H|9P^t<-x187%6OoJJI{ER_p`DxXQvrJOS zmY+!j=2Y0M6k-g|emr=TTUmm(;|O5=kzOo$c*&YRTHM*XG0xdrzPJ}n!V1OOcHVK- z+F`;vP+)ma!R$-Aw1T(fQM;`jGZMn6x4L-gu(wnUetPwEpz0t%QEbdyxv{DqQlmk` zG?D11zDx&r#kXtR3wj1m%~u$1ODj;^SeZ)GQHe5teSAfkjF{>hJe2*WXF2ZkBT_ly zf|Zi_G7rYpGzMy?{Q}$STVn6o*nrS;h|x|fUonVbBTQ$Oca(4cU$9JvXE(E&ssRZi zuhFg&r`5|XsAv-nDi`1m^)YPkWJNs_3V^E14K$D0+6hYsm`tj>0!`z zRf6x_eqsDJY^H%E+gCxho-LqU4M9qe?%liV&0pN>-(S4)ndV~BL5Xq0fiy#OyS0;{*2>nQi(%}4H5!9Wq!WNy zjeRmH&m94CK(s3N-t=&UI|0wmOhsx?vhT><*4Al9d~>H|kinmNm&)m>=6+>Ywdc;p zu0tmH(COT)CHlxCPR4bJtBUcf*omp}IA>Bt34rY(pdP21R2Vm`{gE3rE#VRqVl819 zCBFHq3rEL?s5+V;>~ZnCH!)L9ZfMKQbL{bHPrW^P66r_eA65|QjCCIU=xAL0_ui;u z+cA_#e5J^)sVCPBooK5p95{BI&^f$WQcCh&Z@S2tp|NGH;PuB?-K5PZr1f!IVNYh% zKdMx5+^_4p*eN44vQC27&@L7Jt|cGP`REpz`BVN%#lf0b!7DDFAE+{!^0f{&>9jOqbg4JDq}_7)_RayK5U#Rs7^Cc@WxgEB_+c^*Nb52(u&T2P}=gr!^;D= z5wk>tHhE5hAMrX*d+v%QIqf!D_+IiegFrUKXwTa*p0iQgMRiB5(HgATqV%`TxNt|} zc|SoDs8bzjz>-fP=}S^@@Hf7PN!Hfmbjwdk(TGU$c|S=gQi4#&U8HqCXsh=C!c@Wa zTAR*0alX4jwAF4e-VF)F#D%lLj+^d?l5BU_k!XSgp|@YUUKl4?F55Tla6nr)9C_X= zHo}1vUGFg9fbqBbfZOwxNpw6##~mg`+j;GVC)sBtN@e%HP$~iW{359r%kTR)xe_q< zibOQP6Tc>%@-_CaC70FP)gjli_0SoPs{}d<^R#w)p$U@QQ*>yRYS3sK|JDccY4?H9 zB;;@fC@Xg43ZKASR?*Eb2BvGzW?myIiQK8BexWFOIv8&PPoy6Ufl?`;i5bf`N$5r+ zKHut1?Y!~YHL-h{XQlB$PYsFTbwMFMkbT^Jt1wAaM7WOZnu~tqK<&5Vk+f@aB}BJ~ zZCC@nZmXg5m#L^n&Axm(;Y!y)^!tl?ffSLKp{*LQknNOV?iU6r^X5GPYRzamRu7)< zv*LP@-Z|(l%s8F$fJY^O<}o@ncx?Q#6i3?94*1&x_xsfVlHi~i#D-2AHnrCXv21(% zlFNQ~ODmKlO>iu~HV@&mA@W3bZ*;f`UC#qO-=7mopO^Q^o`WXzwUd8-r`H-@A!pGB z)0tt*nw~xmUp8i|rdWDn&{C!othqyH11)&L(jQ+oSGA%HXtCJ8#Y{{ne6mR`dqXB# zdhxDDtVK;ol?Z}>8)h4Z573m7=)Qcmv%J`om>V%S7hZ-)!tngT@4@U_{DfQ`9=FjY zs9qx2Na&M(th)o}@xPh5X&`C$BMLLiKXp2lT8Q>OiSH|6)Y7#A54z+#7qt3mvg+#9 zBI^p78li*wTkl3&po9BeZCCkzRFFd^Y+dwe{}d#Vx9%kfV#N_s@=|ty{_UK!??>Xm z+@CtSK{>m)e$bDvjeBg0QRi_DNBR0Te>ftV|4Jp@TjwwL&F zaInlW88tTCB8QP`Bs(utdV6Evs+J^bks;87Bz0Ey>VAPqm&T2E5tC#P&z+1AL?CMJe@fiE(}&>L}IZ3ac8waSQ&}u|5pfoJkTaEvA2GZ)N@WFS|^66VJZ!}6tZ(e zeSEpP1iL=Akcqf7;ovSfOA;V)AZtj1YAEXoPXjLBIqwdFRU9c&Hy=K)gLm$~H3~?r zU6~8SuoDXfn7bOdHIq4YH)AUWrKPK0BD&<0&jz5ZyGE`^ zy+tI_yC@tu%d!>iH>QGhdqWQcUbt7AS$C22>>N2z4Xa#FS2+&Ps<;Hbxdu$nS&vq| z7B#*Vh(tl&xk&~`TjOh&BX>J5+s~AkC+IIr>@6fK!$(O4D!?1$F=xn~;9Z3468}Y!{di%w*9QV@j->Tko(c2dQ^+r-!B$INl^{eooQV+z0e{0DN53-;2 zmv#uL;0Yp~R%o?l&m?kpx>_n9p=Ue0VdybHtu{fQ~b4+mbMSl8i59B4vU-EowKAtCJgAN8tMy!{~2d8lh7 z%1tT4f1MEkomyapa=j%v-s8!kq{4N!zEyBRYb~Hr?<6*E*tZ>iai;ZP2-)%gV;Bvz z!SI?j<+%Hu!SS2#nqnRVvHS?3wEnq@GR}z*;}kj+eXPuMW$n>6V3pkQ+J-gYw#(qt z6x0V%zCh2BB*gue7U|n)`#Je;-Fi)UNMi4o60|hiAY>K_VaFD5dMJ`pdSpWu^4Vf$ zkMdvzX`LEEEkq^4NAfdeQ-{H*!)}rlA2= zhcuy$F;_=I{ff|(f=sTtL-8@bIY117BhI8yxAlDe>_D~Zuw+vXFA-LIg+4jWEAXeM zk$M_Aask2?Chb9Y8NzP*d-h0G%Xaw0uW2X)781Y*h*>`I>?$e7nNT5k9OB<-0!eA< z{_>xuyTh6`LBt6p48v?(XN+q+&t%NgEYOxb?f8|6q%wj{8c~ObM)-o>_%ncOp{kW= znFq>RmT-y_#PmePS-+P zLTDcpGZ}!`UVLC2yyjm6Etg;$+hk*pNW3U&8?OB|Mq|({$ow!gXP#cb)~M zb6bX;+%iODOYz;aN2kE1ZUf|p0W=a~7F3mVI`q!GTEs|(%fX+{u~h3RTvf5SDnq~i z$RqWkN@Oy~tqLcPxH68t%Rqf5F`U^;$6ezM5~_YMo3TuPUs!l}*rz(K)wMw!ONV*5 z(=1rVrR!k<1-$F2^sq*3IE7x`Oqy`86RUU$zc_M5#G> z;z&e81DEjQOlFhoMAfxjyAyxysmsxm2)IVzqHPBOim*-{JMQ#07$P~o?>@GYz@wh7 zTA%X;&$L(A3R^LU=O17Lxc+#Jq86;SvkGv0+g215u%Qm9S(#CM?wp(p#Z0_?2dBEY z8pN8$os5_n&VL1KeBFXKdSW35oZ2H7u-5K}Ey_!1UDbY5Rf}Umm+ne`-#m9VXyrhh zun&i-A=PLeItiZGh)|JTlWb9|D|RMR*6(eD8Tb;-!-3r^J0~c;xEbD}R%si7ZNraA z{w>zj0>>Dw)gOB?u&li3eu-Tn*ZX>EbbZm68xen94hGd!gR@SPp=xQ&vAr)Vj|};D5+S1iQ8V~I;d_eDl;cX6M674`lb!==-@DXo z#s^MURfGMlg&7Gg*8PscM44oAf}^4h%@918r89h^nhVrS=!L?kM``B0ya-m8o$$Hq z%_gI#g))~0k?$IsVViNTo}UbYX>g2f`!TP@%2{w1oTwFPlQQ+6^(N#CEeh=H`X#3Y z$jA*ERwEeKY4@%k4veh7)$bnw-!=G7M710NlIAbY$Z!0Y(lCJuVpjDOxRhe1c@t+~ zjGcQHt$6BM{zEr_?O()pT?V7a|#;A#(O!En7w|_2b z47bOR6kiLi`0HQ4XGqcDvK}4Gd9nK46xn~XrYF$u+_?ih?A~p*+L2B3zwl{!U0G~8 zU(oj)@QNBO#3BCRLHvjW5%uflavDbCOYC>7t~YkwyZM5z%+RTjf+HX*z z-%#f9L9_vN%c7{Y(m5b=cKEr1M%rn7sQtIbR&%4_on$$+L*D~<%F_{$TlmTQD&Ahe z>fGkPRI;^}*_sQDn$$f}2GbFGpG!ciaQX*jh*29GR#*qolLbbYVl8*@kdb2Lj7S3) zbWB_&o<8}p_T!;;#E;-kNU!oax$vU%3fU!;r?RN*PT=Cp869Z0H@3o|h&++8=(GLt zrgdqa#w@>(6#vZdH<4;a{0DhI%fZLSsMQ)>nXffG7)7H%nMBcM6Lo(`mw`G3Z=D(F zC1vUvzZNf?A>-jiNO1ZD0o6;H=>q5rRcW1{GS6P12_jJ=;tacAg0oxUB{5fnEM5#0 z8uPCHX%7)rh@M*>@ISo}`82qYz6HU$B>&!(Ex}VBW;G+r6@F8K0A19;V3@=mupsuQ zFLa*3ip2gcX;4J4R5BuSYb?QTGnyT#v~1)ZMTbCCwHbc!X~U<;hE(T1CSofgLy&E; z8IW`~P)BM`)$=T+$xV}aHHw~JN9w?(IS^NLkigEXr(|Ypb7@-$wE!-1o4p*MCQ5(|Z8(m1a z^@QRoU`@-?ikA;V=hVbJOzT{7`PL<)Yh}_L9rHU|L0P!B+geXb4w~l!WkbWq)$TxB zwBU8d{mDUnPkHZIdkBdvZOH_Qz0ALJ0cmP@9p|glja?&AL33$TwcmI1hO=UyLhQ^p zp@V-GGLV+uyI<*8*>UbI7lw^C4A*@eWJtUm@M_W%GGs>3B_t&&Lupk}DXXt?Bu*C; z=wnH1T}diQ4B$NKE6Fm5PD#fgjlpm&pt5OZ82F;m+xUQY{-?I%z zl`@p(w)}9E%&`sR^_t9+R-=e`EaFH{>qavNaI77pKWEk9OlM;_YVZzJ@3;u<3p6o( zbrcP{84%nNjx?a|%DW)0ya@S{1yex3XU)j}1R-inyno`*K?$KAQ+F*k7TgpT1e8Qc z=(Zs7RN;bVW%>~|*_x^H4XBh%AS8(g-S9n~BEH5#@aM!B5u}wy9CtPwFMDEx+0QcC zJagzpRG_5Hi-LSe6ZWj3M@uI{|4(23e;f1=7hdtjlVz2*zWc|}E#9nM#ON`(5KE>;eb zHEr}gAaKbDe${LJHJTf(1RBl8moycLV3INghKm_!{m>&?Ymlr+Fxz`s< z<;vyxwSE!o?simWq9ykbHx-2?y2nV}M%N$Imozw$ZD>(M#Wv?~p-U1_mVHtA$QA3o zOVt%*`jL7vP_r_KrxT0+s!D{Ch;>-fw&Gf=h=ZZ|SjL9%}FJt>FJ;Ns3hTQ}}X9Eke8 zK90V68~tJ>yJaOgxS5YP&QHBpPj)tVWfrB-`&?Q!--k86?PZJ^`D>I6l{+yS+KW&C zis288iE1{3#|iOt2pSSeBnxJOIx4*TxqgB%ilQwV?MOH$3O*B#=;-olN>G#{xYj#E zS!`;zk{Poo$S2gFJ}kp7x;W)4fNCZ3W8PWtS4549u)vASE>e9<0k_=R{ymu=HFST@xKGSAc*!1z;FS4K`8hT*~ld>`Wh{* zBHJI-giBD2`s^r-hRmd!koEou8HGF!X4dym5P%Swm19(U>(PP&|v+J zQn0iaTsfvVaoBl9FfE?>8`u9wkY65ya5Me$!Rf=ZzOfd{JsRp-@m**aB()=j73WkVH4oF%{my5Wu4z}f#;nZv!`ikV`-vSB&~d)S3DLxnM$I6iyaS$z;b z{`ugVN{xZRS@YahJl<6UF6Yvy$r!@>?kYMs4W{;gw!Thl-kx085OiY0P@wRUO(-hc ziAkKTAtqYEsGu4`u|mBy1XJDDLztQ6;GSPFxj&jMayE&W&L4b4voyYW*eb1?1=l^> zV9_!YnQixBmRCHwl|FAMfpf|$;Es=6$5uf`Os+ANvT@;`_B)$`-zQSYE)~~GM^wsl zx)2PUF$ZDi#~2ci0y3&P;>y|g=LeO{99#~dZtQ^L8ZbPUU;O-lc{<*|`;$1`xa+mE z0E{^wcf#;qKPub=4{+|#FoWPj1!9?lYNPH;FY@j$oM|;GRLK!ea+tDR;OR*;mIJ3* zf`%J~3)&Sqg9qm3X251bVa$qL`RKS;y?J)(@OO9V!;B!STUAxn+nmnM&b~cs zAU{0P^OuslqTb5X!WL+$m9{5!Y@^{gobjk?;BhBztLsh(cgbt+`jyhX& zvgC3CM3JP|axI*J49sk80)}>fVFfk_v+qTvo+4CKm|tA$2}bT4ax9OAa*Pte4z+PV zH=b|fU3`#HPRI(JPfMX0S%$Q~nz61(As@T)Is*Ss<@UQ*UC>cwW`>+AMjT}KD-gfh zzi!Ckzv@tBzq8}1Ta-T`1G*8Znyj`)bE80ZUuB5VYj(ZRIr-sJ7rslH$R}KKc4QD& zcYvN=fKur6*+$c|WQ~wac4Cb3#Ce=F3IDx>?KKRkJ%j-)-Em2EwOmPQef;FzUS0OR zpGP&~@F52sD_fCbiYq@te&r%ZOc@5>-P-_WJni^|P$y9AemWFOy?cXAe#Fi6jjsNC z651&M$HBqjI!aV|1;wCgDZ#oXL{dORJIy0CMpKuLzqj6HchA~(FtX?NF6RowSD|BL zffV>nHPk^*?%M8(qWnZLecoPRPi7z@t(C?Ry&`;`yDZP^eb2E=sQ`XGoe_lZ1;1^> z6-dNuVP`sSWTa+*bN;dFhJKV)bxRuBP%mD8AAxRt#Lt0FJ(XH1kE^=wwWA#Mk`bfl zY=t>J6Q8K8N<_#T8N)Mfr*JI`-$HR`3BfdK&p2c@oO54PIaT$1uxe5vp(+&Jpd5~3 z$&nAdv`SR83=rHNOsP zF2oVxTyL^ieT!kG=FkF6q@p_52kq9;Jt#Upu&<_ z2Ttk=;unjfW7GaqCING1gYP3q`9&)TuHam>P7wUX0Ws|~wHsUh@9`xhV(0>H`A98C@)E!^x%KmbYr`8dLoK{$J6-rst)y)R&job^;MIJB4iY;hJ&gX@{qmF# z(OA&de3~ouG5-E8kiVPh9c0Pa0AJsB3LI`#{|Di{!Zu-}SO`$IysLP=l60pXc_N&l+SSk3h!$@QQa zN>A0|yUg4*AwU}lHyQF^n}y+|v6TXGAMCdBXmmv(pfKgAAY1hxrjyys1c)w3%iNZ; zyAJx%>~Lg`cb-?`c#W=k56EB!*eMwiN*h2AqjozB5fY6L{@1pGY}PSZ8hTtK9nU4D zpzmpfaQQ5G04h6G+k=2*$amy!BlOqB)^G>6Z&c}xqtueCPBmD zVfFO#T3y>2j(ynkZ}YAAjhySXsuBFDhztYDems1n7f=I=XXVt|qQ{ZQPXd6T7*fBD zs=}=D=A@$`z|_2-Jvu$UktxatKO)3S6y9xoCH4GWBi1<&lCdk%J?$?$2O=D+U(`67UKsudJ!$_!k4eiH z*6)jH-kZUXmz|mNs?=2M>7MiJYr@Tf(o(8#l7$ono-KZ)$Q<5(UD?>#@6du=cDy)g zhs;RBcU8^Jse=)dtb(C6rqbCXQs}9ucAwPgFq;d$ldqmHReaST+N{+>0+}uP6@)an z5^iqX0|NCOonBv2w!bSM>g80Qb~j-Ad5|;V2;V`yA$D>va$eyasQmRVlu%M z+S(5Q0Dx!{Zr=_4se}Um-B;pWOG|6=U~+6spZH~hG-G;%$R~3}hWN5&d1rxFgVS#; zJso#@aT=YX3lZHiiPU)C>ogmwJOrH`te&ALCPk0j%2b^XXBt zjds7z8WlfZtBxe1DDPJd)KNIh!*aG_8z`m<`Zg~w_(O?;FVnj-F$MU3Go|l5qu(Mk za`6rXb^?fI6GZIjd0g={bU2R6cVcmlI3+u)4M|u!yRVG+`^j`#P^va*e~qJy)pXRm zpaAFC(hXATc}a7L=J3Q7Knvvl%#37miuwX@+>*up%6loQsXrJ(S$;ZXWOsr0Rhx4H`%3Xgv@W@T@TQ{dYpfOr9(1@0X(N8){efGJlK;2Q^&#(<;eY+VO6KDllCRr-bYr!Mbls`(#F*_m2-@ zun8!sqr9Ej40Jc_AV>_nEkJ{TeTK>}{)!xBV|UqK+Aua=T zNf`&*uYkvB|tV`nS;hNqn5Rg{a?j;nru9~o7w z)7^^50{Zf-l8dY;)uF-JiDP%-8aHxr@ZhpiiR;qb6i&E{DrDnepgxhbxy8Ywv$e|^ z&2*+}I(zi}2S`y@4b<-zB4I^zJDXui_^|MSizukNowdje)L~XnlAuTu=@8!NN>gM9 zpF649(IdX)*GM+_4lDBZ?#=K4kLSD|!CqSItr$L#vmpgPQe-rQ+j_dE9Tnx(D7J_P#UV8eLr`m$dNFCwXY z(Fpk+*_!`tNwwkaw?9k(lo+YIvslZYu02iY=Jw$0d=(2nQzN49J?XU8kNnFBEnjv;OynF9=z8T8^LeMRzOOW5!KrEM!-e1 zX9PEMS}WIYzUcS%JBnTLta;=%5QEaTFJP6z(1l4zkBoK=B?Ocp!oFJJB}z8rZ# zcxNKx!ElIF_auCT1aqCx34iuxTsBb%895rl^nshCK^RbKH^CY@y0a%N3SH4eVu9x0 zU_r-GGU@J#O^+tFE}o8jzP9->p=w(ua=7Mekd)+r1cUi21mCsVT+n7ZJ2KHXS-)!z zH!5ahTwUGtG-2<%42QJd$UA*eE4wMpKA1)KXFQt0T|oh2RJV=Uk#r`eCUncfz)HWJ z7^8DrIx;Lo9a}L7lox>~A~l=zv?E?otq8i`$2Q01rL|b_dT3g>(dwbs>)MLH((sRe z*nP_=4t6j6FZ;t6HAJfAaY8zSRm{@;t?b?L~$+H#eqYn{@?E zf4A=YBN+O?Uv@ZKyLayBN$b!4|KZ2y4)KTHM$aqgXU;l*{f_JX7$r_+BtCci7e%S~ zb8O)xX$E4w;2$THQ_`O(a(r_t+iPoUmDd_|29V%cethiXn#9mzn2EnT@4f+VFj_ho zYb^5Fd}(=z)Lmw%hPn8NPysv|3IG5mII+Jv#M)?noPGN@-?^Q6C@_7(n2FNczS?n-ta5&UKzSf{g-lj)@R=Vavib8fbS~@Lz2Y*nxV2F+YjL(smI1 z^iq}~`cm0S&63t)z`U^_RA3^gl0&)!hYOGO;&Judfp~|_j_wpoC#!TE(3vECJ9T)AV1#VUd!}p1XDD22C z8Y!T!DOCgc7x{?h^DUM{^i>D#e>mk+5B|t)I-SYwx?ff@cXeM|mA+(kC^yUBSX2;^ zqmvaojGm*;~%n>|o2rgKOz?o|jBI{3JhNl{KzT>-BYmEWg&dd!VPsUG4RVZt@){|2}W&#&}i81`i$|%^~%#o%N zk^R9?oD4A!vnbMDB1C zH`*NPt)AB}0#tPO)h&K_CNul2+{uFxmwe8}m?mykSNp6*M?@dS?WUx>ikS$Y7q_08 zBVV^wjSY5xD_+%bR}An(m++mrLIP zYIUUfPo&i!^Yw{vOGNPY@;GZ)@k=@m9+Zzf1e$^Kb^nL0ci@f$Y`b)0vtzS6wko!5 zo87VPq+{E*t&VNmw(V2z%)H;4b~WOJ^>uo?$gp2t0zq{=U+_$RPiU~1aOw2&{hMng+Vtkj{E-wzYRm(c zM?41RlE}G+s`MX>85<{Ig45vKTKv}3^5>D|91|~7#PX}6BUP7W#ZZ!@v4Tms`TLYv z3z{t(ejkrC`x-t0zp}jYNppl_dSO}P6cGkPgybb7`%01>r}K9#3GsK>5Orv z=~#b}B+vlCuo7ICdv^yd6+$JpSs-F<@W~!I^V6V>6B)NjtKR9NY*5gZn{Xm0%C7{u z!`m7Lj&$MEK*7VKPoc|KVPTcKS&8-8X&*rv)d5khiw_27j){;k$q&vGZ_iXKs1dFi zD+-B*k^fhQ?{+jF&qgcwI9ddwUK&Wva!3#aJl}cgf$`6K$hwilKs8zZ?27Hch194ZJ>I}D4!J0mlm4yA&ao~w1Q_JPXeEC2QPRFSy9uOjEvSRuSBJEU4=;| ztO!;J0n3K_GtiAXpb}x{?G4p>hDC*rrlzOyx3c}JY07>(p3R=sxP<8c>v;d4bN)@t zg#bkJCt1fkL!*rn3o3xLl-*_nu*DLf12GAcjflsNu(1y+pdBNUI@K*h){5L>;OmPiv{ zN?2eS3oICIWtjqiUw40h?VSCGJJy3&H5*7wgdUaxg*M?m!ECy1>dr zX+V*&Gv`#d7 z2!`%}gKiXREsw!Pb^r&74kJW(5iEotWEdHugEnZVaC#j;yMUagL*maA;JHPg+Y;e; zYylQt#R7>F5G^+Hu?a)-h|+eB?=+o?i-)l&^w%iZK@+pb_ok;lGw^W(wPF}C` z^v%sbi~jr<4g4NOnmNvpd&AW#xB*xgCB zjnvhb4Xjt8f=s{yNy?)L#BZfO`E;@XPzlQQI8e@Quz0owSc#%*z`oEA^^h>Y;FV#Q)pa_B1&N8k@Q0$qWtd<^ei`!b83(y zWZkh-5{I5__?wrFyv6B$iBK3=%{f9ELdfY_v)$iNG2NaNP7@tRlzj znkfB)q%Mk&q3~DM^6i7QlghyxP1>y@Tenvd;^Qmh3u=Sqw76@#u^X5Biw$plukk7R z>Qd7}VAR^WkQ$P|0^zlxICWicYRFrhI&aX7ef!%Iht`4at<&2Arqxbz%FT_Z^JB?t z!FV{~^4-pG&0->I^2ocr!UHXlWJksuRQvojsYm@X$?nxF6qedfY>IM4lz+>D$B(@Q zFJbaGfJVFCt@xulO4pfjobAT*L&IuP7_wb6%{iR}S=opM%KDS5lh>`)1rI?Aqgoc1 z118ay>NVSNH?aRmusU}4viQe>`(RTM%KJ`cB}NMH}ynDBh4l5?fvQ2X+z}T#5B?{u%FeheR~iOZ{O>F zY<$W4%oLP&ufgSG@RQ2)=g^SE@twde=fUG`9eJtva5^n18}-kvty=;c-b*hYjV7F9 zaxCyGpD(>SvIT!zE4EfP=N~7+YfpCEV#DMOU=N!^{izkw;^owEzm!?G&M8GZm;LkL z;EP3zge(_!>;7hi#+TXUxl_iJ)iC%?ewpbV2SPgLK7xmMaL=ILk;;bl>B-Y14`cR8 z0{d+k+WuINlnqOE&$Ocg2;*`eu_o@)U*cmB*FZ)v)9|3 ztxZ228rbj zEcw9xn{DF}%?d(@LVa!11X@LTzIp-^tf@Sss^QzGPCUqRF){mNVe5PE#(9%eML0BD)MwvKQ|?C!U|^VbDOTgsapaa4FaQHJriYFuGu&~Mj0gaBI~ z>tW6oKK}$}e{^e*=2}p8P1^Rp*^|l|0Z&h89YTgoK2+)^T(c}NvnnvNKwKs&PR5dh zzOAhQre72j<%}}ne5qI)NRC3dGTxVrSH4h+zi1F}A!+6OFXaUM>A+kEPgS5*5|o(X_=#*|GZi#EUMPqn;(7ViU74)hSm2HlA>MS1B*wsQM5i z2bpDJz`4@|?0^L}Bwm2~p7OD{SQT}2;$dv{Vf5Ni zvq5$c$8at@+dB%oy+JLUxv(F&uoHLwWvBl|qSfxo1ie2k$)VRYPoMWaKg86e|36SW z1Gm7RZ^eS)`g*!0qrHEWPd`Ml@HvI6j07I!=_%HC&U-0s4_vV4=yeo^Fk>~fnj8|YO zMRDYp312bWxZrJQQ@Jb#pCqkew}2MQwam2CgikRgfLxLrmT3Hjx{}2vc_HPpE|~uxAHc8eL%Ur z=MXDGMAF@YM4U)-+B*SVE3+j&O@V~s^Fdu!Dq2<;XE8$aM8W6UhH6YpG||A%(*mab zphr{JZe(?Qt&p-IR-VoSFQ3?x$7TKD7(9-K_KYD3frPQcr@m7cGLj``fabQ1Bcp|c zTrHB%SKh={Yb(G@sSY`UIYi3Er5A+IVl&Lk8Noaqbs_Jfj6)l#pn2P0wODAYMBXat z1QE&rZiI=33w%ifjqkcNC|G4Zn;7Y#d4Dg{UI9Wx?WXn3%dONIt5|E^UTzGA@Yr$q zId|)PKy`MI@q{(BAr+?&XjMt9yRaXS$z=5?=?zrKGp(kEL#Bz&@j-pQ)E1#Tz}&ch zYFJDNru_25!h1H%W+!N<)C?>qmd_!IQMC?DjSx*07Qj~R*P^=jr+>bwgK;}E z*nJ5OwdciH`Q?xvlw%)^GV61!!OBEZYa{-@pi3&p)NiWJo76V9bc2f?9Q?En>PS&?icY8 zjk&0H^Juu0F2a^AbuWMEnms%{8Yc5av3$a4b?(xN`T`>H;)e_RuT0#Uno3Y*`8e=- zc~eWK4SpW*4mTTRG65?uqPc>kOBK5_h<=m0APHJ6BR$dWC$(OQOw9m=uIdfM8>18k ze8ZGqPDiw{nGi6BkHbJyhR=xvC;DCIWlnee&xG>hs)WK>9q&T4MG@3+r(CKq^+dQ_ zh){`$hXZqRWfz2PjdvmuW$wP@8E<<^Y$pxXx`d*14XRgy0G_5l)Qcy=>iyF*^zPYh z?)qs@Y20Az!JPE%KuvuKnXT#nw}=7t7G(cR5))ee1z(+zC|j8O)&v&rA4nNs07WN^ zV`)+%;Svh5Akn6#uO9=nO1Xj)WEtyzaaOFxqZ$|yiZj3!`QH77`q|q#N{R;tr3nI^LUJt9R#`3Zc z6k0_aVuI-r8)Dmeg_vBrEqTuIv0WzI9Z2zkIhY9mE>(Z?BYT6W_&__Uf)_i%z!M!+ zvfdWJYt{`bOs10hv%fQi66urojx5|N72~eT2n8{oS%vywnRml7a{^()y;~R>eW0{Q zdfnsFq-PP%@HIkUh&+^aNH!+7@gKf^VtegNxF-`SFRsuB3UC@tu0M?z2mQkf&Q)JP zo*~niv^&l1p};Whr_n+%T{8w5u3md5M+tZxu;+w{138&7MX4{jzwU7D3?$U&zt|*< z(+oe0m6i9=X(E%mZ0jA>_r zfw=?HN{Pg+ZX3FH)H@VA$0L*Mo9nadK~KaG;`@+177Kg1H_FyPpTEqV{ywzJ$HXku zfAbI)mk|$N$_sFhO(O>uT7it-&QSUs(Tdb{Hi@I#UiKeHVY;u4fAAbf@`Ey#zr6B@ z`7G!JJIDQJ&)g=YN6oc+3iPVNo%hVC`wAI;WADsMdocTWIh)yY45e>)gt23?`+t?~OMVRYcA=@K8S5{FAWumX1&cI$7At5Xd?LFv7 z&Mpxl?E)DB%9CA&6nOajhMi?Nx1vK3@Wh1t2EcO^Vfo&_jxpI2gg3ivXMh+*VST`T zQh{gPq+X0T+%XyFT2KxF;>wCMLR!5Att@2_Fj#TXG7;1a`N5%j<>JKYZ+aq?i}EXS z?cW6egAG8R0 z76A6S`vo7I5ljSJfR(D3n(9=3LEslruS%8HH^8R$B5COkqMTFm8h3781gk8yNhqUU+x2l_J~Tc6k8|*zMqvn zF)dCXZ0;}Z0eu1Gto3zJhkrJxuK}QAyjMgY^Iis`{`APQDP6ZY#FcYTMd!5InKg3up^ix=ob$Z zjHqe1oZWLQkiZFyEXDSk*NYN`bzeEM=`-<$i#PRfg8KKeq6nELG-RBA(fcC}ZOaZM zT7V(E)RmfLo5AOfb#LOs9^mf({ z+x{e;K7LczpLSdUT`sWWlE}((VG78E-dx{5_@j-nZ&a|#v&v!5q-mXS+}x}G?*YN&Hyvm2yKK^;YB-QL8maffWg`X zQc{XXDHC;}z<#ES=Wc|II7!>XgG*bMkkjFyCQ+A_56t;|smj=Lg#Y_>y7ja;LxYOl z@kplGs_^!^-2$gR^F#aW`jnvT45kv5_2VExULeh@@C}!8%@ubzieqbpnpf!(wUzuQ z%H^|wsJJ-GKswVtzr)m5a@}`B@=rjk>og&&S1XjztWi%URGkft!Y!M5BdlnI8>P|W zvw%9HfF>TzhNKcU`&bQ{$ETi@Wv~!5}QlFE8E7jrotr4|X=t%8P+CUcu~s53eSR#;uT=tt|(kSZxV1Xp|p( z!;k>g$c<7zAM3l0M)|a8^3+5)k;nA*nhU};8g_i9?sQ*)u9-Vw!!6z zrBDttnLeNn5dg-=qw?;%;)zVjS?*~0UM9JT=BXq<<(}v$CLshv?k_UMt|&@XJ3ow=!vOxwTibUd3^a_%z6!kp@Qj!5fs zig<^FuV_FjjQIu$fPi|bs5l$&VrOglfr%U0vKdfDAttNKdt1k(qPDz)0DHusc}{t5 zb67nR1_2F6d4Ak+S8#DzZdfnRS|VOt-`14a_XO46;Q^Ruh-;leNoGSwxj2>0JRMJ2 z6i>rTk}G$u1;b6NC>d`yZpZuh>6mQe;b}}Ik7ctv6pM=RggEbLZN15Kw|+)xZLxEA zVu%12qx8~TQk)1815zzTR6G*IgCP8EqmLC8buiMYT2=KITOzlQm>#o;aikr+~n0t%Jjh-(V}yG0M)0URTl< zAku9RMT5>rdUFu=5!LAI;vhUCu~*RFw14hjxfta6VlTX45v;nGxvru(a`l(FIuviT ztNPC+qVz4|ZXhDfi`#Bz5c!b9_OPvk;fBH)Ui$Cz6LPM;GDk0?5&RKOaxM>d$?cRH zy@)$K`}t9}U&5pdHSK5A4)8G*{nwLsofF981`@u+9UauS4-G72tZ|}6Arkj6Cb7D2 zNS#!K7OEizg}ally*Mu(o()wEnGDp?cE&Fj@t1;dkLjIn?u~Y2SmKC^CmM50BjL?P za*t7lww4(7gkO=)BJ<1J!%PKP3^Xx}eVE0|LVn#fE?;ytqx1T_owbA<(#PB~A=6g_AvtkCVhx3F7-^yP%XVd&$#% z8XuhRWgYplY`|q6qkoOk`TeMobKp8V88!)3hZ(&>>TJ0`}?`zR&{*3$xYcF+4+fnT(^%VI@gQkFeQ$y<1Ug{TJbgw>#Ys+T0b^-0w+<}Gd>vVP#NW%Yn zTC+U!i`NkHSKa%Hrw}P3JN;&r1C$=b5c@0Cr;_F*Tp_$ZhmqLF?oPvee(S1`Vcu3oE@eNXGcomxYi^2!ha_sL7nps6P3_L? zEkW#k;GqEbyQYl$p2xWK&Jk#K*BO4oB~>DgP7PB^5V9IpN}L9@ZXJa3hSfVict805nQddoM50ey!Fa9#*-P*rh(ZAt$5&oEs|F0= z-|#)Dc3EN~={b5fpaA=5$wm`T#03nYv)^hTCKHsAB*$9UXtKqd0ZgmZO&s6NDB;aL ziTnmK8oAZnES6LUw8KsVwWWMnp87ALu2!o0bhD+b6=OFKqgC9%}VDZtSFh3phVCb$p0RMJ7+d80O>6klDz?9@vCRfTPS(Iyxu zJHt5DFNLdkX?nB!A;dG<6v%_+d)M0TrT}y1X%(y}S_h(l^(MCZ@|yqXVe{aH#58@g$;xgdb^I*_ftEMPuC9UQSzlK0S10tL;|k#8Q_Hx{=JT>3fVOQQy=cWQ<}PmeZ3MDzZ7n60B`;ab{rnLJ7um;pXl> zv&+=8FTS+XTIR3cRC}>m0Qd(6$j4`F8x>WB$-s1QmJEUux{PF7gcL z%bzk&QD%B3ze3tASJ!m}jm^2u-^k+$aeI*`gCRWKSqA8WrfHB_xX9C){Abs&jt_Tx zIh;AIe$(uq3O2wzJ)I! zjHCvYltAVts~yj6tcxm2^On30?Oz{Hda6PKXj5Y!V-04O zIzcL1mczT>ugD#R6y0VVFeSP+-AK(MZM^Namnyzw5UyD1!KruC@(X3^6CXpApdMl6 z(vNl%Zcp|ONqwR7B%er7w07KTJaM&qc(##R!36f_qvzt6(j9kiqWsI3 zhYrVJM;JL<{_sgPJ=(RYqP#$yeY`Rdf({^Mqdk4F1E5EkEG#byRwWojs0&t_Ta3iA z$;3>TF*nBoh8cNjxW}SD3(&*i1+`2{Pj@0d4lcN~p#Hu$_SPivwgEpw*;LZcCuSL1 z7j%_JzGP2$hg)gh_I7+C?X&`OP7(ZuCpl_j10B zUoCn=D`XPGf)PZOe<)Qz_~$4BMEb+uY3(hK6Y2wHkY)G68|`&GKfwsSZb+POvg{g1 z*~r^+z~D_sc*dQnLFA5gBi%ui#_VGhjI_GmW=l;nIj%$;|!?0<1ow!zU z`_xZcB>xz&19CZKkIOH638$hH;$iaWF5g8!UKY=+C#i#SK9{nR(%0!P3i<5H+4#A# zXG?+~aa0B>U#M8A0d@X9kq5s~*f!2sD`*r9j1jV-NhB5O16{uXAq%@`sUF+vYB~iV znN>BtTQPw3yNVrMcHDONL@563mb?7u-fud}1}i<91*{`qRxfw8h2 zdq!^AM3%J(9WYK*)o4wz`CD%ZV((WER*pl5H734@fC`b(1FbJoHm&`e{9r7qf*x3^ zu2}uGgs1yc@6At2znOkH1P3thXQq(+E`ORy3+euAoY6CnXSQK1N_XW3?|9sS!ZhOW z7}cRgMm5QRT#e|8N3oIp*t zWI7J6Gn#flV`?i{&k5;$dvQ+FZS_f;hcLRj6@qa#G@z0_vQ}`&?~y?^B6ffXdmKgk zOb5EsnoR7VudT1OJ}_)+b~@B|KKcTKdLz@GPju4YSnw7x%nP%sZ!?&S<#A;_ota#> z+$pd=uEhP`EP3m6-!FlUOb>$`s3ZtV$ahH&UA>MX5`O^Lhy0bfv~~yA??T&=UE8*z zwKZ<%-tQ;v?~%FduA-)NZjT?v`TL7MDQ+qnyRzT><>)2x^R9@(GS-(AfSExeO1xl;pXBhfqE(1Iah=%$ z9UL*pX+`#N%zQk}DTht&JlP(&Wxa-P=}u9b37zQ@-vU?TwS>7^>aLMh&cm>{H+%Uv zy+NNa#aq-HgZ|0XK6vbjyCjNmj{|8y$T8D@%f}~&%YVtAFpB@;^lBfy8(S<`gsKsx z^vPk+rv2^BjokPzef}~>(~Yb#y|m^1i|_=%OswSeX`8PrL^sNG1Dicv!!{%U$JJIo z8&Dq|?XO3O#20?F30MYzK) z#6GNOfqSg&_fr>1Tsilj31ML{`xfGK8Rs$cTT8ttIgDl$h(_|toW3-ZeY4lS0gbm1 z?;731HDxq~^%Y6!PwZkI!9NIhb_&w_TOSea>;Cy>%SoL*e3^3F+JGvT7N6{??a2PC$s)lta;V_) zgwyl#0x#9@Z6Z6!*yuM;CKAx_a`#zS_dzWK7X*Mh| zBr{D_q%B@3yr&C;TX1>QgTwyltd6p>@9DZ6iX*8h8vm6$jxYeCsoXiPHu_d32;goD zDC5!d-j}`VezvLJek^)Ioob?RC@lE<;~9xQ6nEfx(LE^LMa(VOsA2u)uIbic)}?$+ zixrNgmm=22R#@<)vWMIZlDu?ZDf-qwACdeG;;y4~_H0FKQBx?W1DW^J`)n@rTBf9< zB?dEwPmgw0urLF8StPP&eO`KZzPgzCk|GgF1T;4ww^+O+c3y2*LmT3>i^QY*=Kg*+ zJAf_HNyGt|S7iLX{)v#Rdc1qnxV!$_?vDi6x@QH@#B2BA6ynL9l&K7iFJ<9iW?{LE zDajEm9ys;&T$z%2=EkWuL6UoLvNk)r8~byu^8DjnoaCA$EXchK88SwH~)qr-jh-`VHC zMdije&kw%&Q(I=A;hFdVndB5Ifl}PQ_%rA4!@tplogWpHp9bGjD-Y5tQu&O7n!m1S zKoj_v-ClM>7I|NM1Fe6cQcX4MvsfiKN^eAgu}%u>8^YK%yc#2P^aKd7aWc+CpmGTA zX0(qc!}i-8l7~&p-ciM)BQvdVMg}k}%}2ak8Zh*I2x{A1#7ZPC78m(s0e& zEV3Qn@iGCE675D+AEcn#ul_T(8`nt8Smz^qh%5b_KV(By;ip^+cV%?@n$84oj=-(i zxkmA3SzI!@O`nbj4PdXaB--#;`>O5mIlyU6G^TNn)RP;-h3SGPIG#DO%yYSY^}j4Y zi=mANDmf-IIplu&bx;4_wj(CMOVU+`{cHy=+3#>M=RfDe_bCS489L+ z*ZTOA1LN0fDcS7@(bj%%q@@6uz+=P64yb6`;f)*Gh?XNX6x`nM4??~6L$6@FwN6(r zilNGo9{!Dh5;}}zX(or(_G>3rOvkIrvI7pB3BKd}Z+nyTX=m&9=KcSagBo>zG~}g! zFJ~F;Ahf#r6k_7$SV>KYXNhn(O@uor&)UY)H0<)og#W(vwtT<6n+eAcXZK`vuvo$y zjG)UL3);Kl_)~1j=!sK1l;mfpuWV@dM>C^z4)KRFgJdpo(4duFbbk_fcTPk~n~MlQ zRGhSkRM0!|QUw$~2y9H10)K zSjRzy_SR#7&97zE)yi_Jo%$XbD9E6)0*0C0X-9Dmsz^735riLJm3-~y_v{Ia8j}S$ zETPU2G)|f-PO*b*;v@Pu=rMc`wO?J-wN>raiqFcVe5Nq1v0Lo^35I!s-&E{6Tk4wg zxk#dEC5Z!o=mk}v8+dg!$qZ^w`<$0r3p(1>ZvV%P6VHX7#6>Y6^@j}qFQ~!#vXTH0 zWIVLaf|ugGQla8|Bk1$GaUBB7isQ8KtPge@2y-rF=WfDL?V6oCsc*qE3z}<7WxvB& z>mR$j6PB<*8Yh{10Iv zQWa?&OPYI4wS<`Y`3y^AF;vW*+|o|}j+n|lR&lW9;%H^;JJCbbr{xsKcCxtJ-a6d< z86NNc^!`P$OfZe8 zwNyyrpzM2$71fqK!!{JGEB1G|7#yh+$MFpO$Jy`Khh_bifNciZM}CPwu5$>+_s0&F zHNxuSAg`R(`sQWAW#b%t?1-3doG{iHH69qV?LgHb?CZk#y~~)h%U7 zQE~V{FQL(^(*1y1nGGlek4fZw#8pJ5gt{w(Sgh`2MC1GH`ci*okH}@dXNL&`{QfZ0 z0A}Sg1?TW`6X7yFI|GO75oFQq=2jW9baIDtZ7Xwx1v38>qRf6rvDXKJ{9gds>axzL z=GQ1T_%H2rsnuoWe!CpTp;k^<1ULixPsONC{3|RmtDG1yFKB2kw^ob5Csg`S{GuBs zPQJgESUS^aG2fcUqEq1Z#ACPP?!S;}M;>Wz6spmHtuXe?Q?%UWBUwhXhjY<#X`d89fd)Ip*7!W?7)ooAw_m3yo<_9mOeejR_)XdC} z??$En_%s6@nmH@fYs`)&zE99!)g$iu;2A8`Gg`mNf86CDQiX1}IhM3v`BVwNeMw14 zt-QCSqUV@~6P&Bx%SqYb14WFeaG?bpT=!qSaIQDef%@ReQ+63?0XV7e)5b* zy$pf|2?It*=^^$v8h|`)13PuCcDN66vd4jY^o@ei17KEECS{1+R#q13-35h3;@BZO{j zW3LWPz9ncK%1J!*SX0sD>Y3_{H%NaOG3-!WCrRNlYo{M_Y%rZw$YAGP9 zY<*K@YS-6gi6=X}t3;pWpMHYePj@CS=(@HUWCo`?u6ClFM?^ zjl>uN@Tq)k`cyS-AMrKb09^6C|Bw}ojowYNTsqD7wlYcO$}QZlsBhLgmhf03_qiE} zh4lUOwXuPdhCC1v{j02~DyaoZr-RD`;mfbjN5?N=DqayV{oDW6 z>Vj^VbtiMjX-VZEVS96g?stNPg^jYs%*@tU16v0+2Kr=E-BLf9E&lf4Y%#740=`@% zOQOrO4`ve4(31JimbruX5d?L+wtJCX2iPXw-tZr+>`3oPj0@Z9!fso?86O5H&OVJL zH4#gBto_Y?93sq;x^U*^Mf8UD;#fXozxzy_NQ4Mgj*ZO_b>@YugQG}EIG#eXtNA(b z30RrkiCSQ^ZN~#Q9|PY=bF%}zE0UpvKx;Pea{kDZ-J(dKU@l&0IlOZ>ro%_MIy{ML zjE#X=actr0OT=JmYZ&WF8I~uv#EQQkUz*4I;B?cn9|HSy*L%7f*iab9zv0f^ChoYB z=okdc6!v`@-;6K6NimI@CJp)r%!btGf;JI3R9b4I)P;e8@BKiS(CdWc>F8K-eQ9i{ z%YiTL;o3{kY(`1&Vhi<=Fa%Bp&Q=!6Nd>D{ z7B{-LC%oX)>|pr8IOd^!OK@ZGuZFGr$8=jENzCTtArCf7TJ(~fm(b^H`5PsP-qu1> zqeC)T8ea&7{E_Y4bwYL1+!$^r$K`dEv2?$^a}eCd(+R#p=+@5ac*o&;*v-T4h94`E z>+;;zdVkQ3TUXVnsgD9KDEJk=MTMEHC#Y2-CYJ)4b}E0ItcjTEusCJG$Bi zm_E6%OA+1NeLaM(KuCCWkf>Z-?{xZYGd3#!>NxbM``-faKlhMso&VfJQn)#)dEb3S z{8^@XlyeHJD=R-Vb-k6o=$SHsX^!SkR_9cY-}A9bw?JQKZR4shTh#pS1uE|Mi-s~< zI-}?F?hrr9VD`VZrH6iZu%Ooh2>w!FGxc48nHac`IGp=jkj%C`UMxp@mc+E+`+Lt- z&xh4P)-}Qt8lfYNq?94qF|S{oa2^A*7-@Xz(^zyyyKL&Qk3HBGXmWl_7sE`hozQc5 z6!y_wd6+5@_;1GY>rlbIU`wn^cPqapE^P%L>`M8{c^!)fXp%1=`Zw1~V##Iv-Q6vT z&2dCxnglXHS~rO_YB*B*RxBBdHwtrD_<6R^4B#b$8|SOO8o>26khnU)p{5~`ZE5o7 zu6iWW@`kN5Z)-7zUq=rOu(4CTa*+U0`5T{n*f!Y5op7JtsbEu5u9iVRf2^N|4oTz` z@Lc^Hror zRQ>tCZVDyJ1_L8;K|w+FW{>3wNR;;cNvexw8M3+Y2%3!5)>djDDAb!mdf6M#J%K#$ z6Z0D#wm8Vh*=49+KTe+}Zmk=!JO6PSb@`~}A6@Z$fS(cy(n{c0pt@;Rv3)3HN2oK9 zbN8pA#SMKO!EI5mj+=C2%QcCf3pKV1nEXrXNbsgh`k;5_Ap@@xeJemz<;iTx<|HZ8 zNn_S})_>aV&(x&_-dWlV9mAeo>?#Z5kfomHR~H9?mrDfrh$S1A(@zg#bAtJRAFaeZ z%2<8p+FF`AS)$JJcnM(psTf&9u8W-br8UH({sMua=8GQ#bNEdjq@eJ@mCFi}|{lXITBKdB4vbPF?hmJqx=KCO zR_DjQqm?jHZUoqZ-PnwN`qxEU*vMp*BKu@@+~8t%-(l+ew&;*O6i&5@_%e z@J_%=m}vX5k+|p=Lk-lm?O}&-Tei}h;&&48!aWVdJ^Y!}vmzwiNckh`+NEBf-&6Sb#{Z~JFwtu!t&EWG8YwMs2Zlh4iInjMD# zdva|`Y3;3D68C0*VP+t~X5tg^khW@Mc3ajQu$P9NRC>JiDXxaPdo-RiCqHY9W%um9 zN)pct*2ZI#I+Q==bx}!o_&N5egU^>(cLx&Rug|@iJ+duPkJ@vHDjWubcw~zq{`&8M zxZ57dR17G7cYFx^;C0KmiiC;T?P%AbkH!t(+zd={Xer;P-kR^l7$wu}v_V|na)efW zuXmdI@{t%$ZOxWZ&W8j?mr`G3r@RP5fM0T(M-vvv@Kg3b*mLj#232Wv_%fV(Qm3M& z$?17!alsCIitQX-6))fQHvi${^zp0fLd7kk9DdO|1a;8*^P+s<#$&G@)nzp@d$0C} zysouIb!Khms2ny_QvI8AO6J=w{!^F{TBggA&cN6il|wX&WUODlCg$u5iI0JQ!aXi< zML}~UZ%Aq?j6fuXDfenX`JXXswaSyPuRNkH!FE4yUBc}ZBdk5towYZi4hVM+c5{Av`d=(7n8yaIT+{M&R!YpZ%JbnM=lDA>hXpTW$S+Av3*>YPLbC01P2gl+hbpLegL(cbxV*fx;;wM;a z{zT-ovZg&;6m%@DuvPOjL#{=;f+631pz;EG50LMd#1-Y|O%#ZDd3bo73ByouJ(F)K zJuxx%=<7C8{B6am_=Ahh<^^Ls96dW_`~TnbtNo|vUxe%E+Aiwx%@dH7MO0K&q}8bP zTL!u{n=k%(zS@k$ZS^1TZ|zd~-PGkFn$NR8etz_$CdO zumm$Jx-sDw-U|5QqG{x7Ra4o&#Z>678A9g0p^Epr1^fQjn?{$~cjJ#E>2XN4E`2h@ z!O34!R1Fn4#s{XUMEVc}h}g$nRV`~*zDEy%NeJxBy<5ue#22X*$ z?}pO{rkj)W`y^+(WC=0Rwze{|Z4` zb+5In>(icvE>?&-){})FQQ;rLz$cDkUQbPDG{B;7Q#i=wp>3g6 zOoi1G#5mL6u?#cs#9`zHyv*sHIE*Je(8Y9u+)r$YHyy#jmzu#znIc+*jLRUuGt+Rz zeRVOr)bd;9&oQPv_Xk9VYXnrj&pE0Wa#8Pgdh zuaCA!*Ga3z*cTAAH&?-LHYf>2QC^LUN-b=}#?f8K>jG^5Up)r3F!g*9uNkn1FW&j; z_PGD2+xG|C>q+G_#x*pQQ>)e~+op{8Hce_;xol4GHUkS8Se;wu&y^#xzhz}_xdFPw z-pqhM&u?y4MI7B_AII31%v<{%|J6c9{ttm+@x=A-4dH4!wc6f(kU-W;%K=2ZhC2@l zI%NQ{pu8h0#9^7Kk0Z0e>jxX(Wl({fj$lPOd@GMVgmP9g{E3?-II>xdX{kJRi-{a3 zEJ-p3;ckCB*+w>d@U0+|ifhZQ%_!d*7kApBgo%?yq>rn(JhCg{tG|UONOuHoQLQNE z@HXA1xUnymuvex|ETaqoOGO1-sMGUd<_@cO4&eAM-O|0wi^3eL#7=Zh0+Jm%o#wUn z31$B>?&FssIeSLRkT9Ht#I2Lv?-hm5N`V+zHqn3T^(pRgX+VEZofydtN18E*)}J2u z#=cJ6lK7hyqTHn%9d0%&_}<&8_QsO!HwevL-Ko#ZUelG7707ZrA)A|>T;deOhhg&~ zI)JP!-IrJQ{i~zuwea&_J=5yP8|&*C73@AzLB6@|?Mj)uUh1aK28)f&jLZDNS(`}% zTmh+Pq2{)O_G;`H_SSdUQIzP*Ng#RhmJO5_zYj(u=NA^<4^lR7b8z2~Ry7RX&;p<$MVr-iKzaoWh$1`YxU@OxqvKFkOB z)~6>1e@vLS*$?bp+_>zwkg~^DWsLQ)N+8K8W0OuM`2RisXp2Zz%+Zw)6@_%^Se&KE zvYNa6DeTt>lJR%QOlzGx3e{iBAkNnyFqe=* z8`D02Eo4>PGYD+w`ePS2^%%csk`y_rt|RnvQg@WxFlhStj*f*k5}j`1iDOS-S9Pp9 zYYD_R-TO|s^KbEYaJf|tN%A}+357TudI%yYxQ(E0Sp<32gYea-UllnR2GRvX&Cjit zeg$bz#2br?kxsOcB;RRdY!{6dhXR2A4DxApgMRpKby8U%V*f-iu5zHayCOWl=AY9W zHjs6HPy!$X`|v*kNp1qKMV3M@a=bA~DO3%YF%Gwnt{Qrfqp}F~TjpX#f{?W^nrhI*rDEW; zXseryx34X@B8Y{ow1{O?bH;Ays(8bkr!1ipvSDyb!A0G#Go*RW(Ef`QTay6IB^8u#Xyklv&jAOeM4rrzU zzH9yC&Tett5(%>JXX9G1V7(9(xo_03VLBie$_@^Dcde&#%grn7>M8Mms56)YKL2XnZ_7^gJiZ96 z^Blkb_}@B!@%`@+*aOTK+nqi=NYpvXc=V|1MW5s-NWl9$#=wMe*Q4`<@eA}#|5ym} zrlh1~L|45&?Y794Fcnh*w`?vx>vv9fQq3r>J`Ov+g)m4f(B|ywEA-5Gdw;ioUcv&W zNHnD<+mOwb0c}b>mGZ|5cs+)HEqX+GC;~dxTQEuW7MDlKv}|DSZjZmG&d!@s8jEWp zGbTNpr9WYgB~2|ivq5SmSIjBjCD>TV6@p?eKhW1Ntj;fv6;o+9V?VWxTp4O1g z(pi^r+{B zwZne+0EqMo1=M_(9V!pwIM2{G^=XR?7Sj94Ki-^Qul)H!PrM(&WqhC_nP1(|kzWNv zfP|3JH8J)@&}l|yqveU)aHropub404BU@Wjs!7Vltd*C0iX!dr&sqy4DTDlzqM}kd zpLH^ZM7lC;uocRx&WUj^_P|0Uon2T!Q{S6+0@q&EY^`7!ou{&oW_x~4(%u&?dl11! zPX5b;wp|_5re+6MI_q5zQ?C}zcEoIT1u)1iNpR6Md1?frgPe0EMas(mlZ=;+Puufl zjR>sgG8D(1QHB8O8cOllzND`=t~;d+1rtNgCjglV!@jtzH%%vna&Pq{zP_Fd{0p34 zsUcqK2BZ!SS5&GXL~wZN^l#3nbsaM9OhpF-X#Q2J;i&CQWO z>)C&OQh1<)`+?I(;)!S03;FO>dv#j!^}8XNDgqTN~6g?>MXvOVS8ys*1&k8&3+4?Zj3`J0DRrr@EC z){U~x+)9%S{4Lv^*xHfJtqk$aow|=4+&dY2zLx2Ney4VDAnn}AsAQq>RrSf-eC8!m z$W{ojNyzKY8x*9k;;$JK0x%^Xg3_I@2n}Sc(mg9}Y|U^K4p*Goshn(-CW%d4*$jpA z4R48(&jt3cT$jgy*4@sMuN~JmR(qJ$ldp4y?r=w(a!@d~TDLR)Cxhy_b)^Ip1F=e+ zTz$2FON)APxEQsi%$;fJ{54BYLN}4g;g-h6>Q*(S3i<_Ol{V&TUX&#R^Pl&=z7nq` zL7%~jB@u`vG&OMy7~h@`{nnlqC;#^u`2Sw=2l{0|v28yzXfEFMfr?i|NC;yf?O)#K zvc&WAglJXIgYc&BgYiFB@~0;}%GW_+Mako%>*VC5E9mVix57|cZ8q~uK)4Z-;j_C@ zY!8NStpcITSw0CnuE{ga`4+$ZF;dS;3^aHf?=-hfPt>; z?(u_}^;kQY z!~4UDca#|5PoL0^wP>g|j~TZKhQ2Ww&44OV;#(&$rKin(8ClowNo|RP{a*N+x9jwz zbkC)+TkG~>>%d3LKx+VeVo}N?d~trKT0raTsu!^CrZ=fcK;(X9WPNjX{9a)63a||g zhd57&KvWCE^Ws{^tyfZzuyNp0B+ygT2p~o-Q6w8#)ixFSr8dJxv5|?9K5&CJy5R7e zzKZ*&hzp^ZokCd{c@Bi<4d$h}$)eMBLawNBq)IW%Y6f;gVQnZ3vcXpU@wdJHR>!~f@xA;ZF#m3P%o4ew_y{lwc;IX=^=zQ60h!&$o-^Gw=M0VNW2=B zKr+5nR*&t@ac4is_-W^EKbSkCp==3Y}heVG0q#Bn*&;sI)AQ|mrXz1q#e@nMwQ z$Nc<@+aG27JRj6%rp~}G0|qhOe`p3Y@SkY;)QP_Wr=v&*gt5PSuc-J2Am}HZ#(hjo zOgb(cZd^`c-!8`|%sp-SKQ^j(Hh;-mXdvHR_o-&(lJkhF4L7|*_0>!oi)!)@`wpuT zlL>Tt{f`!a2^smhE%m{F+Q>eVcxaqVa9-d{{ovB-FsW$y9&cyn4j{49d> zfz)wwpd7TRh(Ckv6P<_8EOH1v%0Crmhy=O2>wLth#ik&fgvlW;E9WX(*jqJK!U#^u zuiTL9n~2ls#q%U!CIOvrSbOpUdw_F;Uis0`)rzB& zkDs@l2lQgFY-Cvvl5c<{90H?>B$J=Zy}Y~(AKGXvQ=~?ZuNnKOdB!~@5!g%?Ui)}6 z*1w~;1xo+bD9#Ev4ayLla(s*3HvUIge9uDyieotx?pww&INj5ldw3K0RLW2QC#Ae$ z;m~(xPs+USA5wQn#;HEkU*MGSAO_pjyakCKMkAKKh{$~vGe4GvRD*ma4Uo$LCaM{V z`0>9s7fW7pNIdwIjYxZxT5?CX6|QBu!@5SXdhGNcKe`@N2hD%^hP>IGDvekHZp?1o z{21f52!FCa-*2`CEZHzo@#%1+*eDz;^LiRh5tV#qJgqO`P%>0{7OtWzkJ!Ce`R3P)6)=8Q0M3!M@L8h zrWL5C8mC0O44?!YN9EhnudkEf z;UZhA--lxnN-~A}0Pec;MR2MfOZ3U80a~j;X5jZ@jkKcHu+xQ4brW;WS?0#hT&7GH zfP@+=SphQ0v{p?$V}pl@HYeqp4Q|zlQ-ivLO~N2ZD*z#yb9k?wA&~n+8iu$;&-M7d zys#epN4Sw(zijv3Qss)aoj7gTOz5|LB4dj*JWGrRbqTsAaM6Ho>I*A?h@}%8tKluW zO}Gc|3K$K&U`F2DEno9M|o&5BIMQoh<8d2+307&P|bAnZl$nt7G2cx z)Enq8#pccE``-hy-Qcr}*GqLnT4d-0R)4}e|E~w<4+nX;hoN=2(_>fO#W=#?V$9yw zV)a-M5fL$(H&N6;uPUosK#x$b-aZ^zHS;UT0|CBw&RLP{f~^6NV(|tZn5_>CNKa5hfzw z7Zcjyo11m%%%-ul+09#0kwR+u@fdw^;n`=2U-<36nawFrcG_1rc`U5Qv)$n$e<-E5 znGtp&E)MP?=Xc9H^YxCx&z5GC%BsTnp2Q$ z2j3jNI)iZ?hu%N$aoHGUu!iuk&Xx{L=b3?^-RtE(Rwxgs+Q@uqwtZfNy*vc$#3Z3g@UN5JJ2I*=gR>Igk+!@&{WOS$~eFq6~- zN`};whjA~lCGxl8_?k6Xrcq}Rd=R!d@dX5-JS^&uFK-c!p(yC)Hdgh-%yiJDMHIqT zP9n?}7B|KMr*{j75E1U-_PCOU)nuJqCwjyIpVCiA$NxF#|@J;3Q z|CM`*=)ZRgC)_`JKWSRslD5=ChPM=CUGQCoe$hDH>|Q1J>f*8m8$C1ATAweSwHBRa z@Bs{W>4mQwS9ob&We!E#$myJz4Yj>F zLg8_gmIJbc$zVr^zP{9Ns_%G5J#DB2`@2y?@~$s0PA*fL-oLvsco~7?UO{VKb20Q@ zmdcD>`?iFc8RVZaSy%thFW;q7$`>h~{KI#Q4h9!G)cOFSku>Umia#jCv9mq~L!kDq zuSL{XP~A{{d^WxV6gu+vw{PB&{!eGX|8ley0kxc+Ga>&P!J2UHfQhAS4%__KohNz! z?oRZG8!!?f+Y%E|!V&RDfiM5%CSRZq1eor~)dQr4aOd{-Ak|c=l>ag?P+aLFRKLj- zZi^&c;g_1t{3!x=ngk<>3l!?_>`agu5SzPoa-o%4*?SY+uNuj9p#r04$hf6H{;;F5 z`ofGtIs}(B_5xE={&uPQPTgJNRL;ZuHD^lLnaf4*`B$9Z?g)5o5G$YwzxXJy3 z7>Nx}&dsDF4!_27EHjqt)gyGkBh*SO1&LdmPd~WYHLeb+YghSLh+R(@o{V5drXPJo z9=l3@S<1B_(nS#0AvH!B#Z5R5ut_Zz9;{Q}l04Yda21+5dI@~&DxC?;WPv*r^m*sh zu8xj$;?4ZH7yNtWVVfeMjwYz7XX7jfxLa9_M@{~@H`(sBXNjVfMp)DR&-K){Wj7zB zm6;Arp6`iVl+8F2a@RD^qX)6IlXq57P_XA|(nU9KC4>|}uJER25Kx!v@t-d&qj+E{ zi_OZ``uM`3pNf;L@8vrr&_U>t8y4iH}m&fAUml5zz6M)Mf zgJS^Sa%EMuTiR2ri}QP;hy21KgT7{!qg82nakMi^Y~s=7eE$=uu(XFK6l=(6f9CM= z_Oz^DF_B!MjqSFXtMrt8p}v*)RzPCcXdtmiC?2kX#+hVZs_Sqj0aM!md34qC3y;**?aexvPH$C zTuZfHzwqJxW%p(sCK^f>;vW~s)*lC2D2MNf)#+S$8yf=lr;=f0&X&;&mV}rKknn@z z5aSrmYm&Cx%2qfP@>$|63_O#X&GSY4^(YLMk5SI{B=vOY?4eAs6vM?~l&L@5KX|^Zi8xL@>QN*)XK`w0KmABUhLg%>4#LG{`ObmRRLvxPtu6un! zn`d%rUFuQG%PNsn)<>hLHF(s|kgBWfRGG;^QRwp`=+8wuY74S_KJ_$+dT)_1~m z8yB*fI9T;}2`3Te#$$Sd|KPdM(1A}Qnx2tX7JNb-QNx}7NtZ0Go)|eJys*8)5tb_c} zgtM8^^aFaU$%D3Wr_iU3)&~a{Sy=T25{Zd3uzxS6yF_;fCehRVXsi(*OiIjf44 z`|2`XXC3QKZg-pmu;RCK2ZM*Z2;v6&FKwB>)prk#HU`;KNgdJ@62CShoi~Hxi32SD z-Tg>=;oxXLf1y?7fDCPS9_)NbgeF+ETdUb}|92YtarF7}-7v=p z9k2j<_80%%2>Q8Wa$n==>1n>x#XtY-WT8Vyg)e|x50JCrdDXg#!{ba2Ok`gQYcN>I z6bk4YHN1YiheXga#n8x(HOa2koEQZ1RLh1hHRi(Ea+`@h($CMWpL<+6fM*hR=lRr% zKhN5+6{wlA_Q3=H0uA2ph=~HvZLh@eeP3iQ`@VJAdE&a!To~p!Jq1UXbeNm_nl~*V zFGRd&MvT;YwxUfgt)TmvEvpAJow$zAiAAzK5+Nanp>^Ur$aa7fEUA1%Sag<=Qc94^ zQ`YKc+h=r~g%@=LOSj3^R4owO_bn7Iq=)W}+AB$y;hN?0t#w5@W+jluHeXQ>N2r<@F(;3(p zd%1@`p-%1ra{?ZFhMKt5^QN>D%J=?*;-b(ahk@i13yrUI9ZVt5&S~;)&DPm(Pi%-c zt`99onc(1|%%q1O*w)YtSq#Lvh_S+WaAVZKU;q

73l5xhUdS|J>e>O6W6f04oJF z*JzxDqZEWqHN2);Rz34&Yyr96ZiZCnLJ()3V4L`s3foXJX3N!SMS)F0f=loME+(#Q z{rW6W??P=^O|{nIm+hZIS=sZ`Q|vQ>hDg35H|Hsb9o~O~t@)%(>0g()lAoG>f<0d5+lIgx#uBR9BCest`*x zD@IxT2W|oCs72A_N6%*tjfTd$%=2T)x+p^N-)zAr5A99ug|QzdWr+k>b+^}=l4Qi# zS0>R6(ZA{yetn8&q7ls2z5m*#2mHC5Q&oVME0{JvXHnzga*9E{%t?Rz_4i11|!CZbqKzOuCy#O*lvJ^obXA()=B%~26^~B)8k3sedu?o`r zUGDE6w3_YF{rlX(K7A)_Dvv6A#sRxT>C&msg6A8ql2*|5`sh{hLOVJ(wT1sl2p5 zIxc~=wPi3%b%1~kCl22$J2@My(^g-_Og_XKQ1xL47t!}sb4yhQO(LjaIH?K57#a@Y z2v*IpX{VAeYblb*Ee*4;J*Q2JUGMalem$ssDQ~@^@csJN=s|R=takPhaNT0M;6oaU z_WO~MI+8?FsbRI%NXy5gmb$CXMn_!F$3xHQs+UXn=b5;7#3QFPWtLoi=fBy9yd36^ zYlp4;zq9xKCl9dXY%h8Dc`go=CqDTnwR#B`KGfCH#|l%flt!Y;Og-w&Sc}c?Bm&lB z?StB%z5X1GOt>LUp1()z!}=A7Ah@a95r%C$llcY}Vg!#V(uBfvH2(z-Xd2{4f+Bhz`UIGVq3=Uwf9# zFEmj4XIai_wLl(=$02hcI}bHMs}6w!s{JxomGp*U-+jgTUsim7#Q=fNW#d|-2{cfZ z(%Eg0MhDSS`0^9ytK6;6ex_H)*#~ty$8iW2My+Oq&mtWxBpyOWf)-@O4q-}T7V;7a zXYuAjRWQUM^FoU#TG4u;zsvr`Vi2 z?KdBOg=W=2M*A;YqRJ7mBUa_FaY_fkUxs+J(y_8jVbM{F*^GIDHA|nMF zgTrN4*0CuTlZ}}H`C#Uy#i_Ms?jD94{L3#vlF_^hs!nQjsQ-4gal#3qmOZv!RG4i( zLJ1F709*q96 z%51mBr){eVd7GdwwYRB+fKt$S+s@>wvjv+uDu^* zF;^w`f1pN5h{8)WL`2j{T|WHAcB5UpZ(?)WgH>_dusSe8;KdtPPPbKW{9n&$KrwFM z@k}!M>i##!j1$WI2~<&hr7B!zt(a2A#l=a}$UvE6tMiC1Cy5P)OeH!oEsGMQf(LCx@8VcSzbzX| zQ6j^%6vC_9nXu1#Eyzi$l)LRjY5n!s{3%8!-NdH{GbP%`NJgJ2h=O?8UW6KBbOI85 zjzK{3Nb#X{KPp#t4g+yNya`+>H7UwH0Ax0`egH23obi36xFnXDlp zhD2UYB^%#H*k^ueumnDqfiNViwo^!RDjC-fS?-P>4eCW(j>ENKar24g?}N|`t%#}A zwxAOd(e4+6S_h`wk1B=MvL2CYG%Weh;5+yOG`K5t$cWvuXMwnTn35z04u1yw>Kn$W zLf3dF^WgGPMR5=_!yx`lQemu`Pu<8)mhk@4{_*eR*r9P+{>8=bD@}ac5gxN*)JI6Z zWxje2hoR9xXpJWTvB0Jln3_=aM)aTrXIUDO@POHV>v`*VJg*Y!?S3-o0R{0!Eq?!q zuny7;1AZ%h-m+~d*IGQS1DlMAy=o!ha8u0u?n`0gpBORtkI3NYm5c}Lw!$Bq;TRTZ zaGO?{D<&w%t*HmoRc_qmtY+3O&x83~v%_7-yt*4^^v^WZSPji|bHO~_>PD%xAZjxp z-e!l!qV~x24pnHfQ$vUBQ`9)LetzBl;M+6?2`0`T`=|P}*u;O|njMBu9!~Zm^Td*p z@`8l6+^b__9uy&sR`-2_nCM&VYU=ESB_w`>@C4tCmSJ@sSLhZyD+|)f=LLt4Lix5D z7RHXD-~Vl0o6e2GeG4fSN>HU2@9c-fHbV~CIR0tm&w0I3d0+O`I7@CNWXvW`$W7Uw ztr-w6&nWRP7OeAo=h1$Y3$feLr#{5+=YG-mdy!QH*o^qs-EEmeLrE#z>Qh#rc%KY{ z9D{IzXH|_j?S9)IZE-hi36sq9C3Gz>T+gv)T<`_BTemBB<$Oe@tM~{H1}wLWDM9ET zOPAOK{S(`?FNprLo8ge=O#@2~q8zV2N7lLqoJ4xY_BGF|COQ?k5vdjYPUwr>#dC5) zU4hWZs|>@BpZ?tA%0KRQdk>b+IZiXuBdjWWe+?;j3N7E1XeOZ9iEK(kc7n6q-X$ON8|e+@YwC^g|}Ae?tu9Ug_1AV#@NzNrgkBc&J#|8UQA12Z5go~346T> zll%a2-dy*_VgCI{tF<;ex^#LS(#GQb1z^?wzZZDW|0@2oIW5ufSBuQX>;15x#3XimG3SLNn)hZ;v#XU1Wo-a@Fz3$pv7rBl`h4wsa7kPIR**O zg)o`AF=G6{7uH-;XL?RHw~5oVjxMV6jrz58@sS^9G+Wa%Sz^-8IzwR=z^e1xnR8Tc`K0x{OC&SJi7EE%QXxFzgYmke?T*2Bs~7QJ4nGF_gY{HQen`Gq)7w^$ zXo(Lq*U2iXwzTgCF~Cz;0M0r?=9v1W?iv3epnF{DM%r4inm^~0M{`yg<#6&%UY>Dx z*a(4=sk+uW*a>Gb5Jnt5VNoY0Wraty`NmjM|Cr>c8cO%BZkXQyQM93Zo<56))${qw zoel8qu($wwMfu|=R{mo`;$xWi(`}<2$JX3f+g%aj>&o)c({ZL0o77|h2I%ViQ)Tv- z?Al3)jZ62|XIst@Idp0F z2w`Y2Vo^$fakz=HG9sQ6dXm)7dqe;s5^nq7qvZOA>7SeSVj|XMK=OTZa0BA)`hG)a z4z=v8xV|}-Tn6D03vjuaxX*IzU)WwFvP*eYJX;_RiIz>uO@r+R*W9fhCGh~o(?vy~ zzlyr++-h9Wp;}3^?*<>fkty!I9f0?GbBUxq zDG6z7tV?&k^H`LX4}2D$elFG=`%Zhh{VSM_phmB)#-3KJzZ*7A&h17O%%7(~(if`+ zQjWBOsk!M;wym*|4TW)dkg{|bj+S<~;@8n>((gEmrz4yz`jOY`-n&8z_+xMP-Je(b&t zFGrUXgLuVZ5XA#jW*Woxj%!bOR))ZU9i@1h-?XfB;2>5;{|}f3t-{n5^dd1jh-AZ) z!WEQq-nPCs$>%o-V3YEC`Kx3;o;sV=FUpi||443!&LUC{P-m zsv_YkZrb^4u8fr>;YY2fxEfxJv`(naHfMg$@CT;RMJ%p`;k1WMgXKgM(!eMFtGEf` zAh3k|uO{$6U11j+9~20MFfKpNSZ+M}n-NoVwI9QT^)dc$XYwFHDBdE^3(Dnq>TNk_ z6i6fAD~e$p0^YBkHl4<*5fQ3fr!+@vcZq%;!!|OG~SyrUvQR zMrU{^70hxjH$g=i23J#09L(~BL;|b}FT2x1a)amwYP%^+{o$Q=CXg|GJvz=kMr%>b zqdCpn?V|Xy(H?Tfio^6 z@$eCpjKqMWg=_#sxN@ia?KIfb6w|fR+v>u-k?W4ZVE?(=K$V8-bz4&TaHeP6<>BxA zFXK%|+9_-OqS@B-1fQw%^m2Y6JBTbD~5r>9sNF89uo85+Tomxj)K4+?pmOtyG zj16dn1??i@ga1Gxj%^owI~UTDSZVyFhqP6t{(i_EdZc-rxPZY@>L-L1pvFF3+(>>2crA6MLP3?w~ zT_}squIc0W)E}e(HvLM?5XMH+mhOG-Zd-PKBO&`Oe(aExdq+_Pp^~mGFgIg!0%SYA z&>Q~cj+PgHoV;@X*=d`#cttucQV#Ro$2zgS;2>e<$b|O#;RS|^#EoM3_^BJJ=hPe3 zdP#DWp~zHGSC>+FOE@U8*9vR0;6|8VMq{r%*UsPP>Du%UtH#FW9tblRKUTt8;%&un zhPT8|GgL@~6VB_IbUX?L@dn;{#|(9B&^e*A6O%3_oTuVp#RVW+7%Tx{S1s|e-3*HmtYzf*#ta2&pQ zC@d_6eUg`tdbw{A%j6fgGE}}+lxC}G2^ak9mNkK zLoqN8um`aQGyfj_OkSjTu;|Ef1Y||y$mN6^0?V1)M>X*J9zU>o@O+p!HE-|NjA72m@5=( zbY1^7!2%;8WIyS2-@U53gHHQ&vqPNCX_x$l>M#@^@J3{0CF6|;Mo9A0KUZbi?HoJH0I}v-@(8yzw0oHLJOc&1 z73Vf?;ISr5kOcB(n3q%=Pa;@E#dTXNX8}1Ma;H0&E)w+SR4Gv|V$sb^Yhhy~&@&=fN0`fcpdM zcV*>y!i#~--YAjC0?Qb|gQ8wh?EypkW^hXj`#Q*>`S><1K9@a)8}K`|)!_H>JoAa; zQ{79fa}$fpHnQ)$xX{TATbjHOI0VY<9jl_&vNGWN4-azZov23j7oEZAumj_OO1)q3 zazM8}C`oWxv{{iT8&aoQ?%y7V9sVND(s#nDC5z2K?{97brqfe+sc~cER0Pd6; zQ}w6Wmc+2{od?LE;T;JM6+djsb7o7zqiqPwK6~2Ybi?4T1U@K<%lcDp4e&OTccaHZ z%XVxzJeOj~oov|W#)1+v1Qc&f=)Fvg5g&Zpm>2YwBIXNu!ZztS;S|q?cfLwP^5ZTZ zJL4=~qw-0+oIO}h6NU=Sbm7wV;%0tKKnL=nQ<@Cvdh}8ap9)C-3Aj@M!;2GhQ0ct4 zEbsgjpV(9PTuVFEUxAj*VEk&79p|0--yK_s7R>k5bi2n(v&k1of&)OIZ91}L4#e@) zb)Zg(;z&jgtXf&MJMM{kDVbS_G5p38fbrQ#%*$iM|F+V~YT(Ijpy*agOrps?+5l&0 z_#=<8u0pII$Y-x(G^!mL~ts8Ne4DMDn8$B{|s&H-p}qV$oh~${xfC zUF!^2%8g=zvfBzr%7cPz_z0_8h*t5O3{*J{0!6ZZ%D04D?-l~k$gvdsOM^Fh%UbuS z&%EW&(0B`lbRP9{oMXA*rKH_JOw%F#s~BlZ=$9id=g&@_nFwYC@aerk5VGUUzwWmj z*@sWoF%k1bF<|caGK)<{9&5_@Ph`t=qjN&-nxoB)&Q!l$l9N(?2T4=-C8Z|^1A+9u zYdwUxbr^CtoG@NI>tK<;eFgc|TG6_{zd`N0_wjOk@1?=q*EEjP$e2k#UAPIyXQWa( z56c4V)x(9s0!YaH+&c-tY&A3c)1y18>$`p(GiEC>(ZN?%T1m~02eQ#2ZaKq?=l)#7 zR%k2c<)M6zBWso+pD|$23$5$*CYw#8?RD;F&87(ng~Rc64;io!Wy!EZsGQ zA4zJJ!~KvEmr`avT|<56pBQdi$1+M%qF5<8p6fOxl!pyELkVN<$`fXu^wRZ&d&qorZu?qHRyM|W*$c0+x(=$0DX?utot=ZDuy#6P5`gri?QVX;p`*4+%~_-e z=fYRlt3I(jD#!v&4DW))sz3mp=73H`M14_Dc`aGU2d*_iBc&jd>~<1}$m8BF%&Xk6 zZfhAAmT1n+QJiX1%6Km6lL^-$*NN=7Wp8UIOtfsvX{Ke>PVp40s3he)i~79+4KPd)6ZXKnJl~Z1Svz4UY)hfYf8omiGLFvr0ugJSOF8Jc z1~E>-mvc|lfAWMbN|04&3Yx*cV!mdYyTrLfCcp$;$yq*9&%Y3zN}ZUJgOswnVxRK- zh8Y>74n%m_VIV70Q+r5m4Mqk;HQZ) zf#4G}sGTY<0b<4PcVdMM`@k@*gjIbX!~}t7oEbqK z&@=;?Lc3+3n_jAil&nr4m!Pi$$_+w~P15KQtHrXtFl_y`>a9FMKi>afU%o(UTPCmD zxxU@Sn7aSA*U4VYtkCcZc70fq8eDTP>gXx%`J<+$7UDveaprz6-E$NUK#&G(AnQ&)(b^*^lz?U&=VI7BVl_rZ z^$x;P4H(QJ%#}AG#8NGKDW3D>&p#4RS#Re&%w!@sSrp9i{z}dXS@1+4oW3C3$wB%l zfki>g$!w}0n6e;Mn`PUKB-5YwNCigC101IAVNUyO!sIXy>t|rubAAYjWr0f71KcLB zG|cTYDh1IXH?2E#bLq+m2NlrEDVM}seSb^eIV z#mr6K1Dl;Fqo^LR2<+GbkR?apSMt=nU!;%M`q5!KjNv^0^l|a8_*;wSHXl)Mx3Xmr zC3R%t14&pmkrvaUuO;>AZnHQY{^w#Jzy^k`>lpMYxZPq_tts~2aVFPrPA>_ayy%$w*SQPspQ|A{3C(2_c> zr=lff->B+{!#*Oo(c9rxBEe;`a3lIe>`T#PUWTn(*4AJ?MQhTJ7J;GtWHAFXr)M<5 ze!@&>-{9*vhB9p4NW65E=2Iw#cfv=ZymsPFOgh9GA9L|-gb|e>(EG)fbSWch!X9Wz z9^I4bTGEl~EGY>6r>FwX6H>I4lA`i%#CQ$Q602g4j{-j*SdmXy3u>tfD~63y++kKL z3=hhLP7=N-g@0HUG)Y#MYl5C+uDFG&Pwk02iajBUAT0+$;HYkQr05w}OXIBI!IQ<% zKF9yMu|kDZ%_!pM$uFqtYZOlvXc0E8taU)U{q-wz)yjv4*vi;Rb(Le%n{H}0$t1gH zO7B=QBUo2?w$byq4#f$95#-p&0gnezBLr5s<{^p)h7g|?nE9L~-DGdOKm{fNh8!O> z*5v7Tb<6IhoQ5$38YTd#;H~2)rRYY+2I}hg_3*TiEShDJuAbj?XnL8M5-T$(%uIhk zGd_*BT*f%Hs3sYsxS0Cl`Vnd!sv-UcS0!D8_xq=o)~`$^6Y~uMvU*Tado8lU22Gj! z-#^w!P5oZJkWperQ)|uXqshy~Kx*o@>PJ(L)8EcQt3a zE>yK?cOX)EmmiOJ<|z6heiRu^*{ns0oI&C!DS?HcY_gX)=tkh4SxL#ti-@sDD%!Lg ziYh1~(9ongDY>o)Ugl+Zs*T(Tttxg%7Cg{5Y|mlcru>+N4y9ea_O2;V9%R z86&WKfu<<)mWN5TC(Vq7>j0oV6_F>?-yL`x_iJ{QFY9sm}QX>SW5XXsXctfJHpwy<*nHbOey z%CbCfdOeiXv^pgTSCIeTZr2k3-LAQc-tntb8p84UzTTg}(`41(j>H2Su7L3{1AtZ= z7RTLz{ML}Eeh#mDQ(*I-cpVe^zH5FK=klMR$QVC9$yVHnkh_sEMaqqiwGScBh!mzD zY9F@G7@lsufk|Y2CyGVB7Q|NUe}vlhJr^{5I1y%tIhfNCx0Cpj#`8inpo4Vy9PV^V zEUp}0Pn_&)IcgLj+K+A?{j@w;UKhBxPN|+sjaycO-F;I|kB2QlL(5lnR*}xO(+yQ- z!Dn3Q6)D$&l~l9`!xVsh_5TP4dwtX1RF&pJOK7@c{QB z79nE^L39J;Y|OMUIg_vwHkbnN*A&fW+31Y~S&Q-|)CbI~|yIvNuQ9bTrAz9-g8Z znYIyy{KCtoxZDkYYX7#zp=&EBC;)kTdyCBCJUl)slr_?~xanw=3mk~^w%505_{rP| zFi5BrfOaL4B^i>pwGXu@DTN`}h4Rd<*QZVk+hhj!@@zLYUwvdv%Dh1@7S--Anv2IheWqC+V$Fho~^&1d<4pIK$Y6vaW5i zhZ0Q9P$%Es67bQp2UrPDdQf{Ok@&~{!U?*S41(?%z0oJ(R#FBGSQc&%dN4?E(lfq< z<-nvW(zdbpO=-rU_*lpw=#P?Fs*%dN2NE_iwF2Y|Yg>B}8CI8)tF~*hxk?+`WM6CT z+LDt{Cn^+^o}#Hnb-jexM+~ilF2^sl7G+eeL`fSF$oKQCRXQ~B)uOG3*-ndXkzmfw zXKg<)LD<*(3OsfMTnjOkn5CqQlw}qN!7Rz=h*dC?fuT=t4n!i1!($lXr7*X5U;m)_ z9($O1ZUx$firsx)N{oX~^dTobN&xP?#WBS}tB>Z(ckWe{f7bZS)M>1XofuouJd8~A zy_o_`3rs1D3%bZ^+cRZHk8&e7MWX3m4qFD{ELo3fo(R@VeS51OU~*FK#x~|6>*Dvr zNGD|pg=MLz61c*1?rp{@)`6td^w0;)jfHK<-JD}qTLMEZoURK|#gC(|eSv7FV_?75 zSmBhR*Rn7+(tSe#CpI@E?7&W?h?d7b zPua}<me>)fMf!E8Ne0 z7znVUcNZR8M~copN8UZ=*U(^*jB1WN9jAc?}$2;9ps32Sa0Wl^P z%IB-MBc%IJrUN7SX`0v_-Ioyfpby;^oz8|_ynagHetvMuHf^F0OH`9lKEH=94nm<1 z^&N>Q($Uw381#y+OS5znwr={F$fQcJdph&?C)bCIh2R00f6Q`Y249q6VxtDuE`%us zaBZtX9v=%sn~_sZR25bfvNn$t&-r0piHy|r^_fo6NJbqswfxAc#|r)H^v?{S6wl>J z0t{cVOO7wKaSk^kvra>JZtnTHW!YP?-^Aeuacnf%1M&%FCv+5k`!KGI&y{27K0`3h zwYZXvjiSyu+m2lB;cwNH{T_QXz3S1+K!X8H|eRj)x7(CIWc-QsVgM;0}3z%?+h=wSKgqgk| zo#^7Ms1SEvW@6)MY4N$e)V6T2%j&IW)8$kNqssnpcHYB{UFQA7rA;4;;x3r3q1B1S z()bP}iIDtte-{FyHU|fhh9Fx6BU@K`TbPMiQ5SNA>?|GvwOxb@8XRDTEr?VL=I=ZI z7VfE}61vqB%C^sTnHC}k5MrWN{fl+Efd>*}#k=+)(IPVwQr8W9#B#WW7c?C&C`ufN zEPRedy`e#fFfzo|%eE_F(KsnKTE*C9gecGVxBFSXBQYVVRr2o5pCaxenm`^N#m1*O z?RCfYYwSr5MaFV&q>9Ekk06#_V_6~J$K6^A*mB zbl}5DM5?v!WzpKcB0kptu~RJmGm6eG{8FRA%1i9Z6Ri4uDhUJ!!hEma@3AYRghX%_ zSy}7I*h6wmNs-zKE#{E5<5*dp0Q|Q<&GOq4l&`U~rxfcvE#kAOlR(J(qu#!CR~tw7 z@e~(ih8Dn-9(-hWuijIk3$s!9ZoF91@K|Ou&gAs^r>Q5Oes_A>$tN1m1FxC3k$Z}s zVBY^6uP{H0Ql6NeOMU*P{a-KYKLgGGZBCumA?Y?cxk%sf>0F3DpHj=u^SNxXa6nA( z=xqLTI*In;+Z5(b>cPc~fy4@)hwtkzlwC@X>;xiinF2wpuJ^ZR_;0Vwa<_$vyJeZi zsXsj>eFH^dkC+>J!r(B}P)Nt@p~%C-=Jp$15gi{N0U|{}<4c|{O`)7cX^7C#VmG9~ z-_kj#5-Y%Iz1b?Mqq?SMP}mw8i>n7Y(PGCDZ_4x8i}87Tgh(&S+4}7#@F@tjZf!#E zBTDa17@`H#xor2nFP~DpT_VE9wT|TQyX^%N*S|kq%~(l#k$_5<%)*z_(Y`Dr`rq&p zYL%4{!1Vd8RkBT=z?b%_vtV|tK<778R{ZB7Z;fS^!_YoxL#fsX&VQP1Eql9UKD4po zaBy88c{~GicjhADM+$v$T^;)URv_X}f}@??DraEcy4+bWMIIxOt-^U!YQp@m+^lpp zYy43&b;@}>mBqS+s};fUDGzcJCuF!%7iFd!wb+}YS>btj<|1xw2RAg`V`uFjY;r21 zQjfkKKi60A6oFZbfa!8=D%ftUs5FF+T}z>*8CgMH)?e;TRhoFkrWKyVU3iaTK5Tsp zNX>)z$Dd*Exd4@=kb0_LSAoqDw?1UC5qU9moT~j8Z%t=LLbw7k%qqG_v>@iL8vXK2 zH+1NLBacz<;S(>4yb#1p&JlehzsqM^7^YwqriXRbcD)lzVA4(4`^wDcdWom=F%)7p z*Zn0VvujTp3LDA?&(h9+bD|ll6E`*?#AS=_dV?eNm`l&vVhw4r$wRhyj|-{Qpj<|{ zneK-9${P>YkWrDHji`Fvx=zM{Kw5RZ4EtxJjX6M+qCM~u%e~}wir91D6D7}kf8Sj# z3iPjt{qW8Vcp-aR7{;`CPBbd9Fa5~GqwO>>Jxo5B4CT~uuyy`OYGd%_RZZ<)5aBzM zY6i+I$gCit^5q~aRz5B2xCG>PmFsT=FerzhgBc(W5eQl`O|EeHC>aNq@DiT8O`&RX zynS7NRVE}k2=}Crkxiq-#TmQ5YsfCMDI2Rh+0p9w#gfd{i8cjOi+`+iUvmT$tl2nc z;Z6in=%}v`HQ}nMt;oPgOvnm-1|W$pe=K6f!REkcj|p+XpNAWtFzNPbH*9mQRbHeI#dXCqMh4xwk~qS2k{W##9}KC~M>r8S z!jhWUPeZirU$3Z$ksOyZzMqn8+QcJCeEKf0?4@YTwkq8%5tA<@A5>l)!)ZpvQeEZn zA)WkrtgGx-j6+zgpcAgVsqA6?^zMgT1$+_uQ`Jwz1d7@pD|7skX3ReNWH^+pGs7el zDUXM!rI|J;1{Kuu;Mem6xe32~G_@)aPmH!uKd!?tFu9?MkDii|CGL1mVBqoA<_(?JLV=S6UzDI6K;>WN8daM`r$?*F3BQ(OM{+r~~BhGC`=Ddq3y@OsGx>c3Bg_g^30xnLh)9EI57h#S?^F}+$ufAIC3|BrFG1Z> z=xFfTb-z8odxSR=REm(F&nJFD^8ZDAq5Aq>2G!QH{tpw|ems!EV9^5TH0JVsoFe!I zF%nB4CV6)d?}yOx&L&W1xnI@kng(qc0|mLg03 zXrcN}kt77k^KgYmJDLwgAoKtIk#>1p)1Y9WwIc#uNl9&qhzwC5rjcw=FO}Jy6S_F9 z-MqM;o+o7BY`q_+gC7*j27eUy5d>LN1?6L4cU) z08Eqa>3R!2K8r48;M3Dd11%er(rOmx9%^~KflTw81YC{ISqm7?o!q0cvR1WQbY~j- z{SCI7d#g$Rbfy-|cqBp+xO!fl{Tvr!eN({^RT$e**oT-T%uiA%>WUoy_kPmt=;=Px zg#vd|!s3^g_@wKALF7Xm{V)Rw=Kg{hEH7f?4T&YD0HGDUoi48j%L0UwvVtJKp4zPM z9+8q9Z&PYAbL_=sTeR4C`U2JBxG;3Xi$mIUkiJ&=|HHyJBDs z;@VVdsluhj_8(l@$6DzHZsxuop*^_Fkv-dHwgp{g7bdH9WackyL;wdyKd$aS;7pqf z+@zj32%8C{Ck(iuuz^Vsf&vZX)=MgcQi;)eZ8$}NbROgAf(m;!LC>92i>o^5oxi@* zLiQ@4fE)S5X}SB@;hol1sngNRy7)!78qsV>#XnOt)?irZhaeqL=NiJWFoVZl^^`Lm z8urFTE-7{7?7w@aijDf{c4s1{jh|P0e-a&%VN`v=*cH+Mdo&dc(}Wdoj2rrEG5-c! zrby69Mkm^N>+_F9L_UstH)j?H zwZZmIBIR#mvsO;ba(EvG0em&ME3)9Vqp{$UKne)}-%u4`aa;j|Ns@Ep@Q{HAF^L*j zn*O7m#_}1+3N9-D6_I-mR-+HFC`1Q*CZ!3yT2g(Fh?#?L*ERYoz8M)0*{%|7}#b>D~@s(;pv9)c9^YEx= zp#I-7Mww1c2r5%Jj`k-YYCQT1*9Vd-pSqcSeq@J{c5a+Api>OW$Hau2A-8MYb_37? zi+HR(W5M*}`AlzmvVVM_wr`zqelf^VP zWw?WSZg0*oBYwyug!CH$?jlULhBX5eb3B{7C&1_kq2E70oVqc_23k{gMmNV=)YaNy z!IXnOHIPZ_)vG!|17hn^yC-&2^$3H=cL(drU#QK-j zx_WG;7jFZ*=B=Yu-qu#N!w!;PCl1l#@v+KAgmCFetYu61X;9TaNqe+G^X=ayz3EFq z$`(*P{bJ50zB=tS<?U3TyW$IzIbIH?w+@a*}YF$6Kk-GbyNT%m?4C zDQG@>v;JjHWL%cnPyu4gE`Qmu65VnU4AbJ=hivfm``2of;3<(ULD2tGkp6c`V&MG) z)Nt23*_V|zy**xmF4X-XRm3B#qv|&)@I4~R^FQw@G4_Y9+5!K4&&=G9=bpp*CS^Nv z5v=t1u%F~MgiQbK>2qdtGnZQN=UYSjYQL$QU<0In+jO~ap9x223-=R+RK$rS@Xz4& z-u_jH?Uv^{r(Q&3ki{MxSh5)R=mwSFek|U**R~+#HX|w=`|)~2U=k3@s9F9kfezNd zy!a{Rn0<&-vJ;xvO+sMoQ!LI@FdbxNZH;2;|9T55OL7c`HvD4zo`6YcEqq6o~v`C6dG`A}7KF=m*V>s%O|-lLhcW4>~cg_~e~X zOzn4g{?!PL6uy`be!F@1&Uw3-cc=;Pa>s}t90#n6T;3_pn{G*E?q+0WQg%O!7r6_} zL}DVpa7E>#eNV(G%aT_+x@R~UVjI@lYBY*d6MQ8m6{!Yd8hN)i^PUP|9F4bxjMb$Q zV7w-&T{^tIh7801Bds|0DjPDqMG@B>+2`sWBJ{wRIWig;k`3(<74KMtMiMe`a`pR5 zZH$yB0pF0Jw;Wh!*xv!3B*>+nCRwd!r-G7$%qAjO|AzSYoJ~&*v*O6Vh_m;?j2&c) z+mkB+dR-`@o3F-Y8i|ggLCicG z-?LHvAy)``&h8KJhW4CMt=Ea-%* z61WU3ZS(H5jUI~Az<3NJa$#5urHdQ8tl;m1S~ZU*{&4#ywP|H#Miz82c0w}5OouM& zpZZN~bmIOq4S%7d7T>uVcx`dp?hPJnCp#=@UUy=Iq-~#obya{yT3)rZhtx{PnhBS+$}#aI!PVIox4ID%NUcSP-M4V8di~Yv#}EWHB$h$n1V^r^ z0Zaq(!m^jIFUMf!`z zX97j#SBbT)<1^=aWqii7{@jjxX>7026n^Pac@TxfswD(k8}iSWJe*w8u3W0$OF)t= zOps4Mn-L%>Y<0*Vf3UG1fctULK7HKU#Ka zUGLM;aG#|!_K$eX8~#-iko{$g8L4*-ACw$&A*WR@yOr5__vyp_25UooroY;rNUB4m zTB^WP-uu91PIuv_)6Pdrd|3ZUnb;cf$c#6tw;GmhUlcg82J? z+xGukCFhC%LzUL70zciq?rlC*Y zQP#FXE>|`q977PvVYOC5N-BJSeMmN}{Pk`V^FOZn)BOdzPGT$A^4%9)Z`A#HWn9=c zX6O9*_3a={)l%u@!blZcP;d={<>+T`0~<&nfVDkoL~V z((&7y#JTGbKhy>e5#z%O(Ta`MKWzJ;eyj|4D9$+sc8Ty|$s4zt&X6wEAj?C1->bvHbap>2E&@~aW{VzzJV5#@MNQirMf4d+P) zZ?WAH@=r3$pdZK^R4Z;Uz+8kLMF!kZK=RlI7Wggk7qlh((vqEj3)!ozCxtt! zP4S7wC`p@DqK_?NG96Ol`H^aN1g86O5NnC2Hb63VyG9eT9{-Iu^UkuLV~yr=UJ=8K zt*@4=h8$}v(DMJu5?RHTFdmE`BILH`pZocT33lzjW>GLR08{ufMEHlMkpVl&pAB#I z@aWIl$b9!*oNTL(WVvVHZ%0Q#C9x5daZHpMpP!+pA}T{lP~xg-@T;AOMU2(M8jS(M z&VHIAvV@Z9x$SC8QzaiZiARg<){0WGCc%f9#6BkT0kdSF1h7fkoik)1%)!dK6Cd zEk`oJ(GC-2CkO^z>jDt&Lhad{$GAoPlp^p*nS)tzBRDbQyAMXTIP+pXb{hy@iMcPsNd`kkI723BS(quztjKx`GyOXH6zr*Owwpdq_tLv07U^lJMkG z6UnyElcKIhpFa80ssO29SX9)8-=Y1>&3M+LU($AqXx(=W-=Uswqa^?q{$K#v8Sf7o zdc~Pfgl1%^6f8Ku;qb40#&XqE3!x78pxejgosD8Lrrpc}&9a}Z(>cE$Pmp{@F`6k| zFypkx3L9{j?QRLj=Siat`2-sXKpeZ&o4fo1{f@bmpRkV`w*!)rV=ZWEsraGMT|;TX zBx;WnLjktCwo0c)xGV9s;T}_Oe;hK^l5UTA#tM;31fmUtV@|%Gad@jBdi<(#$OMr? za!$e?=aG)OFa%Vr=&KVgHc6Q+vI#NOA6Iy%1eYB0&Rn4-tfiHwzVcMw<*sx`Gln*Z z8WgMX(@n$;!=Y2ON}r{uw{D74wk@YQMtK`9w#h`;M9O?DdP?gq)a#@tyvNyhP@O8* zN9>1Ra_lEE%)tC6<5h{ZnMWLHvz9MM+4t}O&w8h=Tp!lSw#qu6+}`W2V5>|+nqV=# z(f^s-zZ^p4fBm=gUWIZLAq0iSub|$J!b`tND-Ycu1)e9+F`i;v zV)XYtD&Du?_vh^^XsSHQ_grq9#Ft%#I7KF~?x%pdni?sI!2%tz)A3OPlNa{g?RW33 zw}yF(cqHEK;AiFd7&e_9hh-vwn=Z5ug>7>PZH9v{NAA&o#EJiBt{#r3+^bOg#a9H0 zfm+wxn1dU^VmX_mzX!3>0M3&F&f>FdZ#pxLOZI{GjZRE+yz6wpJ|NWL53fwsHk9di z&aJp^s`Hhqy29v`)ME|u>aSdi4=f)7|Km(HrooyoLUDNpz^}MZX(e*Iq%YGlbvZu| z{WzoXC_DV8#R2^tX)%v-_|l(PSSkK|trQn_d^FUPb0>+0Ui$P)u(r)Y~zA22hq_ z{oz5z4C+Je*y2K+b=j`{?rkw2$!Yr2z9xWkrp|!q~X|EBQ?#l9;coYQbDc>#7k;$97>9-GRvi z$*uDIBhtbY9pa6l#%Ow#)~A5}*$aX;)RRNq&)R7zTU8mjv_$DlQOfT9SBCnF>e$0q zwj}azGbJ}+cHg9SBy|U_aOUh_#M9Bmg55QZq_Ryp;*XzX{wkWku}n_mp`q07(1{Mx z3=)|UKFin+C-_L>j(ztICZ*>)_GQ^z&d2~yB;~p_jU8MOU;pKI2&r0eCM6i3;H_*n zkN=pGRoPg{Z~#6DaQ}%BMs>7Of==-)`z!)zb*7C3=p#Yd)=HR?K(mjhe}QTOdigLl zb4&4R9PuT9>M2qt7bB0FuozOQ=#A*JmIl%qV|pNimIF4aow&k@>>RBVIr67Yl{Ti> z%W#-4wZz#c@$D6?bpSD35A5c`h}wD@f@{GIf^=Ll$e1}G)>m}bN!uuVAp5g6F63u* zpZZ)`vm(J*tk?Y?eH!+vlPbOM_Ib?4a6rjPd6h}K^6|NLfv-UM2x;P0w_coD_Y~aY zMl8!GIN0wWG^{GzpPO8jTic&$bp#7@LV6S1h6HR#@-*wE>;C+Ld_B_e6cobq4M;jk zxlm816FAI5ZP=nd(S(yOlR+KaRkQzW^vTDD1y?&rD`yN_R>eAMd9uWb30a%x^cRTQ zLjvd1bx_e6W!0ahdvyJF4YLVfg@|Y7@Wz`Kt?e_gGIBjFYLg)#TzUh^3s8NwEJ&KB^{AA zE!7(``4ZFNcjEc~Gh7rS{7-v~x-S+BbCZZYhlPUuNgY<2@^*qcNv@9z-9wj=Pxb+E zp`W=`gJwC4->u{TeJLXv+G6(HB#H}GgVzw8kdgCko#d?WB(QGSU$Hoz3O8076CgO0 zZfGp6Uo3yKXjP~SXzJ-Pik$=nUJfE6MN0DDtyzrjrHJ9ix@m@QFDazK99P%Wrv})x zmj9?e3)X1I9>bopt5H6xp*|>{O4vEcvPew1L%Mt*e$LRfMV{=#ZN>Hs?C-fCWx%^X0|Z{g&d8nT3f(oOYd=Eo3aiyNN*;)MxD1nF&q#j?hx4 zBCZ>UY|3N^-KwykMQfAu}JM*Z0|xK-y?k2lLbr#sXYcqRS20 zs?!Ppy7kfzLS?Rb^5sr`z#8#RKx^4kW>MUV@oycB#x+!zp(;tPs5}1R8Q}CDXWHji z4{*1b*W@%SE<3pHk8@ARNa8B@X1aJ|bJB>U6WBsx0` z(9?Gz@UUso;&wAb(s*(X7b-D6MEJGIz9tBFW`&*kcDK*w0(n8S8@=47dpY;_*V;bqbKqg2cvSG{{L}X;H7u2=nDq4zZfG^oKYF*Ln&}_1tYs0d_~coAlOAtfRmd-<#EZ=#YzAz?1=!@G&ov1T zzL9~7$7-dEIw={M5iSv5OECUURLQ`MfzWf6#Ss{hVIwOSHZ~4D$f+ow6$hk{hCQnC zcKw0u=G56azp_O#2i9d@ zgUTUYt=K=D7{=uqM}(cavWff;+?S>#B!#Nek!n2+PSfUAMn!d&KOr5+M^wD@sFvaH zXYzx99C`F%L`T5oW3jr+(^eT}6-f@#zccXTH=Qa-`MMOv>GKx^SfhRRn`&f2^)6uz zpVJGb{MYDWRq{Jjn1?zuQq%0B?G@$(jV~fh^_3_5cz##~RB0`FR~w3HeRF>!^uVtM z<9LAWRw@nvN-!0GtrHPI83<11ZpahEh(W<>MY#+*7FQyUg2>f14kXFhDT(Z2@1M!#erzHcDc73+ zPd3W`u{CMxA~O)?3C3B!Nd3Cu9wJK115g3(8)jx;`~UW{^kNxAM^} zn;Es195^T|D{G*yctnW5l;&Cugut51%QudI&05QA8yYeVEydcgcHBr?jecAwV%1bu zB0*^$qY4qan=b4Oby!&j*-;WRyg}UEt%s}qCfmi2cB#2{V8=bpG?o{s)PdU0PF*c0 zpkah{gq@gHyUR8k@Ar7bn<7qx+N5IvfRet4#-j{8>89UtRmELn+;U^xj~_p9G3eT* zPGkXsOtqDArK7Z)H$K9DHai+dy4%{czdOIY?}TIGCuc)=Q-tqDlzMHi1;AKyR zu$P``!Fy+gOzn*5FC5p;N@165BW*Tefbs^cRR&b`VwSlfb8+hZX2^4b7XV8#-`oSg z{gb0FE$M-@&l635n_r~WyJKjr21NmrcBoA|}P+^VyX8vv&?cIAqSP$h7jWV*^*uN%wij2wxP6Fd#{C4LN6=~%nHb?WHS zkF(+jliei?HsG6)fuL2W)t8c6dc1h8Wch;DQzd~+fulam5l>?_wuty;t z2hBm%|BzbNzM$9nvOwyas-$bBkvG=ZUw=kb5S#yfh6Mp3u{R@cCp``?CIoYcW^D); zg0d8$A0YCSzR^z1u;d^~GiPr|Z#1Y;Gt0ZEaq$s7Or*VQ4o;bo_L67M^vQn9^EJVKEdVB>_cD zwa7i|Sz%8-d722&r?=gj|CZ01MHGA-pv;$S^`B_i{Tj0(`~NNdLQQO-WcJ-jUPoas z2&fHom@ykOwANDLv>_=bFq>`z89#>y(j=Q1L%mnyUcVSgB{I&U zOnkJUAFj|w#+d0+Ob?z5$mN8*RZWX2}2M_;lsUMBEz#Qahd9ROB@4z_v`E^u9aanVwsr?YOD5w{Pt#q?*Dk#NU z>2Fo%*~C`oOS57K)D4FExOCe3B-@CRX9Gp0+A2+QFwmv+5M4ssstC?YInxQ`0kFR* z*Zg|OUk*QIt&BLWekUIa%2N-TLH;wVP%}UqU>3%sNL9fF%gw zsQL@=fauwz@`F{Vb=LG7M(YZ|izGxscJPlF-jQre2wQl%7tPIFdc0cli~%x@&0v-* zeDXHtOdZ{n*cc5X*%)jlteMyh?Y!wC2(>YA9K%!O2AkC9a+j&_cWOmwI|3nG#n+>l zGJxwmplider2^ukFbR`gtChr+^h5 zk?KDB`6RQDnvfhyvc&?Ta{qy~D#$(-BPP*)UsO|Zk%U03 zy;rAa-NJC3?{Gt~6x`s9BsjiQtdMPfT;7!;0+Uuq#&Co{u1WJNlr|f&W8tN!*7>CF(rbhBr24e!S?{rD^zEa{o?{UTRH>pzF&{mtbWKIHz>D^3 zQnX>bq5+@fs^WAwo7kaqz}?)iFeb!29W^^>x&%rn%yX3l&LGTSPk=(Bil<+`z-3#Hu~yq|l2s*ppZoB5Vn) zlQSpN>c)1&F0h56JucKKTrTs7#x?~4P@?RGlndrNZ4HzGn0M-WLDoo)A6=;`4+xQ* zkg)}lHiD8*!cF5&!m*MAtqST6BJx5VWhtGay%-^pflD4kKRrby$PF~-Nv$~57AsK4 zCxxxz152*cZgsRks0Rl+?#mLf*$;=;KY!mg*4v>kiVaWmMf$6psB+*B56Hu(6vB$2hR{a;ir+QTMb>Dc~uWoBr8LHdOs)oNcv|j2}+k1ucO5M z?JupK!MC@39EX%A#npxH9Y0p!{!d*Ta1jFFJ3c$jUl@~-Nf0?9RYjEirsm7CT^bZa zKsJ65;cE*YEE;GWjU$*Qa6-_l3@~Twm`tNbH7w2FjiE1iV1HIZorDG)Ywi%t7hj(? zjgzN|6$bp|FFpGhO2A5NM>08dbseLSn2B{>fe%oY zWT{5Moz7r201j~;iOEvESoYX<_(E3#Up3!@5=CE^dH=&T>RAA%A|{~3ao0bACcPg| zitZih?4j$8dWVeD3>phjwBW?={~W;{SL&S)t)-)Hi{M9AaW$QOWI_(~u|69{wa*JH z8nD1#4!={(Q~$&RHrzJC70>V_ymX?Qn(P{5o$6K}XgXYQn#V|Lvl(jRycS_3oR)KQ zJPze?CaJT`{MEo40rNupol()W+CVzM;-6lsn2IC00|<6Th`>Ir$1@eP-nm!f2X=LW zz!xo>qh}BdRUrciXqwJ0@zSzs`aL0Xu$1 zG_Y#CQhTZkGK9mNwnAdR$%`4r=bE^Jvk-6k*r9^lhwUsDY7lTrBc@P73*EE+389}6784Qp?c+cCW;pf^U&FIHyqQa zyXi}mdxWW{lofvfQL*X>$U^z!Vx|+iwcVVLj@rD?#Ka7{g)#rkYW$aOqCXvFkKrGnqT&r6B z*nPO>wAF`5>_nQq^?Ej%9=RcGLz4B_VT1hnqBuusfjhS|Z<=Le&oNP_TIgH_F(6SV z8-e*QGJ!iEQ4sfd`L+Bn>*ZwOnK|kXfcUG$rSGKw6mb_uj!f4uc!kCi>}f8p)f(2SoDAEm=I=FwMxM?{V0^jVsl zKAwwIbawsE8iRC2vn_bv5JLN+hX}8P%7}kxaMZ+URHKXPh-Y{GVT?fqu=>>hL^OVN zhxg+rzFIq!jmf9?qxcbW@NRJ-x>HlR${Nxj?-CIKMKYy7!{TAtm4x9*t79`8pB^$v zW7iw8RDLN0g~gf=hJ}4lua?WC>&=8Ij{3)S(b2aX@l7r|4|g@FUzkR~#`@x2spP9p zp9=-yc^G?$n5@5qLDMlJ`^!`Iw^}!<@;71pd+<6*OVHCVhwdsWSY){!8u*9H=W&Xl z5oNwDQ!|4t#JhmT+2|uQ@qn{ALP^JV`~Xg4hq2{PHTD*AjhRlc`zVK^5!Sjxh?4=j zv(c-hXc(O!YzqDs^9WeDaEvHYG#sCx?SfiiHYp?*3QipzX@iN3f&w$Li}sf!h4OHOdbHqR9|w#8}h#{iT`E zCf{|jFvXDpv?)R}5QF2GD_oH=0Pb1koV*3srtg0`NpP8E(c%MY^w zvpU|JR+@jmH4KGn{5#2`tXge!2kjZ6XhjVWobZnhtE!t(nj^^Wj7_vD!i8$aP*G=l zDLO?R;C+iC=^;~tmxCaT@1c=xe}!F@iDaJc)^F~XP$Xq!G9Vs1ufk(@`TjFWgGlpT zvv;@_()vwc|M&)r%Atkmd)@}Id8@k5UH2rzOK>31dr=*ivkviDQ2CNRI^B7b2*SE( z%5IVihvm_u9_(miM!Kq=!1}66>Y0=;#+DRXlF$57(!DT9Mr^A;pb*|E5_R3{IT7>rTMUt|8&pb@>ewgXT<}gE06l!qZyrU~@Wtd^b-(>|8{a z9(C#KSsRLMl+%k)()8fPaL+wW|CocM#dhyr;6=Ch>rK!Fcd~_?cJtccE99YtL4IL{Hy_%z?qpU+&i5O6@ZlNu#K~YGPwki`4`sjVW6RY2O_)mU zcZ(IKdyRkSTfxtdEo>V#Scj$q^LOcJ8xJ>=L1R`nn{Tm>($gkNJM)a>Y1`X91-|zm z4@6@b8LJqydu*LYNgBqVDFPx03P{ZRKaKa6^UzR2|IXq7oa7K7jjJI1hxUAx-!1>v zYAs2)t>d+8+*~-EWoC6!5{P*uq3vEB{6Y-cxxm<9!SkD7X9dYUyG0wtCYL?1eW1~7 zn8O*&@2|%FlugppSgE1NUkhWAzW##ks<0ZB+f1U8MsmjID*(pZPicL_9P#o&bLo5H zIzM-N3@vi(BM|i>28?_U>Hg!;@QbCH1Iq|7NBi27k-Yuy_RhIxWznH6EB&aqkd+~= zeW>dWuE^DneL(-VrmfC1=RQnm--(hNok0&#R{)zq2Yqf-)koXf5qWZfu`le4olbEV z!2o9uY7*=o(@>QUIk;71;MxuU$2w`XvE_+Z@G<78Do|3gw~edv`IHWIe__vgKElD| z2`;&D0d=GH$%zt-&fVL^{UMG4tzvk|)u!hV-cK)E{I=a&$swuBF;Dxx3;bSZj*9wJ zD}9h5J@Dq_I@eF4+9yW7dGr!ebo#01=CVy^);e@2beHXAW7U+E(QcO{7c%(!6Xq@f zINZwQ6*2!j7;YNG0kix+Pr@A#a5>XxH}9>(s1`N?<>$RbU z7+3@fl$%{{3SsVl69(ENh;y|edklf5g?tWtoem9=!~q=hJKrP-Da2bgA~1_@bVja< z`yv5o<3o~Xumbaw!7f+5efA_#l705vCS_z4f4C!>#D^tuqwJgR8uJGcHbIYx^H>8N zh*>VLq9kw3sgNU1@AIppDYabfw+W(_G*4nN1HZmI3`g;8_Hc&j4p2kBw_Y# zTmlj&i4S7q;^Bpm;b0Tp$>@hXIFsREMQ-q}RM0lLx#LO4SCdq2-lh#!7il8UfX6}v zp6VzDFbhqvbCruWcB9Y{abh7#ruDvU@8FsB!|%yz4p0q-dnoHB?%8cu9UNvvC|dKi z4$ajonk8BLD6{&qoNof5=v59!=YAntH6i(8R=F=|pzEffh+Ztf|Adpl-1K0xV)`g2 z-^q+2X_usyNkQeXz}nG>_${-D_F~IyRfdCmsUElPQQtW2TO&t}>Gkg=Pl%zDY9kzE zH*n^4<{@yI)~}Dz&g&`P#QD1ZtcQMrs?ufrbcp`fj#0%$Anc>;1jw>tk=1)w2pJ>?8O1eXlGWdD1<4IPcrb zUYCHXF=|l5mFVuaUBB54uTapIiLfcs6zGbl?>QLM_llo*^Yk#pw|y^g!C9Yuk9nc` zmS?qg)E06(y0=Tc?&-ty(m1 zLGGfHQK;8D$trj4&Fwrw0{_~rK*#P&UEP<(u_-Sd|lj8a&l#fF4hCd*L92-C9Ify*z zA_)z%!`mwn*OH2qV&>){pmFXp=6%NFl1eO`|J-B@@!qVsRuZ)-`uE`AmXuF}_D7OU zPT~DN_u8@RJCc22-N7M&>+lnBYO?8bxBI!F6bi+^?#6vC=$56nwReU#kh8M{jv~u| zVIZ(zZ{Iy!YAy})#0lLOK!cDjT+kt#2|7xp$jfagVdZBalo%XruONVv=!?d@Tz~o% zHfO8jA$>6SJ2#lu`|L2tXV5fvbjP&8J;B4&KoZr5tEJK9!ubOesmsN^$mu=&2e1Vb zv(WUGYWRqVKQ=Lr{fcM+`h0#$w%K#gXjA?+tkBc>TJQXwf>m4Me3}F;DT0(+yY*bC zwt3G-0mtGm%|7SEwO%&fyjHSeyiT7YsJd zN8Gg+8n|AXgz*L?Im*j@-d(drP4`8M`iiEYSDcfd?!?OI+Ykwaf2FnE_a|!A-&})( zD#KR~>PR;N?Twe1q*;(&r zrv1gZJgssSO%&zGlR)9yH~x3DRPAa?&e}x55+M##BwD``w7t=ii17XHgRAQVScGSW zrok7vKP(R{t=0qcJ7$@r1h-6j-d-MuVzuh~`NMgieUH%*Qvc=h-hS!1+i=kB@wt+J zivVrX{=dr`PuNH&S-xM)*Ak`|TfKzP0M+`wCwqU6CI=83ha+^S0CKrzk#sm5{}!b1 zmZaa_W~n%{H`<%fOY+G!1bUf!1Iz<33Ax{V8tu0@jWB7v%S^(IQt0nRk4`t*%=9n3 zIYS)NNvY+HV+_3=Ws@wuG1yNbKf4^k^9uFi-ti>0N=;IWaZp3G(+f>m68cVrTfiwUB-Rik)QU&Z^f*Z zyn9dEbn@#&_IsE1_PVjNz&id;S6u(?Q1?HYA~pF<|GCv+m_%6p&s0S|to+LU1Fw&I zrW9~ha+{zTJ3neLvg)vr$ogHK$6Wt3`T0+W3iIuk#R(J?*^&CgHN7@w@59ejUUv`% zQtbGTYYCB7-4^ptv~ngq=se8nGCy)-O{h1hoPLv9I-8((d}eKBY8|~RCCS`^sd`RJ zZV95Rum5YBs|nb@{~|S)sIAN4f+2dmi7%#_8_71GD3z{%duHvCnbg7KF)yHH8+3`F zWQhFrZ;~sqk;4s!^9G#WH2ueRGn>=rS8m|S1^xqHjSlo79&`^tY2;Y>G`e+R7Od$h zN&P4SmPaOz2g(9%S~Qb@6(R6=tVHq`bjXH(p#QX=gk1aC=olgh-&PM^v0dBl5kLr< zZ7Q0(7wAibg?Tp^{VLpkzt8y~!2Yoba<(g)Ioe+}Bc3b>fueF+b9pWjENzOG;LPkMdtuYF&LJN9q5AnmhwFop=poNr^2uHFBX zGPw}K_~KW@FT%$n07g&0fAQ^~yIp*iyAk1!Y4lVi@jmp^U+d&;yaw<1v$`CvE+%?(HWW@g-{{ys9bKzJM8UVu z-H(I*4?BTpb(|O5N$q`bk9OzxQd^++rPttA3-Xt@f4~3s6^g&V%vlJ4ntGzV!HQ3H z9b1B{cE?`-Q}F-qmxJF7`eM*4Rqz$4gwEv)8JLc~ZW{-`dMV`#u1MBF*9nUa_}+au z-iA=wbuY*7GV&;70|JDd`~NStzB($(u6rAX4hfNNL{U;HX%JLUQVHos8itUrp#`KQ zln_u9=^ln2Kw3(qTe?fSzB9hh?|tB1-yg1p!(#6HoU_l4YhQcsGnbQV&w%wt{P`(g zY`zZppWg1eN0gDDra2~7LNSc9C-eQIigx+kXDR&pG= z)JiWP@Hm^dbLCE?rM_JLrLi$_)c(Ei-A+HKQ!I1%Ja$&PZgqp;VG=FOKsATSVV>?g z3w5FzUcVoEx&1S682PtHz?Ce-}7<8 zWGQ{umt%a#&XOc}Cb`47NrH0b0*b;E`*Pas;$FKL zdK?DHIkAw`*br*Rw?lrj8|v%gVBTV50VJ2q4aG+x8j4##;`p7HeR6|#H2KIk8hQM% zUp8es_BABiA9xUuQWI9;g->G#v1BTloBDc6`WI4#d3RiGOGl$!_Yb#6*`H_z5k~c}b5GUj+H@sU=rQL)V_&?;AE3)M*9`i8 zFv?(vvya>i@B)r-1Nc35vC|R>TY%kBP810xZAw@<%^jn zf%KHVvQ13;u2bRdsp(b3P+TEzkE;S4$8=R#qwR1%+_?E#PtQYN18XVlc<1eM{_&J( zi$%lL1{u&HLiFS2aj_z+xc7;vL$?iId&s%_80L;twrz2)ttv_dD{h!nrbE6TsiB~G7rZ1RQmn&#aF0%fX?ec|frc?HH}(%Rc1PDEbxn^@gu z^DE`wbc<`Kp+!L!nsRa%~#UyOJtDJdaA=S#Z= zJcg8X&@n#X;52E#y0^?h#(MX?>lTCK%&2$l1Kqg;MqigBZU3E*i?uI4$?0wg%s3p% zWBvGiG!>q=GJ77lFONMk;u3Wn*`AP4lchD^x(}~YNgHsjJDXoiq%J9q`H=v}ne6P@ z4v?H(xAzjT4 zyP%K}mL9EQYASj-WsLK1WEqk3ilm#1J|gO6@NAFX!{K^YV(xEL{?=BIet59)pHY%R zakXI|T~!WDJ~ICzK4IFfXYkt^pI^L!s?|hj0n|U~H?vux|(2wDlG!IWI(iKdxMbt1;mRL0D`fg;?Qhw5Oqe#*S zeXL?ZHJDgU_t>>tCg!^qE|)I;a@WQqxng!)0|BDP50s@f`!}F3&+scuZA~9kk!tl% z1t%BX=sw3Z6{yEpHfoL*I8zLa%a2-8*-FC_GZK=l0+lkfU&Y#N1HChJ}QiBi~MusHKW5}OxPT$o|;TP z(7h?rqBj?7pfNonr9x;%d5%O;DHKb>L%Q-BQzPB4<9t6p>-U zOkha`xrf}w@=a0xy&)bt!?QJqM=mV;G33Pk$e)JbNZOxEZ4XPv>zWL+K0sqK>{DN* zy=LNnd%UXc?FnJ9JOf4l7hF@vyTP0k4C^E2wtP+{PHPN*ut1^YJI~a9K2EXSpfJMn z;r$p}6=M1tA3RYP?doTfuBV$E`t}Cl@>5=J_KY6l)Xw+$CH{EaD-v~5gsT9-zOm-RhD(Zq2nhEne%?Gdb?=z$k+qefn``+WR3fwwimr zJVI===iO&|d4oiNQvRPukKD$;VBrj_<1H zsNvP`(aaz}KS#`)+h3&b znaI#2#b0aAk}oN?wHNu_s5k!e9$^eKvT&96o{zf7@6Z0d2jdoT>^B09oJv*rd2tsL zXT&!QU%T8@&@90iA*3li+#+;4ZoHRiT19neoukeh7lGf{tJJ5NpvO?CpcyrtLKqiL z+*)eexIK_Va#|xy_;A^i^g4OSemuRdt{_?eLBjX4K$1saZ5~&wNClMi9hxf32+iCN zRhWXcY`oylesM#_M>iZ>Z?pNGvV`Tjz{J$Kwe8K+R}~C3%dxJbKChL&BtZKQ(5oLH z@2V3ot4>O-DKZY*kMNl$i+c*PYvgGb*=giIfn0BMMVAO%kXa?8*B_2CMvvQkD8U%a zhsOS#U&7Vk_IdSuuZrK|oQ$dvL)lsAqXU3>xeozDTq6>nFbJNo;>SxNLF&1-E znd*InPxnmkYU06gO+rmeomKbE)Wic`j69jMDDwis--qI_IS+_?TrJF~hGxeue)#`c zTWiu#V^~e03eNiCbWs&xBp8qBym7bC%6;lcS(ulk&mU|6ULjqMrxB zAB$9fR@g}ii@7QLNjXhoxYV4uC4xF&e|>DRQ!LW?g~k}@t&bs}!1)jqk8b_o>p7gD z6UcaXW4M@ZBMcAPAqYz@`o7OA^KB@yMNfiAbDu1QNO?rxlH%MZDg48U{Acr?WCx#4 zf_q+bcM$!Pe$}s;{a0R>vU7}`*c=H7J-FLd~8tY@WJ zCK`3Nlt#`v*`nPCVw*FJwjXFL#;}fjzd3q8GKlMi%N%Qgw9@jEwn*-4<_kJ)F48f2 zb;9t3O3Yhfln=Ld(nxeOs2g8K!bSb*`_O*mrc&mV~H@ z_uh6qaYXPMSQbnRq>uDlKswWV^oNkLw4LLB?bJNQ}R zc;xoG+n--`C9$~&EUlx0GNoE%28^Hwia;);b^n0E;w?ioj8l(4e5}05&~w}~p67g9 zFLAaWC-O!d!)hN9lcD&x;lY#UQ6Hkx9dN`1c5tE~rzmUwWb|&C>ccnmG}-^mb;uVv$hHdfS~iGO;u8 zQ4~lQ(pG;~`s{RXr7LUN?vP%ocm?{PE1-YNYAht8$-9tDFlB=+_E%ChLtT#=nzGsXgDX@mH0t$N)d-FEs}jzZl!P{lGYY$tgBM$T%Irh2u~=&zxz)nR{r^8>O+=e9aHB(6K0w&fS9@4}m;^GN>JHeE7Pe}9xRO5T8=AaPj| z=#WwX8MXP$ukDBreCEFgb?hG>vBXdv?C3$QTw&FYkd&X-xcGmAOM}-oOq_M5I+;Pe z)^#e$6K6Gob4Rz^b0T&V){847FzOlaY)6!Gy}X7bSHoQBTH*F&{h9LEgRZ;!^xV0V zGzP?(>${DWJBy7I@Eb~f>{5?;Znz|#VX|_Tv@g_ZZ`b#!I-juX(BbT%rE zuBZgUtnn}W@BdoT`*F)D>* z!M?E|2ZO-tl7a*uEQy||&TfrNDkB!EJ+bJ`?T#~^mWK5yyrFVhf{AnV#7m=!vWz_6 zsv#}NLSB&a3y>9@1}g4?uHtaa|h?(;F-G<7ryIRzhE1;AbhAAl`7y z@K>Va;)PTjvZnkNSo03$M$dX8x{0v4w-G~DCxM?xsL~)c-x%GP~^yutB zuHkLgj$@MAhtbX5`9<#4*U)1SE znseM&3p|C#ZG9ouU)`)Z$EZHYx3J{&nYE2q5q4>pH|NTPd@{?zP={6JUdBWJ&yo6% zu%`I=HNwfH`gKtDgHF+KJF~ENZ^4sP3qeJ3w#7YOn@p^cp8t{1bpvXDQGC7J?}r+X zg~>OyMaOhqGSQl9yQ*^evUKZKd<2K@e$E)xvtpWc|GG*3VG)N(Ht46Y#))#lih(ad z%J*;itvNH9RSen1n)K6&CY0OTjFN?Nuc*i0N8@}>+bt|U!+qQ`s^)*v886v7s<03# zPCFe*i~FmRQ0a`LR{7%SGv&wEbT;{Hwm**A_T^=2?Vw&TI>~x>^zmhTH+0QCmqCyI zvvu2aQ+lDr35P8)g8H5P^6FD^dv5-tb$TYFj0#yj-J|iLQSHy+jd%Oo2u5>smnu0= zuYViBQ!0l`y?~S?Mn2RGAB)-B-L=>oUZmPNmA9Cv+Uw)as^h+t+PeDVG$Tl#FE9Mf z-_l^BhCL2a>IE{*8Lm^F)eeTv+}H0FI8aU3@Dvgdbw~fnXrMW4MsBuOHv%H@t#h84M-;)y2pXmGL5la4^0J-=#BWQY?^b9>kek}b ze|R@v9yB*;FFNFguHW|z+P%K+J@Fb=BsbR6m7Qd+T>5K5zgHt?vcb^iOaW0Zwz*!a zQTO4&MSeIZ+M{9`AWAMO?;CzcU0@rBHCPT2%jqtccSqcvDewMMo;`Ozs|Kn>5)V`V9{JTSilBd-9K)$D-D2M4b+RO@4x*A>hU}{`m8BoTUZ1 z<3{FUjr-0iY4O6-n`;xH(SlJ-d>jPyBF@MFaUPkhWFj?OinwIuRg~N@q1RJP*Ad2P7w+|#Hd?k*Tq!EQBFkh;0M=;iP4zvIo2^gm__CPk_;OmXMsv)wp9 zOvp2)n>R7~`ud=Hl%P)tGC|TtS%9FSyK-co!s+FialBn9*R?DMVx)s#rtM&NANzQi z+J0l@mL|bhH^Rl5F=Dv0sdf86Jz8Hgri5G$Mj3X+1@R{;^+=KXqtdy4kw~zsp%nBeFFObv;@Szs!NYhcT6Pw11a;? znU3`S*&Tjcyk_-X9NNLb1;(`7fLC2Y?CI!ldrnhNTV+qKQJyP`8n zbdX2n+w4<9*C_%#E3^(Z*A3%St8F-=Q}ME15b#)V4BfahbYml;B5(aGmOtSZm3~1I zZsta54m_H7FD4-&xO23;QJgYfCb64-n!jE=eS=3){XD9a=d4iNK z=gZOYFr$GgMyt|H%E4qM(S$fSI8c<`;(Q;vw7y#_7;9sqSxH_5Tqgn5yx%{h^V{6) zHwCe6=eyB6_yen5M54&!;KI%7b*faD>9(Q zd;0Kva|6ftS!3nl)a0ty;?s%s^_T_gyX}RyIDx{JhjOnGJo@hGnT-|Hko|K70J`VP zO@6;>q1?-x>a+KIO~yJ_)`>QRTb&Bu<7RFKsAhjrpA}BA9%-4>cgpn*>Ic-DumLw! zee4q6nP0Wzx=oFZMZSZ$x+5o~f9jS+oNG5LkyXX~I)C{ExVSLTPmxsebSnVMa(jE* z;@7vBvl7`#$N3_=36Wf@u^RW%{9C-d*Vo4?S|3b#y(_e?2IISIiEcfMlL9HgZ4m9u zt$H~-{>xVQtI}aC=|x;B1#t^^!C|CG0rFFkeGm5!_f4a?+#$%u#%pNOJ|@DRW=ytf zL_haC#()8cA!@dAI@B6_ep>7ImM>Xk=s2hNO#cm#9B_b|T43}F|q8yTP>!Hy!0t!wv z$=3SafDE-4FRt%zOjvWV=)i68G(sIVHGCdDems&#)wZ`fL~LKPjXj*>&=h=a`rOt= zRNUi_1*-F2meGEzYg=e2E&+xXmVTX=JEwNp(jig$%fC@Xh1yB8DE5BsntN*HlFO4muF)tVuVUPzj}4M+9{9YaPSNon zEPWjf_z0GXyfCXj;Nm%wTqg~fjy4Q5&AQ3TLc?r_?SB*B{eteg|8xW^VuU)Bkl?YL zt|ewj@J&`*O8)~krc5>4KujEFq1SBEVd~P*4x+%oK$c85b3ZL^zF{0%Jxu4)veB-wiHG8 zqgVSrhlrC5{KvZEsTj^Q27wlbl4}7OG%{uJ951!ak~Rb4E%J3L;41PVGo450ZMuj` zeWfIVRp|L+k{c9wY$lQ^MRzbOg`p)cW{PsSguYMKOIkAGi?;7qO%K1=*V%aZ;e<3U z&TsnA5jQNXA_RKNq=_UhKI~7K^Y^$rQR$)5lh#KuaLR|GFPXksm|=d|lu^m}4SnG{ zR`+_QFKE4Dq3MMy6F(dI3~3B5j~eF8tsEAwF-WGp&KqOO{l>_N6P04)$AN2lA<=K^ zZ}N*>H;PZbS*9zKi%Y0D8W?@+Gl_N;(SDr0HL>((8CZ02lxC2wN7cji0ulWb8fFnLXJ;dT>Ypbupq`|qR&_K^lYN6qJ{ z%fz3b#k22kKHs?2>OC(cyD>FbkXkX6TTi5c15%zk7QyfJs+_lnvaR&51kh6G6e`0S zemn&Uw}EyILBegv{082UlzC#Lgi>J&)L<(?@1#VSB7D7I={c0{wu}$D%4>)6xJ-X) z2yq<`OITBMY~M7I9Pf)l=DP}v<99`uZqXucSuwLY2#gOqw%r~3E?v9!x}Mp5(PPe( z^=4M%QQMo`&^1MZ=<;d#xnET`M!wK6iHeS$3|G*9=xkxm+IifDA61Iad-DF`RtZQB zs^^BkREK-=@lr-T(fIIH!@DJ{EQpWovsLjuQ{fB^vjpnrzdX5e2Y$YYF5hr94Q^(6 z*GjViCUEwHI%;fq^7g*Ue=F4bsLzBMV+wZ)N4a{9p$=J&e`fzcWic}Hc7kw zMk$i`MP0LhmiuiP1%r+AQeMq#XDOv}_8QIa#k7Df$@f81WP%ayx}%RPzT(jRy%Casb5 z79gAXB!UVCBBZpf?LSZc$5CnJNH1Dk;t=ODUD{Z zXIy^Y)uD+On1N3k@WQd54PZSRIO+UgV^G>FCnUgKoDeZwLB3JP1#q~r!mbD7m5dlM z+fZY9jQONSJwEc`@|Lj!u1{MA=Jl2gBy{+31-@Im;}dRFYrM6`)>$rFlO>c~_q@HRSV6|H z=ZU>?Ma|8-=xyx~{6x`1t&bNK7us^mDXv)+*hvMc*qrA?-R>0_&DL~Y-i^QO-9NUO z{qp?l8>CMY6#aJJUmKBAR<4*1R#&y59r^bBvx)X8pgV5QgfM;rx$dQ@dP#}HX^fK- zBn)GFt+Yp5=LtMI;_X}Uud}IdDD^mz`L6rJM8cF98sM?!k z>uo^)64zFJ(Od0me|k`Vv3HPd?ejnP2*gYpQrwvdpf2RF64^2$rAC@=F;K6}|C*SS zi6Fcry*=CE@Jye``*XE3KgkXJER9xQ-I{7+NOVa*!Kid37#Z#o9XB!UkSd~AbL40< zqtDB&a=S;SCC}OLLXfGTajK_l8oG}IC^0nd!N!P6ABhxe+_tH zka%qf&VFd8x96*{pahzK!fi6rL$GLQxA0=R$KFco-o9e8pqBbDh`a6L?SWP)wSsqY zR}2&_K@*edzk+OytM1?Q%8!KCPxtVM? zr#J>8H*rIJ1#)|d)WTA?W?Tv+yo%fq7`&mJQ)_|BUmiUBPg%l_j^BU{;V7y(IzfSE zAKdrCM+}sSYmmE6ydE<8v()!q(3WO+r#G!d&vhaQ^fe#W|9UX#Sr}xuI+))C_)LAp zCb9&D=cl{MzDYYqi#w=%AehSoS9yCZ&h{dc}4H|2fP?W3Lum7!;(1 z`||UXYo6EWbJVi^fk7Bq{hL<*F9{n3=>Z-^U*MnD|D!_OF^7p_nQ3L{;aYQUKs7xt zeWge+KDfny>H{4R55nz3hgQ+4#E zuewTdBz)qybSO{}*`r5+LD0M~RMeLG$?Irme~;I!Lo-=1*u!fb%<0@bxSf=K#Xo_* z!UP^BwR!5}cX+9{{*McA1x31p1_6DQxj%y9TM)He8`5=p8)Qug3Mh*7s$D8MQtlZ+ zM4XrW7P^vk2Hr~&VXNd}S`QZ)*;%+XU3*~I;E(0DnqQHJwE%iH6T>Y`t)B)folDR> zu!{f(r^VF56cYgU?!m^0MuDzy4dB5Yf=10I@%D8M0=p)_Rnsi9Af@2Zx3Oi@T5d`2 zdWqiVJ*|1SbwBCmlZY)>qAuktFDoPi5;ol=WlA@%NET)RT5dXmhxXv83UFJCUDmaW zV#7{KCcq>VUo9$*Y|SpX&LU9Z{L22&4Pu|kZdFt=p;n=|@r1A?R=lBEyFgMV )MRNTYm0AdM?p&)b?nF^33 zWOUIQgWHy#tyq9Oqesa#Gp048#m%9oyYK$>^rNL7k^wE)Tkvi0>YJzVqH8?3Z#t`Q zV?*8U7skcPQ#p5?V3bN@o*qnIE-x&OHw7*!*YI|0gZHq|1Gws+a``#2N{!$kaJeb% zs@DaRML5P0A+#Ka{s94J4#K`4$T;@d)xP-w_>d0ezy<}^HFT7NsHAn!VY``f7QB1W zYIASCX0fqPy`ZAXmHpjP21LhmcRhZNE0=&N4GWP)-NY8mJDz5})SDrvpimagW#1_` z#s9!|)U-hn;FWn+VdTc=BAVOHM}S_??*C}Y4LojpKG8rs{D&-q`*#V@_}f5do_4A! z9c*nb5#TDleW~x>?B6$1YJQPl6}>Js@?;03d&=kUIh#zo0EFoHWQSk!%9|jGLCAD8 z^SCeWO8g5uB{Lk83bp|g$BX zd4e4q8%sh?9)HqNSa;kZEm$yLe{p8NvAB%dvynxVYh16NZ0|@&3fS0&>cP4<7kv@i zO~jI#OM?gplibjg-{CxK9y<)Uf$$IL4)^(=Uf$UoP;l~Crx0uRE7lv?hc&FR)%9Si ztY{wefG=4o16!%p))H2J^DXFQ%)-Wo0|4j7tTZ31c<8y9&&8Rmb+}f(wm|Mehv;pk z6bsMFfgJTtX|}0hgv00BIsjt8eRY^3z}$;5?Y|aF(!TEK6>t<68M7wqeSK>H{DhbZ zrj1TbeG7UP4`|Dy+StR2bi1Bkk<>O>@V-Z(g{oI$>LZRK_uW_4)JG`O0d5d3F>&i^ zVZG1ygcx`8(?Z`f0z5oC7s8E!7N#)r7w2MB>~)|`Glj&_zS(Ye4=^`amkVtTO!{w&)Zn%L4{hX}TU`*8WEG7Cs08y|VEh|pX2T`P~ zfKhS-n1IsGo~~VH0RzDj7SL7)&!-Znco6IGv3sw6P({#-QUK`fxYHTMS_J>Jh6O$M z*Xq6AdM1NedtIPob@nY2pTc$MqIq~5fHr;uBMX040n)0UY(`3CQ`m=E&(<{YDCz0n zqGModPJ{l1QAe}a1=y~p&RF3vW6-9Whjr^PITeZlJJxgG4@Fuvo-{`$Vd(r~3yR(W_tczCJbN^irx*bE(G z<4?7>Rj_kMG%Mf7#u^W;_2$$z<%1fXdeNT>oc=YAIoL)}?t8bm5v%Apc0QTTF@R+4Qq{8LQ@glQr zTS2BU{*qzv!^Hmvz-)+zQ90-3SC^YkN1klf+dmuwjUY|($wnY0;7D7?d3RJg@yKp7 z-M!mB`a~&N@ET}YM2x!uD#x^v+kiU7vSD*aJV!u{SlQSBkM|lgGqbPfQ?QAo6cpGD z9MLUXGfmq+<$1JdpK5$P8hysFN`vy`36`jS73fqDi7{YQfU9$+Nt`kC^xusHdy1td zWu${%Tcn+E#lt}J0)rqAf&0QmRXrCgf6u%-W!8?`T}T#NkZ@IVxp@}5;}GZ_P{oFA zrk<-IzE;Mekr#S)axe=F)T1}oZ=LVcz$XAbjR$x`;;7gjoNhAP*wX}VJsQyFysS4S zU$PLir!>*W(0NwU!6hU#yuWN>b~8X$<4MV=;?)eekSGdKOC8e71&G>zErb+|((u+8 zPy=>G&w}vD^UUVN55@xkP_P}Bmm04KxUyomT18ZGjO1F}H%Cer8I?zhjAYg(YiVa& zqh#~7%ZW~^fTQ*Pu1u>GEn)NU^&NmAwSrz!)?8AnLxsM&ozvpq*(*gi5E!@Q6TY{_ z+;nKbqSS28oE2()%~|^&5cvPz2GvSqEErASb+SB6saxn7rAuc zWk;pIw^Lmw?-Os#x`>(de^u0&hZ{CP#EzFf9|CKf_vMa`e97Z4S7QR^YmIv4mKC#hYz^P=4^nAN-JnL9tz9= z4Qh0e5%GEzH%c#p;36RZE`}|DZ<-(v&X0g0F>DSaKLi1hvIX_$CvO`8QfC)$3(n&H zN8fYzUjXxwu3%TsN$I`6cl*kHU@8KxEY{%OPn)Y+s0kO9DWWLb+ni!tUS1yLb=;b6 zfF~xjJcRZ3s+|7mSEj%}>t7nkrA**4h^*eIB6u+2+%H*I;CZk?QC5s!0PsEZ)*miO zgvQi}z2*MaeaBgECzhLRY^^!th^UFu9tmsaO)!llFP}Ac@@mik^9PYmzM+9ZzS3oN zgr#i==1&dsmH;I?xU;ohF{RPj=8O0ZWJUJNvv_1q+0C~h3?8_s(N*d%K?@f^*=988 z@Sf1TTLJu3wLH!HOq+c0*c35$UN=TPx0xWeTAT@RD_>>kn3`s)zw|lXZXGT$3sNJHV+a$@YIUheXbK>Gs3Iub2LB5 z3pl#uFv2v1E3M8`2Fs$?=IIsBGTY4(^kXVc_Gn^?C4c#aSl^HR@vE@A4b=GNA< zoc<-D|4Q&*ZIK4VX!jjKhUFzK41R3A4IQ(s7&+Ij7|4M-~`pL3Jq*&UdAvTpo^7{ zj+~{g3djWzkomOQZq7>*3`}nbnhtG=^c`Vg>O@^fJc;9_o7P>ee{F2518eOcISA16 zsN(!QE$w(-NArn;+9klxK@H9D+o3%e}zTD?VT$;~{@!LGj zn*tR+(jBaiQOX1p3d^gP*<4=Z4LX#Qwy14&>+);dfkJ?Jti%GHR>~eKC=j9)DcISr zm+qj09o&~1*sb-7elK|G>Q;Z2a)lp4B}3W`bS&NmoZsYaa3yVKlCNq5lJ_WH+-{ZS z;Fag}LOK(9Dxz&Cs$LJW$L1eWA+pWpmh0EbHh*Q4HG(`OkoSJy#?mc9nGmA#-nZzq zxswPSqgs!>ypDXw`401p(1go}149Ccm11~Ex%4t^SqesGINb|e!ErE2PP5vD%VllY zh?(}lvR~-zFT(*mp7h2I({&L6r6{xjQ+R?YqJ1#7{^GoPeYWM~0w7xw;%o!Yud0cD zkg;4ZD_TjvyrO78k%kuD(>PYk)wl*sXfw)>WgY{etZz-o&sAk(oJ#}D%Yt4rvil*+oiI^+m< z5p+fx<>sRJ1|lwNp}=u$(3a)3;Lxl3Gt&KzXLa$TalxuFP^$3y_btohhTq!LZ2^v_ zF;785W7V)r{!-Uyz%Gb@Qlgfux9{pQ-(6Q3c6s6DQTng-1BMXF3jw#Qv~~per?A*t z^2UqZsT3u^A>1$}%02j)DFv(|%Hf{!Ik|Rf{+93NJ=;1j6^~M^Q&=bTmmE$4esM%9m?D`N+9)*UJ~Y3W_km zQ+EhKJ&a=ugkT^>DKwBQvzr)&EWdwGmFsBz>s#N4)QD~XzF)dqbm{=3dH&|lRogkO zhIbEX)V2@gc^1TQtpH_0c-mM80n2 zPi=W_9r%wrR|dCfbV3HP!yn3W=78IAX^H30f$MHZbonjhrXz5@Kp>!z;RO6UV}Jnd zXC%a!2c9F88_${$24ldeMS|#v;oxL1Sd|UQl$4gbZ;>xR}nC?W%J2*0Dm-?MUkys}^V!Futc!O*kPn14@9NEC&5 z(GO|mUOq-7H7Uwo2HBvH%#~zRW~h;7L*NUk5Cp_^x9bjn6Z|SPU<8s$kn$$&F`r(o zM>#kmWq86IFnSXOH9OA;sd+Jri;K-;f)lQ|NTz=J~Tzu>y=-4fFjVe+annT{J)<$ z<7HDHFZ3UDDg3*W>4tcBY&bNBRC6z_IT}e#2{7KIARE9#g3{p>tZKInf_xR91ZC>h zFI-fZh!O;V)!%>hTQ^)ZV<}oebXiNI zze$8*4NO)QJBa`PGB045%i*&)~UsA6Ok`#S4%AA0PM}z__KGEG&&V zHv}LuhJal~Q3V9@2m2Tp@oxX{5Wo@<4+F5VT3Gny|CM$?CP_yzdul9_N-l5bj*j8s z*33RhJwQb?;WpIVwSQiWgK+p80AhTU6&EG3AOQ9y+d#|`p|bOP1Gu1iMc)Gt#CSp< zeU2^vtc~XvMOG|7|vOcD*_S-g9^J;b@3no zA$Q7c^nZO0#RLWkHd=0dQ%HykAb%V}1ft;@B_4ZL1mlARdf}i7d$fS9-Zh!u(l|p0 zXx46)YnLwr$P>RrRqX8zT+!VshL|5cR_=}w=Kw>bS!4(cvH>8yJxnW&!coQVWb?kl zHGEXlr(g?+=5)WZsxS1tYU0&R&EKpz-&?SqteR z&+2fuJQLOJPzDgmB_cw@=>3NjrBpquQ7-BcFHfr4d&!#;5|a?hU>lH{muNQsIdl3v9e4>KUDXQmFOeXyKAF8CkTrYbJ+E07y1wcuQ-t-2ovK{~WUmy%2MI{opLDVp7aB-X%v;tz36688?SGf<-v@@=* z1c<6CVNTQKIGHIL>#I_#74hVVz8#Pa#7+mteYY*c8(iA_(TQIw)BKQNL`}r^?~iKE zk5>^(pTd=5(G`H;_vnX0>UKzPKG9rqS^ko! zy|fP?#GV4-F+~Y5dI!sEPy6GQ6~0BSaQ55h%Xgp25G$CN(T;dbKJE^fTr>!TiH*$nrGwzoL^5Qfw3mE?oo(aB&+z zM7oiqvY&s!+iR z@HnI@>?XEA!}#N6PJw*YyY~RLLr8TGnWIBp#br=8?(+o2&l%K?B`^FZ1Ne`vMNR?W zV7cHjREyNQa}T&bHNb}l8H2}wn>E0@!&!q937-QlJ&{REM6t#vLL-502H0QD!30Xv zSo4elKL+L9Y0SF-ye?IoL_7QPo<;(9V(h;3vf1UO-NA)?mW_hfzISwVEHdk)$knd| zkHByISr)9yiM*G<^@LWyDiH7@Qh$G5g`RU40cCTdh01fB2tTiJbI5g^J&z=u&@Q){ z-<0n?y}Xyu7L+}_ZHMmmf=bNo z-v1dz)EH)i9`$#k-Vo5C^+S*y(5%_zv2(TeEP9#7#$tS8|wH6zYprnTn+X5HgoiMh?ofvlDu?^thNdLWjT;mWUUijrczIRN#o zXB>izkY03RUU{lCtkP-mlWY`&#+eW}NmVyfdUh48uL>W4%I^F0czyAyw6y>FvPO>D zU4j|#9c16+`)e*Q5c&cvP$^8={<8n%cYy~68fXr*<1l)mwL0Xj1P<5$*5&yFt*$nE zEX2#9qs27)haLV{1g9<^lZHP~x`A8B=}apGWjfwl4MB;K&jOHFG=Psu?vnkazWl`K z9+by+1X=oXX@%E?;vr2T#H}cMy0rxb&_Q30GB;w{U)3F-ewC9H6thK5$c(=WCs&}k z-o}AKbH-i9Y=^5|3(r!r^=qrJ>_D2bl)VBVH3XOZ3@L~?>hhN^ys*x=^f<7gz>7eD z>99zv%wNCM#16)t5CAL1--#`U^D&Xo(M15z1=_=X4M5(0Cy3&Xe3a)?~Hl2j}^05ODP*`W~V`=CoZz;)S362AyvP7_SO2bZ0m zFJAx7B~Kgu;yRVM2R|whD4Jl@K{!lh0!002h*cojj{@)}g3Iwz9@sNWUgE49r8e+FwRCyuVqY#sbqqoN0Dc9=c{ABZag`M8rr_=8-^?i=a2qwTl zNqa`ZagYV4|FbgCsV!e3kbNb9$BzYP=kF{~PI2jR5S}e?&1X?8H#xabafA;jnFdyV z22yXwbD6HCn-Ga77*ik=QlAE@CZ(pQlyY0;xc;=#VaeB%!&~B0h0=04@{G zTou3Nzu*4toQA;I=LY}OtagNbk7bSruFz4z3E7}22<5brRXvkHJd5z_GE0{#`= zpLvV(Bc^+fb8-X|0QfX98Uq~JwH^@ifM-%5u4rpqRG%W450-%Mzb^m+C8T)NXoYFt3faW;v`Og2&?|NHX8w!^%cH1@{>qJGj zrIKlY2?!kh?~Ov&{{3p;m6m}0l&bScy5!_b!6**F{7xuP@-WyK0D}qs@@BYttp|Uf z%KHt-p~w{=*&6D5wrX%LAl!9(8dEc)(=Z>PFBI^{j~;2p({A3^sNAL`xB@DYgrYR( zc8+ED<$I@Xxcu`A0+^y=0OhLXYm>D#o^;VJ3nMbap4j{^$ z{QxQ6R4hAyZ)9NEStWYrWWJ{RcUxh6DE5oOomIL^&$%uXhJ1hD1PYSB#^7vFL{04{ zE0Nm-FhEl1W{*M<3jjJZ#j}w46qKm{&4gBjb64R4Y7!l|uVxu2_X)st$%0&(1cO$W zOd)n;{{bK?9`VDI7l9#-j3$Y`YtE>>{u!$D#eXNjGEU5d;NxW@TE*E2#G8d0;jQHf;h zc0iKSTG9Y-ETt2>!11p(Q$d1fl5Y{@%>&o;qG!@m>ZSUZ&5NEouG-jWC7ujc63k4> zWx3J=K@oOvuy-g_Zk!CaP8Oqm!hIIQT{}bq!MjyEG`8Pp{ zDcn<82n+m7_ECvH>QPev{pH~lUJZ!J#Dv!zEAPEtQ5$#a5x+Q5G9#9r=3ycsAwiRr zl#GjuJH?WHCxZ){#Z$?bx;_DRtXW#e$48=iJS;3s(|HwKPbRXG=qB`38c&D*SF`WMB6Ri9fi7S^XbaxfpdV*FthEJ9w&m)qoTCz z%p~PM23+xWf7*f9C*Z`ioU1|dP0N{ph)9B7N#^*@KfCw$`=PgBLoykJHfp>dN^_`{4prg5x^wfaB=ogtE>%riJ{>(A%Wx4z(ar;X;67 zeVYHwZm&#vh-u-~{r-MHS_6u1Bzpg;^XJhItW^SbO|a9|%)@3l27GlsxAA z%56#!#EgCSuGn=`A3#nHm|b|2LhUS8T{-a^IMK#?{F;*ml76(5{^R5a6L-b%ba zT;tA1U^72IkIJ6hb6L|oKLIICqXt{Rk@L0moin_$B6DJN(y{K3)?DW7mqiNdj|&>K z#k1MT$jDSa=^X%}u>U@1H7O*E0S4w_(_fq}N;(XvznrN%*$~|mxg&lZav?okwOY_r z_>DtC%e)7Hly|!_vYkMPWP2tLUE8H$}T2#^nmt| zo)&Q%6-WSS|G;Kp4+aIRhCH*HOgpp7hdq)hi0(Fc&4YFUb;Tue?i&3+5dU8*uztdi7tLq^BQ| z0G!DDCGDH*tSWM3kSq=uAchoozGb}ve&x(sAB#_juJ)G7EY7EDk^{M)_ z$3ogqo;UowqO${$o43Gq-@J8u^~&@9UWslN2tyPOQ3wXAI28=(>-of&s4D&rPfcAN zl?*%th_|pdihe8hPWffG;*~vDz74)MjZo8t2lMI!{YZo+@B)K_SvWZfM@B}nPO{}L zf{+fFsL~0*yTb$*oI$(7j?4M#pCK7pz!x$$-ln~>&;NN(28ddzEzpT~I5b4pnacG6kpIwTQ7&rKz|LzB= z#`Hmw|KkDW|M+(QYYTps_`4-bI98|pTfF}*r~mzSkSA42{r{iO|DE>xAjqsz^G zTd7<2eBSZ8VN#Vod-d3WC{_zCFMQB(lC*%XZTq&?{MM{#2fs=1+)__hp!^%BuCA_d zY&kf2HlV}CTSw>rZMZs@K8p9&*W3(`c>eRoZQ@eJrKX}ZtVBaL7AhttWVaSFUGE&8 z>^l2<5B*7H?3KW+GQG3%vTu4U>zTdOj|+D>GeVkSH9AmfYnWxa4YHb=`0`PC=WLUe zi{NKJ?0c)G^BtULRv-gwjQDGNb2BkCH1xO3-}uYh{F$^jR~unw&Gn>(?9h_iP?2~7 z0s>)?kvn!r!m&>aC-B1S)4}JZ(y{?R@v&fF=c7u5;oo85hei z9!pgeu_gGxvdv4c6av4nlmXb1IheFDjF#z20aA~bv4+boM2bxZzb6ui+A%`StX7`y ze@y$|X-)U~TP1JG!>{H488J(|07yD&l|Q7#aM*&N^d=G|pz{y2aQ(Uc*rOYB-wARo zHa^bI&gRmWvw%P#fL>~B>N9L%{>|z#fA!a_2<%x+t*#@8B?N`Yq*Qs;6M?H4H{*ZZDakxIc4V`O`{$hmLTf1IUdc|TAaxKgnOU{&73 z&!M^&QzPQP0T>_eNxjOM3j@CthS7@ya5*N%$7Mf%W!)_PR?HppB~Em%{(qLh|Mj~2{@qu7 zG1+vq{uxEd&0w6c@Ni=}O$iBT$Z<&+ZWh1WPyp-QIw7=^-RAi@=G$=31;=`%znj!1 z#Sb{Yy>Eu&viE#q1ise|6$-f(z8C_bqZcjdgs#1yY1hx^jr8bBeii7%@3hgdypMJG6~K&uty zJ9&WM+tN4J1|QV(dY#IsWE5#`o|Dr@=)o0OZhO=@d)$`&>wcA-Wemi_=Ri|vx$Gsm zdE!H!a|zdWv~1tW+m&|M;t1_pwG4LX>IEr>dtA`r<||L4p1)%i@Fs868-|q(u_?$#f0#&BD>^c`8vTUHE0+Bp)@t_8u6r!7dGR^l z6Nni6w(@#-Edz)jdvNbvU?guWE=HlFqhn-em$kNL@*5r=1^|J;x1z!In2&mJ`g3@M z-a|hKMVim}E_Sn;uK#!2{=eqUlJsxK+4zZ>f%gCGR{=ATfAOv+;7UZ_Rn=9|t9Z** z&NMM7_^84jE}>cShIoRZ!2JzQ+ZLc;kTXL7yWE3Q_;~1Mv_VbT&ZRyaVqt(if${+= zLKng-%BjE*K6RTT(cBomn zx_|SlpY9RY?(TSN1|vnbGqkF{NMwnBrFMUngMU}-DSuh=CN{R`T*dDMW}u_nlfDJs z!f&S9ly^alf=&<*(+dSTWDs(511luV&FSRew8~zQc;sISBakKthNmN~#IWC~p6u z17PKqvyV9qk|7LnaU96@a8;I^`z~9hrh*g}Z;y){&A^Gpog0K**{GZZ$hxJ!9 zbX)!a|KxVwIjr&$%}bwhJ}zB^~a=;6*bU2pkv0T?t{^j?vv>i&Q77D)(%>_H0~7vMi_K z^u~D)4Ko9eeL|hiRcYjww=E&X3K0-xZm{!$Pv%c7*w-uldEyqJZ-bo|7_Iv6&2`+yj6?P3%oWTcQ{_5eC+WEQP*X$Zg0uls}k zj15Sz@RZl#sWT*@J(!}fF~{a&QR-i&sz%A3)2`Cq{*vX#f|?CWeX>9UP0h?4@g}Ot z?{uz%6kC&e0^dxQbF&Q}jxnAq<@j^A6K@KewtMn-eKYoWG0H561r!mRJHKa~Qy{x? z5<*gbr)_{<)`o5G4+=oV>%v?l!GmH}A7A`P{e$ zgo8(Emf5jkq4(#3{Sm~Fhe0u2G&_H=REa3lg z`0jJ6EOjoXDobhqo*X*QX)ktzSt@O=bk(iIMh)s} zj8ACvqDpK!U&Qs90d8-`e}7Hx{OrsHmhK_vq9pU(}AbIGkcXhE!gAvT#0=`OXXnUhf` za7y+R*oT6K{bwKI2>Fj6c5=H0n`FyBlp`Lb?f#Y@p`iF+0yLnd{aC#iUyvXN3% zb~aAIZ=n{QX$)%J`z>|;Ld1OPe7JU{G^UvUwgw!mn~ih3GpO`Bf1yS8 zz*0h=0M`|WiT<8$#>~dF>m5lMyWIZ7u>}z{n{vu7zsXWUB(V^=*bUCru4t9uvvHlS z4j|iasLBL_2xuHQQ!PaHVN`^!vaw&i32Md8<%+{V&Ecto>#4k>NFt7JbZjrYRx9IH zB7-#^V}cEw?>*M~;bmbet1An|bNwrr9y90jU2BuT+!h9p9dq`t1uyYj)#H0TcXp>2$pYHSF81qr?iI%~jIrM4Y< zE3>n)>wOn7AwtP-1l|@3KTA}*QEH1XU$|}0J}xp}8DXNymb1J=Mh>#u$l+LpT$Cy3su=4yD?-)=)WyHZdyUd8uB$Yt+F>} z)LnS+tOBD&58S#OPI$X4+&g8*StGxVi7g`8?@k92CBE`@k}nhTIS*UUzr8=>} z?L8e`Snpuuw9%e*Z@2e|h=>kX1(AJiUZLN3U$(M@34Of}x+#v-ZDNF3oLDAwU{&0)5}0ZjA=$t&d)jqHpa`rG*)LlxP& z7@J9A>Y-KWoC!h)l8LC^yWEtc&*aYBf94%>0&}xVP@qdF)XmfmpT87Zw7qxg4~3{5 z`e1CL?Tj9ii=EX%n&D*hj93hD2fLXuqy|IeDDpu1?`e5rIgr|NBx?}vbBqxKQE zQg4E=R-919HBR|QhFORp7JuHLoK%v#r1d9MfEo;c$>M2$OnENsWJjCbs?EV`8xU-g z69p=cuiV8!9u%qHQpS!_b{|`P^m&#VIbJKgImE&#O#GO zl>{X91skQZ+?SE4Pv@@YB5I{eakO0K{SyxXL{J$_E2+m-IrQ z9!`i=O4|F%Sf~;$W|yjxE`LtQ6D*KsJfaQf{nlbMqC0fPnfly-Z`nC@^KsF(U&6G6 zKX5oJOG^58j`(ojO5j?r#5i$pjeMOWWjQuA}e92IxB19NgUWBCU_AJSUycnQdR8(OPPb$ zFJTux&Z-Z@#WY;Wk?f*lTO{hXo>@D{C$8d9I249`2(d3noGK z0!Vsy7P$~Tz+}E0pv4B7zO{zbq5THmSFicqoSfU%3=t;Lz%eKL?AvKXR)!u#^6>yJ zPKU%Fw~VFky#@yMq{tHslFu)?RpNmId2H}X$DJ2j3^7aAFe5UXc5CVQ>~94N?fD#W zImAZz{JP?suAN~zzIJj@D+UN2==#4FXeOYy{c-Kw-?Ze*UEzL?VNHgadvMNPlRrey z5{H*1cIOV)j~^TLIwL+*G8A8e$*x-g_be1*9I7NnNi{Fnv080)ex%$BeB7WSe^a;N z=Kt1b^2&DjwaJq+&=)qL8QQZY+nvKd0tDaYC-5%IgPgHcSl9)ik3s2%wHOL znR|jFtAE@`ED6G$MI0=sA~KMU!R>wY^dAcBr`5f4{5%t$3@Ns>)lk~3m9jVU z<#G~ERvl_jzo{st)6eZuFT7I{RukdFJ|Cpw-sP+j5g^y>7~_dp+#^`SXOy?(OeFJa z5BnIAtpA5~)*n%4)cemA;w8jbw3Sk%va_6aexH}5RbN6QKF%8w` z@dT^`b$;AnM08IXt59CHP>8p%rgpHt*m-$De~4EBRW{BQ2s`&Bml5!x1U=KXUr449 zpMu+7MNnD15i7pT7u>0Spg#j(iQ{D)`j(XbJ%eLUH>j5Eu1bgFP>Q4xpYksJrz@cy zwZ0y@?K`NiCNf_wDGRZ#H!jO`y<6j}upZ;{{ototlz_ z`*gIKrWdx+S4RGruv_}XIviIL`ea(=!p1JDf^Bt79P1gOmG7O_yVrb0n-eH~b;ucO zvT$3Ld>4yY=`rT^^?E63AbjV8;~QZBggN?|Vb|iTht&4S<}!vT2GS4O zA(5sl=G4){P%1a{SEADRzqJ4nSOu1X$So{DE{u=ZX;!2fyOI;$y-}*zvPDx|iNIfu z8d5inO%lSnNm#y_-I4?Xo}SMd_^AYp%h6D|$Hykq$;*H0oSdge2x%Y5b{uz8lQf0I z@@|&zbfBcK)&T>uPiF>5jU_pfJ7WU({f&Nwp3s5%eY@iJ)39Cxq?EM%&-M4mOw(5x zt7F$Daw}UF`k@R?!K`!i`NX-a@?U>#1h8wt4AG2zXx37&%6qEWedjWV=*Pv863kG= z6|)S{Ob1%4h=ucXlu7J?pK_Xk8vv5)SOd_~$NwCn4F5vUQvZRTH-BOO6<0{q75|Nv zNup@W*f@9iYh1a`IODD#C;jvyE_c7oOx$IZ^%3~{kE>y zgr-x5H|W?d9prU@>A$v7DzS~6Ar{WwG1*&5F}ta7vvI_8{)d+9j?s?xhu9ZjRJ1JN&JD?(n`kwQx*KObo73>Bxw&;o%Y;L;Iwa>Gar z!?SW9*=@{4KcR79bYm0WDx7^-W)PtA-mnPfOa(inqZxQ5Kjoy}zTajSSdXy{pzcbB zYizdA7l{_=|NiEH3Izc(A+W~k#}>a2@#STRL^-E}7Bp7sN%N?d0|pQo#o;UKL07+y%}6_fovT@H-iP-z0<`YjaIq}c>orU_Ckt;A{7S8^=U<7*&$ z3hn2cd!g*o{*0D;BAUhN57_KfoSbf~wUV3#apC-W*tH_r zc)z6_lQk^`z^(hyfXaM>ji1Rs9Z$bKfSlDwtWx?NRz&mj7O46RmOO z*gLLxy#*_802z72ceHR$7{qWdJWqfWg@fZEAeczNbG5Gj#o~%9-ZwuqY9=(DFk^5l zCx<}9bUa*|z}XrwmbJ$L`$LAloV7hMrMFjcc71a==YO?eWgq8i6wxP!^fyfOHC?Lh93H>_R>33yIwa03CX6tFM9FfU`uF#A zGeGN6&U#{AHX0kp&_6MOe&~yH2Xy%03#}{?Nh?nZHlW=FkV*mh)b)$Q7ir?--~^jF zoZZCFGcC^)9y3=9n}l=#s$J1z?gwH62f7=Fz{Pu_1$nc}hVi+cr}di7Yc=mi3q0G` zf}cM)5PrBxI+f7Q)={icS|tD^x)n5X!i~{E!CgKReL+%blrz(~CzwCCAe_1YyzJ1? zMo1|AdO+;DABCKSQUFM7xRJsdJnh2Yip(wWUdTZAZZyQntZLFN($jt_nOZJ&0hVV| zRBa_8J(2r1C^!32#nvO{v;M`D-Tu;{OZWqI=#uIkAt1$)bkw(t{+5%f%=l>(Zi1tofW zZBWOQ?mk^y;1{;LbMvzmRa^&tqX{-xWtG<2O?e6JmmoWdDPSDQmIb_=@%Yfj#{}~} zDrG-qB=4VoaNUx(x|(|{6{1;4#oXFBaR`fu61Vr=Iue`QMf(SVcSQNiX5zLoPSgFv z>7^^d*|%QG;_e;fntmrxdG%q9)MXaN@;Znmjqj7x7?XIc`z)yEQeD0#qKcQ>}`#v2B+5 zaeF*qFKzRepylr4&=Rx^r$5$MHi6iPVab-toT1w3_Q zss(E$=^6^p?&VG}VvO;~()#-p#k065b1>_)40o4T3f5*}rh!KM(mK|l9QAK<0YUZS zd?=atxdEp$9`+6-wF-HTWD`>Lyi6_;T12`2&#NyJ`S-ggON+H`JgN0%E^wJjuvgNV zCG(X_rT2ReY40!AO{FtnS#q@6thYs0Fc6*HB^>S-8!?!UrxqYQRMIF^LUZ!vpyZh( zSB#g+`xqGzE~kPdUK5M;R|;)-C8va4kG$o;0i^a;Oy|#{sp>RoBzWsRF5#4XX|Apkm1+Ljc6=Z%Y!+RvqOvr z@u2QNRs8dFfwq}*$YVc;z=xxDf!n)~T#lk2W@#G)UeXxO&6ntVE`kfykYZ% zLv?Cu!^-?_pX075O_3q^?4i_2wqO+3CGaRRo5?o;q+Lh8`rcXr%X4>?Nd!hl)lA_) zoTIFSN@gndwSYW*T;D#pesCXhyq^=|)n?VdB3HjV2`K3I8OSA+S&tVAqMkb@7IMcZ z<&C%g!dfiX0I9~5)C{mrIlB6|H1bov2P&2au;(rlUyUXvWTQ%5Yz|*W0?1rO-CVb6 z7W(s$y0KI%*=6!ly1N%2TIYbSs(YIP1n}t?mn)Q$Y2ssY!5$Rqm=SP?g_@JQ&9tGvD)wyo z?q~#rk9=vTniA%lx@Xh!8e84+w1u<1qihVOUKH<^m+x|Pub~HDTx_!rGpXEFB40_l ztj_ZBgzcv+7*XG`{yaF}d-@Fw{*5;T|1tf!CyG&W0(v=<)QNU9ai>=K`!C&a_l`O! zwnt$1-Tv##(})cVV4mD4CpkVjmt3J^r-C&}@q zd41i@rFDwPlyfWQTY8sQlGHYMhZ3t6k@rj1gir>6;Ws_YQ)=V&WNU;egP^iYK(u(8 zUK5=aFn~Np3p{`B3iDB4hPHH)4Pq)cT03&?!os==9$fF3Z)*FeZ*`;;&3(dx3uf;y}BLj+dd zkgD?~ZS3x%;9NYZBb(=R27{E(Hj&H+e2QS;1^=Y)_Y}B%*?2wLT2(aiK9_v`K^`v; z=UN>V97F}2@+~F*+!y-gl0GQ4r?T=Yt^u9CcnjQ@!Q9@**hRbF2%^0hNcOaaQ_Zhi z0D}taWJ-GDSZXQ-v?7;gC$Gqq{wh>nSo&B2_t%hITgz=#Hm0R+=y>J>W>@z)=Z+jCQ#ea8U1ebv zpZwErcXD#0swCOAt3$&ZPczOZYHh2U!5Bf6|71R(Uw+;jh){Nb!+c!MC2yNes7Bl% z*N{rs8nd%VmN#f8i*v93JDN0$PN%;~>WmjO8(PMt(@BY?#ba-Ctm#TPGUbhg6a?AZ z$RdOJoEKDQX-^~jh5N)3Iz4QKYRSD$+`-+Y6`u0tYR=878SKPY*L=uByZB`#Y}$UW zxHt;AqnmPyX!K-4KAe>=LRX|kB|d$3T~llxUZzOtzz~RyWp{6H3u4$w3h;(O`g%Pj z*Z8oXrmClhI>|$aNZ-UfB!EwBN$t^-7$rY<;6{%6Ov-SXbbNHv(cqh&a2Yw*02#>OeuP#?10(VQcxXy`VS@l=WqN)aFE1RX!j|JC?Y|q219ZC zAJD=_;|_Of8MVVOZolJJBJxgk(VIJiqs9cUalEVl&N<{j8zAfL?=rm_RtopvP^JFA`5WqVsK-9P9axHTFp(f@+!JF zwQ%0TJ0j&*_`+^H`qU5ZCPhxOoJ)Yu+4~#+!M@sC5eVd5B@`R$WRzcIxLaz{Fy|WE z{tW^@U_;~t5cZD0$aMFM*gYgnBI2)*mpy5uvfxk|JFMuH@TKilYIBpgFmm0NUn4?V z|Bvuwp=Z!cG%44oOfH_O!M44~*_E3_td9P6+kRr1rFcTc#8+A{`A%~0ddOs&Xwb2H zEK!4~G5aaX;i$Q|d^%RPwJicqAp4?aaQ&vBJ5AFXqbLV^ACl)~Ms^J!;lR<>nD`eG zoOAi;ej=G29XoOP*6?`Y=C#J}F|Hc`$XS&|wQTf7+VneVNsI61&Y$W(!ZDe@mLtKp z!Ua{Rt{Wya5ph-$rx&tr*10kp2+*b*CJ+sbWORBAU;91u-wQ`1C1Hw-iy?aW*%_bY z4D6RT2uWS*46>&;?&@@7psa6L*R5n+O1%?eQBijbm9dN1&Ph|{)8;)WO#j*s^a zE_?v~$O0_4Fc(F8X*Kj^=TMm5B7Oj+?OvSY1Hh*Q0dAy;%vW3{U4= zk#E>VO{Z~#(GWC2{hsHhL&J7JrW@R%zok}-s%FVj`0NLLV|5l(jPD*s8l9GUC7jtQ zWJ#{q1EnJNtRuW9`E+obx%K#qrJ7$8%i}b_!EEo*RJ+_(f9M9m{3fF?5AHcDllb@x zm7oq`{=2k^febF#O$e-sx@;nbK;KwJ5w2~3HBp}C=KByT15SXJOWZIxbhK%cFzbEo;AEZiABp?S65el9fI#jKaD>HnNOsah&Py+UXkU=$ z&d+v2nxpb3(>mU-Jc3vf)FN&bw(B{Hs}KFL)V$VFYDxVeHYk+8{x*o?+?Z*s-*f?; zGdH*PExZ%FRO56hLqK(27Wr-}f;7z3KiO@6FOVj!>1Q(a0g(X z@5o_Dq>F=@L^z3o7%{9}Xj*e>dw==bonl!vu;J1n2*^FwxGNt=zg1rjQI-(~r0N^9 zJzahxS!L{YUGEXM^~ULucWT~H_6PZiZbotz+{n+4=-LEJ2!;la@8_dFru9l=FAq-P zlbV7irkog$rx&dBrz0_4b1CIeo-)|QX1vVMa=D!V8#hfUbSLhxHWrkc z(_UHq=8ytK1JddlB))c;QP$IBVTzg75-fN^H&**6QSnz zpS5}6jf@6fBg0U^D}Q{-Xe`j9DTmxl&ufo6q#XzCVE~x2J0UaDC=u&?a%hc0z7qi* zCrs4}hZF8TH$RRbIGM~4bPg}%q+R>}9`I-k+~J@96xu7b0^^9QiWC+VYBM1je7|BO z)=1hULM~B9gMH@nX-VcCo~_gm8kAw3a8?KIT+C6KWGNcHTvWFep=^kf_GrXnD*^w9 zM>M?wBmxBPs?MWZlS>IcUUDa6D=s5N5+Tu5{h$pDZ=;BaD#ynOT+H{6?bc=AWL@R=AptR1(fVB_Nhz?OxLlgmn zAM@PziR&ku;iY$IH&M$gB)B06;xD6d*!R)zQHcIJpUxt)R^~g?s4xGVE4w{2%%>R5 zg8dU*(4V&chsV(LUX=LZ!$zN?N$QcbIv(|auF>vtSP;#d^I_HlkxeXIr@|xJ&ivsRnO0aI_kyOa- zL8zaFs+Yh)y;9hp8FgbpGzrObtqwh|1%fUm8!*hy(LqeC0Vm#0v%dzcgHemy_Da%t ztP*qv7OV`MTU<%CDD;M&8NCugt2dcO^ViKe2dh12+Hou>oe(!r6aml!TB4j9&hz8t zP_CW-*m)FLkl$SR%f;SP<6&xA{N+%aJgFCncnk;ju-iENb6}G)aV!(DCE^0qvgxXs z=vL%vm(81Xc~J$h;5@azmMgcoB0ea(U&ZLVQ{R-f{sZQuww4tA!a(?+7b(|q`D;3? z)JFgT%g2F(MdOaiM=Le_bw>Nf*l>?M%}PwKCsr2>H zE@06(@!Zt4Ohtm4>y~;<X^d8(kOM2+D$cc|9o9 zBrqvm(mLiH$=w%-rr_mQl``ibG*&;Jy9dsfGv#u3j}Z+5qpkv2{x~|$1i!f#%7OpG zOOCVuJxduZ=#6XsL&*Fl4OnLy+agE_V?l85QX;?%0lBvy(ae?P7D_u`NH=^dPW!dc zvY+$8h9%R-7a=p#9m)|dt&uf#^z@6sZ0FP1EiMgTDD_}M&f_{fAUbo-0Y(Nfx`etF zSzVpQZM^GvQ(nH@S{A4D3pk4cF_2h_u0TQyUy|o^=+cur{3AO|d9;i@nKQv8Z7~dP zUn~y@+2E`H&giu}_fkc+tk^^hS!Qp3swVE0JO+jNR(Ul}5J)W_h%{#dgG%N#qSaac z;7`*T#sujOof1lbh}h!+zpYgobumI}>Je_~RC>^bB8HhaL0mB9mHU+rFW5N(C91Q> zFu1r$T|1ZA)<<3*#gSDk&hJ+7`YveQOMZQ2RrIkC{5kpx$7T4nl1ldK zJBh2GU1+oWdCYO3t*y`H=fWv2sFDntKaIZHwxQqd8;2!n$fuI2a>lg0qv+2S7O9h> zRcN!Fq`G0|qKP@jW=o4lVFPocp*BmjRopjv-qi8HO&DCAC(ZIIMFg&_Fk2A6AFPd;Oya)OO=Oa|{ugvPP zdadS#q z^VV%CLUO_&4xt79eNG#;>#b;ubhW6mid*6}oDrY}AAPJVNF#*)b3(7Fm&JUOu)>A8 zYUD9)LLM6g>^((Q(d5O-POG{c_=}A-LHwhlO?@c18S;igbpFU2_dXk9N!J&)n4J+Um4pV4eMT8VP}Q z8tNrNPmH0hokJG7!HL?XWt5lGaiaRR_CW0ndWnCSuY(fs@>*TpG|LfG--xlKQbzyU z8HOO#&X?HUr7AA%lG#Ydsp6SFc@)_+pGf@7*rEqfm5ZCVJCXb#Gai6J^&|S^-#-M9 z%7TKSiAJ@&93n?Ho$CiBo@zINlsc^>q;D4X@^d|exKOYo~mK9v126~ zQE$p137|20`^@D8Ca?vSsj?Mchd>nK+r&UCk>18c4`)|FfZp(>ZKZbhxOZ}hT-&SZ zjt5f22UQT!CmljKnVR_Ic(1CWAJQ;D^-pf^itPE4t-8hxO?KNVjQe6(zf*{G;w)u* zDWtIQpGAd`xko$cRvI0`3)|PWT!`v_C}crF{d4))Mo=TJplV_1ZIiW%kQpHf9^$tu zXA*FgZ={0`OV`{|pl`O{J7Hxn^u6O%>^uPb1MfuN{akzxJt4VrqMxU7lVxRcfl(e9 zaRQf7Gp4xtM192g`Rst7tc^^@RC#&$4SVrb^TTyftjIVKlo9@~#QHQuT za>0`ZFD%6*a+3yZeI8i|jiEquHxY9#upeW>q`<#Wvhf}UQ2@dKLBO?x*v@ttdePWi z28GRnWt*5zLDel`8tl*4rUxug&f^cd5HCcl0;|RS@%mJ@1^skWoU6jC_+1x^SRV1p{#pP6y- zsU_J^I9p{Um(EJRhT@YXhxm_$Z|r!Vu=d9Xz7D+S7uhV-kNk~aw4M=mfo~eifFYFO zWEF1Py8xE@$wkQ7#amh)CZ$@2H+CA*$p3-NaG$p?rkYapn^KFT=QEXd9RUSJm}G(< z-g1a~H%fGf9IRvI>TrC=^@DfX=NKtpdzW`_WS}b}{VU>mq`M?0*3s&VPnhlnvy_85 z7uUKZZ(}?9HmeE8eFD*K$(4&JNr=yP!!#&Gs>Dwp*|ZxQNHyKjmw9ZWo3miZljzr;f1{Y5?du>b(R56 zq^jJDl?>5`l4ls$$BciXayw-e;d#H>>f6y?sJ2l!t4_k56!QfdLm|^C9j8@%f*W*vZ2^0CT)>3WIipcr65V3{TmsHW(o-|4)E^`YTXRoaX z^l56Rcqrq;#*C`kXQYu2-iCqJ#348M*Gdy$;#9q7R8zrXbCv z_Q2>5S*tuJ#b^`yAyZe^2YqRUvw0MZshNI=|Ec@wrK zuX-QLRk3h`oFEbscZRu`k8N=>;_Dnuv(*bO*LmYhT{{5=Ab29cfyf|6^JD!J15V`9 zDMBH>kAQOQnh^K?kkWEj7+dC&dl$)%s8nUGg$3j<3z{^JJwjhGLITDBE6#+gKuVJh zTR2?XfgD~f7cdJlS(T0$b*UB0ZEZ6&Prc*L__Y_7zUMotmiGQ^BAcl;Es9_Li&HJ8 zI6Dbyvi{ixt8;)DO5i>Y<)1%)93kG_9Z0;n->|m8nmZ)Gs=@?fN!7o)T=j^=+CL4D zRWBQam5cvu(0tmQJNl-Z3nPYHk7Y(YM_AEY5DJ8EL;) z925qB_!3##nI;6&IBuC`5kQy$2;``yuFB_!+{3LALO3r7RrShlrhad%1ANyFi0PV= zxnN`gyZN!oSIEZGU^qeBuO{mIdxi#8@iR<@8R_m6i%XZMh$Myx7 z%bhMJ?Cg7G02?9~PJS8<7l)Uj24enf_a6EBeSG`>#{vY1CH>{!*bSFj{#o=8 z{*5v&Jw2*_O4Y+`I*(BFD?chea3N=!@ddV-Sorv_{9^aP;)maK1tzFC;UIxEU(+f_4}x??Ru2CBC1A!mN814 zrUB8}sb9G2%6@j3q!am^Lu0pG;SlJ6cw~(4c1=aF6Y_yTt z=!4Qn^^pdKJ16t!PlO0|Gd?Ov$llI%)naziIMlGS;wi7;8dtGpJy<65+VZ!aQSees zKWyJYNS~Bz#}txX4)SCOk!*5|J#57Q`dti#(a;*SgDVs0Nild}8&Ssf7s_pJ^^Mgc z1P?IBlr>k>f-`%D`y@l29`hc8b9L3Z;;-7tc7IL}zu&%bY{?j8caubv&8w=zWv}SA ze;a4Kg{wre5UGY9T8zzmEyFqy2&|LI39Hh|ZZjlr=Kw{FItwjO{!&S2AtB&n>W^53 zdX6JQN^3yKODqPQ{K>AKk5;K~f~25a0x(%%z$me7E}UW;2hA}k2ffU#b#vcn%-I*g z7*STnU7fU>JEM-IrGG4qOQnKw7PwiuSJN47nR#GP*&K)gK`huVl?9BrvE!@3yb94y z%C)E(#$}t=vBeaX%NpngnTTYh=w)Eq=;}pPATS0J*O*&f{Tbn|g9?)P{2>YwsNy2# zP2vNK38R_jxeq5T?r`5b+ehQn@xoYBw-DsY#oqICE|SwU2v3G*UohGl{+CQs)qNPO zjkB%(-Dgh_XQ?ZtpmcK_(4;GnMq*NfE_ro=LZ%KS%c8H-qUQ2<$hhQ&#Vh||xa3!V zRF>iG&Em4aF`aLj;hJA!OMBDP3BUhL%8{ugkV>63`EeG^`eSfK57jNl)C%1s2rt4e zIvy(y##;$maCc0~7k%=VD>H_=_9luAf}Fa`7oltDh*Hi$kQ38F{V$czw|8NK8~!~R z_Mzh&L5gSiVks-(?O3fOa^}@9ad((QLy|im$-!0#m@K-05jg`tBYF z+&?$;3_f1ldo0HW@zG)cv0sO#m_`FL9_&j;oiDxc^C3y2P_?o3 zro+wz5g<1C@T#rW0?pA?8;J&fRLPgJpZ}zu%5|#Y*`(2Bm za`24J7Qfh|G@gT`;*lnJMq!YNHDmxk*lI$x_8L5JJ4iJ+&&m1<-}veMFALFnO&R07 zGE|2X<}n!$S6*4!1IikDI5ADy9f+AxvSPohav5y3oQwB3puUy&g5|g{->H!vot;gI z+xWay&KfYX6QZ}Is-Y1A;W5B#EqA+^IW(ZXS@?45o&898{nXApP@Cy@|I(4wUhTZ2?zu4n~H}x z6*-gvFOm^=`ANJMYsRZdxUru&+fozgv1^1$zBA7)N$2%K@99&-hMgHU&<3oKoJ zM=IH=fCN^?xUxVEQ>Y}D8DkGKTpjmNlda`$WOGG%%OO`3JU2odzAanBiJ!g$)2+CA zXMtdc2XsXiuulqPr2?BN}`oQnr|KlX(A3?r`Tqcm8pYH*Q zRC6qMr%?eFt4@$G;|j@EeZviv8?PKss?t{f^ZFzCa<1aOu9{A>6lhA%SX8iFuS2hV zIqe6-vp0!KzOyJ=3&xyt5mo_JH%j1?hVnrhlyk>N)V|!|$zuw9(HKjsR6;ilmztzxfjY9G>Kb zOq{CUtBSzGJ443B3-gb{H2aIBlVU2IiR-i%bib*3ogdwb z*6e&EfkhbxRNSF?c?dxzxzWgn>0#j(G|(^M2N(ESP5!+i(r1=XP0awMMUM%Av50*e z=D@LNjW*4=qoq(B!O!%Zva|wyp=$1y;OGVuGbJ`w5Mb2|CAWI2H{MN8*GZcsdd>(# zbX1=z+D7w7={1~#9hhFQiTrbw!DxME<$=*VgCr6TdiFbk=@0h`?^N}{)Sw4tWaTufvJ!HC5y7&>Dp*T+mbIlZw^S(CBU<3nQ?2%vxZ+YvwK)5f;{sCjHF_J@ z^!>1{WNHqe>d4py zonYq{BUpNIYK?zU$>H@wuKnw*%)R&~sGzBmVpfr%LC1$l0@c^3zHGAT>8sHzptOeT z?`+8on*p-mIMP(*T6)w&%elOn8r2vbbwfJuo``z}iNu)qx2&ZlK_6>te#UH5>!6QV z@0?X2pxwX|ad^wWi$5~j;2$zF?!ThD4z|FdJ7lzT=TrW1VaVeUCeyj46}3Li&WPno zK9w^tt_jyD-u3u|r?nxna|^wFqRmqeqL0=5s0q_)_(nW!SZ5bZ^WsdT{#OY zRud1`KZb<=PEqX8r6gQ(k(a>hu)!Rm?%0uU|3d<_PAz>aN-2QsK5Bk976$9iE_DKi7tzpj(Ph+4w z=U+n$H8{s{b3S8WMvt@$m4f*4sYw(ZbJGJ*U~68;*C_{PlZ65*NT>H{Oy*Yk$&4P8 zg`%@M2Ld~Kgr7ZYc1zshBmZO~_Bwn|DEvJo{5fCme5=xV)>&+!hpWMKXQYaIH!Wkb z$ilFcaW_5P$--$7mgLGwAw>LsUGVAQwdQtZ^3$j1zpwhcnOfiDWU$(TSJ2yT8A0Wx zG&&m6wuI7JwsX}qfix<>Z-G!XSPPv@3rpS-P8B*_N1?ndTZv54BdUeZnPz?U4WnkClfPo z`ZM=oPEhFz`U`KK*-#rd>HF48Aw_w6poWCRXQ@j$9N%9(PA>XTk7}) zkicc9eTxl(B#E+>t~dTxGj^6ry;?#Ct3u$a`QmPwC@Uz0aJubZbXthLeM>m$f$2A| zrMLP0B*p!NG=|c1h?2Ce%aP2)282HkKSQ^yTzbWaXBYN19qVT#hK)v215@7VJ_EE1z+)4{D{2cGXVv9uGCSEmnMB+*1`%K+}GJR8yxq za)OV9!3s9Ts~pXIHu=0IY%RW)^JIopU;f7JRWF-MG3HZ0uMnEYy`>me&h}$5B1r@M zA1`Q~2PpIF4U_`jyV%TEr|<0Z0U8ofhAgCe>)&H#r9suZ zvxslXHaO)iSyCe?jd4cst3y6s8Hnkc^*C^dur$4K{=4`gM zlf6@B9&GA(^iB=Ls}_p24r94)@sx#hC<Jt~9oodiHOE>_;fd< zJ|T=Y--~h|g`VM;LQ0p1kI_C{#k{ukVx_~^_;dAjiT|E7|KVld6rgsVNFl(H}DYx+j7?7G4HcJfO7 zxRj(9_kOfe=L1U;jTHc_z1FiqoM|SZ6`NV11vNquq~bxPSYjO^2pYHQ&u3ZJ38&1} zQhSjUr8N?OcF6L12)jqVEkP==1Iv84U<`@%cgo#| z@B{`5=0${1)o(t?GxO4@dTERZu#Xpky5gL4DX9)p>C*dxvHaMA&TnaogU;()Hpsax1zw5W4{u&{AnGF&LIZ*G+4s4c0_%ko^RsV>|QMevZ z{Ol2)@oQHI0c>~SN6Vc57y^Ch0gyK1s8u3-cj&yj6P$(T*EM-SlI0^yyj73~I2uO+ z8EtO#iQdfUED1jb^^jpcwBIBCW!u0ISzZ zDdqOD7m+U~AhWMf@Z(1b>XZhIM7Gxx_(T>1q^foSs%0u_IB(}7IH@Ka^b&Ly^acAI zyjf=H{!lFPv(7!dZ|xcWkxG5t>bv;@FF_V}5RvQ0*&3H+u05V7#F0!90@$LpmICRf z`C=3i$q!5fn!ufnYoi0KJ{VngveMX9KWFI!Jnp)aePOvw{c*EM?h9_Nthj5!dDH0X z=|znMc`OIfL?492oqHE2M`LNg@f5-FurN^f{`nMc6%M+fFz$X6$eYZ|p&?{N_ybAv z(rLdY(QgRq3-wexO`-FgLz#N~{pKKhVurpl;3pa!%32rs6glOUmN|{DC zUrFU(@lyMJ2U`ZhXlFyH{656vDh5$WY{~R)e;zl;QuYR44;pve{}vs4y-v_#^)Z7yA};xJ_aQ3awIN z2#CaPJ^>~y=f2P62fNhZRGuVAjiAp-_QA`QmzoCpuTK%r9c%hSWu(-d_7T2&{0egE ziKd~nLENc$jKZi4w*IOI1^cLNkbLDHvU%Kia{!R<6A9=I!Qi;HsKH}9S_c~1UKjJ~2k&oaBxq|9f2NS_Wj-JzF0T&JaT^Co zRf}5iXCFWK>1+m6d^Vgzbpq&bROPLFVT;Q$b=n$v>soT=`|Fnx+98_u-UpK(U*rMG zS<)LcZ=iGM6a&AA%VHqOT>L>@x+F&}LtePA((|INf?-n~Pfar|}|Zh_a^7VXQh;O0@w&o}HfQa(Dobf#S$nzKYq}o09tZv-}Xt6&qe~TE0k2*tr zfxGoV?ehpaGS)qS+2d;=m9(KAzVdR3zn(7<`20mN3jQRU05$Tf80`JK_9D<`81Z>_ zlYwweJRzHon?$antI^k48eAKeMtqiOXio^0taFczROSNP6N+BffG8Yy)L< zAKgr1Vuh^>7}G*R_+e{n;Q8Bxvy?L?uFD5~q2f_df1Zttx(o@p{Md4WM7Ve$0ad16 z(XyWBuH!A;kpq3tq?Yd$FqoJldeXEnS5qW*MJXzI^zfI74^0bYQHG;`>+Zom_1a~c zlA|OodcPp&XObK~D%NTkiQTD=hq%#Q#ZC96FfKt=A#K;f=yqI7aTvC)CHXRa$K{)`5}ie!IEf!wRM(`5i^N~M9jc~gP^KLvfw^+#OHWZ7U| z`=`!QSaC^7(Ncy(U@S>`$jei+)X>0>D~8mAHgMQ;ksZFnTa29$R%2KYCF!*?wa=e= zn)I0;xw&O4xO2ybmi@Y2DyhT;a3>3~EtoC4sJeG4a<~1A6`Ii)_Z=PRNGTuTddKq< z?AHmT)P+MC;R?`YBI>PU4FW@YJtIfg5Zv9{-WlPBU5P#Wi<#$Q=-3 zJnn!nBq(>TTGc7UO$y6oo{|vbT)>A*wqe1TL(18Z$@4BV3hudj=7j*gVsj9cEH`_i z2fvz+?BAVl_7>g1f_A&8QaR!s@WWCh^>-2I*U|Wlk%CtS(Pff`lFt9c$)iO{MxMM= zunJ~)M71N9B2fx|;LegrMALLiE9!$6cqnCy{FQVW^jg$6yu?r{<>(ZG6y8vW%H6=E z3Y*dRx4IA)pF8TvER)JaYe2v(K%r zCf~EDM{#Ew-5waMO??iS^IVQ{8|2&2;#+Qq8{(fHr>Kflinn@VV#irq5009qANn~@@zD}en4<#&UJ6g@Zso5)KA#>3oIa#N4kR# z@_)PlK8}(;OAFeY=IBNudlE~}SwI_fKNo42v6^kt=PAuQ@B@OYpbKsU{(-aBNxu61 z1NjSyaV5R zXzn9IXR8Hm&i3>ooP;#D8ppIqFTF(rBc;`H$sW(PYtR&YdR6YquC(0sF1Vv6)^3i> zwf$Uojl6K#A&PJGkfx|5jf~&-EeYWl%%eFZ8+S_ir6-GR-8d2(mhp=OHe)(eNigA% z`^kEUCEqr)DyWsw>GzT7_HE+-d#`#UQ_Ss*U@E}>Qvn@X@G>xdJEobrc~_=E5ypKV zmgdhEDXt|6*q<=8O>q0mV9i+e!&^>vZyXT0_PcIQ#INgrPqN_2UExA~RltY1*)>)B z<01!zc{ZVN5iK|KDfQNg%~&4<{0Kz^fBj-#Dd;@}Ff|H499A5N6}05(IXFcP=5s9* zf#I*yi%>f1^EvvFuB@P^$eTm)HrJ@T*TzDtSe&0~)s>=^59{UxTZeFOmQlOQx zl}1*8yC(p=LzbK-EU13hNjgJ z69Co&Y;JNCV$m<-($cO|#>zUT!^U$YsaT=n8f4 z2@*lP+QG9ljm+B8^`*jXf2SuLr7alu`o*BYcwbOl*f5uuOrq*i5XbuL?hp%)dA`m{ zv#@u$s{|C8?;6WwbRdR2>@q&4{!J0?<0HCj=hU;iHBs_WY**de!!?r66fl^`qo_)K zM{fziN-vl-xOGPIr{rP%Hx@%wPA-$rVC)r z&LG4z@pX9Dyp0w(+ayBABQH%Pgu+dgSh<`i+FTQu!9cgr<-Agpz;ROB)$rM&lj=?* zD?RTkVfoIDWkg2ec~{h@Et`zk#6xWPPZ!>$t7-?>68zMmAV?HGr=ALBD#iPnYNhOC zQp50wEvHG5G-vJ3FWK~oyB}`#O!!CQKE;mZjNb-bf(hj7smjrXGAVR(c~XwH3+dA& zUtNR!t@+L@qEXCK^v%=TnT(wcZppnj-F)e#=IeYwklPWmD_=JTL5il(NAeOVZtmsW zT$rMTQIdTu5Z_c49crFq@VK7P$-LCl-ErVp#joOr#{We0Oa2 zA>WaQj`%}ln#6T#r}UO@ZtySFRLQ z43abeDT50qucgprP(OY6VHsEs5>VwXEk&U2tKzBs@sSC}kU+d|#k^^yEW;K7ibQB> zlAzU_Vp5DJ;MoR)tLMN~Xw70G`fFu;S^-wm#m3hMf6i*(++FXR9Eh>4>b=IVCFC;m zeWHPL9nTPwY@I+z3wVor5}|O4N8MGs8a2B-jAIUWjdGN%=06RA+fMK1r$*kwK;*R7 zpuzn}#{8M!44d$qg}Wa;=qvN*FL`{Z%chy8!R_@Aov=pM>3hm=#6R9${Jz_hnQygB ztH_t%6_E{gwKiIlx+tKjIu7p*#C-%@3zyI@d>BuXMV%6VhbGg2laN>1agpp_UaJv?;|EnFbChROAb0(`uhWOIsV zHWIgMPO#Ng>yuxGysP7F-&>zcYqdI$3l-P8c?$SYd|jnR6&3VX>}U4#fJ?|?Ftt#& z%z9Ty4Q%+`G~V+m)m!;>|I+W_YM)fdp2A$s2Rqi)7Nfgq%mR;=F((bIw(Sa82IgJT zi4qif#dPjSj?>VI$>y}YZva>F5>Ijw-pp#lHDiiG=q^EP991Onda^LW1Fwt_WhSUV zRmDLv)$O#rB_*y|9r@U#LgBU~-cL4QD7Y4^^xKCZnKA^`?_)o2&QH1W10G(J@2gBv zwC3y#^g*5mNH*aOSC1h)RYzE@t`x%z3ixS28%M5HPKSTZ{<+48KL6UG-=!jcBa>vD-f^>c3n52Ov_rX^+dasp zR~M~C6PJ9z%24a0Ip30ZdVL?ooAdZJU6DEu6Il61SDD7}7tViNDHx51*7R}qV!Zgg z0DJ27s_7)Nn+h&HXr)*?uL(SjTN^uq|Hg) z0vDC@6;jJf1MQ!x(2)|bbH$hy<1Tz%ph8IIbJREj=3#iGHv4YfV)gnUVesPwNW&sd ze&v1C91dI-hJ-X=8>srRv|mv{zQBp+_bUZ-*if1}Hk`*#KbH|r_dFjVjCa(y92{S& zJj~h9lOeSvkn4j%Q(Y>*jTeP$&l|(%?$qFw^Z{+8HEh{oM%_~5R}TB!N5n0z^2lGo zxcWWjiC@=;=CVN;#|!lYDH{+W7;YmTd`tX2UzL8aLl{O`pnge-i=%n1E z%+pv?rMjGIu<5aGzUl6eDzYQEHpvPh<6No}?-3!Ei=>n_H&Wc<$;T<@_+z#F1mk4k z{EERUZOI<^w(O1HUc*P zJpw6nfO3xRg%?F8MvEyxy(X(YhF+AwZPR-JHHD1JS0_0>b|##s#I9c=BF!w3WjrZ2 z2rl2Fvip1P`3Z9F=hv`_tTS1!F6*mct2|%(sDY!28hZOBweHx2{sN5&%4nt^FTzK1 zfxqEnb8qA2xTElt+hUuTT)1sk=n}aMseYCCaaKxcmIpLHXVP*qyJ3xyi^9dWomDn2 z^n884YaYE4n`saX48B@<*%-P=jOxjb?#NqNUIhn9?aMby=Sqo0OMJbud^QGrMZbHa z;JKS=Z);@B2{HzoIaTr?2ypA?^e>%BP3Yye+Uu>2KfOc7UvxPXaGTpKcuw{)1a_Sm zXt_I;g&ycI7Ph-d^ zXRrFkeVo?9-q2~|y-)ebplS6ZWX807Zhi7;S?C>Hvz4Gsb}z!hoh_3;Q=x+-?X5T$ zl#_~ma=7@EJzDqe#FqQ}A#pLmXZGKez{tcI{^$F=rgF^__ZV={3l9qOM_pPb|9)Uy zXL|IP_?lx35461q1>mox|4R-2PZ?f&&HwKpm;Jv7nI$eE0TH;!n#>Y9O;Yvx8uO0$ z_>Ec$$!ssA?Kg?w4#OCwWK@X+D72}pY)7SE74)US^DX{J0=gR+ zo9MmIP~bLEp}Way54{5ijcf@Q7BSF@FVfKW)3K2c!1D9mRI!oXlfJ+F-s_e!48_8T z?MIA_&7o+~Rt2`kAv+jKdY}e(ElR?ztgM{Hh~iKuAm@DBxp0@~ePCGi*Iq)tYjj=% z{f>WXci(XJ_+GGnff3r`tk8&v(dD|!Ka!N?h}U(u9aiyCffPt+PZbc}QH~Al!%DU) zAEUV;4@nNe;y_fob2|mtUO8=TafN5*3&a!X5bA1##ifC9bXm?7M~4Ei&=iu-Y1q!O zYd@aQv;7dOeZrt;W5dvWV>Q?4M=8dL8c(D|!`8@Oi*FK_79V;<5%lB8)~qY~M-SQb3VA^| z!{)^7D$}cf5m_^Vfs%w>b})r!d(o3bksNCk;y@OzC}nJsa8V$CDnr|QUu0OD6UB<$ z#$D}O*zDz{hT1i>m;ZD@;(umMHmF^P;<%aXZx){VV@htv-(RqKaL?QSBQX-5d zMY1HQsMDo7%G8YA5RIO-_OdD4f z*RJ5!ET4=Y!63VT&WC+@1e~ikt#hxuRG1o+w6KE}lVLLj?UK{uB4P|@URFjBf!y;Txb5%a4_7a0i=4E&g0Kd`TP|q(8choE{r;BPbNr zG~tA4r!)!;x_WeGF0x0u$MDCO1=5%A1%Ipd04;2HS{p%bM_;L zC)s^}I0;mm`fg<5rlsDesE94mjs8GdKw&^$oaKgtZ#iwnxI1@4IcKWmf5&{xS%{IK zkrcdpl$|k6e&*4Msh5At7{Dlca$2M-=9Kb8 z2WxSx&YaYPR%srusPFFOg|knVa6?*R`^%y+@P3<$VUif3laS`=RO27@-S#@gZ;Ztj zUM$DyR12Qir3N;3chHYjA>p&)NSVx7F~}OqTwEt8-UM&J`<%h~Db^b(MgwE&P+11h zm}9PfP?(v2rKJ}@PAdx^7#hMGNXjyO#hBsa<|Ns>aCrlrv%S~M6tw5M7t$ph8=O=X zX_lKCP0E5dv*;uv`Qg-kA$D)5O6>e2_~TFxk~n8Th<^s8dD)R1qhS$I$a&38*W)R0 z?fg7!@i>%;4_%{S6Zha$dXg?=eR~sw%XIzt82L0)%OfPXyMHs(eTd`E!<(L`R4js+ zD}-Fge(ma{-ttc=XBV%1*JcJzHY;fDVw~$VgVF_Me$YK0#Qt%OJkR`Z$j<@idzuzU ztdUFsrRLF%jg6ODk^8EbHK7#`#0G~|;r$S;4|60&o7ygh5aMI{!J_cG!3 zV^cz_B6nz@vq2gh7@fU}EoWrVpq6TkoSELMzV3Q&mK}zpzvFc{6}B!qu$l7jI~(Xp z*#YapeSKfBv;N!<4c{iXcmKtY=)U1@D+UzCZ5CdJ_4hO>epA0?iMA3GMjcSrX8st* z*=!DI9g4QHl0peC{j-Op;UePWp`zsoR-!%Hbb@ankiA9v9!&N7GbGMKDa&i7hf52-PL0OVx1%{(NhoChydJ3L zkn`7@l}%;Y-G}n=C|SHBMoh-uzGuJ#u0`b(9e#_-u+Gy((CuU)I_gyK15T#!hI5Bu zkvGptM7G6>6P50_lWr#C=e)>ou^l)N2*jdX6d?RhJxLNe8fEZ_|2t};Kz`vx-U|kx zqsQ-cu0mcxv70wmbJwR`bzX;)GJ;vmQFOy3tYg7R{Tt_-qVKA`JUALL5I)C-s(?Jvx&V(t49R*aV#CBE@D;lO9(kJL5_vwWr`}&v_S>=Dqn&C>_Ick!Z3?_F})NJju+NRW`P6H^NkYAQL5;wD;%QN^~ z-gW#D%lLLYaNVLebK;wXpTukX-uvjL0?p{^7?&VQn)Jybf1_$02Ia7D*wBuXEwOMo+}9o+IY1fV+jxBeXEhqtqmO8% zetYmOieS%Xn^)%q%cmbPe#EZZyVERM<&)hxYFTIa?b-Ec*M~yTakkm#isC`Z9bcs1 zu1EgJr6STW z=>XK0-Zp~#wGSKCvDuKhnzJCj$Og#6UEWZb)d8p5<&jj(eZ1n=ea$$M&hYc);}>Y! zpXxSqni4JgdHwh{3o_@iA7LD`2=n%aTUz@uW6NIaFQXVb54Hk?J|?Wn_Clrp?*iq! z9Jma{UoJBoPnW1#B^)GwA4vwlAjGtW?LOQ`+@p`;**|$|Me-Hn>CU84S`lQ7=Yvwn zm{c_^{^twA#Igqc{aRRYoLTX&GO(%)?*=B~)F+LVd=VxU?UR#jxuHx2*af6NUs=A{ zmh+G=i7>1NUmm@ltJs2W5LZrA`k@U{c^@A1Zq6Ln<7om!pL{R7sSM#MKuFaNeaDf_ z8w8`*e%5+CCU0`Z;%_@;IZ0nH5uWweT|7 zO|(Havu0Q!QzdGT!9OoD5Tkup$fW-?lG|%Y#+E{t93I33k7sw~YpM!$4iFW6=ba%l z%$hqdNdS|Cyu{$rh3x$km|`|p%EP&@vo)kH93)V%$bz9zDH#6HXr;vA;0c}KZZITM zOcG&7Y0;?5rh5g5vc(lJQfDS6WXl{Or_~*%bW`RXa!JoZTq>$aDa=wsMtQ$RrTlBw zXrG2nPsN0yH_P)BL3uiUYLGFg+Gd_RU~p%!cY(yiBmPA~8a_)hs>W1{I!kcS-9Liz zk9c2>6V>$or{?e*v|3HJDp3w`-SzjQ5(~R-zBy1!c#bH~Li%V<>J>iM9Kzl75%QHq zk_I^1BBanWzu#egD0>ncKBuCW>zHxvU5LK|p!)th+aPCodz{P+`;{M6$`PvAv$0Ksrr&E?IiW{;qgXVxwrocSCf*VHxQo1uJ%)}zzj$f@S zHJQ%s<94RQW!vSRAPLR%ZJ0bAv^awy<(@Of#~{z^r-m(Pl#kfGT`ZfEH6L-6w-Oda z2c=O-gx+yerO+^rPkMTaw)@o!S@Ts|d(TROSPH)XWOYKm)B?hG|DRyEZ z*gG6$Nun1>WCU$jHApYl*i7Up2^H##2ckMP#O9J1XpV0AfAiqhQ z9{}IeBvfvWfX#SBN&Oc$-(u^BWBk^2#53}~hi-4P<&F+-p;dn!f9r#Qf8 zX{}6NW5pcbF%iXb2RYZM=!>I!hwR|!DO-(RXuNk`5pA0ec-OHe^O--dXmY#gy+>Ik zNi(DoU^Ok;4Jxb^W;%*UF*5W7;|qcWw7KFpLMVk>3_3MDuylZd zWH_iNnIX#$;lfEsKAB&*>;dqEDZkNjb1_D;^^h~45zhc{9|Y93wY|Q}AHr09|1x>% zaBY}piiDc}pm+9~#64prJNPyLOC{)(TPV2aM`EwI) zVD47Q{%W4o#+#WVNtB|NwzAnXzr;RtFCU}54U?zEXY%Tf4%#*B?JxU>ywtgx2@Aqb z_r&)4Vn3G+Ui~K_R&= zX7|oZ%8B8T`56_MNH~z79fiMf)(7zK9n|QiJpf7f4Yp{cXoGQ$q)Oo z6zjF=J{EMOPDO7Rl~t{2KI72E$i{bFlc>k9?ASno>~f&pcoEWh%SC@iv|;kW^O z>pA%qj^mGjP!?Tj{7{1G>`Mv5!gT=Fm_>2=L{u7eAvyJEydF%O?Fl- znS#UD0~Tket}pR+&K{J7p!ZpWnIv%GeF zohv|2Y_S%16h||sd{_BFsobOvH)`9Z|XCQ!2R>AE~MshpCAF&?;0^;H$9{@ zEPA<^>xEWAc)0`-{T)WRvL}NggjJE{h)nr9xfqA5wv06)#l45sJWK+>U$%VV?1?tA z+W4ke4itUob{*>}^bM27BR>50lb^G+OFpw{eIMxjO}2g8Z+X0(FUsZ)v&Wx5-Bs0? zoql3qV0>OKIeC4!YXc+2dWmb2QlvpZzEVJ0ML+Pt%|523R` zqGtfK0C#Kl@6JBzr8-uPn_S&l*>GxQitsckT!!*R8^+u)1AS5wrnZh}BSKf9JNJ5* zUoi4r*Qeh+OGo}nZK$(JAjk7@XqRJBdF(LLi>&N@?S?whtYwIsmtm%U__Brl+C`f( znCb$htE=lOU7pfEx&>QF;p2L_;w2a0bVhWby%y*1FiHQ3&XKS)ReTYGMgy4Jwmq=A z_fQywId^RpVDP+V)u9t|RCYa?m-sun%TVF-~I=!CTqzA1@@U_lq4GLf9un36k z@TI2A@q3mqeG>nd z^`-uxF&;Fn-ajQ4sNQ51;FLLRIO4klsThA+(lG!^#wQo<;!es_RLz|exn(v+r2H*j z*8=n6x?{O53m>}`x&Q^JisvgH#D`+KRH-92y`lc__~$;CVeM#jgHnH}E}q`At&nCO zvUr=*Z!c&Rem;&wz0HqzJ|_LR7Ks{ixG|O`dIt+`J|EEx;%?F*&B?xEWjTWvmOHQ?7i|FT>CygQ zwNhPzJgJ^;h#y6G{Rz)I(kUz(f{^k^(IWs6Uo zm~^*=YqRlaMk8LR9RUHo-+{UcmDvof^p|+njZ}W*A=u|PkEcMtr%Dt}fV@FE<-65gwYFS{ zRn+^fQ{F(A!;&c`%ckn=`0~qaBez|`TpbSO=m_W@kLlZm(f(m%Np0*WpEkU;!w@Kx zX3M>Z7NQO(^Wn1$8a|938M>AjiFSEQ1BBgJLUw?TgV@g77r8rwd8E}NvpaLfu7>-_ zvl$->w`+1}ubsc4Un*0|ho?gE-&(u8-c!wfjCfIQ!~FmNnwUP7s|7ycQ8wV+o)lVl zuP;ypy|(HEnGUtPE+jsktl(?8Vk~cdP|(?b;J;$=jH(w@}&ekP66**K+0H;f1ro?*!wLU?9T09 zOx4!QJCGrMhzC`$ot<7o*Ved1ek?{>t9Ry4+_a{R7c@8{>q zMgxwK6DqX3Fevyz=fw=EdijM&y*Rv?|DK|?^_>?#9t3$8v^{y;_SNIEw{xz=8r!^m z1~5PZ`+I;Yi=MLmIqlKb9wlB1WLE->BHF@L3`0i;|28?@A=@$ z?cG}E!r#~^9Lte2?(G+MT4rp+SADf+$I!bP0J;WZiW0mr^}PIMk8*$g2|m|3JM$4P z+vno_o#Aq@d-Erm^UrFLh<|Z)E}KSoN4Jl;v9gIqw_CpHU67%MVE8j8LYhGwb)}R6 zv^kv5{V+&L@V{E{O9cY9t;!R1_-v}d7VB+>R;0Un@1y@k*T?^hjBDB z-ufK4{FmV6JE~$zWbEb3POgj_TEAtBjr`m%f#)&B7p8Epw|@u8mUU_hdNk6%W7WMz zMP|>ZZTQm@#)qsLl1iqbxbMO^Yh?$(CH`bq6>9a~&w*4856JC|J; zNK%?)-dnZz%2SEk=g!g0IWJ4x0vuVp$jz(|az0Ysbfqoj_=qTc+2YUPsWM5n5S~Pe z*s|+=JT{~_+ScC(wSUGGZu{ULzAvtlX_6clQ9Z;ZIdNczjD@;e?h+msRdp=#X)R*O z5D8NQ)jyat9FXX}b;p|vrcZ^JJS2iR;<#Nh!FeA&$plP%1k}1K!+*EmZ|lVtbC5)a zxO1Tql3465V{nUcW>dFp9$@-^&7s{CIL$->$QvCrPpA6PpdY$X%JUFlv>fv_S*SNf z0H;j2!ggz-3@f zd+{51cru8EYczVjF!-%jtI)nkT8?CiYU_JfeEunOxii|E#?Hy@_8{P^yPM;mz725O zJNr3BBv7H<9mlFmI$4CGoT9e1iy!iMDV1v^lA7#*2=>X=$s?yiOOE&ptvAV&BwyWs5_^y`k6r;xX4D}g-@0xyN#!)x#|M26{wJdNhYI|U-XmYxfue5 z49Z3tz6Ol4RygL0?*pb9y$wJkOSbX29H4K^vmdf%mcUcyKYSVnp4BG$R7f}s!_Zj} zyT<# zq1&d%^<4Qz&lh1v$d$xJQYq_+W0T*Lgy?k!9HhBME+sevKZ||5YzZ$}5-$F0_t`~H z)nvVQWx+W$|7w<|)4%f1|G(0N^S{GrNE(86m4RH*hotKDZg|U=1Fhw4HD-BtI1rXS zb#&im<+DvoKrP4D{^x#b2>a_>cB)E@qZRn(y>ZG{?_!q1g5jesMr3g$8E{TWBD6~5 zp2xKNJ%t49@zR$P7J0(mY~Dz^Vo@w52>%(3n`95@7Uo8vZRW0>nyFW>2|~b^!Rbq? zA*hrJZf>y82?gCp1m<%lxaOtAC%FzJhbsoPjPCenQHLW-r?)*uTROXEX@ zrQ5bP))X#|4QrD)yhbZT5_70``1$S=XGr)qOOXRDm5pfU9)*lZ!xh#(m*X8B^TB*I z%kwUW+#B;HYrA8eOw&7uVq2%2J~IAroj=;#-iu96B`~as0NFsK=J%fdGuRVto zIKCzTc^lBFWN+spYmL+zS=3v!e52Wl4!_4x$uIvQqk&61;gjLsJSSFQ$Z`!17OvGX z-JPTCH<01n%XuO?8-uM=6^nl%5$dj_FJ)!zXUZMjEaI%MzVKS@%bso*mcPe?-lKH4 zwkl;wifIG+n@ZJEiM0reSNN)1+5@87;IwDQjZO`|-z!7M?zEj$|9I==rk{_M(*;r!-$@H{Y9Lpoa0JJvj;H1_=5A5J|q!EJ`57#+@fkrp9*Sp1^g}TmSm5ho1kF2i>YHQ)b#ogVdK!HMWin|slR@~h!CAd@EDca%%3dJcF zf6 z=N?);>rwda89S=dOqwky2MQ7{7H+SmJ2dAB>E~d3ohVlSg|^x%JR<)A;X z*#ydLYP`4nKSRqdh7lNup2nvAcd)V}UwYc*Pu7~r5?gFmddoI>ikAzb8?*=kMcxnI z`)B7>8-N^h@;X-y0gtzKq6uP;7I6YuuOoS>A{48Oo zSW65wzW6CTZ_vu>LZeT%6KdCm0p9-@>}gJTyy!*5!gE+58*uQk&MyeB1ir9ey_Qg= ziPP*LiD4lBDlR$12=7xnqTe^n@sb8;#dspdfrV;mY^Y24TTZqKOD{E@AHvHnRN)7g z-0LURdE?{sc3a)94?1v#B4;M4?F9|WHWr860y>FKNHS)czhl7a5YJy2`KSQltcwvsHiYwVU)eFAk1Okt)?msSWLd$p6I?t3-&XTw@h`8jpR26` z;@e}z5tO~96qvg7x6M*3rYQ!a?*P`)*huuqr3(rSnhS*>PtkQsxaAfMJR9ECtEOa- z|AnadQcmb@&s+2L&JbaEw5mk*P*@PKKO=QnT)HS1=53#OX8+qGpJL{%c=1n3>#R|w zs5BLZO{YP*lCL(^^BqAF22C#U|0Fc)&~HnBKjm63>ckJwva~!nXR_Pj)*Na zhU-RLp2-B84ff26akz@JI$;bh$p&vm^_teu4}FB=cFb9X>g65cd9at8gA%4ta&cLg zM?nDhTtSSpVUF7h1gXaJKLc{Y+CQ1#3p4;HCz(itKCKjn#Q-Z$^N^Q2P^`FL=F4J7 zyk~Mo<8h9$c6dQ<+Tmfq4v06^uTi}du#o_~V2264VKE2z_nVCf`KC)hmWnu#)~67( zc?L`(_UQv9xHHnwZ^#Hq>B-?s<;?KFFHJdgYXM=0KGYtYPCJ5iB%WKc?BaU_O_xtH z7gKABHMR%o#VOg3D=igw?^b4kPSaBF=N>ZsT|YctGFohYwyJWsjQ<}|*vlYHYJHOD z?~&u~{@;Nmi_PtHSxD=*Ci2C_1)5T9ItTST)3WRe8Cc&5Jf)8 zg9t7cS7e$nMw17#h$@{sG)w9QXwPc(uWFW>W)JC3_>}nbhZ_pJ)jg(gp}5&X%CNn^y*JDBs(A7Njk7pn36*uLGA(AN7hYgh4RRZ2RQ`##0gCJX_1HLhhmo) z`(}93FUN$!vE!18uM(>0a@C$ZJaF(6wmGZz266a>W5{w?@skfHurSf(M<;+!7fwR= z>5fHWF+d4?hnk+GGkg=n>fysV)1IP_DHBbOe$jhG6MayLovJazSM}(> z^|rH$*ZG16rNfV2+@7m0MhfPC`}!<;BCub#o*umX;LR!bpao(}>IXyq@%`3&45$JO z=fY%#<5==`Q)$f?0sfa)@*`sD33ayPF@73Vx^>AE+)8kA=focBA}o~RY&a_NC2yTz zSxE|&GFieq=i8sv3ijL!&0n+GjI((-LF5HGwDZoUrNf!r=}dZ)TTiCrsKPFFx+>Z(ACCsoWflq#g1#^}C2$Hk~%%Xy?bSvoR)?>_v{ER2%oEeQHPHs$ooj@F|7+J9+hGi64`j?A-^}tTkCaZ zk}c#;As4g#0_|8T@=Tcf?{z5XA4$whhO`jpf4hep(fu4-4e&q{5NXw$oz@HVq?!cC z?2++I^X0}=u_GY#9RZ3~`(4j&`fC{!xLz9eh_vu=7*~@5dEtS_`j#_qNW~6jEO3dS zt_YY6mJ7<~K>+L_67;@i@^Qyj-3^C7v8HcY%I-oB_&%;DOCkdL`P4fl?HYD0~4T`%toNRSQAaUn8>j=L-2YiA8F{Bu}+P}GFRnlsF?PkkpxgZkYVGcP7X=^kkbw7~( zqvOFkN7W~xLKUT)&e5L}Mzqal057waQ-<$L6-dt&Z{m%ne9|~L^_4LpZ@FuJfFbBq zuFX~)mJqeLkM)QeH!z@B?>L8po{?yA2r4+derJG~E;%4U^h?rbev7T1;rLW8J{5}% z)epccUm4RCW2(FLSY^F=&FzsF;nCtwEfn`Rt3u57{hH=|pOn|`VlA2Y8Cjr+FQve2 zK)HHMTL8ZEUwRpVY|zSV;@m0=3|c;-k3A(I^a1`BhKROvLBN>71NwIfg711EJ^Mo4 zyKTITF3H?w2wI?0HB#0yhCUU7soI{Tt6+`Fu&!f!%W8v^4UB>6vpJ6UW866MGXQvr zi*z`VzD4w4(I(X{%pAYCcqMO2`ig5U*=RuzUT7k!O8A}<`!f1Y1;naZ`;d%>>ayd3iJ?M?wdNawYKKKNf1>YT*NDHSf!^#LaK)LKWYGpsCGjv8ZGrW0x}ezIr_@2* zOwTQd^YG~h@jcDcgSt{ZXCV&{24;)%)+<5pV@B2I+e+}8BP;?u&McHuV}yC-^qgX2 zr!UVOXhfgfq4dCWe?I$VGRxKP!5`#4C)e!|l7((|r+JJ~Wg;?8Q6C}<@j)^_hrR{R zakvogWgpxZ_`A--+*Onf;m;)Vb!g+giVMGG;ja^%$NorkRM{TNfVAm=Fh=s3c+UQc zDNJbp`Yt#+kLd6a2pj(VI2q(%5&@w=lAO#2Xiu(w$sMXCb8%vou*#mAl}sck6ulfX zb&kvujBpix;Dw*_BP?BNgA3moXxY=ywm3IylN~r4k_WT5kWIemKiBpP!XkSRSLyYv z8P1)%E85CjQ=kIq-iX9-52dxNpZNH}Hs3skqY3G@vwf-(!tbh5-M4Hl+T+Vn>dn7` z;%3hpe{ovEmOwq&Mnd5Egw|ouMiYqoG`-vAF8*KSL+c+;Cei;Mpxu^U;Y#%?)a2+@iQuUpoK zX*P;&PwKVcem^gFtql_2LihFo@x7I1+r03iB@ItkF>xT*@ z2~F%jV^Shwn}K&+s)@06NIEC|h_-W%sbnym)Zm2@>Q6@6_iCZ4seM>4`O2y5&bxl|@F=R68 z{o1#1r{*xEug6o4A|{sO_q_7qt1ZE?0;fMMUhiNMr>l(7!Jr$|xiW1`IRqm`W#yh( zSCbId1xcx_3|f`%HZ_rZJ*m$0_+OEF5jKC~L|s{3`{h|Ybo+qLU8s@W!r#5KW6*>3 zzZDJ-RMz12sE#tsd;J4%o54>NBHr+m9iKEoL&!cRuH7L+ISmZ)WA8mvYZa2_l5u`+ zu^~~_O2h}a{F0|N7PWn%PHcYmqu3Z2S8e@WTXxLMwq)dokB;Fz+6=dL?lZr#cHo1Y zflzgV`h)<q5Hb5-0RkASQJVW6j zxOpkMN|{Ki$#RQ_tFb^u?nkvj{LYH7`qxALZtR(*sM?=;jl%>paVg%mV=czN*pV&j zmw2m$nRP-;Y^r)_E14T19DtvLVq^DO;D&j%@06^p zVwt*LT|jTz@pxa<3>>Gwze;3+zulPms?Cj(#3fi|vkKsF{YsiMczmmXkvYVwg}iF~ zmY(MiO*=J&^oHOA!fzmddQvhdhXm1Vahwk&&`5grXlg-iP_F?x&D%xArq_hI=-?nqVhvNVpaUE7h??fiOs{tv< zA+Bjv1Nyh@SW?`3j3GbO@`W$%AL6WzKgi{MM{0t(?I6V1ky7Tsl-&DXd#sFh+d__Cm4s@dGudaI>k;B8doSdTAE%&UQ%o6GaIjbGObIkpOXYFP$2k3g z)hnvohuLx|qRkAQ6s3iOVgRQ^w$bBwZ46z;; z$;JZwz7nFu;5;(@o8hT&vbtlwv4Gbi+0E~}3|0V$$qM3U*iIqQsslf;seGc!n> z9N=zGPqwCKGB;?2SB}(k(Zwm;SNXd4mx&G#)TH*?FHV(H*hzEC?C$v1PopvPlS3JW8H4@Jc!SY9cfbfK`zUkRu@tG;MY zIu1C>ZBc^+xtt*Sjg=WM98{e0@9C`x=t7_)+!3A~JHNUqo+?Aznbed-e_=5{_gG6h z&9+tnEHWCqig)ayQTUc@%T(i-E%T_1F(UU4+?uyyF=Pbqj~@~wKlpHhRV=btyjcfk z_Ww*cETSqjQSiST={$pSz*3JGkJo}E;ZPY+ipLZ3qd26_B#{-UqTKh7cBAtGGrx*G z;G?}$>VhJcJ5-Qw4Y}Mzna{CL_8-2aI}C3e7gJL-@3P1k0Ci;{4j8tJLX%nCR&n^P zDT%P$Z%eas2N<6dmXF&JQt5l<>9j{G8Gl;y7*Kk-y_S?I4G5Ft|I1){)|Dr6OMkLdusK68$HE|qu7Yli zUSH;=4M#jjrMOy&Lvfpj+&5tG2i%HN_suxM7V>d+16l7y1JCx&7DaizhPOJ@#Df7p zeHy<{%zceC=~*e5)JUJok>KNk`=7sTN>7aDCsD){x`S7HG{uNxf{Zq}tMHwTYDD!K zoS}^q6o*us6t?43=KhjQLP5=*CXP2_T&jo@#!8vF6Zm$QN%CDhyRIv;+srOK%g+L} zf|Jo{sGrw^4%6Ku^NUyvTd>tCLl^FHr27JStrLTwCr#cbI~VshKttGNOcq$XUr~sd zmi&83A_QWVFOPPqsGM4=au8a+OafE2hy}hrdl6ZcSPnxIhuXr&N?Ug3+x(hDB>~$5 z_%OQ#nLEL=ex&pUKYQG_xVJhqPjUr*-qNoKF{QCY_*+k8-VVParQ>r zi-9G#^j`f*K{XnumT1-QHPWUo6!7{4XJ##fbr>K83UqRCcIS{j0EWKUs*!Q4&klH73uozwH6w32cQtre<+Bh6gyDV4@e36k_Ev zl$e_T_{yT!0L!(wmxE3S)>jAO9$xz+?DQfFO8g$IH1}twZfzt-l+kN&Ob272(F;zz zAB!-zGLb}D=M(1o#dfkjj)8bzFV1j9=G9lqheNpVAS-}v$)VeUAtRtGZ5vL8WC;XM zqDy6vQ6sLbh7#*5pymRp{_+#r;4$82grSokha|nX9DpVeJ zFG%biT$%1!eC<5;%vgRkLop$xz68XVF@TNesbS$JBe*DybW8AM1OGz#cf@^ zjqaOjPVjRBC2Ael>W1qnO)s%XJU z4UE{e{7URBTh$IRdGs;PgS70wi13lxyk;9t{(Rrt#kBX)n^f}?gA#g|aJ()vYZuar zU{CtHc{VmpyU{toylIO7dxsL2U;ETpWtD4`p^7BUibEL7apNY%hPaxtmUrj9f<7+? zM)E^l=I&yb;-0?TZ-lnbY)Rp8afUhnn7<}H^^>BtK9!s<#a$5?v(jbz#Rb^pF@xV7d7-e@ zpQ{O2IeQG2gfvPPiv(3(JqEr!*v64@Bl!6Ez(7`At5QleBDhuX!`wu?d@oH?!CWADGD5t zFgxu`tI)x%7}Ll@N?vrd?*yO9#rf8CKT1&=yZGyc;(F_m;HA^~eUhYxCmAp7MT^Ju z6rNegQqi@>uLz_?ciU`BTcbQav3x4__i{6ZS!p!4hknep8z= z^u0n+L4iU)WWg4_HFhgoVHR<*+a4G2iBDOJ_5&=Np;*noAmzFREIVWosz1sToB~l= zInSjC0{1R9uxY|>mTY5`FFuIArJ}uC^~!8ks{Si+e9S3PyS zGwb=_Xa4Wc{&{{{BFYpG5MK4a`V^1N&KW9nUe}E(SbP_kocINAriLM=r#INH_5Otl z06+5dq#Pe7`cy7@p=M|KK@~qE>-xrxYhXisi_S93G0&hu)o!64pQqaRZ+He<`PX3X zc&cTS`HCuJ1XN)g!K7=PS(og5Z;yO_KO>8#zt#-T_STa98AJvEJe3SMYJ2)L^M)WP z)Rg|f&vYbfBqYYx$0iI^8#VmTR1+2;tgX@3$cDL+Rk#B`29^{-73nao&?G!PnV=aQ z_7{#7x#XMRAI2f>K?q2?KsNmIlY1?Cc+{7yHBv>^3!A4rH8MBDNdiSVjqReKDx$e*-Q}YEm53fNI{Q$p?w(PXcod63n)Ap`}t{2}E1pUQmKl#mFm$yTU{U@glypC?(FD-hl&qTkfoAV{_E@=dH zThU@&Mp11Xm+9dNq*$cb89uW$5qQlf&A~{XVkH{>Ztx=))-@JCy78!BPH74?D|#Vc zE*FmPje@gelctSv+jPk#T=y^PLK3)8{6<-m+-0j4r zi0|(bH+Ft%K}rmpHOc_ST|OU-0dKSfh%Aj6j`mkATHmeNjyH9#y6y;%%ii|vtLbjc zL|cOR+IT~yrHSEId8MG}2Q2S62_(@?m=oDhYwQ+Tj8vN=`e}Udk}FSjGgY{xOX#J1 zTWoI{#xNkm)3`c&x-SM@<>{P#(3Atso1TQ~{+^%d7w^y3iC@M`#-c`WsD(Q>+&Y<# zzEv@FbAY?=ABc;3Dtta|Fz5Ve@?F9?Sq!bBJ(eCz*2X9W=``O`h@Cf4CM3rg9pUpc zXZ3m>DHhmM#KhZn+y7l}WE-OFgi2Y~dv6{OT{K!pVjZs_3 zJ_-U!v0-YyYI}J;UDCN;aCR}ZqX$tpx>g}W;)Xa!G<2+C4&zux&K$uCmIM-vSu+B!m1#w1j?$gzWIIf9SM&I*Q+t{3^9Lj$VkH zvn&{&(lTptA37|UsadMp{GN75hIBUR?8fh#!l`!j4p9ab!`lbad*f6uL{5JwGs(*> zrAzL7*{LYBhWOtFiY2ZwUow8$Sj@2b*Zg6|cHUte0`t<*u~@*sz*q;k@T-60R6B{4 ziErZ;<>9c)NYxF-(pVtn^-^(s0<6Cwp&AM{*P<%CCu3;-lKo`dP!c*S?Pvf?)MU9>k0P)F(vaFmT`nPe?%;4CkH|BBzVfxT?U~cqQtSUba@51ok zF4ossp|&<0UFo`pJstNEecSjd zD6+$N%N9X2kz>b=KN&dfNlktTk%~E@=FfAVzz9S$Y!6cOBX)m?tQ4S#wC1ZtP-^q#k$B&C1<%DGoPOV%TEY=4C2 zZr{@9&SSyNg+kKBjO&>}eNP@qkixa!mIlh!U^(in2$Z`G;lCI02wOo>q`K|@K&VT( zbx<*dfE8f=j)YeAtL5PIA0-;Qll=vDW>LhOuKBX2HmB?%}OLCA%rlptt8S z)z?TsI_hj| zR(LVMN>?{6;T+oCzyC<SgHs;b;9!22%6Wy*7W;4$q?OWZrTl{D5>)^eMe?dWv~F^%nP=e9|>c9TS3fam3 ztg?ZPfx4>q1;gHtWXNqM#-$=T?kPZihu9he=_i9~M0ta+5)fu})FJ&k>AHD8`h~F) zYCx}`C6mci7;?-ec{6s%Sn6IazYFkIy;CuA*J~?56c~@5;drr&ar{TOfk?l;RYABXb>rO1A)wY9mP_C)_V=?1M1~c#Kcp^bIJsH?Bv>nj3U!n`tIS z$+HQZhBYPPdg{*fZmaE;9>whS%uR9wp}f|#G_NQ@tB)kXEj}35q@LY%&tz6-hEe(H zGB3_YmqC~DTbr9)kL_cMyi)g@vE$}AW7>Peqn{tiCn3@bkoO8`7%tsTKj4w%w@Rj~ zyiXy~EP{>lA;m4EunG8;dRqPty^w<82nGqtcngyyzVBnWsxmtmleOZcKy?);EsW=o z4dnpu?YA!8@Vos`f3IK!K#<%ATjo*BVQ+Ma5t@DTau}fr@q*k6m)@O!)#pKBQJGF) zKgC|-%5TC0*W;PlE^0{>SI>+F$9DI(KeJC+_6!^+iWA;JB+*IO%05kqif>+ z{lvUaJ-2CBNuK=V%6X3Hd|a;@%*Ysnt${yBF&I!ExUW zH)La^(l`!cbtq+x*N1l<4Y)!~oe&9|?vC>qajUM=4*9Rd|B$-sfN@o^30pHEz4J1; zxU@t0l;1f-s)$bs#ToH6v1hkptz0og8Hj9{&HOR-R@TxZG2H;=2zK)3Q^?es*b%s5 zcrg#Unzq(AF1{UN_p#!~lAWGgVdBAEK{m#-S5~Gh)%MR;!2uqvrldQi!>I`escjTu zgI7#?cNhp{bwiGuGBhwB__btl={|Hr=e&sKnvd{29SQ>NU*|H6Wo!Mo#Fw&vKDxeY zbs03^P~;qv7I2UG2YD7WQpP6Ens(2yeom^pwti?u60h64xc#vzNbDw96swU?!5EF* z@UAJm89-$zVHyv~@}%2wC=0}0h^!3$S9Ka!1Y<9qZsN|h^BKbbt3qB+V8QImWWRMe zw+5c2JiftzjUUQFDW*WzGYlj+-_6GZ4X(S~vtCBh1jg z#6pIEwuW{xvpJEK>_$b!i;^;TMi{C+oUyLwBRaKGc(GvlX=Xw86S(8Oj3;;>%P$TM zo8jmM(M*ZddFaVv9r7dv)1))7vmXaRFP~j=Oo2eYz69#ne>|A3Y8NTxpUGP6WB&Yw zNTabu7OF&^#L+mtQU$&^lfM-e!1$eaOO(l$^iy8f=QptW}VsB zc`n_2JtQ4q z`k%y6W<7Axwvh~>8HpJBHY1`*R}HxJ4W3Fq_gpBz!JjcFA3cXjb1Hj@g+so49ExFD zl^49gBlGigpPe961<%+7t$aL^%V!P=OIcw3_B#dPM{g;X{oTqWy2jtY3Uq(E`t^pZ zbq@lXiBEFsw*jtz3;KeT?xAJJ`}LiTbBD%^NEQPo}oa@g**42`X<)hPF#SinX z_6&V$FK7x9AhphRP%Fsh_X@2mi<5=dlC#2cb38&U9$qv>b(O+lJkV;y4`mg&F_ru= zgHvR%iYioJ5X4{`rE}s*UNbBm;|f3kVEtIz30%yGa48)yWDE# z8H7!h>o-j8+~x2R1_qBiC!KLOM_z8=i5p3mo}3o0p)N}R)oC7)$J*9lhXKV*$4Cgh zY6%5(2uGE-=Z25dXjRzfXxkkrJo))IVEPI_lQ~rG{ahwHD*@Z#jc zDF5G8=CNxr!}>p~j2klx9u--znS`2C<`oA)+27ZdQFqF>T$CKla@3MkL%bG$x}NW! zaxMlIFBsl(&f~hDGn1ReP_iF=lmCLxh0g~$VMCMwEOL#xb#alaHa=|gjWLzHVS5Pc zn+Le5&B0q9m(@esspaQtNz3T4bzZ+CbdE%L0SA;6<@!5+?`-~%(qiZwm7&5RF>SI3 zgTnW*xOx{57U%hylHJ2}!eT1rqY&Xgwh)0QnWV_@G)Q+R#)EfoCb&sVl>Nd4qjo&b z+t860KFWF-JtFJnn1r$p;tNDVEZ>aml236m^&B1=^IIfNrf{4J8{MK$3U)av)Mv!1 z(u3ciLFcg>_YAYe|BzC0ru|eZZ{J^QY?R06F-W9Ks;ci~LH25Wi4y*K7K!P#uHwUl zL{z*!-Tz~F!jMX$i2)Z~25oPDpFykDL$TuLPUyF97WehE@&5!g@bFx!>UFtyZow7+$6Fj{%nT0BS4gC+ zK-%9$e3VIKB{U_$UPU>1|JzqGOOw2iUd+JWjk&BhtF%*U*4R%f%h3@o>{XM0{PQ(s zU=y8vE7P0+y}p|@pjz)G&1`;e`=C7Gk8qnuP(=VWsOHn0^pfpN?cmKvv(YiWD!INr zZiB>ghrPYU*Z~|VPxZ;?Ztv-1s}Q}oW?qHeLEksPl!4Byl7{{yG5zDJ%^%EeXr)DC zIiPe_i45ES#|7Xzwzj-}yTRE7FGVYY>8GOQTzav|+`wF}iKurMP~|e5Q^$1$U7KL) z!cg`h&gnT7q_cBJUia~n_o^!DhoVS}>r##RS9xXG8J9}xuqF}E z6hT8ztoon9wNqMf7@7p0Ci2H<<_Ys_XsISES_@9zOUBWjs$Z9`X+ZeJR)1j;NbyVp zs7?JS_sE0{yf8`fRq?woGW{cw)QXyw{u=5ivb$A#arRD~VWi{*sT*p8=mC8`NoZ>& zwKPa884NvJ%rY#j*MDul35WLJCgp!RX-}(3LUv{Aickf?r2AYWiyqy85 zgW}Pyg{qF-{UcY4UF15-bBt&GB|4qWJ}B%jUVV6TN4PzHqV&#I&dA(~I)F)(;3A%;_5s%(USiYZ*bD{aecB1aqB-&;t3EsQ_=QcHx|9Rb>Un zE#o{2AuP&7)86o#wWGVP6TJx$XY-{!$NzMT^EGwYCi9A&e4O$B;i#`Al062H^P^M^ zJHhMtj#b%w+#D9FS>HOu!KtW$J{c=T@Izs;_g=rpyf73y{#Ut(WJqtZbIqH>PNT)Q z$%ZCg00=-PMEg~!1*Q;jvgL5QdU2=&xybu`mRH1v-4YfmXb$9 z-QfrYbsW!+xG^Np6*s}5yzsaAZ|dI;eF%U{NekP2tJc}Zg}tEdTfp)jIU!(;VTzwN z+yq|{!|ChgObUy#5W8dz2M7J!d!&9Uh=!+2tl-9C`1YoJw8(8LV?N&w-q0{x5*{tG znwRdqK9is~D`AEvulnw1^pmoOLODs2c2U}dqu+BH-}0inJjC#K&*Hz%4b|>w(o^*z zH%JGs^0%pLxtwopNN2|Ih|YIXmUB|f;39sB$ZFdc4PNS-T1e&{N!bM ztU;=KC=5dJgvC<)>z}JBt4vg6f!fHLUXN0dtC`%EN$Keku6YM#D>kRAJlVQ?AA=7i zOSOJrmS+gRK`*!syhOrrSqn}+dqF$<^;H`q6s23MFKc!mgRMdstAN$q(B>O>#Nx+h zxB}E-qbz>@Nr`PH+Gc#I2_s+gwx_}m^X-{1dtdyNHllK&2D(~vBFGC3lp$K2kQN1& z%<*H2fgC@e~QM44zbNy6NS4{u>3lekz66mmcih-_v)Yihcn$_< z0TqM<4xJ{4j>Ny@jC(-Er`OO>Dw^-4=Bm{Z#QLOf?v$DcQc^f)sLd&`g_S8G;ZQX-`5kW13`2fKbAhu5(YIl!5 zhT{NLek+>Iq+&c-d};aECtTVSvF+!Gi*}9p8KFTMsaw5YIpO?vB%R_-jtVm#*}0{- zZzqh8NpZ5**kQSdE^Y8&lIE8LUWz6(ovlbSqZKo4d&Ig6v4`*eO+`7gk?iw11NVyK zlE|UJ)LOL!RXcf^FQ!aDP1_IfGz=UM?ZT;l5EfpL(k;QQz}dnxDi%i&HBNf+PpX=Rv4PSK*`7?Ee~6BRqTIdiAnk!yUV218 z-UkTs?87g9(QnVf79NVbuMAh5cRgFVja%r^;i_NP$Gs1yA@i;7oMW6hzi@N7V&=u2 zkpEq$n#fM`o*98Px7G{)y64PeEVE*?3OK)8-jKC2(iY576|Cdv`JRTgMTujpffJ`o zmyLYDHTh{h3^Q97XMKLbDq93hinufv$qS;0)5XgMDkJrFCHoQx;A5N}R%+;+%G%qr z4skds=u6=YJy~j;lExDy`7zu;5u$JCCB)$j~b zAEkWH&P>cO)QiG)>vA+4O48mB?$HRLpsymE9YB1r?zHubT|aU$mnBrI5Y$aLg^1R& zHY1cw=bQuFEi4pVQ-yWo6ekw$9=+Et@aKPqE-w4tq^63P$91(Bw_4jg-p*Qp z^nzop3!Gb-fAo8I&h=w1aS*Mm_zEqHwNAAA;>pjl^&i*0fRc0&I2FWCqfrSg7~Ggj zs3NC~H+D8c^v(#bO9>H%R}fLBmyzsl94X?nK`v-l?2#E+;m5HG__~4c8UTpK?7kFP z3E{=Y3kB+62q$VMkUMpI_*_r8icF|)tZY`zVrD%zilrk)Dn4}5X-0*b;tJ%8w|hZT1NCIj}oi zy681U-o?Q*myBl-z-aU$^JN&Z0pR`Y-&%jqo_^X9Ix(RRa8oLxG5uUEQIXllHRwmA zVk+^gHqqlTMh4J*Rmkl+PRt%ufNdap{!`MpujhMsQ?|HN?cw}2&Nmv_31X#S{5b;) z{NA=cDY2X+?ZqAkUW$lyCpLyHIxdyR<@scJGo^hw)K8N2APc0393ylP*nXVE@6V93 zSz};69{t%5II_=4tZhzqmo*?j(N?vpE8R)A19zFj6uu zxByzzF*lN1ZSLvMk+1fX!6zqfOhpJTB_js8{}Ny1{5=-=1(m8eZFo&S9He@M_s=FH z&Fv=1i8*gD#Efp_qF7NBM2smw-S?T$hOTVsGguhilgwVxOwo4ry>1Q&nY^4-gYvD2 zzL~Mq5PF!vVZ?0ZlG)X<0Ka_wMt;W|!n5-oSh)6{<@P`_OS`|`9Hu!BwHD-T@?Kv0 zcMJib2>(9HGwW9&3lzmd;U(($Cq^3GnQ*qm8g8umX;)AlGO`go(-JAaE2Rw&X!+uQ*$9XLe(XsXK^o&?@RS^?P^3H;G7cx;Qm{o8u# zk?7G+;i3I6Mbq(qqZ(yL8ItL34nVOlm4ZWN{C-dAD0CK)xQWzn)d6N zcKe?)gvy7C%a?r!(7IJu4CX|EAlXPv63uq1%~THE*)}m33LN}Ag@#lG^_l1DMPMp; z{k+4~1-$H=MDJQTF`kjenYL5sl;QSU9qw(Ca_ljxOPo>2k7q`{H*DJKesHHaa-eNQ z_;-2LB7AUaiIp&dPW`(>IU%1SA=3n0YeZwSv$1`xs~{We#u8i(*kB)v>{-Db`-Wge zq(eu?g-6`$PjP(lt@XTl^B_ofvd^p5zhN8YLSvrKv%}EWE_B>G=b0? z0$B%deU=sayg`l^q4K)u1`C2X+A(%32jC*K8Ee2!w;V9cYxqLW?Y+`h=An(vO$@fi zS{yH-8t03!zT8OKQ1|ud3BLUTWpc5TWE81S0GGGi#%yJbiP1rpxiNRpQ+~K)k3U zFUsx+g-R|gd@9;~(?${z>wBYY1}*Ab9Qzht;{z8p(N+FY__N|pZ;b^-mzAj$VD#lY z_ILC!WxJeF6Z2Wv4wQ3za%qXuH6aiVS*|yed&fDmD5pu%FicXVN4#{|5eBD}C3H<8 zWTqTXw!|=r5O8m9?I^0K$i{g!_pJefSFgF_-9>L&gptTo^vapp17V~?C54%yUWN1F zT&<_4MKhr9$F-gjKRuBx>GI!SIY2xkeZM6S(P8nh!8(PQBfrQcSGDQNY6bWA(=Aa6 z8{31*p%s`FRj%C=+CM-j1Bz4PtZV9e@ia*mqg#^bqI!oZuhw=r{v)= zr1T-4o2}eOYqP2!q=keh&}c$8`MZ-<1#eB5PP9s>yO3A>Z3W<^xEJ+%J;t~02~OIt z&=J5Lh~;0{PBMg>rE|Q{bGYw&JTmWRDfgihirGGG;{KF3A&?VifZ9hQMsviqHD)3m zY35A#pdp&(LxNZ@iLCexiSLONpB9I5xsG*P1C9I^U4#OVGI?~y1*A8fd(#Y4k@v!# z-Hy)LPwgRiL4gA3I?sjy0k2<}R>(Q;Pny=ao}!u6>mLYHy~DbPSHAJn<4D}{6HsZp z{LYV5kL2J7j|+ETkgnl1N4DszW}4Cxv{f*tvlR^r;?@Cv$S=$u79WLncx~|sx3-g^ zaxP2WMoVl^uaWKE?Lk702$9O41@k>Qk%x>#j$ z?6d{z49D>8rcQR7L*UNv7GyxavelKEV(@wdC1B@Y;Cdg^U7B5fJ}RWEB{$??;P?eJ zlm&hvw;xe$#muTH z&1cQ^ebu$9_QBPO_cQbKM=5te4X!exiBoL*qekr=cecH@<2BQMW(L_$NG?>cT>8#u z1WNU{WjBMeJO5`wdf_C3-Kk&0y>F`*|1%-M*e&BMHP*u# zkYF!6_Eymq!1X-(@-!EX332oJ?Z&3lxHPTNjs zkc3naFc~4b(~cU9(i_vVxv+t|x%Y`V`R+*ZL=6OO-~xFX4ctDnSvoG9|w zM-d`l5V-5Z>vPa^{rI_q%v(!v|-g5Eb( zn-_m87)tr#qR6kOfo<*yb9f`DCr4ys!HE!a<^Hs8#Ep}#R)h6t$0N?)Do9UV6qetD zTuK;*Lxa<@kfNQ+8r+qw&$b7?o6yJ}tWUD)wKyDcwbFTi5%sjvX!+v#G%@<;RUzb) zj*&Z=iESG^%ZOrlP>w3-n?E}Dh4zar$y~=nH3$S+&#{eoQ#eD+d0mQrhw{8mLxg;{ zi3NIF72yXE7Bxr=^8*@t)B{E*D~aa_rqii=N51m{yrI5Y;-WGlqhuO-4PqmcYrywK zR7shi%eTC#b3dPjW@YA*eRbWf>`Y0->0A1|HkTrzM%)tHrP^=hv& zmq>tIWCLWST(byr6(((S`pCyD7i_yDJ}gBC1Ss9 zBcoQ-C1kkY*J4P0QyOdkwoSvPvQNT51Dj7cX+3!bQ2WWF8ovrJ7+)!hw;o~LWOMOU z%vhw%+h`uE?vVKT^&~efQ>@M}UG@gwWgyU=3(fo!bbTfaoY}BT_7>|5H4kcAHvJBe zb1V6UllM*9oV=S%jb1Ue=5*bp_dM|U(U;1h@ep2ylz_itbti9=&N^>8Jk>`-ic+G| zQW+`RTTV+3`pcshFg_V{b@-GQ1eJ>=*{JQO={enz$nSaX(J(N97tj^(2Dfxh_rOj} zr{~w$VI?KZqVFzuq|nA|-kqP{!grwhUhl(qw8pw|j)p2oqk>Sw=tWK_ehzc?tfcn5 z7bH3D;&v-%i7>RsS*Gf}H-|ET9+1rMYZNw8>d9{crq+)(BI3@|1<&+pLj-QS*-lu> z!l9iad^88`kmOuX#FtYNhg}y1dr57hQZbZ-sW+J^9SlD=-?bW3)s!{%9~5aJ`RSiy z#mGACatKP7_24hy{g$hmI?x#`$0BZc?CdAn_~l51@p)Wnjjn^xp)ag^MI-FcU1Swo$n-+MKjO?0L&xd!&xJoShaER@ExJU^Z#oWh*6DxPGZV6G|5iPWH8Pc*Pdw zLAXqsn<=OO!Q_Qr@MsMqJThG=!+tu#LnAvcoUrN!FA0DJ5u4H=F`+$Jg#Z?qNZ#HF zOB3udbQTj06DVISUHt5j%Ibl?|36&4RZyH;6E2Lq2ZzBmI0ScxK#+vs?!gIzyW8NJ z;1&q(GPn~6&fxCuuK(<+|D3b;cQ>^z=H{){tGgdV=sgtFG4KJBO{ci|pz8k=iyFsH z-8$hnk{_#SAU*n6Y#mMn+^A-ZRj+xSti`VIdI4rp8ljD_z%ab9%K0{6F5Ob}JP3|k zH)Z<%cU!dFk2YU@!v5-XqEz=^#*G<)@!Pb0+HgfQ=N^Aglr3|Ox%kWVJ43$*i{n>| z_cnd9rDk0M1Tr{w7XnR^fcGh+JvJW{43H2IR*{a^s(Ace9yqh;AmEaNsU z88&UhKs+O4epPMB6zC#gVyGp)G~R)!2ldrg=-s|)0erqD_JJoJGUA_<=ETyTQV6=` ziwsp;qf~i5YkG@x16lM$AXS|Fx5`%oA@2M`B}Uu=1QIPv$SJddk>*Y#+R1)Y$t1JHviWPah&-r3;lL4g2~BA_v6cU-%X(Hs~Z-}t&vPWW2Fx%3%lW#wwWVhif_UoS0BtxER?BJ=O=XZh zd!`{>Y*cWkVz5peBSdq*^kl`BA08uqUM?ymVNv3DGgd;8jFgS#8u^|L$qoX8!P_?LlY$Vpkd&#|1j2B3BK@OF}&Uv5>DQ%{o2 z#Gp>z#E*m(X9-~dJtAeUPAN()EH2a5`GS1`Osx-zyK;d5wTEJ^D8;lOYu%r+a|Y$h6YvF68y=g{sc=mK)8ye zyj_Mt4@v7wU^tlv4MP`ZxPw5=sT4Df4;$#)COg0~hoYn_MIRaDocjsHav0i5!#qee zG2N5@h5p>Bj#EfMJ`Ns&aYsPD-_lm3FD%XsFD}4oKMbW}m;mtH39AYJC8&6*ESTv;t8F(+ z*awA27a-*ArRB@>@;`g!lN+6k%3eUea4v~W&CB0*tO$QH3#uZ=nfstpmAESZ>eCne za>A z@&+Fs_>FnV;o#-`L@}Vrs91alRiwWRbU-%ALw%NB6!tl|xz>E+FqIT=`uYJ3an|*# zzX>kA!z7}rM@Njds9*ApdHrB4N956Z!Q4nws8F=aY-O#Hvh_69T%bHWsgi#ngvJfztpCG=oJG_u^hZY+EOpjX7_;Q)oDM-=9 zx!>D6YrfhU-OJr3P;2P{MWkM+qwviCC_aK)D3I7*Gm}j{9aKpNYNl8~{7Tx;54BLh zAtCtV=W?WsZ{hjEj807cFx>YIcC2A#f9KRX_nP=3et z3|UTPr9cC>12KaywCx$E-nZ3=@M7o?7&57-%1Ju#>Gpf&>J*039uTHxGRf%ZFdq&e zY~G||mA4sJU&5&^tOd#+g1a_01Em+l1iYqE=Psq1g+?a0y0I%*o;K~W8TkzPN+`91 zyj~HZq-jZWZO+*>6pet)jHtmw*ssC*=c^`ieY>SBAM*L+G)4l#qq~ozH@Zfrt=0b+ zOBnb+d(C3*q5IO`PHH4mF$|IU+BBo@btI|>;ILr-U^mfDp@5q>ja=AIx@iCw%7wy1cHnv;&aA^5#hH67s`H@ezSUx0e|YRNmh z>VoF?V+`4n>V|yzaQe6e>#?TA$YYj5HCKOmw~6pgBtG#2qujffgvrluuls0S$&d4b~yQQ>O5;br2%G_2t1u>{_z<|e~EfQl~v3MlvI81i2Ulrf9bAak{8Kbpwf^Huu2lQ+nk;JwO+ zAU46;#KN3Gn`bJurmENSd=2E)4PFFeHUX!1svVc?XpMfT(O6ONE+149RqyNcB2+RH z^<5hjGY#70xbc>2uKtYbEq|~a!Ep89ZKnJsAfJzecx~@ViBg62!4p5#i1=@41)ewK zWE`^J%my9p-Monm5i9_SE5;a6rWePY>t|wD^hHb?yYt*bw~ZkA?nITuDyFt5qvBfv z@O9f@j<|JyfD<7^#3~4!=tEW8h$SVMS?Sbe^9ucAw#Aml7uxphk2ekOF*y5|Vkgt4 z_@q_8UtBoD_uxGFO?dsu!}dltSk0bM#`eX)&bs;i44uM&BR>275bt-KxsugD-<4C# z#PN+s)EIF(pDjVIL=a9;&L=6xZO=g(CMJ}Fg98*y)JZXFIrLRG_7xMBlsWY(L(%HA z_;Z%|Id~O$!Az&Xq%m@LN&x_YQz{{v%;tDzWp-ks^{|rqXAPL}FUX6>q(FBrruXc! zp0bGV$yJ(Lz&$bTA?!#FPF@g2tSnsT%dWvrlLoIpFfqj^J*Oy1>XLs=Nx zYK&ke3R60m*l3ycrYnraA;PIJPga5<*4#5+rCxlt5Q(;`1b^%pAEOj@2F101Fnl@Q zNTZw-p>HEnH#FkX&3IUnQe#_RK5HEfWP21YSiD^G@?9_!VMPTwAAW_aJb_5xfGzWr zdoI`?^)ky)`zI+o&P0U>QT9$Ku|_-G%~nt~Q=QNyTGaa2pOLqDEhw>|Ks{r86>q*E zJ(YV;ssFT}W5xaEUj1fcKUnme3 zYHz`>*6;gaCpb(uH4DBoa3=pq{%u$+G$(67X=dNdfXsAfwNBvfz*xp3wbu(6D3VFV zt{BvK{`0acv>Bs{B52{Y1{|Itid6>X?wFzTn%+M2%(i-k{fZGCyWe-iDeW(D+9Jxdc8eO-&XOmP&y3ej zN=wTF|De|?H4>m~?)9}hE{!%J;b+jbtzyZhR$znU^~Ao8&=6E~3HS{x+%XF0HIF@p zm7A*6y+U5D8{1lqbA!Qk-JUj``2~zbvnIRbw-&3FAg=+L?ecQ^xy{f9S6|$JrK4Pf zciHA|PeYbjF|b|fApIBP5LPcx585z|8lwGsFaFPO_e{96_B)D>*JE0d1$x+bojSL7 z)fp-g4UX_%m%43k=Z)s**?HtxzvEEg#1IT}b9zirB~#wZB5hAWRn@)^XP3JCs?HV> zEUdM$au&GkxB-({v&S`hE1DV5UPob*dZEA1TCwg~$Jr#;P&_8Rt=>T2NYh*ta9Gx{hHFSioIgc2P zxo+e3{%klKmt*lQ@PCbmE!#m!AMrN+>VR2yWE=AmZBr79i>3OI4Qilm8xg=Rn^u}) zVPtQP<>(@RmZgQyQlZ-8##w?>5Y5;1Ml^n0;@Fx=fcGfVVzt4{V2(}g#KKx=8bt^) z1$rFs(u(SB##6OLJ4P-q>mwi|(+Ufd!6I8lzG}urx|g@cIHyBJQc_U2rG5*|!51X` z&7WN#7h&gw?s`Ji7==;U1(~aB*}EH59JTeM8U4dYb_~*M4j5kN{V90FUhxe!G0>9Y zs7h$y9V|SZ%GR`=F0r0b$R9ohRJ*=u9a?(z%_bH;_D@wm!sKCx~RAb`_}+r~!D!b-IB0DVAT^ zQ*Iu*3uU-}Xx3Wb8jJ-`9KzYKf^}?xp$iW!Fn5zxD*fIZVu9!01J-xsaiA;CV$CSB zX;bs}ck>IbkM1PC2V_W4C;iK*l5?C7<+$OlYo8!@Wz&1Ht;laps?8samQ~<%mfKCR z#z3ppX)0CD>ZpYPg|D><+*qX0U9o4Q9_1H!xo&V5c zH0uV73Wydk=2O0XgvtzjUuM2j(Mjspw+#l?B%@QzMU2ZyUyF^EEamH;G@8p(d9xe{ zSXeP`0S`1Vl+=p3&UTL1mG?1N^0Abs*tI8PfK}%_h2B3B3P6iHq=nuom3v}mEFTKI zm1)9#Vb)M`<%kTH+6LD|itZqf@Fo|lW1`(ICK_C8(JG1^bB%F8Uxvy#+Y+cJ1kHw_ zjQtD6B^DtS@~Nv>$fWnr%5iw{GI*gE*79H2u&$4Xz4UD=Hs3A8q3rgNULwmi_bGo5 zwg)(hw+jqDBc_RVHY~TbjU{B7;8?35t4dGGaY5z{YW8~KP z+e6D+(_F%f)PL)Pumn_|y#h;_=o|IFHR7$j>o&BG(`Gqif4PCo@(nEWX6B6bcfg%M zsNVmz%7_yhO;YJ_NKvp==*GeNWCuq<{095pkCatf)wWCdz&|mMS&|I-2Iy1M8Yyi8 zeuq-MFI2K1ok<9XOo|0Ndc)oZO zis4T(I<(4LcPH%XHObbk!Xq>w05jTL5^N|mI%XdL+4&l2$YaZ}zuu&a?CDSPJn#W; zJ*+JJri}zLB-T*RQ@tn(@3Mta9-%7~L9aj;LKWhCqhb9{_iO9M=UZDDi(8BI6{C@A zy-msmz~J1relWpj+5C;dt=QfLT69jUez!jJlVJBNj=<#yN)wMgN6BW$3Zdam)9@EJUe=mhBQhnyg0^6 zR&kI=W>@S6^2z|yJFU&rZCyMq%#|$EEMs0zgxwjmm3KO6uaC-wacs_6r`U$l71`fo zBWwz>=!E!N9PeGg>~ER*SoGjnytyOqA__#l&MK~ouZum0TsnbG&L1<&^xwhq8{~x8 z=;s+Q!eAo({v6WHV)Qw^zfv|kJY2l`iYG>D=V9)c`l%~ZAI;nd(me0*u82tel;qgz z3Ny_mCA;q=hXon%|)xTa`%VsU+L+zm$Zf4{x4_t&?_TW?s*`F-c(G?vfXmOlAf2dVRo?%_+) zo#837*s{cg98AL@{YgdnqM*b3&wOkLX#nFhscY`oy#-Vggy4-dO-0@uDbdiX>{Wi02< zmJebCC4*GhjC&hfAEq0s zM*pjEV*hIV*7L*T_(oN|NDfeL)M+SOMT|LMUE|=Wi0}iek)H)+y{m)M0Cs%gM!WHz zALGY0&wD~qMBDYfO*z~-EP%Kte;~-^Qdm>U3Q+5Hj}o6>7m#2Kw}U5STTT?-0Se!q z$Ue=s$ufo?%f3JZoLoLnh0S#YARazWh_k=bxk|cz=I`9!3tODP+Q}1zCaf!V z6FR(j-*CL&gT!sa6$t5()0S)Gh;Tp|tb67AiiL|7ARKorg#2gOsj{2+$%9;$Xu}x+ zj{TTJJd?1ec#!6;$Oet|w?gR4JdjC^}vuVzRQeLj&hi3<;%TZu6V`!D3SEz=${3am+ZU`LNIR z$&wZ|-CB;QsXC^{I2j}F$9OD;9)F<~51jiV3hABKs$2?_rQ6`Q@0ibw?Nn4$FkgNt zcQ+%q_LARA7Wf_v(C=HV(Vf}qOCuBfdImU{l&8f?q6;Rw1Yc76H&!Q z;icas(ZBynMK8=(MM)GfHop5q{;c;hrqxN5UZ*MxH%qeSO>tzgL-SvcbKn~lluCC? zd2OrYAX(vIez@l7r8uv2_7Np{NoDY8C_A&WZJdg z0Th|n@5qY~N!i@4`L!RuagrxnW@}jwvS-a~!4aZlBd|`2HobYfBG;HP*MsHq0h#D7 z_gVmRK4O$4Kf`YPtt%AtN{T>;Rah6V^G5X-C`V`v;kLGj;)$$Yb`}E5Vb(?}UI-8+ z7n!4kDU)+guQ-E`Br^|S<+6S@cs0lt_TGuAo%!m16OUmd$eAL^@yC^!l}Ge?u|9e2+>T9A6fOjjjy%4F6;9GQXV^Dd&TI%oUgrcxAwZ- z#WX~{6n-$+{I_#sEZ_#>7q@lB6GAgOW&pz&nyZ%c(Au%{w#Q)J=c144d^Z4)<4$EP zKxhF4Dtv)3DforyOM}X{Ixet2pQoghvl9gI1aW{v_HF8xVb*Y*nO_j8P@K!$gZ{Ku z8lNsBtxu^?S#Xq}IZt^&+P$*Taq(%bsMMHAsW-f>*b3TQZDaERd zj;PN5Bv9A86NaV9ayr(s{UEcNc1mA%P{0{-AbEju#si!4H$6&z%kjNJ@t5eQgid+l z@$tZ}&{J9|PYO_(Icu?zKZT*&s!hHt4+F3x~&FH`cvVCj{QbP%`L>dwruKp%CFA4;7Em#?8N=CNf2j z2?vZhanFfBnJ>@+rEr$FO?bXPKs)`W}_ZbP(XgUYu#aN=Yh&tQ6Gb$0)=TI(ul=;x4kuAqbeAt@`)}GRQv0NhHmrA zK#vdzKc08N*la4JqKRR-*Gg7@6uVG5bC+aMwX#(Nd0}>sKG`inQ0j zY>mtBk&(l;{C7hu=wWXdB?xs_=v3vuI>t}(-ttD41H2lMj(Vck|_{O@udtLmP)-9jywt8fXtP8gP>uzx`xU{oYsG z^-0nE2jtw(m73uUZ?0Dyj<@a`DSM81u+8}2wDBGBSEu9x^(+L9=s4N6Up2%Ji$FwV zP_VhXw(Jj~`u%`Bo}@`na?82n%1RPGWN@?baQf4QWz}%?B4~E*eEpF}1y-Nit{6Qg zmbKFVG+{psEpp#Md9{MQW=cPXf&Q2(V0ZjrpfEbAI@LHs#2g`U za7ealzKWDZ`RfLJAV1vAF?`Krij!lKz+Z|vty4wN1h0NsnlY zPMtsZ4ufCY9(QGqpPZO%NaI)ftH!{X4)irALrOtdTSjFpkBq7cuZFcN;9Hz5`!p8 zV$WULlL1WWzTzch^>-NU?Uki|x#|TcO&?9EOS>Cd`%0K&R+vgy;jRpzcCrMdK z8TMF7t)^_~`>wS~dM0oMC+GK79NoHMBIi;W_og>c)%c+#-an4;LExW@A!$Jc#asZT9-?sbF*+ zb^RSJz7hLh&nk2$)~fN(%v(fR?yK|Pnb*Az?%TI-B}P>g&msoZev8s2liwm_HePs_ zx6hL&_MhbdKi{d*ouyV%4(PqO%@UXyc>#W8SRP2EyeTPZxRVPPpa2B&3JXRL0toQo zUGDj*sPIs#Fg=hDtJ4zVJ4IoepL%S#HE8b3BZTmlENXh~->|ENM6f4t<@WoC#jxy1 z-V|^KVa1RywQi57I!6#<`qA46j0P3Bu%wqjlO2>;s_^+H4djTu_cg~XXp$%sDB=P> zMbqK2$Z(Q2iGG^iOHlsY5L7A#uL}3aKBud68h&@%CX_0~HUx1dD#6m*%J$jfX%ByW zMSxyCLo=(RR9!t49Fe(G{pc4$P&T_BiCb{~G&4AP0L?Dt>tv(~au8zaD0;3~f;tsE zxB8OblQ4KPj>%z5w8z4)Q%SHkqe{ygpBTDdeSWQ~I(bCZu${#GRHh{rtoat+eicw# z3WlmKs#MQ_@ObKn0Fvm+mvH6y^k;k&?;va5P_etYC`;o zp?v1M+UECW>p$+zbN=*lS@uU9X-p@&NMzOfZq;|Z-Lc}T#ry1_`Rqdcu-{Iut!6Mz z-3h4m{UQ@5Cia8SZ&L?S7}N`jWlnE=zq`M6q@O2T<-)Zd7)n~kZQ?b-Si?zZ(d-;x z)&rppI6qz3@_Y`&SelvNdG??AG;FeCX>Rynf~}OQ$Y_-j?*aPg_`X=Odt=FDHywB| zl}Q)7IvN1I}%+eckLz;p4wnxHgucdc>tYQA>n#{)m9jMIjX!+{Ow4!OpqZuOB{ zQ|x1cfgzw@g)CPgwmXJX-hWx2PoMpoJ;s;~UW~pwpv?SKx@pW84Kb*i#{9{PQfiEb zgh3pTCylgKC-0yC)JII@Q69IrGDIN)OEg9rZI$DRwjPX=8F$QHC%7$%bnv)znLRse zne@u}`nEJK9z~YpgI$*0jM6)x3YqL^JNuB_^|-J5V|)`hl93DOY+JkXx$GDYCwUyj zQDDQCp4j+f8`7Q#val$0I`t??)(1>0mB{g5L^dfnl6~xeuSJ4ewehBiE%v}?w;gPkP{Rc;6}z->CCLeR z-C^docp@8oeY^g<@W?+x%;R1RqWYHj_szq zu$+~UJF!=cn2~HE#wHRu*n$A{E+%7qPBdb~SW83+>m4bZ#It9VW|AxymhRJRg z$mI6Jad_UqcACY+-lw$}DkOS`ob?_(PoF7G)`k7~h7`UGtmgH5_367Hd;9h0kONlZ zXCleZ{e&mi-^2=G;a9!}x&gKtF4(%Va*0d;6z#;0?mds|j6#~_Z;^`?5{xU|XStWf zRJDh3)iE7VJEwOAq%3@}V={mpUSo#x9@YA-FclQ=hDwVnN50!kw6`4C`#$T7<5U%; zHg%Q+kV$b3+E*>e%kmH8@lFviRz_QEC6j(9Wo4hm`&hVZ>!g4@R$bS3V(jZx*QXe@ zWjFK<=I*BJdt}!ZTxlPEW3qJCYqgtha`_Z`jW0u&4O3v0QlPAaVztLlH$GOsTfV<9 zCr{cJCm%uf_sZ_rHA~@xnUhAXd&Hj<@4nhWA$lwfHxGOE%E7+Kcepkc1fw$(JaH>S zHL2gtcd@(ai872 zyp48tJziotrw%=2J7@eA1b;^1rcXT%oi`&vebA=0+FNIC4W{BW0L4(-@!%FJENT5r08 zo6emc$DmfIVIT{O8-*L&1eO@Z+XPR)qVUb7aQu($f#T>?x4~74qPxSlKPEfoR;FVh zRF4PuyWY}O)SZvX9Ul+A6%2r$P6R)WG4Fv6T|~;KZti_XDfBP8Z0}EB#7+9DsqqR} zaHA;KasgUC#)0e`m!lFsr7+yQCxqH%IdjIz20ltTxfY+tF^jGnQ7zsKwx>ajEh9^89h$>*(Le9^0LB_kB8fOa;$zr{@&WyC5p8@5`s}c2@ zy~9~lRTYMpg4zBrzCL$<8`J=<)@6&~wG?$Yks%2VytMq(tkLQH-54`z#}a2CI`d+N zVOt|j2HMs$r@l1Sfe+)b^?iG0(%y|S^7)M@7ox~!UgnsGneElQCO;(_YkL|5-a=j@ z>cVRvd}siRr95&9j5C6`zfOSJ;+z9MJ z*IA8!gm{hZG5AfG56*@NwQ^m;n|RIb5Vf+L&|tOAF7CpHGkK?3Q8g2&iYBcKHwS@? zD9Cpd+#0*$2vqp=@LNE*9MjPl(&#JjaC)cqdI%X(_yjE#`pskFk{n&NYp3VLwQOex zr}Ve37T3LLMt+elL%C8{h_yp*Au7uti%BmbKmF6TvzHt%nhr@x~@>aSudhef07-!YO;aTSDo z1aZ-m@0yA}O|^X@wdUk@oYBWLJ~%z#l?!IO?ZTBg)+z6XF@ZdASm(;c%QG$nT5K|< zXe)x5JN<}r-^FmbA}j!VtHooV#+t=?#2k*UmVhFxt-165vOZnJT2i7xd}GX7A{w() zk-cAnRe}lZ<|c~}5JTL5w~)rhwc^iqSO{$`kA$xc)t!;K^N%+#IIU_iV->oY_a#`-@@&7?!`ygO!R2SF zA}Ry2AsJokVNi+^=J?d?YiBO_U!}XaxHwPsC3sv_qUX&swp{T5`W59DXYT@&4b^;A zCURpYN-UfH_Bs;&9Ynkv??vZmi-_jv&~`WTdsRAV+YvM|H(S9DF%ZNq9T z*11Xpslz_Yy>YxJ107;$$vX;?O@h0DI6+&+DKj>C$# z=VevvsTL>=pP!eS+>9JKJKrD+w7(Nf#d9|Cq*zSxB>6BQWg6@D4ez>UE9Q!S0@ii* z-6L$ICqlO;sr5AOpBL>L#sKxy-mySZZ{~6~M_bP9)T$>KRjAS?D|dt6gt~JMP4ZK_ z&N^5|AMetYPa5Fg5a3|jYYWp#y7l?OH5A!u?a~`q`4}CRU6HaGlqA-c;BwPq#~j{p zLR*qoY}-U_!dNweS+$X-+8)};x)?Ou=*BY_^PduN^G}JG$uV^a_-~0&K}3mpygi8y z2|;KkcWVw}bE5y%Yw=(rD59V+8e~%uFuD_n_K-JqX}q=RQw<+}d)!qDp!e7*tx<^j zVCL+~Efv`wA8Dz40Fb1z>y2e#yiJ2piH`=Vi*6aqBaCF!R=1ZE{J`QE8)Xfp6+^5d zY{7A&qh2<|Ze^t4$GXa&tUB)*69{Qzi}28MI#t6CGZ#Extwd?m^1zLiff{%asjyXV z#1_FOHzD9)g|1h_-eo67Vzd%rS6t_{(Sy`QGd&LLJ=&U~DOGm}u5+ny-xJB3?_{u} zGFV0G#;{rNdQ!|ptS^}{Ml`fNnF5y0kyVCVjjaq2I2=)_q$CJY3$zk@R;D-ACP}-4 z*WAzqwkV769WwC3yGgLxnkU2AASRiMT89A?<#%zz&4hUOL)8yn%219>Y;=hqsPGZ$Ka^4t3cPVQMWK5}4^%VjU^ zJWD~aCNtEd3LJeYJ^!E?J=?Lyrxpi(K64g2o!G#0>LXU5yP26R5z#{KoX(A^t}~7j z@F$1U89i@e)j59|j3?cRwx*-A^*obHB=iGvaX^mUq;WDw9ZzE4*dzu%;5^!kIhMsvQ=#rgkhQ zLxpznP7KSI*)QUO4O1^>5W>hS`5OxKXq-s`WZp|SfPvzk&Z^U5c^mFz05j|-=?pR%L;(6L+F6dhS$qF;}z=PI_ zO(zty8#di_bw|6+gNxzQPXyaP!O+aiRw2(`w1-^0TGwM*6Q(U@l8vD;Ub&x|c8GX9 zymky!mi$GZByYq~30HoA5G>tE_oQxd0I+|&lL%%F?7oJoZBQa#<#?n7Zt(v&hXC$6&x%Lv9g0&e9iVaE;(vee@fMr1IsTanYG}Q0U z#2)Wg=O+S=s#*!NuKZAVuh_L&auTv-0zJqDP zs3x={jRio)oDsRAxPeYGolONKx7FD6H(5i9suDSLp*P`+lROKdWrK`Vt0!HvUc!x< zNL*CYfiB_n!<(nOjlksa>2)Zk=r91=*Fvtna}=(yYuvo}aYOMeCh2oof;G0W*&&IO z$I<%x2c(xLYzfWG(B1ltee2S2;pNz4kcN2!e!j&fM{YTezIR7}cNczNsP1~n5~Ay} zg;(GYUpqjI*Cga_T7zq~0S&eP`r+4a#p*kRz=2qG{5A0B7D3R?~c6zko z4sEbnBj(Wu1tdE<&K*LwxN(&Kgg4xBSd+O5l1fENOcpx{1I}^9W<%80ey`gz^%mA! zYPbts47|#GAYTq0Nag!*AF#y(^?}MHLoeew-wfuLj%m@pdz{Eh760x1k#KCzIcM6B z8hxJo=g9eN0sj_Q+y}|gbCL^lkxAadf}TM+JplQ%ovcjU2dhfNnP1{gS3cOS{)Qa0 z%%Le3a^#yiTkTMVqyS15c_mdAys;C|8>v3tK<^8c-lugLI+*kg1nGcO(-55konrdI zPZfFE!pnzusC4l|ihdjh;-Cq$Wct9R*PX7{B0SqtzkA&32YYPArgdD{k=xg3fIkGu zTNnmRCWctHsZLe=c^D7p+5cshY~dxw>SlaNN_}jGKzG(~Okj?{BFv4PwYZX^-Gn^L z9|xl-D6t}j2|9pTdLSw1<$x*h%k8w8i_#PoD-u)T7fhTEfv=p!y$bbLg!1JAf zXT=BdiBoYni&rtH?C@Z`xIeZpH{v3oBga;v4*M(DzeU25U|BQiM;cG;Zx!*Cd_5Lm z%SB$Xym(`(FllAG$Z9;XnhvS1R4RJ2`e+Z@9kFnmc7TTYOZdU zaQI(87yc^lU3_N8Ifn6ygP@0YShnX56ZgUL1a^FJ1yVj`&5ybqh^z$DURbsVf<=Wi zFErd_5nZ

RXYInW0f^`HZyvSe1c4{mA0|;U=^(YzKWXyXLE;C1jCtyWtAjE>$Kj z46{CYU8Kb#2>I^^32f*NnHE=ll}5hp#P!1vG@yikDkJXSSDEPwLGKTNKUOn4(=ttU zeq-v+%SW_0!jJ#ImBNIv~a6?)*GLT2Q&pC^}Cz|pxy_=&C( zTIIv&mF1J=7EE_LF+s~0)iI&c^v>}lOr}Fq`^d0|mcu_}P@L^Uf*DLyRMdGe`q-fF z9_FBY+1*C4#Zn(VZ7d)AaFYhim`9)r_fM=t2XSxPDU@M>*|=nM|OD zBnmz`h3_<8iG2`y_774K!xa)r1f1rLU8=CHKnUk!Y)<;~mjh({K{D^5eHa@2=4-Ye zWrW@ZC|-hdCs>$^^UTe+76Fx&c~%D{q62$1Ut+L|a<%dSh&?Hea_0SKxdK*x=ZPIi zZhr3#zD%|Ed#Z49UbA8@5})iT1`TJRlpRp1-T#PPRT@I@iV6D8(53s7FM)R8`GpU2 zXP-~^Eh>9W{l0svnsB+Ob;TyioDwy-zeV1<=e5%Bi>%%u*kZ{|L)1Mxc1GkN_VfW@ zJx4|VJ6s@8p!}0#R9%cH5J3SE5fLuj@S(S0am7=6aB0$S?G-!jY~3bM-*@~CGi=w& zta}7o5;Gg~{N`8icUAMklKu2ljytkdTEq`~vC<|cc>xD;uRzK%r%L4geEN7A{roG? zom9BqUE|5)XD^j(9{rNL{GQ9djlPY*2Mb zHTctl?%a59_8zOBPlR5RU$Vc3cHOfe9$$7i_`bxSUKN;KmU&+XK+eXRfQY}-FMvhs+z&5N^k@fSfU-qwD}{vmFi z__<c-HumM>8ra&bPugD{}?&ZJy?v;Rt(Uk!C3e=ir#r5ghiA0HMyqc317Lv1EZLPSi~O7!P$ zSc+W|3mKNd1Y_$#({1xd=U?d+cgKS(CycnSYwW!z{04Drnf!SE`YTbPlx@Ex_1+F1 zeVV?y4-#_cJHHqOM;n|qZXv6Sc1~>$2B8S zxZWY%poOaj$)cOdEL)>hYV^a*CCyLN_`+EzRHf)ULh3{v+y!s0; zgNNc7di1&(8G!GbglVJOlhYE=2TyKlQjUm;ntDx1V$2NS9xCt)<=K1*)J2 z$*T^98fDKbc52Ra#XlkW4Qv&gg^VBBPrmfAv%nlP z2)mDw+)vBG-`sScP?QC$R8P4wDTz{i`K2PAlL_>xHzv`r$JN;3o0l6Uu==TcQMhb% zyv3d*@I~*Vsz==b6*Ducg@pyNpee%c3lML2OaQB~UL^7^SwKbzf>L)GCvV+4v$hOTiMF;JWhXcr^pE!-0QJBJayo_^)>obtE_x1SYi2b56gZ&+c@T%_E z=9o)aeyGFccbE>ao;L;dsbiKiUcKtrBkAPXFt{y9QNlLnnO$~3Kgw7MGl(T8xS|E& z{cq%-SC$)Hv6%ZCm(V-*7r=rE&)+%v@75w)min_y2!~w@$3y)zxUW zEtApv;s2d>F*(h=R|6Z;^vtA3Uz^esi7wpnJRf&@4vnw*5;#1)2%oz>8WU8(5dTgQ z?l|JvM9Hj$Ck)YQJ zo>tmHQ)8z!JXlIDpjkc!iW8;#Ur$%2eNyMpC0 zVGZxRM;G5!csq8^fesE+?&+evrp6Jqob62V(=)Cb=hx1U#e}UfKb7dE&SQ%Wd&#QW zLQ2n831R4P{oi0&MQ!#jO^|ON_Hwq0tSEZ3utfsf^=8-Q3y#$gOfq@s*S@9q%lbJM zgq3t?I`37xD`s*miTCl2`vmUj-I~3DtBe7tv+SlFrUK&B1O>H(?V~zV`u)tYgBo^xR}AEemA+S=#+#WN^=oRQpUyZZ zrX9_H5gJC|o=6$+{Z+umj+v+G7zp_sxl2vdE4b?ua#tl`y~5)3Yp3T+=ero+aT()# zM5juB+(rpxJ4Kyy`fE9tq0pWpN|UKvRslSo4R?kDAx{LUBg1gEc8>3OG)&?eK_Rb< zxut-Js-+;m4buV#Zp}yN80!BwlK;N|MEP@=*B`4ZCaZ{wI5p1xESWkOu(lor5%1QK~ zzlcE&k0JlyC5l&#CxlyB*q5EAnwpIsH6a%}b*18yP6w8@S_J1MU*D->*Jr-%ot+J+ zhIq>T02{$+mx?INO7LPwOx$fYtT6bJ(M)uH1*hB7toEXN7s;*+(?p{Yp8pfj?DZ6e zFm2H|I5c5_PkGoSkG4c@4z--=Lec)m_x_EIFFf(Yc3m$l!PeLdP7o%>FpGurzi!wz zLMOF=j4DLwX{qLKZSZHOE+(#KXxyTNe(`e=^#Y-HA9>Ua4X`chOuoNqV0^;tB5EwC zreG?!mdz^zN^RRSXX8N=dwP+c#ACG@0dD?X4c>$M0=G0ounjev5 zyDGKdY7E)nA{5MsX=3r}wo;PFY{ElTS#J&knl~PI5sX*h6Xk$*FZUf~z_DT5jtDD% zGFI`lnnZHD#UaAZ&^Nb>$LJ(j97iP8s#(zOUrl)|?mvK>oJzf*^1n`@fLsGy2ugPH z+by-2>)egU`h)vqt!=0up17k=6HBZEA($f3>m%jQzLQXRAqvon@;krfc7WJQ=j+## zLh9wj(4qb57q?=!3eMF8Q9aOTf$(mKEtqKUKxW-kWUD`4h22>UIP-C40XKN;gOn(E z_;TPIBTj2!^E_o*e!hhujx+9&9BJHijV(9qBJGP~;=SmCoDZ1w zyOIyt29^Qf<_U>**_U6am5%Y5>L;WTud@xuZ&4#m3!{X!ata zA$h@d?xV%KrqypHN{~SAiEPIO($mf_gc-j`hS)japNfVo{?ZijHW5FztvA2u=&>FA z`AZ`+1czI3UFu@ydj zX1WJ+@c(1)E!*PUvb9m%65QS09SV0#kPsvkE`bDhDXj1yg@+K_N$}uSI3yvsyL;j8 zr`CS=?(WrRzkQt_a6VGTGv^#W?r~3c$=YAtEgYzvyDCWw5iyZ$)-1Etn+f51HWr5Z zD5ThE@2|aAx0&UHd>E@4EPNbD0s;Iu@Dg3n@@6tmXU=OEE1w2rXNOCx06oPdeK1D< zrN?^ZH&-7ED}=#82C=-f?kERzipK}B1~^TpmNj?g{W+bd#h54Y3CpvfTx z{g+etdNChV+FId4WTlXGnID2azPo}5<@jcoDnZeQ`c2C6=Z`6ct)}us#6qQlIQcY~KwY7tT z5S8rvE{sKwwU>+czq_qm-aj8IKv&z`3tvo3=zyD+71lI|!&)m|9yN(IfpEfWx9E&H zUK%|4P!q#Ey4UQ(9#_?k&vplHE{j;#Cx2OulAL0&4gT=%{QUWaUJDL*5^Cpa-9v8I zEBnkN#DpVjGtL$UEL%I9&TXuvmR7;?cgfGMB)2f5WFSW#Gti^|<}2D@lM~wTBa2O2 zZ+FM3i{e^nk`6gh6(up+QPpno)4Rt^!tE9 zl(9&6c+%!6+y0;63LkNYb`^`apRof%(6#5C6qJ|T@L!|5x zBvTI(19v&CdpSKW(h)(%4C{jT-LIJ0xNmhbg$UO#>M*<{upA9eR zN0obWkpbKP>fZmO5qNS;*S&&Z zV)5}0`W;#Wv;yn)XP17I9_0A1*eOS?Fy1~V3lZIK!8bTi7f5((3EM* zw%Xy)daPCiV1De@ldVc502Z&0Thym0#I+hDWHqId+5hS@?6;sf+PvMD-fPyi;sNW` zC|zUL&+H$L_Vw1Rc^Wcv))yS}A+SDbq%P$3n|=qK;Z*eLi7tgZH_~=O8;@f{8empl zSEjCg#3%KW7CTqm7!vFF(QIJ=Al?tyx)Ek%55}E@4ZZ|X10FhBo2-)^aINgx)Xyn;z@_0aF#?^fcEX?tWc^!=)SwZ&lmKhdviUc692iv&UGu-;=;39_z1lL z`eG(wB4MS4#_>vMGWG^@{hC+N?OYS(thkfps$h8z=+{WKOtE?KJK4UT{+KB2g-kg2 zrjlPyp37|p-f3Z8-{;s+^ec;eD_)qFgb~Y1xT-P5A=c`}7s5Fme`7>zRpW^O3R6Nh56Yx20>2xh5@ zx#e_DRK)VhE8s>P8&%07#?c5tbO7pbedM8`eliO2e%KJ!Mj;Ywv5nB$&~N|Ax^O-g zD2eVl%y4QNUc)4>o^#M%Mq?5^?(ZvD*QVG5Z)G7TSmUny0P&6SX}_d2rWGBMR^x4$i9a-iNJtBQr_YYmL4SuP@-H z*IloL$LF%H_>;+{pt+m*GgePt5=Qz0{xr4)1pApZf+ALGph%=xhxSD{WILu$#%+L1 zruk$B@Lm9G5C!XpM)Z+*2+x%f|55xiO-$5gxAb-EfD3i)uS!N`lzy}Vv^1$o#v|oF zas44hAH0W0YlCA0JG+FdKYAlQ;ZWD8eIcQqZomzeit(hf*z+sROst+|>=zp`ykNYC zVc?x^y$Ev9yGDuRN7pj5Nte&<;4tWcDVi|_U760#pybvi!{Y}Ovy#QKCCG{-gL~9{ z&yC)mnpNa#61ok}JE&J_HseL+j433{C;|jtajRuN- zn4LJk+>8E084vs9?)rE+v(bf4J6Y4Ll%t_S1L{LT@J;fJPjA#jp4{z0K#Fg>bj!AR z8lWK%tCnAXXY*up5m~>j^vSkwbRsWeXb+m!4l05S4Q1qlrfxd>uHwPP1a$UM9^Wjk z;eM``%;3*cXKY|gN!&SjtQ<2P(4CfJ(?MBPlFfK&ykYLJ;T<&TTJE8yEU~U#Lq6R& zf>01N+OCCSlCgpsa;P1kHR%dj*9@dIe5T;ryQ{2JaRtsClN!|Wf`|YT5(lbL4F7`# zfOAoSwNE*ij_zM#e?r#D^q16>F}b$=-9r!aqWWn#`EG1w$hpu~(~sH;a1L_h`6Lmz znmFM4{CNF>O6dQ)J8s&V=R7%0Gb^ae2egzrG&ibLV3Bj&s@zynzt2FxiXtOJ z5RXo>3H+utRPVhTWo2&owV?QlP3wK^@AXKLfHDaErbtuHeiu>)7O9RQ`=Z#>m8kPY z{Wx{AC-t31XBY|N8YIbm>9Q{J1{7dKu*vOv!&D$b2|7mwuQp%v{rTbwc9s0=7DW9a z&C*_)^&fbnBx&DQ40KWK`*b_2#kCoS5AVwRX8%Z;?V+yhdK*!$gq<;MWKtgW2nWWHH$JV~+C6S+vdP2Ae2Wz*-s30XL$1VW z^w(G^gbFXFt3L&R1doI3FOf^zTq!K6jggrO6|$}~BfinglpklKim51PF4PC(%10}8 zj8RAJJeVH8*Zt-ts_%pzk%K+u9WB;XzNPicAK#W4k)l8uM9T^h7B`j0M+$!&%#&2B zPBFQfm_NZ99V=fF`X%nSVl|pMR<5@oEVOd~6EwceghFg=3NasiF3lVq;FAdhozOwh4Qpv(w6TZEwF?>Qz70wSBONKk>1|kei5|A$qu-B z=Ss|hB+wGJ2VGyBJ*&KM|6ZGovZ!`fw#&mJG)z;zqYm>#njr27rSL8~;xAt)WAt;6 zPT|w%s?Lye_h`-f0_tcbF7-kO}N&R_UO8&F_}n5uIrgu!Lr}m5Kx(f zp$EID%9M{8B9#>(?LX`wX6Ux|PD2YpVkHm8ttsNO_03oCG8#-t2eE#eX5t6E$-O4l*sJs-{@S*CJ4kaNYZus zG@G)cWo8HbuVjpMYLpIAByi%T@7Ru@_fCpl{%0#KLFu>7{~E%sn!Q6rUU*6!$)5B6 zTi&Y-eojS-4FdkK#*B`Mxny8lLIZo1J_#}xYbb-T%SQsgk#bo{Il{{3?y@_PFS4+N zcZqzn9@LjmRZeD{0h86W7oesnb=HN|x$4~%#?k0Lb>%4nQB@H3 zAIjR5|J_qQ3b4xgC`91uGV3(z3F=5I+njtUxq10nShdo?{*iM&RV9HE zbvZZrY}H$K6xWj~&le1ZLIacsQk6mYRs?P|1r4gLn&RHqTp{&qo~61+2T)v;_G)OG zu0$N<_y|aZ*Ktf)iob&}S+$2(o;rf?%@--JbukNtz+*^2+3Tl6Gb!J1x1}FSWHUAL zX3k;p?;nndRs&{^y5U7AelM8epVnuN0N(9I^`md#C-Bx8Ul_vMVOks{A>{xTYgueS z#l1hgk$eZ|oY{TXVwDuM^O&>*(hZyrF!f#8IY6mpT`8-=G}UOk79958yp=VgeY}@7 z)SUeT5d>tXj%$i9muLm4(zI3YW->7qoF{kCXU1nfXY3%xMGEB}5C`Cep69p^C1VL+ z+L}OgT=(krKjxCUubdhIufoKnx9*BsQxGza8l?&2V@bA;DH-GUayOhc(+ELQ<$5D> z)ZXNHhjxUGALr0~$&kO@-gQCpN9q@R&#;k}#j+m`O`6Ud;~8CCq#PpkKa}MD8*KZ{ zUsWP?t#Wa$f76zZicXs}lL%}|5%Uq&j)@nwfyxtcDXgbf@OHCTyPaAjmI_n(lC3@p ziHV?xlxp2{{_eUHIuSKd?7&7oBjYH>sHKx2&yfH?0N%p)YA{_a+6dMR4*8|xC2|Rn$Er}1HF7=1PokM87ZK5K-0L$?SeL|rNlF4LRL;c0%Cb~3;WTk&vYA|3Qc}$M7T|ES+?hXh&%GkJS{sKnYise5U~w=lFSqB zdI7=~Z`>Ha0F^cL%-=*r;9Kz$Rx{q05dB`h5%~1hRF^b9NGz`eHdLJZ?YaRkt+oIazuWh^6 ze>>i_R7)@e-ds{t>~I8W_|cYh)cO~QDWXn~OATC)Z>ef)40yc3eFTlY$qS;ibb93k zpE`U`I_jNt^t_32XI>F#x?EMfGVd9*8A?o$IuCkvjf{g5%$YrnAMd{#T4}Z1Gnbam zuT6Vzp^%$dMgDO+i(rT&^f6R}cA~?(XLzz|>L{e%3hMQVqo~R%{$7Gtkl+KHZKaL_NtP*b?_y3X|!* z`U(zdoXuaL1Z9oSAp=6p41yX}PNpC%TYYHaUSqay z&68S*7sWdEy?LXqAnvr#<9Sr7|8d7_SJ_f!buH%3BNg9V zY$F$|ST!=O?Rp}^^I~NUBJ_kz{kaY224G=w=%fe@L=Z7Wj%;2*ksj4iQF-ftx9{ll z`by<>i)m}T{v*C!r6f||&&whE&$*C9KOJqtmuN+1ycc8G)q)(Ou^ll;wVGeSvR35! z!$HU-^klRI=97Efpv`RN>&D%LU6uaFY#(<~*^M3|l(M#xAE3CP9jR39FS0^_XKPb@ z68B0%b^c|Kd$YWo1XL57u%Q|dZR-Yx4LRrUm6w1%FWyP_<4v=eiPx6kIo_Y9BRveU zMak7Zs0qJDykzR=hTqfBt{F_$a1A$~y_WTFL311%e)UPHVx9-*J>r!As#Q6lXkVH_b8qZ{UT4)t2- zwYA$F2-pa*8{+x2o(!csa(AfKiiThu0pSB@XRoz;M#ch6P~u`M3KuL#*{jKt)du8+ zSaiKf+H>hC*Ip@L;$10R(trvn6W}P-shU|0r;MWncsFfdhI%zb5S`T3J0Oo**I4Mi zB15pwBPh=~g7SPN01mR)r4@MrdEzx~bfRX_$_~?B+tEg;i=4(p`K&3_3LH@iF~4H3 z8Hfw8{SeTL%`OY3Heg+uKpi1V9x&xd2YER3=aN_vkkhJk{(;Is?g9v;ShL>nk17Xu zLb`fs0Mq;;yFzylX?N`tLEq#uh7e=izt$w{ta7?m6deS2mF#5;;s7ppGp!13p2`zIJOGBXx3Z|z!6?f(@lQBl;wlZ%f{Sg z-#I*J4BSLQ>DoIO$+M0K}da1*2kFRnN^v6rq8PV(L(_BGrsF0u@wZtQhuB?el)T#3M6CnmMs6lcH-q@&h4wZB6_o`@@mIUWuums zt1F8)C>*)3@28rVS7=+%O)MYfU-M}HtY|-*{gn_r+dDPb{cEDo$#$ZpZ+>&K78M^q zaA$A~C+5pm*3~71SML*HHS}{p$Y#ix>d};0$99#nTPE@^_7AF>Cv-M9ub3Ls=-9Pm zQt(+B_d1bgSxqs0GA=BOds&t92tX?s0+Q_2O+Ur&WGYE`5ZVBG^wX zXxGy2dc7IFt-hsJjL5TZoigd|baok9y&oT+vPR`F-_nwA`i@~?i&UOs9HybqxE`e<_KmgM z^WNezJ!}ZrU=CZAcAI_~kuHK)Cj5at%C_N~SVJ2DJ6pI2$i9D15fxTF@-=CWRu2kcPs@EEI zxN+K_bCP|eTMWj}l$c!Ij?Az&V?!u>D@h=^7Ja(1ZqB9bGQ@c1#6d6K9wIR4bW)(s zd8infEfuUt92T>E(M0+BJNJ#dr}eiB(+UK#9)NX&qWNzOA5UA0YF&w;{7NCj9uLQ1 z-~FG2bH-@)r_NwR)(?DL8?63&{BK()Ev*jjb8A(zZrKtNip*@x*84?0tJW}QFoRP9 z@&_YlKwQH~F6qOD%D6wYV%lSCbRM{&zK^b%PqUUNNL>Y1Nda1r_5HG{{TMt^DsuLYh`a72bl`z76^2^->C*Q3Zxn|a7;G-iz5 zD!qJ81#==ycJ{%j&~eST?&U0Z(Vk8%(bG-J~SNJS!FV3<-`isGx#kJ zWd8I&uS7+Po%(j|-DL^(29Y8+Z?sA*wZYJt?J^L`ERzF$7~JW~w>l<{v;dcxr*~w# zyBi9uuIx=0@hj^EBP6-dGrA-V5_$a5Ye_D^6-XfHkJG;>mpRc5h|S!5VMRgdociBO z#&57TH#b+4GgPuVtsi}GO^~4pPsdgq;Tt-V$WAj!aZPRXc=th(u$0Ee)-VzrK zC5CU$V+je1yQ#cdbkZ-&~k#dlW0lSJ$X~FTfE2 z$S!`e`#u{#nJJCI$@=lP2D}|WkN1+^{G4v}oB^)^@1+u6h)FO7*tGMQu#|LL6RCRP za5c{@rwP}CjG37^uf2je(Sad|DEk~*Yj(Z@$w2`3 z`X;;(M*OGGs{_-qvG2B0l{>skzbIFb(1rjGm2xBJ zhe1;_s1iK4czvDeZb!RFsp#xvIlphzh6Qaxzn*G5W;sq(lPnd`_C{Ulslz@V(mYah zc{^mEdbVnqu}Z1z&FryR6|!%z?Wu3jjhe!zs)xxMGl=8Pm{Y8nHh;djc*5r+q#~%p zwl;Pq+CP<0hbG(WQH{rnvb4Q#bFlj4Wg#e?#5h_h2dvsKpE?d_up7xK%yRb78M*Up z1SRw6b)Um4vR8p0mQ|YeiZirNf9t6pr&hIY{P~-=a!KKUtCNnO<&XvWKYiF=Bi0X& zsJmBlZN7rpfp;G~JUo8pTm_VkjjbuExXC;kSqTG-fTQ@X`R&a#)K7STwk#KkLseCE z<@^P3m>4d>oAI^X@|mh@3MRTgm0Yt692VZJvbR8s5jMPpJ|z%*K2OI;O-kqO+c0{C zh}`|rKx;Hke6_t$7x{+Z2A9rA^GheNao&vs%6!JEePx7MA7hD@ObQJbzPWM}i#dEE zH=_qNLn4o|jvwnsExw!?tK_%zYE*!^#-HH0s(YY?#yIVNo^cx{V{awkf3>JVSnMdpRg}rtZrVtrRRT%Auh_q^Cihe(~Sc>w9rZ(bSiaIk< ztbX!(*K{2HsSp%-0>Mv2p@5@T1F`B-4pdd-5#?LPJ&_QYig}$a8$)2Mg%eTcAvH-z zec8x>k9qHxQG55kNPjv8UNxa1#iDJR|9QXWe`}Wd1(h#bcz5d2bS|`hkh?((rsGgG z&C}eoq=4cWn*kH1zzy`p8+U}KOM9*do|7Hb z#Rei3*8?>|`40vq@=uF8rXeCof-C@lx{Gc$zhH!JVa5v!Q|QpbL_lubVUCaJ$aa3Ig#cDG*J zwD%cU^OU;44;zpCqbU5Z0iVlT&c9SBz;CGQ-&M#KYteOpT2)n5Rv!52L5)g+6!V?0 z>TbDwNaM>-1v_-mb|TCxJOU7|oc}F{Sic*7ErGsc;zjr;j@QIJE^_aJongk zKq{~m5L7M|;}J$nDsL6~;@Wt1gz6>Lv=wt@MuL|23)fG{F7%_W)-!q+5?o&I!ygFH zXQj9dfielIyspa^;xwh~1PNVYl6&5@2xT>j*OvGglSkt*<$4l@Jt}?+0_Z*87b-x@ z7$Y~wD3m|6&OO7-Y<1H%cX{zHfADwa+*|J62VzT^^XV_{=CM+s2{|IONyRU~-No<)K z`fKHy(29bFEfGQ>bn-uzt@2nNN;OE85q}VB-j(>gip8}4v+-j9oM-Df0PCy7y($q- zmCtg+?euLv^+b$8!V1y77 z)$Aho2zMhOIyBeuTtd5N=jY-s#e#Igf$Y%uXi9>CicIkbZ3p+=MuN_9_Ou|$wVBv`k2K^BA>?J;dhdxmXmOBFR>QPC`pU?p_0FwjQ9o-@IvMSZP*}EuwG{=)BQ@ z$fBFJ-+Da2rs~Su@)M#fM?UYB0j0x&-9$coYS*Uy(T0#?RuA*2a@XcKv(~jomJIRz|$yag>~dsFe&U z?G0QP%uH$sArzEFBqlIb9I@G1d6Sc_V0xDj8z|4!anl&!`t>iUo+li!q(vXV}$ZB@5q$iRe1luU+}J z-MX@5`w>dnMZHE=)>GF@!95eGxa6T91I|m293^1w4+-_R+^e1H)U;p5fjzZnLjas4 zIH48_Av_VOYBcnqDY)OoT9;A9OlKR@IaIQ@OS}7{;NdsLqjzcRGU3wotF^9em63tB*eMM z<8%vy@P?^kt2r^(A_&4BO%=!Q4=|P#4)8HO7?MO0@il(6LEaK~?>m5}KwMlI$Rzie z;s&z^UWO?g$ITp6qmUo5&}#7wra%MFNjZ23BV|hlW0&fyaj@qAF9Fek%4T8$SWG`= zoOG*c?=%7Y)S;RYjIi*6_vo3Q_FULr3_t0vc{Hzj0M;D1!)yJyGe0!D#ETIQ?{mA* zBY0P>xujNwgErG$Y;l7Zwn9M93{6Wsv!IvZo+lEnU6`^ZFNB#5R0jI*h#CWOsXfQU z416DB7WkcCi&qA1>%x{=N!vl!Pck|7=cs?nG@v7J(8^VSw8Z%6w71s z1(}#+K2L)VPXp%_&sZEWb?lt3aiPp&`&s)*RAFIUZ;7YO8VmMJtV9oP2MJF~V7TrF*4BW}uS z=qXKXd%xF^{H@siUmSruW)yFdU#ynR;lf$4pu=8p)Rwx?x4|uWpCOGXG0AFWGcG=b zr%9Od7X(HkStfTrhB*Ztpx&p8uTxPd06ZEuk&tS8Z2~NM!v^BKA43?I8Q=s7?5^(& zZ;6hTomS+kw_Z-1gq)V4fA0P~PYHRN*c4W&M!x#aU3#vZkd}W6f1!&4D8#e}c6>p6a<5EUr58r#D|X ze5K^u8E-qU#5B)b63jGDY~8R|iJn%uHX0l`#+&tdA*bc4iSw^sL=}<1v(w*;RxU6Oyfbg^Xc~8huuv0x-+{#%T4r?Pj;AWItpRV#!Qpn< zAx-MXUCo3ip@XJDPtmnI?@=8P-twf}NinX%K=#&dX;oNb0k-L2sPk2WC2Y0^4t(tK zwi9eSl^Pp+i(4fVtf_(ppyU{x)5ghRB$tkfxwh zamQ-+JX5*g)Yph2B`V06I9?t2Vt5Toq{o<=3^1b@1bnu44?9o5#Hhe{n-8K1Z^nk@ zJNsvD9{KsNg%cWJnQe_-jS+Nk zwSQE1Y7iBg%;@1ly`R<|7ICpLn!=A37Z+#8bw?5>VUXKAaB7D$JG!_SyZt@Xu2MRT z#Eh&$4&5kYe{BDTru?8?r>KT zq{0e7mSZ|IH1T3;B3e&+v!*sYiU+}Ikrkv`_M7#K>#bg6ETp|$@?KF=R_?#H{_$n$ zZ-K>0{9i*wL9vzFz`yZ;{C5SYQ$`7#X6mO#(gXwoPlrTT=!h`r$aWRksN}>fe+6Mi zz&yO>@`HaL;f*q#M1>+?sgoRI#;%*-AmXfV|KJoep%FX8;t5-gjfq@HrlTR(@=xMw zi!}O}keD6PRF3rNIL2LE`&|=70op}G>j+tsk@~9UFEHnNcur zAVl!K6o%$Kvw9ijmRN@9Be5elx7~-2viAOnUn_isrl)fAT-d+PKibt4t<{t~NAX+r zKj?1vk3n_HZh%X;s@G2L>C!V}^p$Wzc8{EiVU~2(v*jL3|G6brlQ0h^0R~Y(Tb!_+ z$s=IC7NXE90W|({NZdNMrb>7+D85(09?|H652#hnn@Elu9@f0^m%jPk&;@5MS|O>* zvZCh3sGFuI)4>P>Ck$~P34(31DlpJP;dth{0zs;fK102Jlm!p7;$WL$Q%;PG9&Q{7 zYW5GIjO(GHGmOu87iTT`8v?UP-u?uO-=t7tU7>ye4*ax~kelpKykjqVs0}T_xcd{F zq!TeqFE!vapyMJ81qns|xScUN*aHPckzB>0z>4y+%8DGe5!)Wy4k@)dHj&kqBO)qP zKW?AFOG+?Z2|9FlPmcd0OYxXl{{xA)YVjU)4drdVKXlhF(qUDEnFMoq0`t}Tc@(EV zKuW!k@rj8)ame)-TP?LC*$KWZ{*cpc(KjF2A&u)kPmEslR%&h_>q-lg4Ust)B_)_Z z8|h5qD{Iy9hkw4HF8kl_cK(uGh2P!GW7xD(Z-d;u@tKZ+2?3wmZ~jhC?>imC2dR)Z zUJRpy$}T16_+%Xva=#?-QZhc}UiIL;Eeo$WB_d+dq;%dPP;7vqIxhqHUf&^@;l_**Iqz70sZ9ISis4Z>vC6R9{`W4U$LY0yU*bKsS zm*oP5SK~sUpYg|;^yNnNM|S*`&w;2texa?O4fBRDg)xDv6=b**rM6v{Qq!gL0_er} z_2HK=pY@x|6a8m|iMp!E%H-J^vSnx9o|kZL0;i8u=qS6{)`vs0IN?^&L-XxO%Mp9} zDHR0nCry^^gZr?IZFsdcNy&_6^EGhF$Y~a0Ph`8#J8h^ zcLv{rW4_8sSKSz#ORt&UsZAEG!(V|_rKvIktKlR1MRCct2z91z^VGFyc_hZblDT+X z6rfxNn*^21HQA>V_rBREAJuQgZp+fB2wt5*b3V%8^N`jlx2pb3V();e)agO;Gy@F@T6zY;OFJPHg>1<%r5HQB4^2{eB?g2V{*OPeW<-$YZR_lfQO6 z*hB*-n=mzOy1?*D@`t|8q>zxWIR{lqJQXE6|J;I3%-6Dg4TN`BA2w3|lJ~#uO(b1H zi`KY8zIw&)F2zLlCiT4$C4<@QRKALcXo-sQwHi%5!>Lf+sj;c^`+GjeVZ`rgxFQmW|RAB~j*k$6FKQljKg|L@QIkFWpRYQ(lt-a1HZH_B4}w`Tv}KKMU- z;F3?EU2~||?mqAIKgQrce+6u%3ExVqz!!Ch|M`ReSnB`3i2t&&|NCxro@$yeHo5xT zZl#^V+hh59W%>gk_xb=XmF!Nbv0_fJZX{foN`~8~vpN2; z!~d_J1`|ef!U-{J92Z24-G_uRaB!rz4`~G7zswGL@?G?pc(|L1q^Me7IaNwWQAz{|C8n5y1N*;z$8!hNUxUetx3AYOhTb3XJI9NxZZh+I7-xzl<42045blQ0L})ETK8~bxswu(+ zDtY()M}z;gPrqb?*XKiVrP)j@0PWW+LC;^j4~|-o`*pRX!gUIi*AA*W;AGe?&UY7WDo0rNipX@O_> zO}qIqC-4bqjR6QL>s&?SGDpmMC^~Nc=C5Qe>D~E)jfhUs75`9~$33kiX<-Xr(Ou!4wo+`AIw1jaBYSF+}d;hUC;k^fRPNN-}N}lO`3HpYh}uYtHJp2Cvf8cfG_fUwnEi3s|MSP?D08v@48+KLk&2 zOL$%92~?F^0&T`8-TxP(^G~axO^@^{gOrdkw5_deL*1dE2eQY-+X@r#hmW%n?-$UN zqu?m5W_d4nO&OTQ`CaZ?++Q7H9WXKR*$z`(?2ak;O&7nWEH`dGV}fmcn_`ArtBU)I zcD?_rwGzWZf%AtUs|AU-4`#hWuxK6wlr4A>d=}7Z|UmxcJ zmr@d;6CX83alZ>YfB2|G(^1R8$=yK}Z3jZV4XY&Bz$`#78<`u}(EC-`Ui0S34*f}W zwIE))+wAF#^cK;KXSUV5$`a6p33rSkE3%Hz|Le|oo~o?&k(%{7HZHMX1L16C0MGrm zmW(;R+vJOIaXxb|Ce~0@6?+1B5CMCI;s{w@u{EuqGBjj?j+(!Sxia*}l84&n1nAGF zef=6e$lF16dU~2m`3;T?Qc0XlY$ihOdw$rFc_RpetAm69=~yjI62ia?wf7>p(xlb0 zD+E;?zN;?xhm9M}JK4$Es3<7Gye4*;vwzw1n&X@X(|_|S|5W}}F($r%JCBRw6*Hr& z&19|h%GM)}hSXR114NeW?CfXfK<9R1UN{Yr>uf>6MBz||Xo2tf_Rpaq?k2iEu3~4& zX?~Ag?N`~7!aSJ;Br!# z#*}bb4St9^qMXK=s7CqB4UAydEna?D!)v)`QPQ&S5Kju{xdv_TIw zi_XO=t9P%m7!y`zQr+dALC8p9(# z6|exF*!dqpe(xh=*AL7rEDTIcijd+`#k{y~?2U)>ku+4#X!;VjCV<{%wd;nmG3c;9 z=|`18mFX98_icWaiCSANHA1y(`h%aTtZZI7fAR-mc9HPVu=NP=SEoYd{*1Nna&bLy z@H2PK`=PIl(pIDi;SnUx%X8NOioGb-D-d}4BsJqD+)XfZ>`Ra}}q;21v z4XXhBgPv|}Zx@RseBnQEa_hgl+%5sV+vij}c$ckRQz1EdLH3^<0vVE1S7z%Y9GgLS zYBy2*nxT^0q~$H-{nsYfjedGDu^U`ls|uFlFMt00QGE64CBXaWCEz3E?blzP2h(d$ z_s66bG4<`=-+p7b+~jHABzQP<4wA@6!l$-qgI~JbS_q}ivLrS0uPKa&!BO^zRaWA= z052Gua4Yn7o3c651`|ur`u_2HCC0r9ZjF;fQtF#CF@T@XQ*F&y8?edvWXBbcW^0fK z(|FUu-=AGeYGtO9n)=WiH97^%YPiH!%aluy;p2nhRE(?TwZlo!ROanJ>5TO!yJ!#h zE6;((aG1+_ptW_OixtN#JQjfy2SSt$>=Tgqbr(}botn2u+8)jZRf-}uaAh9oGzZ|9 zjtq*GzN=+v@jO6_{?QYO%j-;v1ELzBsTxef@j70PQ;wrh~50CRcw`>?n~Id?ZWN`NcEgJw{)X?1mhu=zG727>b?dbJx7{^QeFxLt;1Jtn#cy_}F)#a{zlJR| zR?X^8IGQfIR>4h$O6gI_|4Cf`U61}NgZ(WVEI;?5B#TAG#UeF-EQwho;qzyf1vZwk zdikA)u+;?z(+q<5|^5sN4xntb`iMKhy60- zj1yk=;Ud*~2Za3l7ArbXyPup;^v#~fXx4r8{JTjXn zlL>4q#4NHi68a+TG+wC2pifL{yxXyp4eAZ=5^+9FdtTt`6M?)Yy-e%DfCwnD1XlBZAzU3g= z=oBJXPQjw12v_{Ckj8vD(LV|F3eleJW6TOQ`DPt~L1K{W zP$b+eRi)7DgZArXg^m8W=MzDm@$)aY=Q}ayC3e-1N+yvx{%4=Km?U)Qxwu|>DCsD= zUw;X~CC!dOJ07UE=uszd;p5bPF0Jn3mL!t`q;{&zASU@xyX$rw8UVs~@fqrAcEj=Hnf_=6kcfWc z4aW%3dT~2ZqXaX)Gf`f$6-+Hw&Q&+8zZdM)2<(Sd#U$RqP*cj*nb zUS0F8j>piA5Z(~jTy2T`d7}G|-5u^6t4!c4OyIjbc%Ym(dH%nYzy2ps1~Z_rd0ZSV zQTo4FK_x9>HzLNydg(!WgKi=VAFL~FnuU9PCg&x=ArevuyCokg6{=|Ql3S%950_hB zV_|OS!C!d;Ud=AeM!sO^ivuE-hd!2QUE^}oYI86bWMXORuk&o4G4?3FkN+UNP;bxr z4Q6}@0Qk}PtRlXxFw*X;A?s4IBJ~)nj?=U&@cD^qh6N+VF zlfTFP6soOSC>Y7Wj4A-A$YfK7NIJXn0G?lHQ{`3vYzZk>8o0WD5l+(L5-DyTsI~E=kcWHA5ZbKrC!OU$;=>omsK=Na@O-uB{6Xh~Jm(nlr17$by{c4fN^G-r{HO!e`;yoAN?cX~+4xXdFb>AEytIZXRblM7^0e zdSkA`%ID&pX#|-ubDm-6!8{TWx#*+V-xSAN0n1cnBh)*_+wH}ruV&Z=L}X@mmpysx zAv(}l?Zn%kDMhb(VEvrO$TI&AUvI$_XVi6Rixc%XKLmbbamCa_t|UjYpp{TBRcUxaQ(AG1*3$>|GjA! zM<)-1qu?OqX|WkE*uR+3RDIq+i?JYdBOw%py?C{*&fVfNlErBdSJEGkLnKJsVgfc1 zlm{v?v+e_nV&lzlBHqU66H@C&#YvdPAH0u60aUc2DXS;OzDnm(?@+)QSXfg1H8xsW zo03XOvb}0yR{#(KObjOtWs>^E7%SON7OoB;mJ<1xD}->28X6x>T8{3iWW5ok*)P19 z#u#(d?mSL=uCA`aL{r+k@nrWR2<-NITu$B_GtjpRdc4egmUCs^BuhqNVuX6gO{wd) zd%@!9TGfuJ2#s??6s?Ih#W*u$Qh9E0hz$0ve}O*@^2{61hOsZn-I9mXu&sEoGP1fY zLjztUyx$DxoB#GmQSw)SkM}HR44YOTdh9DG`13gSDC?RSRzq;~N2r<~-0Gvskv3mE zn$*`6tQKJ2DTM-Qo{kt^7cO1*YgrGhG46iZ@M4~yU=oB09MvJ@;i%Y{s|uS>jKmsa zr6vl66)iKqgvm?=8Wvr!j(I1uUoeF(`kru~aZ~*}t+$|np$m@Y!1l!5)4H(bq7J}g zfg_j7urdH=azl4&-TeY4FqrwVk$FUE@8ZmZ@083Z$K_ECEaTi`hCHwY3*&E-w{o{4 zCJ4Kv#QA=>yK1E^Xs_;#*F7w|o+^uG-+$Z!4BEA=j3k;zY^6+)i_YT!qOub`p3mJw z+4x)5xkB^NfBfjv9Bl4ip?hxqzlzk5H$SQ2yE0#4oZ5(924%eMbWEgT#o&oY{IcIs z6`j$K1c4X|gMhtFjun0|M;~{R!N^C}7LrrqGbUCwi}gM>z!DSMbf#NlSRYM<7;8}d ziN_L0%K}*17HHspUB@`&dj4Y*$574UAm1=JAu* ziXowo!DO%O4HmGr1E!C5)(CrYzRBniM_>Rv%=rt=z&Qe&;6k{=WoHlG=~B%`6rvSB zkK)O{66%u??&RwGTNA#buLVa5w?d9}k^jvC{6Pj>Fl8mOToyVt}gm7S=MCdaM+uUOjo*aCHFk=r#;ICk{{cSV8 z@J&i-QMaNNP3{Owy`j=>eMofR%dnaagGQ*O1=Jr>& zGt8fbICnDkbhkU)%bjvVM(tgcTmqoT@L-CV?wkpgkWy-C_-2}30Fo|q%I#uAFt)I$ z_rBd$j^(85P^$*4-1y4O6Na;4)f!7fM%@hlY0Y}vS3`qe8}(P6u$I{Tx>b{VkaJHc ze`$NzA;s4i6Py=s$vqE30 zB;?b(GMg4@)i$>M7sm44w~56qGTvx_)YcFsZ~WZ1SQr|@I#UIT5eRsLA6dO1o`b1| z!nlBvR-02csUJo9JF5|=46GxX{AOwCgWMB7v^IE)GC7c5!R-yPS(c%&5t32})C z1%J=1A@8wCs__?xU#RtN`BGwFJK$0XD}^71(O~6Tt9&kAm%PD>ktlUd|KSk`94c`c z^It!6aC_v!Hk*Q7X&Y)o4z>7zjj@aO1%4xJS=uUX{x&l>w6>Tg|zd9 z0&rSq>Kss|9fCPtVEQPvu76MTwxaTW-$<=+g zJ9xS6^>S@6(Dl5gg^d;MPkMTMNNGL|?49D@=gVF{2dCU5AmRUWKD+*vp@%zBWrpc@ z(+06f>&RHDT@(iJrr4*qASFKb#RG$#v8|3bIT(7Gv`H|CfU?w|rDyQuRiq@}H;@!^ zrootYv+Qv@*}J#+hdcJ=^rK+VzAiKoHY=(T57{mfjA$;xT=cl&abA5Y*lJn-(-^)y zKkH@wK}~14n*ZIm*apIFgGK4Ko&6D$)c4iI^WMRsA(9qTpEC*98DT#yqt0&$eeiI+ zD?4HK3j*42FPK*`wk5>0IX>NbW3DXkO3?-5huNwfATcrFkn>ZszI$1fO((vG404ZM%BVX}&X}VDmQdtgC`E04&TvV>>>%8w$Vxr9OK7wkmq_`o5 zH7hv`M;c6ql4}4{4$Srtg(fjW4?-v1nm=MJHkQ|;-K=`9aWVGq&&u4h(q_CxL5u^f ze)`M#PI7HFnZJ@GB9F-o<6*->RKpmsfarv-W=)Yq;Yz8_PAemph6D541Rm+nM(s3P zIen%6Q=uyQc}rpo&OkYfKT?900;-Hq$n5Vu72o2H2{NNI-GEbQTR?fpvo&-OrEUCa zrMp^nd1y~-3P8TB`)ed%JD{v0#eDD?`!_&sU$hLWqwQT4zDJNca9}}kB5-K z`*kWVm7XI0K|XMH^r@6phW+mHf;sWIcv zm(B(0nHWB;(6EY;Ys@aU`t^GR=?>;Q1x0Zqmu(5N{BFE`HoRSn5}K&K!o8HicQ0{! zUMJpxDQBX%5~f-uHSZ)s(ZC8)F5@!wE+cGzDc{hrhJTOqfxVLQuF=c=UaY$XPV35F zx9{!O9eUcLQO0`#fXzJ-em6p|)-U+F6U7eIfcBhV3$L?j;;AMw;-fIeOXWGl7`h(8 zH%gEJ|LraxwpN$*Q!+^Ol*D=v_)&McJ64d$#RP!odAG>Puw3W$5IKhRJuU`xTfbLNU z1dtn!CK|xq{k@V7F@+iHlYN@&60DV`L(c3Dx%xM2xEnP21tVSb=10Ubz$qV}`&)PK za=e)KpN{J5Rt?o9hS2es!TAvR;Wgj8cKGmlQ@f#0=myIZ#xunul64UhNg-yWfE`jo z9LAw3$L+(;DS_Y52jm}M9i<0?o@*dfEb`!WB=sCN5m8i7VU250wdi~NH1)N6jSw3n z!{hY3KyYF)W@7oplf8l~R*U~uuy6!v48_aA%O5Vv+z?3y215gKMZ#GhL${Ki-ez|d z=ql&nyKvUGZNGE+tM{8S?IveQ{r}}1!8|*uWxi1&k=?jzl}vu|e-6`MFTa@~2FnT< z+_lRynVk#lvL(12Cs2xf@!@)c(6-I|^hgrm!;cV%d4)~w2sScQgut);<4OHZkmsC< z*rIYD!(Vc*?)>nOQl+X9{P0hyxFK??Sr=apl!!72C;|mWB zA-?ImpQNN=c0BgQb`IvgWc7B-2%qE2W9=xSMwU*LWCz5)Awah9NLIuRtR*(o=K`CZ zIC*`prwRjwkhCk9aT-G+&S+~@?2(#COcF7%q#nStXlu_x9biB|7yJdP(di*o2(dNQ z_Rp=GmCNB`D9Si$jX1%7LG0Bf82!)It zlUuw))B7t>?~|I65?@*AfOLlyCOv>z7iMJw7nT%XFmi~7fsyuDE9S%&0FUB%#FuI zzak@$Xt4CqYKX&Ye;(2_3Wg?Nb{Len^s!9@e!n|nume0cbYLNAl_#u!#YR&|{N&Q^ zzU{-h?S4KzP8zIkFmS#Ni*}-?F1Q^p^`l)7(0WEbTOObo(YqwL_<|#$Vsa;;qRU`+ z*&cB6C1Px)B59>1e+=`LEmH`+{d=7`X}w*e#L(_q5~(^aX_reT^HY(3#6k2(?kOfV zol6NC`(R;(#OIu?(D@xN)*GQHTx3pI>z^qrk>~WLt!bawtkwxIh`^9`eiASzu^C{- zZunLcR5$cym_Pg!eNM(5b5Y57XwWLsotlCPMVO63bKcN9Hvl*--^U$Y4JI{|lm;vw zAZ$oF?16JJ4+ZMj0-T7H`Q(kB$hRi;D@plXOyHU4wl*bOg8#@hM@_I`q`=yD-_J2$Ow+TYgSN>?9iyIM4#pNj-wpg}3&;wbu~R zFQ|yd-shNqk=Lf5WVC|Fdr`+GtWP8DZY$GP;`CnK51T(GWlgg!Tfr^y9-Zx(ZzZ9SK4&C^y`&s zpxjhC0x$AR(FR*{67Jr15*kpfWHjkA+(`!TW0nnQ;u$8khYg9PWaU?7ifr|jEn`?| ztTx)5Dc7_VZd-p9KqYljqKdbUYZuK2No$T?th&8iwD#Q(2yAFHx#e8k!+NgMmJ0!y zGGb|4ln8E&HbO7boKFFyeY0uX2J26@lLfsSPM@O1*kE%<*!4NdGTL>a2llfK;3#TY zch8s9ogz$eTB`@k0=r8y{&dU6x~pCwqljV}h;(jF6HpzZlrLzXZd9QDXXmX(jPTSS z8T>h(ghO(*XLmF07P7U;&0%eXI%9+3e4ry9Z^a(k3vC96ANas0gIHxZ=Vo~@LAK`O__v5kOdw=)C?tp^_sV|!+@BaF%OG9(yvn5veT#I zBj*v+3|vH;3YamHUQR}#JyR_*jJ&6dxCDe&(s*pRqHF+qYPcZ)oUuvmN2G@i@@-2? z-MaVhB@Mhq5i=@3`;$X7BBguxFvY#B1SLCRU|K{UfIJdzt2aIO1FML!1&e8~`L@3k zJpztH6dO<=8Nw<3@!YbMn%XytoJy^%YK4yhyK7(^ZN0mKPe%MR%U{ZFcP_I_uYM$? z{u$fE9JuTRCWdD?H!oJurqt}v$V~)|)1%)<-L|*yO!2=(M<-qBKWz~_^1;vIs`zWB z`Byy>tA+%$w)9JDbcK%1$UokmO7;Ou<B)_?#rqNuBD(e3E` zfUV$^3Zy!uNmulZnf5bDkF&Y^t=*aK{g z!N>b^Z>8D)z(0IJQPXvWL;{}5@yIAp%lIy5+V{Lb&k8j5J18X)@~WH(n^r@RjR{dK z97Yyp{gpy`GmYjO~JDavN0?^-x`i>*_d{3y;NJ@qoFzaeBZAAWS`_Jl&@?c zEcQl1hd4hX$6v|tUuZkk9MAdyZ*zR0S@3&+K6lCg~ez9j73NAv41#1*TXAc(v>ntHuiW9zNm++_b z%_8>!BJY}m4;yD%4xnH|A;DuG7KXQ}+$xNf>OHRI%B@sK15+ZuBRny(29ftrkVzBp zyKFR8*Xx8;UESlmFV!+`oXH-4e=Y_8?E~oSzul(k4u;+;&t;k2#n$aOTFVb}AeSq8 zADr=4RR&GH8(s{HtN#}H5_VGfp!5zrPIo#iu&=SC@wF(!ucRLxq$DuPF`E8jphj4B zIX3IOy$pmsG~LE@#$RSX(Q3`~kiufjUt+SI$852`-1yWPT_mtqT|ugY&${Eg3iXqH zbdQ?fzRV|QbRp7tOnjWwlL3{uQxIEf&fSj)j9LT4#EgROE$^Re_Ax0t7?u+c!k#tJ z(|5amE@;11XiNOhew&ECM>-p*oc+hF<0BmSEvEC4Lk3qheAZ0_Xxm->YzX#r^-TnO zE*lEYaR?|#5N`!;WHiFoJFl@n`;oZTons983fGGt;i&i*)A=`m9a|*|@wti@!%8ShHQALtnZ9tVG0vf1S=d=n-!*76 zyk}sb6$s+PQ3|AF>iQt($V*->|CrQf_f})8nMS0IXj;-f`ZC`6xQyXgW@s0A1gk1^ z%;hL)_thT}cBF6`kODeC%;ML%Zr*YqDKNRethDcqH4 z7|b2v0v@O&kM010OipJDgS}Y83tHgUF@>hf+0w{u;jwYaOJ0fr0U9PtJyZ?`WQjt2 zG^`ps&`!Hb_;-85il#3Mjz+u)w;w;LT}PtDINI7Xx&`M)^*kTpf94?$g%?HI1=4=E zwLO4VE_Jzdry8OuttJ#`BkoikHamVX9r91?Vi2Wmsg>)Elq8}+CCvRG zv>#K5H)xHm`A6M{Svg*_IiGEoeI&dokP)q|i+$xC0L)~?DUuyrOJ&fri@*)|oLLVR z$h>o)vkrPWUwi3&@HWU``=G5(U9c4j{wUb^1hCc1jzNI6GXHoKiM_4B0-@;G(I=6g zB?XFcVHrovnvd#m1(S04A$wN-&x0D?tT8}B)r|V2=Uoyq(SSRN^vyk_T$$zc@|T5) zPsqa`a_tOqfV3mX!DYog@)Z%Mh9#9dE5j<*yEeVg!2_iExECIsOk z69nD3#la>MX7t09(tDdomR(Yk_}RC4TaE8^FmFxc#RW?Y47)Cx>q@p5FX~U_MAI(6Nh0TZ6Sn&13#92jP6}`uo=DD%ferUAHu*PRpC3Q}eew|n z=9R81hGyYNF-Q>w-bW>9wvp$4`!eOKVmcw9{QNj*%|9F;j4k?-!zKs4AL~xZF)_lY z{GWB@(!K2C^VTSKC4)k`zN>oL{XY4~sW5B>{lm9sf;7QA=5L30bu$&{(pP5@#w#?I zc!>SyzVm}r2&Q>ua4%mgNaYo7)!E!snz3#p??Gh~@O!5F1J z^$7B3XU14pokYrn%U{*Vzj(eLyzo&ks3KKP=FW%9hdAqS1rog$q^HV|%0q!-m?gvd z_)SIt=F{p7>aA3s1sFRHU z2J*I)FwWfvi_+Sb1j?v?F3jKoW34~I4+ezV z``-#aEk|@O>}uQ&4Gu;_WhJ69D=J%uBF;9y_T&e@=zV?9nx$#8?)_?#WoXpC?*+om zeItyTE20!H-wU4ox+O{&y6;ym$8O0Op}Y6Qd}=`Wq(J+1id4?Dy2Znu$ad>I=;Lsh zyJ-5%+%)G$XXAsG!w>84+95t=g%k!3^=S>4^mR=ZsM!tQ)e8zLM7g|DoYjkKKqUj-k6cG+&E_|XOvLX09%T3*!r_Nen-iTU|l?d$e;+y9;&9a+lzaG z+Q(9MgG>qdzZ2@lj)jt2AgA6<^PWbV5kQEI@x8Vm^vH%TSdxaWcNkmP-4W8 z73OuvY+OhSphJtGN*qsnJm#|!LuL{dWJ8rk)R)W}j&L{YCe+sVMq>V&R|(K7KR}OR zaxED9)9P(5x=*bwQ)_(Dr`&{y*gIjx3n!nH)35^%{7L#(7O3HibQX^@jB|O1U^MNO z5qi65FK`}DBe5n_IXPrMMFjFERccQX??1hP<|z3<^6JChMhK=OB!}(fyu!;bnW%Js zLE?8s)>uFAsB3w+^KxBE8+Z9M@s_`P&6Zunx$#siUV#oeLYpY3Ow)BeK4ZCh#{s8F zrr}YgF&guexw}b{Y*=xA6cb4mKT06Te~l(V5x-;~V@vnU&35CtXkGBYso=#H6xGmpF*SAjV!6yb3bp`sA;yDDV5m=UF}3hpZ(5GU-m06UblK4>1x+CyOQhWEJWKY!uV zwCY#E&mIf(63GfkEVDnj1VWwEzHKWKS2B;ZtvxBZHfb;=T<9s)rLh^O(!uaWOqLa$ z{{T=!-3Zoc^rD)prs#RGxVnNyW4?GCj|CC{)p6B6a(Q%0@io)H@*^8Gs(xzG4Y5LB zUntcolwZucm>foT^`M*cvHfE+4z_@;1MXrW#ozVA66Df7#qKsE{hObYFJCTlJohGlOQn;6&U22+5Kd&}gAIHN>8sR3Nubj}vV}>2~awFBfJMAK<;uRQr76m-4TsW?) zV#yFmVMSQfh~H@L1?|04!>J;M{_NWY|E^f_4k>{heP9L;i^VShg566?UUXS2OpUgw3nx-N8N^Lv5daWQ&Q2TGUXMH&wcp+0^VvYT zZ!^I(S7r{*0z(VxzS z{Q#lBu2RXQk96QRheI!fL(hnrx-YK{hDv)y0)YF>i}-`}_2}#&fj`^i9z(9db5WZD zjLTJ-W-AiPkGkIaQp)l~3{IrwmR5ixlGVh-1@0umre)q3zsk!Y?xd!r#VY+hJspGW z4R(+1^95zDka#ZX9K1r7l9&}0)K#4J5o*b7_q--byA{Egy1Ry_#=u1Qq~@0dzDG}! zdCrVNTmYJ*+j9gJ!ifxJ>Ps@y)5z3S7tv3x{WJnpGRUQ8`{wGFI@} z3V!>|z{V;n)!==)k-N*%Rqzj)kZbGJ0P~N(tcEvY{bl-co#e zQ!UBaEf1U5sx+%$=VvaJiE6O(Z^z`{k-+X^Mlx*38Mx0?X-d$syQ*m@HKnq6NHDz4 z5fhT=b~fM$?dqNY>N(6!v~}@&H;>g{k6tYGyUQOA&wW}~Hvy(J_y!BHF{0(YCRb_h z@8S=xHu_|^Q!+0%-}5nln64xKGJfNI=?uN8Mp2?xD~>kMr{Mu(_)FkJb@OW>(s-ts z;A!_vt12}Yz`LNmH%D=$m|o{)UN2L&hYTP5M!V|6{KeTj?y%W;vgIEZR9+wO_2?7C zA(a2k0(_qCvxUJ}E>8p|AJ}ztoE95xORKX7Q&T=gEpmgTk9a9wJLiZ$gl=VrxF;LJ z3@$#G<9xV(bp_{j^S^^sEmMoZ+MmDQ9?>`+|@b7vAk(3Df5$0i(VW?-`Wb_;Icec2ep(397RkBtH4u^6v zvXx0NAELNSk*CG%C3AvuQpj~?Sifmjj^$ufmHP>eU%KL@eb->$QoBr!B~9Kei|}eT z5Ak|ee?DQavKd6lMZ#!V8oSaz{!uP#7}#0mGqms`)bS%FWMt9H7_6xUo9eDQKt-7p9(;>cj z!3){?hB&|^*NKpF!5G6+$&_?yH74v{;)QSZ^`q4Z6gnB`z16YkZ}PBKN_&HbXOfjA zOVxt=j?^{d>BxYZX3e(A8o6VKpi+0)&c)PDBc@pCoqKt9Awcgj zQ8#Dy;|p{@B-R$&c?gqJRkeN7)b&rLKIjx}Iq<)%cCumOy1FF$$@eg>m)%0Hi|mFS zkR{~J3KoV4w);BB$&Do0WaVk1=OiO#NG6xRhc&al_1&$lp&b3Nf8m-Olz{3XMd}t zN;r?ZVhVGWdM)Ms@Cp^98N%?nx2626;(E`XYYFO(`Nnv_nECcQ)7b(gfGjwaZ;LJ? z2!t5#w^%hg9D3a}Fs0pDz9Qp2QQ{wB6=) zQF^FvsgoQCiP+>OO8*dHoBrX)iJ9HZHRMFt#7<6>jZx#Gb-L2oG`o2`8~~yO`kA{< zWwudBi>Ig7Qpm_-|7>Y#d1F8Dc~x$b3+9VZ(?iPQwhe<}XrpI@!LN^Nik^!whf)E| zA5ec*Z#g>I)fsqd2di&tt_U!X-;ar@f5w^P2Hff@gY$;;XDpcWiL$!mLZO@jf&wSN(HnP(bZiQjO3=Xbfl;B0gpNvz5wxC5dOf+#!L=gP4%<;s0B}_H`+WB} zSiXuEeXb+R2cXtKs+9Q3yr5-Z+l$ZseAX=asn5!%`*~jQ>wbxAQvoCWSkNp6xkqYU z+H8HzdesBG)g3_uD^Ff#b&IWP|5Xd63&Q$hd9!l(EB4HRMNNLTPNI~7&^4cz>I(M5 ziQ)~%JjX1BZKuH@A#fc@inET#=Yrd!daYhtAb?;SzMXF zw@}bGzU|n~)waQmZXD(+-8?1Iwi#WEJi%Ode3I7Ss$^lVeLI=HmtWKLqJbw`Ys$?o~$qf-77rlDOX z6O?#*Jj>ReR>Nkg`)_l87)`{+m#ULlP8{9BD!V`**9z0D8=~{XE!|zqEy{=!Gm-Y$ zwf-E>*l!yPoWsCXhsi3nk9(_xy$8fqvwtw0uG?vxcSmWa6@PsZU-h##{LLa3M|7_C z3Fa!^%I_(V2#3O?`o_Po(0>&4FO&uqG78s9qCrBh%GZ)Yq*U<(^nVdR-mAW4{Jmli z`?P9TRgEGmoIK^2qr8Y{@{iB@`&kC`oXnNtAP$ZXFu; zKI0uOz-jCHpi$^?jd#WJkC9JzXWz7|w#m18p1FZ%nGI@YWZ-jtCA0G|0`pJo7Jvzh z0Knib|Du*iba8jy+5B*nl&$ZoH`dFc{2 zScE*)lAvv6ly+&_PNTm-2hkb=d?@=kYTpv?8ir$gn!*>Y&s947%}k9PBD6=9bSGay z+g#L(U=I0>!>}+u&E%nG&|cvrz1giUH8Tb9EJMHM z1ZJs{l0q61_2H^Mp>))g8^8ml^QS)v#n#H^rvhLoa75kwwFqemvF%nroZ=~bI({@7kbRU{nxwC^)!DwM>}uk!IN&4g4Bk=>R)I$k`B z$A)S@nW#}`zBGF7-wW6cTo1HqexhROQ1uw71Cztxp66WizAv9WuGA{Wy#M&diZxX; z#H3fB5^E~FJW*xv?!hTBWTb>5Q;CK(I0vFTuLN?)tZXtYlh01M+8KPLr$_P78eq$} z8dl22Z&o1-udY173|;=nEJsS!q0Cj9kQrw?cUa-HMmhW^di|Lvoa&9MeAsOof`NGL zvb*zy0(HrU?=8r{sZ3;=++(3L3ZLxY21!Go#u*XrLUpKHLsjhw483;#3UWhG3g@a^ zhtO)Wjrs&5n-J>XH|;cI4V&o)@AcxAB+fHK1iZ6An)Cu>B<^S<2)MZ zisJlSMFRpf!8@349?hO7-&nHeOy`%+#Ydm~-R!UwPfFkLo1(L8uATn8p;N^xaMh%N6q24}rx2%OLU3sg}Tj>}G#W z=N>a2OT&Mcq|Wss<8zu3jF%0)FAqS6nU;6jYXbj6Z^~wdOzibDsIDbUiDWIz_`cJk z_FT+pgQ_a^+5ozlIMvkHPryR_j^y^DRgdUaliek!DaY+zF+_~}9eNjZN-N;aeF>A= zD&+Jtq@vpGcq&8))T2igt7lFi89agIi}rH`{6>#U?=UC2m_@8Jm+wfcoQzI|lEDx9 zKuEb?U;HzT&CWOFwE5uLZ{@QbKON3$B5rj5n8$>wzI3R?gQmC=b4HLi@u|?6BePSS z4+a`m)6&6YAynL34K@zGA>t!kQDQ;qZKO2Itjt6JW=UPXu#Z131s?o)e90Gg#|Y%5 zV>r@3S#Bd`ky!~%T=k_WW@RFjPq>)mr(*@Yd4s^lhMT?G@z!(619S@89h)Bop~Rtn zDr9)Ca7dcvwby4Z!WlE|;lj!L^x;53`L!Lx{+iCA4*VyCIs=^Ev(&9_VD7;;eJx{c zAR&h_3oEM66gJbtbIKNre`;+qSOH7i$#IQ7N5rMHFsyp^t!`b+X*)`pzWVp1Y~-`y zsGWuIAr5H?V1WbAK+KVzE!3)mIxPFZJ$oi1W#o6G=i&b3s2l=9zscL%5WahO{k8F6 z%BpM(=hdaT%)ztss zybKnm&H>X*9K=m23hNfsJ?Q|Lmz>^d!>H3Qlxa(WfQ;8cJeHVc-8YZlaNlPNmYh-G zfwVu6rTj?H{x2{HWrza+3<1D{V*3c zMe|tXK_D5PrtVp&84>S`o#wJU#;C(lka-}p;HH7a``+va5UtLJ3fOH9xAXsTxUT8L0Mx#4bFaNR zvYv8-Ht{wwA?wUha*2`v<>M-mou?SsS)%%uM5!W+ICE_|c>(&?W{G4Nngyp{{tKNk zN@(Hl+%`-G;^Xs#jHUK*^laT$e&xHDVCLmHLv`+F6MyofW`PIpP!~8^`{#C$GdF6L zJKA@7lF$_aj`B>0C65H%!AddBIW_f$9)-m!$ zUJJax*TTc>4b5kAIxPuzfu(^yeFmr`ip~PZ0gL3$mUG_m|Iq#lzt5M|`d@a9>`J9~v5(xSY`kzdf+D zw#oZ6EvmSJMt-DOhp!7KDO3F1MCDUeNs~70eUf2nEdFeEPPtf`4WyhxMjwVzkx-p-9s`3b3!Qd8nu_F8iEc}S=TS#-qpPh<$Vp{*OrBK70 z9F>mZZ%?CKJ(+wx_M+mu6|(6^7d1(clegP*b_}dUPPdK^$X08D<%JoBWn`Q>s?s)6 zNYh8T5sh{%7Dg%ZPAa@+!NEAB8+H5MYw0G2zhP>Z=wL4&Mhf)Z6g{bhxtj5h7`D4!;=f7&qyUy6YIQ6&$ z1IT5SA;27qe_0s6oql@B{_m>@@(CoEIrqST^?=wMGI)B zGTOWJNmx&eaBrV;Vy@9)Pf^Wdj}Jt!h$t<-iKFzqs0~mhwv$FFbkz!fKJdMB@y+|# zwzp%owb~$wPlU95uZbd@xO9gKib8#eEc{|Yw9IK*gJ#$Z-$O=A!|pB1x~iMXGETbi z9!Vfw;4BI$1uf?GC-Fz<=p3nCK^=$=Xm^q;+SF`pL~7GOZd^x6$^zU>?B zr&tMDOcE6qT%s0ODv)^Z5@;P>xh&k#I7!VB-o8^H&4ZhGg)^`;icRnQr{R=_WO)sO zn$K}}8%J*2ot_f!NQoF2tTY=w~$sory99Z4f9f;%y;BC7$Eepu*nK_UtF z53V0P3=ixPv5qLTYUv%lY^Y^IU)S$GG4FTLPGJvd;J^zFmC%4lc7-9}k-x>kD1E}f zxq4^1C*!b`9#)wn%uWG_ovcb2N`LwCmJ<-`R$ zz5V8wB#M6#w3HVW(e6IgW(6nHghBZ1xY_oxM3$$w*>Q=ZF}fGI3K4qzBwYD-;$tn! ztYnC+AKAxP#1~Mi6({a|^TellLHHJHKh`&#!lY-udA%dMqT7CyZ>HVY3*_-AvooVd z7f}nn!{OrYIVjF;ecb}`fu=26S8yF&)D#4B7Qa0Ohfy}GjZE)cYR=KL_+ks4ds5rCQ(U#Q|8;G~!jwI2JB=7Dx%eyJ zWn)2FJRW#+LAaf<>)Im%)!4i^lJ!Jves+?m!!Y+9O`JwOW2>C4W>3+r>WLT z6#+1;Opzr;q6rtS+%;K6ec7O*yUgIdCstA;MdTJ7N`uu)*qwK^l$nWgWu++Ezf=Z) zz2HYFDWK!U8@qh0x; zErW0j52I*DM#cRs%uAg0BTE*zrf59j!Iep2(eT!^QveoB@4D#NDLxkt?F?;Q&8kF) zSOtR!_|Vu|-bPw&)IH2=I6JFT@$9h(XoOb(Lr>bz6i|u=>2$z`E?@>|5^zrLsnZ#5 zH6^N)c#nCJe7i2j)`^aYOO-$x<|AcZtfMpV(4pM0F>4dk0nI?zGcfW(xwxt9ORkdN z7xO5Y34qbD|NSAr+-Uqw|AL+DU;&w(|2t2~{|hM$0g+=T{&DK{@NuK+i5O@XSO7zO zbOVF)OS+4{$ynnWiXK@7#H&v{#}N5nIDV-_ydG!K`1B|CC;EE_WX7};eYT=(fyXuR z6_uL+syGylO_S;;nE}3XzQGwcp>$pN$_YnaPooP;pRE@dUJxd*W#^+;&OV#nM;45U zr-y1apb6!jNPhcSZUM_7*ZD|oDsWg=gcK&?5U<;N;`~ZO%5MK^H7lnFB7JtofQd&z z-Gv4WxkY&QT~G0*2aPD4Y%6#zyFP7hOB`nQhDEWAI~e!e1Hw=XuI>PIaq z3$3@|Td!}Bv8A>d0#*Ed8io*B^+ufBDQ{3`@%JXWe2q*!xMQrjJJzxFZAT4U9DE*AGa#=%gB3qpRhEDJFHIj!Vn= zk%TPh1&wmgaW8hgDhugY*CAgX1_^8(@IRlMI`DZ;-bR*{;$RJG{+p@&e*t)*n%S_j zUUu_5zsmgoCw=`3lPp%CW~YYJ40^&xY-@@H-En1zzJ5Q_cijhMV6LUMDZ^GvFzWS= zr40Ka{za_($AEP3*|4nCOcC|h94u;EhVPjlF`+a@>QGaqz!?m2Ko>H%@QUCdap1?M z5)*_$5v!~lVul>Pa}{a^0{#;>W-RVA=bDrhCRPFGc}{(ORuTEdZyDO`d)*c_4UX^~ z^4qg7oz))5!-%8D;t>qyxl@Y>MS^aYmyl>3czY4>lN_a~}4o&`pupkTJz z7adqD2VXT&cT%FpyFvBS`w}URat51C_>p_ER;Z*)dsCZ)GuqMl3_l{O@tNkWimFpc zR9+CW`|g;;!&SNo43r`eZ6$LQl-Ro$h968KQe`N_(%i>$g9owbI9@_%dYjf zH6s|N%~jwlmf@t+yCg`a%Q3ZDp;BzRm}`Haj+PhD9Y=}*0q(`qSolL;5F1J1w@VP! zuLpb&Tsy$N6gGU*ZEAcIVIXhpTCaMMo3MF@rS7(==F5&fshl)N`!YBoMw;Qjf-Z!e zONn71D5A+ZU~WA9Rmp^c|13K@boUu{`B^VzremKy$A3^yTZl(`1?n?jvT{;|h;F8C~AG8=*zIcEU>OinV6lxG)uv z`UR^KApGAffbHsVx_(l@irn8svmE~M8^@4BpP3U3|CQOb-p3o2j8zbvQbR^M>e;+? zNXs>iL4h8X>y}!{B~Tb`pbAm&g%k~oI6dyRAF(Htf<63yuyvMEQMOUPpBaYkuAxDY zRzQXxKpJI`?h-*zx_dynQ9;Q;kZz<~q$Q=hySq8#d)8U&d7k%t;ltc~U=91euD$pF zxA&hRpvIjiu6|-XDeESxgrRPEPo*1zxtrsvYmcvGkDA+RwcBVfxv_PvHSP2<{? z%*?=d6thMIlzlE#KRrTH)ecz30(}W)h(&RbFu%*DPbOIcl>z3u=K=tkoQ7NU6j0fqkAw#;D!fuOm(x?Qora zq4nlYqK=)}STytf-5~7PZf2YU?PzDasrDkhq1=3mgtg?$ql@5to)@7f9eTOy;)vla z1&f{W;>#Jic1PEiH~e%Dc=q?+{6fhQwJhcuS_Th&Dd(SDo6}D6M!}IKG~tdqLNb*B zP5eOzm>jQ$_qwm`4Bqrnkz29PB}c8Tyr&uKlwjj2G)wq}&H!pNgMFMozkdpq%oTr~ z_GW3!%xeK-^-cTDG7Yi=2?|XMQzEO{^e7s`$eDPdU9?tJGk{j{%9Qu6J4Yz(mZ;1e z@*8=xJ8QnH3%0+4GM)h74MA8QD2?^-nJOm7{HcQGCD; zM?d0`Jg&78p1m3ouaMW*j^O?l32Q~tuFuSLq^-?b&2KXR z>b>tXnrdQ7j`bUwH>Mp~un|~H;1^hJXxRJv`bDt#L}o30Yez0J4!FK`CK(UUOGkIY zeFAAT>|bHTQ{GK0^}MU8_C5|) zOEf90m1@p}zjpSDhdN<6$V+R-dGm({<5D6(a$AulmsUGnS`n4Y6VJPXGb!KB$o4@?eY!h-XSWpNgVqs1RCqw1#&(zM%QVoyV}gryxlZoh@h zKhbYBY*Xf#b(j2RGv3krMB>+&cKV_bpXb3`7Q|68KBQOpQ^*i{l>p zEuFw7riG->`&Ht05-Gi=ydjRgC5pDZn!_}^)2}@?Rv(w~vCA6T^$@Zq#0Na7Hz}Z-w8l*sp6!sbL4VfLgyrmZ0I$Hbyo#qJnTo1;ElK9L%gA{0TA*Y_ZktXzg zGPTID1zwKc5oLBnF7DpH7)ohL6sbx-y1oWK>)DbQZP;i}>z6{l8g#TC&L*ZP*1hQ5 zlM^uek1w}>DF~zb&ihPo;myBYGXBRf&z~bMJ)MA$kM9<>t&lO4P^^$6hgUd-W%6|D zb3h04oCzNic=ln8fUvdAe!9Iu^97>`mZqX?_(ayeK+7w5bXPaI#h{G+Z6H&imsPX) zrCL31)0*vmphY(f8EsA-`+$>b!sYA8mZC^VF5+glkHLI{L&g{vACD(4+67lJmGNT~ zGYUWrXK3vZYVFh2Ny=&6kL#J!=VwL0Iy=rg`w9|->$zoba-*-iBlPPuVVg8R$-V`d4VmM@G*}0I*%Q< zwVeFVu>Y=_qM~0&gMnWF=bsoB{{G8Sa>iD-tr0fC_tRvuUNylv^8#WD98N?O`AoL* z1D#)Fmyex})+aE8aO5-=prc!2Ks#5AaM~z{vC@S|v-;STjG?t)H+J2=hNHd~1D4!^ z=c&~er1Fgl`?SghWRGS0mZeN?IR5^7rcTo!V>AFNtrugHJ<)TQQaoTO0IBOD=pR`)Y%S*}1k^!~AUzH4Qvu&e45k4fa6_@vxHRT-9D(}51puog$52T3nT+W`8@8g_hlrM5XN|o# zEisy`9tyk-ht=?^@6IvTE2n|u0o5|TWwPevJ?aXus73u?UdT+Y1px@$YR5q24)0k9 z(Gd=Uy7M(Fs5S3OE%#5S=d4?M@Hr;J8&)qx!HahLp9zX77C?javKc1Wv!Ym^>{!a8 zCDmXd^&7_&6qlFgo0}%Uh=|`hpn`dwA?zA>wy`@Hy0O?j+Xly1=n=~>q~zb!Qe?yx znb`EU-)~F!yz|6+>KSWJ_VRku0-A$A#;*23=GuoPm0k-=?b=n-k+1QB4 z6p&M>n2DD^_D!YLKodCcvKaoVicke7KPEU{CCS4v^7KFQaH03Q%MGZc)NIPEAjE#< zNdZb{msT(0tQ)hY%ilS;SFJS{hA5GH z#HVZh@aRqu+qUB%WBWc*8NTQ5*R;Sv@G(FrO(4*Bv}SW3x5+j&wcO4hK<~FqVKhbC zJ@CQHgpoCtw&At0-xBEBT54Vd&)1qlp|ODCPk^}J(C`W30~I4w+0@WYRMW#YVBoNV z9<(mTXTFi!TuoPIiRSilHpxR`4nNx47vCh8`?`$1g~^pQrEh))-I<+gdiIcxBY_Chy5%pLvFkgVS}4c8n+H^a}eh^67osae^MJ8ht4z>e52GmRo%{r^AtY|-Ty$X1{J{BFIF2l+$-=(Pu9yrS2<_)KD^#BvTfod)j8YrTI1N=MJMKfrv+ov^7KxF;P? z^7(r42jKLl5c1=`0|UL-THem^7;ZrqO}ORvyy;y$w;$@ixo~tk5W1LnyHmvUiQEu9 z@%^gGk1)CplZ2Zp7h2%tsM>}ephlvQnVA`c+A{hpUBdhKxbQ!XI#K4)DSwR%7ktpZ z%>3sG%I`Lptk(DDnba5hn=z|Ps5trIW6)!x^a+e#Z;!ZEJ^@6khVibukpEdT)Md2e`f@&da)# zcWMzAXDRvGC=B&rGubG?!>sQ;+fA+7?*RRP!{T#M~1uD4)KO06v7QD=&0UV$pGDLkngK3{<`GRpk z1+FbPkhrnh-OQmY=9y>`nS>I7s;I$NrJ;y)Q>=SDBn7)VJ_ag0g!53;WfAlKd-yr=@%s^sfwBx0b3x2`_ z)xvw+$yYT0v|GM({~MSs2I?oi|FUKKAA`ZxheA*ipNV|2eieWAf!Jk*I+NStq3qqa zs(b1m#Jb-k%R{XWxod&9ECv?8!%F$HT}WW@F+!3+j*&gRqJZjMmo47H({#X(F3QY@ zTdV>_eCvpAmPPp(D2S5GLID|*Dlkf2h4{vvwtB+0Eo=2CJZo=Od5bqou{S9o5W;JW zfU7fL!H|xdm~Pj37%Tv%5hB65auu&Kj`NZ=P0wyMbwBqQkescI5It80+XeODW6XjH zukiNlFtKe7mJR&2_Ppk0jz|&K1+#u~VdA>en1Qus-d51qIi@53uta>rn0aP zw)qfa0f4A5ZsPcYO6vNR5I|%NG6z2_ltnE8l9oRv1-@Tp^m37grMUASp62{|Ct4>> zUJ%G$(m_N+6R~{nNhFc^bymt~A?o23V(U+?#X(Fz)?rx}U!nCL*7oe0FH*S%q+H{D z=7}MJ%Hxt>{H)DnkWDi{#0yZ!m@ELZJ5_$<+Zu0i&I*%X_qgg}F^knYB=$89zQlyUte|wC8ts5 zv^M+`n8|g}rvtFVoX2X+E*uf1!Z3Q;Rg5IpfU`9`s{cZwBAEo+t zJRcojfVek!%u{mFKpDo+AvHhEcT~J4s>IEehE{V6C!RwdK!UzusiM+4a^GL?uLs0q zR#zvde>ITwZ4(1LE@k3(&Q4!(r`+-6>_!K5R03Y6iD15F?Y+eg{AlqJVz+vo#anZn zHl^pW(uqG>WQ^aRCByH(&T73Kku@(S@LyNU#J_%*HHL1^fBqYrh=)edsMc7Ko&etF zFp!1jO2F_{$97{(wf7S?-?wz_oDmZ$E?gmj=(y%DOCvUo$HP}3b;1$LOB@j~NX>hA z-mCsK?}`yr>1=jE>0Tet6(u_S%!gq0b)(ySqfa!%5ln#EVmF}t5!%^D`+Xr_q;8Rs z*psYh(k zt{(QN25vX%22jY&zNf`reWRSs+HoGA6}7mOZja8H@TYhhW*vz5`)eUK*U_gc#I=*q zFKG&pc%_%jG^-H(JEksIeXO{+$T+VSH(wTHX$RS5VsYf?sHV8S1YUYwI3ACdB3i4k zY5-0L@Rhikt}_NFH8u6&(YZ_qv!~fEVPSuC zxubltlgneg)O#bZJL@KHw@YUi_TMu;pp;x+@UL`4fq?F!?kwA8kQD!_Wp0w1`V?)< z{;WlqIQn??DwIp2pG@7Bh)f2NiJ>Wh1tD38DzXQO_j|CkR`q2D{< zKqS1#(!{D2V(ZzSl(>^3L{@#Sx*XqQHaKlAiQs6-J2^eA*TMZ6Ac{Vm?a7t*Xk~5b zqFNoc%>q;g&vc8(`a!(9A5I^+KhH0t*qt4=l_U#=Cw<2oug#5Gocc&h(m^$hCNd}x z=V3;8{3tCbt58fcg`q?J{4_-Ta!@DL$Qq|8jHyQUdaciF{VqOA~nOSfrBCANRQ_a6vWK{L^oKEE4zj*@7EumrC{wWG#4>D?S zYa19K;7a8EF#qJ`b+_?3%6e4T%~?3ZMmz?Ufq%=`cnHZ5hO}R8(Sc3XO&It3eKjVV zI_>(*t7^q!FRZ3X!xdRM6jr8KHWwOt=6BCSq01x7nG>fQQ046xETNCjG9N!YT{d@d z3$;0pPNf9d$95lmW{rW0DTM1bHUc>-_+T5KG4^<-X9m(qT6BvjG8}!E)D|ZV>n6^n zfW7p|_o_r(Mbw(@l(4uGxsAnh>AZT`sjW^rSNGW7s$Tb{yME^dNgPxmR4`3=-KNB` zt(6@k4H}=Qk=442V@iMIalZ+QJ5yaY&o7Nf&dz0lQYCJW+Fea{#oTG%#@d^8O`K6n z9(yt+F?KkvOJah0d!-0Go|L@5#};bvkLJbaG+z%~Bk6jJ;za%#kgW^aK(v|rY889C z*?)fiwEP&yP}zM_ zn=>0|@b!%YmpZjN1NSGo^44se2X}wy2SSZ5jAWS9Hg?Z?FwczwUu@72Y zS3h-zmTP+hoqnc=wbFRZO#{a=-+EkYOA>Nv+HJF4YB`PCJ3dYdr(uI232UH|%h)~c zPPjU;*%xLG>T5xR-3dz`t+!a-ULJ+T#Dt{Dna#0lAyPNsFFKo=%h`pI?lFi6FB6Cn z*oH3ks3(?o(dRAsBmXHKQ2_S-s|izcJb10BX7M1{`1>zV$^0@6o2~-JZ-K6*dS4Dh zt!J}dyum5<{M;hcpqxL;=&B!YB2!K6B!|exF2&Ic&M9}zU+Yk|m0a4w-@kE!ydD@v z?K5hos*~Gp)xS{@(Rmx*J@*tep31Fz*Y{H!$^6c5sccSyVCP7VOcmS6o#RIXgL`f{ zt{}?D)Fz@H<1HkEuy4;7ZaLjrB14%sDNr^m0h*b!o+K|tYuLV9Go=5D-tf!7A#W7l zt=!6tVV;a2nAz0b`q8fayA|JJ8$t?&*`}4og~#LMF}%B9)~qM+Ga{USjCfzV^(&>$ zl|u$eGR*rn$Kk6_@gm5E5>9bT`JTvv)1u^2GhgXouMCk8mcLkml;y1A$;j`%`h3S3z!&y1kXRA32PzHD(Cu+} zDZ6HH=vq_Qa&UDG?mO|QeJvSS*c4)bfY+iYh1&MMsD0hp){z*)etv(*qd}`jLHRmO zH)J=;s{0&ErrI6*wU`H^m;l{HOtFG-!v^%>H(f{Hb7rw60;gA`aUVY;3~NB$j=zIu zM{_+K9_Wwj_2X6+5!`E_!m)_dD6-YpIV?INj7jkHv548!ceNsVajEHOksZg|Qs0Z= z)1AB=T-*R4wk<)sG>B3=<<_Qhii(1y^|AgBEc#!P>w}Y9kjWj$eN$5q>8_E4Sw_kL zk(!8TyO5eO)MGWs_a&#@d@Y88w-|W!Lb^FXrtIR>;)_|*5%}B~Smu5f;@kYQtWOy+ z*Thoh-XAp9wck42j!9<|g2wBKtPnunwwo01s04g_#el1sbq}q_G$4rCpNb%*w9iC2 zQGL1Gd%fD9a?#QdJ->P3dokx;>$bcUdkK%nIb?U&PLa zexETN=<0Xe{YA|^?E`i=`ZQlro=_YEV}X zb+`6_;tLgvGRtxfC9bH)r4NQ~g+bEzKo12{EMpFjFC5R=M7qvwL+r`Hn{W1!NuvAHRX}q=KEJL~wUg zDl)Js2yHiQ)@;L+@Mc44vBljj$d_K~my}mPYLupKCcNaG9<S=^)pD9kuR9~sN(SEIKaGzH}4as05J#~a`+ad2`9tgJkdx?}h1tGoBVm+bBX zA|&@o>i*XyL0cSvcyE7t{I!SnpJm@)^W%94`gpNHC`yvda@(6mncOfKtEC!>jv8oX zckV#ziv;&&D@AWwt9tmstFG-;45IB%&Lp{s=Q5q2X7;**Q^hCoLk0Y?SUL0~mp;r( zzA|TKsh8E-?jdZ;$k4}tTq4W}eIlf*u7Js2gB)H@_DL2KbRgWtR*E%RKi}_{f8*s$ zrv)M`B__9mN8V>9`*RH>d9y>!)y6KVl2cvf3V@X3N)HC*q9kh@i43)MnANTsuwnOC zWQ-#A9B&ek%5I|X*tS{Pv#_3~voQ&*lRf3NlMC@?dJTTgB!DCtA^a`Z`NzAvOq8)* z49vdF!?BbeyzzuR?ro)Pj=f&?PqaF1uJkTrY3-kAul0GlE1)bh!?b$&*dFPLQSS<6 zloLbJf=HbzD&sz;MH{BP30Q{lN+@7X66+7w;i+;{hymHAF+P9pZB-SiW7^LLMEcBo z0&IEn+jrVTiHWmKunFE+$r$Vc#Mlu*dRrdpX+gi=aiI-)PYkp*z=nmbSRQZ%*d{5Q zohVCYqubM~B1Mu9*M`Kp1fbey%?V-cA7^<|as#%Iwl-A-J^eJ+WvC;3{hv;x$y z^Aa_XgQaVJ&nyq{?pEueYsM~{TWd`Y*RKVy-~F*3$`*Np@_aJV2^@yr(tiX(->+!Z?-pki(D&6I8Uvsg42nlXa zPRFv<#wdCgZHP&DRf*xWXK4CDO~%gdS-Qbq+^6I={+Y2J#DdBJ6M%y#&S6c3<89Z$ z4f_`Yq`U{wvl62t(eyKuNrUyQ|J0Tb?&SPsdk-b|>)Zc%B9CZ^2BCHtx1+3x;Q6oE zZcqQHIW6erlY}bxDysXM1cf$zygGPS70Q?qD=Zn~)NWPGvZ&=xqLa>Q)5x@?V)sFIYEkpbP_-NBoz>38p-cn&?7V=fRu*9M7Zu;DzO2L%zo z_L;21>GH>&fKWboCRf+v5tKfH$y@yK!u1QT@7t#5qv|G_uNgLTg)b3Ob%Svpv>dwF z1+Ph4Spe6)j{>|3u>voX5}{&)jGK|R9WOrdfw61}vZc^RRuDmzRNa}cN%L&(masN|HNcsoJtVKY)n*5SZ`m?w@IoKKY7fL(hw-uA0>(^P zFLdKS_P(k7dGTj2k6fK*PSpK5PoLiEBZ=pB;hkloYY3qqkBh$Mm)X&w8`u%NTN48Y z7a(7pbRPDtL`~h%!ZdO}2Ngm|=T0h(Iaz?vIxLD(m@Y@>NE`L`vi_t*Q_LRqcYeod zRn=rc>?Y27X?1aqv+2(^SK1k&`9a>y&z|73+x=QX`L2~--1Ky`-hG7kvp-oKbI;1; zbV*#0f(Gwv-2b}YtgNkZrgtzEy$Djz&dTxVYUrvZB_bS@M)i&E0)LoH`2L==8kWOc z{$uaoff@^1%owDNeI(yY6OR@n+Mlow80TL6MIsWKM){Ou99+g{7nj%NR++g{3L0)v zK4q1Dzm0o}_vv!HFa>bHOtPfE+2Sm<55vis{Mb6;_$iYh8()b;K0&M#6>iWIktF z3?A>SQmhg*pqkY-c+2;4X)crnFg-VeS77$GlUTp`m8ZEO$eVSO(Km2aD;H^4=_J3C!i_u!BbWGT7>-Vo{(0|?- z`$P~f1$XPuU|eDfiVzdOD>-K3p8l29&ZZZ9p*fh>q-X(SHZ zcCK%hMQR^;&64N3N1>S$lvQGsNTTGcWvFD6dqe=n0z8jb1!=##B(iKQs9EQZbRL0BvaAC-<4%`AmcAzk=Bx#cRnz2ud%RxsL%~E@T zB-QNqbi=~e8sll0u7P{$taD~UQs}9(h!b06>AMd$$M|maEx(BqMy1c~vibZ!oH1iA z8i+UF*l{i|FFP)_0{87!95?t6h$Yc}Oq9F}!LjW%{hXRqAR{bjJ%hhL;C5LVOg=q~ zP7!FLbP>RnVo4|w8C-CM4f75dx4q$5yw%5W!lV~5UG0vaZ1G^m(Xk2plCeO}Zn$PV z1Wv^YA=dpvQ?Ovict(6WEHdjZUmjP`;qKFUT7CtaMMqlG#T055=SVZ$rUO46<6Gv* z!D9@QeM%V8l3a5o;051Z+K6X!Vk83xT$(p=yxD>IMw%8LW)jKUvG-UmEtQa3q`YUVm3CdSfa#E>Akd0-)42h8Z-I zK;9}HRaUCK`6+47A2UDL`R$o!fH*pm^URb#bVn?+fH2U{2MDb+G|jP+7C$VG9w8p% zuQxI9k@N%#w!Pa9;d&(-z;s53p+&)Q72BSO+w>M|oxw9|Z(_t@$wjb)e)`mAZ+>~t z-md?j??J1<|0G&x$H^%I|CWAG0*@=%7a>~;4&|p%DAmcvAZ6Gm8g@HKgnw4K74zvz zADrir%I;!^)X_>UpCgkP5en;qom?pJw=!>hm3*eN2=17ZYsB@nWr%aYJ;ZrK)S9NH zB&T&1C)A{MK2C@++JZtxwH{2e#s5OS$*G$O4RZRAgTO={m;R5|7Z^*m| zLH7#b*!!>0<&uNflc7V80)j~6uEDkbc-_jFc{RUF$u!q4kZL^S3bZ2QI67#lCou9- zOy5&0&T`Pm3X=2Ujn$Xq#~3ryKsZ^-#fYW514i6e&@U6>3H9gh82VmQ4YNjFaRWRI zWI(`n$&BE0ui?~V6cruA6f!UTdF6tNtU7~*KV)7Z=8?Bm_LYvzA0X+GXA{gr>~+@5 zn+wZya0_JaSLSn5l#@yxCj)>we26{CbZGMHp<}t}u1?p9>oKpmIC`hxZm@dwY%OlNH1#1iP=8rOo%SqIa^LQ&{Z1qLLt)|;7V zo`kcOrlCjE+9D)|jG@PCozM(ZD|~hbb(nbOJ(D`75%}eLvC<^%fCK$lYRO}8}dnWVWz2~)P0vuZjNh}LVnxTJ^lpo=NCEZf+;R(l9rTpJ%Z zXnr$PyY0xhKA0M&5WWSiF)e#OM!|8Bm;rG1{@1ZTZ;X!j&f9xpeD>?VJ65RbobNjT zQG%MOVDZ1d)rIEqI&5HRW<6PcmF4Iyb-I+19m!LKep*KkSs1suB++dQ8oHmG-#l8f z0&#FWil>Puv6Ch*#Z9}`rJv@4Ri*M0e|&p1WQ)1HTBl-t-S-}EXNoE#_KM&d-1b2t zK#e#iNW-lEQd(R#Q2ZVuk(;cBaU2TLsrgy$$?*xrqD-;w$BwLqWMN+TncX6dzq7*@ zM3|4fR!KPL2nx(~fSBw_SfiZU3aO~wZb(96?sCB6^1rIwA9qYSnbIxxi|0?`$*hzn zn7ZX+o`JMiIdR`&HC4A5N;OuAr?OAV0~}xY!)Bn830@mmw@-$m8oJ|PRhibkKS9&W zC5PVFHaWW)K|NA@a@hWaWVn2>$yM7prVbau^Cvu|xDlhviZ7Efw-heKZAJv4-m4uG z8h#;8188*K^btt*I{dARRqkw84<7_wP{A|+>0xBNO{>F~rTp3w&yFG7Ok<-dLh@GS z{p16~BiXV&)D;Z%zZV~XNZ~bERRr|wXUJsSqxB}8Yl2U|E@WCiG&vMh6;lNmo9S5# zxd|Pp7*p!lPdPJ1XZHU1Je6*Y(PaM=p3jbIQC)uT=b~K74L7k+8iWg`I^yJN50Vkd z^G5)LM25Db12`w>uV*-R^>G9$&PHWG{x#$9O;c)&YXGO!brS~1>^27z0Etq-!{q2MW9h#ek1_3u+SCGoW z-^na#uAy)G=y?s%uBV^fUm-Uy8ah#}2P*ApgVptH-TqIyKxgpWW0|uNi~w^(G_I!{ zIkign_n!~?nz@WA8SZPz&yHui&e*?F4c7$uqz_mR(=@a2gSj^HH4AV6v(~^1*0^Tt5`j<`!9^*QAtK6@CGjkV`BC-w`VDtdwh9i6o#D4J}cv&4DU~; zS7U&VRdj{6Qhlo*Hm?Ym$iE%xEtG(qv7WwuBW6PGiu0B%<50-pN%Q$p&ilLdM4s#IN|G9i$U1SAX@QLVY zMClIYo~uJW9-NCYerI?jNe~eO`gDweA`c+-H^6}@2D7M_r0GFxc^ry15bPKu=Faxx z0RiP~aj_V?<&D(Ih`)U0Kt@Ak#5zHvR^*209O{G)E>@wj#$t-1r5reAS2ZL8o`;wS z%9r$To}4l+#RCZLtQhv|Txr)^^6L@NY~%SLKdA%R3k=Knt}X_l>laI5626<5j>KzR zOawv^zZx6WnXkogBs{h(RGMxSfXtWGKC4zekMx`-N_sGsdi4Q)0Mm2 zD5-8yg&6xkT5hfQO5%RK2zI!^g6Hrws0h^wsqvww&#pGhpn!@^Oe;V#q4Zo7D+j3Y zehF2#p&6cl=B1qyBgU6cLM>W)GX&CQ;P-tA;cx|C5vsV6JK?Xbyt5|jcM&4C z=WQKaQZcE}6TV%}6LnPY^&fp&L`&db5Q|*9?e~9FG?X|Y4F7B2{tM+io%w3tVyj4r zBt?E@%3`zFME0`{@RLo41L9^7q%8aST?lF--eqSTpZ>g#YP^;)F1^*r0z#@}%5zWS z^7Dmm-byi?9ESP_;je|>qRQ2GbsJIp2hEXMSq1R{EE<++C5-_FYo4ADLn6bOg@EWisN9u0h^~3t2TTLlME?Aq@GIRrS!yoIGJv`!k z{(KpsqI60x8DxPa&70d8oN433&D$)dfW%IPA80#}<7G4hZs;2p52NNmS6DKs`k*m~ zY?$Wu`>gxAfTxg#3!7gMna*;y!~Jpq8iwx&YJoSm_--w;%x%vi;>2s+*-P#x(Id{q z5NT1*fnwm)k!9+lW2DfvGf}g#58#AXWO3{I3GBBtP>w8KF@~3B^hUn5+LSGiz(QnXGu&OyNojP^5AzaKziC!3*7|!dNd7(z0WYCM2*DX0TdZb|b#niqENFqQ3mp`Xy zdu>|{lN2R%(SEg$23!Lk$iSC)zI*QtaWitroI}z|MNJeL=-gQ(Q7{Dwk-KH&^WqUk zxPmwTenhHXhB*ngQvEBMa7z^B7kTZi=IAs=~izR zWBYGR00_fP{0tQaYS+|w=P3G-4IYq|{$8xBmAlQ^qH$augM`|*sIxpi zcr}a#Apa6Mw*cBml+|JYKR@dz&?+3N(NTj?PvCvuJ&;vbHCdv8dj@6J#G>CSgTkLc z7P`t4$)xHW%bt&ra}bW*TWwmL0GtZd2lab8Vf)UD=S++&(xNk)HyoTJQ>53c<_J7V zxeiC!JGE22?T;)Mgc6N>Wa#80ipa^d_7OR)Oo311-+*T$@o0lBf(iqUs%&R4&r2Zj z8~`6kLbj&xBw%Xmo3u4cczM>KjI`X@lA?6XfOri?x6+1(^j-qyqyTz%&-03Y@|JgE zxCh7m#0Q#K?FM9+FYh{u5u633(97jg99XtD7Dv~P#=%FuZbgp(i%JZDh~DlujkA#6 zrXkUx`gQEM$+jexEdmbZSaZ(3p9T(6<9wCP8^#qa?VDqTl=KU+ZD%SQ4$gZu@L=Yt zj_!b@*)$XqrM2JF8bcjqpE+?(E&HE(YMG`Aaq!u>D zK#<+hBf?0fIOozsh@^>4oaiprsjsTVvfo8dFduSRex`}<1bbbEnJj1vB%d-cx5V}l ztB*h{(NuABU{(W$AnS4;S}AeQ+w*N0RPELYXJgkdCz7N>jd{VgH?AXiv)cvkzFlLPw(jl5OoUw3U2$Xze!GXq0N5JKkMQDQK2a{kthks!P!|B zB@QI=nB0T9@h}XVE`cJlIUi=0bG{AmZ}XwSOI#lR_G+PUV-SrQJ{Sij9E9UWh5bp( zLJ5!pdEeRyt7H4oIbA%_Jt@@gaPAaT+tZZB;udNEM26&?WsuYP$0Hw~->I(FzvUy` zI7=+eCXHsM_bv*MY>s1L6M2~++VhG&0yoCco?=_K5F4Brbj*k5yGyx<(*dXlF`d;p zt4Kr!HGAvshf30FDy2G|F5ZuLF`9(^o)E)I7M*VlM4+Y$uJPp$(fs$LnAvINRHuuI zVo^11+8a(`gH*V>FKWBpu=i+}RxPKy*{~3_nolf9I_CT_L0m)FtTaK1;$eQ6S}w2j z@U?`g{{x4-{5rrh66Y58EvtImO*1Jbg9vs{yiPMI%|wih%&kT0my_~eOxg@sWToso z4vXFx^bjk>IUPZin6=pLt92z&TMx}Ln!~z1UlA@%HS%wJe`B9Eg4$4X#PN&*MY41> zSiDDxk%sO85IMZ{v<-+?Jm?%GrtH>9pJS>&K3h&x_H(GHLEhzc4Al_! z8e%h26#X7bo^$?@RQV@>G}08;mLQY{t=zM-%X*muAb)G>@1wn7nhEun$(1MX&r{E^S`(Z-J|!Gw^%ne}#O=Z!%uPFIlW8 zbaJ39j7jI?5ec2rQ5*%XLPNRt28!e1B#X9@H`~Z0rJ8sK=9qga`q^&JYEz!@^sV)F z%D422xR6q@Li53;Tx^rYmAZ!b21{trQ}Q!}VTz%^l?ot9Sl0D6m|=0Jm<8Z9WY-_K zU84eEGzvAp?{)j|GBzpnbk(D$KOyvd^(U?^hOhY(hqr04{8z=a>HJk(GHTsz`c;)@--Obi%CS$z_W-a zf=fFauW>$Qd*{@x;fHWu5TL~*($)W@9xQi)yXHmAK?@QlmPTLg7ihd-S8iO7Y&-IQ z=?TS&Yv!|G8p$)@0Tr^J#$UbwJ+i-|60#wCDj-nqGF%#V=Un76rRR(+Js>iAD-{=F zbq}!HU-9)>dnfozbvwu)R^y4gs=uQes){%;n8yuk0FPOgyAX`j`h&bqCIWOAOP6*n zQyi`qAAKTt9Kpp-VESeT1Ur*P;o=iau$@KjeI81CYbX6jQe=ngsR|Gy%5lJiJ5bOh zGWY_$Uxy)`2Og7}j?RVi?z^#!SM>SLiDuqUt?%AE`%fFXu@;{QKC)svqqO0P|FVjA z#*NkV9vpYhoctN$GltnMnR8#2E#~x0|3%UP>!mz^TZkaPu!SoRLD}?nM3Jci&>%>P@xvjLhu$9heb(9D;Tkne7Csjyz zx5=!zL%oc_G9*Kge4O?`J!EJY%haxrsZz7A+i<85m2u93CTcj(+@y>j5=k)O4zs7j zCgb2r(>eu6qB!<@>M#8MXr1rFrO$JSNw6v1!*7Z-!$XF;)|J`yf(M4Yo6++GxUM8M zQ>cf?qCFfXy7XVQKty^k&HUd~Kv6420mi|<_T&LOKQoRGGw5qEbr8pRWD{^i1>VbU z@*L%wKtCrm#%{9>z=xU;PB_8!F3$e1pJ^7bP7P!)CLEm;W>1UGIt%`3p!CV)YG8=W zUTME6RduV}M%X}7jo5eeu5&m^3W)t=M9j(&aN7sS#1AmQL`({hpi?Oy7L$~0GH*`S zi_{0By@=3-XBr#iB`%Z~&FabB>92n4CFOwgupsC@p!rXSSEG{-I}EuX4Hfi4ZD+p! ziGX;nI(R)ft&R^6qAa*#HJ0+-76$*I4)0|`Ox8L-NEKhv)|5YBlG2O1aHOU|j{d|h ztn>um1w>Mwc2t3;8*!S(4_D1m1Ck=3qxIDEOTRy!7zzS93;}emUxuf5sW4I9VD9%; z*r(s6;}Z(6u`lzbqS^+dn3^1Z>_RG zRht6jY!&@!&%P7S@)%P(^q`D)WVL)Klg*j28UIY35hTP2I3<#3veZ0AIExB{OEc!< zn%jLkkOc$*lGl|!G3J~n6u^iH1P%x8_Om}V=d$XCoO{9%qpU-22ULgLF>O>q&#p_n zhmSp->b1AA$#tb+v}s30cwgBO-h+V6Dh6OMGveo1N3a58%m7VVo4B8DoedAifb8yeF%Y35!?_fu?eEXkDR;T0l|4wiy zIN`mxE->MX)Bd92kq`5V(Qqf#y0TCxr#@}+H`aX22u6snzplW3AqogpE_zd*4B^RF zu30bPz%Zp&8c7swFSR*2?7K z-8T-WZ@Mp=<8YT2XmEA&0D50t`e^9Usw;PFxQh0^Z1v1$m zH!YsyCIRVvjIyWKb*^jvn9_Rq+X`kai(z_QGk|wRrI;Y_3Bcaw=+v@ygeeZ<_qR+ z2)N~1wp#ZiHOJaJmVlR0(z~7Z>Q0L!jSp?1U!M7PF025)Gzk9l4mhOvI~-!rSi3&{ zk9*<&UK9Ro=)IpF2hr~(FOKd@mHUi|T04akTdVkk&H4sp$;uV2gG zk=5l#JxU5?5sQ?rhQ#m_yicK9xetJJL$nCZkC+#tl-;WFm{;>{e;fTxf50T0ZDP_O zr=m~Rj89?>Li^CXKn7$TlvB8(TP4sBBcurPDs+ChAUzN>!cVmk7c3WM8)E1-;dbac z$);c@;&s^K`=fT-u(38vWkCvPIK@im43@yMMwQJ?jk&+(J<+i*l^gM)4QRQEK%gab zUyT^96xCK|BH6ibVFj;x4H%%l z44b^2xi{DtDOFMVKzZI(#&8eYK3A$JS*6d4cS&L))0%RJ&VPh`J$@Wi+A1eO{(Fn`gb|u zQ(F@gn9Hmj^0jkF$!>SHu6)E*(HXUiIqGYdI^D+(Fo6zYuI_(RR06ev;d^vf^`p3o zF?`cI5r(O#%ac4RAjepCTe#szJQGz0*S18TDxe}mepO9(k{LKL)cpe z#kn?Jqqw^VcMTHU-JJj-IKf?lJA@z;+}$05yAxaocXuD$?VJ6)=Q-cr`>i_X@7yy} zQ#IXv^|jXO)qzMTry6eTfY0%SY0KlyKh4Z&p4@oD~{l8 z9j5ybgIQ@|!+P!st;Y~CnRiC=SRm7%#sY6kcApPvA+6im+-$YpLQZY1Uf~Mytptem zxRos3r=U+4A~0b>p@b%lMn2K4B`a@2C0F`iWqy@w?C~quo>cgr-l@BWhG8d3TNAsF z&=iaTg(g+iH zx1D=LvR~>E{3}S(5P^G!R(T8~v3W!tKbkozRwkwnC!1;KSw=P-y5Uw>y|Sx;qHDR@SB@Ps;x3>m_`$m~(XmA&@NB+VeDBG$R|%dvjH4tg%=c-& zIAtzk2GPx7O>%#^qgqOQ9kAW$cf(a)n6ejnx!=1AlhT{f5MsB1lWlZR_K%RiI)h-t zj+xlbK$u7JOrJ>@>uYXA%`71B1jyeQ?|Ic^l7;!<$roje0-p{Ll3Jcb9lWO2q>XA$ zK1bD?Y7x6yVOo~+B#_v8t;zgGl^||~hr)niA1;*^&C@_Wl2z~@0Sn5y>f1J>Hyk-I&-g4X;U&O96mZkD64hG(V}vN zmcEPSkjjb5BV!O}r;mGHd}vqWp_EdZ7Dg-^do}YKU4Xhlpbl*^W-Bd?k0H+CESabP zse&cuBahTyl@GYU)(mG8xyddGZ<62lFf-7sK3*oyqRM%v=kU2=F1L{%?-*W4#baIc ztIs~ZXo31n4zjc;4kb92j>^)MjW$aV!J~gbT^zJCf61E!cZ^RSd>iTOT#)oj8lrBe zp5UpaM+>MHeNR)NAYtz4KwaC|yOQO4DmXM^gkSh(TI77nh%jm2{>fKe3^0Jt0iq^T z^PYcET+)F&>W~f&4vJZVAD+1@yU(%VYty6D$UW+~XHZw7xXA z2fgqA7HDt?!n0f62|^bZH8O(7Hfe@rUsAufNuQ~k;fGvY&^mVP>GEGF8Tz9z0 z2hPa{GROg>;^T-kBJefC7^g+nJ1x%u@u_tWznBQJDT4Zli@HQX$PR~2&R*$<2E%=> zAXLtgw;kU!kF0m*9U0$3iON(HwMK!AJy1&Om*>^AMVeC+dt ztoSG({X(Kx^>=#;pejjPVV2>~AyFIThN@fd^y7|@;>Ok?lx7S5srd2M?0!GJVFSwz z5D`m3wuh5d+!jHgWcEAttBr)fB-h2sd9Ek}UzU?ylrRk)wm_;cb}cT`0w znX719x1>+U9lvmT%dZ_YJKYe8Sm)xAP}M#DFjBC7;Y4JQlHyRotd1y$7t? zgT+#Fq$s|w$T2SvL-NUCo!>Jd!lB~Gpk`yGFq2_O4<}kuRq?=c;Z($Ga+lF4vIwde zlSnp5IAxS-@#&#Z(JXSS=5-Ezt`e%zd7I~@E~t)dc|hXb2C zz1k~kiv8Q8_d2yy(Omhl9=cwQ^vY~y=rV}M>Izae?p<1<6+Vww7dMW*ozcB~cQI{O z>sM1{noKT7)3DHRD59S~V{KG`0d;}lW`P(lC=$IKLFil)H0S6oEfd;5E4{GyJW9Pc zowu8+zxg#x9Rk_u@Q?_?4vLAxN8fzcLpnU_vkyJq^=l1|^LTDv6+jATz z{v=C?hva>Z5@LN=%Ng$@*B_mn#Tf@!ug6KrR5UIAF?V>keQL#oErXQa$OZXzqOT#d zy|Oh`E?;)45D%#~71tbg*{CcC!O~Gf=i3l#enhYW0fa!Us-&6*?&n`UzoCk<=zOn_ z^^VBn7|u<8gR!|Mmf4a=y*wHr;$@?j7v-sdl`3gW9|wv2O4wcRJ#}$wIl8v_wqZ#P zHf{)=4E0C)S@~GwauE9^5SMrRYHxDTA2tr8plH2y92Ol7-{Nbb@*&7zk9hAl=4nAT zCX0c?qa0h4#osIc=9&6m6ORr`<0Sv;iT*S3SWsM?AJE0>NBl(Umf0TCzu^OeyZ&@5 zcuu_KAp>VOK`MWa51AiYTw~KCz?-NI5uTTA^ct6~x(z&^R?@757BKLLvH6*ya}(3C zUwlfOdnn*_VPN8hVmm@A6S815NGMTqz4M)EYQD!Y->E);b5d{`feBRB9hi@|{s8_Qk&H((Cs;B%-1P z+dh*UcW#SHL-lefk?85{G>J6z?ku6!AMv1+IdYL#Q@Z~=f}VlBn({d#?sH9yB?k%V zB;!-%8N)W43;hZ7>@~Ib5BdvV4OE}Xw`y{ zhZ|iYI>G#4?)3*@HV`a9*EB||vQKRIxD>I4E#>){nyO7adg6doXgR$L_=q@Vb70gv z#k6=WXO&?6h$$d_+bV`qsru}^l#Jsm-(eNa5_WPr@3$WAV{pii(`7mSqimV%qENU& z4XPDMN{(*%8i&5>$0VJ9eArCDv?`>TnTmQ}eh1U$E|jJXTQ;O{(5m(!gRf;`tAybO za^LBJZ(mfsGIB0rTx|MS(kOY_6CvUVM}$H+_FyI#QQ5Eur0c;Kd19J=>#zH>5+)dZXbyf=oN=g+7h1V?o#LPz59;h2wE9PE#vV+3!Ps&QSrm|bwNJWmFx*- zt$Xe`k(ISGfz0a~mX?#I&XZnuGL|9=&3GA?C9|b8H6!Nx{&Q>$#n0o4alp_A`)Si! zBVD-2c?z{SK)st`6UPyW7d8V*IZ#~%5`IOw!(!k3^uMn`{y%Tw-NPB$XpRxX2JT%7 zRGMwY?Y+E)1~-n%g-D8nQUJdcCBsLh^JZZ*GxA;i!p;50pOf8-5{qYB z?WEW2);|CE?dJ>QHr~j>>UpGJKO@cBNFw_F>_=W!lLE`pDZYTSv;*S-Xp^EOm7vP) zobDXMUr=zQXEfpU8i(gL{>-0EZCMH031L&veu z$`tLT`*uOISnjIyrs@n^_mt&K1StkHu z+TY)vm+t#t$(3o3j^dR$A*G@IaQoP=DP64w48F0&mh*FczaC6~xL8{NIM}oyLK_-H zTCWjt-7?Q0mhkul%AZ&X3>2W-tp@EkTe)ZU{^%clFtuZV@A6OkVp1vQ?-vZ4vrU`e zO7$$eEprd#*;FU{3U>yV->OoxW(5@kt9;vzFt>+tbEBIhh+Zq5<}2z<#YoZXR(58H z#N4(l+;?NG4BnyWS-B=aWy%p-dDJ$v+boq9&`_Ni^9l>W^poz}fReY)=Q z7jn`gjRYYd%Ldn2Bwm{m?kIL<(CD6n_I(l-M0VgHXR9ap3_WFO29_;2m%<|Ff!^a`-vc^^@+)oi@>mv8olMGS_<7xctd z6D%UA)L8xg04-J}+%`sIUKuXqE4*}?KsB*V4)9Bv-qo8c=me9Dha%F)Z{^`*z7zj9 z4gL!GO@AqHD!gf#y5qb4PjdV>5gd_TI&l_TU77F=T__fmNk)%_g4~dZ+8Wc{UOZJ}XS8Ui_kEp@y02k#ujx(N=y9cF04dI-$7+^B1BW<= zj~8q0tHN^Y4Ox7GFsalN_2|j;1*0Gyj^b{hQjf zn=%zWXW9*Sg`%yQA9>*B*f;7*46}4WjUtr3x?QOs)=`t}j0h^9@ysMZbo7%$ z%gI&7Z%_IWrx?AHi1BjiFJXYY8K(7y0ML;1?1hMK3MQeMzWknN`zH%@R^O($j=FWY z3flrhB}nVVY-W!HSqUc_Pbym#1x*xI%bx?~F71Yp&FW zkyaJe*Gf^cHEZH5>(kdpf$_dX>!DQBozGoexM&}FJ{8xK+2#{qbJ%JWG*7}{A+2J? z&{lu-TBeSwdFP^K`ONCs|AQOgfNQjNzz%(d;#kSbbFn6AEswJg)SguRmhqd`_;a2- z?86u0S{C56P|1{gzJl|&DdfB5@63>(%d<&`2sShoK%K_78cV0NUsTFKhI?wG*BgsN zAxoI(h^TM(ok6QkZ)_{~1c?#rFI=a(wt6MgR&*VXaGjo17mqKB{s(eiFMBez&rxhZ z4PO_ItQvFznY<5$&r?k6wMKfBG||w!Ml%Z&>=(76U!UiYT|f!16S1vb=_FhBvwzQ~ zZU-4~F8K*eOs@g{e%$;g&o;gU(kLWdbVH-Zq@)aurE&m%{bE1g7TVsw^th01isK=U zrW^99B(zanOZ{aj=BkNH@#a+!c?lktGRw&k_SxY^7)i>K`@?SxpQ}q)heUB*eN_ST88fFg> zgqg+%v^;UZ1tyhw6TI{Lt&ugW0XD5Q4;qG{xz*t#114ma?v!j0l7)Pmj*l=|=cc3Z z+J13MCtlaaL^k}-SS*shZ2yYhb94-Zhhe$9@1rpuSX4OB4xZ8Ta-R;RSaWNo2}9!v zo#`dX3ZF%d6c0F>FH6x=>Ws0y{e3ibTAzeY7&Bg4op_JWME)e5 zKe+Y#t!wLa1ApjT83?UN)64698@!xXq61aIPNZ2_i&|Ur5JfW96r^{3%CxU1`@#RJ z?VZtG^z-8of%#@UfS_V12-iV(h&o59+xpKJ@6H&ekCozJT#z4j8}z0NBxC5)Ul!(*B*m??Bo{Hgw1%Cp@zQ`=WKL-ZtWxoUXzfL@1X z4ewXsGUn8q>dL1v0H0X8qd;Nja#Ad1F2 zSK>A=S#BHJT)R)*M@>9zFU;MjdXKHn9(q$$g!=kTuhGVqk+ND3jnfIB?ycAq*R*{U zjpnGbScu@ULYmp|WjE7HWAH7Q`Z{HDLdO_tspA)wx=OW>lNiFDHwk~ZA-xY zI%BpnSNuZoC)R1q@=(fX!+7UAvLy9niy2PM7Ad8bk*^fa5pFrcZHOv<@_6m?yhpTs zjzKU7Z*CmW2~v0+TUjcfa&lGF?RPW>`_@d4%N@?_W_fw{!q*9!%$J|RGFMmgmT62; zF<74`aP@_V?QX8#zM%K-Y0_F>MFkr4{NP0YXe(K6BrXK0(+_hw)=a;TFI|_zmUhuf z^|O=~c(jrRLiFn#iIk)>=jf|Tl{Ms3J9~~vXUoV58VxHOrxIZJx%<-t34TpQGMqZk z?-hBf0UmJ3yV|#p+HtF7?!y;}rz;|E7AbwMYe^o6XGHPph6S(fIy-RLaNK*_wAR0R z>%H`R8?RbB`U4w)W?LzP>=7HqjKCe!7^)YR0NzY#Ix0gCL& zTlxm!^cvUedDYbex8uLfI5Nr{t(XQti^ZoI)v@N$io zrS%5s?QXvO#ma?9P>Jio+aQ9K7iuH6;vuN#0v}XsKl!`nAxIoCfs9hMIX(c1vZ*;L z@{#K&r`GkU74ugEVrA)P6v~e3GV-6I>x_9jSS{^s$-a?JW0gQo<)(`(A-yHd*n?<` z#x9YvkIBNe6pWNVE`EOb&45!clyvRLFU_*>`G5iyRx zCYmczR4ni|`g*`?XZf7a<1EsWL|Y0(%a@O6pSjl8~XHo zS~zg~7RG!q!>$yLV^YW)1CG&~Q zP&9zF!VAgBfV}*T(IbURx*tr>_T+e^tqxme$S?7`*`HG#6QZ?sbyJaZFn7%2TDZ9! zGATnt-LcGvd6~b<6>?P;Y&CAx1WPkL!G;t7bS*uhQW0o%E}Fy!Nx0`F1J#KOZqgku zZ45!;4*ySC$N2-8bSw?^mQvMsH8Kr!uao!mkjf01i%GnYt{x;=A@b~$*273u=Dx?C zvp5GkrTn&fR=#-(!wLeCc_qZqAAX$3%WBDl$;s~+9h_XA8p?isLz)&eE9q0ThJS~h z0A^o2od7C_X_|e|#gcu*Vyf_%FFn``&MSHeG&Yzf>&QY@Y4CmlRz6nXm(9SONXH46 zqwUz<lQz-vCovIW;6<~={;LIuvM+4MG!{ZF8S&8=P+f`qqHdcJ zHqe2KP+iO&VW-NIpyTJgF;Q^hQ(zG2X9oqJwglXN; zir3*0zg4%*4o

kN8$Sc&6wk*fOC>88KB`7JnxG+wjk(+68QH;IBi-=v{4YmAac z6FM?`twhgv&|N0yPL&woJ8fR5**9NNu6LPzwEDnTf~b;l8nb$l3Ru>VaA;7T*r9UZ zYW2V`Gt=T8&sOd#&(19>f~%_9&(3h@3bNa@pZaHmsQg!h=o~$}7W2OvL>w*AoFraN zm{r?{jEUq^^MMilflKEOK$+(K(cRbLnsLk1v zA7^NtwdmGMpQqV*l@(me0rPqMk+C$jnsMi>C~w-3Ixdmvmk~qf#Y}$yw&Nf8p(jmA zv7X&QD@Fba4v%PeK#6nR2#f9bm{r7%{qq45lMJ&`eY=cIm(_~rzP^hO2(i)HD1IS0V32aEy&?U;;z0ZrK{d1?tGXxl zL!1f`=`)NJ%~n;pY41FVq>QxV_@OPOiEecxEn;F*+VQcq{fD_U_PHOhoun{x$yhFy zG*mj=u>|hn;>3%LXM&NIa zB^|~f-1s#Pu-Hx(`dl5H^ogP#nWd1;iY-)qP=`RXK2teMu-(bZcC;C(!JVSa6#a-9 z=Q4}G(Y+b&i+>m$FN}jVcj~u9PN|~rq75|FzO;x+p>hhpEc}W!X zxkp7`%nbtu2BaI$*F!7$M^fg6uOyZ)CHw@rytSsOhE<}e(M9%yFSqi1%|+5XK45*j z%RUNPX_}Namfy8dXLd=q-4;goxn|q&y~edrCQt}X7Y>SHdp=ciPjBX_68XTukd1FI zb8cwrjd9On8gx7spcZeb^BB0a!B}`bj4<`S!}uNkKcL*-aECUAO|U2G3bxa=)8Egk z|H1FxXy=DunC7drm&HYn+Eq5eZ8ax`U}qz~#Upp`{41A4U4b0yo_6EWgLj@@azW)|+A z&7LF5m2X=Y@0QMVZ7Ip{{omOA4Gl*_k?$7|Ts<)Zm|Jhwl>Cuk>*TnLiwkTKOeLD1O=H~e0t^g@ z8^y^}$BiFsS8Fph-ql>*;54jx+CXg?kov{otqc|-P;%PlXUN@}c2BhrEBDP;msW}& z<`WOdUa-q%sN3L>8yA6G+AF>GT>6$?TZXyp=y{*V36mOfEY=+Ebx%4=AQv(qWX(G& zQq5Ph%FYuv9qQG>F#)QH(P=mJ+V8bI)NUH*S+Ese>ApGc6Ak*dP!|18p2HT0BpgN) zK469LS`h!)r<*`r)g&8=ga9+A#7N)CQ?-^(tYWi!XR&=v-CtRLbw{oF6MN>MS)Ant z-Pq8i&Ze8(yLP6H8iB4sV8P zM?oqOCmVC-jyxxS@2EC;3Ic-cx^hF0?EC+IDEx&?U+8KW&HvVJzZzjG+OV0#y1(N|1@|Nen=Z`zZ1ez*Qq@&WHNe2#~_B3BT<#G4;=Bi zDPl#DrKFhQ{{tmF8*nly0P8m%YQ$>&VsaSQvF}c+Y~OoOg6U3Zk-6T~-Pw!}uVaaF zJa$kaA)WKZjsfSq@%qe}&uh-R`a}yMyfH_Ka{r1ahsO^VK3PH@e^%R{OId)%$i5eU zAg~zd*FPrl?h_5^kN^r^Y@BCEO0g1{rsh_4v7AHhIFuqcBDRF{0gO7FxSWifIFah; znIoo}8+hGGOcrIkwn-J|!5LUkwactG-t~V`FR&?%rgSjejxXPqzbV@baAq;=4f-IvL{g<9qn zZ_@p$*@_>1Dx&SpB~=o*uzY*y!09bLpVKT5Y5!W;crv~a@OJwNgKhpA3ONQgPli?} zr}0&$gjpu@om6Mu#J44QrPdIJ6VG0P)q@l_(bCv<&htF2!$j0cp8>%XNN7h#WJ1&8baxV`&>- zNhO}}MW#N0)iakBt?A95&G-A&cJ`)fj6r?nK@QvojAyy6OWsW?mqsqo3v+j`wU3X| z2o&a23P7G1vpQhqj&geGjah~rLCZ<*`lfHaR-6;^ir8_)gb>~!x;sQW`(Yx@LqX^S zl1Nj_(8`3SKM^lEYW;!FLp-7v*j-mxrRyFiTAKhYq2TGw@`M{I6;94WuLul-(P%2Y z%PM55sO)Rbr7oK0T`r3z^i`TQUx6%{_eFg9&dOMk9639h)t!3ZsiLNaSx!oV?2RY7 zF{&2iAoxfuNd(xumd^1qwBGsX8+qud@`ERig-3^83c02*D?w zZDuJJ4HIyBm$(fb)~)IdqQ@lHG&@t>k=|=2*-hvd5w}T7qjw6iVcx8A#T^?Aa6uN0c(mLFcMZ_CL9fYxfcNjd8Zpp(1 z`nFQ%W_yTg*b|F7xu>Y{gu%YtNICgc0mMNS5j4u%(T7;)OG{nPGdb3GpoyOo^1dX^ zRp~JU_3`K@@&l@^I@UNn>#{Sgj@Ihb_@SN1@>|m~)@(@|n3W@-1Eww;p9m89)Sa2E z2>0^4O0Pqv=A$fvk$*@)dV+AHvq8;NJY<*C@=&-h*EZTAFR(SqCvp=t^q3O*SmO=` znUpbFG<0gvm{p{4%`z_(qM-&Wvt?R(#g!mPxE8ZVCuK-!1|C1vbmMX%DVJ8rZ093- z<+&xRA^sc z)77I8ivh~jpI}eqr<_ScA5cyv<0wScDef~>(!uBlR zopr=$YcI;`xzwySoa$nnnc~{`NED_l<|;JUZF;0JBPC$_V&chid~<+u8fPQBS{h(D z(?Bokw39sGN@w#0^|=nP#hERKQ_RZ&YmD|D?^J*M!Cjl~+s3Hpwu8t;&&m1KV-P+^ zuI{hMW@6`akqW)Kb6ECpfy&A;-e(lOjCYjtx7t8mgJ(cdZ~EC2{#r+@{oX$+tRqrJ z9un`O+b>~W9=m_?)R4v9=Tdf0z4!xH!^Q3WJv?l8niHt+(tA_NnMOMF~1$wgY0egQZuza@7IK~m&{LYm#`?N*Y; zW(oGYu4KH&TR%fvqC7bE(q9Y^Ia~k*&Z|AI&2@@t>yGnfEaz&91Dk6>H>-4ZefgqG zCK3*B^8=Y*_K9BIac3vXE!@pFmcw&|9Kj#1J@jEec$ohnMjtX_bLl z_{$T!kHcx04+%{JF0f^o{*5)!SmgwSX7|!7VtC{VA4{8QPMA=CC-Tb79MD#d<Cjx!7)uxXm(53?EcKYay5P2NM#T>v`4E zfpHJi3-534gy_Oo1g;_?<~yRg^-ksM|LjmValU|2LiLBOmePO4E-$PfIue3q^J(p< ztgP%Nc>kG&;+JMfRXFf63gO4^-tSm3_^Vgbh8euUD^qRev99aAyU9GNT_7gt>YFqx zRnH+PCO`nrB6YQF2FJz*+`6jln6zuUEq|59$@=_bQbHvEt5r#P^_tCgYbguw{zc0` z>%!ewT@DpRuPDt<>#)A9;mv*5z)j%?M+zW`)399Ed_NgcSk67OIE}kFt1l{h$jW+o z#YTRUpTqOf)feq*S|KN_Pc06ykhBAcUpyC~WStP?GWPn46(foQB%mSFPZYZ{Z=_NQ z=DHz*j_M$A>dG&Qw0C$E zJ_iOP<6jL?MU%fbDl3`pEmTc;ElT0lBjQBK*GKi3NStTcP>I?oDE%xGty%RXi}PKc zv-LTV{@!{o-d{{)WVV?cU=%0@)dT}G%AaJMUG|YZa|RI($cM-pHdJ)`$Z0 zF!g@1QdIjqlXl5sqo@kDvv$2zg$u|A26hav8tAKSAM6^vD+DKgOYO*?_?Ef~%j1}& zLk{#8ff**z-;iEVkJIqE1FIG!M$_TV+LS8DkyM)e%VTd0>n4?U-J6?4YEulBk1MM6 z(E5gf7Sl79PFJ|Lp#$>tLPlFY=i$+TjYcbTcz7mlC)>-pA{~pMlZ+W9vUJgIR`Uv@ zvsH(do%U}9`p-2^jhO`SS~;U=i@aY+swc$n0-JT6lL?#eE>5y9wnwYfIvh}2Ujm%) z2%B<={@*_Je@^(~0w}b`%(&=rrR9GHF!+k~tY$PAMFQ5FCZ?ni82FqTz$LyLW$0kh zxEm8MS!SAJ4boq_^tUk|{*o)~J@Vfb{RWHLFb8v%YGzDnC4v)W&&9eq*GE^Ou%G-~ zp}@@8evBvF2m4%kuOvw&*B*seEzp0f9B05UcWeQey| zHkD=-vK{eO9)6^PazH5H3Oj}NQ|cc4etqr5d*T2-^iPm?nc zaF9W5781pPk_gN%LHW9ei~b*9+srO6uibU44+i{IMaft4qlHIa?nw(a_!chjLJcRI z6pHjkYJt-0uI%EslR9n6Fd*=H)TMEk5SyMzWa3>p-CA0X0XNFw-Xq)tH0?))zG~vrh%+ zc3g7s*5b~+TSHG-PK~$#om{YOpAnz*0oF5QJ8l)cwvf*`CsR=9G+vs<9N~@W^h#XY zEJVHl_5g#s0>B=Q$Nd><<4Mh^c8NjqB3kP)6dv&5S4hwaAnQs{VxskBw+_20+XX|K zX2oL4wRR2u`#M!RbNvMA1c{dNe$aludm@%$ye;|RJ_Zmq!*Skd zpB{q%2C}BMA>xMkA(hTTbwVQ|YM~Fa&&9NMxA@+DPtvsCQ!~rE`Hw;b?(J64xt_m4 zSJaCFW-E&}!TIL5p5HI%uZ{tu(}6}le)QPS~nBuq9^N(OlL#1vcQ@eKbfXH zom);{*P;pLiqNypazT{Rwec=mV>aGdacL)^;Qlj2s%&P95V~3hk?H82Hc&(QG;g)&EvCN=Ps&`fCYAf z(7L#uwhdH}Pb6L$Z1Cb+mv1-)H!9bgnk9u(o&cLYU!eB3+lg~(q`%%vEg$;jQOSJA zq)gKXcS=NUw~mG*6M>b!BL)anV_7qYu%Ow1TGmrPOs(Y)6S~Qnr-Q=DE6r11gjAs` z>oCi7VUTEb=~>jmx%v-pvk8otP87qv73VEIes^RwYnq|nwNFBM9EHUrEOct+1W81O zV|Ssujn!jj6pPpd8kv#qI@!6pISAcfD+FFsgvye_kS$`W}v*!nQpX>=# z75|~dMo0wkg~fGS(r#-0N8^t2m(=7J-zi~%XcCT=EkM^N-*v`cV5f{Nh}UF1z;oO` z`H1eWU~naU)fI%}LHu`53KortsBdom35p>21ZAiXtB~E5a5HMaceG#1P|Nz(p;nU1 zLqt#RCmy4jyKK2> z*ro*C8vFQOUT z=&!u`jW6D|-tape@*!F!8;L$G5}Q8>BW+}NXTOK^s{NX8dS`8b_~uuu%EtvsC7|D- z>HXT##Mf>L`?7tnl9A%-nL-!dGeU#*bJ|L|6?iKw{8HO7!_b;LzkYpsr}XXJBlkpK zbJ!JI7rGeN;X0JVrPPVe%x|K>MDJ>lfwn5bkJMmIF%<_@rr|k}iuCgxy{8{kR8$ps z`FniBvVDweB_Lj&I4oFzFToY_NGKE&13&e!Yg2nA_Mlt_wfzqS9dz4nOA6Q?`s^td z)L1cd<`+vU=|qb(w*Cpbh%vgl@(-WVFedp zglKMW7}zQbS^u$qQP`5@G-nnWl7e%=wbVfC-IBX}s>Jc0CO(r&uPlVU!RT!S+NpNO zbP*;=4ufF~5AfzUQE|EuQjeNsbr!is~j9gMn~v} z*2EM?6}{WAyqk74iR^fG&7omYEWul)+UFXZm??sQOWKm~kImj_tR@3Os(qc196|6+cNO58}^lxfUXp;DL^hY{~-I z_={!99i0Jn3BblCyE9H?l0LJAGW@c4t^qEKZoOylhebpsk&(Z^Knyen`d+J%4D<$w zEP8aes01`;mLZ|=a@P26Tdwep-t}f)W}G>vU-L$yx|ZbWYq+$eur_gD&F(~BOeeVM zc}z)hcdF#UN6UE2FQl(lHABI#83}9@NT@&*NZQf5=3MuQOg2dnk_ma%wTSF=I`3rT zU<}=X?bSK%+oFenMFn|XaEc|;S z(IDY^R#Ip;y<~Z&vR@ERUPhQLtjcYAHG;@*hM{@W;Kv>BOyV8S00;_n{d8aV3%_SM z8cMm0S6nSaij*96g{dFj_!Y!7_1)f~{%E0h!#2>=lJh|Lmg=O*mWqdGp~CB*vH1TN zrFnUjt@|zomV@#qq2r@D$Tz4_GgX`EgJ91(T#f637RJszCkw)K(Bf4 z!9%Z(>c#{qLKhD(Dr{4>i z7s$Kx>!XWzfA(qK^@Jd3NSxLYJ};ZiBzDk4+0sYnR2M`XrhHXa;U8Jwr^cH%n_69! zyw2)3Es|vaSQvYo_-TxvP(nhE)S?(KK9yi3oJe9xeO1)X>_cppc@`Pj@Dmp6#-5bA zmS&XJTE|xihsx=Ob;?>UU(WY(tr^UtxYqi$7r?$G}(uo)t5!UaQ0b&|p+`+At|4c>N+c zQZoTT22wLy&ov&~9f8C=5Ggnpu#7Zu^k9aHxRWb?|`Mm0d_^h;Z1}c=F zyfa3B4AY_xgC!w#|`cD4T(U7Jz+7ko7$%+KTW&kXnrzLE&M+!?I1oT3DZy!-U@ z_97x9Q~kbU+u{mE93>S0>LQp+vw~nV7p&^foV$+E!w$EJ`o#mCWow|gc#4~``v|&@ z@A?Uzye0?|Dht;EV_HK>EFF=!>_Sc5C1!D1uU4GtEAioo6Jg%B8)dm7iCTdc&jbi77Vb>l>3En)b?9upqK<*YFO z(5m1QS~HG^zr0nnQ%ifL;0oexRPsGEgr>Ny9gcJRK=#igu|7&A{*;#d?pFRC?Gamh zdhySXaETL3>@PAC^Ozb(2=kl`#YOc26xLMQ3~D}x8=-kcs!)5O2vx$}yC@+M#@Yn44xMdUfdeyw$h1NXcJqWnagyHQPg z?;BK?@ULUGfq@LDnPQcLAJ-FwM#~L-G)>$c)@9E)Czhjz7W$DwKq=QuW4-Kwsif7* zuhH6_k>H)Vv>iXU3r@Ybe>Mb+_^cq(=P);NS#RFbMoqa?a%rq&HTr3#RdRG}ITL~L zYianrC%wR|+ki+HoWZ@=$P%#y2@RyDXWftVL>@wA&l=u_q4PQJFSdHW`z%uYc;{aA zX0l6YtUo^?(eyM3E_8FI5&iDvJV1f=&|G=@Qcjf~;jFw`1~-jpinF-*Y>W*0;twHU zA)XHDR&m#e0Y(z~my%{Nx%LeZq7Sl=LoUW^*A)F9SzYXAdh}*agTg$B7FygpT?_Ee zrGq7N6x=?n6(mQ6@k&P#%V}$WNH1Emp(v09x(V+Xex(61{R?$?M*}Co-T4L1D)HZ# zv;WQ-y=w%XcRok$jV8~4*Rda;oRm&%D8IfXa?-jMiv_D^cLg)zN|jJWb3SX&-!A>ph#3o~(W zMUZFE=pLL+f^|{3z3Q7cP4k6x(f_c88M~UGE*H5mnl4~6g*pe5MnsVQVdSTquz?;j z?yVoKg%vmM6CxtwH-{E;Lo4jJ0e<_#okBw$jHb8GgRy0_91964s+Ff7c zCO3aac`6#jB#a*deL29O4Y=MEuk`k1QOM?vPg)Dn<%y?T6i0O8++&eIear`t;#_>o z35l4kgo8SACCmTOe$##^yzCCLGm0U4KSu&-$k;%FafsToNf?kwxfcP)ms9IhT;BFFqpkpKWhVr5dRO$$9=QmfXO zqWaE+dJ*jCA2}V@2=Zg(rCtK-y-X!1PBi0ZAy|!POqA|EL3Abru84exR}wIT)Y#g- z7(Y5VZ#(dCEkkazv1auoEcGGzeG%R|wbYwTvYf)D{!eXh85Kv{tqJ2AJVq}R#Hu5Ks3-FhFwkdeu|o=p@FqWGW%61fbh9(8KM8s#^J zxwJc*W(35}o;&;^S^DhwE@EPeEWEF=n<7C*5dz@x?LMPeRoIN=N3rJ*&QHqIi7H+< zzr=BQZi*q01c|slLtDSq{VZT(UI-R9ykvHVGQQ?xyQM1-X!fRA;)vbhcj*uT}lTf(Bpnc8&=R>m;u_pP+(w zux5j{QHi11ishSBhGGnemPstRBj}NZ%>RnpsQnE5OiTI%+>bGx2rLCn;SuSWBIwG; zItxz)W6}Q~o|ck&hHt)Rs}c8p}EFt*5hly$#>DR(g1T*TL`xlbwONne1Fx ztblq?>N7Aa+VGZORB|J{kA6cLqD)y&l1;~ft7k4{0M$}VfLwhrx>R|6)cpU*y#O!e zEy5&F0f!3mdOqkgVB{mCGB(K|&-w_sDW>^X;_36^5<}#KgE@QtiKj-h3pD6)4;FYC z@;f+JZ46Wnv`J(<3F**6AV;0G^XzK2X(xGOBdk(^Pn#$}h44QJcmpMkub;LMGY5(lIg7t^E9Av7Q&;*b(3y1G{1 zqh8Ev>g%g}2H@G;suT-gt4jKkrPTx$drj!@cF#xQF}mm!yQ}`N=r8u)$1xtBg5wze z(lZ$2yczooEnSDP4il;FM|E8FSISBVzLZqz?)7Nbw*EaT&)nuULX)>Ei zC50~&zkQQ3cu~^v;41V8*%i9FN#hGu^Gy{1P=_I5Sk9K~j(pw&T$OvPCKVC`T90v! zuVHma!?Ourt7> zG`l8$dm@3+hDkM#FLd0+f<-K0 zsNRO|RIAmS?i${uh!5K`Gpim_qMg_X7QXMk9Uf+?TOS! zS*(I&J0-+is-yfP<-U@Vb)#o=pEFf8&oyUBznkGpBJ?WR(C_krDF3m0H<(LMM5_pPe-W6Sl&}vv-@_CWmf{D>EFuds>Dq2aagLpSywwio#0K%W8P&N5@i+<-1x2-EZIQ5 zpNZ?{$M%N+@Ex6f_tu10@UzHzox1|W876y`i>%v{ay?3dn|aT}KeGzw`I@(%t>-+B z2Y`C~GTSK>+M{7Q&gsqvQ(?h!_-lZTRfO+q$UEHFSz zCdjyGmF4`p-ak3K7Mh<0Rf4`V9r+3tfps@!C}_z<^ek$)Roe+@$PdX6!>O)%9yKR&*Vcb0Uy8@b9g_1O~rY1|kW zPtg(zn}%8)1WRjG(4rDK*FGt1ozBktJ=HF^2F4y*-wc^Dx?sK|krdjnSJtTT%fDrR z4_WP8Od7lUJj%o7Gx(E6*IZ5Zb;%YbrdZIMjw&h{&%7=kenDURK()Bpf{;qr_m?wx zaICeNNJ0ViIzqW$8OZq zCi6=OMu{K>q=3|tPSHQvuMqOCrw*&CX&Kw+$$D_v-{q)hd%aSF&+7mkWg`g;;wh!_ zzyMNDe!mFM*0N{>55r2>+dV8hTtUiSV{NaXya|TyQIP_eV`pj?hwC^tU%`GFdmN zpa$Wr0Ee#jDgz zQUMQ_*5Fg(4{RAvDeMf)Y8@N4;JovJSw0z6-9%I10@WKpp_k(fn!+mt`neq(&+ z9t@Vqe2tIB5ZQqeLr!+vsUQ2_5Y=H}*o(sruRaQ>p{zJWa<&)ky20gfsj1~(`x^;3( zr(ak+N28Eve_XDLPS#nkxBc9a=nUOSUODoG7A}04+JU`ds8G z>6Gj*=d-1ybjU>g60jEauZMj^nxj|>d+la+HUAemd9bD~L8 z@3LBF$h}9tTT^C2VYKB7|1^SqJ9Q#B_e?(!^TT?Ju-b;){tPs6NMRA3=|4`2Zn${W zn5$~Z9PbQu@$+p@DO8+3q$YMW0`bD9+k{Jj@7V?5wQ=i?t`SL5Y?Y7gNpBpJv{%KSJ^dbJp@ zDeV%$7oc7Vf@Z9oD}L*P$7xqR7c!7;(s=hke9zYX2}}c|-bzD&O;!wq*o=`T-uRwc zjGVOk!rq0l&ZZd3d=hRj9@oy!-Xj66+oV=M1E-)&a_IK51tGt#nb-_Gg3O@f|65P;JVV zQ-Y;Hz-kasdGjZ%;peA6vjjTw^iOdFrLDUEC7RVCib(4Ccr2>zE&s1P4jShYXLQ}m z@}A;a9#|8*k2wv@mRD25gesINsd0EO#)77Xn!FELR9?RUrc(*`g7~vO6&J9wlM$Cs zpl5SdOMu$8deH+fFU(A=VTe62#ETkz=_@W##khkne3hnM+3+Oj zt|tjsqHxSJTzatura3N<8C}<2F5Gh;8NU%s)ELN8xcKt?C3aXI=YPUIf zzL%3!XcJ)EW_|!P3}ur~%>=ftVv{)WQ7QV|Vat$6ulmW!Sx~kJ-?{w-i*?iy-vXC@(stH*vhb;7u z+6B^8j#R7}H-2&?_M7JS5~VQuD48MQT8Ati3P0?mvdvlAA7}mstW9$goJ8u+2B>m+ zSZq3orDt5?ZQ4MGzkd@fOn%IGY4n-~yY=af@MH>uMrN(a<4TiPDPMpSM}8CRqbVZl zJBHIWm`hQpY(AZ6_-jLZv%M6>>hrd+?FV6;f8+X|e{ubp059j={{|yLk|FXGZSkFX z^*;q>M#zGiqt02wa^KH@A72iZHoqm9@qNB7ojGY3w};%$G0{Kkl$kolu5|C^m8}YW ziW8pSrt9!9->Ik(bs8|AB0Q~i<_xL}xL?f&weJFg={R2D1F!OCOV#k$P=qndx0?zw zCeUAEi0Lcy0&iMDv$jAF$8u>PiGoB(({eA}ywi|)^8oFII!92C1lM=L#n849O(-%rhVN!BD1*=#_M+zhK&m(cCUI?u&~Ts*e_Ycs69`- z#wSpUtbeaYKApfI9o(NeAb4Ak|GCi>-U#yAt?d!@^aIUDguJrp4lc5O&Lm*moxvp5 zi;P{?~e01n0FUVr|I`Dmc1h6t~DO04cTY_YU|?q9ZB% zm?js!PiqpoWvfky0UV^BbnXg~=sv_}1y4~Kv<<_(5AmDIVBDG@=H};;%U`_Wlu&`H z{=cBtP50tCE5l>%)V%}=;GF;I!r#X~1q%#cN#1b^igT3NV-H)$RyeSfZ0=PgIOVHJ%ijW?x@2Lt?&U_VTHl!Vv~U2k zuVDDiJUOd=-=23o*sk3EOCazC^cISYwfzrGW&aXA|Lg1PGg=#zt!(e?Xc`zOc%5mI zw|qi4{PkMWei>H@5vqFnUA2+t{S`(MmzA;$sX8$joWTgh)n)a?HB<)VtMrq{+u$yx z8N{&4*5=bHZND9KOLh`$u;x;8cqYSp~8?)T|>-q)l%#!zaelR5%| zuE#Oa+srjV!l1VHo0sh4OUlP1>)c8NSKnYTkGA&lh@5t#Qa)`fHU8`hP5q3_C$!d) zZh*&T42cMaUZDO_W;D)bCcw`EBWDEyS;3OU=t@j7>yvpAQH!(KBoVQm%GM$h<%xtR+K38eTeG}D{=@Mh zhl$k$PlDJvcDu7iAD4qd)I^>RF&$TH@k!=Bhc55SQGR5_PF#LPUgsmPX=x#bPVM{j z2~}jwUfzC&lyw~BQ9ZU^RH-18p%@^qhVy#RJC+GsN5w|d)lch*As2%7x3sjWbTdI^ zJr<+)YU8Q?FIY!)#vpWV)+8Xf{1K>CSOKW32#GUbGy(hW2RTA zU9uy*aoh=clN$;khX=)g5?v0WjVhL6%{@x=cRg;xMXi^2=_cJn)R2=2bl>#N?`r3ZM>ukt4aa zsPd5}-*sSc-Ts3k@^se2+|)NPuys9eeky2A@r3x57j+J7MQ;h%SMju1oMng=pzHWj zFiPfRk8gzOUWKAX-Q>6RVf1xPG?CQ|-G8a0+vp)Q#(j}FexDcUo|Nv){5FN+{U*Rb zC)u5CtZK7f;zj>KE_FySNyvt??C^J8{F54)x0eGbEhT!CIvWVDOKL?u69E~l*GX%M z-JE-y$MQL{=)`EjjE3B6mf}}@8`iH`don5QlF1qM>2znhrK^BO`Y-W=>?-IDOra7mfcRP^#&wH}oFt zYQ zublyt0%)8hBXI{e4i={9uPHLWO9?`iH!4x=>{ONHY)V;$BxfvH_LgL|wDZ`IYZ8zD-k zB($hS8OG12!u=TY%Ha*_fpDaLgmbate3~U6Br8sS;w)~zxMnrFFO5F#s=hz1W~gM5 zyOAJgnT3I&uv3(g)S*yO9q1O61mURm-7|Z7+7su$)y)ZqHhMOIXXqsOpnRwZq*0M6>g}qwe#OG4r47W1TIEwq zJtI7Ddu?_)RFQT>`pWfYF1k5b{ZQjVPG9&B$7ERy=p zdphn<`qfk!`c7@?;b{(6idyBf6|F8UuvHg}ly_I547&P*f>&6ibi>7ai-`du%d}cr zt7$;<<2305XD}{~9lYL6VRq>}`XlJeTPlO;7Y&D|1AodOl6002qxa?#mV6VgY7w5G zB-N1jsVly_>25203W*g)%r}Cw$FBgJhsz4wyF1a`^4cK(t#pw&=4%w333f&atLXToBCZb`_=lHY=2iUg9}MK-OI>nnkKbYviiY z9%3hwh2XqK6%95s6LR{swAGbJ1#1o-5ZmQ+W5RwuGy!MuV3pIGx*J}9Dktr`q41i% zw@isf-u!b;Pn@T0YPkj>fZO(ZjoHJoWt4T@ZNwI+LF@WCh*G$dchJk%lrct~+W01< z`XqmeV@R>AES% z8J7HINZMy}At$%Jy!T-*&-JW6mFBm>!NKUbxPEOcRlP~Q!=DQ_{GNg}7avaV zJF0tY1bxzh`1r6Y;j`gCos*f96^{Rb|ST zWku_eWrM%FvjgI^Uf)|8!i*2dlJ{zSU*ZjH>x&Sicc{tI-BGN)cTp|ql&yfT^bRV<>taPIQ3XXOct3PKKbjp;zl(@+Td6sy5lN*(;fE}q;gF{I=7O7db_ zxAki=(GFAlk@@YL?hOx54hiwbFrj3pYZ<8+|IuZJdDdneOmvj)DZ#7kl%CpkSAi=W z%qVos5+~yDL@*^g2o4Sut|hPXZ(X2vZdwL#N=ei+??Jo9)23C{;;I05Cw|&*zi-|5_ar* z)_o{#9!!ZiJ;mJI-4oJ@N;tpYioepo?E8z)wEI3Bl+=70{Kn!!?{9)1;(yw%AEB{9 zzT?uVz2z{|WjFu5bI87C!>T=gC>N*QxVs*9)i0b2vD4(XIY zUF9G95&h1bDp*Rc7wy%%lqfJBy92irFy;*s!GjO()ydCUIQ~7CulpRT&r*N|#Y7|5 z&A0d=$|iuV6YhD)cp9XVaSMfv$8d{|*SWsFziGTsFArr0IS_sr57}6TrwnzTF4bYA zSf+{HcW$|LYcH6ESnuYO#h*6;BOF2Z%56_iiFZS>c|H&IYi2|1UJ?#Uwo&}oX>?(s zcU>_B)$;5z)%65<+9awuQ|%}*!O7etRKlgNRUndxkq^EraAs#4(y;1%Ta&5p3e@eD#8re_n?s)s(;|RY-YfDk-!eTZ?8c8 zEddj_R5nf^pDysctcJXdN#gAChLjyElVDvA=J%KvLY5hV)I^QY;?jl(%=x4ywE0pG zOI2GabkX60U%zyMY#`=vYfroUoPke1I!SN0W+`5Y27%`-F2q=e9z7?HlYU`Z@gq0h zFj=Gh^*dLk`#(n4Fo);u?NgfDSLQ2%Og09~37;fE6KjhU=e0=!Uc38bG z8o8rH6eFCS!tCWKit@YS$%|~9G&_j7tD?tzcff-C2dMsQF5`RSjDUF|=FE_dN^G`(b4z-_KH2(u27Rq_ zr$Ge?!zM5eV4Wy;Vz`y;5Fn_$4A3h0p%2OK5J>KJE_N5T&?n`p@m`ggZasN(G+NbZ z^rYeq62NhOvT5jUe4IavH$VyUqI#;hK0Cf=QzKsl9T~=V=J54Ldke2=bXERrK<)G5 zqaTHGtKsB;je&J9zjb+vfbisG0#YFllq`&$yOU5>PoUyVnXb3o7WH!vVP1P4t`mt_ zn9G&u{-4z7u`#vhbA8e%msjI61`K(NUMV~%BX>SFL{N8S(Icq4 z1dFpt*1hvOp+=CF*zXSXl!+8FvM7n|9K1l<-rl}K>EFy{U5Na;BgbeZ z5Xx&4`*p=ADz}QP>&E(*@D{eKY(d-Id#yS*Y!BHVjK?lb!4}$dp+ITuVCC_QvHBu{ zmj-Kuwn1j1VQ44={V4P0GqjJz~FTH){Pvop0G<$-hEE*E#eD66cY) zbS!<^P26E3kIOk~Uxpxa*W#GSFQRCW3=X?<$4HUv_p3!M?&#z>J|9uFUO9OhAu&aV zR^N6r8nJp^qCEq*WHc$~Mv^fLRL;l&RNf$lB>C%XM#c{}rJeomc2Yoi9#gC3?ib}7%8{?4!GK`?y{kjP~C#J|UY zU{IHJ5Tszik3sOXRz1Xgt!pL4`6*3XVlIgs7qZbPjWnUq z!1mTZ6*s;mt}CZ?()6|Q$C20utzK;3aPblhS}=h<^wfb@tUhW1a7%dT4!v#Jf{KNk z^5QU&_+9CXH2F#^T+mQS`S4WwB^f1O{a*GKOYo$3;BU{tCYEfzkC6`Dyx)6PpOJnt znM2C#^cn~oMEt=&Pxinp*__vVk>l-G3)eQVIHj%W?Z+h&6E>?fILrPW$`JEN-W^U# z_LVkwXo~6K1E{&RME{t{vn0QuY-y#8;>J(^`gPAc-Or6W+q=ET&vaA3Dt6>)d;sX3 z@V6VX{0o7xVda-`5Ag4!(6HGRKP({KkcRN;wwg2^!I=zh^n3L%RyT3|pOJ-mYic!r z-bZkhgemIJd>4~icK>Vi>V8)ETC@k7QAzQm_E$`Ybp*;*eUZbsdDR(*-S8LRI22pl z%XC5VKiR|!sg{c9^OawY;XmT;=072u|i94Cfg?WkM&Z0BC>0x$t5vls3E{r_Nbn{-j36C@qI`41Y_ z{S__Y#(}HXAXS4_U&51CcXkss8yQG5{td{|SjKoz?eA#*Z}%IyLcUG89f`TV^&Vr9 zaa6ggXiGBoI`O;)88mwGpBVCxXp(!&K$`wPol$xq2C#W0$ePJZ21ErkUT8TlH0#gL zecZZf*WYn${p(yyZ2q)@fPc8VcWyY0KT4$zB9ul_MI;Fc-XH-UOx{IejD-&Gv=sL_ zFL=V>>++_Ge#D*9POP%~`J=hHrVJ7Fyr!BJV(;9#`y02zz;p@bgOT=pa9o^PuZ%xu zkp-((B3_m6(>S+;tS4K;(Q-WUNqUtC^#}KYn`8Sq`I=*U+Zv9)5sbef{Xav&ot{+K zXc1@UEfvgYY}wp+=KxLtw@scn{G7h+C(b{EV{OHIfo)#Uo)W_E8(oJ8>G^bLXFg2F z1rUi^GWLVW0e$yhB

_@KxI_*VG}^z5`-2{JcGTmkuetcLKBw{XK31lw-BAzuu~l z@*ld#2eI)d>(5yXtV$dF_8_OWa;Je_?7RZ>Vu3Z{|8%hu9e617yWc|S4mlS|BECZ! zyY_BB22SeGGSHb!bFUdU!@fwsDDUD^pP-j7(2upj%d2mi7}D($s!B?=0ju6pIX)e zLn;z!^~L&!!?~Lyd-gmC-1x&v`v3Kkak2H6$}yHEr{*fR*i(5iV<8um^O^2!b*NQEV41V9vC z+ylnzMS9RUWSW7s0YohJwu}iV6xXyUW3cSMFHlSn+qeD`HV6Is1YUk}brNVe&2ZB! zPV}ZMW8~P&v&v;DJ8Y=fySGTjFk2u1b8m>5$Ia~`T!e>*7t(e}l8#MC5@v@tV8^r5 zcdYd2wrlcEEsL!$UyxhO(@B4^u{c@pXxW=+l+7vnqRDf-e;cz^M&kKr^B)^rJEo%o zkrZ!irHWQEf2eR4m(f+shmKWcN!@SyRROaptv1q$iuq!7x^(u?8+p0vKuOnYz0qxUyKMQCF|R~Ws|rJJ4%9@U85u3r*l&4#Dx{|g zI%51>`@z_|a5K%Fu=;j_(~}D*u;tZS69{qM28|rhG~wq6R)uB?Y!>FANK%ggxqo=C z1w_S5_h)kYiqs+%7L+Mp;lWllPD7^WDr! zyL#b{B(wF`0g)s;@5y|dd%j{5E*{fo`%;ot;#$l+6Wqh%%(nqYuhcuMm@m1Nz zv@FVeU9LHG^(HMrapY15oPmhaD*|NBE`3B&jRl&=KljNpZi0{AC0?~0B~~|oe2|f; zb=mY=YCeq?dfg^fNPO__vl)jFlcXC_3(-(BJFmH;Y_gK(@WXBHx;L8$WgksPR-%lE`Jyp;u^aiiSem0WK_H|QG;7^86Ab*Rcb zv%*=Ct3FaZ81n6MCsm(E7qM4?xUWHhu?Fj(KrYdN3vP3#D7Qb<Y*Z9Iw=#ksZVT=@tHa}YQ4ljO;#qh z`Mse!zxfxV=D>`(Viux#{{FM}?<5|pz;nRL$0M*m3X}xq7(!S@4!p(8N;XpkW_ycf zTTz7pjS&u0o)1sprhxc3Hr32vt$4PlW>@2OK$Ot&_TDBvTuQXRyS3V!fxO)&rHHv6 zY}9m~Wid+Gg^3+?!(jG!M^CT9jWvy07t3)*)7n^a^43eyC41TzllZIH@bTmk)x)_# zp~CE(qoNVOqD)zuO{RMSPmEK7h9$*6P%_GpKsrjzFgvC}BEH_4kF8A+1w<($EHXAW z40A3k1)*sI^@TSn0;MwB2|~8iU!x(an^<&_Q+rsH+@kp3c|D0k3;%RDD}&!lFS^zEiL_5KO=MZAQ{q#K!t| zODYe0)wG73AQ-W(*jxBVcm7PRY@O$mx>Zd$WsG8)%#i6s3K;m7ie1rQE7>RH0@reAEx7c5C7F)dqokCJI@ts}&8usW!Q%z%%)3 zLWVz3oYYeA0y}_^r(Z(OIiB>44YC*g)&B_9PC_LKnXmQ6UU|0muv*`6U4P!O#^nh8 zp#CDm8AlUuWq^|^n>@B1`5lZv!vG?%yv8{AZ_YXEYzS{D0LZl_5>IuE`G#S z7j&LMA!QQ_1E<;+(lM0TGEf)IZ%OJTA?H^sc6^DD1n!FH8--~0L{S#~5HopC)EiRJ z$Lt0qVpOFT??dDREHi@o->$w$XdsTF`qF&GrYL!4F1N1_%(yZa@E1xqUaIz*Rs*dj zi5`#2Ek{BriLFagO!TJOn=fhYhBAI~X#H{B{m1_Ga_dmfr{l9+e;SPzNezz73P>RL zfx`KlrT9O>IU_k(Akl=~rdJAx;&DL}nk2*Cvcm4+;gSEl%V?@!CQG9rT{x&+8j(Wp}*S8YIcoRuQEjHfP7k9x>dnuQ9zA!s` z#NVjPVCTkAb(cm@)J8w)lML2llG>&t)TN|+`4#(V`G+V?LM&4N6eUVA&;K$_{P(ZF zbr5>9=mFoLGrnr+hL(;(%nTU{divh7FGrS~N6Lu7Uk(VN@%RTs8f2<}nn*%twoPPz p=@-bnvmEIEACBJt|Mk?b6dyB099Q*YDPW) Date: Fri, 3 Jul 2020 11:53:37 +0300 Subject: [PATCH 060/168] Resize volume by changing pvc size if enabled in config. (#958) * Try to resize pvc if resizing pv has failed * added config option to switch between storage resize strategies * changes according to requests * Update pkg/controller/operator_config.go Co-authored-by: Felix Kunde * enable_storage_resize documented added examples to the default configuration and helm value files * enable_storage_resize renamed to volume_resize_mode, off by default * volume_resize_mode renamed to storage_resize_mode * Update pkg/apis/acid.zalan.do/v1/crds.go * pkg/cluster/volumes.go updated * Update docs/reference/operator_parameters.md * Update manifests/postgresql-operator-default-configuration.yaml * Update pkg/controller/operator_config.go * Update pkg/util/config/config.go * Update charts/postgres-operator/values-crd.yaml * Update charts/postgres-operator/values.yaml * Update docs/reference/operator_parameters.md * added logging if no changes required Co-authored-by: Felix Kunde --- charts/postgres-operator/values-crd.yaml | 2 + charts/postgres-operator/values.yaml | 2 + docs/reference/operator_parameters.md | 6 +++ manifests/configmap.yaml | 1 + manifests/operatorconfiguration.crd.yaml | 6 +++ ...gresql-operator-default-configuration.yaml | 1 + pkg/apis/acid.zalan.do/v1/crds.go | 14 +++++ .../v1/operator_configuration_type.go | 1 + pkg/cluster/sync.go | 51 +++++++++++++++---- pkg/cluster/volumes.go | 50 +++++++++++++++++- pkg/controller/operator_config.go | 1 + pkg/util/config/config.go | 1 + 12 files changed, 125 insertions(+), 11 deletions(-) diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 14287fdaf..db26e6d98 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -124,6 +124,8 @@ configKubernetes: # whether the Spilo container should run in privileged mode spilo_privileged: false + # storage resize strategy, available options are: ebs, pvc, off + storage_resize_mode: ebs # operator watches for postgres objects in the given namespace watched_namespace: "*" # listen to all namespaces diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index cce0b79c8..eb5c10b0e 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -115,6 +115,8 @@ configKubernetes: # whether the Spilo container should run in privileged mode spilo_privileged: "false" + # storage resize strategy, available options are: ebs, pvc, off + storage_resize_mode: ebs # operator watches for postgres objects in the given namespace watched_namespace: "*" # listen to all namespaces diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index f8189f913..7e5196d56 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -333,6 +333,12 @@ configuration they are grouped under the `kubernetes` key. of stateful sets of PG clusters. The default is `ordered_ready`, the second possible value is `parallel`. +* **storage_resize_mode** + defines how operator handels the difference between requested volume size and + actual size. Available options are: ebs - tries to resize EBS volume, pvc - + changes PVC definition, off - disables resize of the volumes. Default is "ebs". + When using OpenShift please use one of the other available options. + ## Kubernetes resource requests This group allows you to configure resource requests for the Postgres pods. diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 963aea96b..d666b0383 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -97,6 +97,7 @@ data: # set_memory_request_to_limit: "false" # spilo_fsgroup: 103 spilo_privileged: "false" + # storage_resize_mode: "off" super_username: postgres # team_admin_role: "admin" # team_api_role_configuration: "log_statement:all" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 364ea6d5a..346eabb4a 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -168,6 +168,12 @@ spec: type: integer spilo_privileged: type: boolean + storage_resize_mode: + type: string + enum: + - "ebs" + - "pvc" + - "off" toleration: type: object additionalProperties: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index ab6a23113..cb7b1ed11 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -59,6 +59,7 @@ configuration: secret_name_template: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}" # spilo_fsgroup: 103 spilo_privileged: false + storage_resize_mode: ebs # toleration: {} # watched_namespace: "" postgres_pod_resources: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 43410ed3b..bc38d6dfd 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -980,6 +980,20 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation "spilo_privileged": { Type: "boolean", }, + "storage_resize_mode": { + Type: "string", + Enum: []apiextv1beta1.JSON{ + { + Raw: []byte(`"ebs"`), + }, + { + Raw: []byte(`"pvc"`), + }, + { + Raw: []byte(`"off"`), + }, + }, + }, "toleration": { Type: "object", AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 2dd0bbb50..5ac5a4677 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -53,6 +53,7 @@ type KubernetesMetaConfiguration struct { WatchedNamespace string `json:"watched_namespace,omitempty"` PDBNameFormat config.StringTemplate `json:"pdb_name_format,omitempty"` EnablePodDisruptionBudget *bool `json:"enable_pod_disruption_budget,omitempty"` + StorageResizeMode string `json:"storage_resize_mode,omitempty"` EnableInitContainers *bool `json:"enable_init_containers,omitempty"` EnableSidecars *bool `json:"enable_sidecars,omitempty"` SecretNameTemplate config.StringTemplate `json:"secret_name_template,omitempty"` diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index e49bd4537..b03b5d494 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -57,16 +57,26 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { return err } - // potentially enlarge volumes before changing the statefulset. By doing that - // in this order we make sure the operator is not stuck waiting for a pod that - // cannot start because it ran out of disk space. - // TODO: handle the case of the cluster that is downsized and enlarged again - // (there will be a volume from the old pod for which we can't act before the - // the statefulset modification is concluded) - c.logger.Debugf("syncing persistent volumes") - if err = c.syncVolumes(); err != nil { - err = fmt.Errorf("could not sync persistent volumes: %v", err) - return err + if c.OpConfig.StorageResizeMode == "pvc" { + c.logger.Debugf("syncing persistent volume claims") + if err = c.syncVolumeClaims(); err != nil { + err = fmt.Errorf("could not sync persistent volume claims: %v", err) + return err + } + } else if c.OpConfig.StorageResizeMode == "ebs" { + // potentially enlarge volumes before changing the statefulset. By doing that + // in this order we make sure the operator is not stuck waiting for a pod that + // cannot start because it ran out of disk space. + // TODO: handle the case of the cluster that is downsized and enlarged again + // (there will be a volume from the old pod for which we can't act before the + // the statefulset modification is concluded) + c.logger.Debugf("syncing persistent volumes") + if err = c.syncVolumes(); err != nil { + err = fmt.Errorf("could not sync persistent volumes: %v", err) + return err + } + } else { + c.logger.Infof("Storage resize is disabled (storage_resize_mode is off). Skipping volume sync.") } if err = c.enforceMinResourceLimits(&c.Spec); err != nil { @@ -571,6 +581,27 @@ func (c *Cluster) syncRoles() (err error) { return nil } +// syncVolumeClaims reads all persistent volume claims and checks that their size matches the one declared in the statefulset. +func (c *Cluster) syncVolumeClaims() error { + c.setProcessName("syncing volume claims") + + act, err := c.volumeClaimsNeedResizing(c.Spec.Volume) + if err != nil { + return fmt.Errorf("could not compare size of the volume claims: %v", err) + } + if !act { + c.logger.Infof("volume claims don't require changes") + return nil + } + if err := c.resizeVolumeClaims(c.Spec.Volume); err != nil { + return fmt.Errorf("could not sync volume claims: %v", err) + } + + c.logger.Infof("volume claims have been synced successfully") + + return nil +} + // syncVolumes reads all persistent volumes and checks that their size matches the one declared in the statefulset. func (c *Cluster) syncVolumes() error { c.setProcessName("syncing volumes") diff --git a/pkg/cluster/volumes.go b/pkg/cluster/volumes.go index a5bfe6c2d..d5c08c2e2 100644 --- a/pkg/cluster/volumes.go +++ b/pkg/cluster/volumes.go @@ -52,6 +52,35 @@ func (c *Cluster) deletePersistentVolumeClaims() error { return nil } +func (c *Cluster) resizeVolumeClaims(newVolume acidv1.Volume) error { + c.logger.Debugln("resizing PVCs") + pvcs, err := c.listPersistentVolumeClaims() + if err != nil { + return err + } + newQuantity, err := resource.ParseQuantity(newVolume.Size) + if err != nil { + return fmt.Errorf("could not parse volume size: %v", err) + } + _, newSize, err := c.listVolumesWithManifestSize(newVolume) + for _, pvc := range pvcs { + volumeSize := quantityToGigabyte(pvc.Spec.Resources.Requests[v1.ResourceStorage]) + if volumeSize >= newSize { + if volumeSize > newSize { + c.logger.Warningf("cannot shrink persistent volume") + } + continue + } + pvc.Spec.Resources.Requests[v1.ResourceStorage] = newQuantity + c.logger.Debugf("updating persistent volume claim definition for volume %q", pvc.Name) + if _, err := c.KubeClient.PersistentVolumeClaims(pvc.Namespace).Update(context.TODO(), &pvc, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("could not update persistent volume claim: %q", err) + } + c.logger.Debugf("successfully updated persistent volume claim %q", pvc.Name) + } + return nil +} + func (c *Cluster) listPersistentVolumes() ([]*v1.PersistentVolume, error) { result := make([]*v1.PersistentVolume, 0) @@ -150,7 +179,7 @@ func (c *Cluster) resizeVolumes(newVolume acidv1.Volume, resizers []volumes.Volu c.logger.Debugf("successfully updated persistent volume %q", pv.Name) } if !compatible { - c.logger.Warningf("volume %q is incompatible with all available resizing providers", pv.Name) + c.logger.Warningf("volume %q is incompatible with all available resizing providers, consider switching storage_resize_mode to pvc or off", pv.Name) totalIncompatible++ } } @@ -160,6 +189,25 @@ func (c *Cluster) resizeVolumes(newVolume acidv1.Volume, resizers []volumes.Volu return nil } +func (c *Cluster) volumeClaimsNeedResizing(newVolume acidv1.Volume) (bool, error) { + newSize, err := resource.ParseQuantity(newVolume.Size) + manifestSize := quantityToGigabyte(newSize) + if err != nil { + return false, fmt.Errorf("could not parse volume size from the manifest: %v", err) + } + pvcs, err := c.listPersistentVolumeClaims() + if err != nil { + return false, fmt.Errorf("could not receive persistent volume claims: %v", err) + } + for _, pvc := range pvcs { + currentSize := quantityToGigabyte(pvc.Spec.Resources.Requests[v1.ResourceStorage]) + if currentSize != manifestSize { + return true, nil + } + } + return false, nil +} + func (c *Cluster) volumesNeedResizing(newVolume acidv1.Volume) (bool, error) { vols, manifestSize, err := c.listVolumesWithManifestSize(newVolume) if err != nil { diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index bfb0e6dcc..a5a91dba7 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -65,6 +65,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.WatchedNamespace = fromCRD.Kubernetes.WatchedNamespace result.PDBNameFormat = fromCRD.Kubernetes.PDBNameFormat result.EnablePodDisruptionBudget = util.CoalesceBool(fromCRD.Kubernetes.EnablePodDisruptionBudget, util.True()) + result.StorageResizeMode = util.Coalesce(fromCRD.Kubernetes.StorageResizeMode, "ebs") result.EnableInitContainers = util.CoalesceBool(fromCRD.Kubernetes.EnableInitContainers, util.True()) result.EnableSidecars = util.CoalesceBool(fromCRD.Kubernetes.EnableSidecars, util.True()) result.SecretNameTemplate = fromCRD.Kubernetes.SecretNameTemplate diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 01057f236..bf1f5b70a 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -142,6 +142,7 @@ type Config struct { CustomPodAnnotations map[string]string `name:"custom_pod_annotations"` EnablePodAntiAffinity bool `name:"enable_pod_antiaffinity" default:"false"` PodAntiAffinityTopologyKey string `name:"pod_antiaffinity_topology_key" default:"kubernetes.io/hostname"` + StorageResizeMode string `name:"storage_resize_mode" default:"ebs"` // deprecated and kept for backward compatibility EnableLoadBalancer *bool `name:"enable_load_balancer"` MasterDNSNameFormat StringTemplate `name:"master_dns_name_format" default:"{cluster}.{team}.{hostedzone}"` From c10d30903e049bc75ce29e0a9342ff45434deeb5 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 8 Jul 2020 11:56:58 +0200 Subject: [PATCH 061/168] bump pgBouncer image (#1050) Co-authored-by: Felix Kunde --- charts/postgres-operator/values-crd.yaml | 2 +- charts/postgres-operator/values.yaml | 2 +- manifests/configmap.yaml | 2 +- manifests/postgresql-operator-default-configuration.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index db26e6d98..2652d02e1 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -273,7 +273,7 @@ configConnectionPooler: # db user for pooler to use connection_pooler_user: "pooler" # docker image - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-8" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-9" # max db connections the pooler should hold connection_pooler_max_db_connections: 60 # default pooling mode diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index eb5c10b0e..7e83a32fa 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -265,7 +265,7 @@ configConnectionPooler: # db user for pooler to use connection_pooler_user: "pooler" # docker image - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-8" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-9" # max db connections the pooler should hold connection_pooler_max_db_connections: "60" # default pooling mode diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index d666b0383..2af4c8f8b 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -15,7 +15,7 @@ data: # connection_pooler_default_cpu_request: "500m" # connection_pooler_default_memory_limit: 100Mi # connection_pooler_default_memory_request: 100Mi - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-8" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-9" # connection_pooler_max_db_connections: 60 # connection_pooler_mode: "transaction" # connection_pooler_number_of_instances: 2 diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index cb7b1ed11..2cd71fff3 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -129,7 +129,7 @@ configuration: connection_pooler_default_cpu_request: "500m" connection_pooler_default_memory_limit: 100Mi connection_pooler_default_memory_request: 100Mi - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-8" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-9" # connection_pooler_max_db_connections: 60 connection_pooler_mode: "transaction" connection_pooler_number_of_instances: 2 From b80f9767d1982bd863172f7cbf5bd2786f8b920e Mon Sep 17 00:00:00 2001 From: Igor Yanchenko <1504692+yanchenko-igor@users.noreply.github.com> Date: Fri, 10 Jul 2020 10:07:25 +0300 Subject: [PATCH 062/168] test coverage (#1055) --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 091c875ba..a52769c91 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,5 +18,6 @@ install: script: - hack/verify-codegen.sh - - travis_wait 20 goveralls -service=travis-ci -package ./pkg/... -v + - travis_wait 20 go test -race -covermode atomic -coverprofile=profile.cov ./pkg/... -v + - goveralls -coverprofile=profile.cov -service=travis-ci -v - make e2e From 375963424d1e8b78f6fed2cacfce6b2e27656b3b Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 10 Jul 2020 15:07:42 +0200 Subject: [PATCH 063/168] delete secrets the right way (#1054) * delete secrets the right way * make a one function * continue deleting secrets even if one delete fails Co-authored-by: Felix Kunde --- pkg/cluster/cluster.go | 6 ++---- pkg/cluster/resources.go | 27 ++++++++++++++++++--------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 44c3e9b62..ef728a728 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -797,10 +797,8 @@ func (c *Cluster) Delete() { c.logger.Warningf("could not delete statefulset: %v", err) } - for _, obj := range c.Secrets { - if err := c.deleteSecret(obj); err != nil { - c.logger.Warningf("could not delete secret: %v", err) - } + if err := c.deleteSecrets(); err != nil { + c.logger.Warningf("could not delete secrets: %v", err) } if err := c.deletePodDisruptionBudget(); err != nil { diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index 5c35058c2..c75457a5a 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -725,17 +725,26 @@ func (c *Cluster) deleteEndpoint(role PostgresRole) error { return nil } -func (c *Cluster) deleteSecret(secret *v1.Secret) error { - c.setProcessName("deleting secret %q", util.NameFromMeta(secret.ObjectMeta)) - c.logger.Debugf("deleting secret %q", util.NameFromMeta(secret.ObjectMeta)) - err := c.KubeClient.Secrets(secret.Namespace).Delete(context.TODO(), secret.Name, c.deleteOptions) - if err != nil { - return err +func (c *Cluster) deleteSecrets() error { + c.setProcessName("deleting secrets") + var errors []string + errorCount := 0 + for uid, secret := range c.Secrets { + c.logger.Debugf("deleting secret %q", util.NameFromMeta(secret.ObjectMeta)) + err := c.KubeClient.Secrets(secret.Namespace).Delete(context.TODO(), secret.Name, c.deleteOptions) + if err != nil { + errors = append(errors, fmt.Sprintf("could not delete secret %q: %v", util.NameFromMeta(secret.ObjectMeta), err)) + errorCount++ + } + c.logger.Infof("secret %q has been deleted", util.NameFromMeta(secret.ObjectMeta)) + c.Secrets[uid] = nil } - c.logger.Infof("secret %q has been deleted", util.NameFromMeta(secret.ObjectMeta)) - delete(c.Secrets, secret.UID) - return err + if errorCount > 0 { + return fmt.Errorf("could not delete all secrets: %v", errors) + } + + return nil } func (c *Cluster) createRoles() (err error) { From ec932f88d826861d8b7d2512a92e55ffdb29bb13 Mon Sep 17 00:00:00 2001 From: Toon Sevrin Date: Wed, 15 Jul 2020 13:53:10 +0200 Subject: [PATCH 064/168] Port-forward service instead of pod (#1040) --- docs/quickstart.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/quickstart.md b/docs/quickstart.md index d2c88b9a4..034d32e39 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -160,7 +160,7 @@ You can now access the web interface by port forwarding the UI pod (mind the label selector) and enter `localhost:8081` in your browser: ```bash -kubectl port-forward "$(kubectl get pod -l name=postgres-operator-ui --output='name')" 8081 +kubectl port-forward svc/postgres-operator-ui 8081:8081 ``` Available option are explained in detail in the [UI docs](operator-ui.md). From 002b47ec3248685080762db0c25ee314bf50c060 Mon Sep 17 00:00:00 2001 From: Igor Yanchenko <1504692+yanchenko-igor@users.noreply.github.com> Date: Thu, 16 Jul 2020 15:43:57 +0300 Subject: [PATCH 065/168] Use scram-sha-256 hash if postgresql parameter password_encryption set to do so. (#995) * Use scram-sha-256 hash if postgresql parameter password_encryption set to do so. * test fixed * Refactoring * code style --- go.mod | 1 + pkg/cluster/cluster.go | 6 +++- pkg/util/users/users.go | 11 ++++---- pkg/util/util.go | 61 ++++++++++++++++++++++++++++++++++++++--- pkg/util/util_test.go | 28 ++++++++++++++----- 5 files changed, 90 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index 041f90706..6e8cd8ef4 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/sirupsen/logrus v1.6.0 github.com/stretchr/testify v1.5.1 golang.org/x/mod v0.3.0 // indirect + golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 golang.org/x/tools v0.0.0-20200615222825-6aa8f57aacd9 // indirect gopkg.in/yaml.v2 v2.2.8 k8s.io/api v0.18.3 diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index ef728a728..a88cde53e 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -124,6 +124,10 @@ func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgres return fmt.Sprintf("%s-%s", e.PodName, e.ResourceVersion), nil }) + password_encryption, ok := pgSpec.Spec.PostgresqlParam.Parameters["password_encryption"] + if !ok { + password_encryption = "md5" + } cluster := &Cluster{ Config: cfg, @@ -135,7 +139,7 @@ func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgres Secrets: make(map[types.UID]*v1.Secret), Services: make(map[PostgresRole]*v1.Service), Endpoints: make(map[PostgresRole]*v1.Endpoints)}, - userSyncStrategy: users.DefaultUserSyncStrategy{}, + userSyncStrategy: users.DefaultUserSyncStrategy{password_encryption}, deleteOptions: metav1.DeleteOptions{PropagationPolicy: &deletePropagationPolicy}, podEventsQueue: podEventsQueue, KubeClient: kubeClient, diff --git a/pkg/util/users/users.go b/pkg/util/users/users.go index 345caa001..166e90264 100644 --- a/pkg/util/users/users.go +++ b/pkg/util/users/users.go @@ -28,6 +28,7 @@ const ( // an existing roles of another role membership, nor it removes the already assigned flag // (except for the NOLOGIN). TODO: process other NOflags, i.e. NOSUPERUSER correctly. type DefaultUserSyncStrategy struct { + PasswordEncryption string } // ProduceSyncRequests figures out the types of changes that need to happen with the given users. @@ -45,7 +46,7 @@ func (strategy DefaultUserSyncStrategy) ProduceSyncRequests(dbUsers spec.PgUserM } } else { r := spec.PgSyncUserRequest{} - newMD5Password := util.PGUserPassword(newUser) + newMD5Password := util.NewEncryptor(strategy.PasswordEncryption).PGUserPassword(newUser) if dbUser.Password != newMD5Password { r.User.Password = newMD5Password @@ -140,7 +141,7 @@ func (strategy DefaultUserSyncStrategy) createPgUser(user spec.PgUser, db *sql.D if user.Password == "" { userPassword = "PASSWORD NULL" } else { - userPassword = fmt.Sprintf(passwordTemplate, util.PGUserPassword(user)) + userPassword = fmt.Sprintf(passwordTemplate, util.NewEncryptor(strategy.PasswordEncryption).PGUserPassword(user)) } query := fmt.Sprintf(createUserSQL, user.Name, strings.Join(userFlags, " "), userPassword) @@ -155,7 +156,7 @@ func (strategy DefaultUserSyncStrategy) alterPgUser(user spec.PgUser, db *sql.DB var resultStmt []string if user.Password != "" || len(user.Flags) > 0 { - alterStmt := produceAlterStmt(user) + alterStmt := produceAlterStmt(user, strategy.PasswordEncryption) resultStmt = append(resultStmt, alterStmt) } if len(user.MemberOf) > 0 { @@ -174,14 +175,14 @@ func (strategy DefaultUserSyncStrategy) alterPgUser(user spec.PgUser, db *sql.DB return nil } -func produceAlterStmt(user spec.PgUser) string { +func produceAlterStmt(user spec.PgUser, encryption string) string { // ALTER ROLE ... LOGIN ENCRYPTED PASSWORD .. result := make([]string, 0) password := user.Password flags := user.Flags if password != "" { - result = append(result, fmt.Sprintf(passwordTemplate, util.PGUserPassword(user))) + result = append(result, fmt.Sprintf(passwordTemplate, util.NewEncryptor(encryption).PGUserPassword(user))) } if len(flags) != 0 { result = append(result, strings.Join(flags, " ")) diff --git a/pkg/util/util.go b/pkg/util/util.go index ff1be4e68..abb9be01f 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -1,8 +1,11 @@ package util import ( + "crypto/hmac" "crypto/md5" // #nosec we need it to for PostgreSQL md5 passwords cryptoRand "crypto/rand" + "crypto/sha256" + "encoding/base64" "encoding/hex" "fmt" "math/big" @@ -16,10 +19,14 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/zalando/postgres-operator/pkg/spec" + "golang.org/x/crypto/pbkdf2" ) const ( - md5prefix = "md5" + md5prefix = "md5" + scramsha256prefix = "SCRAM-SHA-256" + saltlength = 16 + iterations = 4096 ) var passwordChars = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") @@ -61,16 +68,62 @@ func NameFromMeta(meta metav1.ObjectMeta) spec.NamespacedName { } } -// PGUserPassword is used to generate md5 password hash for a given user. It does nothing for already hashed passwords. -func PGUserPassword(user spec.PgUser) string { - if (len(user.Password) == md5.Size*2+len(md5prefix) && user.Password[:3] == md5prefix) || user.Password == "" { +type Hasher func(user spec.PgUser) string +type Random func(n int) string + +type Encryptor struct { + encrypt Hasher + random Random +} + +func NewEncryptor(encryption string) *Encryptor { + e := Encryptor{random: RandomPassword} + m := map[string]Hasher{ + "md5": e.PGUserPasswordMD5, + "scram-sha-256": e.PGUserPasswordScramSHA256, + } + hasher, ok := m[encryption] + if !ok { + hasher = e.PGUserPasswordMD5 + } + e.encrypt = hasher + return &e +} + +func (e *Encryptor) PGUserPassword(user spec.PgUser) string { + if (len(user.Password) == md5.Size*2+len(md5prefix) && user.Password[:3] == md5prefix) || + (len(user.Password) > len(scramsha256prefix) && user.Password[:len(scramsha256prefix)] == scramsha256prefix) || user.Password == "" { // Avoid processing already encrypted or empty passwords return user.Password } + return e.encrypt(user) +} + +func (e *Encryptor) PGUserPasswordMD5(user spec.PgUser) string { s := md5.Sum([]byte(user.Password + user.Name)) // #nosec, using md5 since PostgreSQL uses it for hashing passwords. return md5prefix + hex.EncodeToString(s[:]) } +func (e *Encryptor) PGUserPasswordScramSHA256(user spec.PgUser) string { + salt := []byte(e.random(saltlength)) + key := pbkdf2.Key([]byte(user.Password), salt, iterations, 32, sha256.New) + mac := hmac.New(sha256.New, key) + mac.Write([]byte("Server Key")) + serverKey := mac.Sum(nil) + mac = hmac.New(sha256.New, key) + mac.Write([]byte("Client Key")) + clientKey := mac.Sum(nil) + storedKey := sha256.Sum256(clientKey) + pass := fmt.Sprintf("%s$%v:%s$%s:%s", + scramsha256prefix, + iterations, + base64.StdEncoding.EncodeToString(salt), + base64.StdEncoding.EncodeToString(storedKey[:]), + base64.StdEncoding.EncodeToString(serverKey), + ) + return pass +} + // Diff returns diffs between 2 objects func Diff(a, b interface{}) []string { return pretty.Diff(a, b) diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index 1f86ea1b4..a9d25112b 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -12,20 +12,27 @@ import ( ) var pgUsers = []struct { - in spec.PgUser - out string + in spec.PgUser + outmd5 string + outscramsha256 string }{{spec.PgUser{ Name: "test", Password: "password", Flags: []string{}, MemberOf: []string{}}, - "md587f77988ccb5aa917c93201ba314fcd4"}, + "md587f77988ccb5aa917c93201ba314fcd4", "SCRAM-SHA-256$4096:c2FsdA==$lF4cRm/Jky763CN4HtxdHnjV4Q8AWTNlKvGmEFFU8IQ=:ub8OgRsftnk2ccDMOt7ffHXNcikRkQkq1lh4xaAqrSw="}, {spec.PgUser{ Name: "test", Password: "md592f413f3974bdf3799bb6fecb5f9f2c6", Flags: []string{}, MemberOf: []string{}}, - "md592f413f3974bdf3799bb6fecb5f9f2c6"}} + "md592f413f3974bdf3799bb6fecb5f9f2c6", "md592f413f3974bdf3799bb6fecb5f9f2c6"}, + {spec.PgUser{ + Name: "test", + Password: "SCRAM-SHA-256$4096:S1ByZWhvYVV5VDlJNGZoVw==$ozLevu5k0pAQYRrSY+vZhetO6+/oB+qZvuutOdXR94U=:yADwhy0LGloXzh5RaVwLMFyUokwI17VkHVfKVuHu0Zs=", + Flags: []string{}, + MemberOf: []string{}}, + "SCRAM-SHA-256$4096:S1ByZWhvYVV5VDlJNGZoVw==$ozLevu5k0pAQYRrSY+vZhetO6+/oB+qZvuutOdXR94U=:yADwhy0LGloXzh5RaVwLMFyUokwI17VkHVfKVuHu0Zs=", "SCRAM-SHA-256$4096:S1ByZWhvYVV5VDlJNGZoVw==$ozLevu5k0pAQYRrSY+vZhetO6+/oB+qZvuutOdXR94U=:yADwhy0LGloXzh5RaVwLMFyUokwI17VkHVfKVuHu0Zs="}} var prettyDiffTest = []struct { inA interface{} @@ -107,9 +114,16 @@ func TestNameFromMeta(t *testing.T) { func TestPGUserPassword(t *testing.T) { for _, tt := range pgUsers { - pwd := PGUserPassword(tt.in) - if pwd != tt.out { - t.Errorf("PgUserPassword expected: %q, got: %q", tt.out, pwd) + e := NewEncryptor("md5") + pwd := e.PGUserPassword(tt.in) + if pwd != tt.outmd5 { + t.Errorf("PgUserPassword expected: %q, got: %q", tt.outmd5, pwd) + } + e = NewEncryptor("scram-sha-256") + e.random = func(n int) string { return "salt" } + pwd = e.PGUserPassword(tt.in) + if pwd != tt.outscramsha256 { + t.Errorf("PgUserPassword expected: %q, got: %q", tt.outscramsha256, pwd) } } } From 102a3536497cf9deae466dce7c02c3e6bb4569a3 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 29 Jul 2020 15:57:55 +0200 Subject: [PATCH 066/168] update dependencies (#1080) --- go.mod | 15 +++++++-------- go.sum | 42 +++++++++++++++++++++++------------------- 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/go.mod b/go.mod index 6e8cd8ef4..49ba3682b 100644 --- a/go.mod +++ b/go.mod @@ -9,13 +9,12 @@ require ( github.com/r3labs/diff v1.1.0 github.com/sirupsen/logrus v1.6.0 github.com/stretchr/testify v1.5.1 - golang.org/x/mod v0.3.0 // indirect - golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 - golang.org/x/tools v0.0.0-20200615222825-6aa8f57aacd9 // indirect + golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 + golang.org/x/tools v0.0.0-20200729041821-df70183b1872 // indirect gopkg.in/yaml.v2 v2.2.8 - k8s.io/api v0.18.3 - k8s.io/apiextensions-apiserver v0.18.3 - k8s.io/apimachinery v0.18.3 - k8s.io/client-go v0.18.3 - k8s.io/code-generator v0.18.3 + k8s.io/api v0.18.6 + k8s.io/apiextensions-apiserver v0.18.6 + k8s.io/apimachinery v0.18.6 + k8s.io/client-go v0.18.6 + k8s.io/code-generator v0.18.6 ) diff --git a/go.sum b/go.sum index b3d154b98..389608b82 100644 --- a/go.sum +++ b/go.sum @@ -291,7 +291,7 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -312,13 +312,13 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 h1:/Tl7pH94bvbAAHBdZJT947M/+gp0+CqQXDtMRC0fseo= golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -339,8 +339,8 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= @@ -351,6 +351,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -368,6 +369,8 @@ golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 h1:HmbHVPwrPEKPGLAcHSrMe6+hqSUlvZU0rab6x5EXfGU= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -392,8 +395,8 @@ golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200615222825-6aa8f57aacd9 h1:cwgUY+1ja2qxWb2dyaCoixaA66WGWmrijSlxaM+JM/g= -golang.org/x/tools v0.0.0-20200615222825-6aa8f57aacd9/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729041821-df70183b1872 h1:/U95VAvB4ZsR91rpZX2MwiKpejhWr+UxJ+N2VlJuESk= +golang.org/x/tools v0.0.0-20200729041821-df70183b1872/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= @@ -435,19 +438,20 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.18.3 h1:2AJaUQdgUZLoDZHrun21PW2Nx9+ll6cUzvn3IKhSIn0= -k8s.io/api v0.18.3/go.mod h1:UOaMwERbqJMfeeeHc8XJKawj4P9TgDRnViIqqBeH2QA= -k8s.io/apiextensions-apiserver v0.18.3 h1:h6oZO+iAgg0HjxmuNnguNdKNB9+wv3O1EBDdDWJViQ0= -k8s.io/apiextensions-apiserver v0.18.3/go.mod h1:TMsNGs7DYpMXd+8MOCX8KzPOCx8fnZMoIGB24m03+JE= -k8s.io/apimachinery v0.18.3 h1:pOGcbVAhxADgUYnjS08EFXs9QMl8qaH5U4fr5LGUrSk= -k8s.io/apimachinery v0.18.3/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= -k8s.io/apiserver v0.18.3/go.mod h1:tHQRmthRPLUtwqsOnJJMoI8SW3lnoReZeE861lH8vUw= -k8s.io/client-go v0.18.3 h1:QaJzz92tsN67oorwzmoB0a9r9ZVHuD5ryjbCKP0U22k= -k8s.io/client-go v0.18.3/go.mod h1:4a/dpQEvzAhT1BbuWW09qvIaGw6Gbu1gZYiQZIi1DMw= -k8s.io/code-generator v0.18.3 h1:5H57pYEbkMMXCLKD16YQH3yDPAbVLweUsB1M3m70D1c= -k8s.io/code-generator v0.18.3/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c= -k8s.io/component-base v0.18.3/go.mod h1:bp5GzGR0aGkYEfTj+eTY0AN/vXTgkJdQXjNTTVUaa3k= +k8s.io/api v0.18.6 h1:osqrAXbOQjkKIWDTjrqxWQ3w0GkKb1KA1XkUGHHYpeE= +k8s.io/api v0.18.6/go.mod h1:eeyxr+cwCjMdLAmr2W3RyDI0VvTawSg/3RFFBEnmZGI= +k8s.io/apiextensions-apiserver v0.18.6 h1:vDlk7cyFsDyfwn2rNAO2DbmUbvXy5yT5GE3rrqOzaMo= +k8s.io/apiextensions-apiserver v0.18.6/go.mod h1:lv89S7fUysXjLZO7ke783xOwVTm6lKizADfvUM/SS/M= +k8s.io/apimachinery v0.18.6 h1:RtFHnfGNfd1N0LeSrKCUznz5xtUP1elRGvHJbL3Ntag= +k8s.io/apimachinery v0.18.6/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= +k8s.io/apiserver v0.18.6/go.mod h1:Zt2XvTHuaZjBz6EFYzpp+X4hTmgWGy8AthNVnTdm3Wg= +k8s.io/client-go v0.18.6 h1:I+oWqJbibLSGsZj8Xs8F0aWVXJVIoUHWaaJV3kUN/Zw= +k8s.io/client-go v0.18.6/go.mod h1:/fwtGLjYMS1MaM5oi+eXhKwG+1UHidUEXRh6cNsdO0Q= +k8s.io/code-generator v0.18.6 h1:QdfvGfs4gUCS1dru+rLbCKIFxYEV0IRfF8MXwY/ozLk= +k8s.io/code-generator v0.18.6/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c= +k8s.io/component-base v0.18.6/go.mod h1:knSVsibPR5K6EW2XOjEHik6sdU5nCvKMrzMt2D4In14= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200114144118-36b2048a9120 h1:RPscN6KhmG54S33L+lr3GS+oD1jmchIU0ll519K6FA4= k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= From ece341d5160d30cb8e70ce521a615b74c8d7c4d9 Mon Sep 17 00:00:00 2001 From: Christian Rohmann Date: Thu, 30 Jul 2020 10:48:16 +0200 Subject: [PATCH 067/168] Allow pod environment variables to also be sourced from a secret (#946) * Extend operator configuration to allow for a pod_environment_secret just like pod_environment_configmap * Add all keys from PodEnvironmentSecrets as ENV vars (using SecretKeyRef to protect the value) * Apply envVars from pod_environment_configmap and pod_environment_secrets before doing the global settings from the operator config. This allows them to be overriden by the user (via configmap / secret) * Add ability use a Secret for custom pod envVars (via pod_environment_secret) to admin documentation * Add pod_environment_secret to Helm chart values.yaml * Add unit tests for PodEnvironmentConfigMap and PodEnvironmentSecret - highly inspired by @kupson and his very similar PR #481 * Added new parameter pod_environment_secret to operatorconfig CRD and configmap examples * Add pod_environment_secret to the operationconfiguration CRD Co-authored-by: Christian Rohmann --- .../crds/operatorconfigurations.yaml | 2 + charts/postgres-operator/values-crd.yaml | 2 + charts/postgres-operator/values.yaml | 2 + docs/administrator.md | 64 +++++- manifests/configmap.yaml | 1 + manifests/operatorconfiguration.crd.yaml | 2 + ...gresql-operator-default-configuration.yaml | 1 + pkg/apis/acid.zalan.do/v1/crds.go | 3 + .../v1/operator_configuration_type.go | 1 + pkg/cluster/k8sres.go | 161 +++++++++----- pkg/cluster/k8sres_test.go | 208 ++++++++++++++++++ pkg/controller/operator_config.go | 1 + pkg/util/config/config.go | 1 + 13 files changed, 394 insertions(+), 55 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index ffcef7b4a..89f495367 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -149,6 +149,8 @@ spec: type: string pod_environment_configmap: type: string + pod_environment_secret: + type: string pod_management_policy: type: string enum: diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 2652d02e1..44a7f315b 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -104,6 +104,8 @@ configKubernetes: pod_antiaffinity_topology_key: "kubernetes.io/hostname" # namespaced name of the ConfigMap with environment variables to populate on every pod # pod_environment_configmap: "default/my-custom-config" + # name of the Secret (in cluster namespace) with environment variables to populate on every pod + # pod_environment_secret: "my-custom-secret" # specify the pod management policy of stateful sets of Postgres clusters pod_management_policy: "ordered_ready" diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 7e83a32fa..b64495bee 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -95,6 +95,8 @@ configKubernetes: pod_antiaffinity_topology_key: "kubernetes.io/hostname" # namespaced name of the ConfigMap with environment variables to populate on every pod # pod_environment_configmap: "default/my-custom-config" + # name of the Secret (in cluster namespace) with environment variables to populate on every pod + # pod_environment_secret: "my-custom-secret" # specify the pod management policy of stateful sets of Postgres clusters pod_management_policy: "ordered_ready" diff --git a/docs/administrator.md b/docs/administrator.md index e2c2e01eb..b3d4d9efa 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -319,11 +319,18 @@ spec: ## Custom Pod Environment Variables - -It is possible to configure a ConfigMap which is used by the Postgres pods as +It is possible to configure a ConfigMap as well as a Secret which are used by the Postgres pods as an additional provider for environment variables. One use case is to customize -the Spilo image and configure it with environment variables. The ConfigMap with -the additional settings is referenced in the operator's main configuration. +the Spilo image and configure it with environment variables. Another case could be to provide custom +cloud provider or backup settings. + +In general the Operator will give preference to the globally configured variables, to not have the custom +ones interfere with core functionality. Variables with the 'WAL_' and 'LOG_' prefix can be overwritten though, to allow +backup and logshipping to be specified differently. + + +### Via ConfigMap +The ConfigMap with the additional settings is referenced in the operator's main configuration. A namespace can be specified along with the name. If left out, the configured default namespace of your K8s client will be used and if the ConfigMap is not found there, the Postgres cluster's namespace is taken when different: @@ -365,7 +372,54 @@ data: MY_CUSTOM_VAR: value ``` -This ConfigMap is then added as a source of environment variables to the +The key-value pairs of the ConfigMap are then added as environment variables to the +Postgres StatefulSet/pods. + + +### Via Secret +The Secret with the additional variables is referenced in the operator's main configuration. +To protect the values of the secret from being exposed in the pod spec they are each referenced +as SecretKeyRef. +This does not allow for the secret to be in a different namespace as the pods though + +**postgres-operator ConfigMap** + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-operator +data: + # referencing secret with custom environment variables + pod_environment_secret: postgres-pod-secrets +``` + +**OperatorConfiguration** + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: OperatorConfiguration +metadata: + name: postgresql-operator-configuration +configuration: + kubernetes: + # referencing secret with custom environment variables + pod_environment_secret: postgres-pod-secrets +``` + +**referenced Secret `postgres-pod-secrets`** + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: postgres-pod-secrets + namespace: default +data: + MY_CUSTOM_VAR: dmFsdWU= +``` + +The key-value pairs of the Secret are all accessible as environment variables to the Postgres StatefulSet/pods. ## Limiting the number of min and max instances in clusters diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 2af4c8f8b..d1c1b3d17 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -74,6 +74,7 @@ data: # pod_antiaffinity_topology_key: "kubernetes.io/hostname" pod_deletion_wait_timeout: 10m # pod_environment_configmap: "default/my-custom-config" + # pod_environment_secret: "my-custom-secret" pod_label_wait_timeout: 10m pod_management_policy: "ordered_ready" pod_role_label: spilo-role diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 346eabb4a..2b6e8ae67 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -145,6 +145,8 @@ spec: type: string pod_environment_configmap: type: string + pod_environment_secret: + type: string pod_management_policy: type: string enum: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 2cd71fff3..f7eba1f6c 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -49,6 +49,7 @@ configuration: pdb_name_format: "postgres-{cluster}-pdb" pod_antiaffinity_topology_key: "kubernetes.io/hostname" # pod_environment_configmap: "default/my-custom-config" + # pod_environment_secret: "my-custom-secret" pod_management_policy: "ordered_ready" # pod_priority_class_name: "" pod_role_label: spilo-role diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index bc38d6dfd..6f907e266 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -942,6 +942,9 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation "pod_environment_configmap": { Type: "string", }, + "pod_environment_secret": { + Type: "string", + }, "pod_management_policy": { Type: "string", Enum: []apiextv1beta1.JSON{ diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 5ac5a4677..e6e13cbd3 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -70,6 +70,7 @@ type KubernetesMetaConfiguration struct { // TODO: use a proper toleration structure? PodToleration map[string]string `json:"toleration,omitempty"` PodEnvironmentConfigMap spec.NamespacedName `json:"pod_environment_configmap,omitempty"` + PodEnvironmentSecret string `json:"pod_environment_secret,omitempty"` PodPriorityClassName string `json:"pod_priority_class_name,omitempty"` MasterPodMoveTimeout Duration `json:"master_pod_move_timeout,omitempty"` EnablePodAntiAffinity bool `json:"enable_pod_antiaffinity,omitempty"` diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index ef20da062..21875f953 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -7,6 +7,7 @@ import ( "path" "sort" "strconv" + "strings" "github.com/sirupsen/logrus" @@ -20,7 +21,6 @@ import ( acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/spec" - pkgspec "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/constants" @@ -715,6 +715,30 @@ func (c *Cluster) generateSpiloPodEnvVars(uid types.UID, spiloConfiguration stri envVars = append(envVars, v1.EnvVar{Name: "SPILO_CONFIGURATION", Value: spiloConfiguration}) } + if c.patroniUsesKubernetes() { + envVars = append(envVars, v1.EnvVar{Name: "DCS_ENABLE_KUBERNETES_API", Value: "true"}) + } else { + envVars = append(envVars, v1.EnvVar{Name: "ETCD_HOST", Value: c.OpConfig.EtcdHost}) + } + + if c.patroniKubernetesUseConfigMaps() { + envVars = append(envVars, v1.EnvVar{Name: "KUBERNETES_USE_CONFIGMAPS", Value: "true"}) + } + + if cloneDescription.ClusterName != "" { + envVars = append(envVars, c.generateCloneEnvironment(cloneDescription)...) + } + + if c.Spec.StandbyCluster != nil { + envVars = append(envVars, c.generateStandbyEnvironment(standbyDescription)...) + } + + // add vars taken from pod_environment_configmap and pod_environment_secret first + // (to allow them to override the globals set in the operator config) + if len(customPodEnvVarsList) > 0 { + envVars = append(envVars, customPodEnvVarsList...) + } + if c.OpConfig.WALES3Bucket != "" { envVars = append(envVars, v1.EnvVar{Name: "WAL_S3_BUCKET", Value: c.OpConfig.WALES3Bucket}) envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))}) @@ -737,28 +761,6 @@ func (c *Cluster) generateSpiloPodEnvVars(uid types.UID, spiloConfiguration stri envVars = append(envVars, v1.EnvVar{Name: "LOG_BUCKET_SCOPE_PREFIX", Value: ""}) } - if c.patroniUsesKubernetes() { - envVars = append(envVars, v1.EnvVar{Name: "DCS_ENABLE_KUBERNETES_API", Value: "true"}) - } else { - envVars = append(envVars, v1.EnvVar{Name: "ETCD_HOST", Value: c.OpConfig.EtcdHost}) - } - - if c.patroniKubernetesUseConfigMaps() { - envVars = append(envVars, v1.EnvVar{Name: "KUBERNETES_USE_CONFIGMAPS", Value: "true"}) - } - - if cloneDescription.ClusterName != "" { - envVars = append(envVars, c.generateCloneEnvironment(cloneDescription)...) - } - - if c.Spec.StandbyCluster != nil { - envVars = append(envVars, c.generateStandbyEnvironment(standbyDescription)...) - } - - if len(customPodEnvVarsList) > 0 { - envVars = append(envVars, customPodEnvVarsList...) - } - return envVars } @@ -777,13 +779,81 @@ func deduplicateEnvVars(input []v1.EnvVar, containerName string, logger *logrus. result = append(result, input[i]) } else if names[va.Name] == 1 { names[va.Name]++ - logger.Warningf("variable %q is defined in %q more than once, the subsequent definitions are ignored", - va.Name, containerName) + + // Some variables (those to configure the WAL_ and LOG_ shipping) may be overriden, only log as info + if strings.HasPrefix(va.Name, "WAL_") || strings.HasPrefix(va.Name, "LOG_") { + logger.Infof("global variable %q has been overwritten by configmap/secret for container %q", + va.Name, containerName) + } else { + logger.Warningf("variable %q is defined in %q more than once, the subsequent definitions are ignored", + va.Name, containerName) + } } } return result } +// Return list of variables the pod recieved from the configured ConfigMap +func (c *Cluster) getPodEnvironmentConfigMapVariables() ([]v1.EnvVar, error) { + configMapPodEnvVarsList := make([]v1.EnvVar, 0) + + if c.OpConfig.PodEnvironmentConfigMap.Name == "" { + return configMapPodEnvVarsList, nil + } + + cm, err := c.KubeClient.ConfigMaps(c.OpConfig.PodEnvironmentConfigMap.Namespace).Get( + context.TODO(), + c.OpConfig.PodEnvironmentConfigMap.Name, + metav1.GetOptions{}) + if err != nil { + // if not found, try again using the cluster's namespace if it's different (old behavior) + if k8sutil.ResourceNotFound(err) && c.Namespace != c.OpConfig.PodEnvironmentConfigMap.Namespace { + cm, err = c.KubeClient.ConfigMaps(c.Namespace).Get( + context.TODO(), + c.OpConfig.PodEnvironmentConfigMap.Name, + metav1.GetOptions{}) + } + if err != nil { + return nil, fmt.Errorf("could not read PodEnvironmentConfigMap: %v", err) + } + } + for k, v := range cm.Data { + configMapPodEnvVarsList = append(configMapPodEnvVarsList, v1.EnvVar{Name: k, Value: v}) + } + return configMapPodEnvVarsList, nil +} + +// Return list of variables the pod recieved from the configured Secret +func (c *Cluster) getPodEnvironmentSecretVariables() ([]v1.EnvVar, error) { + secretPodEnvVarsList := make([]v1.EnvVar, 0) + + if c.OpConfig.PodEnvironmentSecret == "" { + return secretPodEnvVarsList, nil + } + + secret, err := c.KubeClient.Secrets(c.OpConfig.PodEnvironmentSecret).Get( + context.TODO(), + c.OpConfig.PodEnvironmentSecret, + metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("could not read Secret PodEnvironmentSecretName: %v", err) + } + + for k := range secret.Data { + secretPodEnvVarsList = append(secretPodEnvVarsList, + v1.EnvVar{Name: k, ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: c.OpConfig.PodEnvironmentSecret, + }, + Key: k, + }, + }}) + } + + return secretPodEnvVarsList, nil +} + func getSidecarContainer(sidecar acidv1.Sidecar, index int, resources *v1.ResourceRequirements) *v1.Container { name := sidecar.Name if name == "" { @@ -943,32 +1013,23 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef initContainers = spec.InitContainers } - customPodEnvVarsList := make([]v1.EnvVar, 0) - - if c.OpConfig.PodEnvironmentConfigMap != (pkgspec.NamespacedName{}) { - var cm *v1.ConfigMap - cm, err = c.KubeClient.ConfigMaps(c.OpConfig.PodEnvironmentConfigMap.Namespace).Get( - context.TODO(), - c.OpConfig.PodEnvironmentConfigMap.Name, - metav1.GetOptions{}) - if err != nil { - // if not found, try again using the cluster's namespace if it's different (old behavior) - if k8sutil.ResourceNotFound(err) && c.Namespace != c.OpConfig.PodEnvironmentConfigMap.Namespace { - cm, err = c.KubeClient.ConfigMaps(c.Namespace).Get( - context.TODO(), - c.OpConfig.PodEnvironmentConfigMap.Name, - metav1.GetOptions{}) - } - if err != nil { - return nil, fmt.Errorf("could not read PodEnvironmentConfigMap: %v", err) - } - } - for k, v := range cm.Data { - customPodEnvVarsList = append(customPodEnvVarsList, v1.EnvVar{Name: k, Value: v}) - } - sort.Slice(customPodEnvVarsList, - func(i, j int) bool { return customPodEnvVarsList[i].Name < customPodEnvVarsList[j].Name }) + // fetch env vars from custom ConfigMap + configMapEnvVarsList, err := c.getPodEnvironmentConfigMapVariables() + if err != nil { + return nil, err } + + // fetch env vars from custom ConfigMap + secretEnvVarsList, err := c.getPodEnvironmentSecretVariables() + if err != nil { + return nil, err + } + + // concat all custom pod env vars and sort them + customPodEnvVarsList := append(configMapEnvVarsList, secretEnvVarsList...) + sort.Slice(customPodEnvVarsList, + func(i, j int) bool { return customPodEnvVarsList[i].Name < customPodEnvVarsList[j].Name }) + if spec.StandbyCluster != nil && spec.StandbyCluster.S3WalPath == "" { return nil, fmt.Errorf("s3_wal_path is empty for standby cluster") } diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index ff830a1f5..f324a9bd3 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -1,6 +1,7 @@ package cluster import ( + "context" "errors" "fmt" "reflect" @@ -10,6 +11,7 @@ import ( "github.com/stretchr/testify/assert" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/constants" @@ -22,6 +24,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + v1core "k8s.io/client-go/kubernetes/typed/core/v1" ) // For testing purposes @@ -713,6 +716,211 @@ func TestSecretVolume(t *testing.T) { } } +const ( + testPodEnvironmentConfigMapName = "pod_env_cm" + testPodEnvironmentSecretName = "pod_env_sc" +) + +type mockSecret struct { + v1core.SecretInterface +} + +type mockConfigMap struct { + v1core.ConfigMapInterface +} + +func (c *mockSecret) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.Secret, error) { + if name != testPodEnvironmentSecretName { + return nil, fmt.Errorf("Secret PodEnvironmentSecret not found") + } + secret := &v1.Secret{} + secret.Name = testPodEnvironmentSecretName + secret.Data = map[string][]byte{ + "minio_access_key": []byte("alpha"), + "minio_secret_key": []byte("beta"), + } + return secret, nil +} + +func (c *mockConfigMap) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.ConfigMap, error) { + if name != testPodEnvironmentConfigMapName { + return nil, fmt.Errorf("NotFound") + } + configmap := &v1.ConfigMap{} + configmap.Name = testPodEnvironmentConfigMapName + configmap.Data = map[string]string{ + "foo": "bar", + } + return configmap, nil +} + +type MockSecretGetter struct { +} + +type MockConfigMapsGetter struct { +} + +func (c *MockSecretGetter) Secrets(namespace string) v1core.SecretInterface { + return &mockSecret{} +} + +func (c *MockConfigMapsGetter) ConfigMaps(namespace string) v1core.ConfigMapInterface { + return &mockConfigMap{} +} + +func newMockKubernetesClient() k8sutil.KubernetesClient { + return k8sutil.KubernetesClient{ + SecretsGetter: &MockSecretGetter{}, + ConfigMapsGetter: &MockConfigMapsGetter{}, + } +} +func newMockCluster(opConfig config.Config) *Cluster { + cluster := &Cluster{ + Config: Config{OpConfig: opConfig}, + KubeClient: newMockKubernetesClient(), + } + return cluster +} + +func TestPodEnvironmentConfigMapVariables(t *testing.T) { + testName := "TestPodEnvironmentConfigMapVariables" + tests := []struct { + subTest string + opConfig config.Config + envVars []v1.EnvVar + err error + }{ + { + subTest: "no PodEnvironmentConfigMap", + envVars: []v1.EnvVar{}, + }, + { + subTest: "missing PodEnvironmentConfigMap", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentConfigMap: spec.NamespacedName{ + Name: "idonotexist", + }, + }, + }, + err: fmt.Errorf("could not read PodEnvironmentConfigMap: NotFound"), + }, + { + subTest: "simple PodEnvironmentConfigMap", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentConfigMap: spec.NamespacedName{ + Name: testPodEnvironmentConfigMapName, + }, + }, + }, + envVars: []v1.EnvVar{ + { + Name: "foo", + Value: "bar", + }, + }, + }, + } + for _, tt := range tests { + c := newMockCluster(tt.opConfig) + vars, err := c.getPodEnvironmentConfigMapVariables() + if !reflect.DeepEqual(vars, tt.envVars) { + t.Errorf("%s %s: expected `%v` but got `%v`", + testName, tt.subTest, tt.envVars, vars) + } + if tt.err != nil { + if err.Error() != tt.err.Error() { + t.Errorf("%s %s: expected error `%v` but got `%v`", + testName, tt.subTest, tt.err, err) + } + } else { + if err != nil { + t.Errorf("%s %s: expected no error but got error: `%v`", + testName, tt.subTest, err) + } + } + } +} + +// Test if the keys of an existing secret are properly referenced +func TestPodEnvironmentSecretVariables(t *testing.T) { + testName := "TestPodEnvironmentSecretVariables" + tests := []struct { + subTest string + opConfig config.Config + envVars []v1.EnvVar + err error + }{ + { + subTest: "No PodEnvironmentSecret configured", + envVars: []v1.EnvVar{}, + }, + { + subTest: "Secret referenced by PodEnvironmentSecret does not exist", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentSecret: "idonotexist", + }, + }, + err: fmt.Errorf("could not read Secret PodEnvironmentSecretName: Secret PodEnvironmentSecret not found"), + }, + { + subTest: "Pod environment vars reference all keys from secret configured by PodEnvironmentSecret", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentSecret: testPodEnvironmentSecretName, + }, + }, + envVars: []v1.EnvVar{ + { + Name: "minio_access_key", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: testPodEnvironmentSecretName, + }, + Key: "minio_access_key", + }, + }, + }, + { + Name: "minio_secret_key", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: testPodEnvironmentSecretName, + }, + Key: "minio_secret_key", + }, + }, + }, + }, + }, + } + + for _, tt := range tests { + c := newMockCluster(tt.opConfig) + vars, err := c.getPodEnvironmentSecretVariables() + if !reflect.DeepEqual(vars, tt.envVars) { + t.Errorf("%s %s: expected `%v` but got `%v`", + testName, tt.subTest, tt.envVars, vars) + } + if tt.err != nil { + if err.Error() != tt.err.Error() { + t.Errorf("%s %s: expected error `%v` but got `%v`", + testName, tt.subTest, tt.err, err) + } + } else { + if err != nil { + t.Errorf("%s %s: expected no error but got error: `%v`", + testName, tt.subTest, err) + } + } + } + +} + func testResources(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { cpuReq := podSpec.Spec.Containers[0].Resources.Requests["cpu"] if cpuReq.String() != cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultCPURequest { diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index a5a91dba7..e2d8636a1 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -58,6 +58,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.PodServiceAccountDefinition = fromCRD.Kubernetes.PodServiceAccountDefinition result.PodServiceAccountRoleBindingDefinition = fromCRD.Kubernetes.PodServiceAccountRoleBindingDefinition result.PodEnvironmentConfigMap = fromCRD.Kubernetes.PodEnvironmentConfigMap + result.PodEnvironmentSecret = fromCRD.Kubernetes.PodEnvironmentSecret result.PodTerminateGracePeriod = util.CoalesceDuration(time.Duration(fromCRD.Kubernetes.PodTerminateGracePeriod), "5m") result.SpiloPrivileged = fromCRD.Kubernetes.SpiloPrivileged result.SpiloFSGroup = fromCRD.Kubernetes.SpiloFSGroup diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index bf1f5b70a..6cab8af45 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -45,6 +45,7 @@ type Resources struct { MinCPULimit string `name:"min_cpu_limit" default:"250m"` MinMemoryLimit string `name:"min_memory_limit" default:"250Mi"` PodEnvironmentConfigMap spec.NamespacedName `name:"pod_environment_configmap"` + PodEnvironmentSecret string `name:"pod_environment_secret"` NodeReadinessLabel map[string]string `name:"node_readiness_label" default:""` MaxInstances int32 `name:"max_instances" default:"-1"` MinInstances int32 `name:"min_instances" default:"-1"` From aab9b0aff9ac43d7559f51213ff504fc24835e66 Mon Sep 17 00:00:00 2001 From: Allison Richardet Date: Thu, 30 Jul 2020 04:08:33 -0500 Subject: [PATCH 068/168] chart ui: fix target namespace to allow '*' (#1082) --- charts/postgres-operator-ui/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/postgres-operator-ui/templates/deployment.yaml b/charts/postgres-operator-ui/templates/deployment.yaml index 00610c799..4c6d46689 100644 --- a/charts/postgres-operator-ui/templates/deployment.yaml +++ b/charts/postgres-operator-ui/templates/deployment.yaml @@ -46,7 +46,7 @@ spec: - name: "RESOURCES_VISIBLE" value: "{{ .Values.envs.resourcesVisible }}" - name: "TARGET_NAMESPACE" - value: {{ .Values.envs.targetNamespace }} + value: "{{ .Values.envs.targetNamespace }}" - name: "TEAMS" value: |- [ From 3bee590d439d6f697166441093e3ef2f1ba2ddcd Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 30 Jul 2020 13:35:37 +0200 Subject: [PATCH 069/168] fix index in TestGenerateSpiloPodEnvVarswq (#1084) Co-authored-by: Felix Kunde --- pkg/cluster/k8sres_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index f324a9bd3..7261d5902 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -119,17 +119,17 @@ func TestGenerateSpiloPodEnvVars(t *testing.T) { expectedValuesGSBucket := []ExpectedValue{ ExpectedValue{ - envIndex: 14, + envIndex: 15, envVarConstant: "WAL_GS_BUCKET", envVarValue: "wale-gs-bucket", }, ExpectedValue{ - envIndex: 15, + envIndex: 16, envVarConstant: "WAL_BUCKET_SCOPE_SUFFIX", envVarValue: "/SomeUUID", }, ExpectedValue{ - envIndex: 16, + envIndex: 17, envVarConstant: "WAL_BUCKET_SCOPE_PREFIX", envVarValue: "", }, @@ -137,22 +137,22 @@ func TestGenerateSpiloPodEnvVars(t *testing.T) { expectedValuesGCPCreds := []ExpectedValue{ ExpectedValue{ - envIndex: 14, + envIndex: 15, envVarConstant: "WAL_GS_BUCKET", envVarValue: "wale-gs-bucket", }, ExpectedValue{ - envIndex: 15, + envIndex: 16, envVarConstant: "WAL_BUCKET_SCOPE_SUFFIX", envVarValue: "/SomeUUID", }, ExpectedValue{ - envIndex: 16, + envIndex: 17, envVarConstant: "WAL_BUCKET_SCOPE_PREFIX", envVarValue: "", }, ExpectedValue{ - envIndex: 17, + envIndex: 18, envVarConstant: "GOOGLE_APPLICATION_CREDENTIALS", envVarValue: "some_path_to_credentials", }, From 47b11f7f8993a3f4bd39d7aa63aacdd3f66774f3 Mon Sep 17 00:00:00 2001 From: hlihhovac Date: Thu, 30 Jul 2020 16:31:29 +0200 Subject: [PATCH 070/168] change Clone attribute of PostgresSpec to *CloneDescription (#1020) * change Clone attribute of PostgresSpec to *ConnectionPooler * update go.mod from master * fix TestConnectionPoolerSynchronization() * Update pkg/apis/acid.zalan.do/v1/postgresql_type.go Co-authored-by: Felix Kunde Co-authored-by: Pavlo Golub Co-authored-by: Felix Kunde --- .gitignore | 1 + pkg/apis/acid.zalan.do/v1/marshal.go | 5 +- pkg/apis/acid.zalan.do/v1/postgresql_type.go | 2 +- pkg/apis/acid.zalan.do/v1/util.go | 2 +- pkg/apis/acid.zalan.do/v1/util_test.go | 12 ++-- .../acid.zalan.do/v1/zz_generated.deepcopy.go | 6 +- pkg/cluster/k8sres.go | 4 +- pkg/cluster/sync_test.go | 59 ++++++++++--------- 8 files changed, 50 insertions(+), 41 deletions(-) diff --git a/.gitignore b/.gitignore index 0fdb50756..559c92499 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ _testmain.go /docker/build/ /github.com/ .idea +.vscode scm-source.json diff --git a/pkg/apis/acid.zalan.do/v1/marshal.go b/pkg/apis/acid.zalan.do/v1/marshal.go index 336b0da41..9521082fc 100644 --- a/pkg/apis/acid.zalan.do/v1/marshal.go +++ b/pkg/apis/acid.zalan.do/v1/marshal.go @@ -112,8 +112,9 @@ func (p *Postgresql) UnmarshalJSON(data []byte) error { if clusterName, err := extractClusterName(tmp2.ObjectMeta.Name, tmp2.Spec.TeamID); err != nil { tmp2.Error = err.Error() - tmp2.Status.PostgresClusterStatus = ClusterStatusInvalid - } else if err := validateCloneClusterDescription(&tmp2.Spec.Clone); err != nil { + tmp2.Status = PostgresStatus{PostgresClusterStatus: ClusterStatusInvalid} + } else if err := validateCloneClusterDescription(tmp2.Spec.Clone); err != nil { + tmp2.Error = err.Error() tmp2.Status.PostgresClusterStatus = ClusterStatusInvalid } else { diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 5df82e947..24ef24d63 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -53,7 +53,7 @@ type PostgresSpec struct { NumberOfInstances int32 `json:"numberOfInstances"` Users map[string]UserFlags `json:"users"` MaintenanceWindows []MaintenanceWindow `json:"maintenanceWindows,omitempty"` - Clone CloneDescription `json:"clone"` + Clone *CloneDescription `json:"clone"` ClusterName string `json:"-"` Databases map[string]string `json:"databases,omitempty"` PreparedDatabases map[string]PreparedDatabase `json:"preparedDatabases,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/util.go b/pkg/apis/acid.zalan.do/v1/util.go index db6efcd71..a795ec685 100644 --- a/pkg/apis/acid.zalan.do/v1/util.go +++ b/pkg/apis/acid.zalan.do/v1/util.go @@ -72,7 +72,7 @@ func extractClusterName(clusterName string, teamName string) (string, error) { func validateCloneClusterDescription(clone *CloneDescription) error { // when cloning from the basebackup (no end timestamp) check that the cluster name is a valid service name - if clone.ClusterName != "" && clone.EndTimestamp == "" { + if clone != nil && clone.ClusterName != "" && clone.EndTimestamp == "" { if !serviceNameRegex.MatchString(clone.ClusterName) { return fmt.Errorf("clone cluster name must confirm to DNS-1035, regex used for validation is %q", serviceNameRegexString) diff --git a/pkg/apis/acid.zalan.do/v1/util_test.go b/pkg/apis/acid.zalan.do/v1/util_test.go index 28e9e8ca4..bf6875a82 100644 --- a/pkg/apis/acid.zalan.do/v1/util_test.go +++ b/pkg/apis/acid.zalan.do/v1/util_test.go @@ -163,7 +163,7 @@ var unmarshalCluster = []struct { "kind": "Postgresql","apiVersion": "acid.zalan.do/v1", "metadata": {"name": "acid-testcluster1"}, "spec": {"teamId": 100}}`), &tmp).Error(), }, - marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":{}},"status":"Invalid"}`), + marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":null},"status":"Invalid"}`), err: nil}, { about: "example with /status subresource", @@ -184,7 +184,7 @@ var unmarshalCluster = []struct { "kind": "Postgresql","apiVersion": "acid.zalan.do/v1", "metadata": {"name": "acid-testcluster1"}, "spec": {"teamId": 100}}`), &tmp).Error(), }, - marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":{}},"status":{"PostgresClusterStatus":"Invalid"}}`), + marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":null},"status":{"PostgresClusterStatus":"Invalid"}}`), err: nil}, { about: "example with detailed input manifest and deprecated pod_priority_class_name -> podPriorityClassName", @@ -327,7 +327,7 @@ var unmarshalCluster = []struct { EndTime: mustParseTime("05:15"), }, }, - Clone: CloneDescription{ + Clone: &CloneDescription{ ClusterName: "acid-batman", }, ClusterName: "testcluster1", @@ -351,7 +351,7 @@ var unmarshalCluster = []struct { Status: PostgresStatus{PostgresClusterStatus: ClusterStatusInvalid}, Error: errors.New("name must match {TEAM}-{NAME} format").Error(), }, - marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"teapot-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null} ,"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"acid","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":{}},"status":{"PostgresClusterStatus":"Invalid"}}`), + marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"teapot-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null} ,"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"acid","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":null},"status":{"PostgresClusterStatus":"Invalid"}}`), err: nil}, { about: "example with clone", @@ -366,7 +366,7 @@ var unmarshalCluster = []struct { }, Spec: PostgresSpec{ TeamID: "acid", - Clone: CloneDescription{ + Clone: &CloneDescription{ ClusterName: "team-batman", }, ClusterName: "testcluster1", @@ -405,7 +405,7 @@ var unmarshalCluster = []struct { err: errors.New("unexpected end of JSON input")}, { about: "expect error on JSON with field's value malformatted", - in: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster","creationTimestamp":qaz},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"acid","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":{}},"status":{"PostgresClusterStatus":"Invalid"}}`), + in: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster","creationTimestamp":qaz},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"acid","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":null},"status":{"PostgresClusterStatus":"Invalid"}}`), out: Postgresql{}, marshal: []byte{}, err: errors.New("invalid character 'q' looking for beginning of value"), diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 5879c9b73..064ced184 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -567,7 +567,11 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - in.Clone.DeepCopyInto(&out.Clone) + if in.Clone != nil { + in, out := &in.Clone, &out.Clone + *out = new(CloneDescription) + (*in).DeepCopyInto(*out) + } if in.Databases != nil { in, out := &in.Databases, &out.Databases *out = make(map[string]string, len(*in)) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 21875f953..0f9a1a5bc 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -725,7 +725,7 @@ func (c *Cluster) generateSpiloPodEnvVars(uid types.UID, spiloConfiguration stri envVars = append(envVars, v1.EnvVar{Name: "KUBERNETES_USE_CONFIGMAPS", Value: "true"}) } - if cloneDescription.ClusterName != "" { + if cloneDescription != nil && cloneDescription.ClusterName != "" { envVars = append(envVars, c.generateCloneEnvironment(cloneDescription)...) } @@ -1065,7 +1065,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef spiloEnvVars := c.generateSpiloPodEnvVars( c.Postgresql.GetUID(), spiloConfiguration, - &spec.Clone, + spec.Clone, spec.StandbyCluster, customPodEnvVarsList, ) diff --git a/pkg/cluster/sync_test.go b/pkg/cluster/sync_test.go index 3a7317938..d9248ae33 100644 --- a/pkg/cluster/sync_test.go +++ b/pkg/cluster/sync_test.go @@ -63,23 +63,26 @@ func noEmptySync(cluster *Cluster, err error, reason SyncReason) error { func TestConnectionPoolerSynchronization(t *testing.T) { testName := "Test connection pooler synchronization" - var cluster = New( - Config{ - OpConfig: config.Config{ - ProtectedRoles: []string{"admin"}, - Auth: config.Auth{ - SuperUsername: superUserName, - ReplicationUsername: replicationUserName, + newCluster := func() *Cluster { + return New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", + NumberOfInstances: int32ToPointer(1), + }, }, - ConnectionPooler: config.ConnectionPooler{ - ConnectionPoolerDefaultCPURequest: "100m", - ConnectionPoolerDefaultCPULimit: "100m", - ConnectionPoolerDefaultMemoryRequest: "100Mi", - ConnectionPoolerDefaultMemoryLimit: "100Mi", - NumberOfInstances: int32ToPointer(1), - }, - }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) + } + cluster := newCluster() cluster.Statefulset = &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ @@ -87,20 +90,20 @@ func TestConnectionPoolerSynchronization(t *testing.T) { }, } - clusterMissingObjects := *cluster + clusterMissingObjects := newCluster() clusterMissingObjects.KubeClient = k8sutil.ClientMissingObjects() - clusterMock := *cluster + clusterMock := newCluster() clusterMock.KubeClient = k8sutil.NewMockKubernetesClient() - clusterDirtyMock := *cluster + clusterDirtyMock := newCluster() clusterDirtyMock.KubeClient = k8sutil.NewMockKubernetesClient() clusterDirtyMock.ConnectionPooler = &ConnectionPoolerObjects{ Deployment: &appsv1.Deployment{}, Service: &v1.Service{}, } - clusterNewDefaultsMock := *cluster + clusterNewDefaultsMock := newCluster() clusterNewDefaultsMock.KubeClient = k8sutil.NewMockKubernetesClient() tests := []struct { @@ -124,7 +127,7 @@ func TestConnectionPoolerSynchronization(t *testing.T) { ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, - cluster: &clusterMissingObjects, + cluster: clusterMissingObjects, defaultImage: "pooler:1.0", defaultInstances: 1, check: objectsAreSaved, @@ -139,7 +142,7 @@ func TestConnectionPoolerSynchronization(t *testing.T) { EnableConnectionPooler: boolToPointer(true), }, }, - cluster: &clusterMissingObjects, + cluster: clusterMissingObjects, defaultImage: "pooler:1.0", defaultInstances: 1, check: objectsAreSaved, @@ -154,7 +157,7 @@ func TestConnectionPoolerSynchronization(t *testing.T) { ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, - cluster: &clusterMissingObjects, + cluster: clusterMissingObjects, defaultImage: "pooler:1.0", defaultInstances: 1, check: objectsAreSaved, @@ -169,7 +172,7 @@ func TestConnectionPoolerSynchronization(t *testing.T) { newSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{}, }, - cluster: &clusterMock, + cluster: clusterMock, defaultImage: "pooler:1.0", defaultInstances: 1, check: objectsAreDeleted, @@ -182,7 +185,7 @@ func TestConnectionPoolerSynchronization(t *testing.T) { newSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{}, }, - cluster: &clusterDirtyMock, + cluster: clusterDirtyMock, defaultImage: "pooler:1.0", defaultInstances: 1, check: objectsAreDeleted, @@ -203,7 +206,7 @@ func TestConnectionPoolerSynchronization(t *testing.T) { }, }, }, - cluster: &clusterMock, + cluster: clusterMock, defaultImage: "pooler:1.0", defaultInstances: 1, check: deploymentUpdated, @@ -220,7 +223,7 @@ func TestConnectionPoolerSynchronization(t *testing.T) { ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, - cluster: &clusterNewDefaultsMock, + cluster: clusterNewDefaultsMock, defaultImage: "pooler:2.0", defaultInstances: 2, check: deploymentUpdated, @@ -239,7 +242,7 @@ func TestConnectionPoolerSynchronization(t *testing.T) { ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, - cluster: &clusterMock, + cluster: clusterMock, defaultImage: "pooler:1.0", defaultInstances: 1, check: noEmptySync, From f3ddce81d50d5d816c1976d8b0f635b045e2de0f Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 30 Jul 2020 17:48:15 +0200 Subject: [PATCH 071/168] fix random order for pod environment tests (#1085) --- pkg/cluster/k8sres.go | 2 +- pkg/cluster/k8sres_test.go | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 0f9a1a5bc..d7878942c 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -780,7 +780,7 @@ func deduplicateEnvVars(input []v1.EnvVar, containerName string, logger *logrus. } else if names[va.Name] == 1 { names[va.Name]++ - // Some variables (those to configure the WAL_ and LOG_ shipping) may be overriden, only log as info + // Some variables (those to configure the WAL_ and LOG_ shipping) may be overwritten, only log as info if strings.HasPrefix(va.Name, "WAL_") || strings.HasPrefix(va.Name, "LOG_") { logger.Infof("global variable %q has been overwritten by configmap/secret for container %q", va.Name, containerName) diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 7261d5902..1e474fbf5 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "reflect" + "sort" "testing" @@ -749,7 +750,8 @@ func (c *mockConfigMap) Get(ctx context.Context, name string, options metav1.Get configmap := &v1.ConfigMap{} configmap.Name = testPodEnvironmentConfigMapName configmap.Data = map[string]string{ - "foo": "bar", + "foo1": "bar1", + "foo2": "bar2", } return configmap, nil } @@ -816,8 +818,12 @@ func TestPodEnvironmentConfigMapVariables(t *testing.T) { }, envVars: []v1.EnvVar{ { - Name: "foo", - Value: "bar", + Name: "foo1", + Value: "bar1", + }, + { + Name: "foo2", + Value: "bar2", }, }, }, @@ -825,6 +831,7 @@ func TestPodEnvironmentConfigMapVariables(t *testing.T) { for _, tt := range tests { c := newMockCluster(tt.opConfig) vars, err := c.getPodEnvironmentConfigMapVariables() + sort.Slice(vars, func(i, j int) bool { return vars[i].Name < vars[j].Name }) if !reflect.DeepEqual(vars, tt.envVars) { t.Errorf("%s %s: expected `%v` but got `%v`", testName, tt.subTest, tt.envVars, vars) @@ -902,6 +909,7 @@ func TestPodEnvironmentSecretVariables(t *testing.T) { for _, tt := range tests { c := newMockCluster(tt.opConfig) vars, err := c.getPodEnvironmentSecretVariables() + sort.Slice(vars, func(i, j int) bool { return vars[i].Name < vars[j].Name }) if !reflect.DeepEqual(vars, tt.envVars) { t.Errorf("%s %s: expected `%v` but got `%v`", testName, tt.subTest, tt.envVars, vars) From 7cf2fae6df5dfde03bd9ac87e9ab4493bab4aeef Mon Sep 17 00:00:00 2001 From: Dmitry Dolgov <9erthalion6@gmail.com> Date: Wed, 5 Aug 2020 14:18:56 +0200 Subject: [PATCH 072/168] [WIP] Extend infrastructure roles handling (#1064) Extend infrastructure roles handling Postgres Operator uses infrastructure roles to provide access to a database for external users e.g. for monitoring purposes. Such infrastructure roles are expected to be present in the form of k8s secrets with the following content: inrole1: some_encrypted_role password1: some_encrypted_password user1: some_entrypted_name inrole2: some_encrypted_role password2: some_encrypted_password user2: some_entrypted_name The format of this content is implied implicitly and not flexible enough. In case if we do not have possibility to change the format of a secret we want to use in the Operator, we need to recreate it in this format. To address this lets make the format of secret content explicitly. The idea is to introduce a new configuration option for the Operator. infrastructure_roles_secrets: - secretname: k8s_secret_name userkey: some_encrypted_name passwordkey: some_encrypted_password rolekey: some_encrypted_role - secretname: k8s_secret_name userkey: some_encrypted_name passwordkey: some_encrypted_password rolekey: some_encrypted_role This would allow Operator to use any avalable secrets to prepare infrastructure roles. To make it backward compatible simulate the old behaviour if the new option is not present. The new configuration option is intended be used mainly from CRD, but it's also available via Operator ConfigMap in a limited fashion. For ConfigMap one can put there only a string with one secret definition in the following format (as a string): infrastructure_roles_secrets: | secretname: k8s_secret_name, userkey: some_encrypted_name, passwordkey: some_encrypted_password, rolekey: some_encrypted_role Note than only one secret could be specified this way, no multiple secrets are allowed. Eventually the resulting list of infrastructure roles would be a total sum of all supported ways to describe it, namely legacy via infrastructure_roles_secret_name and infrastructure_roles_secrets from both ConfigMap and CRD. --- .../crds/operatorconfigurations.yaml | 22 + docs/reference/operator_parameters.md | 10 +- docs/user.md | 61 +- e2e/Dockerfile | 3 + e2e/exec.sh | 2 + e2e/tests/test_e2e.py | 1038 +++++++++-------- manifests/configmap.yaml | 3 +- manifests/infrastructure-roles-new.yaml | 14 + manifests/operatorconfiguration.crd.yaml | 22 + ...gresql-operator-default-configuration.yaml | 8 + pkg/apis/acid.zalan.do/v1/crds.go | 31 +- .../v1/operator_configuration_type.go | 45 +- .../acid.zalan.do/v1/zz_generated.deepcopy.go | 12 + pkg/controller/controller.go | 3 +- pkg/controller/operator_config.go | 15 + pkg/controller/util.go | 292 ++++- pkg/controller/util_test.go | 299 ++++- pkg/spec/types.go | 4 + pkg/util/config/config.go | 42 +- pkg/util/k8sutil/k8sutil.go | 70 +- 20 files changed, 1375 insertions(+), 621 deletions(-) create mode 100755 e2e/exec.sh create mode 100644 manifests/infrastructure-roles-new.yaml diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 89f495367..3218decd7 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -131,6 +131,28 @@ spec: type: boolean infrastructure_roles_secret_name: type: string + infrastructure_roles_secrets: + type: array + nullable: true + items: + type: object + required: + - secretname + - userkey + - passwordkey + properties: + secretname: + type: string + userkey: + type: string + passwordkey: + type: string + rolekey: + type: string + details: + type: string + template: + type: boolean inherited_labels: type: array items: diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 7e5196d56..20771078f 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -252,8 +252,14 @@ configuration they are grouped under the `kubernetes` key. teams API. The default is `postgresql-operator`. * **infrastructure_roles_secret_name** - namespaced name of the secret containing infrastructure roles names and - passwords. + *deprecated*: namespaced name of the secret containing infrastructure roles + with user names, passwords and role membership. + +* **infrastructure_roles_secrets** + array of infrastructure role definitions which reference existing secrets + and specify the key names from which user name, password and role membership + are extracted. For the ConfigMap this has to be a string which allows + referencing only one infrastructure roles secret. The default is empty. * **pod_role_label** name of the label assigned to the Postgres pods (and services/endpoints) by diff --git a/docs/user.md b/docs/user.md index 3683fdf61..a4b1424b8 100644 --- a/docs/user.md +++ b/docs/user.md @@ -150,23 +150,62 @@ user. There are two ways to define them: #### Infrastructure roles secret -The infrastructure roles secret is specified by the `infrastructure_roles_secret_name` -parameter. The role definition looks like this (values are base64 encoded): +Infrastructure roles can be specified by the `infrastructure_roles_secrets` +parameter where you can reference multiple existing secrets. Prior to `v1.6.0` +the operator could only reference one secret with the +`infrastructure_roles_secret_name` option. However, this secret could contain +multiple roles using the same set of keys plus incrementing index. ```yaml -user1: ZGJ1c2Vy -password1: c2VjcmV0 -inrole1: b3BlcmF0b3I= +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-infrastructure-roles +data: + user1: ZGJ1c2Vy + password1: c2VjcmV0 + inrole1: b3BlcmF0b3I= + user2: ... ``` The block above describes the infrastructure role 'dbuser' with password -'secret' that is a member of the 'operator' role. For the following definitions -one must increase the index, i.e. the next role will be defined as 'user2' and -so on. The resulting role will automatically be a login role. +'secret' that is a member of the 'operator' role. The resulting role will +automatically be a login role. -Note that with definitions that solely use the infrastructure roles secret -there is no way to specify role options (like superuser or nologin) or role -memberships. This is where the ConfigMap comes into play. +With the new option users can configure the names of secret keys that contain +the user name, password etc. The secret itself is referenced by the +`secretname` key. If the secret uses a template for multiple roles as described +above list them separately. + +```yaml +apiVersion: v1 +kind: OperatorConfiguration +metadata: + name: postgresql-operator-configuration +configuration: + kubernetes: + infrastructure_roles_secrets: + - secretname: "postgresql-infrastructure-roles" + userkey: "user1" + passwordkey: "password1" + rolekey: "inrole1" + - secretname: "postgresql-infrastructure-roles" + userkey: "user2" + ... +``` + +Note, only the CRD-based configuration allows for referencing multiple secrets. +As of now, the ConfigMap is restricted to either one or the existing template +option with `infrastructure_roles_secret_name`. Please, refer to the example +manifests to understand how `infrastructure_roles_secrets` has to be configured +for the [configmap](../manifests/configmap.yaml) or [CRD configuration](../manifests/postgresql-operator-default-configuration.yaml). + +If both `infrastructure_roles_secret_name` and `infrastructure_roles_secrets` +are defined the operator will create roles for both of them. So make sure, +they do not collide. Note also, that with definitions that solely use the +infrastructure roles secret there is no way to specify role options (like +superuser or nologin) or role memberships. This is where the additional +ConfigMap comes into play. #### Secret plus ConfigMap diff --git a/e2e/Dockerfile b/e2e/Dockerfile index 236942d04..a250ea9cb 100644 --- a/e2e/Dockerfile +++ b/e2e/Dockerfile @@ -1,7 +1,10 @@ +# An image to perform the actual test. Do not forget to copy all necessary test +# files here. FROM ubuntu:18.04 LABEL maintainer="Team ACID @ Zalando " COPY manifests ./manifests +COPY exec.sh ./exec.sh COPY requirements.txt tests ./ RUN apt-get update \ diff --git a/e2e/exec.sh b/e2e/exec.sh new file mode 100755 index 000000000..56276bc3c --- /dev/null +++ b/e2e/exec.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +kubectl exec -it $1 -- sh -c "$2" diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 18b9852c4..4cd1c6a30 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -1,3 +1,4 @@ +import json import unittest import time import timeout_decorator @@ -50,7 +51,8 @@ class EndToEndTestCase(unittest.TestCase): for filename in ["operator-service-account-rbac.yaml", "configmap.yaml", - "postgres-operator.yaml"]: + "postgres-operator.yaml", + "infrastructure-roles-new.yaml"]: result = k8s.create_with_kubectl("manifests/" + filename) print("stdout: {}, stderr: {}".format(result.stdout, result.stderr)) @@ -69,507 +71,548 @@ class EndToEndTestCase(unittest.TestCase): print('Operator log: {}'.format(k8s.get_operator_log())) raise + # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + # def test_enable_disable_connection_pooler(self): + # ''' + # For a database without connection pooler, then turns it on, scale up, + # turn off and on again. Test with different ways of doing this (via + # enableConnectionPooler or connectionPooler configuration section). At + # the end turn connection pooler off to not interfere with other tests. + # ''' + # k8s = self.k8s + # service_labels = { + # 'cluster-name': 'acid-minimal-cluster', + # } + # pod_labels = dict({ + # 'connection-pooler': 'acid-minimal-cluster-pooler', + # }) + + # pod_selector = to_selector(pod_labels) + # service_selector = to_selector(service_labels) + + # try: + # # enable connection pooler + # k8s.api.custom_objects_api.patch_namespaced_custom_object( + # 'acid.zalan.do', 'v1', 'default', + # 'postgresqls', 'acid-minimal-cluster', + # { + # 'spec': { + # 'enableConnectionPooler': True, + # } + # }) + # k8s.wait_for_pod_start(pod_selector) + + # pods = k8s.api.core_v1.list_namespaced_pod( + # 'default', label_selector=pod_selector + # ).items + + # self.assertTrue(pods, 'No connection pooler pods') + + # k8s.wait_for_service(service_selector) + # services = k8s.api.core_v1.list_namespaced_service( + # 'default', label_selector=service_selector + # ).items + # services = [ + # s for s in services + # if s.metadata.name.endswith('pooler') + # ] + + # self.assertTrue(services, 'No connection pooler service') + + # # scale up connection pooler deployment + # k8s.api.custom_objects_api.patch_namespaced_custom_object( + # 'acid.zalan.do', 'v1', 'default', + # 'postgresqls', 'acid-minimal-cluster', + # { + # 'spec': { + # 'connectionPooler': { + # 'numberOfInstances': 2, + # }, + # } + # }) + + # k8s.wait_for_running_pods(pod_selector, 2) + + # # turn it off, keeping configuration section + # k8s.api.custom_objects_api.patch_namespaced_custom_object( + # 'acid.zalan.do', 'v1', 'default', + # 'postgresqls', 'acid-minimal-cluster', + # { + # 'spec': { + # 'enableConnectionPooler': False, + # } + # }) + # k8s.wait_for_pods_to_stop(pod_selector) + + # except timeout_decorator.TimeoutError: + # print('Operator log: {}'.format(k8s.get_operator_log())) + # raise + + # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + # def test_enable_load_balancer(self): + # ''' + # Test if services are updated when enabling/disabling load balancers + # ''' + + # k8s = self.k8s + # cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + + # # enable load balancer services + # pg_patch_enable_lbs = { + # "spec": { + # "enableMasterLoadBalancer": True, + # "enableReplicaLoadBalancer": True + # } + # } + # k8s.api.custom_objects_api.patch_namespaced_custom_object( + # "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_lbs) + # # wait for service recreation + # time.sleep(60) + + # master_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=master') + # self.assertEqual(master_svc_type, 'LoadBalancer', + # "Expected LoadBalancer service type for master, found {}".format(master_svc_type)) + + # repl_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=replica') + # self.assertEqual(repl_svc_type, 'LoadBalancer', + # "Expected LoadBalancer service type for replica, found {}".format(repl_svc_type)) + + # # disable load balancer services again + # pg_patch_disable_lbs = { + # "spec": { + # "enableMasterLoadBalancer": False, + # "enableReplicaLoadBalancer": False + # } + # } + # k8s.api.custom_objects_api.patch_namespaced_custom_object( + # "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_lbs) + # # wait for service recreation + # time.sleep(60) + + # master_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=master') + # self.assertEqual(master_svc_type, 'ClusterIP', + # "Expected ClusterIP service type for master, found {}".format(master_svc_type)) + + # repl_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=replica') + # self.assertEqual(repl_svc_type, 'ClusterIP', + # "Expected ClusterIP service type for replica, found {}".format(repl_svc_type)) + + # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + # def test_lazy_spilo_upgrade(self): + # ''' + # Test lazy upgrade for the Spilo image: operator changes a stateful set but lets pods run with the old image + # until they are recreated for reasons other than operator's activity. That works because the operator configures + # stateful sets to use "onDelete" pod update policy. + + # The test covers: + # 1) enabling lazy upgrade in existing operator deployment + # 2) forcing the normal rolling upgrade by changing the operator configmap and restarting its pod + # ''' + + # k8s = self.k8s + + # # update docker image in config and enable the lazy upgrade + # conf_image = "registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p114" + # patch_lazy_spilo_upgrade = { + # "data": { + # "docker_image": conf_image, + # "enable_lazy_spilo_upgrade": "true" + # } + # } + # k8s.update_config(patch_lazy_spilo_upgrade) + + # pod0 = 'acid-minimal-cluster-0' + # pod1 = 'acid-minimal-cluster-1' + + # # restart the pod to get a container with the new image + # k8s.api.core_v1.delete_namespaced_pod(pod0, 'default') + # time.sleep(60) + + # # lazy update works if the restarted pod and older pods run different Spilo versions + # new_image = k8s.get_effective_pod_image(pod0) + # old_image = k8s.get_effective_pod_image(pod1) + # self.assertNotEqual(new_image, old_image, "Lazy updated failed: pods have the same image {}".format(new_image)) + + # # sanity check + # assert_msg = "Image {} of a new pod differs from {} in operator conf".format(new_image, conf_image) + # self.assertEqual(new_image, conf_image, assert_msg) + + # # clean up + # unpatch_lazy_spilo_upgrade = { + # "data": { + # "enable_lazy_spilo_upgrade": "false", + # } + # } + # k8s.update_config(unpatch_lazy_spilo_upgrade) + + # # at this point operator will complete the normal rolling upgrade + # # so we additonally test if disabling the lazy upgrade - forcing the normal rolling upgrade - works + + # # XXX there is no easy way to wait until the end of Sync() + # time.sleep(60) + + # image0 = k8s.get_effective_pod_image(pod0) + # image1 = k8s.get_effective_pod_image(pod1) + + # assert_msg = "Disabling lazy upgrade failed: pods still have different images {} and {}".format(image0, image1) + # self.assertEqual(image0, image1, assert_msg) + + # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + # def test_logical_backup_cron_job(self): + # ''' + # Ensure we can (a) create the cron job at user request for a specific PG cluster + # (b) update the cluster-wide image for the logical backup pod + # (c) delete the job at user request + + # Limitations: + # (a) Does not run the actual batch job because there is no S3 mock to upload backups to + # (b) Assumes 'acid-minimal-cluster' exists as defined in setUp + # ''' + + # k8s = self.k8s + + # # create the cron job + # schedule = "7 7 7 7 *" + # pg_patch_enable_backup = { + # "spec": { + # "enableLogicalBackup": True, + # "logicalBackupSchedule": schedule + # } + # } + # k8s.api.custom_objects_api.patch_namespaced_custom_object( + # "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_backup) + # k8s.wait_for_logical_backup_job_creation() + + # jobs = k8s.get_logical_backup_job().items + # self.assertEqual(1, len(jobs), "Expected 1 logical backup job, found {}".format(len(jobs))) + + # job = jobs[0] + # self.assertEqual(job.metadata.name, "logical-backup-acid-minimal-cluster", + # "Expected job name {}, found {}" + # .format("logical-backup-acid-minimal-cluster", job.metadata.name)) + # self.assertEqual(job.spec.schedule, schedule, + # "Expected {} schedule, found {}" + # .format(schedule, job.spec.schedule)) + + # # update the cluster-wide image of the logical backup pod + # image = "test-image-name" + # patch_logical_backup_image = { + # "data": { + # "logical_backup_docker_image": image, + # } + # } + # k8s.update_config(patch_logical_backup_image) + + # jobs = k8s.get_logical_backup_job().items + # actual_image = jobs[0].spec.job_template.spec.template.spec.containers[0].image + # self.assertEqual(actual_image, image, + # "Expected job image {}, found {}".format(image, actual_image)) + + # # delete the logical backup cron job + # pg_patch_disable_backup = { + # "spec": { + # "enableLogicalBackup": False, + # } + # } + # k8s.api.custom_objects_api.patch_namespaced_custom_object( + # "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_backup) + # k8s.wait_for_logical_backup_job_deletion() + # jobs = k8s.get_logical_backup_job().items + # self.assertEqual(0, len(jobs), + # "Expected 0 logical backup jobs, found {}".format(len(jobs))) + + # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + # def test_min_resource_limits(self): + # ''' + # Lower resource limits below configured minimum and let operator fix it + # ''' + # k8s = self.k8s + # cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + # labels = 'spilo-role=master,' + cluster_label + # _, failover_targets = k8s.get_pg_nodes(cluster_label) + + # # configure minimum boundaries for CPU and memory limits + # minCPULimit = '500m' + # minMemoryLimit = '500Mi' + # patch_min_resource_limits = { + # "data": { + # "min_cpu_limit": minCPULimit, + # "min_memory_limit": minMemoryLimit + # } + # } + # k8s.update_config(patch_min_resource_limits) + + # # lower resource limits below minimum + # pg_patch_resources = { + # "spec": { + # "resources": { + # "requests": { + # "cpu": "10m", + # "memory": "50Mi" + # }, + # "limits": { + # "cpu": "200m", + # "memory": "200Mi" + # } + # } + # } + # } + # k8s.api.custom_objects_api.patch_namespaced_custom_object( + # "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_resources) + # k8s.wait_for_pod_failover(failover_targets, labels) + # k8s.wait_for_pod_start('spilo-role=replica') + + # pods = k8s.api.core_v1.list_namespaced_pod( + # 'default', label_selector=labels).items + # self.assert_master_is_unique() + # masterPod = pods[0] + + # self.assertEqual(masterPod.spec.containers[0].resources.limits['cpu'], minCPULimit, + # "Expected CPU limit {}, found {}" + # .format(minCPULimit, masterPod.spec.containers[0].resources.limits['cpu'])) + # self.assertEqual(masterPod.spec.containers[0].resources.limits['memory'], minMemoryLimit, + # "Expected memory limit {}, found {}" + # .format(minMemoryLimit, masterPod.spec.containers[0].resources.limits['memory'])) + + # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + # def test_multi_namespace_support(self): + # ''' + # Create a customized Postgres cluster in a non-default namespace. + # ''' + # k8s = self.k8s + + # with open("manifests/complete-postgres-manifest.yaml", 'r+') as f: + # pg_manifest = yaml.safe_load(f) + # pg_manifest["metadata"]["namespace"] = self.namespace + # yaml.dump(pg_manifest, f, Dumper=yaml.Dumper) + + # k8s.create_with_kubectl("manifests/complete-postgres-manifest.yaml") + # k8s.wait_for_pod_start("spilo-role=master", self.namespace) + # self.assert_master_is_unique(self.namespace, "acid-test-cluster") + + # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + # def test_node_readiness_label(self): + # ''' + # Remove node readiness label from master node. This must cause a failover. + # ''' + # k8s = self.k8s + # cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + # readiness_label = 'lifecycle-status' + # readiness_value = 'ready' + + # # get nodes of master and replica(s) (expected target of new master) + # current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label) + # num_replicas = len(current_replica_nodes) + # failover_targets = self.get_failover_targets(current_master_node, current_replica_nodes) + + # # add node_readiness_label to potential failover nodes + # patch_readiness_label = { + # "metadata": { + # "labels": { + # readiness_label: readiness_value + # } + # } + # } + # for failover_target in failover_targets: + # k8s.api.core_v1.patch_node(failover_target, patch_readiness_label) + + # # define node_readiness_label in config map which should trigger a failover of the master + # patch_readiness_label_config = { + # "data": { + # "node_readiness_label": readiness_label + ':' + readiness_value, + # } + # } + # k8s.update_config(patch_readiness_label_config) + # new_master_node, new_replica_nodes = self.assert_failover( + # current_master_node, num_replicas, failover_targets, cluster_label) + + # # patch also node where master ran before + # k8s.api.core_v1.patch_node(current_master_node, patch_readiness_label) + + # # wait a little before proceeding with the pod distribution test + # time.sleep(30) + + # # toggle pod anti affinity to move replica away from master node + # self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) + + # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + # def test_scaling(self): + # ''' + # Scale up from 2 to 3 and back to 2 pods by updating the Postgres manifest at runtime. + # ''' + # k8s = self.k8s + # labels = "application=spilo,cluster-name=acid-minimal-cluster" + + # k8s.wait_for_pg_to_scale(3) + # self.assertEqual(3, k8s.count_pods_with_label(labels)) + # self.assert_master_is_unique() + + # k8s.wait_for_pg_to_scale(2) + # self.assertEqual(2, k8s.count_pods_with_label(labels)) + # self.assert_master_is_unique() + + # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + # def test_service_annotations(self): + # ''' + # Create a Postgres cluster with service annotations and check them. + # ''' + # k8s = self.k8s + # patch_custom_service_annotations = { + # "data": { + # "custom_service_annotations": "foo:bar", + # } + # } + # k8s.update_config(patch_custom_service_annotations) + + # pg_patch_custom_annotations = { + # "spec": { + # "serviceAnnotations": { + # "annotation.key": "value", + # "foo": "bar", + # } + # } + # } + # k8s.api.custom_objects_api.patch_namespaced_custom_object( + # "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_custom_annotations) + + # # wait a little before proceeding + # time.sleep(30) + # annotations = { + # "annotation.key": "value", + # "foo": "bar", + # } + # self.assertTrue(k8s.check_service_annotations( + # "cluster-name=acid-minimal-cluster,spilo-role=master", annotations)) + # self.assertTrue(k8s.check_service_annotations( + # "cluster-name=acid-minimal-cluster,spilo-role=replica", annotations)) + + # # clean up + # unpatch_custom_service_annotations = { + # "data": { + # "custom_service_annotations": "", + # } + # } + # k8s.update_config(unpatch_custom_service_annotations) + + # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + # def test_statefulset_annotation_propagation(self): + # ''' + # Inject annotation to Postgresql CRD and check it's propagation to stateful set + # ''' + # k8s = self.k8s + # cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + + # patch_sset_propagate_annotations = { + # "data": { + # "downscaler_annotations": "deployment-time,downscaler/*", + # } + # } + # k8s.update_config(patch_sset_propagate_annotations) + + # pg_crd_annotations = { + # "metadata": { + # "annotations": { + # "deployment-time": "2020-04-30 12:00:00", + # "downscaler/downtime_replicas": "0", + # }, + # } + # } + # k8s.api.custom_objects_api.patch_namespaced_custom_object( + # "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_crd_annotations) + + # # wait a little before proceeding + # time.sleep(60) + # annotations = { + # "deployment-time": "2020-04-30 12:00:00", + # "downscaler/downtime_replicas": "0", + # } + # self.assertTrue(k8s.check_statefulset_annotations(cluster_label, annotations)) + + # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + # def test_taint_based_eviction(self): + # ''' + # Add taint "postgres=:NoExecute" to node with master. This must cause a failover. + # ''' + # k8s = self.k8s + # cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + + # # get nodes of master and replica(s) (expected target of new master) + # current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label) + # num_replicas = len(current_replica_nodes) + # failover_targets = self.get_failover_targets(current_master_node, current_replica_nodes) + + # # taint node with postgres=:NoExecute to force failover + # body = { + # "spec": { + # "taints": [ + # { + # "effect": "NoExecute", + # "key": "postgres" + # } + # ] + # } + # } + + # # patch node and test if master is failing over to one of the expected nodes + # k8s.api.core_v1.patch_node(current_master_node, body) + # new_master_node, new_replica_nodes = self.assert_failover( + # current_master_node, num_replicas, failover_targets, cluster_label) + + # # add toleration to pods + # patch_toleration_config = { + # "data": { + # "toleration": "key:postgres,operator:Exists,effect:NoExecute" + # } + # } + # k8s.update_config(patch_toleration_config) + + # # wait a little before proceeding with the pod distribution test + # time.sleep(30) + + # # toggle pod anti affinity to move replica away from master node + # self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_enable_disable_connection_pooler(self): + def test_infrastructure_roles(self): ''' - For a database without connection pooler, then turns it on, scale up, - turn off and on again. Test with different ways of doing this (via - enableConnectionPooler or connectionPooler configuration section). At - the end turn connection pooler off to not interfere with other tests. + Test using external secrets for infrastructure roles ''' k8s = self.k8s - service_labels = { - 'cluster-name': 'acid-minimal-cluster', + # update infrastructure roles description + secret_name = "postgresql-infrastructure-roles-old" + roles = "secretname: postgresql-infrastructure-roles-new, userkey: user, rolekey: role, passwordkey: password" + patch_infrastructure_roles = { + "data": { + "infrastructure_roles_secret_name": secret_name, + "infrastructure_roles_secrets": roles, + }, } - pod_labels = dict({ - 'connection-pooler': 'acid-minimal-cluster-pooler', + k8s.update_config(patch_infrastructure_roles) + + # wait a little before proceeding + time.sleep(30) + + # check that new roles are represented in the config by requesting the + # operator configuration via API + operator_pod = k8s.get_operator_pod() + get_config_cmd = "wget --quiet -O - localhost:8080/config" + result = k8s.exec_with_kubectl(operator_pod.metadata.name, get_config_cmd) + roles_dict = (json.loads(result.stdout) + .get("controller", {}) + .get("InfrastructureRoles")) + + self.assertTrue("robot_zmon_acid_monitoring_new" in roles_dict) + role = roles_dict["robot_zmon_acid_monitoring_new"] + role.pop("Password", None) + self.assertDictEqual(role, { + "Name": "robot_zmon_acid_monitoring_new", + "Flags": None, + "MemberOf": ["robot_zmon_new"], + "Parameters": None, + "AdminRole": "", + "Origin": 2, }) - pod_selector = to_selector(pod_labels) - service_selector = to_selector(service_labels) - - try: - # enable connection pooler - k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresqls', 'acid-minimal-cluster', - { - 'spec': { - 'enableConnectionPooler': True, - } - }) - k8s.wait_for_pod_start(pod_selector) - - pods = k8s.api.core_v1.list_namespaced_pod( - 'default', label_selector=pod_selector - ).items - - self.assertTrue(pods, 'No connection pooler pods') - - k8s.wait_for_service(service_selector) - services = k8s.api.core_v1.list_namespaced_service( - 'default', label_selector=service_selector - ).items - services = [ - s for s in services - if s.metadata.name.endswith('pooler') - ] - - self.assertTrue(services, 'No connection pooler service') - - # scale up connection pooler deployment - k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresqls', 'acid-minimal-cluster', - { - 'spec': { - 'connectionPooler': { - 'numberOfInstances': 2, - }, - } - }) - - k8s.wait_for_running_pods(pod_selector, 2) - - # turn it off, keeping configuration section - k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresqls', 'acid-minimal-cluster', - { - 'spec': { - 'enableConnectionPooler': False, - } - }) - k8s.wait_for_pods_to_stop(pod_selector) - - except timeout_decorator.TimeoutError: - print('Operator log: {}'.format(k8s.get_operator_log())) - raise - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_enable_load_balancer(self): - ''' - Test if services are updated when enabling/disabling load balancers - ''' - - k8s = self.k8s - cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' - - # enable load balancer services - pg_patch_enable_lbs = { - "spec": { - "enableMasterLoadBalancer": True, - "enableReplicaLoadBalancer": True - } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_lbs) - # wait for service recreation - time.sleep(60) - - master_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=master') - self.assertEqual(master_svc_type, 'LoadBalancer', - "Expected LoadBalancer service type for master, found {}".format(master_svc_type)) - - repl_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=replica') - self.assertEqual(repl_svc_type, 'LoadBalancer', - "Expected LoadBalancer service type for replica, found {}".format(repl_svc_type)) - - # disable load balancer services again - pg_patch_disable_lbs = { - "spec": { - "enableMasterLoadBalancer": False, - "enableReplicaLoadBalancer": False - } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_lbs) - # wait for service recreation - time.sleep(60) - - master_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=master') - self.assertEqual(master_svc_type, 'ClusterIP', - "Expected ClusterIP service type for master, found {}".format(master_svc_type)) - - repl_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=replica') - self.assertEqual(repl_svc_type, 'ClusterIP', - "Expected ClusterIP service type for replica, found {}".format(repl_svc_type)) - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_lazy_spilo_upgrade(self): - ''' - Test lazy upgrade for the Spilo image: operator changes a stateful set but lets pods run with the old image - until they are recreated for reasons other than operator's activity. That works because the operator configures - stateful sets to use "onDelete" pod update policy. - - The test covers: - 1) enabling lazy upgrade in existing operator deployment - 2) forcing the normal rolling upgrade by changing the operator configmap and restarting its pod - ''' - - k8s = self.k8s - - # update docker image in config and enable the lazy upgrade - conf_image = "registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p114" - patch_lazy_spilo_upgrade = { - "data": { - "docker_image": conf_image, - "enable_lazy_spilo_upgrade": "true" - } - } - k8s.update_config(patch_lazy_spilo_upgrade) - - pod0 = 'acid-minimal-cluster-0' - pod1 = 'acid-minimal-cluster-1' - - # restart the pod to get a container with the new image - k8s.api.core_v1.delete_namespaced_pod(pod0, 'default') - time.sleep(60) - - # lazy update works if the restarted pod and older pods run different Spilo versions - new_image = k8s.get_effective_pod_image(pod0) - old_image = k8s.get_effective_pod_image(pod1) - self.assertNotEqual(new_image, old_image, "Lazy updated failed: pods have the same image {}".format(new_image)) - - # sanity check - assert_msg = "Image {} of a new pod differs from {} in operator conf".format(new_image, conf_image) - self.assertEqual(new_image, conf_image, assert_msg) - - # clean up - unpatch_lazy_spilo_upgrade = { - "data": { - "enable_lazy_spilo_upgrade": "false", - } - } - k8s.update_config(unpatch_lazy_spilo_upgrade) - - # at this point operator will complete the normal rolling upgrade - # so we additonally test if disabling the lazy upgrade - forcing the normal rolling upgrade - works - - # XXX there is no easy way to wait until the end of Sync() - time.sleep(60) - - image0 = k8s.get_effective_pod_image(pod0) - image1 = k8s.get_effective_pod_image(pod1) - - assert_msg = "Disabling lazy upgrade failed: pods still have different images {} and {}".format(image0, image1) - self.assertEqual(image0, image1, assert_msg) - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_logical_backup_cron_job(self): - ''' - Ensure we can (a) create the cron job at user request for a specific PG cluster - (b) update the cluster-wide image for the logical backup pod - (c) delete the job at user request - - Limitations: - (a) Does not run the actual batch job because there is no S3 mock to upload backups to - (b) Assumes 'acid-minimal-cluster' exists as defined in setUp - ''' - - k8s = self.k8s - - # create the cron job - schedule = "7 7 7 7 *" - pg_patch_enable_backup = { - "spec": { - "enableLogicalBackup": True, - "logicalBackupSchedule": schedule - } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_backup) - k8s.wait_for_logical_backup_job_creation() - - jobs = k8s.get_logical_backup_job().items - self.assertEqual(1, len(jobs), "Expected 1 logical backup job, found {}".format(len(jobs))) - - job = jobs[0] - self.assertEqual(job.metadata.name, "logical-backup-acid-minimal-cluster", - "Expected job name {}, found {}" - .format("logical-backup-acid-minimal-cluster", job.metadata.name)) - self.assertEqual(job.spec.schedule, schedule, - "Expected {} schedule, found {}" - .format(schedule, job.spec.schedule)) - - # update the cluster-wide image of the logical backup pod - image = "test-image-name" - patch_logical_backup_image = { - "data": { - "logical_backup_docker_image": image, - } - } - k8s.update_config(patch_logical_backup_image) - - jobs = k8s.get_logical_backup_job().items - actual_image = jobs[0].spec.job_template.spec.template.spec.containers[0].image - self.assertEqual(actual_image, image, - "Expected job image {}, found {}".format(image, actual_image)) - - # delete the logical backup cron job - pg_patch_disable_backup = { - "spec": { - "enableLogicalBackup": False, - } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_backup) - k8s.wait_for_logical_backup_job_deletion() - jobs = k8s.get_logical_backup_job().items - self.assertEqual(0, len(jobs), - "Expected 0 logical backup jobs, found {}".format(len(jobs))) - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_min_resource_limits(self): - ''' - Lower resource limits below configured minimum and let operator fix it - ''' - k8s = self.k8s - cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' - labels = 'spilo-role=master,' + cluster_label - _, failover_targets = k8s.get_pg_nodes(cluster_label) - - # configure minimum boundaries for CPU and memory limits - minCPULimit = '500m' - minMemoryLimit = '500Mi' - patch_min_resource_limits = { - "data": { - "min_cpu_limit": minCPULimit, - "min_memory_limit": minMemoryLimit - } - } - k8s.update_config(patch_min_resource_limits) - - # lower resource limits below minimum - pg_patch_resources = { - "spec": { - "resources": { - "requests": { - "cpu": "10m", - "memory": "50Mi" - }, - "limits": { - "cpu": "200m", - "memory": "200Mi" - } - } - } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_resources) - k8s.wait_for_pod_failover(failover_targets, labels) - k8s.wait_for_pod_start('spilo-role=replica') - - pods = k8s.api.core_v1.list_namespaced_pod( - 'default', label_selector=labels).items - self.assert_master_is_unique() - masterPod = pods[0] - - self.assertEqual(masterPod.spec.containers[0].resources.limits['cpu'], minCPULimit, - "Expected CPU limit {}, found {}" - .format(minCPULimit, masterPod.spec.containers[0].resources.limits['cpu'])) - self.assertEqual(masterPod.spec.containers[0].resources.limits['memory'], minMemoryLimit, - "Expected memory limit {}, found {}" - .format(minMemoryLimit, masterPod.spec.containers[0].resources.limits['memory'])) - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_multi_namespace_support(self): - ''' - Create a customized Postgres cluster in a non-default namespace. - ''' - k8s = self.k8s - - with open("manifests/complete-postgres-manifest.yaml", 'r+') as f: - pg_manifest = yaml.safe_load(f) - pg_manifest["metadata"]["namespace"] = self.namespace - yaml.dump(pg_manifest, f, Dumper=yaml.Dumper) - - k8s.create_with_kubectl("manifests/complete-postgres-manifest.yaml") - k8s.wait_for_pod_start("spilo-role=master", self.namespace) - self.assert_master_is_unique(self.namespace, "acid-test-cluster") - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_node_readiness_label(self): - ''' - Remove node readiness label from master node. This must cause a failover. - ''' - k8s = self.k8s - cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' - readiness_label = 'lifecycle-status' - readiness_value = 'ready' - - # get nodes of master and replica(s) (expected target of new master) - current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label) - num_replicas = len(current_replica_nodes) - failover_targets = self.get_failover_targets(current_master_node, current_replica_nodes) - - # add node_readiness_label to potential failover nodes - patch_readiness_label = { - "metadata": { - "labels": { - readiness_label: readiness_value - } - } - } - for failover_target in failover_targets: - k8s.api.core_v1.patch_node(failover_target, patch_readiness_label) - - # define node_readiness_label in config map which should trigger a failover of the master - patch_readiness_label_config = { - "data": { - "node_readiness_label": readiness_label + ':' + readiness_value, - } - } - k8s.update_config(patch_readiness_label_config) - new_master_node, new_replica_nodes = self.assert_failover( - current_master_node, num_replicas, failover_targets, cluster_label) - - # patch also node where master ran before - k8s.api.core_v1.patch_node(current_master_node, patch_readiness_label) - - # wait a little before proceeding with the pod distribution test - time.sleep(30) - - # toggle pod anti affinity to move replica away from master node - self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_scaling(self): - ''' - Scale up from 2 to 3 and back to 2 pods by updating the Postgres manifest at runtime. - ''' - k8s = self.k8s - labels = "application=spilo,cluster-name=acid-minimal-cluster" - - k8s.wait_for_pg_to_scale(3) - self.assertEqual(3, k8s.count_pods_with_label(labels)) - self.assert_master_is_unique() - - k8s.wait_for_pg_to_scale(2) - self.assertEqual(2, k8s.count_pods_with_label(labels)) - self.assert_master_is_unique() - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_service_annotations(self): - ''' - Create a Postgres cluster with service annotations and check them. - ''' - k8s = self.k8s - patch_custom_service_annotations = { - "data": { - "custom_service_annotations": "foo:bar", - } - } - k8s.update_config(patch_custom_service_annotations) - - pg_patch_custom_annotations = { - "spec": { - "serviceAnnotations": { - "annotation.key": "value", - "foo": "bar", - } - } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_custom_annotations) - - # wait a little before proceeding - time.sleep(30) - annotations = { - "annotation.key": "value", - "foo": "bar", - } - self.assertTrue(k8s.check_service_annotations( - "cluster-name=acid-minimal-cluster,spilo-role=master", annotations)) - self.assertTrue(k8s.check_service_annotations( - "cluster-name=acid-minimal-cluster,spilo-role=replica", annotations)) - - # clean up - unpatch_custom_service_annotations = { - "data": { - "custom_service_annotations": "", - } - } - k8s.update_config(unpatch_custom_service_annotations) - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_statefulset_annotation_propagation(self): - ''' - Inject annotation to Postgresql CRD and check it's propagation to stateful set - ''' - k8s = self.k8s - cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' - - patch_sset_propagate_annotations = { - "data": { - "downscaler_annotations": "deployment-time,downscaler/*", - } - } - k8s.update_config(patch_sset_propagate_annotations) - - pg_crd_annotations = { - "metadata": { - "annotations": { - "deployment-time": "2020-04-30 12:00:00", - "downscaler/downtime_replicas": "0", - }, - } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_crd_annotations) - - # wait a little before proceeding - time.sleep(60) - annotations = { - "deployment-time": "2020-04-30 12:00:00", - "downscaler/downtime_replicas": "0", - } - self.assertTrue(k8s.check_statefulset_annotations(cluster_label, annotations)) - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_taint_based_eviction(self): - ''' - Add taint "postgres=:NoExecute" to node with master. This must cause a failover. - ''' - k8s = self.k8s - cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' - - # get nodes of master and replica(s) (expected target of new master) - current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label) - num_replicas = len(current_replica_nodes) - failover_targets = self.get_failover_targets(current_master_node, current_replica_nodes) - - # taint node with postgres=:NoExecute to force failover - body = { - "spec": { - "taints": [ - { - "effect": "NoExecute", - "key": "postgres" - } - ] - } - } - - # patch node and test if master is failing over to one of the expected nodes - k8s.api.core_v1.patch_node(current_master_node, body) - new_master_node, new_replica_nodes = self.assert_failover( - current_master_node, num_replicas, failover_targets, cluster_label) - - # add toleration to pods - patch_toleration_config = { - "data": { - "toleration": "key:postgres,operator:Exists,effect:NoExecute" - } - } - k8s.update_config(patch_toleration_config) - - # wait a little before proceeding with the pod distribution test - time.sleep(30) - - # toggle pod anti affinity to move replica away from master node - self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) - def get_failover_targets(self, master_node, replica_nodes): ''' If all pods live on the same node, failover will happen to other worker(s) @@ -820,6 +863,11 @@ class K8s: stdout=subprocess.PIPE, stderr=subprocess.PIPE) + def exec_with_kubectl(self, pod, cmd): + return subprocess.run(["./exec.sh", pod, cmd], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + def get_effective_pod_image(self, pod_name, namespace='default'): ''' Get the Spilo image pod currently uses. In case of lazy rolling updates diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index d1c1b3d17..1210d5015 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -47,7 +47,8 @@ data: # etcd_host: "" # gcp_credentials: "" # kubernetes_use_configmaps: "false" - # infrastructure_roles_secret_name: postgresql-infrastructure-roles + # infrastructure_roles_secret_name: "postgresql-infrastructure-roles" + # infrastructure_roles_secrets: "secretname:monitoring-roles,userkey:user,passwordkey:password,rolekey:inrole" # inherited_labels: application,environment # kube_iam_role: "" # log_s3_bucket: "" diff --git a/manifests/infrastructure-roles-new.yaml b/manifests/infrastructure-roles-new.yaml new file mode 100644 index 000000000..e4f378396 --- /dev/null +++ b/manifests/infrastructure-roles-new.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +data: + # infrastructure role definition in the new format + # robot_zmon_acid_monitoring_new + user: cm9ib3Rfem1vbl9hY2lkX21vbml0b3JpbmdfbmV3 + # robot_zmon_new + role: cm9ib3Rfem1vbl9uZXc= + # foobar_new + password: Zm9vYmFyX25ldw== +kind: Secret +metadata: + name: postgresql-infrastructure-roles-new + namespace: default +type: Opaque diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 2b6e8ae67..55b7653ef 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -127,6 +127,28 @@ spec: type: boolean infrastructure_roles_secret_name: type: string + infrastructure_roles_secrets: + type: array + nullable: true + items: + type: object + required: + - secretname + - userkey + - passwordkey + properties: + secretname: + type: string + userkey: + type: string + passwordkey: + type: string + rolekey: + type: string + details: + type: string + template: + type: boolean inherited_labels: type: array items: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index f7eba1f6c..c0dce42ee 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -39,6 +39,14 @@ configuration: enable_pod_disruption_budget: true enable_sidecars: true # infrastructure_roles_secret_name: "postgresql-infrastructure-roles" + # infrastructure_roles_secrets: + # - secretname: "monitoring-roles" + # userkey: "user" + # passwordkey: "password" + # rolekey: "inrole" + # - secretname: "other-infrastructure-role" + # userkey: "other-user-key" + # passwordkey: "other-password-key" # inherited_labels: # - application # - environment diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 6f907e266..c22ed25c0 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -911,6 +911,35 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation "infrastructure_roles_secret_name": { Type: "string", }, + "infrastructure_roles_secrets": { + Type: "array", + Items: &apiextv1beta1.JSONSchemaPropsOrArray{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "object", + Required: []string{"secretname", "userkey", "passwordkey"}, + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "secretname": { + Type: "string", + }, + "userkey": { + Type: "string", + }, + "passwordkey": { + Type: "string", + }, + "rolekey": { + Type: "string", + }, + "details": { + Type: "string", + }, + "template": { + Type: "boolean", + }, + }, + }, + }, + }, "inherited_labels": { Type: "array", Items: &apiextv1beta1.JSONSchemaPropsOrArray{ @@ -983,7 +1012,7 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation "spilo_privileged": { Type: "boolean", }, - "storage_resize_mode": { + "storage_resize_mode": { Type: "string", Enum: []apiextv1beta1.JSON{ { diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index e6e13cbd3..ea08f2ff3 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -45,28 +45,29 @@ type PostgresUsersConfiguration struct { type KubernetesMetaConfiguration struct { PodServiceAccountName string `json:"pod_service_account_name,omitempty"` // TODO: change it to the proper json - PodServiceAccountDefinition string `json:"pod_service_account_definition,omitempty"` - PodServiceAccountRoleBindingDefinition string `json:"pod_service_account_role_binding_definition,omitempty"` - PodTerminateGracePeriod Duration `json:"pod_terminate_grace_period,omitempty"` - SpiloPrivileged bool `json:"spilo_privileged,omitempty"` - SpiloFSGroup *int64 `json:"spilo_fsgroup,omitempty"` - WatchedNamespace string `json:"watched_namespace,omitempty"` - PDBNameFormat config.StringTemplate `json:"pdb_name_format,omitempty"` - EnablePodDisruptionBudget *bool `json:"enable_pod_disruption_budget,omitempty"` - StorageResizeMode string `json:"storage_resize_mode,omitempty"` - EnableInitContainers *bool `json:"enable_init_containers,omitempty"` - EnableSidecars *bool `json:"enable_sidecars,omitempty"` - SecretNameTemplate config.StringTemplate `json:"secret_name_template,omitempty"` - ClusterDomain string `json:"cluster_domain,omitempty"` - OAuthTokenSecretName spec.NamespacedName `json:"oauth_token_secret_name,omitempty"` - InfrastructureRolesSecretName spec.NamespacedName `json:"infrastructure_roles_secret_name,omitempty"` - PodRoleLabel string `json:"pod_role_label,omitempty"` - ClusterLabels map[string]string `json:"cluster_labels,omitempty"` - InheritedLabels []string `json:"inherited_labels,omitempty"` - DownscalerAnnotations []string `json:"downscaler_annotations,omitempty"` - ClusterNameLabel string `json:"cluster_name_label,omitempty"` - NodeReadinessLabel map[string]string `json:"node_readiness_label,omitempty"` - CustomPodAnnotations map[string]string `json:"custom_pod_annotations,omitempty"` + PodServiceAccountDefinition string `json:"pod_service_account_definition,omitempty"` + PodServiceAccountRoleBindingDefinition string `json:"pod_service_account_role_binding_definition,omitempty"` + PodTerminateGracePeriod Duration `json:"pod_terminate_grace_period,omitempty"` + SpiloPrivileged bool `json:"spilo_privileged,omitempty"` + SpiloFSGroup *int64 `json:"spilo_fsgroup,omitempty"` + WatchedNamespace string `json:"watched_namespace,omitempty"` + PDBNameFormat config.StringTemplate `json:"pdb_name_format,omitempty"` + EnablePodDisruptionBudget *bool `json:"enable_pod_disruption_budget,omitempty"` + StorageResizeMode string `json:"storage_resize_mode,omitempty"` + EnableInitContainers *bool `json:"enable_init_containers,omitempty"` + EnableSidecars *bool `json:"enable_sidecars,omitempty"` + SecretNameTemplate config.StringTemplate `json:"secret_name_template,omitempty"` + ClusterDomain string `json:"cluster_domain,omitempty"` + OAuthTokenSecretName spec.NamespacedName `json:"oauth_token_secret_name,omitempty"` + InfrastructureRolesSecretName spec.NamespacedName `json:"infrastructure_roles_secret_name,omitempty"` + InfrastructureRolesDefs []*config.InfrastructureRole `json:"infrastructure_roles_secrets,omitempty"` + PodRoleLabel string `json:"pod_role_label,omitempty"` + ClusterLabels map[string]string `json:"cluster_labels,omitempty"` + InheritedLabels []string `json:"inherited_labels,omitempty"` + DownscalerAnnotations []string `json:"downscaler_annotations,omitempty"` + ClusterNameLabel string `json:"cluster_name_label,omitempty"` + NodeReadinessLabel map[string]string `json:"node_readiness_label,omitempty"` + CustomPodAnnotations map[string]string `json:"custom_pod_annotations,omitempty"` // TODO: use a proper toleration structure? PodToleration map[string]string `json:"toleration,omitempty"` PodEnvironmentConfigMap spec.NamespacedName `json:"pod_environment_configmap,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 064ced184..efc31d6b6 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -27,6 +27,7 @@ SOFTWARE. package v1 import ( + config "github.com/zalando/postgres-operator/pkg/util/config" corev1 "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -168,6 +169,17 @@ func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfigura } out.OAuthTokenSecretName = in.OAuthTokenSecretName out.InfrastructureRolesSecretName = in.InfrastructureRolesSecretName + if in.InfrastructureRolesDefs != nil { + in, out := &in.InfrastructureRolesDefs, &out.InfrastructureRolesDefs + *out = make([]*config.InfrastructureRole, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(config.InfrastructureRole) + **out = **in + } + } + } if in.ClusterLabels != nil { in, out := &in.ClusterLabels, &out.ClusterLabels *out = make(map[string]string, len(*in)) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 6011d3863..10c817016 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -300,7 +300,8 @@ func (c *Controller) initController() { c.logger.Infof("config: %s", c.opConfig.MustMarshal()) - if infraRoles, err := c.getInfrastructureRoles(&c.opConfig.InfrastructureRolesSecretName); err != nil { + roleDefs := c.getInfrastructureRoleDefinitions() + if infraRoles, err := c.getInfrastructureRoles(roleDefs); err != nil { c.logger.Warningf("could not get infrastructure roles: %v", err) } else { c.config.InfrastructureRoles = infraRoles diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index e2d8636a1..d115aa118 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -71,7 +71,22 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.EnableSidecars = util.CoalesceBool(fromCRD.Kubernetes.EnableSidecars, util.True()) result.SecretNameTemplate = fromCRD.Kubernetes.SecretNameTemplate result.OAuthTokenSecretName = fromCRD.Kubernetes.OAuthTokenSecretName + result.InfrastructureRolesSecretName = fromCRD.Kubernetes.InfrastructureRolesSecretName + if fromCRD.Kubernetes.InfrastructureRolesDefs != nil { + result.InfrastructureRoles = []*config.InfrastructureRole{} + for _, secret := range fromCRD.Kubernetes.InfrastructureRolesDefs { + result.InfrastructureRoles = append( + result.InfrastructureRoles, + &config.InfrastructureRole{ + SecretName: secret.SecretName, + UserKey: secret.UserKey, + RoleKey: secret.RoleKey, + PasswordKey: secret.PasswordKey, + }) + } + } + result.PodRoleLabel = util.Coalesce(fromCRD.Kubernetes.PodRoleLabel, "spilo-role") result.ClusterLabels = util.CoalesceStrMap(fromCRD.Kubernetes.ClusterLabels, map[string]string{"application": "spilo"}) result.InheritedLabels = fromCRD.Kubernetes.InheritedLabels diff --git a/pkg/controller/util.go b/pkg/controller/util.go index 511f02823..6035903dd 100644 --- a/pkg/controller/util.go +++ b/pkg/controller/util.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "strings" v1 "k8s.io/api/core/v1" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" @@ -109,8 +110,161 @@ func readDecodedRole(s string) (*spec.PgUser, error) { return &result, nil } -func (c *Controller) getInfrastructureRoles(rolesSecret *spec.NamespacedName) (map[string]spec.PgUser, error) { - if *rolesSecret == (spec.NamespacedName{}) { +var emptyName = (spec.NamespacedName{}) + +// Return information about what secrets we need to use to create +// infrastructure roles and in which format are they. This is done in +// compatible way, so that the previous logic is not changed, and handles both +// configuration in ConfigMap & CRD. +func (c *Controller) getInfrastructureRoleDefinitions() []*config.InfrastructureRole { + var roleDef config.InfrastructureRole + rolesDefs := c.opConfig.InfrastructureRoles + + if c.opConfig.InfrastructureRolesSecretName == emptyName { + // All the other possibilities require secret name to be present, so if + // it is not, then nothing else to be done here. + return rolesDefs + } + + // check if we can extract something from the configmap config option + if c.opConfig.InfrastructureRolesDefs != "" { + // The configmap option could contain either a role description (in the + // form key1: value1, key2: value2), which has to be used together with + // an old secret name. + + var secretName spec.NamespacedName + var err error + propertySep := "," + valueSep := ":" + + // The field contains the format in which secret is written, let's + // convert it to a proper definition + properties := strings.Split(c.opConfig.InfrastructureRolesDefs, propertySep) + roleDef = config.InfrastructureRole{Template: false} + + for _, property := range properties { + values := strings.Split(property, valueSep) + if len(values) < 2 { + continue + } + name := strings.TrimSpace(values[0]) + value := strings.TrimSpace(values[1]) + + switch name { + case "secretname": + if err = secretName.DecodeWorker(value, "default"); err != nil { + c.logger.Warningf("Could not marshal secret name %s: %v", value, err) + } else { + roleDef.SecretName = secretName + } + case "userkey": + roleDef.UserKey = value + case "passwordkey": + roleDef.PasswordKey = value + case "rolekey": + roleDef.RoleKey = value + default: + c.logger.Warningf("Role description is not known: %s", properties) + } + } + } else { + // At this point we deal with the old format, let's replicate it + // via existing definition structure and remember that it's just a + // template, the real values are in user1,password1,inrole1 etc. + roleDef = config.InfrastructureRole{ + SecretName: c.opConfig.InfrastructureRolesSecretName, + UserKey: "user", + PasswordKey: "password", + RoleKey: "inrole", + Template: true, + } + } + + if roleDef.UserKey != "" && + roleDef.PasswordKey != "" && + roleDef.RoleKey != "" { + rolesDefs = append(rolesDefs, &roleDef) + } + + return rolesDefs +} + +func (c *Controller) getInfrastructureRoles( + rolesSecrets []*config.InfrastructureRole) ( + map[string]spec.PgUser, []error) { + + var errors []error + var noRolesProvided = true + + roles := []spec.PgUser{} + uniqRoles := map[string]spec.PgUser{} + + // To be compatible with the legacy implementation we need to return nil if + // the provided secret name is empty. The equivalent situation in the + // current implementation is an empty rolesSecrets slice or all its items + // are empty. + for _, role := range rolesSecrets { + if role.SecretName != emptyName { + noRolesProvided = false + } + } + + if noRolesProvided { + return nil, nil + } + + for _, secret := range rolesSecrets { + infraRoles, err := c.getInfrastructureRole(secret) + + if err != nil || infraRoles == nil { + c.logger.Debugf("Cannot get infrastructure role: %+v", *secret) + + if err != nil { + errors = append(errors, err) + } + + continue + } + + for _, r := range infraRoles { + roles = append(roles, r) + } + } + + for _, r := range roles { + if _, exists := uniqRoles[r.Name]; exists { + msg := "Conflicting infrastructure roles: roles[%s] = (%q, %q)" + c.logger.Debugf(msg, r.Name, uniqRoles[r.Name], r) + } + + uniqRoles[r.Name] = r + } + + return uniqRoles, errors +} + +// Generate list of users representing one infrastructure role based on its +// description in various K8S objects. An infrastructure role could be +// described by a secret and optionally a config map. The former should contain +// the secret information, i.e. username, password, role. The latter could +// contain an extensive description of the role and even override an +// information obtained from the secret (except a password). +// +// This function returns a list of users to be compatible with the previous +// behaviour, since we don't know how many users are actually encoded in the +// secret if it's a "template" role. If the provided role is not a template +// one, the result would be a list with just one user in it. +// +// FIXME: This dependency on two different objects is rather unnecessary +// complicated, so let's get rid of it via deprecation process. +func (c *Controller) getInfrastructureRole( + infraRole *config.InfrastructureRole) ( + []spec.PgUser, error) { + + rolesSecret := infraRole.SecretName + roles := []spec.PgUser{} + + if rolesSecret == emptyName { // we don't have infrastructure roles defined, bail out return nil, nil } @@ -119,52 +273,98 @@ func (c *Controller) getInfrastructureRoles(rolesSecret *spec.NamespacedName) (m Secrets(rolesSecret.Namespace). Get(context.TODO(), rolesSecret.Name, metav1.GetOptions{}) if err != nil { - c.logger.Debugf("infrastructure roles secret name: %q", *rolesSecret) - return nil, fmt.Errorf("could not get infrastructure roles secret: %v", err) + msg := "could not get infrastructure roles secret %s/%s: %v" + return nil, fmt.Errorf(msg, rolesSecret.Namespace, rolesSecret.Name, err) } secretData := infraRolesSecret.Data - result := make(map[string]spec.PgUser) -Users: - // in worst case we would have one line per user - for i := 1; i <= len(secretData); i++ { - properties := []string{"user", "password", "inrole"} - t := spec.PgUser{Origin: spec.RoleOriginInfrastructure} - for _, p := range properties { - key := fmt.Sprintf("%s%d", p, i) - if val, present := secretData[key]; !present { - if p == "user" { - // exit when the user name with the next sequence id is absent - break Users - } - } else { - s := string(val) - switch p { - case "user": - t.Name = s - case "password": - t.Password = s - case "inrole": - t.MemberOf = append(t.MemberOf, s) - default: - c.logger.Warningf("unknown key %q", p) - } + + if infraRole.Template { + Users: + for i := 1; i <= len(secretData); i++ { + properties := []string{ + infraRole.UserKey, + infraRole.PasswordKey, + infraRole.RoleKey, } - delete(secretData, key) + t := spec.PgUser{Origin: spec.RoleOriginInfrastructure} + for _, p := range properties { + key := fmt.Sprintf("%s%d", p, i) + if val, present := secretData[key]; !present { + if p == "user" { + // exit when the user name with the next sequence id is + // absent + break Users + } + } else { + s := string(val) + switch p { + case "user": + t.Name = s + case "password": + t.Password = s + case "inrole": + t.MemberOf = append(t.MemberOf, s) + default: + c.logger.Warningf("unknown key %q", p) + } + } + // XXX: This is a part of the original implementation, which is + // rather obscure. Why do we delete this key? Wouldn't it be + // used later in comparison for configmap? + delete(secretData, key) + } + + if t.Valid() { + roles = append(roles, t) + } else { + msg := "infrastructure role %q is not complete and ignored" + c.logger.Warningf(msg, t) + } + } + } else { + roleDescr := &spec.PgUser{Origin: spec.RoleOriginInfrastructure} + + if details, exists := secretData[infraRole.Details]; exists { + if err := yaml.Unmarshal(details, &roleDescr); err != nil { + return nil, fmt.Errorf("could not decode yaml role: %v", err) + } + } else { + roleDescr.Name = string(secretData[infraRole.UserKey]) + roleDescr.Password = string(secretData[infraRole.PasswordKey]) + roleDescr.MemberOf = append(roleDescr.MemberOf, string(secretData[infraRole.RoleKey])) } - if t.Name != "" { - if t.Password == "" { - c.logger.Warningf("infrastructure role %q has no password defined and is ignored", t.Name) - continue - } - result[t.Name] = t + if roleDescr.Valid() { + roles = append(roles, *roleDescr) + } else { + msg := "infrastructure role %q is not complete and ignored" + c.logger.Warningf(msg, roleDescr) + + return nil, nil } + + if roleDescr.Name == "" { + msg := "infrastructure role %q has no name defined and is ignored" + c.logger.Warningf(msg, roleDescr.Name) + return nil, nil + } + + if roleDescr.Password == "" { + msg := "infrastructure role %q has no password defined and is ignored" + c.logger.Warningf(msg, roleDescr.Name) + return nil, nil + } + + roles = append(roles, *roleDescr) } - // perhaps we have some map entries with usernames, passwords, let's check if we have those users in the configmap - if infraRolesMap, err := c.KubeClient.ConfigMaps(rolesSecret.Namespace).Get( - context.TODO(), rolesSecret.Name, metav1.GetOptions{}); err == nil { + // Now plot twist. We need to check if there is a configmap with the same + // name and extract a role description if it exists. + infraRolesMap, err := c.KubeClient. + ConfigMaps(rolesSecret.Namespace). + Get(context.TODO(), rolesSecret.Name, metav1.GetOptions{}) + if err == nil { // we have a configmap with username - json description, let's read and decode it for role, s := range infraRolesMap.Data { roleDescr, err := readDecodedRole(s) @@ -182,20 +382,12 @@ Users: } roleDescr.Name = role roleDescr.Origin = spec.RoleOriginInfrastructure - result[role] = *roleDescr + roles = append(roles, *roleDescr) } } - if len(secretData) > 0 { - c.logger.Warningf("%d unprocessed entries in the infrastructure roles secret,"+ - " checking configmap %v", len(secretData), rolesSecret.Name) - c.logger.Info(`infrastructure role entries should be in the {key}{id} format,` + - ` where {key} can be either of "user", "password", "inrole" and the {id}` + - ` a monotonically increasing integer starting with 1`) - c.logger.Debugf("unprocessed entries: %#v", secretData) - } - - return result, nil + // TODO: check for role collisions + return roles, nil } func (c *Controller) podClusterName(pod *v1.Pod) spec.NamespacedName { diff --git a/pkg/controller/util_test.go b/pkg/controller/util_test.go index ef182248e..fd756a0c7 100644 --- a/pkg/controller/util_test.go +++ b/pkg/controller/util_test.go @@ -8,20 +8,25 @@ import ( b64 "encoding/base64" "github.com/zalando/postgres-operator/pkg/spec" + "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/k8sutil" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( - testInfrastructureRolesSecretName = "infrastructureroles-test" + testInfrastructureRolesOldSecretName = "infrastructureroles-old-test" + testInfrastructureRolesNewSecretName = "infrastructureroles-new-test" ) func newUtilTestController() *Controller { controller := NewController(&spec.ControllerConfig{}, "util-test") controller.opConfig.ClusterNameLabel = "cluster-name" controller.opConfig.InfrastructureRolesSecretName = - spec.NamespacedName{Namespace: v1.NamespaceDefault, Name: testInfrastructureRolesSecretName} + spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesOldSecretName, + } controller.opConfig.Workers = 4 controller.KubeClient = k8sutil.NewMockKubernetesClient() return controller @@ -80,24 +85,32 @@ func TestClusterWorkerID(t *testing.T) { } } -func TestGetInfrastructureRoles(t *testing.T) { +// Test functionality of getting infrastructure roles from their description in +// corresponding secrets. Here we test only common stuff (e.g. when a secret do +// not exist, or empty) and the old format. +func TestOldInfrastructureRoleFormat(t *testing.T) { var testTable = []struct { - secretName spec.NamespacedName - expectedRoles map[string]spec.PgUser - expectedError error + secretName spec.NamespacedName + expectedRoles map[string]spec.PgUser + expectedErrors []error }{ { + // empty secret name spec.NamespacedName{}, nil, nil, }, { + // secret does not exist spec.NamespacedName{Namespace: v1.NamespaceDefault, Name: "null"}, - nil, - fmt.Errorf(`could not get infrastructure roles secret: NotFound`), + map[string]spec.PgUser{}, + []error{fmt.Errorf(`could not get infrastructure roles secret default/null: NotFound`)}, }, { - spec.NamespacedName{Namespace: v1.NamespaceDefault, Name: testInfrastructureRolesSecretName}, + spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesOldSecretName, + }, map[string]spec.PgUser{ "testrole": { Name: "testrole", @@ -116,15 +129,269 @@ func TestGetInfrastructureRoles(t *testing.T) { }, } for _, test := range testTable { - roles, err := utilTestController.getInfrastructureRoles(&test.secretName) - if err != test.expectedError { - if err != nil && test.expectedError != nil && err.Error() == test.expectedError.Error() { - continue - } - t.Errorf("expected error '%v' does not match the actual error '%v'", test.expectedError, err) + roles, errors := utilTestController.getInfrastructureRoles( + []*config.InfrastructureRole{ + &config.InfrastructureRole{ + SecretName: test.secretName, + UserKey: "user", + PasswordKey: "password", + RoleKey: "inrole", + Template: true, + }, + }) + + if len(errors) != len(test.expectedErrors) { + t.Errorf("expected error '%v' does not match the actual error '%v'", + test.expectedErrors, errors) } + + for idx := range errors { + err := errors[idx] + expectedErr := test.expectedErrors[idx] + + if err != expectedErr { + if err != nil && expectedErr != nil && err.Error() == expectedErr.Error() { + continue + } + t.Errorf("expected error '%v' does not match the actual error '%v'", + expectedErr, err) + } + } + if !reflect.DeepEqual(roles, test.expectedRoles) { - t.Errorf("expected roles output %v does not match the actual %v", test.expectedRoles, roles) + t.Errorf("expected roles output %#v does not match the actual %#v", + test.expectedRoles, roles) + } + } +} + +// Test functionality of getting infrastructure roles from their description in +// corresponding secrets. Here we test the new format. +func TestNewInfrastructureRoleFormat(t *testing.T) { + var testTable = []struct { + secrets []spec.NamespacedName + expectedRoles map[string]spec.PgUser + expectedErrors []error + }{ + // one secret with one configmap + { + []spec.NamespacedName{ + spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesNewSecretName, + }, + }, + map[string]spec.PgUser{ + "new-test-role": { + Name: "new-test-role", + Origin: spec.RoleOriginInfrastructure, + Password: "new-test-password", + MemberOf: []string{"new-test-inrole"}, + }, + "new-foobar": { + Name: "new-foobar", + Origin: spec.RoleOriginInfrastructure, + Password: b64.StdEncoding.EncodeToString([]byte("password")), + MemberOf: nil, + Flags: []string{"createdb"}, + }, + }, + nil, + }, + // multiple standalone secrets + { + []spec.NamespacedName{ + spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: "infrastructureroles-new-test1", + }, + spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: "infrastructureroles-new-test2", + }, + }, + map[string]spec.PgUser{ + "new-test-role1": { + Name: "new-test-role1", + Origin: spec.RoleOriginInfrastructure, + Password: "new-test-password1", + MemberOf: []string{"new-test-inrole1"}, + }, + "new-test-role2": { + Name: "new-test-role2", + Origin: spec.RoleOriginInfrastructure, + Password: "new-test-password2", + MemberOf: []string{"new-test-inrole2"}, + }, + }, + nil, + }, + } + for _, test := range testTable { + definitions := []*config.InfrastructureRole{} + for _, secret := range test.secrets { + definitions = append(definitions, &config.InfrastructureRole{ + SecretName: secret, + UserKey: "user", + PasswordKey: "password", + RoleKey: "inrole", + Template: false, + }) + } + + roles, errors := utilTestController.getInfrastructureRoles(definitions) + if len(errors) != len(test.expectedErrors) { + t.Errorf("expected error does not match the actual error:\n%+v\n%+v", + test.expectedErrors, errors) + + // Stop and do not do any further checks + return + } + + for idx := range errors { + err := errors[idx] + expectedErr := test.expectedErrors[idx] + + if err != expectedErr { + if err != nil && expectedErr != nil && err.Error() == expectedErr.Error() { + continue + } + t.Errorf("expected error '%v' does not match the actual error '%v'", + expectedErr, err) + } + } + + if !reflect.DeepEqual(roles, test.expectedRoles) { + t.Errorf("expected roles output/the actual:\n%#v\n%#v", + test.expectedRoles, roles) + } + } +} + +// Tests for getting correct infrastructure roles definitions from present +// configuration. E.g. in which secrets for which roles too look. The biggest +// point here is compatibility of old and new formats of defining +// infrastructure roles. +func TestInfrastructureRoleDefinitions(t *testing.T) { + var testTable = []struct { + rolesDefs []*config.InfrastructureRole + roleSecretName spec.NamespacedName + roleSecrets string + expectedDefs []*config.InfrastructureRole + }{ + // only new format + { + []*config.InfrastructureRole{ + &config.InfrastructureRole{ + SecretName: spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesNewSecretName, + }, + UserKey: "user", + PasswordKey: "password", + RoleKey: "inrole", + Template: false, + }, + }, + spec.NamespacedName{}, + "", + []*config.InfrastructureRole{ + &config.InfrastructureRole{ + SecretName: spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesNewSecretName, + }, + UserKey: "user", + PasswordKey: "password", + RoleKey: "inrole", + Template: false, + }, + }, + }, + // only old format + { + []*config.InfrastructureRole{}, + spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesOldSecretName, + }, + "", + []*config.InfrastructureRole{ + &config.InfrastructureRole{ + SecretName: spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesOldSecretName, + }, + UserKey: "user", + PasswordKey: "password", + RoleKey: "inrole", + Template: true, + }, + }, + }, + // only configmap format + { + []*config.InfrastructureRole{}, + spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesOldSecretName, + }, + "secretname: infrastructureroles-old-test, userkey: test-user, passwordkey: test-password, rolekey: test-role, template: false", + []*config.InfrastructureRole{ + &config.InfrastructureRole{ + SecretName: spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesOldSecretName, + }, + UserKey: "test-user", + PasswordKey: "test-password", + RoleKey: "test-role", + Template: false, + }, + }, + }, + // incorrect configmap format + { + []*config.InfrastructureRole{}, + spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesOldSecretName, + }, + "wrong-format", + []*config.InfrastructureRole{}, + }, + // configmap without a secret + { + []*config.InfrastructureRole{}, + spec.NamespacedName{}, + "userkey: test-user, passwordkey: test-password, rolekey: test-role, template: false", + []*config.InfrastructureRole{}, + }, + } + + for _, test := range testTable { + t.Logf("Test: %+v", test) + utilTestController.opConfig.InfrastructureRoles = test.rolesDefs + utilTestController.opConfig.InfrastructureRolesSecretName = test.roleSecretName + utilTestController.opConfig.InfrastructureRolesDefs = test.roleSecrets + + defs := utilTestController.getInfrastructureRoleDefinitions() + if len(defs) != len(test.expectedDefs) { + t.Errorf("expected definitions does not match the actual:\n%#v\n%#v", + test.expectedDefs, defs) + + // Stop and do not do any further checks + return + } + + for idx := range defs { + def := defs[idx] + expectedDef := test.expectedDefs[idx] + + if !reflect.DeepEqual(def, expectedDef) { + t.Errorf("expected definition/the actual:\n%#v\n%#v", + expectedDef, def) + } } } } diff --git a/pkg/spec/types.go b/pkg/spec/types.go index 08008267b..7a2c0ddac 100644 --- a/pkg/spec/types.go +++ b/pkg/spec/types.go @@ -55,6 +55,10 @@ type PgUser struct { AdminRole string `yaml:"admin_role"` } +func (user *PgUser) Valid() bool { + return user.Name != "" && user.Password != "" +} + // PgUserMap maps user names to the definitions. type PgUserMap map[string]PgUser diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 6cab8af45..5f262107f 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -52,16 +52,42 @@ type Resources struct { ShmVolume *bool `name:"enable_shm_volume" default:"true"` } +type InfrastructureRole struct { + // Name of a secret which describes the role, and optionally name of a + // configmap with an extra information + SecretName spec.NamespacedName + + UserKey string + PasswordKey string + RoleKey string + + // This field point out the detailed yaml definition of the role, if exists + Details string + + // Specify if a secret contains multiple fields in the following format: + // + // %(userkey)idx: ... + // %(passwordkey)idx: ... + // %(rolekey)idx: ... + // + // If it does, Name/Password/Role are interpreted not as unique field + // names, but as a template. + + Template bool +} + // Auth describes authentication specific configuration parameters type Auth struct { - SecretNameTemplate StringTemplate `name:"secret_name_template" default:"{username}.{cluster}.credentials.{tprkind}.{tprgroup}"` - PamRoleName string `name:"pam_role_name" default:"zalandos"` - PamConfiguration string `name:"pam_configuration" default:"https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees"` - TeamsAPIUrl string `name:"teams_api_url" default:"https://teams.example.com/api/"` - OAuthTokenSecretName spec.NamespacedName `name:"oauth_token_secret_name" default:"postgresql-operator"` - InfrastructureRolesSecretName spec.NamespacedName `name:"infrastructure_roles_secret_name"` - SuperUsername string `name:"super_username" default:"postgres"` - ReplicationUsername string `name:"replication_username" default:"standby"` + SecretNameTemplate StringTemplate `name:"secret_name_template" default:"{username}.{cluster}.credentials.{tprkind}.{tprgroup}"` + PamRoleName string `name:"pam_role_name" default:"zalandos"` + PamConfiguration string `name:"pam_configuration" default:"https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees"` + TeamsAPIUrl string `name:"teams_api_url" default:"https://teams.example.com/api/"` + OAuthTokenSecretName spec.NamespacedName `name:"oauth_token_secret_name" default:"postgresql-operator"` + InfrastructureRolesSecretName spec.NamespacedName `name:"infrastructure_roles_secret_name"` + InfrastructureRoles []*InfrastructureRole `name:"-"` + InfrastructureRolesDefs string `name:"infrastructure_roles_secrets"` + SuperUsername string `name:"super_username" default:"postgres"` + ReplicationUsername string `name:"replication_username" default:"standby"` } // Scalyr holds the configuration for the Scalyr Agent sidecar for log shipping: diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index 5cde1c3e8..1234ef74a 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -271,31 +271,73 @@ func SameLogicalBackupJob(cur, new *batchv1beta1.CronJob) (match bool, reason st } func (c *mockSecret) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.Secret, error) { - if name != "infrastructureroles-test" { - return nil, fmt.Errorf("NotFound") - } - secret := &v1.Secret{} - secret.Name = "testcluster" - secret.Data = map[string][]byte{ + oldFormatSecret := &v1.Secret{} + oldFormatSecret.Name = "testcluster" + oldFormatSecret.Data = map[string][]byte{ "user1": []byte("testrole"), "password1": []byte("testpassword"), "inrole1": []byte("testinrole"), "foobar": []byte(b64.StdEncoding.EncodeToString([]byte("password"))), } - return secret, nil + + newFormatSecret := &v1.Secret{} + newFormatSecret.Name = "test-secret-new-format" + newFormatSecret.Data = map[string][]byte{ + "user": []byte("new-test-role"), + "password": []byte("new-test-password"), + "inrole": []byte("new-test-inrole"), + "new-foobar": []byte(b64.StdEncoding.EncodeToString([]byte("password"))), + } + + secrets := map[string]*v1.Secret{ + "infrastructureroles-old-test": oldFormatSecret, + "infrastructureroles-new-test": newFormatSecret, + } + + for idx := 1; idx <= 2; idx++ { + newFormatStandaloneSecret := &v1.Secret{} + newFormatStandaloneSecret.Name = fmt.Sprintf("test-secret-new-format%d", idx) + newFormatStandaloneSecret.Data = map[string][]byte{ + "user": []byte(fmt.Sprintf("new-test-role%d", idx)), + "password": []byte(fmt.Sprintf("new-test-password%d", idx)), + "inrole": []byte(fmt.Sprintf("new-test-inrole%d", idx)), + } + + secrets[fmt.Sprintf("infrastructureroles-new-test%d", idx)] = + newFormatStandaloneSecret + } + + if secret, exists := secrets[name]; exists { + return secret, nil + } + + return nil, fmt.Errorf("NotFound") } func (c *mockConfigMap) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.ConfigMap, error) { - if name != "infrastructureroles-test" { - return nil, fmt.Errorf("NotFound") - } - configmap := &v1.ConfigMap{} - configmap.Name = "testcluster" - configmap.Data = map[string]string{ + oldFormatConfigmap := &v1.ConfigMap{} + oldFormatConfigmap.Name = "testcluster" + oldFormatConfigmap.Data = map[string]string{ "foobar": "{}", } - return configmap, nil + + newFormatConfigmap := &v1.ConfigMap{} + newFormatConfigmap.Name = "testcluster" + newFormatConfigmap.Data = map[string]string{ + "new-foobar": "{\"user_flags\": [\"createdb\"]}", + } + + configmaps := map[string]*v1.ConfigMap{ + "infrastructureroles-old-test": oldFormatConfigmap, + "infrastructureroles-new-test": newFormatConfigmap, + } + + if configmap, exists := configmaps[name]; exists { + return configmap, nil + } + + return nil, fmt.Errorf("NotFound") } // Secrets to be mocked From 43163cf83b463e5350073a8acb4bdb622a95b9a4 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 10 Aug 2020 15:08:03 +0200 Subject: [PATCH 073/168] allow using both infrastructure_roles_options (#1090) * allow using both infrastructure_roles_options * new default values for user and role definition * use robot_zmon as parent role * add operator log to debug * right name for old secret * only extract if rolesDefs is empty * set password1 in old infrastructure role * fix new infra rile secret * choose different role key for new secret * set memberof everywhere * reenable all tests * reflect feedback * remove condition for rolesDefs --- .../crds/operatorconfigurations.yaml | 4 + e2e/tests/test_e2e.py | 1007 +++++++++-------- manifests/infrastructure-roles-new.yaml | 2 - manifests/infrastructure-roles.yaml | 6 +- manifests/operatorconfiguration.crd.yaml | 4 + pkg/apis/acid.zalan.do/v1/crds.go | 6 + pkg/cluster/resources.go | 2 +- pkg/controller/util.go | 38 +- pkg/controller/util_test.go | 121 +- pkg/util/config/config.go | 3 + 10 files changed, 650 insertions(+), 543 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 3218decd7..4dde1fc23 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -149,6 +149,10 @@ spec: type: string rolekey: type: string + defaultuservalue: + type: string + defaultrolevalue: + type: string details: type: string template: diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 4cd1c6a30..9f0b946c9 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -52,6 +52,7 @@ class EndToEndTestCase(unittest.TestCase): for filename in ["operator-service-account-rbac.yaml", "configmap.yaml", "postgres-operator.yaml", + "infrastructure-roles.yaml", "infrastructure-roles-new.yaml"]: result = k8s.create_with_kubectl("manifests/" + filename) print("stdout: {}, stderr: {}".format(result.stdout, result.stderr)) @@ -71,506 +72,506 @@ class EndToEndTestCase(unittest.TestCase): print('Operator log: {}'.format(k8s.get_operator_log())) raise - # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - # def test_enable_disable_connection_pooler(self): - # ''' - # For a database without connection pooler, then turns it on, scale up, - # turn off and on again. Test with different ways of doing this (via - # enableConnectionPooler or connectionPooler configuration section). At - # the end turn connection pooler off to not interfere with other tests. - # ''' - # k8s = self.k8s - # service_labels = { - # 'cluster-name': 'acid-minimal-cluster', - # } - # pod_labels = dict({ - # 'connection-pooler': 'acid-minimal-cluster-pooler', - # }) - - # pod_selector = to_selector(pod_labels) - # service_selector = to_selector(service_labels) - - # try: - # # enable connection pooler - # k8s.api.custom_objects_api.patch_namespaced_custom_object( - # 'acid.zalan.do', 'v1', 'default', - # 'postgresqls', 'acid-minimal-cluster', - # { - # 'spec': { - # 'enableConnectionPooler': True, - # } - # }) - # k8s.wait_for_pod_start(pod_selector) - - # pods = k8s.api.core_v1.list_namespaced_pod( - # 'default', label_selector=pod_selector - # ).items - - # self.assertTrue(pods, 'No connection pooler pods') - - # k8s.wait_for_service(service_selector) - # services = k8s.api.core_v1.list_namespaced_service( - # 'default', label_selector=service_selector - # ).items - # services = [ - # s for s in services - # if s.metadata.name.endswith('pooler') - # ] - - # self.assertTrue(services, 'No connection pooler service') - - # # scale up connection pooler deployment - # k8s.api.custom_objects_api.patch_namespaced_custom_object( - # 'acid.zalan.do', 'v1', 'default', - # 'postgresqls', 'acid-minimal-cluster', - # { - # 'spec': { - # 'connectionPooler': { - # 'numberOfInstances': 2, - # }, - # } - # }) - - # k8s.wait_for_running_pods(pod_selector, 2) - - # # turn it off, keeping configuration section - # k8s.api.custom_objects_api.patch_namespaced_custom_object( - # 'acid.zalan.do', 'v1', 'default', - # 'postgresqls', 'acid-minimal-cluster', - # { - # 'spec': { - # 'enableConnectionPooler': False, - # } - # }) - # k8s.wait_for_pods_to_stop(pod_selector) - - # except timeout_decorator.TimeoutError: - # print('Operator log: {}'.format(k8s.get_operator_log())) - # raise - - # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - # def test_enable_load_balancer(self): - # ''' - # Test if services are updated when enabling/disabling load balancers - # ''' - - # k8s = self.k8s - # cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' - - # # enable load balancer services - # pg_patch_enable_lbs = { - # "spec": { - # "enableMasterLoadBalancer": True, - # "enableReplicaLoadBalancer": True - # } - # } - # k8s.api.custom_objects_api.patch_namespaced_custom_object( - # "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_lbs) - # # wait for service recreation - # time.sleep(60) - - # master_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=master') - # self.assertEqual(master_svc_type, 'LoadBalancer', - # "Expected LoadBalancer service type for master, found {}".format(master_svc_type)) - - # repl_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=replica') - # self.assertEqual(repl_svc_type, 'LoadBalancer', - # "Expected LoadBalancer service type for replica, found {}".format(repl_svc_type)) - - # # disable load balancer services again - # pg_patch_disable_lbs = { - # "spec": { - # "enableMasterLoadBalancer": False, - # "enableReplicaLoadBalancer": False - # } - # } - # k8s.api.custom_objects_api.patch_namespaced_custom_object( - # "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_lbs) - # # wait for service recreation - # time.sleep(60) - - # master_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=master') - # self.assertEqual(master_svc_type, 'ClusterIP', - # "Expected ClusterIP service type for master, found {}".format(master_svc_type)) - - # repl_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=replica') - # self.assertEqual(repl_svc_type, 'ClusterIP', - # "Expected ClusterIP service type for replica, found {}".format(repl_svc_type)) - - # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - # def test_lazy_spilo_upgrade(self): - # ''' - # Test lazy upgrade for the Spilo image: operator changes a stateful set but lets pods run with the old image - # until they are recreated for reasons other than operator's activity. That works because the operator configures - # stateful sets to use "onDelete" pod update policy. - - # The test covers: - # 1) enabling lazy upgrade in existing operator deployment - # 2) forcing the normal rolling upgrade by changing the operator configmap and restarting its pod - # ''' - - # k8s = self.k8s - - # # update docker image in config and enable the lazy upgrade - # conf_image = "registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p114" - # patch_lazy_spilo_upgrade = { - # "data": { - # "docker_image": conf_image, - # "enable_lazy_spilo_upgrade": "true" - # } - # } - # k8s.update_config(patch_lazy_spilo_upgrade) - - # pod0 = 'acid-minimal-cluster-0' - # pod1 = 'acid-minimal-cluster-1' - - # # restart the pod to get a container with the new image - # k8s.api.core_v1.delete_namespaced_pod(pod0, 'default') - # time.sleep(60) - - # # lazy update works if the restarted pod and older pods run different Spilo versions - # new_image = k8s.get_effective_pod_image(pod0) - # old_image = k8s.get_effective_pod_image(pod1) - # self.assertNotEqual(new_image, old_image, "Lazy updated failed: pods have the same image {}".format(new_image)) - - # # sanity check - # assert_msg = "Image {} of a new pod differs from {} in operator conf".format(new_image, conf_image) - # self.assertEqual(new_image, conf_image, assert_msg) - - # # clean up - # unpatch_lazy_spilo_upgrade = { - # "data": { - # "enable_lazy_spilo_upgrade": "false", - # } - # } - # k8s.update_config(unpatch_lazy_spilo_upgrade) - - # # at this point operator will complete the normal rolling upgrade - # # so we additonally test if disabling the lazy upgrade - forcing the normal rolling upgrade - works - - # # XXX there is no easy way to wait until the end of Sync() - # time.sleep(60) - - # image0 = k8s.get_effective_pod_image(pod0) - # image1 = k8s.get_effective_pod_image(pod1) - - # assert_msg = "Disabling lazy upgrade failed: pods still have different images {} and {}".format(image0, image1) - # self.assertEqual(image0, image1, assert_msg) - - # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - # def test_logical_backup_cron_job(self): - # ''' - # Ensure we can (a) create the cron job at user request for a specific PG cluster - # (b) update the cluster-wide image for the logical backup pod - # (c) delete the job at user request - - # Limitations: - # (a) Does not run the actual batch job because there is no S3 mock to upload backups to - # (b) Assumes 'acid-minimal-cluster' exists as defined in setUp - # ''' - - # k8s = self.k8s - - # # create the cron job - # schedule = "7 7 7 7 *" - # pg_patch_enable_backup = { - # "spec": { - # "enableLogicalBackup": True, - # "logicalBackupSchedule": schedule - # } - # } - # k8s.api.custom_objects_api.patch_namespaced_custom_object( - # "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_backup) - # k8s.wait_for_logical_backup_job_creation() - - # jobs = k8s.get_logical_backup_job().items - # self.assertEqual(1, len(jobs), "Expected 1 logical backup job, found {}".format(len(jobs))) - - # job = jobs[0] - # self.assertEqual(job.metadata.name, "logical-backup-acid-minimal-cluster", - # "Expected job name {}, found {}" - # .format("logical-backup-acid-minimal-cluster", job.metadata.name)) - # self.assertEqual(job.spec.schedule, schedule, - # "Expected {} schedule, found {}" - # .format(schedule, job.spec.schedule)) - - # # update the cluster-wide image of the logical backup pod - # image = "test-image-name" - # patch_logical_backup_image = { - # "data": { - # "logical_backup_docker_image": image, - # } - # } - # k8s.update_config(patch_logical_backup_image) - - # jobs = k8s.get_logical_backup_job().items - # actual_image = jobs[0].spec.job_template.spec.template.spec.containers[0].image - # self.assertEqual(actual_image, image, - # "Expected job image {}, found {}".format(image, actual_image)) - - # # delete the logical backup cron job - # pg_patch_disable_backup = { - # "spec": { - # "enableLogicalBackup": False, - # } - # } - # k8s.api.custom_objects_api.patch_namespaced_custom_object( - # "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_backup) - # k8s.wait_for_logical_backup_job_deletion() - # jobs = k8s.get_logical_backup_job().items - # self.assertEqual(0, len(jobs), - # "Expected 0 logical backup jobs, found {}".format(len(jobs))) - - # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - # def test_min_resource_limits(self): - # ''' - # Lower resource limits below configured minimum and let operator fix it - # ''' - # k8s = self.k8s - # cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' - # labels = 'spilo-role=master,' + cluster_label - # _, failover_targets = k8s.get_pg_nodes(cluster_label) - - # # configure minimum boundaries for CPU and memory limits - # minCPULimit = '500m' - # minMemoryLimit = '500Mi' - # patch_min_resource_limits = { - # "data": { - # "min_cpu_limit": minCPULimit, - # "min_memory_limit": minMemoryLimit - # } - # } - # k8s.update_config(patch_min_resource_limits) - - # # lower resource limits below minimum - # pg_patch_resources = { - # "spec": { - # "resources": { - # "requests": { - # "cpu": "10m", - # "memory": "50Mi" - # }, - # "limits": { - # "cpu": "200m", - # "memory": "200Mi" - # } - # } - # } - # } - # k8s.api.custom_objects_api.patch_namespaced_custom_object( - # "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_resources) - # k8s.wait_for_pod_failover(failover_targets, labels) - # k8s.wait_for_pod_start('spilo-role=replica') - - # pods = k8s.api.core_v1.list_namespaced_pod( - # 'default', label_selector=labels).items - # self.assert_master_is_unique() - # masterPod = pods[0] - - # self.assertEqual(masterPod.spec.containers[0].resources.limits['cpu'], minCPULimit, - # "Expected CPU limit {}, found {}" - # .format(minCPULimit, masterPod.spec.containers[0].resources.limits['cpu'])) - # self.assertEqual(masterPod.spec.containers[0].resources.limits['memory'], minMemoryLimit, - # "Expected memory limit {}, found {}" - # .format(minMemoryLimit, masterPod.spec.containers[0].resources.limits['memory'])) - - # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - # def test_multi_namespace_support(self): - # ''' - # Create a customized Postgres cluster in a non-default namespace. - # ''' - # k8s = self.k8s - - # with open("manifests/complete-postgres-manifest.yaml", 'r+') as f: - # pg_manifest = yaml.safe_load(f) - # pg_manifest["metadata"]["namespace"] = self.namespace - # yaml.dump(pg_manifest, f, Dumper=yaml.Dumper) - - # k8s.create_with_kubectl("manifests/complete-postgres-manifest.yaml") - # k8s.wait_for_pod_start("spilo-role=master", self.namespace) - # self.assert_master_is_unique(self.namespace, "acid-test-cluster") - - # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - # def test_node_readiness_label(self): - # ''' - # Remove node readiness label from master node. This must cause a failover. - # ''' - # k8s = self.k8s - # cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' - # readiness_label = 'lifecycle-status' - # readiness_value = 'ready' - - # # get nodes of master and replica(s) (expected target of new master) - # current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label) - # num_replicas = len(current_replica_nodes) - # failover_targets = self.get_failover_targets(current_master_node, current_replica_nodes) - - # # add node_readiness_label to potential failover nodes - # patch_readiness_label = { - # "metadata": { - # "labels": { - # readiness_label: readiness_value - # } - # } - # } - # for failover_target in failover_targets: - # k8s.api.core_v1.patch_node(failover_target, patch_readiness_label) - - # # define node_readiness_label in config map which should trigger a failover of the master - # patch_readiness_label_config = { - # "data": { - # "node_readiness_label": readiness_label + ':' + readiness_value, - # } - # } - # k8s.update_config(patch_readiness_label_config) - # new_master_node, new_replica_nodes = self.assert_failover( - # current_master_node, num_replicas, failover_targets, cluster_label) - - # # patch also node where master ran before - # k8s.api.core_v1.patch_node(current_master_node, patch_readiness_label) - - # # wait a little before proceeding with the pod distribution test - # time.sleep(30) - - # # toggle pod anti affinity to move replica away from master node - # self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) - - # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - # def test_scaling(self): - # ''' - # Scale up from 2 to 3 and back to 2 pods by updating the Postgres manifest at runtime. - # ''' - # k8s = self.k8s - # labels = "application=spilo,cluster-name=acid-minimal-cluster" - - # k8s.wait_for_pg_to_scale(3) - # self.assertEqual(3, k8s.count_pods_with_label(labels)) - # self.assert_master_is_unique() - - # k8s.wait_for_pg_to_scale(2) - # self.assertEqual(2, k8s.count_pods_with_label(labels)) - # self.assert_master_is_unique() - - # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - # def test_service_annotations(self): - # ''' - # Create a Postgres cluster with service annotations and check them. - # ''' - # k8s = self.k8s - # patch_custom_service_annotations = { - # "data": { - # "custom_service_annotations": "foo:bar", - # } - # } - # k8s.update_config(patch_custom_service_annotations) - - # pg_patch_custom_annotations = { - # "spec": { - # "serviceAnnotations": { - # "annotation.key": "value", - # "foo": "bar", - # } - # } - # } - # k8s.api.custom_objects_api.patch_namespaced_custom_object( - # "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_custom_annotations) - - # # wait a little before proceeding - # time.sleep(30) - # annotations = { - # "annotation.key": "value", - # "foo": "bar", - # } - # self.assertTrue(k8s.check_service_annotations( - # "cluster-name=acid-minimal-cluster,spilo-role=master", annotations)) - # self.assertTrue(k8s.check_service_annotations( - # "cluster-name=acid-minimal-cluster,spilo-role=replica", annotations)) - - # # clean up - # unpatch_custom_service_annotations = { - # "data": { - # "custom_service_annotations": "", - # } - # } - # k8s.update_config(unpatch_custom_service_annotations) - - # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - # def test_statefulset_annotation_propagation(self): - # ''' - # Inject annotation to Postgresql CRD and check it's propagation to stateful set - # ''' - # k8s = self.k8s - # cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' - - # patch_sset_propagate_annotations = { - # "data": { - # "downscaler_annotations": "deployment-time,downscaler/*", - # } - # } - # k8s.update_config(patch_sset_propagate_annotations) - - # pg_crd_annotations = { - # "metadata": { - # "annotations": { - # "deployment-time": "2020-04-30 12:00:00", - # "downscaler/downtime_replicas": "0", - # }, - # } - # } - # k8s.api.custom_objects_api.patch_namespaced_custom_object( - # "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_crd_annotations) - - # # wait a little before proceeding - # time.sleep(60) - # annotations = { - # "deployment-time": "2020-04-30 12:00:00", - # "downscaler/downtime_replicas": "0", - # } - # self.assertTrue(k8s.check_statefulset_annotations(cluster_label, annotations)) - - # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - # def test_taint_based_eviction(self): - # ''' - # Add taint "postgres=:NoExecute" to node with master. This must cause a failover. - # ''' - # k8s = self.k8s - # cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' - - # # get nodes of master and replica(s) (expected target of new master) - # current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label) - # num_replicas = len(current_replica_nodes) - # failover_targets = self.get_failover_targets(current_master_node, current_replica_nodes) - - # # taint node with postgres=:NoExecute to force failover - # body = { - # "spec": { - # "taints": [ - # { - # "effect": "NoExecute", - # "key": "postgres" - # } - # ] - # } - # } - - # # patch node and test if master is failing over to one of the expected nodes - # k8s.api.core_v1.patch_node(current_master_node, body) - # new_master_node, new_replica_nodes = self.assert_failover( - # current_master_node, num_replicas, failover_targets, cluster_label) - - # # add toleration to pods - # patch_toleration_config = { - # "data": { - # "toleration": "key:postgres,operator:Exists,effect:NoExecute" - # } - # } - # k8s.update_config(patch_toleration_config) - - # # wait a little before proceeding with the pod distribution test - # time.sleep(30) - - # # toggle pod anti affinity to move replica away from master node - # self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_enable_disable_connection_pooler(self): + ''' + For a database without connection pooler, then turns it on, scale up, + turn off and on again. Test with different ways of doing this (via + enableConnectionPooler or connectionPooler configuration section). At + the end turn connection pooler off to not interfere with other tests. + ''' + k8s = self.k8s + service_labels = { + 'cluster-name': 'acid-minimal-cluster', + } + pod_labels = dict({ + 'connection-pooler': 'acid-minimal-cluster-pooler', + }) + + pod_selector = to_selector(pod_labels) + service_selector = to_selector(service_labels) + + try: + # enable connection pooler + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'enableConnectionPooler': True, + } + }) + k8s.wait_for_pod_start(pod_selector) + + pods = k8s.api.core_v1.list_namespaced_pod( + 'default', label_selector=pod_selector + ).items + + self.assertTrue(pods, 'No connection pooler pods') + + k8s.wait_for_service(service_selector) + services = k8s.api.core_v1.list_namespaced_service( + 'default', label_selector=service_selector + ).items + services = [ + s for s in services + if s.metadata.name.endswith('pooler') + ] + + self.assertTrue(services, 'No connection pooler service') + + # scale up connection pooler deployment + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'connectionPooler': { + 'numberOfInstances': 2, + }, + } + }) + + k8s.wait_for_running_pods(pod_selector, 2) + + # turn it off, keeping configuration section + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'enableConnectionPooler': False, + } + }) + k8s.wait_for_pods_to_stop(pod_selector) + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_enable_load_balancer(self): + ''' + Test if services are updated when enabling/disabling load balancers + ''' + + k8s = self.k8s + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + + # enable load balancer services + pg_patch_enable_lbs = { + "spec": { + "enableMasterLoadBalancer": True, + "enableReplicaLoadBalancer": True + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_lbs) + # wait for service recreation + time.sleep(60) + + master_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=master') + self.assertEqual(master_svc_type, 'LoadBalancer', + "Expected LoadBalancer service type for master, found {}".format(master_svc_type)) + + repl_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=replica') + self.assertEqual(repl_svc_type, 'LoadBalancer', + "Expected LoadBalancer service type for replica, found {}".format(repl_svc_type)) + + # disable load balancer services again + pg_patch_disable_lbs = { + "spec": { + "enableMasterLoadBalancer": False, + "enableReplicaLoadBalancer": False + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_lbs) + # wait for service recreation + time.sleep(60) + + master_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=master') + self.assertEqual(master_svc_type, 'ClusterIP', + "Expected ClusterIP service type for master, found {}".format(master_svc_type)) + + repl_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=replica') + self.assertEqual(repl_svc_type, 'ClusterIP', + "Expected ClusterIP service type for replica, found {}".format(repl_svc_type)) + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_lazy_spilo_upgrade(self): + ''' + Test lazy upgrade for the Spilo image: operator changes a stateful set but lets pods run with the old image + until they are recreated for reasons other than operator's activity. That works because the operator configures + stateful sets to use "onDelete" pod update policy. + + The test covers: + 1) enabling lazy upgrade in existing operator deployment + 2) forcing the normal rolling upgrade by changing the operator configmap and restarting its pod + ''' + + k8s = self.k8s + + # update docker image in config and enable the lazy upgrade + conf_image = "registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p114" + patch_lazy_spilo_upgrade = { + "data": { + "docker_image": conf_image, + "enable_lazy_spilo_upgrade": "true" + } + } + k8s.update_config(patch_lazy_spilo_upgrade) + + pod0 = 'acid-minimal-cluster-0' + pod1 = 'acid-minimal-cluster-1' + + # restart the pod to get a container with the new image + k8s.api.core_v1.delete_namespaced_pod(pod0, 'default') + time.sleep(60) + + # lazy update works if the restarted pod and older pods run different Spilo versions + new_image = k8s.get_effective_pod_image(pod0) + old_image = k8s.get_effective_pod_image(pod1) + self.assertNotEqual(new_image, old_image, "Lazy updated failed: pods have the same image {}".format(new_image)) + + # sanity check + assert_msg = "Image {} of a new pod differs from {} in operator conf".format(new_image, conf_image) + self.assertEqual(new_image, conf_image, assert_msg) + + # clean up + unpatch_lazy_spilo_upgrade = { + "data": { + "enable_lazy_spilo_upgrade": "false", + } + } + k8s.update_config(unpatch_lazy_spilo_upgrade) + + # at this point operator will complete the normal rolling upgrade + # so we additonally test if disabling the lazy upgrade - forcing the normal rolling upgrade - works + + # XXX there is no easy way to wait until the end of Sync() + time.sleep(60) + + image0 = k8s.get_effective_pod_image(pod0) + image1 = k8s.get_effective_pod_image(pod1) + + assert_msg = "Disabling lazy upgrade failed: pods still have different images {} and {}".format(image0, image1) + self.assertEqual(image0, image1, assert_msg) + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_logical_backup_cron_job(self): + ''' + Ensure we can (a) create the cron job at user request for a specific PG cluster + (b) update the cluster-wide image for the logical backup pod + (c) delete the job at user request + + Limitations: + (a) Does not run the actual batch job because there is no S3 mock to upload backups to + (b) Assumes 'acid-minimal-cluster' exists as defined in setUp + ''' + + k8s = self.k8s + + # create the cron job + schedule = "7 7 7 7 *" + pg_patch_enable_backup = { + "spec": { + "enableLogicalBackup": True, + "logicalBackupSchedule": schedule + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_backup) + k8s.wait_for_logical_backup_job_creation() + + jobs = k8s.get_logical_backup_job().items + self.assertEqual(1, len(jobs), "Expected 1 logical backup job, found {}".format(len(jobs))) + + job = jobs[0] + self.assertEqual(job.metadata.name, "logical-backup-acid-minimal-cluster", + "Expected job name {}, found {}" + .format("logical-backup-acid-minimal-cluster", job.metadata.name)) + self.assertEqual(job.spec.schedule, schedule, + "Expected {} schedule, found {}" + .format(schedule, job.spec.schedule)) + + # update the cluster-wide image of the logical backup pod + image = "test-image-name" + patch_logical_backup_image = { + "data": { + "logical_backup_docker_image": image, + } + } + k8s.update_config(patch_logical_backup_image) + + jobs = k8s.get_logical_backup_job().items + actual_image = jobs[0].spec.job_template.spec.template.spec.containers[0].image + self.assertEqual(actual_image, image, + "Expected job image {}, found {}".format(image, actual_image)) + + # delete the logical backup cron job + pg_patch_disable_backup = { + "spec": { + "enableLogicalBackup": False, + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_backup) + k8s.wait_for_logical_backup_job_deletion() + jobs = k8s.get_logical_backup_job().items + self.assertEqual(0, len(jobs), + "Expected 0 logical backup jobs, found {}".format(len(jobs))) + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_min_resource_limits(self): + ''' + Lower resource limits below configured minimum and let operator fix it + ''' + k8s = self.k8s + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + labels = 'spilo-role=master,' + cluster_label + _, failover_targets = k8s.get_pg_nodes(cluster_label) + + # configure minimum boundaries for CPU and memory limits + minCPULimit = '500m' + minMemoryLimit = '500Mi' + patch_min_resource_limits = { + "data": { + "min_cpu_limit": minCPULimit, + "min_memory_limit": minMemoryLimit + } + } + k8s.update_config(patch_min_resource_limits) + + # lower resource limits below minimum + pg_patch_resources = { + "spec": { + "resources": { + "requests": { + "cpu": "10m", + "memory": "50Mi" + }, + "limits": { + "cpu": "200m", + "memory": "200Mi" + } + } + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_resources) + k8s.wait_for_pod_failover(failover_targets, labels) + k8s.wait_for_pod_start('spilo-role=replica') + + pods = k8s.api.core_v1.list_namespaced_pod( + 'default', label_selector=labels).items + self.assert_master_is_unique() + masterPod = pods[0] + + self.assertEqual(masterPod.spec.containers[0].resources.limits['cpu'], minCPULimit, + "Expected CPU limit {}, found {}" + .format(minCPULimit, masterPod.spec.containers[0].resources.limits['cpu'])) + self.assertEqual(masterPod.spec.containers[0].resources.limits['memory'], minMemoryLimit, + "Expected memory limit {}, found {}" + .format(minMemoryLimit, masterPod.spec.containers[0].resources.limits['memory'])) + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_multi_namespace_support(self): + ''' + Create a customized Postgres cluster in a non-default namespace. + ''' + k8s = self.k8s + + with open("manifests/complete-postgres-manifest.yaml", 'r+') as f: + pg_manifest = yaml.safe_load(f) + pg_manifest["metadata"]["namespace"] = self.namespace + yaml.dump(pg_manifest, f, Dumper=yaml.Dumper) + + k8s.create_with_kubectl("manifests/complete-postgres-manifest.yaml") + k8s.wait_for_pod_start("spilo-role=master", self.namespace) + self.assert_master_is_unique(self.namespace, "acid-test-cluster") + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_node_readiness_label(self): + ''' + Remove node readiness label from master node. This must cause a failover. + ''' + k8s = self.k8s + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + readiness_label = 'lifecycle-status' + readiness_value = 'ready' + + # get nodes of master and replica(s) (expected target of new master) + current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label) + num_replicas = len(current_replica_nodes) + failover_targets = self.get_failover_targets(current_master_node, current_replica_nodes) + + # add node_readiness_label to potential failover nodes + patch_readiness_label = { + "metadata": { + "labels": { + readiness_label: readiness_value + } + } + } + for failover_target in failover_targets: + k8s.api.core_v1.patch_node(failover_target, patch_readiness_label) + + # define node_readiness_label in config map which should trigger a failover of the master + patch_readiness_label_config = { + "data": { + "node_readiness_label": readiness_label + ':' + readiness_value, + } + } + k8s.update_config(patch_readiness_label_config) + new_master_node, new_replica_nodes = self.assert_failover( + current_master_node, num_replicas, failover_targets, cluster_label) + + # patch also node where master ran before + k8s.api.core_v1.patch_node(current_master_node, patch_readiness_label) + + # wait a little before proceeding with the pod distribution test + time.sleep(30) + + # toggle pod anti affinity to move replica away from master node + self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_scaling(self): + ''' + Scale up from 2 to 3 and back to 2 pods by updating the Postgres manifest at runtime. + ''' + k8s = self.k8s + labels = "application=spilo,cluster-name=acid-minimal-cluster" + + k8s.wait_for_pg_to_scale(3) + self.assertEqual(3, k8s.count_pods_with_label(labels)) + self.assert_master_is_unique() + + k8s.wait_for_pg_to_scale(2) + self.assertEqual(2, k8s.count_pods_with_label(labels)) + self.assert_master_is_unique() + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_service_annotations(self): + ''' + Create a Postgres cluster with service annotations and check them. + ''' + k8s = self.k8s + patch_custom_service_annotations = { + "data": { + "custom_service_annotations": "foo:bar", + } + } + k8s.update_config(patch_custom_service_annotations) + + pg_patch_custom_annotations = { + "spec": { + "serviceAnnotations": { + "annotation.key": "value", + "foo": "bar", + } + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_custom_annotations) + + # wait a little before proceeding + time.sleep(30) + annotations = { + "annotation.key": "value", + "foo": "bar", + } + self.assertTrue(k8s.check_service_annotations( + "cluster-name=acid-minimal-cluster,spilo-role=master", annotations)) + self.assertTrue(k8s.check_service_annotations( + "cluster-name=acid-minimal-cluster,spilo-role=replica", annotations)) + + # clean up + unpatch_custom_service_annotations = { + "data": { + "custom_service_annotations": "", + } + } + k8s.update_config(unpatch_custom_service_annotations) + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_statefulset_annotation_propagation(self): + ''' + Inject annotation to Postgresql CRD and check it's propagation to stateful set + ''' + k8s = self.k8s + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + + patch_sset_propagate_annotations = { + "data": { + "downscaler_annotations": "deployment-time,downscaler/*", + } + } + k8s.update_config(patch_sset_propagate_annotations) + + pg_crd_annotations = { + "metadata": { + "annotations": { + "deployment-time": "2020-04-30 12:00:00", + "downscaler/downtime_replicas": "0", + }, + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_crd_annotations) + + # wait a little before proceeding + time.sleep(60) + annotations = { + "deployment-time": "2020-04-30 12:00:00", + "downscaler/downtime_replicas": "0", + } + self.assertTrue(k8s.check_statefulset_annotations(cluster_label, annotations)) + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_taint_based_eviction(self): + ''' + Add taint "postgres=:NoExecute" to node with master. This must cause a failover. + ''' + k8s = self.k8s + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + + # get nodes of master and replica(s) (expected target of new master) + current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label) + num_replicas = len(current_replica_nodes) + failover_targets = self.get_failover_targets(current_master_node, current_replica_nodes) + + # taint node with postgres=:NoExecute to force failover + body = { + "spec": { + "taints": [ + { + "effect": "NoExecute", + "key": "postgres" + } + ] + } + } + + # patch node and test if master is failing over to one of the expected nodes + k8s.api.core_v1.patch_node(current_master_node, body) + new_master_node, new_replica_nodes = self.assert_failover( + current_master_node, num_replicas, failover_targets, cluster_label) + + # add toleration to pods + patch_toleration_config = { + "data": { + "toleration": "key:postgres,operator:Exists,effect:NoExecute" + } + } + k8s.update_config(patch_toleration_config) + + # wait a little before proceeding with the pod distribution test + time.sleep(30) + + # toggle pod anti affinity to move replica away from master node + self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_infrastructure_roles(self): @@ -579,8 +580,8 @@ class EndToEndTestCase(unittest.TestCase): ''' k8s = self.k8s # update infrastructure roles description - secret_name = "postgresql-infrastructure-roles-old" - roles = "secretname: postgresql-infrastructure-roles-new, userkey: user, rolekey: role, passwordkey: password" + secret_name = "postgresql-infrastructure-roles" + roles = "secretname: postgresql-infrastructure-roles-new, userkey: user, rolekey: memberof, passwordkey: password, defaultrolevalue: robot_zmon" patch_infrastructure_roles = { "data": { "infrastructure_roles_secret_name": secret_name, @@ -607,7 +608,7 @@ class EndToEndTestCase(unittest.TestCase): self.assertDictEqual(role, { "Name": "robot_zmon_acid_monitoring_new", "Flags": None, - "MemberOf": ["robot_zmon_new"], + "MemberOf": ["robot_zmon"], "Parameters": None, "AdminRole": "", "Origin": 2, diff --git a/manifests/infrastructure-roles-new.yaml b/manifests/infrastructure-roles-new.yaml index e4f378396..64b854c6a 100644 --- a/manifests/infrastructure-roles-new.yaml +++ b/manifests/infrastructure-roles-new.yaml @@ -3,8 +3,6 @@ data: # infrastructure role definition in the new format # robot_zmon_acid_monitoring_new user: cm9ib3Rfem1vbl9hY2lkX21vbml0b3JpbmdfbmV3 - # robot_zmon_new - role: cm9ib3Rfem1vbl9uZXc= # foobar_new password: Zm9vYmFyX25ldw== kind: Secret diff --git a/manifests/infrastructure-roles.yaml b/manifests/infrastructure-roles.yaml index 3c2d86850..c66d79139 100644 --- a/manifests/infrastructure-roles.yaml +++ b/manifests/infrastructure-roles.yaml @@ -7,12 +7,14 @@ data: # provide other options in the configmap. # robot_zmon_acid_monitoring user1: cm9ib3Rfem1vbl9hY2lkX21vbml0b3Jpbmc= + # foobar + password1: Zm9vYmFy # robot_zmon inrole1: cm9ib3Rfem1vbg== # testuser user2: dGVzdHVzZXI= - # foobar - password2: Zm9vYmFy + # testpassword + password2: dGVzdHBhc3N3b3Jk # user batman with the password justice # look for other fields in the infrastructure roles configmap batman: anVzdGljZQ== diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 55b7653ef..95c4678a8 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -145,6 +145,10 @@ spec: type: string rolekey: type: string + defaultuservalue: + type: string + defaultrolevalue: + type: string details: type: string template: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index c22ed25c0..b5695bb4e 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -930,6 +930,12 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation "rolekey": { Type: "string", }, + "defaultuservalue": { + Type: "string", + }, + "defaultrolevalue": { + Type: "string", + }, "details": { Type: "string", }, diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index c75457a5a..3066b78c6 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -207,7 +207,7 @@ func (c *Cluster) deleteConnectionPooler() (err error) { serviceName = service.Name } - // set delete propagation policy to foreground, so that all the dependant + // set delete propagation policy to foreground, so that all the dependent // will be deleted. err = c.KubeClient. Services(c.Namespace). diff --git a/pkg/controller/util.go b/pkg/controller/util.go index 6035903dd..e460db2a5 100644 --- a/pkg/controller/util.go +++ b/pkg/controller/util.go @@ -15,6 +15,7 @@ import ( acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/cluster" "github.com/zalando/postgres-operator/pkg/spec" + "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/k8sutil" "gopkg.in/yaml.v2" @@ -118,13 +119,9 @@ var emptyName = (spec.NamespacedName{}) // configuration in ConfigMap & CRD. func (c *Controller) getInfrastructureRoleDefinitions() []*config.InfrastructureRole { var roleDef config.InfrastructureRole - rolesDefs := c.opConfig.InfrastructureRoles - if c.opConfig.InfrastructureRolesSecretName == emptyName { - // All the other possibilities require secret name to be present, so if - // it is not, then nothing else to be done here. - return rolesDefs - } + // take from CRD configuration + rolesDefs := c.opConfig.InfrastructureRoles // check if we can extract something from the configmap config option if c.opConfig.InfrastructureRolesDefs != "" { @@ -163,27 +160,33 @@ func (c *Controller) getInfrastructureRoleDefinitions() []*config.Infrastructure roleDef.PasswordKey = value case "rolekey": roleDef.RoleKey = value + case "defaultuservalue": + roleDef.DefaultUserValue = value + case "defaultrolevalue": + roleDef.DefaultRoleValue = value default: c.logger.Warningf("Role description is not known: %s", properties) } } - } else { + + if roleDef.SecretName != emptyName && + (roleDef.UserKey != "" || roleDef.DefaultUserValue != "") && + roleDef.PasswordKey != "" { + rolesDefs = append(rolesDefs, &roleDef) + } + } + + if c.opConfig.InfrastructureRolesSecretName != emptyName { // At this point we deal with the old format, let's replicate it // via existing definition structure and remember that it's just a // template, the real values are in user1,password1,inrole1 etc. - roleDef = config.InfrastructureRole{ + rolesDefs = append(rolesDefs, &config.InfrastructureRole{ SecretName: c.opConfig.InfrastructureRolesSecretName, UserKey: "user", PasswordKey: "password", RoleKey: "inrole", Template: true, - } - } - - if roleDef.UserKey != "" && - roleDef.PasswordKey != "" && - roleDef.RoleKey != "" { - rolesDefs = append(rolesDefs, &roleDef) + }) } return rolesDefs @@ -330,9 +333,10 @@ func (c *Controller) getInfrastructureRole( return nil, fmt.Errorf("could not decode yaml role: %v", err) } } else { - roleDescr.Name = string(secretData[infraRole.UserKey]) + roleDescr.Name = util.Coalesce(string(secretData[infraRole.UserKey]), infraRole.DefaultUserValue) roleDescr.Password = string(secretData[infraRole.PasswordKey]) - roleDescr.MemberOf = append(roleDescr.MemberOf, string(secretData[infraRole.RoleKey])) + roleDescr.MemberOf = append(roleDescr.MemberOf, + util.Coalesce(string(secretData[infraRole.RoleKey]), infraRole.DefaultRoleValue)) } if roleDescr.Valid() { diff --git a/pkg/controller/util_test.go b/pkg/controller/util_test.go index fd756a0c7..edc05d67e 100644 --- a/pkg/controller/util_test.go +++ b/pkg/controller/util_test.go @@ -279,7 +279,7 @@ func TestInfrastructureRoleDefinitions(t *testing.T) { roleSecrets string expectedDefs []*config.InfrastructureRole }{ - // only new format + // only new CRD format { []*config.InfrastructureRole{ &config.InfrastructureRole{ @@ -287,9 +287,9 @@ func TestInfrastructureRoleDefinitions(t *testing.T) { Namespace: v1.NamespaceDefault, Name: testInfrastructureRolesNewSecretName, }, - UserKey: "user", - PasswordKey: "password", - RoleKey: "inrole", + UserKey: "test-user", + PasswordKey: "test-password", + RoleKey: "test-role", Template: false, }, }, @@ -301,14 +301,50 @@ func TestInfrastructureRoleDefinitions(t *testing.T) { Namespace: v1.NamespaceDefault, Name: testInfrastructureRolesNewSecretName, }, - UserKey: "user", - PasswordKey: "password", - RoleKey: "inrole", + UserKey: "test-user", + PasswordKey: "test-password", + RoleKey: "test-role", Template: false, }, }, }, - // only old format + // only new configmap format + { + []*config.InfrastructureRole{}, + spec.NamespacedName{}, + "secretname: infrastructureroles-new-test, userkey: test-user, passwordkey: test-password, rolekey: test-role", + []*config.InfrastructureRole{ + &config.InfrastructureRole{ + SecretName: spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesNewSecretName, + }, + UserKey: "test-user", + PasswordKey: "test-password", + RoleKey: "test-role", + Template: false, + }, + }, + }, + // new configmap format with defaultRoleValue + { + []*config.InfrastructureRole{}, + spec.NamespacedName{}, + "secretname: infrastructureroles-new-test, userkey: test-user, passwordkey: test-password, defaultrolevalue: test-role", + []*config.InfrastructureRole{ + &config.InfrastructureRole{ + SecretName: spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesNewSecretName, + }, + UserKey: "test-user", + PasswordKey: "test-password", + DefaultRoleValue: "test-role", + Template: false, + }, + }, + }, + // only old CRD and configmap format { []*config.InfrastructureRole{}, spec.NamespacedName{ @@ -329,19 +365,13 @@ func TestInfrastructureRoleDefinitions(t *testing.T) { }, }, }, - // only configmap format + // both formats for CRD { - []*config.InfrastructureRole{}, - spec.NamespacedName{ - Namespace: v1.NamespaceDefault, - Name: testInfrastructureRolesOldSecretName, - }, - "secretname: infrastructureroles-old-test, userkey: test-user, passwordkey: test-password, rolekey: test-role, template: false", []*config.InfrastructureRole{ &config.InfrastructureRole{ SecretName: spec.NamespacedName{ Namespace: v1.NamespaceDefault, - Name: testInfrastructureRolesOldSecretName, + Name: testInfrastructureRolesNewSecretName, }, UserKey: "test-user", PasswordKey: "test-password", @@ -349,14 +379,69 @@ func TestInfrastructureRoleDefinitions(t *testing.T) { Template: false, }, }, + spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesOldSecretName, + }, + "", + []*config.InfrastructureRole{ + &config.InfrastructureRole{ + SecretName: spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesNewSecretName, + }, + UserKey: "test-user", + PasswordKey: "test-password", + RoleKey: "test-role", + Template: false, + }, + &config.InfrastructureRole{ + SecretName: spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesOldSecretName, + }, + UserKey: "user", + PasswordKey: "password", + RoleKey: "inrole", + Template: true, + }, + }, }, - // incorrect configmap format + // both formats for configmap { []*config.InfrastructureRole{}, spec.NamespacedName{ Namespace: v1.NamespaceDefault, Name: testInfrastructureRolesOldSecretName, }, + "secretname: infrastructureroles-new-test, userkey: test-user, passwordkey: test-password, rolekey: test-role", + []*config.InfrastructureRole{ + &config.InfrastructureRole{ + SecretName: spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesNewSecretName, + }, + UserKey: "test-user", + PasswordKey: "test-password", + RoleKey: "test-role", + Template: false, + }, + &config.InfrastructureRole{ + SecretName: spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesOldSecretName, + }, + UserKey: "user", + PasswordKey: "password", + RoleKey: "inrole", + Template: true, + }, + }, + }, + // incorrect configmap format + { + []*config.InfrastructureRole{}, + spec.NamespacedName{}, "wrong-format", []*config.InfrastructureRole{}, }, @@ -364,7 +449,7 @@ func TestInfrastructureRoleDefinitions(t *testing.T) { { []*config.InfrastructureRole{}, spec.NamespacedName{}, - "userkey: test-user, passwordkey: test-password, rolekey: test-role, template: false", + "userkey: test-user, passwordkey: test-password, rolekey: test-role", []*config.InfrastructureRole{}, }, } diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 5f262107f..4fe66910a 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -61,6 +61,9 @@ type InfrastructureRole struct { PasswordKey string RoleKey string + DefaultUserValue string + DefaultRoleValue string + // This field point out the detailed yaml definition of the role, if exists Details string From 0508266219c9aa8235862ecaa089e082aa97672c Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 10 Aug 2020 18:26:26 +0200 Subject: [PATCH 074/168] Remove all secrets on delete incl. pooler (#1091) * fix syncSecrets and remove pooler secret * update log for deleteSecret * use c.credentialSecretName(username) * minor fix --- go.mod | 3 ++- go.sum | 6 ++++-- pkg/cluster/cluster.go | 2 +- pkg/cluster/resources.go | 38 +++++++++++++++++++++++++++++++------- pkg/cluster/sync.go | 1 + 5 files changed, 39 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 49ba3682b..f6cc39f32 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,8 @@ require ( github.com/sirupsen/logrus v1.6.0 github.com/stretchr/testify v1.5.1 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 - golang.org/x/tools v0.0.0-20200729041821-df70183b1872 // indirect + golang.org/x/tools v0.0.0-20200809012840-6f4f008689da // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gopkg.in/yaml.v2 v2.2.8 k8s.io/api v0.18.6 k8s.io/apiextensions-apiserver v0.18.6 diff --git a/go.sum b/go.sum index 389608b82..651fb34fc 100644 --- a/go.sum +++ b/go.sum @@ -395,12 +395,14 @@ golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200729041821-df70183b1872 h1:/U95VAvB4ZsR91rpZX2MwiKpejhWr+UxJ+N2VlJuESk= -golang.org/x/tools v0.0.0-20200729041821-df70183b1872/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200809012840-6f4f008689da h1:ml5G98G4/tdKT1XNq+ky5iSRdKKux0TANlLAzmXT/hg= +golang.org/x/tools v0.0.0-20200809012840-6f4f008689da/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index a88cde53e..19712cccd 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -124,7 +124,7 @@ func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgres return fmt.Sprintf("%s-%s", e.PodName, e.ResourceVersion), nil }) - password_encryption, ok := pgSpec.Spec.PostgresqlParam.Parameters["password_encryption"] + password_encryption, ok := pgSpec.Spec.PostgresqlParam.Parameters["password_encryption"] if !ok { password_encryption = "md5" } diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index 3066b78c6..a9d13c124 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -207,8 +207,6 @@ func (c *Cluster) deleteConnectionPooler() (err error) { serviceName = service.Name } - // set delete propagation policy to foreground, so that all the dependent - // will be deleted. err = c.KubeClient. Services(c.Namespace). Delete(context.TODO(), serviceName, options) @@ -221,6 +219,21 @@ func (c *Cluster) deleteConnectionPooler() (err error) { c.logger.Infof("Connection pooler service %q has been deleted", serviceName) + // Repeat the same for the secret object + secretName := c.credentialSecretName(c.OpConfig.ConnectionPooler.User) + + secret, err := c.KubeClient. + Secrets(c.Namespace). + Get(context.TODO(), secretName, metav1.GetOptions{}) + + if err != nil { + c.logger.Debugf("could not get connection pooler secret %q: %v", secretName, err) + } else { + if err = c.deleteSecret(secret.UID, *secret); err != nil { + return fmt.Errorf("could not delete pooler secret: %v", err) + } + } + c.ConnectionPooler = nil return nil } @@ -730,14 +743,11 @@ func (c *Cluster) deleteSecrets() error { var errors []string errorCount := 0 for uid, secret := range c.Secrets { - c.logger.Debugf("deleting secret %q", util.NameFromMeta(secret.ObjectMeta)) - err := c.KubeClient.Secrets(secret.Namespace).Delete(context.TODO(), secret.Name, c.deleteOptions) + err := c.deleteSecret(uid, *secret) if err != nil { - errors = append(errors, fmt.Sprintf("could not delete secret %q: %v", util.NameFromMeta(secret.ObjectMeta), err)) + errors = append(errors, fmt.Sprintf("%v", err)) errorCount++ } - c.logger.Infof("secret %q has been deleted", util.NameFromMeta(secret.ObjectMeta)) - c.Secrets[uid] = nil } if errorCount > 0 { @@ -747,6 +757,20 @@ func (c *Cluster) deleteSecrets() error { return nil } +func (c *Cluster) deleteSecret(uid types.UID, secret v1.Secret) error { + c.setProcessName("deleting secret") + secretName := util.NameFromMeta(secret.ObjectMeta) + c.logger.Debugf("deleting secret %q", secretName) + err := c.KubeClient.Secrets(secret.Namespace).Delete(context.TODO(), secret.Name, c.deleteOptions) + if err != nil { + return fmt.Errorf("could not delete secret %q: %v", secretName, err) + } + c.logger.Infof("secret %q has been deleted", secretName) + c.Secrets[uid] = nil + + return nil +} + func (c *Cluster) createRoles() (err error) { // TODO: figure out what to do with duplicate names (humans and robots) among pgUsers return c.syncRoles() diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index b03b5d494..056e43043 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -500,6 +500,7 @@ func (c *Cluster) syncSecrets() error { c.logger.Warningf("secret %q does not contain the role %q", secretSpec.Name, secretUsername) continue } + c.Secrets[secret.UID] = secret c.logger.Debugf("secret %q already exists, fetching its password", util.NameFromMeta(secret.ObjectMeta)) if secretUsername == c.systemUsers[constants.SuperuserKeyName].Name { secretUsername = constants.SuperuserKeyName From dfd0dd90ed07365dba7da8f080d266201685fa31 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 11 Aug 2020 10:42:31 +0200 Subject: [PATCH 075/168] set search_path for default roles (#1065) * set search_path for default roles * deployment back to 1.5.0 Co-authored-by: Felix Kunde --- pkg/cluster/cluster.go | 41 +++++++++++++++++++++++-------------- pkg/util/constants/roles.go | 31 ++++++++++++++-------------- pkg/util/users/users.go | 16 ++++++++++----- 3 files changed, 53 insertions(+), 35 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 19712cccd..51c5d3809 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -959,32 +959,42 @@ func (c *Cluster) initPreparedDatabaseRoles() error { } for preparedDbName, preparedDB := range c.Spec.PreparedDatabases { + // get list of prepared schemas to set in search_path + preparedSchemas := preparedDB.PreparedSchemas + if len(preparedDB.PreparedSchemas) == 0 { + preparedSchemas = map[string]acidv1.PreparedSchema{"data": {DefaultRoles: util.True()}} + } + + var searchPath strings.Builder + searchPath.WriteString(constants.DefaultSearchPath) + for preparedSchemaName := range preparedSchemas { + searchPath.WriteString(", " + preparedSchemaName) + } + // default roles per database - if err := c.initDefaultRoles(defaultRoles, "admin", preparedDbName); err != nil { + if err := c.initDefaultRoles(defaultRoles, "admin", preparedDbName, searchPath.String()); err != nil { return fmt.Errorf("could not initialize default roles for database %s: %v", preparedDbName, err) } if preparedDB.DefaultUsers { - if err := c.initDefaultRoles(defaultUsers, "admin", preparedDbName); err != nil { + if err := c.initDefaultRoles(defaultUsers, "admin", preparedDbName, searchPath.String()); err != nil { return fmt.Errorf("could not initialize default roles for database %s: %v", preparedDbName, err) } } // default roles per database schema - preparedSchemas := preparedDB.PreparedSchemas - if len(preparedDB.PreparedSchemas) == 0 { - preparedSchemas = map[string]acidv1.PreparedSchema{"data": {DefaultRoles: util.True()}} - } for preparedSchemaName, preparedSchema := range preparedSchemas { if preparedSchema.DefaultRoles == nil || *preparedSchema.DefaultRoles { if err := c.initDefaultRoles(defaultRoles, preparedDbName+constants.OwnerRoleNameSuffix, - preparedDbName+"_"+preparedSchemaName); err != nil { + preparedDbName+"_"+preparedSchemaName, + constants.DefaultSearchPath+", "+preparedSchemaName); err != nil { return fmt.Errorf("could not initialize default roles for database schema %s: %v", preparedSchemaName, err) } if preparedSchema.DefaultUsers { if err := c.initDefaultRoles(defaultUsers, preparedDbName+constants.OwnerRoleNameSuffix, - preparedDbName+"_"+preparedSchemaName); err != nil { + preparedDbName+"_"+preparedSchemaName, + constants.DefaultSearchPath+", "+preparedSchemaName); err != nil { return fmt.Errorf("could not initialize default users for database schema %s: %v", preparedSchemaName, err) } } @@ -994,7 +1004,7 @@ func (c *Cluster) initPreparedDatabaseRoles() error { return nil } -func (c *Cluster) initDefaultRoles(defaultRoles map[string]string, admin, prefix string) error { +func (c *Cluster) initDefaultRoles(defaultRoles map[string]string, admin, prefix string, searchPath string) error { for defaultRole, inherits := range defaultRoles { @@ -1018,12 +1028,13 @@ func (c *Cluster) initDefaultRoles(defaultRoles map[string]string, admin, prefix } newRole := spec.PgUser{ - Origin: spec.RoleOriginBootstrap, - Name: roleName, - Password: util.RandomPassword(constants.PasswordLength), - Flags: flags, - MemberOf: memberOf, - AdminRole: adminRole, + Origin: spec.RoleOriginBootstrap, + Name: roleName, + Password: util.RandomPassword(constants.PasswordLength), + Flags: flags, + MemberOf: memberOf, + Parameters: map[string]string{"search_path": searchPath}, + AdminRole: adminRole, } if currentRole, present := c.pgUsers[roleName]; present { c.pgUsers[roleName] = c.resolveNameConflict(¤tRole, &newRole) diff --git a/pkg/util/constants/roles.go b/pkg/util/constants/roles.go index 87c9c51ce..dd906fe80 100644 --- a/pkg/util/constants/roles.go +++ b/pkg/util/constants/roles.go @@ -2,20 +2,21 @@ package constants // Roles specific constants const ( - PasswordLength = 64 - SuperuserKeyName = "superuser" + PasswordLength = 64 + SuperuserKeyName = "superuser" ConnectionPoolerUserKeyName = "pooler" - ReplicationUserKeyName = "replication" - RoleFlagSuperuser = "SUPERUSER" - RoleFlagInherit = "INHERIT" - RoleFlagLogin = "LOGIN" - RoleFlagNoLogin = "NOLOGIN" - RoleFlagCreateRole = "CREATEROLE" - RoleFlagCreateDB = "CREATEDB" - RoleFlagReplication = "REPLICATION" - RoleFlagByPassRLS = "BYPASSRLS" - OwnerRoleNameSuffix = "_owner" - ReaderRoleNameSuffix = "_reader" - WriterRoleNameSuffix = "_writer" - UserRoleNameSuffix = "_user" + ReplicationUserKeyName = "replication" + RoleFlagSuperuser = "SUPERUSER" + RoleFlagInherit = "INHERIT" + RoleFlagLogin = "LOGIN" + RoleFlagNoLogin = "NOLOGIN" + RoleFlagCreateRole = "CREATEROLE" + RoleFlagCreateDB = "CREATEDB" + RoleFlagReplication = "REPLICATION" + RoleFlagByPassRLS = "BYPASSRLS" + OwnerRoleNameSuffix = "_owner" + ReaderRoleNameSuffix = "_reader" + WriterRoleNameSuffix = "_writer" + UserRoleNameSuffix = "_user" + DefaultSearchPath = "\"$user\"" ) diff --git a/pkg/util/users/users.go b/pkg/util/users/users.go index 166e90264..5d97336e6 100644 --- a/pkg/util/users/users.go +++ b/pkg/util/users/users.go @@ -114,14 +114,14 @@ func (strategy DefaultUserSyncStrategy) ExecuteSyncRequests(requests []spec.PgSy return nil } -func (strategy DefaultUserSyncStrategy) alterPgUserSet(user spec.PgUser, db *sql.DB) (err error) { + +func (strategy DefaultUserSyncStrategy) alterPgUserSet(user spec.PgUser, db *sql.DB) error { queries := produceAlterRoleSetStmts(user) query := fmt.Sprintf(doBlockStmt, strings.Join(queries, ";")) - if _, err = db.Exec(query); err != nil { - err = fmt.Errorf("dB error: %v, query: %s", err, query) - return + if _, err := db.Exec(query); err != nil { + return fmt.Errorf("dB error: %v, query: %s", err, query) } - return + return nil } func (strategy DefaultUserSyncStrategy) createPgUser(user spec.PgUser, db *sql.DB) error { @@ -149,6 +149,12 @@ func (strategy DefaultUserSyncStrategy) createPgUser(user spec.PgUser, db *sql.D return fmt.Errorf("dB error: %v, query: %s", err, query) } + if len(user.Parameters) > 0 { + if err := strategy.alterPgUserSet(user, db); err != nil { + return fmt.Errorf("incomplete setup for user %s: %v", user.Name, err) + } + } + return nil } From fc9ee76832612d15a569a3ccaee5f0cd834693c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sonay=20=20=C5=9Eevik?= Date: Tue, 11 Aug 2020 14:14:39 +0100 Subject: [PATCH 076/168] UI Service port forwarding internal port is updated to 80 from 8081. (#1096) Fix #1093 --- docs/quickstart.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/quickstart.md b/docs/quickstart.md index 034d32e39..16b587d84 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -160,7 +160,7 @@ You can now access the web interface by port forwarding the UI pod (mind the label selector) and enter `localhost:8081` in your browser: ```bash -kubectl port-forward svc/postgres-operator-ui 8081:8081 +kubectl port-forward svc/postgres-operator-ui 8081:80 ``` Available option are explained in detail in the [UI docs](operator-ui.md). From 808030ad1774e616405f6b6d1747519f0948c4ff Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 12 Aug 2020 15:37:40 +0200 Subject: [PATCH 077/168] update go modules (#1097) --- go.mod | 6 +++--- go.sum | 17 ++++++----------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index f6cc39f32..5d3a760df 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,14 @@ module github.com/zalando/postgres-operator go 1.14 require ( - github.com/aws/aws-sdk-go v1.32.2 - github.com/lib/pq v1.7.0 + github.com/aws/aws-sdk-go v1.34.1 + github.com/lib/pq v1.8.0 github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d github.com/r3labs/diff v1.1.0 github.com/sirupsen/logrus v1.6.0 github.com/stretchr/testify v1.5.1 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 - golang.org/x/tools v0.0.0-20200809012840-6f4f008689da // indirect + golang.org/x/tools v0.0.0-20200811032001-fd80f4dbb3ea // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gopkg.in/yaml.v2 v2.2.8 k8s.io/api v0.18.6 diff --git a/go.sum b/go.sum index 651fb34fc..a12e65ce6 100644 --- a/go.sum +++ b/go.sum @@ -26,8 +26,8 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.32.2 h1:X5/tQ4cuqCCUZgeOh41WFh9Eq5xe32JzWe4PSE2i1ME= -github.com/aws/aws-sdk-go v1.32.2/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.34.1 h1:jM0mJ9JSJyhujwxBNYKrNB8Iwp8N7J2WsQxTR4yPSck= +github.com/aws/aws-sdk-go v1.34.1/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= @@ -198,8 +198,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lib/pq v1.7.0 h1:h93mCPfUSkaul3Ka/VG8uZdmW1uMHDGxzu0NWHuJmHY= -github.com/lib/pq v1.7.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= +github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -308,9 +308,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 h1:/Tl7pH94bvbAAHBdZJT947M/+gp0+CqQXDtMRC0fseo= golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -364,10 +362,8 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f h1:25KHgbfyiSm6vwQLbM3zZIe1v9p/3ea4Rz+nnM5K/i4= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 h1:HmbHVPwrPEKPGLAcHSrMe6+hqSUlvZU0rab6x5EXfGU= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -395,8 +391,8 @@ golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200809012840-6f4f008689da h1:ml5G98G4/tdKT1XNq+ky5iSRdKKux0TANlLAzmXT/hg= -golang.org/x/tools v0.0.0-20200809012840-6f4f008689da/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200811032001-fd80f4dbb3ea h1:9ym67RBRK/wN50W0T3g8g1n8viM1D2ofgWufDlMfWe0= +golang.org/x/tools v0.0.0-20200811032001-fd80f4dbb3ea/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= @@ -440,7 +436,6 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.18.6 h1:osqrAXbOQjkKIWDTjrqxWQ3w0GkKb1KA1XkUGHHYpeE= k8s.io/api v0.18.6/go.mod h1:eeyxr+cwCjMdLAmr2W3RyDI0VvTawSg/3RFFBEnmZGI= From 0d81f972a17ef8697479e93f4ecc6df052b70729 Mon Sep 17 00:00:00 2001 From: Jan Mussler Date: Wed, 12 Aug 2020 15:45:00 +0200 Subject: [PATCH 078/168] Added build and node directory to gitignore file. (#1102) --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 559c92499..b9a730ad8 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,8 @@ __pycache__/ # Distribution / packaging .Python +ui/app/node_modules +ui/operator_ui/static/build build/ develop-eggs/ dist/ From 3ddc56e5b9236de656cd5793b10029a567ff38c1 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 13 Aug 2020 16:36:22 +0200 Subject: [PATCH 079/168] allow delete only if annotations meet configured criteria (#1069) * define annotations for delete protection * change log level and reduce log lines for e2e tests * reduce wait_for_pod_start even further --- .../crds/operatorconfigurations.yaml | 4 + charts/postgres-operator/values-crd.yaml | 6 ++ charts/postgres-operator/values.yaml | 6 ++ docs/administrator.md | 76 +++++++++++++-- docs/reference/operator_parameters.md | 10 ++ e2e/requirements.txt | 4 +- e2e/tests/test_e2e.py | 96 +++++++++++++++++-- manifests/complete-postgres-manifest.yaml | 4 +- manifests/configmap.yaml | 2 + manifests/operatorconfiguration.crd.yaml | 4 + ...gresql-operator-default-configuration.yaml | 2 + pkg/apis/acid.zalan.do/v1/crds.go | 6 ++ .../v1/operator_configuration_type.go | 2 + pkg/controller/controller.go | 32 +++++++ pkg/controller/operator_config.go | 2 + pkg/controller/postgresql.go | 17 ++++ pkg/controller/postgresql_test.go | 87 +++++++++++++++++ pkg/util/config/config.go | 2 + 18 files changed, 343 insertions(+), 19 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 4dde1fc23..c3966b410 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -117,6 +117,10 @@ spec: type: object additionalProperties: type: string + delete_annotation_date_key: + type: string + delete_annotation_name_key: + type: string downscaler_annotations: type: array items: diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 44a7f315b..9f9100cab 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -67,6 +67,12 @@ configKubernetes: # keya: valuea # keyb: valueb + # key name for annotation that compares manifest value with current date + # delete_annotation_date_key: "delete-date" + + # key name for annotation that compares manifest value with cluster name + # delete_annotation_name_key: "delete-clustername" + # list of annotations propagated from cluster manifest to statefulset and deployment # downscaler_annotations: # - deployment-time diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index b64495bee..af918a67f 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -63,6 +63,12 @@ configKubernetes: # annotations attached to each database pod # custom_pod_annotations: "keya:valuea,keyb:valueb" + # key name for annotation that compares manifest value with current date + # delete_annotation_date_key: "delete-date" + + # key name for annotation that compares manifest value with cluster name + # delete_annotation_name_key: "delete-clustername" + # list of annotations propagated from cluster manifest to statefulset and deployment # downscaler_annotations: "deployment-time,downscaler/*" diff --git a/docs/administrator.md b/docs/administrator.md index b3d4d9efa..1a1b5e8f9 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -44,7 +44,7 @@ Once the validation is enabled it can only be disabled manually by editing or patching the CRD manifest: ```bash -zk8 patch crd postgresqls.acid.zalan.do -p '{"spec":{"validation": null}}' +kubectl patch crd postgresqls.acid.zalan.do -p '{"spec":{"validation": null}}' ``` ## Non-default cluster domain @@ -123,6 +123,68 @@ Every other Postgres cluster which lacks the annotation will be ignored by this operator. Conversely, operators without a defined `CONTROLLER_ID` will ignore clusters with defined ownership of another operator. +## Delete protection via annotations + +To avoid accidental deletes of Postgres clusters the operator can check the +manifest for two existing annotations containing the cluster name and/or the +current date (in YYYY-MM-DD format). The name of the annotation keys can be +defined in the configuration. By default, they are not set which disables the +delete protection. Thus, one could choose to only go with one annotation. + +**postgres-operator ConfigMap** + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-operator +data: + delete_annotation_date_key: "delete-date" + delete_annotation_name_key: "delete-clustername" +``` + +**OperatorConfiguration** + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: OperatorConfiguration +metadata: + name: postgresql-operator-configuration +configuration: + kubernetes: + delete_annotation_date_key: "delete-date" + delete_annotation_name_key: "delete-clustername" +``` + +Now, every cluster manifest must contain the configured annotation keys to +trigger the delete process when running `kubectl delete pg`. Note, that the +`Postgresql` resource would still get deleted as K8s' API server does not +block it. Only the operator logs will tell, that the delete criteria wasn't +met. + +**cluster manifest** + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: postgresql +metadata: + name: demo-cluster + annotations: + delete-date: "2020-08-31" + delete-clustername: "demo-cluster" +spec: + ... +``` + +In case, the resource has been deleted accidentally or the annotations were +simply forgotten, it's safe to recreate the cluster with `kubectl create`. +Existing Postgres cluster are not replaced by the operator. But, as the +original cluster still exists the status will show `CreateFailed` at first. +On the next sync event it should change to `Running`. However, as it is in +fact a new resource for K8s, the UID will differ which can trigger a rolling +update of the pods because the UID is used as part of backup path to S3. + + ## Role-based access control for the operator The manifest [`operator-service-account-rbac.yaml`](../manifests/operator-service-account-rbac.yaml) @@ -586,11 +648,11 @@ The configuration paramaters that we will be using are: * `gcp_credentials` * `wal_gs_bucket` -### Generate a K8 secret resource +### Generate a K8s secret resource -Generate the K8 secret resource that will contain your service account's +Generate the K8s secret resource that will contain your service account's credentials. It's highly recommended to use a service account and limit its -scope to just the WAL-E bucket. +scope to just the WAL-E bucket. ```yaml apiVersion: v1 @@ -613,13 +675,13 @@ the operator's configuration is set up like the following: ... aws_or_gcp: additional_secret_mount: "pgsql-wale-creds" - additional_secret_mount_path: "/var/secrets/google" # or where ever you want to mount the file + additional_secret_mount_path: "/var/secrets/google" # or where ever you want to mount the file # aws_region: eu-central-1 # kube_iam_role: "" # log_s3_bucket: "" # wal_s3_bucket: "" - wal_gs_bucket: "postgres-backups-bucket-28302F2" # name of bucket on where to save the WAL-E logs - gcp_credentials: "/var/secrets/google/key.json" # combination of the mount path & key in the K8 resource. (i.e. key.json) + wal_gs_bucket: "postgres-backups-bucket-28302F2" # name of bucket on where to save the WAL-E logs + gcp_credentials: "/var/secrets/google/key.json" # combination of the mount path & key in the K8s resource. (i.e. key.json) ... ``` diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 20771078f..9fa622de8 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -200,6 +200,16 @@ configuration they are grouped under the `kubernetes` key. of a database created by the operator. If the annotation key is also provided by the database definition, the database definition value is used. +* **delete_annotation_date_key** + key name for annotation that compares manifest value with current date in the + YYYY-MM-DD format. Allowed pattern: `'([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]'`. + The default is empty which also disables this delete protection check. + +* **delete_annotation_name_key** + key name for annotation that compares manifest value with Postgres cluster name. + Allowed pattern: `'([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]'`. The default is + empty which also disables this delete protection check. + * **downscaler_annotations** An array of annotations that should be passed from Postgres CRD on to the statefulset and, if exists, to the connection pooler deployment as well. diff --git a/e2e/requirements.txt b/e2e/requirements.txt index 68a8775ff..4f6f5ac5f 100644 --- a/e2e/requirements.txt +++ b/e2e/requirements.txt @@ -1,3 +1,3 @@ -kubernetes==9.0.0 +kubernetes==11.0.0 timeout_decorator==0.4.1 -pyyaml==5.1 +pyyaml==5.3.1 diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 9f0b946c9..49e7da10d 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -7,6 +7,7 @@ import warnings import os import yaml +from datetime import datetime from kubernetes import client, config @@ -614,6 +615,71 @@ class EndToEndTestCase(unittest.TestCase): "Origin": 2, }) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_x_cluster_deletion(self): + ''' + Test deletion with configured protection + ''' + k8s = self.k8s + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + + # configure delete protection + patch_delete_annotations = { + "data": { + "delete_annotation_date_key": "delete-date", + "delete_annotation_name_key": "delete-clustername" + } + } + k8s.update_config(patch_delete_annotations) + + # this delete attempt should be omitted because of missing annotations + k8s.api.custom_objects_api.delete_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster") + + # check that pods and services are still there + k8s.wait_for_running_pods(cluster_label, 2) + k8s.wait_for_service(cluster_label) + + # recreate Postgres cluster resource + k8s.create_with_kubectl("manifests/minimal-postgres-manifest.yaml") + + # wait a little before proceeding + time.sleep(10) + + # add annotations to manifest + deleteDate = datetime.today().strftime('%Y-%m-%d') + pg_patch_delete_annotations = { + "metadata": { + "annotations": { + "delete-date": deleteDate, + "delete-clustername": "acid-minimal-cluster", + } + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_delete_annotations) + + # wait a little before proceeding + time.sleep(10) + k8s.wait_for_running_pods(cluster_label, 2) + k8s.wait_for_service(cluster_label) + + # now delete process should be triggered + k8s.api.custom_objects_api.delete_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster") + + # wait until cluster is deleted + time.sleep(120) + + # check if everything has been deleted + self.assertEqual(0, k8s.count_pods_with_label(cluster_label)) + self.assertEqual(0, k8s.count_services_with_label(cluster_label)) + self.assertEqual(0, k8s.count_endpoints_with_label(cluster_label)) + self.assertEqual(0, k8s.count_statefulsets_with_label(cluster_label)) + self.assertEqual(0, k8s.count_deployments_with_label(cluster_label)) + self.assertEqual(0, k8s.count_pdbs_with_label(cluster_label)) + self.assertEqual(0, k8s.count_secrets_with_label(cluster_label)) + def get_failover_targets(self, master_node, replica_nodes): ''' If all pods live on the same node, failover will happen to other worker(s) @@ -700,11 +766,12 @@ class K8sApi: self.apps_v1 = client.AppsV1Api() self.batch_v1_beta1 = client.BatchV1beta1Api() self.custom_objects_api = client.CustomObjectsApi() + self.policy_v1_beta1 = client.PolicyV1beta1Api() class K8s: ''' - Wraps around K8 api client and helper methods. + Wraps around K8s api client and helper methods. ''' RETRY_TIMEOUT_SEC = 10 @@ -755,14 +822,6 @@ class K8s: if pods: pod_phase = pods[0].status.phase - if pods and pod_phase != 'Running': - pod_name = pods[0].metadata.name - response = self.api.core_v1.read_namespaced_pod( - name=pod_name, - namespace=namespace - ) - print("Pod description {}".format(response)) - time.sleep(self.RETRY_TIMEOUT_SEC) def get_service_type(self, svc_labels, namespace='default'): @@ -824,6 +883,25 @@ class K8s: def count_pods_with_label(self, labels, namespace='default'): return len(self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items) + def count_services_with_label(self, labels, namespace='default'): + return len(self.api.core_v1.list_namespaced_service(namespace, label_selector=labels).items) + + def count_endpoints_with_label(self, labels, namespace='default'): + return len(self.api.core_v1.list_namespaced_endpoints(namespace, label_selector=labels).items) + + def count_secrets_with_label(self, labels, namespace='default'): + return len(self.api.core_v1.list_namespaced_secret(namespace, label_selector=labels).items) + + def count_statefulsets_with_label(self, labels, namespace='default'): + return len(self.api.apps_v1.list_namespaced_stateful_set(namespace, label_selector=labels).items) + + def count_deployments_with_label(self, labels, namespace='default'): + return len(self.api.apps_v1.list_namespaced_deployment(namespace, label_selector=labels).items) + + def count_pdbs_with_label(self, labels, namespace='default'): + return len(self.api.policy_v1_beta1.list_namespaced_pod_disruption_budget( + namespace, label_selector=labels).items) + def wait_for_pod_failover(self, failover_targets, labels, namespace='default'): pod_phase = 'Failing over' new_pod_node = '' diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index e626d6b26..69f7a2d9f 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -6,6 +6,8 @@ metadata: # environment: demo # annotations: # "acid.zalan.do/controller": "second-operator" +# "delete-date": "2020-08-31" # can only be deleted on that day if "delete-date "key is configured +# "delete-clustername": "acid-test-cluster" # can only be deleted when name matches if "delete-clustername" key is configured spec: dockerImage: registry.opensource.zalan.do/acid/spilo-12:1.6-p3 teamId: "acid" @@ -34,7 +36,7 @@ spec: defaultUsers: false postgresql: version: "12" - parameters: # Expert section + parameters: # Expert section shared_buffers: "32MB" max_connections: "10" log_statement: "all" diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 1210d5015..3f3e331c4 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -29,6 +29,8 @@ data: # default_cpu_request: 100m # default_memory_limit: 500Mi # default_memory_request: 100Mi + # delete_annotation_date_key: delete-date + # delete_annotation_name_key: delete-clustername docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p3 # downscaler_annotations: "deployment-time,downscaler/*" # enable_admin_role_for_users: "true" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 95c4678a8..36db2dda8 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -113,6 +113,10 @@ spec: type: object additionalProperties: type: string + delete_annotation_date_key: + type: string + delete_annotation_name_key: + type: string downscaler_annotations: type: array items: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index c0dce42ee..7a029eccd 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -31,6 +31,8 @@ configuration: # custom_pod_annotations: # keya: valuea # keyb: valueb + # delete_annotation_date_key: delete-date + # delete_annotation_name_key: delete-clustername # downscaler_annotations: # - deployment-time # - downscaler/* diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index b5695bb4e..43c313c16 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -888,6 +888,12 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, }, }, + "delete_annotation_date_key": { + Type: "string", + }, + "delete_annotation_name_key": { + Type: "string", + }, "downscaler_annotations": { Type: "array", Items: &apiextv1beta1.JSONSchemaPropsOrArray{ diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index ea08f2ff3..157596123 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -66,6 +66,8 @@ type KubernetesMetaConfiguration struct { InheritedLabels []string `json:"inherited_labels,omitempty"` DownscalerAnnotations []string `json:"downscaler_annotations,omitempty"` ClusterNameLabel string `json:"cluster_name_label,omitempty"` + DeleteAnnotationDateKey string `json:"delete_annotation_date_key,omitempty"` + DeleteAnnotationNameKey string `json:"delete_annotation_name_key,omitempty"` NodeReadinessLabel map[string]string `json:"node_readiness_label,omitempty"` CustomPodAnnotations map[string]string `json:"custom_pod_annotations,omitempty"` // TODO: use a proper toleration structure? diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 10c817016..aa996288c 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "sync" + "time" "github.com/sirupsen/logrus" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" @@ -454,6 +455,37 @@ func (c *Controller) GetReference(postgresql *acidv1.Postgresql) *v1.ObjectRefer return ref } +func (c *Controller) meetsClusterDeleteAnnotations(postgresql *acidv1.Postgresql) error { + + deleteAnnotationDateKey := c.opConfig.DeleteAnnotationDateKey + currentTime := time.Now() + currentDate := currentTime.Format("2006-01-02") // go's reference date + + if deleteAnnotationDateKey != "" { + if deleteDate, ok := postgresql.Annotations[deleteAnnotationDateKey]; ok { + if deleteDate != currentDate { + return fmt.Errorf("annotation %s not matching the current date: got %s, expected %s", deleteAnnotationDateKey, deleteDate, currentDate) + } + } else { + return fmt.Errorf("annotation %s not set in manifest to allow cluster deletion", deleteAnnotationDateKey) + } + } + + deleteAnnotationNameKey := c.opConfig.DeleteAnnotationNameKey + + if deleteAnnotationNameKey != "" { + if clusterName, ok := postgresql.Annotations[deleteAnnotationNameKey]; ok { + if clusterName != postgresql.Name { + return fmt.Errorf("annotation %s not matching the cluster name: got %s, expected %s", deleteAnnotationNameKey, clusterName, postgresql.Name) + } + } else { + return fmt.Errorf("annotation %s not set in manifest to allow cluster deletion", deleteAnnotationNameKey) + } + } + + return nil +} + // hasOwnership returns true if the controller is the "owner" of the postgresql. // Whether it's owner is determined by the value of 'acid.zalan.do/controller' // annotation. If the value matches the controllerID then it owns it, or if the diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index d115aa118..aad9069b1 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -92,6 +92,8 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.InheritedLabels = fromCRD.Kubernetes.InheritedLabels result.DownscalerAnnotations = fromCRD.Kubernetes.DownscalerAnnotations result.ClusterNameLabel = util.Coalesce(fromCRD.Kubernetes.ClusterNameLabel, "cluster-name") + result.DeleteAnnotationDateKey = fromCRD.Kubernetes.DeleteAnnotationDateKey + result.DeleteAnnotationNameKey = fromCRD.Kubernetes.DeleteAnnotationNameKey result.NodeReadinessLabel = fromCRD.Kubernetes.NodeReadinessLabel result.PodPriorityClassName = fromCRD.Kubernetes.PodPriorityClassName result.PodManagementPolicy = util.Coalesce(fromCRD.Kubernetes.PodManagementPolicy, "ordered_ready") diff --git a/pkg/controller/postgresql.go b/pkg/controller/postgresql.go index a41eb0335..c7074c7e4 100644 --- a/pkg/controller/postgresql.go +++ b/pkg/controller/postgresql.go @@ -2,6 +2,7 @@ package controller import ( "context" + "encoding/json" "fmt" "reflect" "strings" @@ -420,6 +421,22 @@ func (c *Controller) queueClusterEvent(informerOldSpec, informerNewSpec *acidv1. clusterError = informerNewSpec.Error } + // only allow deletion if delete annotations are set and conditions are met + if eventType == EventDelete { + if err := c.meetsClusterDeleteAnnotations(informerOldSpec); err != nil { + c.logger.WithField("cluster-name", clusterName).Warnf( + "ignoring %q event for cluster %q - manifest does not fulfill delete requirements: %s", eventType, clusterName, err) + c.logger.WithField("cluster-name", clusterName).Warnf( + "please, recreate Postgresql resource %q and set annotations to delete properly", clusterName) + if currentManifest, marshalErr := json.Marshal(informerOldSpec); marshalErr != nil { + c.logger.WithField("cluster-name", clusterName).Warnf("could not marshal current manifest:\n%+v", informerOldSpec) + } else { + c.logger.WithField("cluster-name", clusterName).Warnf("%s\n", string(currentManifest)) + } + return + } + } + if clusterError != "" && eventType != EventDelete { c.logger.WithField("cluster-name", clusterName).Debugf("skipping %q event for the invalid cluster: %s", eventType, clusterError) diff --git a/pkg/controller/postgresql_test.go b/pkg/controller/postgresql_test.go index b36519c5a..71d23a264 100644 --- a/pkg/controller/postgresql_test.go +++ b/pkg/controller/postgresql_test.go @@ -1,8 +1,10 @@ package controller import ( + "fmt" "reflect" "testing" + "time" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/spec" @@ -90,3 +92,88 @@ func TestMergeDeprecatedPostgreSQLSpecParameters(t *testing.T) { } } } + +func TestMeetsClusterDeleteAnnotations(t *testing.T) { + // set delete annotations in configuration + postgresqlTestController.opConfig.DeleteAnnotationDateKey = "delete-date" + postgresqlTestController.opConfig.DeleteAnnotationNameKey = "delete-clustername" + + currentTime := time.Now() + today := currentTime.Format("2006-01-02") // go's reference date + clusterName := "acid-test-cluster" + + tests := []struct { + name string + pg *acidv1.Postgresql + error string + }{ + { + "Postgres cluster with matching delete annotations", + &acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Annotations: map[string]string{ + "delete-date": today, + "delete-clustername": clusterName, + }, + }, + }, + "", + }, + { + "Postgres cluster with violated delete date annotation", + &acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Annotations: map[string]string{ + "delete-date": "2020-02-02", + "delete-clustername": clusterName, + }, + }, + }, + fmt.Sprintf("annotation delete-date not matching the current date: got 2020-02-02, expected %s", today), + }, + { + "Postgres cluster with violated delete cluster name annotation", + &acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Annotations: map[string]string{ + "delete-date": today, + "delete-clustername": "acid-minimal-cluster", + }, + }, + }, + fmt.Sprintf("annotation delete-clustername not matching the cluster name: got acid-minimal-cluster, expected %s", clusterName), + }, + { + "Postgres cluster with missing delete annotations", + &acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Annotations: map[string]string{}, + }, + }, + "annotation delete-date not set in manifest to allow cluster deletion", + }, + { + "Postgres cluster with missing delete cluster name annotation", + &acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Annotations: map[string]string{ + "delete-date": today, + }, + }, + }, + "annotation delete-clustername not set in manifest to allow cluster deletion", + }, + } + for _, tt := range tests { + if err := postgresqlTestController.meetsClusterDeleteAnnotations(tt.pg); err != nil { + if !reflect.DeepEqual(err.Error(), tt.error) { + t.Errorf("Expected error %q, got: %v", tt.error, err) + } + } + } +} diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 4fe66910a..5f7559929 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -36,6 +36,8 @@ type Resources struct { InheritedLabels []string `name:"inherited_labels" default:""` DownscalerAnnotations []string `name:"downscaler_annotations"` ClusterNameLabel string `name:"cluster_name_label" default:"cluster-name"` + DeleteAnnotationDateKey string `name:"delete_annotation_date_key"` + DeleteAnnotationNameKey string `name:"delete_annotation_name_key"` PodRoleLabel string `name:"pod_role_label" default:"spilo-role"` PodToleration map[string]string `name:"toleration" default:""` DefaultCPURequest string `name:"default_cpu_request" default:"100m"` From dab704c566392b4978622ae8ad20f616e8411dc6 Mon Sep 17 00:00:00 2001 From: Peter Halliday Date: Wed, 26 Aug 2020 05:06:25 -0500 Subject: [PATCH 080/168] Add kustomize support to Postgres UI. (#1086) Co-authored-by: Peter Halliday --- ui/manifests/kustomization.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 ui/manifests/kustomization.yaml diff --git a/ui/manifests/kustomization.yaml b/ui/manifests/kustomization.yaml new file mode 100644 index 000000000..5803f854e --- /dev/null +++ b/ui/manifests/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- deployment.yaml +- ingress.yaml +- service.yaml +- ui-service-account-rbac.yaml From 248ce9fc7823d676d602b117ae36da5dd07bb1b6 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 26 Aug 2020 14:00:14 +0200 Subject: [PATCH 081/168] Update to go 1.14.7 (#1122) * update go version, dependencies, and client-go 1.18.8 --- Makefile | 2 +- delivery.yaml | 2 +- go.mod | 12 ++++++------ go.sum | 35 +++++++++++++++++++++++------------ 4 files changed, 31 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index 69dd240db..7ecf54f7c 100644 --- a/Makefile +++ b/Makefile @@ -79,7 +79,7 @@ scm-source.json: .git tools: GO111MODULE=on go get -u honnef.co/go/tools/cmd/staticcheck - GO111MODULE=on go get k8s.io/client-go@kubernetes-1.16.3 + GO111MODULE=on go get k8s.io/client-go@kubernetes-1.18.8 GO111MODULE=on go mod tidy fmt: diff --git a/delivery.yaml b/delivery.yaml index d0884f982..07c768424 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -12,7 +12,7 @@ pipeline: - desc: 'Install go' cmd: | cd /tmp - wget -q https://storage.googleapis.com/golang/go1.14.linux-amd64.tar.gz -O go.tar.gz + wget -q https://storage.googleapis.com/golang/go1.14.7.linux-amd64.tar.gz -O go.tar.gz tar -xf go.tar.gz mv go /usr/local ln -s /usr/local/go/bin/go /usr/bin/go diff --git a/go.mod b/go.mod index 5d3a760df..74f8dc5e1 100644 --- a/go.mod +++ b/go.mod @@ -3,19 +3,19 @@ module github.com/zalando/postgres-operator go 1.14 require ( - github.com/aws/aws-sdk-go v1.34.1 + github.com/aws/aws-sdk-go v1.34.10 github.com/lib/pq v1.8.0 github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d github.com/r3labs/diff v1.1.0 github.com/sirupsen/logrus v1.6.0 github.com/stretchr/testify v1.5.1 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 - golang.org/x/tools v0.0.0-20200811032001-fd80f4dbb3ea // indirect + golang.org/x/tools v0.0.0-20200826040757-bc8aaaa29e06 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gopkg.in/yaml.v2 v2.2.8 - k8s.io/api v0.18.6 - k8s.io/apiextensions-apiserver v0.18.6 - k8s.io/apimachinery v0.18.6 + k8s.io/api v0.18.8 + k8s.io/apiextensions-apiserver v0.18.0 + k8s.io/apimachinery v0.18.8 k8s.io/client-go v0.18.6 - k8s.io/code-generator v0.18.6 + k8s.io/code-generator v0.18.8 ) diff --git a/go.sum b/go.sum index a12e65ce6..a7787e0fd 100644 --- a/go.sum +++ b/go.sum @@ -26,8 +26,8 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.34.1 h1:jM0mJ9JSJyhujwxBNYKrNB8Iwp8N7J2WsQxTR4yPSck= -github.com/aws/aws-sdk-go v1.34.1/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.34.10 h1:VU78gcf/3wA4HNEDCHidK738l7K0Bals4SJnfnvXOtY= +github.com/aws/aws-sdk-go v1.34.10/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= @@ -64,6 +64,7 @@ github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v0.0.0-20200808040245-162e5629780b/go.mod h1:NAJj0yf/KaRKURN6nyi7A9IZydMivZEm9oQLWNjfKDc= github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -175,6 +176,7 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -391,8 +393,8 @@ golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200811032001-fd80f4dbb3ea h1:9ym67RBRK/wN50W0T3g8g1n8viM1D2ofgWufDlMfWe0= -golang.org/x/tools v0.0.0-20200811032001-fd80f4dbb3ea/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200826040757-bc8aaaa29e06 h1:ChBCbOHeLqK+j+znGPlWCcvx/t2PdxmyPBheVZxXbcc= +golang.org/x/tools v0.0.0-20200826040757-bc8aaaa29e06/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= @@ -436,19 +438,27 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.18.6 h1:osqrAXbOQjkKIWDTjrqxWQ3w0GkKb1KA1XkUGHHYpeE= +k8s.io/api v0.18.0/go.mod h1:q2HRQkfDzHMBZL9l/y9rH63PkQl4vae0xRT+8prbrK8= k8s.io/api v0.18.6/go.mod h1:eeyxr+cwCjMdLAmr2W3RyDI0VvTawSg/3RFFBEnmZGI= -k8s.io/apiextensions-apiserver v0.18.6 h1:vDlk7cyFsDyfwn2rNAO2DbmUbvXy5yT5GE3rrqOzaMo= -k8s.io/apiextensions-apiserver v0.18.6/go.mod h1:lv89S7fUysXjLZO7ke783xOwVTm6lKizADfvUM/SS/M= -k8s.io/apimachinery v0.18.6 h1:RtFHnfGNfd1N0LeSrKCUznz5xtUP1elRGvHJbL3Ntag= +k8s.io/api v0.18.8 h1:aIKUzJPb96f3fKec2lxtY7acZC9gQNDLVhfSGpxBAC4= +k8s.io/api v0.18.8/go.mod h1:d/CXqwWv+Z2XEG1LgceeDmHQwpUJhROPx16SlxJgERY= +k8s.io/apiextensions-apiserver v0.18.0 h1:HN4/P8vpGZFvB5SOMuPPH2Wt9Y/ryX+KRvIyAkchu1Q= +k8s.io/apiextensions-apiserver v0.18.0/go.mod h1:18Cwn1Xws4xnWQNC00FLq1E350b9lUF+aOdIWDOZxgo= +k8s.io/apimachinery v0.18.0/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= k8s.io/apimachinery v0.18.6/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= -k8s.io/apiserver v0.18.6/go.mod h1:Zt2XvTHuaZjBz6EFYzpp+X4hTmgWGy8AthNVnTdm3Wg= +k8s.io/apimachinery v0.18.8 h1:jimPrycCqgx2QPearX3to1JePz7wSbVLq+7PdBTTwQ0= +k8s.io/apimachinery v0.18.8/go.mod h1:6sQd+iHEqmOtALqOFjSWp2KZ9F0wlU/nWm0ZgsYWMig= +k8s.io/apiserver v0.18.0/go.mod h1:3S2O6FeBBd6XTo0njUrLxiqk8GNy6wWOftjhJcXYnjw= +k8s.io/client-go v0.18.0 h1:yqKw4cTUQraZK3fcVCMeSa+lqKwcjZ5wtcOIPnxQno4= +k8s.io/client-go v0.18.0/go.mod h1:uQSYDYs4WhVZ9i6AIoEZuwUggLVEF64HOD37boKAtF8= k8s.io/client-go v0.18.6 h1:I+oWqJbibLSGsZj8Xs8F0aWVXJVIoUHWaaJV3kUN/Zw= k8s.io/client-go v0.18.6/go.mod h1:/fwtGLjYMS1MaM5oi+eXhKwG+1UHidUEXRh6cNsdO0Q= -k8s.io/code-generator v0.18.6 h1:QdfvGfs4gUCS1dru+rLbCKIFxYEV0IRfF8MXwY/ozLk= -k8s.io/code-generator v0.18.6/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c= -k8s.io/component-base v0.18.6/go.mod h1:knSVsibPR5K6EW2XOjEHik6sdU5nCvKMrzMt2D4In14= +k8s.io/code-generator v0.18.0/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= +k8s.io/code-generator v0.18.8 h1:lgO1P1wjikEtzNvj7ia+x1VC4svJ28a/r0wnOLhhOTU= +k8s.io/code-generator v0.18.8/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c= +k8s.io/component-base v0.18.0/go.mod h1:u3BCg0z1uskkzrnAKFzulmYaEpZF7XC9Pf/uFyb1v2c= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200114144118-36b2048a9120 h1:RPscN6KhmG54S33L+lr3GS+oD1jmchIU0ll519K6FA4= k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= @@ -456,6 +466,7 @@ k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUc k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6 h1:Oh3Mzx5pJ+yIumsAD0MOECPVeXsVot0UkiaCGVyfGQY= k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89 h1:d4vVOjXm687F1iLSP2q3lyPPuyvTUt3aVoBpi2DqRsU= From 30c86758a3ce3703648ee2529d4f2f48e6037538 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 28 Aug 2020 12:16:37 +0200 Subject: [PATCH 082/168] update kind and use with old storage class (#1121) * update kind and use with old storage class * specify standard storage class in minimal manifest * remove existing local storage class in kind * fix pod distribution test * exclude k8s master from nodes of interest --- e2e/Makefile | 2 +- e2e/run.sh | 9 +++------ e2e/tests/test_e2e.py | 23 ++++++++++++++--------- manifests/e2e-storage-class.yaml | 8 ++++++++ 4 files changed, 26 insertions(+), 16 deletions(-) create mode 100644 manifests/e2e-storage-class.yaml diff --git a/e2e/Makefile b/e2e/Makefile index 70a2ff4e9..16e3f2f99 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -42,7 +42,7 @@ push: docker tools: docker # install pinned version of 'kind' - GO111MODULE=on go get sigs.k8s.io/kind@v0.5.1 + GO111MODULE=on go get sigs.k8s.io/kind@v0.8.1 e2etest: ./run.sh diff --git a/e2e/run.sh b/e2e/run.sh index c7825bfd3..9d7e2eba7 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -35,25 +35,22 @@ function start_kind(){ kind delete cluster --name ${cluster_name} fi + export KUBECONFIG="${kubeconfig_path}" kind create cluster --name ${cluster_name} --config kind-cluster-postgres-operator-e2e-tests.yaml kind load docker-image "${operator_image}" --name ${cluster_name} kind load docker-image "${e2e_test_image}" --name ${cluster_name} - KUBECONFIG="$(kind get kubeconfig-path --name=${cluster_name})" - export KUBECONFIG } function set_kind_api_server_ip(){ # use the actual kubeconfig to connect to the 'kind' API server # but update the IP address of the API server to the one from the Docker 'bridge' network - cp "${KUBECONFIG}" /tmp readonly local kind_api_server_port=6443 # well-known in the 'kind' codebase - readonly local kind_api_server=$(docker inspect --format "{{ .NetworkSettings.IPAddress }}:${kind_api_server_port}" "${cluster_name}"-control-plane) + readonly local kind_api_server=$(docker inspect --format "{{ .NetworkSettings.Networks.kind.IPAddress }}:${kind_api_server_port}" "${cluster_name}"-control-plane) sed -i "s/server.*$/server: https:\/\/$kind_api_server/g" "${kubeconfig_path}" } function run_tests(){ - - docker run --rm --mount type=bind,source="$(readlink -f ${kubeconfig_path})",target=/root/.kube/config -e OPERATOR_IMAGE="${operator_image}" "${e2e_test_image}" + docker run --rm --network kind --mount type=bind,source="$(readlink -f ${kubeconfig_path})",target=/root/.kube/config -e OPERATOR_IMAGE="${operator_image}" "${e2e_test_image}" } function clean_up(){ diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 49e7da10d..182bdba4d 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -38,6 +38,9 @@ class EndToEndTestCase(unittest.TestCase): # set a single K8s wrapper for all tests k8s = cls.k8s = K8s() + # remove existing local storage class and create hostpath class + k8s.api.storage_v1_api.delete_storage_class("standard") + # operator deploys pod service account there on start up # needed for test_multi_namespace_support() cls.namespace = "test" @@ -54,7 +57,8 @@ class EndToEndTestCase(unittest.TestCase): "configmap.yaml", "postgres-operator.yaml", "infrastructure-roles.yaml", - "infrastructure-roles-new.yaml"]: + "infrastructure-roles-new.yaml", + "e2e-storage-class.yaml"]: result = k8s.create_with_kubectl("manifests/" + filename) print("stdout: {}, stderr: {}".format(result.stdout, result.stderr)) @@ -600,8 +604,8 @@ class EndToEndTestCase(unittest.TestCase): get_config_cmd = "wget --quiet -O - localhost:8080/config" result = k8s.exec_with_kubectl(operator_pod.metadata.name, get_config_cmd) roles_dict = (json.loads(result.stdout) - .get("controller", {}) - .get("InfrastructureRoles")) + .get("controller", {}) + .get("InfrastructureRoles")) self.assertTrue("robot_zmon_acid_monitoring_new" in roles_dict) role = roles_dict["robot_zmon_acid_monitoring_new"] @@ -685,12 +689,13 @@ class EndToEndTestCase(unittest.TestCase): If all pods live on the same node, failover will happen to other worker(s) ''' k8s = self.k8s + k8s_master_exclusion = 'kubernetes.io/hostname!=postgres-operator-e2e-tests-control-plane' failover_targets = [x for x in replica_nodes if x != master_node] if len(failover_targets) == 0: - nodes = k8s.api.core_v1.list_node() + nodes = k8s.api.core_v1.list_node(label_selector=k8s_master_exclusion) for n in nodes.items: - if "node-role.kubernetes.io/master" not in n.metadata.labels and n.metadata.name != master_node: + if n.metadata.name != master_node: failover_targets.append(n.metadata.name) return failover_targets @@ -738,8 +743,7 @@ class EndToEndTestCase(unittest.TestCase): } } k8s.update_config(patch_enable_antiaffinity) - self.assert_failover( - master_node, len(replica_nodes), failover_targets, cluster_label) + self.assert_failover(master_node, len(replica_nodes), failover_targets, cluster_label) # now disable pod anti affintiy again which will cause yet another failover patch_disable_antiaffinity = { @@ -767,6 +771,7 @@ class K8sApi: self.batch_v1_beta1 = client.BatchV1beta1Api() self.custom_objects_api = client.CustomObjectsApi() self.policy_v1_beta1 = client.PolicyV1beta1Api() + self.storage_v1_api = client.StorageV1Api() class K8s: @@ -944,8 +949,8 @@ class K8s: def exec_with_kubectl(self, pod, cmd): return subprocess.run(["./exec.sh", pod, cmd], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) def get_effective_pod_image(self, pod_name, namespace='default'): ''' diff --git a/manifests/e2e-storage-class.yaml b/manifests/e2e-storage-class.yaml new file mode 100644 index 000000000..c8d941341 --- /dev/null +++ b/manifests/e2e-storage-class.yaml @@ -0,0 +1,8 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + namespace: kube-system + name: standard + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: kubernetes.io/host-path From 5e93aabea607b37002856b39d9e1b6368840350a Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 28 Aug 2020 14:57:19 +0200 Subject: [PATCH 083/168] improve e2e test debugging (#1107) * print operator log in most tests when they time out --- e2e/tests/test_e2e.py | 581 +++++++++++++++++++++++------------------- 1 file changed, 322 insertions(+), 259 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 182bdba4d..ce2d392e1 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -163,45 +163,96 @@ class EndToEndTestCase(unittest.TestCase): k8s = self.k8s cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' - # enable load balancer services - pg_patch_enable_lbs = { - "spec": { - "enableMasterLoadBalancer": True, - "enableReplicaLoadBalancer": True + try: + # enable load balancer services + pg_patch_enable_lbs = { + "spec": { + "enableMasterLoadBalancer": True, + "enableReplicaLoadBalancer": True + } } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_lbs) - # wait for service recreation - time.sleep(60) + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_lbs) + # wait for service recreation + time.sleep(60) - master_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=master') - self.assertEqual(master_svc_type, 'LoadBalancer', - "Expected LoadBalancer service type for master, found {}".format(master_svc_type)) + master_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=master') + self.assertEqual(master_svc_type, 'LoadBalancer', + "Expected LoadBalancer service type for master, found {}".format(master_svc_type)) - repl_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=replica') - self.assertEqual(repl_svc_type, 'LoadBalancer', - "Expected LoadBalancer service type for replica, found {}".format(repl_svc_type)) + repl_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=replica') + self.assertEqual(repl_svc_type, 'LoadBalancer', + "Expected LoadBalancer service type for replica, found {}".format(repl_svc_type)) - # disable load balancer services again - pg_patch_disable_lbs = { - "spec": { - "enableMasterLoadBalancer": False, - "enableReplicaLoadBalancer": False + # disable load balancer services again + pg_patch_disable_lbs = { + "spec": { + "enableMasterLoadBalancer": False, + "enableReplicaLoadBalancer": False + } } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_lbs) + # wait for service recreation + time.sleep(60) + + master_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=master') + self.assertEqual(master_svc_type, 'ClusterIP', + "Expected ClusterIP service type for master, found {}".format(master_svc_type)) + + repl_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=replica') + self.assertEqual(repl_svc_type, 'ClusterIP', + "Expected ClusterIP service type for replica, found {}".format(repl_svc_type)) + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_infrastructure_roles(self): + ''' + Test using external secrets for infrastructure roles + ''' + k8s = self.k8s + # update infrastructure roles description + secret_name = "postgresql-infrastructure-roles" + roles = "secretname: postgresql-infrastructure-roles-new, userkey: user, rolekey: memberof, passwordkey: password, defaultrolevalue: robot_zmon" + patch_infrastructure_roles = { + "data": { + "infrastructure_roles_secret_name": secret_name, + "infrastructure_roles_secrets": roles, + }, } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_lbs) - # wait for service recreation - time.sleep(60) + k8s.update_config(patch_infrastructure_roles) - master_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=master') - self.assertEqual(master_svc_type, 'ClusterIP', - "Expected ClusterIP service type for master, found {}".format(master_svc_type)) + # wait a little before proceeding + time.sleep(30) - repl_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=replica') - self.assertEqual(repl_svc_type, 'ClusterIP', - "Expected ClusterIP service type for replica, found {}".format(repl_svc_type)) + try: + # check that new roles are represented in the config by requesting the + # operator configuration via API + operator_pod = k8s.get_operator_pod() + get_config_cmd = "wget --quiet -O - localhost:8080/config" + result = k8s.exec_with_kubectl(operator_pod.metadata.name, get_config_cmd) + roles_dict = (json.loads(result.stdout) + .get("controller", {}) + .get("InfrastructureRoles")) + + self.assertTrue("robot_zmon_acid_monitoring_new" in roles_dict) + role = roles_dict["robot_zmon_acid_monitoring_new"] + role.pop("Password", None) + self.assertDictEqual(role, { + "Name": "robot_zmon_acid_monitoring_new", + "Flags": None, + "MemberOf": ["robot_zmon"], + "Parameters": None, + "AdminRole": "", + "Origin": 2, + }) + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_lazy_spilo_upgrade(self): @@ -230,38 +281,44 @@ class EndToEndTestCase(unittest.TestCase): pod0 = 'acid-minimal-cluster-0' pod1 = 'acid-minimal-cluster-1' - # restart the pod to get a container with the new image - k8s.api.core_v1.delete_namespaced_pod(pod0, 'default') - time.sleep(60) + try: + # restart the pod to get a container with the new image + k8s.api.core_v1.delete_namespaced_pod(pod0, 'default') + time.sleep(60) - # lazy update works if the restarted pod and older pods run different Spilo versions - new_image = k8s.get_effective_pod_image(pod0) - old_image = k8s.get_effective_pod_image(pod1) - self.assertNotEqual(new_image, old_image, "Lazy updated failed: pods have the same image {}".format(new_image)) + # lazy update works if the restarted pod and older pods run different Spilo versions + new_image = k8s.get_effective_pod_image(pod0) + old_image = k8s.get_effective_pod_image(pod1) + self.assertNotEqual(new_image, old_image, + "Lazy updated failed: pods have the same image {}".format(new_image)) - # sanity check - assert_msg = "Image {} of a new pod differs from {} in operator conf".format(new_image, conf_image) - self.assertEqual(new_image, conf_image, assert_msg) + # sanity check + assert_msg = "Image {} of a new pod differs from {} in operator conf".format(new_image, conf_image) + self.assertEqual(new_image, conf_image, assert_msg) - # clean up - unpatch_lazy_spilo_upgrade = { - "data": { - "enable_lazy_spilo_upgrade": "false", + # clean up + unpatch_lazy_spilo_upgrade = { + "data": { + "enable_lazy_spilo_upgrade": "false", + } } - } - k8s.update_config(unpatch_lazy_spilo_upgrade) + k8s.update_config(unpatch_lazy_spilo_upgrade) - # at this point operator will complete the normal rolling upgrade - # so we additonally test if disabling the lazy upgrade - forcing the normal rolling upgrade - works + # at this point operator will complete the normal rolling upgrade + # so we additonally test if disabling the lazy upgrade - forcing the normal rolling upgrade - works - # XXX there is no easy way to wait until the end of Sync() - time.sleep(60) + # XXX there is no easy way to wait until the end of Sync() + time.sleep(60) - image0 = k8s.get_effective_pod_image(pod0) - image1 = k8s.get_effective_pod_image(pod1) + image0 = k8s.get_effective_pod_image(pod0) + image1 = k8s.get_effective_pod_image(pod1) - assert_msg = "Disabling lazy upgrade failed: pods still have different images {} and {}".format(image0, image1) - self.assertEqual(image0, image1, assert_msg) + assert_msg = "Disabling lazy upgrade failed: pods still have different images {} and {}".format(image0, image1) + self.assertEqual(image0, image1, assert_msg) + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_logical_backup_cron_job(self): @@ -287,45 +344,51 @@ class EndToEndTestCase(unittest.TestCase): } k8s.api.custom_objects_api.patch_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_backup) - k8s.wait_for_logical_backup_job_creation() - jobs = k8s.get_logical_backup_job().items - self.assertEqual(1, len(jobs), "Expected 1 logical backup job, found {}".format(len(jobs))) + try: + k8s.wait_for_logical_backup_job_creation() - job = jobs[0] - self.assertEqual(job.metadata.name, "logical-backup-acid-minimal-cluster", - "Expected job name {}, found {}" - .format("logical-backup-acid-minimal-cluster", job.metadata.name)) - self.assertEqual(job.spec.schedule, schedule, - "Expected {} schedule, found {}" - .format(schedule, job.spec.schedule)) + jobs = k8s.get_logical_backup_job().items + self.assertEqual(1, len(jobs), "Expected 1 logical backup job, found {}".format(len(jobs))) - # update the cluster-wide image of the logical backup pod - image = "test-image-name" - patch_logical_backup_image = { - "data": { - "logical_backup_docker_image": image, + job = jobs[0] + self.assertEqual(job.metadata.name, "logical-backup-acid-minimal-cluster", + "Expected job name {}, found {}" + .format("logical-backup-acid-minimal-cluster", job.metadata.name)) + self.assertEqual(job.spec.schedule, schedule, + "Expected {} schedule, found {}" + .format(schedule, job.spec.schedule)) + + # update the cluster-wide image of the logical backup pod + image = "test-image-name" + patch_logical_backup_image = { + "data": { + "logical_backup_docker_image": image, + } } - } - k8s.update_config(patch_logical_backup_image) + k8s.update_config(patch_logical_backup_image) - jobs = k8s.get_logical_backup_job().items - actual_image = jobs[0].spec.job_template.spec.template.spec.containers[0].image - self.assertEqual(actual_image, image, - "Expected job image {}, found {}".format(image, actual_image)) + jobs = k8s.get_logical_backup_job().items + actual_image = jobs[0].spec.job_template.spec.template.spec.containers[0].image + self.assertEqual(actual_image, image, + "Expected job image {}, found {}".format(image, actual_image)) - # delete the logical backup cron job - pg_patch_disable_backup = { - "spec": { - "enableLogicalBackup": False, + # delete the logical backup cron job + pg_patch_disable_backup = { + "spec": { + "enableLogicalBackup": False, + } } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_backup) - k8s.wait_for_logical_backup_job_deletion() - jobs = k8s.get_logical_backup_job().items - self.assertEqual(0, len(jobs), - "Expected 0 logical backup jobs, found {}".format(len(jobs))) + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_backup) + k8s.wait_for_logical_backup_job_deletion() + jobs = k8s.get_logical_backup_job().items + self.assertEqual(0, len(jobs), + "Expected 0 logical backup jobs, found {}".format(len(jobs))) + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_min_resource_limits(self): @@ -365,20 +428,26 @@ class EndToEndTestCase(unittest.TestCase): } k8s.api.custom_objects_api.patch_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_resources) - k8s.wait_for_pod_failover(failover_targets, labels) - k8s.wait_for_pod_start('spilo-role=replica') - pods = k8s.api.core_v1.list_namespaced_pod( - 'default', label_selector=labels).items - self.assert_master_is_unique() - masterPod = pods[0] + try: + k8s.wait_for_pod_failover(failover_targets, labels) + k8s.wait_for_pod_start('spilo-role=replica') - self.assertEqual(masterPod.spec.containers[0].resources.limits['cpu'], minCPULimit, - "Expected CPU limit {}, found {}" - .format(minCPULimit, masterPod.spec.containers[0].resources.limits['cpu'])) - self.assertEqual(masterPod.spec.containers[0].resources.limits['memory'], minMemoryLimit, - "Expected memory limit {}, found {}" - .format(minMemoryLimit, masterPod.spec.containers[0].resources.limits['memory'])) + pods = k8s.api.core_v1.list_namespaced_pod( + 'default', label_selector=labels).items + self.assert_master_is_unique() + masterPod = pods[0] + + self.assertEqual(masterPod.spec.containers[0].resources.limits['cpu'], minCPULimit, + "Expected CPU limit {}, found {}" + .format(minCPULimit, masterPod.spec.containers[0].resources.limits['cpu'])) + self.assertEqual(masterPod.spec.containers[0].resources.limits['memory'], minMemoryLimit, + "Expected memory limit {}, found {}" + .format(minMemoryLimit, masterPod.spec.containers[0].resources.limits['memory'])) + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_multi_namespace_support(self): @@ -392,9 +461,14 @@ class EndToEndTestCase(unittest.TestCase): pg_manifest["metadata"]["namespace"] = self.namespace yaml.dump(pg_manifest, f, Dumper=yaml.Dumper) - k8s.create_with_kubectl("manifests/complete-postgres-manifest.yaml") - k8s.wait_for_pod_start("spilo-role=master", self.namespace) - self.assert_master_is_unique(self.namespace, "acid-test-cluster") + try: + k8s.create_with_kubectl("manifests/complete-postgres-manifest.yaml") + k8s.wait_for_pod_start("spilo-role=master", self.namespace) + self.assert_master_is_unique(self.namespace, "acid-test-cluster") + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_node_readiness_label(self): @@ -406,40 +480,45 @@ class EndToEndTestCase(unittest.TestCase): readiness_label = 'lifecycle-status' readiness_value = 'ready' - # get nodes of master and replica(s) (expected target of new master) - current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label) - num_replicas = len(current_replica_nodes) - failover_targets = self.get_failover_targets(current_master_node, current_replica_nodes) + try: + # get nodes of master and replica(s) (expected target of new master) + current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label) + num_replicas = len(current_replica_nodes) + failover_targets = self.get_failover_targets(current_master_node, current_replica_nodes) - # add node_readiness_label to potential failover nodes - patch_readiness_label = { - "metadata": { - "labels": { - readiness_label: readiness_value + # add node_readiness_label to potential failover nodes + patch_readiness_label = { + "metadata": { + "labels": { + readiness_label: readiness_value + } } } - } - for failover_target in failover_targets: - k8s.api.core_v1.patch_node(failover_target, patch_readiness_label) + for failover_target in failover_targets: + k8s.api.core_v1.patch_node(failover_target, patch_readiness_label) - # define node_readiness_label in config map which should trigger a failover of the master - patch_readiness_label_config = { - "data": { - "node_readiness_label": readiness_label + ':' + readiness_value, + # define node_readiness_label in config map which should trigger a failover of the master + patch_readiness_label_config = { + "data": { + "node_readiness_label": readiness_label + ':' + readiness_value, + } } - } - k8s.update_config(patch_readiness_label_config) - new_master_node, new_replica_nodes = self.assert_failover( - current_master_node, num_replicas, failover_targets, cluster_label) + k8s.update_config(patch_readiness_label_config) + new_master_node, new_replica_nodes = self.assert_failover( + current_master_node, num_replicas, failover_targets, cluster_label) - # patch also node where master ran before - k8s.api.core_v1.patch_node(current_master_node, patch_readiness_label) + # patch also node where master ran before + k8s.api.core_v1.patch_node(current_master_node, patch_readiness_label) - # wait a little before proceeding with the pod distribution test - time.sleep(30) + # wait a little before proceeding with the pod distribution test + time.sleep(30) - # toggle pod anti affinity to move replica away from master node - self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) + # toggle pod anti affinity to move replica away from master node + self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_scaling(self): @@ -449,13 +528,18 @@ class EndToEndTestCase(unittest.TestCase): k8s = self.k8s labels = "application=spilo,cluster-name=acid-minimal-cluster" - k8s.wait_for_pg_to_scale(3) - self.assertEqual(3, k8s.count_pods_with_label(labels)) - self.assert_master_is_unique() + try: + k8s.wait_for_pg_to_scale(3) + self.assertEqual(3, k8s.count_pods_with_label(labels)) + self.assert_master_is_unique() - k8s.wait_for_pg_to_scale(2) - self.assertEqual(2, k8s.count_pods_with_label(labels)) - self.assert_master_is_unique() + k8s.wait_for_pg_to_scale(2) + self.assertEqual(2, k8s.count_pods_with_label(labels)) + self.assert_master_is_unique() + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_service_annotations(self): @@ -470,27 +554,32 @@ class EndToEndTestCase(unittest.TestCase): } k8s.update_config(patch_custom_service_annotations) - pg_patch_custom_annotations = { - "spec": { - "serviceAnnotations": { - "annotation.key": "value", - "foo": "bar", + try: + pg_patch_custom_annotations = { + "spec": { + "serviceAnnotations": { + "annotation.key": "value", + "foo": "bar", + } } } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_custom_annotations) + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_custom_annotations) - # wait a little before proceeding - time.sleep(30) - annotations = { - "annotation.key": "value", - "foo": "bar", - } - self.assertTrue(k8s.check_service_annotations( - "cluster-name=acid-minimal-cluster,spilo-role=master", annotations)) - self.assertTrue(k8s.check_service_annotations( - "cluster-name=acid-minimal-cluster,spilo-role=replica", annotations)) + # wait a little before proceeding + time.sleep(30) + annotations = { + "annotation.key": "value", + "foo": "bar", + } + self.assertTrue(k8s.check_service_annotations( + "cluster-name=acid-minimal-cluster,spilo-role=master", annotations)) + self.assertTrue(k8s.check_service_annotations( + "cluster-name=acid-minimal-cluster,spilo-role=replica", annotations)) + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise # clean up unpatch_custom_service_annotations = { @@ -515,24 +604,29 @@ class EndToEndTestCase(unittest.TestCase): } k8s.update_config(patch_sset_propagate_annotations) - pg_crd_annotations = { - "metadata": { - "annotations": { - "deployment-time": "2020-04-30 12:00:00", - "downscaler/downtime_replicas": "0", - }, + try: + pg_crd_annotations = { + "metadata": { + "annotations": { + "deployment-time": "2020-04-30 12:00:00", + "downscaler/downtime_replicas": "0", + }, + } } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_crd_annotations) + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_crd_annotations) - # wait a little before proceeding - time.sleep(60) - annotations = { - "deployment-time": "2020-04-30 12:00:00", - "downscaler/downtime_replicas": "0", - } - self.assertTrue(k8s.check_statefulset_annotations(cluster_label, annotations)) + # wait a little before proceeding + time.sleep(60) + annotations = { + "deployment-time": "2020-04-30 12:00:00", + "downscaler/downtime_replicas": "0", + } + self.assertTrue(k8s.check_statefulset_annotations(cluster_label, annotations)) + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_taint_based_eviction(self): @@ -559,65 +653,29 @@ class EndToEndTestCase(unittest.TestCase): } } - # patch node and test if master is failing over to one of the expected nodes - k8s.api.core_v1.patch_node(current_master_node, body) - new_master_node, new_replica_nodes = self.assert_failover( - current_master_node, num_replicas, failover_targets, cluster_label) + try: + # patch node and test if master is failing over to one of the expected nodes + k8s.api.core_v1.patch_node(current_master_node, body) + new_master_node, new_replica_nodes = self.assert_failover( + current_master_node, num_replicas, failover_targets, cluster_label) - # add toleration to pods - patch_toleration_config = { - "data": { - "toleration": "key:postgres,operator:Exists,effect:NoExecute" + # add toleration to pods + patch_toleration_config = { + "data": { + "toleration": "key:postgres,operator:Exists,effect:NoExecute" + } } - } - k8s.update_config(patch_toleration_config) + k8s.update_config(patch_toleration_config) - # wait a little before proceeding with the pod distribution test - time.sleep(30) + # wait a little before proceeding with the pod distribution test + time.sleep(30) - # toggle pod anti affinity to move replica away from master node - self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) + # toggle pod anti affinity to move replica away from master node + self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_infrastructure_roles(self): - ''' - Test using external secrets for infrastructure roles - ''' - k8s = self.k8s - # update infrastructure roles description - secret_name = "postgresql-infrastructure-roles" - roles = "secretname: postgresql-infrastructure-roles-new, userkey: user, rolekey: memberof, passwordkey: password, defaultrolevalue: robot_zmon" - patch_infrastructure_roles = { - "data": { - "infrastructure_roles_secret_name": secret_name, - "infrastructure_roles_secrets": roles, - }, - } - k8s.update_config(patch_infrastructure_roles) - - # wait a little before proceeding - time.sleep(30) - - # check that new roles are represented in the config by requesting the - # operator configuration via API - operator_pod = k8s.get_operator_pod() - get_config_cmd = "wget --quiet -O - localhost:8080/config" - result = k8s.exec_with_kubectl(operator_pod.metadata.name, get_config_cmd) - roles_dict = (json.loads(result.stdout) - .get("controller", {}) - .get("InfrastructureRoles")) - - self.assertTrue("robot_zmon_acid_monitoring_new" in roles_dict) - role = roles_dict["robot_zmon_acid_monitoring_new"] - role.pop("Password", None) - self.assertDictEqual(role, { - "Name": "robot_zmon_acid_monitoring_new", - "Flags": None, - "MemberOf": ["robot_zmon"], - "Parameters": None, - "AdminRole": "", - "Origin": 2, - }) + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_x_cluster_deletion(self): @@ -636,53 +694,58 @@ class EndToEndTestCase(unittest.TestCase): } k8s.update_config(patch_delete_annotations) - # this delete attempt should be omitted because of missing annotations - k8s.api.custom_objects_api.delete_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster") + try: + # this delete attempt should be omitted because of missing annotations + k8s.api.custom_objects_api.delete_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster") - # check that pods and services are still there - k8s.wait_for_running_pods(cluster_label, 2) - k8s.wait_for_service(cluster_label) + # check that pods and services are still there + k8s.wait_for_running_pods(cluster_label, 2) + k8s.wait_for_service(cluster_label) - # recreate Postgres cluster resource - k8s.create_with_kubectl("manifests/minimal-postgres-manifest.yaml") + # recreate Postgres cluster resource + k8s.create_with_kubectl("manifests/minimal-postgres-manifest.yaml") - # wait a little before proceeding - time.sleep(10) + # wait a little before proceeding + time.sleep(10) - # add annotations to manifest - deleteDate = datetime.today().strftime('%Y-%m-%d') - pg_patch_delete_annotations = { - "metadata": { - "annotations": { - "delete-date": deleteDate, - "delete-clustername": "acid-minimal-cluster", + # add annotations to manifest + deleteDate = datetime.today().strftime('%Y-%m-%d') + pg_patch_delete_annotations = { + "metadata": { + "annotations": { + "delete-date": deleteDate, + "delete-clustername": "acid-minimal-cluster", + } } } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_delete_annotations) + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_delete_annotations) - # wait a little before proceeding - time.sleep(10) - k8s.wait_for_running_pods(cluster_label, 2) - k8s.wait_for_service(cluster_label) + # wait a little before proceeding + time.sleep(10) + k8s.wait_for_running_pods(cluster_label, 2) + k8s.wait_for_service(cluster_label) - # now delete process should be triggered - k8s.api.custom_objects_api.delete_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster") + # now delete process should be triggered + k8s.api.custom_objects_api.delete_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster") - # wait until cluster is deleted - time.sleep(120) + # wait until cluster is deleted + time.sleep(120) - # check if everything has been deleted - self.assertEqual(0, k8s.count_pods_with_label(cluster_label)) - self.assertEqual(0, k8s.count_services_with_label(cluster_label)) - self.assertEqual(0, k8s.count_endpoints_with_label(cluster_label)) - self.assertEqual(0, k8s.count_statefulsets_with_label(cluster_label)) - self.assertEqual(0, k8s.count_deployments_with_label(cluster_label)) - self.assertEqual(0, k8s.count_pdbs_with_label(cluster_label)) - self.assertEqual(0, k8s.count_secrets_with_label(cluster_label)) + # check if everything has been deleted + self.assertEqual(0, k8s.count_pods_with_label(cluster_label)) + self.assertEqual(0, k8s.count_services_with_label(cluster_label)) + self.assertEqual(0, k8s.count_endpoints_with_label(cluster_label)) + self.assertEqual(0, k8s.count_statefulsets_with_label(cluster_label)) + self.assertEqual(0, k8s.count_deployments_with_label(cluster_label)) + self.assertEqual(0, k8s.count_pdbs_with_label(cluster_label)) + self.assertEqual(0, k8s.count_secrets_with_label(cluster_label)) + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise def get_failover_targets(self, master_node, replica_nodes): ''' From e03e9f919a0f5b8effc6ab3a6ca91f051566c196 Mon Sep 17 00:00:00 2001 From: hlihhovac Date: Mon, 31 Aug 2020 12:28:52 +0200 Subject: [PATCH 084/168] add missing omitempty directive to the attributes of PostgresSpec (#1128) Co-authored-by: Pavlo Golub --- pkg/apis/acid.zalan.do/v1/postgresql_type.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 24ef24d63..b9ac6b660 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -53,7 +53,7 @@ type PostgresSpec struct { NumberOfInstances int32 `json:"numberOfInstances"` Users map[string]UserFlags `json:"users"` MaintenanceWindows []MaintenanceWindow `json:"maintenanceWindows,omitempty"` - Clone *CloneDescription `json:"clone"` + Clone *CloneDescription `json:"clone,omitempty"` ClusterName string `json:"-"` Databases map[string]string `json:"databases,omitempty"` PreparedDatabases map[string]PreparedDatabase `json:"preparedDatabases,omitempty"` @@ -64,10 +64,10 @@ type PostgresSpec struct { ShmVolume *bool `json:"enableShmVolume,omitempty"` EnableLogicalBackup bool `json:"enableLogicalBackup,omitempty"` LogicalBackupSchedule string `json:"logicalBackupSchedule,omitempty"` - StandbyCluster *StandbyDescription `json:"standby"` - PodAnnotations map[string]string `json:"podAnnotations"` - ServiceAnnotations map[string]string `json:"serviceAnnotations"` - TLS *TLSDescription `json:"tls"` + StandbyCluster *StandbyDescription `json:"standby,omitempty"` + PodAnnotations map[string]string `json:"podAnnotations,omitempty"` + ServiceAnnotations map[string]string `json:"serviceAnnotations,omitempty"` + TLS *TLSDescription `json:"tls,omitempty"` AdditionalVolumes []AdditionalVolume `json:"additionalVolumes,omitempty"` // deprecated json tags From 03437b63749e9b3bd51cdc9fd6b2f38bd6bfa8d6 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 3 Sep 2020 08:02:46 +0200 Subject: [PATCH 085/168] Update issue templates (#1051) * Update issue templates To help us helping them * update the template * some updates * or not on --- .../postgres-operator-issue-template.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/postgres-operator-issue-template.md diff --git a/.github/ISSUE_TEMPLATE/postgres-operator-issue-template.md b/.github/ISSUE_TEMPLATE/postgres-operator-issue-template.md new file mode 100644 index 000000000..ff7567d2d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/postgres-operator-issue-template.md @@ -0,0 +1,19 @@ +--- +name: Postgres Operator issue template +about: How are you using the operator? +title: '' +labels: '' +assignees: '' + +--- + +Please, answer some short questions which should help us to understand your problem / question better? + +- **Which image of the operator are you using?** e.g. registry.opensource.zalan.do/acid/postgres-operator:v1.5.0 +- **Where do you run it - cloud or metal? Kubernetes or OpenShift?** [AWS K8s | GCP ... | Bare Metal K8s] +- **Are you running Postgres Operator in production?** [yes | no] +- **Type of issue?** [Bug report, question, feature request, etc.] + +Some general remarks when posting a bug report: +- Please, check the operator, pod (Patroni) and postgresql logs first. When copy-pasting many log lines please do it in a separate GitHub gist together with your Postgres CRD and configuration manifest. +- If you feel this issue might be more related to the [Spilo](https://github.com/zalando/spilo/issues) docker image or [Patroni](https://github.com/zalando/patroni/issues), consider opening issues in the respective repos. From d8884a40038eef397aed0e504b5241820ef65c5f Mon Sep 17 00:00:00 2001 From: Igor Yanchenko <1504692+yanchenko-igor@users.noreply.github.com> Date: Tue, 15 Sep 2020 14:19:22 +0300 Subject: [PATCH 086/168] Allow to overwrite default ExternalTrafficPolicy for the service (#1136) * Allow to overwrite default ExternalTrafficPolicy for the service --- docs/reference/operator_parameters.md | 3 + pkg/apis/acid.zalan.do/v1/crds.go | 11 +++ .../v1/operator_configuration_type.go | 1 + pkg/cluster/k8sres.go | 1 + pkg/cluster/k8sres_test.go | 80 +++++++++++++++++++ pkg/controller/operator_config.go | 1 + pkg/util/config/config.go | 2 + 7 files changed, 99 insertions(+) diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 9fa622de8..a1ec0fab3 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -460,6 +460,9 @@ In the CRD-based configuration they are grouped under the `load_balancer` key. replaced with the hosted zone (the value of the `db_hosted_zone` parameter). No other placeholders are allowed. +* **external_traffic_policy** define external traffic policy for the load +balancer, it will default to `Cluster` if undefined. + ## AWS or GCP interaction The options in this group configure operator interactions with non-Kubernetes diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 43c313c16..d49399f6e 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -1129,6 +1129,17 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation "replica_dns_name_format": { Type: "string", }, + "external_traffic_policy": { + Type: "string", + Enum: []apiextv1beta1.JSON{ + { + Raw: []byte(`"Cluster"`), + }, + { + Raw: []byte(`"Local"`), + }, + }, + }, }, }, "aws_or_gcp": { diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 157596123..2351b16aa 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -109,6 +109,7 @@ type LoadBalancerConfiguration struct { CustomServiceAnnotations map[string]string `json:"custom_service_annotations,omitempty"` MasterDNSNameFormat config.StringTemplate `json:"master_dns_name_format,omitempty"` ReplicaDNSNameFormat config.StringTemplate `json:"replica_dns_name_format,omitempty"` + ExternalTrafficPolicy string `json:"external_traffic_policy" default:"Cluster"` } // AWSGCPConfiguration defines the configuration for AWS diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index d7878942c..c7824e5ad 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1619,6 +1619,7 @@ func (c *Cluster) generateService(role PostgresRole, spec *acidv1.PostgresSpec) } c.logger.Debugf("final load balancer source ranges as seen in a service spec (not necessarily applied): %q", serviceSpec.LoadBalancerSourceRanges) + serviceSpec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyType(c.OpConfig.ExternalTrafficPolicy) serviceSpec.Type = v1.ServiceTypeLoadBalancer } else if role == Replica { // before PR #258, the replica service was only created if allocated a LB diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 1e474fbf5..5c92a788f 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -1742,3 +1742,83 @@ func TestSidecars(t *testing.T) { }) } + +func TestGenerateService(t *testing.T) { + var spec acidv1.PostgresSpec + var cluster *Cluster + var enableLB bool = true + spec = acidv1.PostgresSpec{ + TeamID: "myapp", NumberOfInstances: 1, + Resources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + }, + Volume: acidv1.Volume{ + Size: "1G", + }, + Sidecars: []acidv1.Sidecar{ + acidv1.Sidecar{ + Name: "cluster-specific-sidecar", + }, + acidv1.Sidecar{ + Name: "cluster-specific-sidecar-with-resources", + Resources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "210m", Memory: "0.8Gi"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "510m", Memory: "1.4Gi"}, + }, + }, + acidv1.Sidecar{ + Name: "replace-sidecar", + DockerImage: "overwrite-image", + }, + }, + EnableMasterLoadBalancer: &enableLB, + } + + cluster = New( + Config{ + OpConfig: config.Config{ + PodManagementPolicy: "ordered_ready", + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + Resources: config.Resources{ + DefaultCPURequest: "200m", + DefaultCPULimit: "500m", + DefaultMemoryRequest: "0.7Gi", + DefaultMemoryLimit: "1.3Gi", + }, + SidecarImages: map[string]string{ + "deprecated-global-sidecar": "image:123", + }, + SidecarContainers: []v1.Container{ + v1.Container{ + Name: "global-sidecar", + }, + // will be replaced by a cluster specific sidecar with the same name + v1.Container{ + Name: "replace-sidecar", + Image: "replaced-image", + }, + }, + Scalyr: config.Scalyr{ + ScalyrAPIKey: "abc", + ScalyrImage: "scalyr-image", + ScalyrCPURequest: "220m", + ScalyrCPULimit: "520m", + ScalyrMemoryRequest: "0.9Gi", + // ise default memory limit + }, + ExternalTrafficPolicy: "Cluster", + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) + + service := cluster.generateService(Master, &spec) + assert.Equal(t, v1.ServiceExternalTrafficPolicyTypeCluster, service.Spec.ExternalTrafficPolicy) + cluster.OpConfig.ExternalTrafficPolicy = "Local" + service = cluster.generateService(Master, &spec) + assert.Equal(t, v1.ServiceExternalTrafficPolicyTypeLocal, service.Spec.ExternalTrafficPolicy) + +} diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index aad9069b1..51cd9737f 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -124,6 +124,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.CustomServiceAnnotations = fromCRD.LoadBalancer.CustomServiceAnnotations result.MasterDNSNameFormat = fromCRD.LoadBalancer.MasterDNSNameFormat result.ReplicaDNSNameFormat = fromCRD.LoadBalancer.ReplicaDNSNameFormat + result.ExternalTrafficPolicy = util.Coalesce(fromCRD.LoadBalancer.ExternalTrafficPolicy, "Cluster") // AWS or GCP config result.WALES3Bucket = fromCRD.AWSGCP.WALES3Bucket diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 5f7559929..3255e61bf 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -175,6 +175,8 @@ type Config struct { EnablePodAntiAffinity bool `name:"enable_pod_antiaffinity" default:"false"` PodAntiAffinityTopologyKey string `name:"pod_antiaffinity_topology_key" default:"kubernetes.io/hostname"` StorageResizeMode string `name:"storage_resize_mode" default:"ebs"` + // ExternalTrafficPolicy for load balancer + ExternalTrafficPolicy string `name:"external_traffic_policy" default:"Cluster"` // deprecated and kept for backward compatibility EnableLoadBalancer *bool `name:"enable_load_balancer"` MasterDNSNameFormat StringTemplate `name:"master_dns_name_format" default:"{cluster}.{team}.{hostedzone}"` From d09e418b56472411ee794726aa1c2dfa1b8f7443 Mon Sep 17 00:00:00 2001 From: Rico Berger Date: Tue, 15 Sep 2020 13:27:59 +0200 Subject: [PATCH 087/168] Set user and group in security context (#1083) * Set user and group in security context --- .../crds/operatorconfigurations.yaml | 4 +++ .../postgres-operator/crds/postgresqls.yaml | 4 +++ charts/postgres-operator/values-crd.yaml | 3 +++ charts/postgres-operator/values.yaml | 3 +++ docs/reference/cluster_manifest.md | 10 +++++++ docs/reference/operator_parameters.md | 10 +++++++ manifests/complete-postgres-manifest.yaml | 2 ++ manifests/configmap.yaml | 2 ++ manifests/operatorconfiguration.crd.yaml | 4 +++ ...gresql-operator-default-configuration.yaml | 2 ++ manifests/postgresql.crd.yaml | 4 +++ pkg/apis/acid.zalan.do/v1/crds.go | 12 +++++++++ .../v1/operator_configuration_type.go | 2 ++ pkg/apis/acid.zalan.do/v1/postgresql_type.go | 4 ++- .../acid.zalan.do/v1/zz_generated.deepcopy.go | 20 ++++++++++++++ pkg/cluster/k8sres.go | 26 ++++++++++++++++++- pkg/cluster/k8sres_test.go | 6 ++++- pkg/controller/operator_config.go | 2 ++ pkg/util/config/config.go | 2 ++ 19 files changed, 119 insertions(+), 3 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index c3966b410..24e476c11 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -200,6 +200,10 @@ spec: type: string secret_name_template: type: string + spilo_runasuser: + type: integer + spilo_runasgroup: + type: integer spilo_fsgroup: type: integer spilo_privileged: diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 6df2de723..0d444e568 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -374,6 +374,10 @@ spec: items: type: object additionalProperties: true + spiloRunAsUser: + type: integer + spiloRunAsGroup: + type: integer spiloFSGroup: type: integer standby: diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 9f9100cab..1aeff87ff 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -127,6 +127,9 @@ configKubernetes: pod_terminate_grace_period: 5m # template for database user secrets generated by the operator secret_name_template: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}" + # set user and group for the spilo container (required to run Spilo as non-root process) + # spilo_runasuser: "101" + # spilo_runasgroup: "103" # group ID with write-access to volumes (required to run Spilo as non-root process) # spilo_fsgroup: 103 diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index af918a67f..f72f375bf 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -118,6 +118,9 @@ configKubernetes: pod_terminate_grace_period: 5m # template for database user secrets generated by the operator secret_name_template: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}" + # set user and group for the spilo container (required to run Spilo as non-root process) + # spilo_runasuser: "101" + # spilo_runasgroup: "103" # group ID with write-access to volumes (required to run Spilo as non-root process) # spilo_fsgroup: "103" diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 576031543..70ab14855 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -65,6 +65,16 @@ These parameters are grouped directly under the `spec` key in the manifest. custom Docker image that overrides the **docker_image** operator parameter. It should be a [Spilo](https://github.com/zalando/spilo) image. Optional. +* **spiloRunAsUser** + sets the user ID which should be used in the container to run the process. + This must be set to run the container without root. By default the container + runs with root. This option only works for Spilo versions >= 1.6-p3. + +* **spiloRunAsGroup** + sets the group ID which should be used in the container to run the process. + This must be set to run the container without root. By default the container + runs with root. This option only works for Spilo versions >= 1.6-p3. + * **spiloFSGroup** the Persistent Volumes for the Spilo pods in the StatefulSet will be owned and writable by the group ID specified. This will override the **spilo_fsgroup** diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index a1ec0fab3..b21f6ac17 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -317,6 +317,16 @@ configuration they are grouped under the `kubernetes` key. that should be assigned to the Postgres pods. The priority class itself must be defined in advance. Default is empty (use the default priority class). +* **spilo_runasuser** + sets the user ID which should be used in the container to run the process. + This must be set to run the container without root. By default the container + runs with root. This option only works for Spilo versions >= 1.6-p3. + +* **spilo_runasgroup** + sets the group ID which should be used in the container to run the process. + This must be set to run the container without root. By default the container + runs with root. This option only works for Spilo versions >= 1.6-p3. + * **spilo_fsgroup** the Persistent Volumes for the Spilo pods in the StatefulSet will be owned and writable by the group ID specified. This is required to run Spilo as a diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 69f7a2d9f..79d1251e6 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -68,6 +68,8 @@ spec: # name: my-config-map enableShmVolume: true +# spiloRunAsUser: 101 +# spiloRunAsGroup: 103 # spiloFSGroup: 103 # podAnnotations: # annotation.key: value diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 3f3e331c4..db39ee33c 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -99,6 +99,8 @@ data: secret_name_template: "{username}.{cluster}.credentials" # sidecar_docker_images: "" # set_memory_request_to_limit: "false" + # spilo_runasuser: 101 + # spilo_runasgroup: 103 # spilo_fsgroup: 103 spilo_privileged: "false" # storage_resize_mode: "off" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 36db2dda8..23ab795ab 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -196,6 +196,10 @@ spec: type: string secret_name_template: type: string + spilo_runasuser: + type: integer + spilo_runasgroup: + type: integer spilo_fsgroup: type: integer spilo_privileged: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 7a029eccd..1fbfff529 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -68,6 +68,8 @@ configuration: # pod_service_account_role_binding_definition: "" pod_terminate_grace_period: 5m secret_name_template: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}" + # spilo_runasuser: 101 + # spilo_runasgroup: 103 # spilo_fsgroup: 103 spilo_privileged: false storage_resize_mode: ebs diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 1d42e7254..97b72a8ca 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -370,6 +370,10 @@ spec: items: type: object additionalProperties: true + spiloRunAsUser: + type: integer + spiloRunAsGroup: + type: integer spiloFSGroup: type: integer standby: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index d49399f6e..b67ee60e2 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -519,6 +519,12 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, }, }, + "spiloRunAsUser": { + Type: "integer", + }, + "spiloRunAsGroup": { + Type: "integer", + }, "spiloFSGroup": { Type: "integer", }, @@ -1018,6 +1024,12 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation "secret_name_template": { Type: "string", }, + "spilo_runasuser": { + Type: "integer", + }, + "spilo_runasgroup": { + Type: "integer", + }, "spilo_fsgroup": { Type: "integer", }, diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 2351b16aa..ca3fa46d7 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -49,6 +49,8 @@ type KubernetesMetaConfiguration struct { PodServiceAccountRoleBindingDefinition string `json:"pod_service_account_role_binding_definition,omitempty"` PodTerminateGracePeriod Duration `json:"pod_terminate_grace_period,omitempty"` SpiloPrivileged bool `json:"spilo_privileged,omitempty"` + SpiloRunAsUser *int64 `json:"spilo_runasuser,omitempty"` + SpiloRunAsGroup *int64 `json:"spilo_runasgroup,omitempty"` SpiloFSGroup *int64 `json:"spilo_fsgroup,omitempty"` WatchedNamespace string `json:"watched_namespace,omitempty"` PDBNameFormat config.StringTemplate `json:"pdb_name_format,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index b9ac6b660..499a4cfda 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -35,7 +35,9 @@ type PostgresSpec struct { TeamID string `json:"teamId"` DockerImage string `json:"dockerImage,omitempty"` - SpiloFSGroup *int64 `json:"spiloFSGroup,omitempty"` + SpiloRunAsUser *int64 `json:"spiloRunAsUser,omitempty"` + SpiloRunAsGroup *int64 `json:"spiloRunAsGroup,omitempty"` + SpiloFSGroup *int64 `json:"spiloFSGroup,omitempty"` // vars that enable load balancers are pointers because it is important to know if any of them is omitted from the Postgres manifest // in that case the var evaluates to nil and the value is taken from the operator config diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index efc31d6b6..34e6b46e8 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -147,6 +147,16 @@ func (in *ConnectionPoolerConfiguration) DeepCopy() *ConnectionPoolerConfigurati // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfiguration) { *out = *in + if in.SpiloRunAsUser != nil { + in, out := &in.SpiloRunAsUser, &out.SpiloRunAsUser + *out = new(int64) + **out = **in + } + if in.SpiloRunAsGroup != nil { + in, out := &in.SpiloRunAsGroup, &out.SpiloRunAsGroup + *out = new(int64) + **out = **in + } if in.SpiloFSGroup != nil { in, out := &in.SpiloFSGroup, &out.SpiloFSGroup *out = new(int64) @@ -527,6 +537,16 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { *out = new(ConnectionPooler) (*in).DeepCopyInto(*out) } + if in.SpiloRunAsUser != nil { + in, out := &in.SpiloRunAsUser, &out.SpiloRunAsUser + *out = new(int64) + **out = **in + } + if in.SpiloRunAsGroup != nil { + in, out := &in.SpiloRunAsGroup, &out.SpiloRunAsGroup + *out = new(int64) + **out = **in + } if in.SpiloFSGroup != nil { in, out := &in.SpiloFSGroup, &out.SpiloFSGroup *out = new(int64) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index c7824e5ad..fef202538 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -557,6 +557,8 @@ func (c *Cluster) generatePodTemplate( initContainers []v1.Container, sidecarContainers []v1.Container, tolerationsSpec *[]v1.Toleration, + spiloRunAsUser *int64, + spiloRunAsGroup *int64, spiloFSGroup *int64, nodeAffinity *v1.Affinity, terminateGracePeriod int64, @@ -576,6 +578,14 @@ func (c *Cluster) generatePodTemplate( containers = append(containers, sidecarContainers...) securityContext := v1.PodSecurityContext{} + if spiloRunAsUser != nil { + securityContext.RunAsUser = spiloRunAsUser + } + + if spiloRunAsGroup != nil { + securityContext.RunAsGroup = spiloRunAsGroup + } + if spiloFSGroup != nil { securityContext.FSGroup = spiloFSGroup } @@ -1073,7 +1083,17 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef // pickup the docker image for the spilo container effectiveDockerImage := util.Coalesce(spec.DockerImage, c.OpConfig.DockerImage) - // determine the FSGroup for the spilo pod + // determine the User, Group and FSGroup for the spilo pod + effectiveRunAsUser := c.OpConfig.Resources.SpiloRunAsUser + if spec.SpiloRunAsUser != nil { + effectiveRunAsUser = spec.SpiloRunAsUser + } + + effectiveRunAsGroup := c.OpConfig.Resources.SpiloRunAsGroup + if spec.SpiloRunAsGroup != nil { + effectiveRunAsGroup = spec.SpiloRunAsGroup + } + effectiveFSGroup := c.OpConfig.Resources.SpiloFSGroup if spec.SpiloFSGroup != nil { effectiveFSGroup = spec.SpiloFSGroup @@ -1217,6 +1237,8 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef initContainers, sidecarContainers, &tolerationSpec, + effectiveRunAsUser, + effectiveRunAsGroup, effectiveFSGroup, nodeAffinity(c.OpConfig.NodeReadinessLabel), int64(c.OpConfig.PodTerminateGracePeriod.Seconds()), @@ -1897,6 +1919,8 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) { []v1.Container{}, &[]v1.Toleration{}, nil, + nil, + nil, nodeAffinity(c.OpConfig.NodeReadinessLabel), int64(c.OpConfig.PodTerminateGracePeriod.Seconds()), c.OpConfig.PodServiceAccountName, diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 5c92a788f..f44b071bb 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -1302,6 +1302,8 @@ func TestTLS(t *testing.T) { var err error var spec acidv1.PostgresSpec var cluster *Cluster + var spiloRunAsUser = int64(101) + var spiloRunAsGroup = int64(103) var spiloFSGroup = int64(103) var additionalVolumes = spec.AdditionalVolumes @@ -1329,7 +1331,9 @@ func TestTLS(t *testing.T) { ReplicationUsername: replicationUserName, }, Resources: config.Resources{ - SpiloFSGroup: &spiloFSGroup, + SpiloRunAsUser: &spiloRunAsUser, + SpiloRunAsGroup: &spiloRunAsGroup, + SpiloFSGroup: &spiloFSGroup, }, }, }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 51cd9737f..7e4880712 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -61,6 +61,8 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.PodEnvironmentSecret = fromCRD.Kubernetes.PodEnvironmentSecret result.PodTerminateGracePeriod = util.CoalesceDuration(time.Duration(fromCRD.Kubernetes.PodTerminateGracePeriod), "5m") result.SpiloPrivileged = fromCRD.Kubernetes.SpiloPrivileged + result.SpiloRunAsUser = fromCRD.Kubernetes.SpiloRunAsUser + result.SpiloRunAsGroup = fromCRD.Kubernetes.SpiloRunAsGroup result.SpiloFSGroup = fromCRD.Kubernetes.SpiloFSGroup result.ClusterDomain = util.Coalesce(fromCRD.Kubernetes.ClusterDomain, "cluster.local") result.WatchedNamespace = fromCRD.Kubernetes.WatchedNamespace diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 3255e61bf..2a2103f5a 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -28,6 +28,8 @@ type Resources struct { PodLabelWaitTimeout time.Duration `name:"pod_label_wait_timeout" default:"10m"` PodDeletionWaitTimeout time.Duration `name:"pod_deletion_wait_timeout" default:"10m"` PodTerminateGracePeriod time.Duration `name:"pod_terminate_grace_period" default:"5m"` + SpiloRunAsUser *int64 `json:"spilo_runasuser,omitempty"` + SpiloRunAsGroup *int64 `json:"spilo_runasgroup,omitempty"` SpiloFSGroup *int64 `name:"spilo_fsgroup"` PodPriorityClassName string `name:"pod_priority_class_name"` ClusterDomain string `name:"cluster_domain" default:"cluster.local"` From ab95eaa6ef9ea072a386c365e7b685b1de6b89a7 Mon Sep 17 00:00:00 2001 From: neelasha-09 <66790082+neelasha-09@users.noreply.github.com> Date: Tue, 22 Sep 2020 20:46:05 +0530 Subject: [PATCH 088/168] Fixes #1130 (#1139) * Fixes #1130 Co-authored-by: Felix Kunde --- pkg/cluster/sync.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 056e43043..fef5b7b66 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -696,12 +696,8 @@ func (c *Cluster) syncPreparedDatabases() error { if err := c.initDbConnWithName(preparedDbName); err != nil { return fmt.Errorf("could not init connection to database %s: %v", preparedDbName, err) } - defer func() { - if err := c.closeDbConn(); err != nil { - c.logger.Errorf("could not close database connection: %v", err) - } - }() + c.logger.Debugf("syncing prepared database %q", preparedDbName) // now, prepare defined schemas preparedSchemas := preparedDB.PreparedSchemas if len(preparedDB.PreparedSchemas) == 0 { @@ -715,6 +711,10 @@ func (c *Cluster) syncPreparedDatabases() error { if err := c.syncExtensions(preparedDB.Extensions); err != nil { return err } + + if err := c.closeDbConn(); err != nil { + c.logger.Errorf("could not close database connection: %v", err) + } } return nil From 2a21cc4393c8de4c2f1a0c666b27fbb3b5bb73f2 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Wed, 23 Sep 2020 17:26:56 +0200 Subject: [PATCH 089/168] Compare Postgres pod priority on Sync (#1144) * compare Postgres pod priority on Sync Co-authored-by: Sergey Dudoladov --- .travis.yml | 2 +- delivery.yaml | 4 ++++ manifests/configmap.yaml | 1 + manifests/postgres-pod-priority-class.yaml | 11 +++++++++++ pkg/cluster/cluster.go | 9 +++++++++ 5 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 manifests/postgres-pod-priority-class.yaml diff --git a/.travis.yml b/.travis.yml index a52769c91..1239596fc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,4 +20,4 @@ script: - hack/verify-codegen.sh - travis_wait 20 go test -race -covermode atomic -coverprofile=profile.cov ./pkg/... -v - goveralls -coverprofile=profile.cov -service=travis-ci -v - - make e2e + - travis_wait 20 make e2e diff --git a/delivery.yaml b/delivery.yaml index 07c768424..d1eec8a2b 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -2,6 +2,10 @@ version: "2017-09-20" pipeline: - id: build-postgres-operator type: script + vm: large + cache: + paths: + - /go/pkg/mod commands: - desc: 'Update' cmd: | diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index db39ee33c..998e0e45f 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -85,6 +85,7 @@ data: pod_service_account_name: "postgres-pod" # pod_service_account_role_binding_definition: "" pod_terminate_grace_period: 5m + # pod_priority_class_name: "postgres-pod-priority" # postgres_superuser_teams: "postgres_superusers" # protected_role_names: "admin" ready_wait_interval: 3s diff --git a/manifests/postgres-pod-priority-class.yaml b/manifests/postgres-pod-priority-class.yaml new file mode 100644 index 000000000..f1b565f21 --- /dev/null +++ b/manifests/postgres-pod-priority-class.yaml @@ -0,0 +1,11 @@ +apiVersion: scheduling.k8s.io/v1 +description: 'This priority class must be used only for databases controlled by the + Postgres operator' +kind: PriorityClass +metadata: + labels: + application: postgres-operator + name: postgres-pod-priority +preemptionPolicy: PreemptLowerPriority +globalDefault: false +value: 1000000 diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 51c5d3809..9b8b51eb0 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -459,6 +459,15 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa } } + // we assume any change in priority happens by rolling out a new priority class + // changing the priority value in an existing class is not supproted + if c.Statefulset.Spec.Template.Spec.PriorityClassName != statefulSet.Spec.Template.Spec.PriorityClassName { + match = false + needsReplace = true + needsRollUpdate = true + reasons = append(reasons, "new statefulset's pod priority class in spec doesn't match the current one") + } + // lazy Spilo update: modify the image in the statefulset itself but let its pods run with the old image // until they are re-created for other reasons, for example node rotation if c.OpConfig.EnableLazySpiloUpgrade && !reflect.DeepEqual(c.Statefulset.Spec.Template.Spec.Containers[0].Image, statefulSet.Spec.Template.Spec.Containers[0].Image) { From ffdb47f53a3749050fe36168a70931b187f36ae1 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 25 Sep 2020 09:46:50 +0200 Subject: [PATCH 090/168] remove outdated GSOC info (#1148) Co-authored-by: Sergey Dudoladov --- README.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/README.md b/README.md index f65a97a23..28532e246 100644 --- a/README.md +++ b/README.md @@ -76,12 +76,6 @@ There is a browser-friendly version of this documentation at * [Postgres manifest reference](docs/reference/cluster_manifest.md) * [Command-line options and environment variables](docs/reference/command_line_and_environment.md) -## Google Summer of Code - -The Postgres Operator made it to the [Google Summer of Code 2019](https://summerofcode.withgoogle.com/organizations/5429926902104064/)! -Check [our ideas](docs/gsoc-2019/ideas.md#google-summer-of-code-2019) -and start discussions in [the issue tracker](https://github.com/zalando/postgres-operator/issues). - ## Community There are two places to get in touch with the community: From 3b6dc4f92d6506ca28d1972ee3cc95144877dacb Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 25 Sep 2020 14:14:19 +0200 Subject: [PATCH 091/168] Improve e2e tests (#1111) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * icnrease vm size * cache deps * switch to the absolute cache path as cdp does not support shell expansion * do not pull non-existing image * manually install kind * add alias to kind * use full kind name * one more name change * install kind with other tools * add bind mounts instead of copying files * test fetching the runner image * build image for pierone * bump up the client-go version to match the master * bump up go version * install pinned version of kind before any test run * do not overwrite local ./manifests during test run * update the docs * fix kind name * update go.* files * fix deps * avoid unnecessary image upload * properly install kind * Change network to host to make it reachable within e2e runner. May not be the right solution though. * Small changes. Also use entrypoint vs cmd. * Bumping spilo. Load before test. * undo incorrect merge from the master Co-authored-by: Sergey Dudoladov Co-authored-by: Jan Mußler --- Makefile | 2 +- docs/developer.md | 8 ++-- e2e/Dockerfile | 20 ++++----- e2e/Makefile | 19 +++++--- ...d-cluster-postgres-operator-e2e-tests.yaml | 2 +- e2e/run.sh | 43 ++++++++++++------- e2e/tests/test_e2e.py | 11 +++-- go.mod | 4 +- go.sum | 21 ++------- manifests/configmap.yaml | 4 +- 10 files changed, 71 insertions(+), 63 deletions(-) diff --git a/Makefile b/Makefile index 7ecf54f7c..29bbb47e6 100644 --- a/Makefile +++ b/Makefile @@ -97,4 +97,4 @@ test: GO111MODULE=on go test ./... e2e: docker # build operator image to be tested - cd e2e; make tools e2etest clean + cd e2e; make e2etest diff --git a/docs/developer.md b/docs/developer.md index 6e0fc33c8..59fbe09a2 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -237,9 +237,11 @@ kubectl logs acid-minimal-cluster-0 ## End-to-end tests -The operator provides reference end-to-end tests (e2e) (as Docker image) to -ensure various infrastructure parts work smoothly together. Each e2e execution -tests a Postgres Operator image built from the current git branch. The test +The operator provides reference end-to-end (e2e) tests to +ensure various infrastructure parts work smoothly together. The test code is available at `e2e/tests`. +The special `registry.opensource.zalan.do/acid/postgres-operator-e2e-tests-runner` image is used to run the tests. The container mounts the local `e2e/tests` directory at runtime, so whatever you modify in your local copy of the tests will be executed by a test runner. By maintaining a separate test runner image we avoid the need to re-build the e2e test image on every build. + +Each e2e execution tests a Postgres Operator image built from the current git branch. The test runner creates a new local K8s cluster using [kind](https://kind.sigs.k8s.io/), utilizes provided manifest examples, and runs e2e tests contained in the `tests` folder. The K8s API client in the container connects to the `kind` cluster via diff --git a/e2e/Dockerfile b/e2e/Dockerfile index a250ea9cb..70e6f0a84 100644 --- a/e2e/Dockerfile +++ b/e2e/Dockerfile @@ -1,11 +1,12 @@ -# An image to perform the actual test. Do not forget to copy all necessary test -# files here. -FROM ubuntu:18.04 +# An image to run e2e tests. +# The image does not include the tests; all necessary files are bind-mounted when a container starts. +FROM ubuntu:20.04 LABEL maintainer="Team ACID @ Zalando " -COPY manifests ./manifests -COPY exec.sh ./exec.sh -COPY requirements.txt tests ./ +ENV TERM xterm-256color + +COPY requirements.txt ./ +COPY scm-source.json ./ RUN apt-get update \ && apt-get install --no-install-recommends -y \ @@ -14,13 +15,10 @@ RUN apt-get update \ python3-pip \ curl \ && pip3 install --no-cache-dir -r requirements.txt \ - && curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl \ + && curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl \ && chmod +x ./kubectl \ && mv ./kubectl /usr/local/bin/kubectl \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -ARG VERSION=dev -RUN sed -i "s/__version__ = .*/__version__ = '${VERSION}'/" ./__init__.py - -CMD ["python3", "-m", "unittest", "discover", "--start-directory", ".", "-v"] +ENTRYPOINT ["python3", "-m", "unittest", "discover", "--start-directory", ".", "-v"] diff --git a/e2e/Makefile b/e2e/Makefile index 16e3f2f99..05ea6a3d6 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -1,6 +1,6 @@ .PHONY: clean copy docker push tools test -BINARY ?= postgres-operator-e2e-tests +BINARY ?= postgres-operator-e2e-tests-runner BUILD_FLAGS ?= -v CGO_ENABLED ?= 0 ifeq ($(RACE),1) @@ -34,15 +34,20 @@ copy: clean mkdir manifests cp ../manifests -r . -docker: copy - docker build --build-arg "VERSION=$(VERSION)" -t "$(IMAGE):$(TAG)" . +docker: scm-source.json + docker build -t "$(IMAGE):$(TAG)" . + +scm-source.json: ../.git + echo '{\n "url": "git:$(GITURL)",\n "revision": "$(GITHEAD)",\n "author": "$(USER)",\n "status": "$(GITSTATUS)"\n}' > scm-source.json push: docker docker push "$(IMAGE):$(TAG)" -tools: docker +tools: # install pinned version of 'kind' - GO111MODULE=on go get sigs.k8s.io/kind@v0.8.1 + # go get must run outside of a dir with a (module-based) Go project ! + # otherwise go get updates project's dependencies and/or behaves differently + cd "/tmp" && GO111MODULE=on go get sigs.k8s.io/kind@v0.8.1 -e2etest: - ./run.sh +e2etest: tools copy clean + ./run.sh main diff --git a/e2e/kind-cluster-postgres-operator-e2e-tests.yaml b/e2e/kind-cluster-postgres-operator-e2e-tests.yaml index a59746fd3..752e993cd 100644 --- a/e2e/kind-cluster-postgres-operator-e2e-tests.yaml +++ b/e2e/kind-cluster-postgres-operator-e2e-tests.yaml @@ -1,5 +1,5 @@ kind: Cluster -apiVersion: kind.sigs.k8s.io/v1alpha3 +apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane - role: worker diff --git a/e2e/run.sh b/e2e/run.sh index 9d7e2eba7..74d842879 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -6,29 +6,29 @@ set -o nounset set -o pipefail IFS=$'\n\t' -cd $(dirname "$0"); - readonly cluster_name="postgres-operator-e2e-tests" readonly kubeconfig_path="/tmp/kind-config-${cluster_name}" +readonly spilo_image="registry.opensource.zalan.do/acid/spilo-12:1.6-p5" + +echo "Clustername: ${cluster_name}" +echo "Kubeconfig path: ${kubeconfig_path}" function pull_images(){ - operator_tag=$(git describe --tags --always --dirty) if [[ -z $(docker images -q registry.opensource.zalan.do/acid/postgres-operator:${operator_tag}) ]] then docker pull registry.opensource.zalan.do/acid/postgres-operator:latest fi - if [[ -z $(docker images -q registry.opensource.zalan.do/acid/postgres-operator-e2e-tests:${operator_tag}) ]] - then - docker pull registry.opensource.zalan.do/acid/postgres-operator-e2e-tests:latest - fi operator_image=$(docker images --filter=reference="registry.opensource.zalan.do/acid/postgres-operator" --format "{{.Repository}}:{{.Tag}}" | head -1) - e2e_test_image=$(docker images --filter=reference="registry.opensource.zalan.do/acid/postgres-operator-e2e-tests" --format "{{.Repository}}:{{.Tag}}" | head -1) + + # this image does not contain the tests; a container mounts them from a local "./tests" dir at start time + e2e_test_runner_image="registry.opensource.zalan.do/acid/postgres-operator-e2e-tests-runner:latest" + docker pull ${e2e_test_runner_image} } function start_kind(){ - + echo "Starting kind for e2e tests" # avoid interference with previous test runs if [[ $(kind get clusters | grep "^${cluster_name}*") != "" ]] then @@ -38,10 +38,12 @@ function start_kind(){ export KUBECONFIG="${kubeconfig_path}" kind create cluster --name ${cluster_name} --config kind-cluster-postgres-operator-e2e-tests.yaml kind load docker-image "${operator_image}" --name ${cluster_name} - kind load docker-image "${e2e_test_image}" --name ${cluster_name} + docker pull "${spilo_image}" + kind load docker-image "${spilo_image}" --name ${cluster_name} } function set_kind_api_server_ip(){ + echo "Setting up kind API server ip" # use the actual kubeconfig to connect to the 'kind' API server # but update the IP address of the API server to the one from the Docker 'bridge' network readonly local kind_api_server_port=6443 # well-known in the 'kind' codebase @@ -50,10 +52,21 @@ function set_kind_api_server_ip(){ } function run_tests(){ - docker run --rm --network kind --mount type=bind,source="$(readlink -f ${kubeconfig_path})",target=/root/.kube/config -e OPERATOR_IMAGE="${operator_image}" "${e2e_test_image}" + echo "Running tests..." + + # tests modify files in ./manifests, so we mount a copy of this directory done by the e2e Makefile + + docker run --rm --network=host -e "TERM=xterm-256color" \ + --mount type=bind,source="$(readlink -f ${kubeconfig_path})",target=/root/.kube/config \ + --mount type=bind,source="$(readlink -f manifests)",target=/manifests \ + --mount type=bind,source="$(readlink -f tests)",target=/tests \ + --mount type=bind,source="$(readlink -f exec.sh)",target=/exec.sh \ + -e OPERATOR_IMAGE="${operator_image}" "${e2e_test_runner_image}" + } function clean_up(){ + echo "Executing cleanup" unset KUBECONFIG kind delete cluster --name ${cluster_name} rm -rf ${kubeconfig_path} @@ -63,11 +76,11 @@ function main(){ trap "clean_up" QUIT TERM EXIT - pull_images - start_kind - set_kind_api_server_ip + time pull_images + time start_kind + time set_kind_api_server_ip run_tests exit 0 } -main "$@" +"$@" diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index ce2d392e1..550d3ced8 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -34,6 +34,7 @@ class EndToEndTestCase(unittest.TestCase): In the case of test failure the cluster will stay to enable manual examination; next invocation of "make test" will re-create it. ''' + print("Test Setup being executed") # set a single K8s wrapper for all tests k8s = cls.k8s = K8s() @@ -216,7 +217,8 @@ class EndToEndTestCase(unittest.TestCase): k8s = self.k8s # update infrastructure roles description secret_name = "postgresql-infrastructure-roles" - roles = "secretname: postgresql-infrastructure-roles-new, userkey: user, rolekey: memberof, passwordkey: password, defaultrolevalue: robot_zmon" + roles = "secretname: postgresql-infrastructure-roles-new, \ + userkey: user, rolekey: memberof, passwordkey: password, defaultrolevalue: robot_zmon" patch_infrastructure_roles = { "data": { "infrastructure_roles_secret_name": secret_name, @@ -313,7 +315,8 @@ class EndToEndTestCase(unittest.TestCase): image0 = k8s.get_effective_pod_image(pod0) image1 = k8s.get_effective_pod_image(pod1) - assert_msg = "Disabling lazy upgrade failed: pods still have different images {} and {}".format(image0, image1) + assert_msg = "Disabling lazy upgrade failed: pods still have different \ + images {} and {}".format(image0, image1) self.assertEqual(image0, image1, assert_msg) except timeout_decorator.TimeoutError: @@ -710,11 +713,11 @@ class EndToEndTestCase(unittest.TestCase): time.sleep(10) # add annotations to manifest - deleteDate = datetime.today().strftime('%Y-%m-%d') + delete_date = datetime.today().strftime('%Y-%m-%d') pg_patch_delete_annotations = { "metadata": { "annotations": { - "delete-date": deleteDate, + "delete-date": delete_date, "delete-clustername": "acid-minimal-cluster", } } diff --git a/go.mod b/go.mod index 74f8dc5e1..91267bfad 100644 --- a/go.mod +++ b/go.mod @@ -10,12 +10,12 @@ require ( github.com/sirupsen/logrus v1.6.0 github.com/stretchr/testify v1.5.1 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 - golang.org/x/tools v0.0.0-20200826040757-bc8aaaa29e06 // indirect + golang.org/x/tools v0.0.0-20200828161849-5deb26317202 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gopkg.in/yaml.v2 v2.2.8 k8s.io/api v0.18.8 k8s.io/apiextensions-apiserver v0.18.0 k8s.io/apimachinery v0.18.8 - k8s.io/client-go v0.18.6 + k8s.io/client-go v0.18.8 k8s.io/code-generator v0.18.8 ) diff --git a/go.sum b/go.sum index a7787e0fd..1a59b280a 100644 --- a/go.sum +++ b/go.sum @@ -137,7 +137,6 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -146,7 +145,6 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -156,7 +154,6 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.1.0 h1:rVsPeBmXbYv4If/cumu1AzZPwV58q433hvONV1UEZoI= github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= @@ -181,7 +178,6 @@ github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2 github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -272,7 +268,6 @@ github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tL github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -281,7 +276,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= @@ -375,7 +369,6 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -393,11 +386,10 @@ golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200826040757-bc8aaaa29e06 h1:ChBCbOHeLqK+j+znGPlWCcvx/t2PdxmyPBheVZxXbcc= -golang.org/x/tools v0.0.0-20200826040757-bc8aaaa29e06/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200828161849-5deb26317202 h1:DrWbY9UUFi/sl/3HkNVoBjDbGfIPZZfgoGsGxOL1EU8= +golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -431,7 +423,6 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -441,20 +432,17 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.18.0/go.mod h1:q2HRQkfDzHMBZL9l/y9rH63PkQl4vae0xRT+8prbrK8= -k8s.io/api v0.18.6/go.mod h1:eeyxr+cwCjMdLAmr2W3RyDI0VvTawSg/3RFFBEnmZGI= k8s.io/api v0.18.8 h1:aIKUzJPb96f3fKec2lxtY7acZC9gQNDLVhfSGpxBAC4= k8s.io/api v0.18.8/go.mod h1:d/CXqwWv+Z2XEG1LgceeDmHQwpUJhROPx16SlxJgERY= k8s.io/apiextensions-apiserver v0.18.0 h1:HN4/P8vpGZFvB5SOMuPPH2Wt9Y/ryX+KRvIyAkchu1Q= k8s.io/apiextensions-apiserver v0.18.0/go.mod h1:18Cwn1Xws4xnWQNC00FLq1E350b9lUF+aOdIWDOZxgo= k8s.io/apimachinery v0.18.0/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= -k8s.io/apimachinery v0.18.6/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= k8s.io/apimachinery v0.18.8 h1:jimPrycCqgx2QPearX3to1JePz7wSbVLq+7PdBTTwQ0= k8s.io/apimachinery v0.18.8/go.mod h1:6sQd+iHEqmOtALqOFjSWp2KZ9F0wlU/nWm0ZgsYWMig= k8s.io/apiserver v0.18.0/go.mod h1:3S2O6FeBBd6XTo0njUrLxiqk8GNy6wWOftjhJcXYnjw= -k8s.io/client-go v0.18.0 h1:yqKw4cTUQraZK3fcVCMeSa+lqKwcjZ5wtcOIPnxQno4= k8s.io/client-go v0.18.0/go.mod h1:uQSYDYs4WhVZ9i6AIoEZuwUggLVEF64HOD37boKAtF8= -k8s.io/client-go v0.18.6 h1:I+oWqJbibLSGsZj8Xs8F0aWVXJVIoUHWaaJV3kUN/Zw= -k8s.io/client-go v0.18.6/go.mod h1:/fwtGLjYMS1MaM5oi+eXhKwG+1UHidUEXRh6cNsdO0Q= +k8s.io/client-go v0.18.8 h1:SdbLpIxk5j5YbFr1b7fq8S7mDgDjYmUxSbszyoesoDM= +k8s.io/client-go v0.18.8/go.mod h1:HqFqMllQ5NnQJNwjro9k5zMyfhZlOwpuTLVrxjkYSxU= k8s.io/code-generator v0.18.0/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= k8s.io/code-generator v0.18.8 h1:lgO1P1wjikEtzNvj7ia+x1VC4svJ28a/r0wnOLhhOTU= k8s.io/code-generator v0.18.8/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c= @@ -475,7 +463,6 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E= sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= -sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 998e0e45f..fc374c754 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -15,7 +15,7 @@ data: # connection_pooler_default_cpu_request: "500m" # connection_pooler_default_memory_limit: 100Mi # connection_pooler_default_memory_request: 100Mi - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-9" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-11" # connection_pooler_max_db_connections: 60 # connection_pooler_mode: "transaction" # connection_pooler_number_of_instances: 2 @@ -31,7 +31,7 @@ data: # default_memory_request: 100Mi # delete_annotation_date_key: delete-date # delete_annotation_name_key: delete-clustername - docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p3 + docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p5 # downscaler_annotations: "deployment-time,downscaler/*" # enable_admin_role_for_users: "true" # enable_crd_validation: "true" From 21475f4547d8ff62eb6cc1e085bc92e1401933a2 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 30 Sep 2020 17:24:14 +0200 Subject: [PATCH 092/168] Cleanup config examples (#1151) * post polishing for latest PRs * update travis and go modules * make deprecation comments in structs less confusing * have separate pod priority class es for operator and database pods --- .travis.yml | 2 +- .../crds/operatorconfigurations.yaml | 5 ++ .../templates/configmap.yaml | 3 + .../templates/operatorconfiguration.yaml | 3 + .../postgres-pod-priority-class.yaml | 15 +++++ charts/postgres-operator/values-crd.yaml | 6 ++ charts/postgres-operator/values.yaml | 6 ++ docs/reference/operator_parameters.md | 16 +++--- go.mod | 3 +- go.sum | 11 ++-- manifests/configmap.yaml | 3 +- manifests/operatorconfiguration.crd.yaml | 5 ++ ...gresql-operator-default-configuration.yaml | 9 +-- pkg/apis/acid.zalan.do/v1/crds.go | 12 ++-- .../v1/operator_configuration_type.go | 27 +++++---- pkg/util/config/config.go | 55 +++++++++---------- 16 files changed, 110 insertions(+), 71 deletions(-) create mode 100644 charts/postgres-operator/templates/postgres-pod-priority-class.yaml diff --git a/.travis.yml b/.travis.yml index 1239596fc..a52769c91 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,4 +20,4 @@ script: - hack/verify-codegen.sh - travis_wait 20 go test -race -covermode atomic -coverprofile=profile.cov ./pkg/... -v - goveralls -coverprofile=profile.cov -service=travis-ci -v - - travis_wait 20 make e2e + - make e2e diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 24e476c11..8b576822c 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -263,6 +263,11 @@ spec: type: boolean enable_replica_load_balancer: type: boolean + external_traffic_policy: + type: string + enum: + - "Cluster" + - "Local" master_dns_name_format: type: string replica_dns_name_format: diff --git a/charts/postgres-operator/templates/configmap.yaml b/charts/postgres-operator/templates/configmap.yaml index 64b55e0df..87fd752b1 100644 --- a/charts/postgres-operator/templates/configmap.yaml +++ b/charts/postgres-operator/templates/configmap.yaml @@ -9,6 +9,9 @@ metadata: app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/instance: {{ .Release.Name }} data: + {{- if .Values.podPriorityClassName }} + pod_priority_class_name: {{ .Values.podPriorityClassName }} + {{- end }} pod_service_account_name: {{ include "postgres-pod.serviceAccountName" . }} {{ toYaml .Values.configGeneral | indent 2 }} {{ toYaml .Values.configUsers | indent 2 }} diff --git a/charts/postgres-operator/templates/operatorconfiguration.yaml b/charts/postgres-operator/templates/operatorconfiguration.yaml index d28d68f9c..0625e1327 100644 --- a/charts/postgres-operator/templates/operatorconfiguration.yaml +++ b/charts/postgres-operator/templates/operatorconfiguration.yaml @@ -13,6 +13,9 @@ configuration: users: {{ toYaml .Values.configUsers | indent 4 }} kubernetes: + {{- if .Values.podPriorityClassName }} + pod_priority_class_name: {{ .Values.podPriorityClassName }} + {{- end }} pod_service_account_name: {{ include "postgres-pod.serviceAccountName" . }} oauth_token_secret_name: {{ template "postgres-operator.fullname" . }} {{ toYaml .Values.configKubernetes | indent 4 }} diff --git a/charts/postgres-operator/templates/postgres-pod-priority-class.yaml b/charts/postgres-operator/templates/postgres-pod-priority-class.yaml new file mode 100644 index 000000000..7ee0f2e55 --- /dev/null +++ b/charts/postgres-operator/templates/postgres-pod-priority-class.yaml @@ -0,0 +1,15 @@ +{{- if .Values.podPriorityClassName }} +apiVersion: scheduling.k8s.io/v1 +description: 'Use only for databases controlled by Postgres operator' +kind: PriorityClass +metadata: + labels: + app.kubernetes.io/name: {{ template "postgres-operator.name" . }} + helm.sh/chart: {{ template "postgres-operator.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + name: {{ .Values.podPriorityClassName }} +preemptionPolicy: PreemptLowerPriority +globalDefault: false +value: 1000000 +{{- end }} diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 1aeff87ff..ffa8b7f51 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -183,6 +183,8 @@ configLoadBalancer: enable_master_load_balancer: false # toggles service type load balancer pointing to the replica pod of the cluster enable_replica_load_balancer: false + # define external traffic policy for the load balancer + external_traffic_policy: "Cluster" # defines the DNS name string template for the master load balancer cluster master_dns_name_format: "{cluster}.{team}.{hostedzone}" # defines the DNS name string template for the replica load balancer cluster @@ -318,8 +320,12 @@ podServiceAccount: # If not set a name is generated using the fullname template and "-pod" suffix name: "postgres-pod" +# priority class for operator pod priorityClassName: "" +# priority class for database pods +podPriorityClassName: "" + resources: limits: cpu: 500m diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index f72f375bf..37eac4254 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -172,6 +172,8 @@ configLoadBalancer: enable_master_load_balancer: "false" # toggles service type load balancer pointing to the replica pod of the cluster enable_replica_load_balancer: "false" + # define external traffic policy for the load balancer + external_traffic_policy: "Cluster" # defines the DNS name string template for the master load balancer cluster master_dns_name_format: '{cluster}.{team}.{hostedzone}' # defines the DNS name string template for the replica load balancer cluster @@ -310,8 +312,12 @@ podServiceAccount: # If not set a name is generated using the fullname template and "-pod" suffix name: "postgres-pod" +# priority class for operator pod priorityClassName: "" +# priority class for database pods +podPriorityClassName: "" + resources: limits: cpu: 500m diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index b21f6ac17..465465432 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -434,6 +434,12 @@ CRD-based configuration. Those options affect the behavior of load balancers created by the operator. In the CRD-based configuration they are grouped under the `load_balancer` key. +* **custom_service_annotations** + This key/value map provides a list of annotations that get attached to each + service of a cluster created by the operator. If the annotation key is also + provided by the cluster definition, the manifest value is used. + Optional. + * **db_hosted_zone** DNS zone for the cluster DNS name when the load balancer is configured for the cluster. Only used when combined with @@ -450,11 +456,8 @@ In the CRD-based configuration they are grouped under the `load_balancer` key. cluster. Can be overridden by individual cluster settings. The default is `false`. -* **custom_service_annotations** - This key/value map provides a list of annotations that get attached to each - service of a cluster created by the operator. If the annotation key is also - provided by the cluster definition, the manifest value is used. - Optional. +* **external_traffic_policy** defines external traffic policy for load + balancers. Allowed values are `Cluster` (default) and `Local`. * **master_dns_name_format** defines the DNS name string template for the master load balancer cluster. The default is @@ -470,9 +473,6 @@ In the CRD-based configuration they are grouped under the `load_balancer` key. replaced with the hosted zone (the value of the `db_hosted_zone` parameter). No other placeholders are allowed. -* **external_traffic_policy** define external traffic policy for the load -balancer, it will default to `Cluster` if undefined. - ## AWS or GCP interaction The options in this group configure operator interactions with non-Kubernetes diff --git a/go.mod b/go.mod index 91267bfad..79c3b9be9 100644 --- a/go.mod +++ b/go.mod @@ -10,8 +10,7 @@ require ( github.com/sirupsen/logrus v1.6.0 github.com/stretchr/testify v1.5.1 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 - golang.org/x/tools v0.0.0-20200828161849-5deb26317202 // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + golang.org/x/tools v0.0.0-20200928201943-a0ef9b62deab // indirect gopkg.in/yaml.v2 v2.2.8 k8s.io/api v0.18.8 k8s.io/apiextensions-apiserver v0.18.0 diff --git a/go.sum b/go.sum index 1a59b280a..2d76a94ee 100644 --- a/go.sum +++ b/go.sum @@ -287,7 +287,7 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -333,8 +333,8 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= @@ -386,11 +386,10 @@ golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200828161849-5deb26317202 h1:DrWbY9UUFi/sl/3HkNVoBjDbGfIPZZfgoGsGxOL1EU8= -golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200928201943-a0ef9b62deab h1:CyH2SDm5ATQiX9gtbMYfvNNed97A9v+TJFnUX/fTaJY= +golang.org/x/tools v0.0.0-20200928201943-a0ef9b62deab/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index fc374c754..970f845bf 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -47,6 +47,7 @@ data: # enable_team_superuser: "false" enable_teams_api: "false" # etcd_host: "" + external_traffic_policy: "Cluster" # gcp_credentials: "" # kubernetes_use_configmaps: "false" # infrastructure_roles_secret_name: "postgresql-infrastructure-roles" @@ -80,12 +81,12 @@ data: # pod_environment_secret: "my-custom-secret" pod_label_wait_timeout: 10m pod_management_policy: "ordered_ready" + # pod_priority_class_name: "postgres-pod-priority" pod_role_label: spilo-role # pod_service_account_definition: "" pod_service_account_name: "postgres-pod" # pod_service_account_role_binding_definition: "" pod_terminate_grace_period: 5m - # pod_priority_class_name: "postgres-pod-priority" # postgres_superuser_teams: "postgres_superusers" # protected_role_names: "admin" ready_wait_interval: 3s diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 23ab795ab..515f87438 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -265,6 +265,11 @@ spec: type: boolean enable_replica_load_balancer: type: boolean + external_traffic_policy: + type: string + enum: + - "Cluster" + - "Local" master_dns_name_format: type: string replica_dns_name_format: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 1fbfff529..5fb77bf76 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -61,7 +61,7 @@ configuration: # pod_environment_configmap: "default/my-custom-config" # pod_environment_secret: "my-custom-secret" pod_management_policy: "ordered_ready" - # pod_priority_class_name: "" + # pod_priority_class_name: "postgres-pod-priority" pod_role_label: spilo-role # pod_service_account_definition: "" pod_service_account_name: postgres-pod @@ -90,12 +90,13 @@ configuration: resource_check_interval: 3s resource_check_timeout: 10m load_balancer: - # db_hosted_zone: "" - enable_master_load_balancer: false - enable_replica_load_balancer: false # custom_service_annotations: # keyx: valuex # keyy: valuey + # db_hosted_zone: "" + enable_master_load_balancer: false + enable_replica_load_balancer: false + external_traffic_policy: "Cluster" master_dns_name_format: "{cluster}.{team}.{hostedzone}" replica_dns_name_format: "{cluster}-repl.{team}.{hostedzone}" aws_or_gcp: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index b67ee60e2..2cfc28856 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -1135,12 +1135,6 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation "enable_replica_load_balancer": { Type: "boolean", }, - "master_dns_name_format": { - Type: "string", - }, - "replica_dns_name_format": { - Type: "string", - }, "external_traffic_policy": { Type: "string", Enum: []apiextv1beta1.JSON{ @@ -1152,6 +1146,12 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, }, }, + "master_dns_name_format": { + Type: "string", + }, + "replica_dns_name_format": { + Type: "string", + }, }, }, "aws_or_gcp": { diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index ca3fa46d7..179b7e751 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -193,20 +193,19 @@ type OperatorLogicalBackupConfiguration struct { // OperatorConfigurationData defines the operation config type OperatorConfigurationData struct { - EnableCRDValidation *bool `json:"enable_crd_validation,omitempty"` - EnableLazySpiloUpgrade bool `json:"enable_lazy_spilo_upgrade,omitempty"` - EtcdHost string `json:"etcd_host,omitempty"` - KubernetesUseConfigMaps bool `json:"kubernetes_use_configmaps,omitempty"` - DockerImage string `json:"docker_image,omitempty"` - Workers uint32 `json:"workers,omitempty"` - MinInstances int32 `json:"min_instances,omitempty"` - MaxInstances int32 `json:"max_instances,omitempty"` - ResyncPeriod Duration `json:"resync_period,omitempty"` - RepairPeriod Duration `json:"repair_period,omitempty"` - SetMemoryRequestToLimit bool `json:"set_memory_request_to_limit,omitempty"` - ShmVolume *bool `json:"enable_shm_volume,omitempty"` - // deprecated in favour of SidecarContainers - SidecarImages map[string]string `json:"sidecar_docker_images,omitempty"` + EnableCRDValidation *bool `json:"enable_crd_validation,omitempty"` + EnableLazySpiloUpgrade bool `json:"enable_lazy_spilo_upgrade,omitempty"` + EtcdHost string `json:"etcd_host,omitempty"` + KubernetesUseConfigMaps bool `json:"kubernetes_use_configmaps,omitempty"` + DockerImage string `json:"docker_image,omitempty"` + Workers uint32 `json:"workers,omitempty"` + MinInstances int32 `json:"min_instances,omitempty"` + MaxInstances int32 `json:"max_instances,omitempty"` + ResyncPeriod Duration `json:"resync_period,omitempty"` + RepairPeriod Duration `json:"repair_period,omitempty"` + SetMemoryRequestToLimit bool `json:"set_memory_request_to_limit,omitempty"` + ShmVolume *bool `json:"enable_shm_volume,omitempty"` + SidecarImages map[string]string `json:"sidecar_docker_images,omitempty"` // deprecated in favour of SidecarContainers SidecarContainers []v1.Container `json:"sidecars,omitempty"` PostgresUsersConfiguration PostgresUsersConfiguration `json:"users"` Kubernetes KubernetesMetaConfiguration `json:"kubernetes"` diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 2a2103f5a..7a1ae8a41 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -143,14 +143,13 @@ type Config struct { LogicalBackup ConnectionPooler - WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to' - KubernetesUseConfigMaps bool `name:"kubernetes_use_configmaps" default:"false"` - EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS - DockerImage string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-12:1.6-p3"` - // deprecated in favour of SidecarContainers - SidecarImages map[string]string `name:"sidecar_docker_images"` - SidecarContainers []v1.Container `name:"sidecars"` - PodServiceAccountName string `name:"pod_service_account_name" default:"postgres-pod"` + WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to' + KubernetesUseConfigMaps bool `name:"kubernetes_use_configmaps" default:"false"` + EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS + DockerImage string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-12:1.6-p3"` + SidecarImages map[string]string `name:"sidecar_docker_images"` // deprecated in favour of SidecarContainers + SidecarContainers []v1.Container `name:"sidecars"` + PodServiceAccountName string `name:"pod_service_account_name" default:"postgres-pod"` // value of this string must be valid JSON or YAML; see initPodServiceAccount PodServiceAccountDefinition string `name:"pod_service_account_definition" default:""` PodServiceAccountRoleBindingDefinition string `name:"pod_service_account_role_binding_definition" default:""` @@ -177,27 +176,25 @@ type Config struct { EnablePodAntiAffinity bool `name:"enable_pod_antiaffinity" default:"false"` PodAntiAffinityTopologyKey string `name:"pod_antiaffinity_topology_key" default:"kubernetes.io/hostname"` StorageResizeMode string `name:"storage_resize_mode" default:"ebs"` - // ExternalTrafficPolicy for load balancer - ExternalTrafficPolicy string `name:"external_traffic_policy" default:"Cluster"` - // deprecated and kept for backward compatibility - EnableLoadBalancer *bool `name:"enable_load_balancer"` - MasterDNSNameFormat StringTemplate `name:"master_dns_name_format" default:"{cluster}.{team}.{hostedzone}"` - ReplicaDNSNameFormat StringTemplate `name:"replica_dns_name_format" default:"{cluster}-repl.{team}.{hostedzone}"` - PDBNameFormat StringTemplate `name:"pdb_name_format" default:"postgres-{cluster}-pdb"` - EnablePodDisruptionBudget *bool `name:"enable_pod_disruption_budget" default:"true"` - EnableInitContainers *bool `name:"enable_init_containers" default:"true"` - EnableSidecars *bool `name:"enable_sidecars" default:"true"` - Workers uint32 `name:"workers" default:"8"` - APIPort int `name:"api_port" default:"8080"` - RingLogLines int `name:"ring_log_lines" default:"100"` - ClusterHistoryEntries int `name:"cluster_history_entries" default:"1000"` - TeamAPIRoleConfiguration map[string]string `name:"team_api_role_configuration" default:"log_statement:all"` - PodTerminateGracePeriod time.Duration `name:"pod_terminate_grace_period" default:"5m"` - PodManagementPolicy string `name:"pod_management_policy" default:"ordered_ready"` - ProtectedRoles []string `name:"protected_role_names" default:"admin"` - PostgresSuperuserTeams []string `name:"postgres_superuser_teams" default:""` - SetMemoryRequestToLimit bool `name:"set_memory_request_to_limit" default:"false"` - EnableLazySpiloUpgrade bool `name:"enable_lazy_spilo_upgrade" default:"false"` + EnableLoadBalancer *bool `name:"enable_load_balancer"` // deprecated and kept for backward compatibility + ExternalTrafficPolicy string `name:"external_traffic_policy" default:"Cluster"` + MasterDNSNameFormat StringTemplate `name:"master_dns_name_format" default:"{cluster}.{team}.{hostedzone}"` + ReplicaDNSNameFormat StringTemplate `name:"replica_dns_name_format" default:"{cluster}-repl.{team}.{hostedzone}"` + PDBNameFormat StringTemplate `name:"pdb_name_format" default:"postgres-{cluster}-pdb"` + EnablePodDisruptionBudget *bool `name:"enable_pod_disruption_budget" default:"true"` + EnableInitContainers *bool `name:"enable_init_containers" default:"true"` + EnableSidecars *bool `name:"enable_sidecars" default:"true"` + Workers uint32 `name:"workers" default:"8"` + APIPort int `name:"api_port" default:"8080"` + RingLogLines int `name:"ring_log_lines" default:"100"` + ClusterHistoryEntries int `name:"cluster_history_entries" default:"1000"` + TeamAPIRoleConfiguration map[string]string `name:"team_api_role_configuration" default:"log_statement:all"` + PodTerminateGracePeriod time.Duration `name:"pod_terminate_grace_period" default:"5m"` + PodManagementPolicy string `name:"pod_management_policy" default:"ordered_ready"` + ProtectedRoles []string `name:"protected_role_names" default:"admin"` + PostgresSuperuserTeams []string `name:"postgres_superuser_teams" default:""` + SetMemoryRequestToLimit bool `name:"set_memory_request_to_limit" default:"false"` + EnableLazySpiloUpgrade bool `name:"enable_lazy_spilo_upgrade" default:"false"` } // MustMarshal marshals the config or panics From 38e15183a22a8bb6d1e7728ad32bad5001b82c48 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 2 Oct 2020 09:31:55 +0200 Subject: [PATCH 093/168] update kind (#1156) Co-authored-by: Sergey Dudoladov --- e2e/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/Makefile b/e2e/Makefile index 05ea6a3d6..a72c6bef0 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -47,7 +47,7 @@ tools: # install pinned version of 'kind' # go get must run outside of a dir with a (module-based) Go project ! # otherwise go get updates project's dependencies and/or behaves differently - cd "/tmp" && GO111MODULE=on go get sigs.k8s.io/kind@v0.8.1 + cd "/tmp" && GO111MODULE=on go get sigs.k8s.io/kind@v0.9.0 e2etest: tools copy clean ./run.sh main From 692c721854e4667102778e8e69e3dd12e47984b5 Mon Sep 17 00:00:00 2001 From: Alex Stockinger Date: Thu, 8 Oct 2020 15:32:15 +0200 Subject: [PATCH 094/168] Introduce ENABLE_JSON_LOGGING env variable (#1158) --- charts/postgres-operator/templates/deployment.yaml | 4 ++++ charts/postgres-operator/values.yaml | 3 +++ cmd/main.go | 7 ++++++- docs/reference/command_line_and_environment.md | 4 ++++ pkg/controller/controller.go | 3 +++ pkg/spec/types.go | 2 ++ 6 files changed, 22 insertions(+), 1 deletion(-) diff --git a/charts/postgres-operator/templates/deployment.yaml b/charts/postgres-operator/templates/deployment.yaml index 2d8eebcb3..9841bf1bc 100644 --- a/charts/postgres-operator/templates/deployment.yaml +++ b/charts/postgres-operator/templates/deployment.yaml @@ -37,6 +37,10 @@ spec: image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} env: + {{- if .Values.enableJsonLogging }} + - name: ENABLE_JSON_LOGGING + value: "true" + {{- end }} {{- if eq .Values.configTarget "ConfigMap" }} - name: CONFIG_MAP_NAME value: {{ template "postgres-operator.fullname" . }} diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 37eac4254..d4acfe1aa 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -15,6 +15,9 @@ podLabels: {} configTarget: "ConfigMap" +# JSON logging format +enableJsonLogging: false + # general configuration parameters configGeneral: # choose if deployment creates/updates CRDs with OpenAPIV3Validation diff --git a/cmd/main.go b/cmd/main.go index a178c187e..376df0bad 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -2,7 +2,7 @@ package main import ( "flag" - "log" + log "github.com/sirupsen/logrus" "os" "os/signal" "sync" @@ -36,6 +36,8 @@ func init() { flag.BoolVar(&config.NoTeamsAPI, "noteamsapi", false, "Disable all access to the teams API") flag.Parse() + config.EnableJsonLogging = os.Getenv("ENABLE_JSON_LOGGING") == "true" + configMapRawName := os.Getenv("CONFIG_MAP_NAME") if configMapRawName != "" { @@ -63,6 +65,9 @@ func init() { func main() { var err error + if config.EnableJsonLogging { + log.SetFormatter(&log.JSONFormatter{}) + } log.SetOutput(os.Stdout) log.Printf("Spilo operator %s\n", version) diff --git a/docs/reference/command_line_and_environment.md b/docs/reference/command_line_and_environment.md index ece29b094..35f47cabf 100644 --- a/docs/reference/command_line_and_environment.md +++ b/docs/reference/command_line_and_environment.md @@ -56,3 +56,7 @@ The following environment variables are accepted by the operator: * **CRD_READY_WAIT_INTERVAL** defines the interval between consecutive attempts waiting for the `postgresql` CRD to be created. The default is 5s. + +* **ENABLE_JSON_LOGGING** + Set to `true` for JSON formatted logging output. + The default is false. diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index aa996288c..8e9f02029 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -71,6 +71,9 @@ type Controller struct { // NewController creates a new controller func NewController(controllerConfig *spec.ControllerConfig, controllerId string) *Controller { logger := logrus.New() + if controllerConfig.EnableJsonLogging { + logger.SetFormatter(&logrus.JSONFormatter{}) + } var myComponentName = "postgres-operator" if controllerId != "" { diff --git a/pkg/spec/types.go b/pkg/spec/types.go index 7a2c0ddac..78c79e1b3 100644 --- a/pkg/spec/types.go +++ b/pkg/spec/types.go @@ -114,6 +114,8 @@ type ControllerConfig struct { CRDReadyWaitTimeout time.Duration ConfigMapName NamespacedName Namespace string + + EnableJsonLogging bool } // cached value for the GetOperatorNamespace From d15f2d339205baddcaea7aa7f6f81bdcc846a5dc Mon Sep 17 00:00:00 2001 From: Dmitry Dolgov <9erthalion6@gmail.com> Date: Thu, 15 Oct 2020 10:16:42 +0200 Subject: [PATCH 095/168] Readiness probe (#1169) Right now there are no readiness probes defined for connection pooler, which means after a pod restart there is a short time window (between a container start and connection pooler starting listening to a socket) when a service can send queries to a new pod, but connection will be refused. The pooler container is rather lightweight and it start to listen immediately, so the time window is small, but still. To fix this add a readiness probe for tcp socket opened by connection pooler. --- pkg/cluster/k8sres.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index fef202538..88eb33efb 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -2216,6 +2216,13 @@ func (c *Cluster) generateConnectionPoolerPodTemplate(spec *acidv1.PostgresSpec) }, }, Env: envVars, + ReadinessProbe: &v1.Probe{ + Handler: v1.Handler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.IntOrString{IntVal: pgPort}, + }, + }, + }, } podTemplate := &v1.PodTemplateSpec{ From 1f5d0995a58b8df0973deb3fbb90b162d6c982a2 Mon Sep 17 00:00:00 2001 From: Dmitry Dolgov <9erthalion6@gmail.com> Date: Mon, 19 Oct 2020 16:18:58 +0200 Subject: [PATCH 096/168] Lookup function installation (#1171) * Lookup function installation Due to reusing a previous database connection without closing it, lookup function installation process was skipping the first database in the list, installing twice into postgres db instead. To prevent that, make internal initDbConnWithName to overwrite a connection object, and return the same object only from initDbConn, which is sort of public interface. Another solution for this would be to modify initDbConnWithName to return a connection object and then generate one temporary connection for each db. It sound feasible but after one attempt it seems it requires a bit more changes around (init, close connections) and doesn't bring anything significantly better on the table. In case if some future changes will prove this wrong, do not hesitate to refactor. Change retry strategy to more insistive one, namely: * retry on the next sync even if we failed to process one database and install pooler appliance. * perform the whole installation unconditionally on update, since the list of target databases could be changed. And for the sake of making it even more robust, also log the case when operator decides to skip installation. Extend connection pooler e2e test with verification that all dbs have required schema installed. --- e2e/exec.sh | 2 +- e2e/tests/test_e2e.py | 78 +++++++++++++++++++++++++++++++++++++- pkg/cluster/cluster.go | 8 +++- pkg/cluster/database.go | 84 ++++++++++++++++++++++++++++------------- pkg/cluster/sync.go | 9 ++++- 5 files changed, 151 insertions(+), 30 deletions(-) diff --git a/e2e/exec.sh b/e2e/exec.sh index 56276bc3c..1ab666e5e 100755 --- a/e2e/exec.sh +++ b/e2e/exec.sh @@ -1,2 +1,2 @@ #!/usr/bin/env bash -kubectl exec -it $1 -- sh -c "$2" +kubectl exec -i $1 -- sh -c "$2" diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 550d3ced8..fc251c430 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -15,6 +15,14 @@ def to_selector(labels): return ",".join(["=".join(l) for l in labels.items()]) +def clean_list(values): + # value is not stripped bytes, strip and convert to a string + clean = lambda v: v.strip().decode() + notNone = lambda v: v + + return list(filter(notNone, map(clean, values))) + + class EndToEndTestCase(unittest.TestCase): ''' Test interaction of the operator with multiple K8s components. @@ -140,6 +148,58 @@ class EndToEndTestCase(unittest.TestCase): k8s.wait_for_running_pods(pod_selector, 2) + # Verify that all the databases have pooler schema installed. + # Do this via psql, since otherwise we need to deal with + # credentials. + dbList = [] + + leader = k8s.get_cluster_leader_pod('acid-minimal-cluster') + dbListQuery = "select datname from pg_database" + schemasQuery = """ + select schema_name + from information_schema.schemata + where schema_name = 'pooler' + """ + exec_query = r"psql -tAq -c \"{}\" -d {}" + + if leader: + try: + q = exec_query.format(dbListQuery, "postgres") + q = "su postgres -c \"{}\"".format(q) + print('Get databases: {}'.format(q)) + result = k8s.exec_with_kubectl(leader.metadata.name, q) + dbList = clean_list(result.stdout.split(b'\n')) + print('dbList: {}, stdout: {}, stderr {}'.format( + dbList, result.stdout, result.stderr + )) + except Exception as ex: + print('Could not get databases: {}'.format(ex)) + print('Stdout: {}'.format(result.stdout)) + print('Stderr: {}'.format(result.stderr)) + + for db in dbList: + if db in ('template0', 'template1'): + continue + + schemas = [] + try: + q = exec_query.format(schemasQuery, db) + q = "su postgres -c \"{}\"".format(q) + print('Get schemas: {}'.format(q)) + result = k8s.exec_with_kubectl(leader.metadata.name, q) + schemas = clean_list(result.stdout.split(b'\n')) + print('schemas: {}, stdout: {}, stderr {}'.format( + schemas, result.stdout, result.stderr + )) + except Exception as ex: + print('Could not get databases: {}'.format(ex)) + print('Stdout: {}'.format(result.stdout)) + print('Stderr: {}'.format(result.stderr)) + + self.assertNotEqual(len(schemas), 0) + else: + print('Could not find leader pod') + # turn it off, keeping configuration section k8s.api.custom_objects_api.patch_namespaced_custom_object( 'acid.zalan.do', 'v1', 'default', @@ -235,7 +295,10 @@ class EndToEndTestCase(unittest.TestCase): # operator configuration via API operator_pod = k8s.get_operator_pod() get_config_cmd = "wget --quiet -O - localhost:8080/config" - result = k8s.exec_with_kubectl(operator_pod.metadata.name, get_config_cmd) + result = k8s.exec_with_kubectl( + operator_pod.metadata.name, + get_config_cmd, + ) roles_dict = (json.loads(result.stdout) .get("controller", {}) .get("InfrastructureRoles")) @@ -862,6 +925,19 @@ class K8s: return master_pod_node, replica_pod_nodes + def get_cluster_leader_pod(self, pg_cluster_name, namespace='default'): + labels = { + 'application': 'spilo', + 'cluster-name': pg_cluster_name, + 'spilo-role': 'master', + } + + pods = self.api.core_v1.list_namespaced_pod( + namespace, label_selector=to_selector(labels)).items + + if pods: + return pods[0] + def wait_for_operator_pod_start(self): self. wait_for_pod_start("name=postgres-operator") # HACK operator must register CRD and/or Sync existing PG clusters after start up diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 9b8b51eb0..6aa1f6fa4 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -780,7 +780,13 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { } } - // sync connection pooler + // Sync connection pooler. Before actually doing sync reset lookup + // installation flag, since manifest updates could add another db which we + // need to process. In the future we may want to do this more careful and + // check which databases we need to process, but even repeating the whole + // installation process should be good enough. + c.ConnectionPooler.LookupFunction = false + if _, err := c.syncConnectionPooler(oldSpec, newSpec, c.installLookupFunction); err != nil { c.logger.Errorf("could not sync connection pooler: %v", err) diff --git a/pkg/cluster/database.go b/pkg/cluster/database.go index 75e2d2097..f51b58a89 100644 --- a/pkg/cluster/database.go +++ b/pkg/cluster/database.go @@ -101,15 +101,20 @@ func (c *Cluster) databaseAccessDisabled() bool { } func (c *Cluster) initDbConn() error { - return c.initDbConnWithName("") -} - -func (c *Cluster) initDbConnWithName(dbname string) error { - c.setProcessName("initializing db connection") if c.pgDb != nil { return nil } + return c.initDbConnWithName("") +} + +// Worker function for connection initialization. This function does not check +// if the connection is already open, if it is then it will be overwritten. +// Callers need to make sure no connection is open, otherwise we could leak +// connections +func (c *Cluster) initDbConnWithName(dbname string) error { + c.setProcessName("initializing db connection") + var conn *sql.DB connstring := c.pgConnectionString(dbname) @@ -145,6 +150,12 @@ func (c *Cluster) initDbConnWithName(dbname string) error { conn.SetMaxOpenConns(1) conn.SetMaxIdleConns(-1) + if c.pgDb != nil { + msg := "Closing an existing connection before opening a new one to %s" + c.logger.Warningf(msg, dbname) + c.closeDbConn() + } + c.pgDb = conn return nil @@ -465,8 +476,11 @@ func (c *Cluster) execCreateOrAlterExtension(extName, schemaName, statement, doi // perform remote authentification. func (c *Cluster) installLookupFunction(poolerSchema, poolerUser string) error { var stmtBytes bytes.Buffer + c.logger.Info("Installing lookup function") + // Open a new connection if not yet done. This connection will be used only + // to get the list of databases, not for the actuall installation. if err := c.initDbConn(); err != nil { return fmt.Errorf("could not init database connection") } @@ -480,37 +494,41 @@ func (c *Cluster) installLookupFunction(poolerSchema, poolerUser string) error { } }() + // List of databases we failed to process. At the moment it function just + // like a flag to retry on the next sync, but in the future we may want to + // retry only necessary parts, so let's keep the list. + failedDatabases := []string{} currentDatabases, err := c.getDatabases() if err != nil { msg := "could not get databases to install pooler lookup function: %v" return fmt.Errorf(msg, err) } + // We've got the list of target databases, now close this connection to + // open a new one to every each of them. + if err := c.closeDbConn(); err != nil { + c.logger.Errorf("could not close database connection: %v", err) + } + templater := template.Must(template.New("sql").Parse(connectionPoolerLookup)) + params := TemplateParams{ + "pooler_schema": poolerSchema, + "pooler_user": poolerUser, + } + + if err := templater.Execute(&stmtBytes, params); err != nil { + msg := "could not prepare sql statement %+v: %v" + return fmt.Errorf(msg, params, err) + } for dbname := range currentDatabases { + if dbname == "template0" || dbname == "template1" { continue } - if err := c.initDbConnWithName(dbname); err != nil { - return fmt.Errorf("could not init database connection to %s", dbname) - } - c.logger.Infof("Install pooler lookup function into %s", dbname) - params := TemplateParams{ - "pooler_schema": poolerSchema, - "pooler_user": poolerUser, - } - - if err := templater.Execute(&stmtBytes, params); err != nil { - c.logger.Errorf("could not prepare sql statement %+v: %v", - params, err) - // process other databases - continue - } - // golang sql will do retries couple of times if pq driver reports // connections issues (driver.ErrBadConn), but since our query is // idempotent, we can retry in a view of other errors (e.g. due to @@ -520,7 +538,20 @@ func (c *Cluster) installLookupFunction(poolerSchema, poolerUser string) error { constants.PostgresConnectTimeout, constants.PostgresConnectRetryTimeout, func() (bool, error) { - if _, err := c.pgDb.Exec(stmtBytes.String()); err != nil { + + // At this moment we are not connected to any database + if err := c.initDbConnWithName(dbname); err != nil { + msg := "could not init database connection to %s" + return false, fmt.Errorf(msg, dbname) + } + defer func() { + if err := c.closeDbConn(); err != nil { + msg := "could not close database connection: %v" + c.logger.Errorf(msg, err) + } + }() + + if _, err = c.pgDb.Exec(stmtBytes.String()); err != nil { msg := fmt.Errorf("could not execute sql statement %s: %v", stmtBytes.String(), err) return false, msg @@ -533,15 +564,16 @@ func (c *Cluster) installLookupFunction(poolerSchema, poolerUser string) error { c.logger.Errorf("could not execute after retries %s: %v", stmtBytes.String(), err) // process other databases + failedDatabases = append(failedDatabases, dbname) continue } c.logger.Infof("pooler lookup function installed into %s", dbname) - if err := c.closeDbConn(); err != nil { - c.logger.Errorf("could not close database connection: %v", err) - } } - c.ConnectionPooler.LookupFunction = true + if len(failedDatabases) == 0 { + c.ConnectionPooler.LookupFunction = true + } + return nil } diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index fef5b7b66..2a3959b1a 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -847,7 +847,9 @@ func (c *Cluster) syncConnectionPooler(oldSpec, var err error if c.ConnectionPooler == nil { - c.ConnectionPooler = &ConnectionPoolerObjects{} + c.ConnectionPooler = &ConnectionPoolerObjects{ + LookupFunction: false, + } } newNeedConnectionPooler := c.needConnectionPoolerWorker(&newSpec.Spec) @@ -885,6 +887,11 @@ func (c *Cluster) syncConnectionPooler(oldSpec, if err = lookup(schema, user); err != nil { return NoSync, err } + } else { + // Lookup function installation seems to be a fragile point, so + // let's log for debugging if we skip it + msg := "Skip lookup function installation, old: %d, already installed %d" + c.logger.Debug(msg, oldNeedConnectionPooler, c.ConnectionPooler.LookupFunction) } if reason, err = c.syncConnectionPoolerWorker(oldSpec, newSpec); err != nil { From a8bfe4eb874f829785ada30085af34853a31f790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=96=B0?= <1148576125@qq.com> Date: Tue, 20 Oct 2020 20:18:22 +0800 Subject: [PATCH 097/168] Remove repeated initialization of Pod ServiceAccount (#1164) Co-authored-by: xin.liu --- pkg/controller/controller.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 8e9f02029..cc08f1587 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -295,7 +295,6 @@ func (c *Controller) initController() { c.logger.Fatalf("could not register Postgres CustomResourceDefinition: %v", err) } - c.initPodServiceAccount() c.initSharedInformers() if c.opConfig.DebugLogging { From 22fa0875e2378466aed60eab6c94fe0f807caf2e Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 22 Oct 2020 08:44:04 +0200 Subject: [PATCH 098/168] add maxLength constraint for CRD (#1175) * add maxLength constraint for CRD --- charts/postgres-operator/crds/postgresqls.yaml | 9 +++++++++ docs/user.md | 2 +- manifests/postgresql.crd.yaml | 9 +++++++++ pkg/apis/acid.zalan.do/v1/crds.go | 13 ++++++++++++- 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 0d444e568..488f17c2b 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -57,6 +57,7 @@ spec: required: - kind - apiVersion + - metadata - spec properties: kind: @@ -67,6 +68,14 @@ spec: type: string enum: - acid.zalan.do/v1 + metadata: + type: object + required: + - name + properties: + name: + type: string + maxLength: 53 spec: type: object required: diff --git a/docs/user.md b/docs/user.md index a4b1424b8..9a9e01b9a 100644 --- a/docs/user.md +++ b/docs/user.md @@ -49,7 +49,7 @@ Note, that the name of the cluster must start with the `teamId` and `-`. At Zalando we use team IDs (nicknames) to lower the chance of duplicate cluster names and colliding entities. The team ID would also be used to query an API to get all members of a team and create [database roles](#teams-api-roles) for -them. +them. Besides, the maximum cluster name length is 53 characters. ## Watch pods being created diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 97b72a8ca..56c010739 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -53,6 +53,7 @@ spec: required: - kind - apiVersion + - metadata - spec properties: kind: @@ -63,6 +64,14 @@ spec: type: string enum: - acid.zalan.do/v1 + metadata: + type: object + required: + - name + properties: + name: + type: string + maxLength: 53 spec: type: object required: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 2cfc28856..a7d9bccf0 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -107,12 +107,13 @@ var min0 = 0.0 var min1 = 1.0 var min2 = 2.0 var minDisable = -1.0 +var maxLength = int64(53) // PostgresCRDResourceValidation to check applied manifest parameters var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ OpenAPIV3Schema: &apiextv1beta1.JSONSchemaProps{ Type: "object", - Required: []string{"kind", "apiVersion", "spec"}, + Required: []string{"kind", "apiVersion", "metadata", "spec"}, Properties: map[string]apiextv1beta1.JSONSchemaProps{ "kind": { Type: "string", @@ -130,6 +131,16 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, }, }, + "metadata": { + Type: "object", + Required: []string{"name"}, + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "name": { + Type: "string", + MaxLength: &maxLength, + }, + }, + }, "spec": { Type: "object", Required: []string{"numberOfInstances", "teamId", "postgresql", "volume"}, From d9f5d1c9dfef1a7e9e00b216553ef77cf6904da4 Mon Sep 17 00:00:00 2001 From: preved911 Date: Thu, 22 Oct 2020 09:49:30 +0300 Subject: [PATCH 099/168] changed PodEnvironmentSecret location namespace (#1177) Signed-off-by: Ildar Valiullin --- pkg/cluster/k8sres.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 88eb33efb..ba22f24c3 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -841,7 +841,7 @@ func (c *Cluster) getPodEnvironmentSecretVariables() ([]v1.EnvVar, error) { return secretPodEnvVarsList, nil } - secret, err := c.KubeClient.Secrets(c.OpConfig.PodEnvironmentSecret).Get( + secret, err := c.KubeClient.Secrets(c.Namespace).Get( context.TODO(), c.OpConfig.PodEnvironmentSecret, metav1.GetOptions{}) From e97235aa398e6e3e561fe04fffcaa2b19deb7103 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 27 Oct 2020 16:59:26 +0100 Subject: [PATCH 100/168] update dependencies oct 2020 (#1184) * update dependencies oct 2020 * update codegen --- Makefile | 2 +- go.mod | 16 +- go.sum | 322 +++++++++++++----- .../clientset/versioned/fake/register.go | 2 +- .../listers/acid.zalan.do/v1/postgresql.go | 5 + 5 files changed, 249 insertions(+), 98 deletions(-) diff --git a/Makefile b/Makefile index 29bbb47e6..1a676adad 100644 --- a/Makefile +++ b/Makefile @@ -79,7 +79,7 @@ scm-source.json: .git tools: GO111MODULE=on go get -u honnef.co/go/tools/cmd/staticcheck - GO111MODULE=on go get k8s.io/client-go@kubernetes-1.18.8 + GO111MODULE=on go get k8s.io/client-go@kubernetes-1.19.3 GO111MODULE=on go mod tidy fmt: diff --git a/go.mod b/go.mod index 79c3b9be9..341af771c 100644 --- a/go.mod +++ b/go.mod @@ -3,18 +3,18 @@ module github.com/zalando/postgres-operator go 1.14 require ( - github.com/aws/aws-sdk-go v1.34.10 + github.com/aws/aws-sdk-go v1.35.15 github.com/lib/pq v1.8.0 github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d github.com/r3labs/diff v1.1.0 - github.com/sirupsen/logrus v1.6.0 + github.com/sirupsen/logrus v1.7.0 github.com/stretchr/testify v1.5.1 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 - golang.org/x/tools v0.0.0-20200928201943-a0ef9b62deab // indirect + golang.org/x/tools v0.0.0-20201026223136-e84cfc6dd5ca // indirect gopkg.in/yaml.v2 v2.2.8 - k8s.io/api v0.18.8 - k8s.io/apiextensions-apiserver v0.18.0 - k8s.io/apimachinery v0.18.8 - k8s.io/client-go v0.18.8 - k8s.io/code-generator v0.18.8 + k8s.io/api v0.19.3 + k8s.io/apiextensions-apiserver v0.19.3 + k8s.io/apimachinery v0.19.3 + k8s.io/client-go v0.19.3 + k8s.io/code-generator v0.19.3 ) diff --git a/go.sum b/go.sum index 2d76a94ee..1f2e5f1d8 100644 --- a/go.sum +++ b/go.sum @@ -1,17 +1,33 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= @@ -21,40 +37,49 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.34.10 h1:VU78gcf/3wA4HNEDCHidK738l7K0Bals4SJnfnvXOtY= -github.com/aws/aws-sdk-go v1.34.10/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.35.15 h1:JdQNM8hJe+9N9xP53S54NDmX8GCaZn8CCJ4LBHfom4U= +github.com/aws/aws-sdk-go v1.35.15/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= @@ -64,19 +89,24 @@ github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v0.0.0-20200808040245-162e5629780b/go.mod h1:NAJj0yf/KaRKURN6nyi7A9IZydMivZEm9oQLWNjfKDc= -github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I= -github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= +github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= @@ -124,45 +154,61 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 h1:5ZkaAPbicIKTF2I64qf5Fh8Aa83Q/dnOafMYV0OMwjA= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.1.0 h1:rVsPeBmXbYv4If/cumu1AzZPwV58q433hvONV1UEZoI= -github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= +github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= @@ -170,28 +216,31 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= -github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= -github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= @@ -210,8 +259,10 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -224,6 +275,7 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -237,85 +289,120 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/r3labs/diff v1.1.0 h1:V53xhrbTHrWFWq3gI4b94AjgEJOerO1+1l0xyHOBi8M= github.com/r3labs/diff v1.1.0/go.mod h1:7WjXasNzi0vJetRcB/RqNl5dlIsmXcTTLmF5IoH6Xig= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -327,51 +414,71 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 h1:pE8b58s1HRDMi8RDc79m0HISf9D4TzseP40cEA6IGfs= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -380,36 +487,77 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200928201943-a0ef9b62deab h1:CyH2SDm5ATQiX9gtbMYfvNNed97A9v+TJFnUX/fTaJY= -golang.org/x/tools v0.0.0-20200928201943-a0ef9b62deab/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20201026223136-e84cfc6dd5ca h1:vL6Mv8VrSxz8azdgLrH/zO/Rd1Bzdk89ZfMVW39gD0Q= +golang.org/x/tools v0.0.0-20201026223136-e84cfc6dd5ca/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= @@ -423,45 +571,43 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.18.0/go.mod h1:q2HRQkfDzHMBZL9l/y9rH63PkQl4vae0xRT+8prbrK8= -k8s.io/api v0.18.8 h1:aIKUzJPb96f3fKec2lxtY7acZC9gQNDLVhfSGpxBAC4= -k8s.io/api v0.18.8/go.mod h1:d/CXqwWv+Z2XEG1LgceeDmHQwpUJhROPx16SlxJgERY= -k8s.io/apiextensions-apiserver v0.18.0 h1:HN4/P8vpGZFvB5SOMuPPH2Wt9Y/ryX+KRvIyAkchu1Q= -k8s.io/apiextensions-apiserver v0.18.0/go.mod h1:18Cwn1Xws4xnWQNC00FLq1E350b9lUF+aOdIWDOZxgo= -k8s.io/apimachinery v0.18.0/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= -k8s.io/apimachinery v0.18.8 h1:jimPrycCqgx2QPearX3to1JePz7wSbVLq+7PdBTTwQ0= -k8s.io/apimachinery v0.18.8/go.mod h1:6sQd+iHEqmOtALqOFjSWp2KZ9F0wlU/nWm0ZgsYWMig= -k8s.io/apiserver v0.18.0/go.mod h1:3S2O6FeBBd6XTo0njUrLxiqk8GNy6wWOftjhJcXYnjw= -k8s.io/client-go v0.18.0/go.mod h1:uQSYDYs4WhVZ9i6AIoEZuwUggLVEF64HOD37boKAtF8= -k8s.io/client-go v0.18.8 h1:SdbLpIxk5j5YbFr1b7fq8S7mDgDjYmUxSbszyoesoDM= -k8s.io/client-go v0.18.8/go.mod h1:HqFqMllQ5NnQJNwjro9k5zMyfhZlOwpuTLVrxjkYSxU= -k8s.io/code-generator v0.18.0/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= -k8s.io/code-generator v0.18.8 h1:lgO1P1wjikEtzNvj7ia+x1VC4svJ28a/r0wnOLhhOTU= -k8s.io/code-generator v0.18.8/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c= -k8s.io/component-base v0.18.0/go.mod h1:u3BCg0z1uskkzrnAKFzulmYaEpZF7XC9Pf/uFyb1v2c= -k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20200114144118-36b2048a9120 h1:RPscN6KhmG54S33L+lr3GS+oD1jmchIU0ll519K6FA4= -k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= -k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= -k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= -k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6 h1:Oh3Mzx5pJ+yIumsAD0MOECPVeXsVot0UkiaCGVyfGQY= -k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= -k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89 h1:d4vVOjXm687F1iLSP2q3lyPPuyvTUt3aVoBpi2DqRsU= -k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0= -sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= -sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E= -sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= +honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +k8s.io/api v0.19.3 h1:GN6ntFnv44Vptj/b+OnMW7FmzkpDoIDLZRvKX3XH9aU= +k8s.io/api v0.19.3/go.mod h1:VF+5FT1B74Pw3KxMdKyinLo+zynBaMBiAfGMuldcNDs= +k8s.io/apiextensions-apiserver v0.19.3 h1:WZxBypSHW4SdXHbdPTS/Jy7L2la6Niggs8BuU5o+avo= +k8s.io/apiextensions-apiserver v0.19.3/go.mod h1:igVEkrE9TzInc1tYE7qSqxaLg/rEAp6B5+k9Q7+IC8Q= +k8s.io/apimachinery v0.19.3 h1:bpIQXlKjB4cB/oNpnNnV+BybGPR7iP5oYpsOTEJ4hgc= +k8s.io/apimachinery v0.19.3/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= +k8s.io/apiserver v0.19.3/go.mod h1:bx6dMm+H6ifgKFpCQT/SAhPwhzoeIMlHIaibomUDec0= +k8s.io/client-go v0.19.3 h1:ctqR1nQ52NUs6LpI0w+a5U+xjYwflFwA13OJKcicMxg= +k8s.io/client-go v0.19.3/go.mod h1:+eEMktZM+MG0KO+PTkci8xnbCZHvj9TqR6Q1XDUIJOM= +k8s.io/code-generator v0.19.3 h1:fTrTpJ8PZog5oo6MmeZtveo89emjQZHiw0ieybz1RSs= +k8s.io/code-generator v0.19.3/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= +k8s.io/component-base v0.19.3/go.mod h1:WhLWSIefQn8W8jxSLl5WNiR6z8oyMe/8Zywg7alOkRc= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14 h1:t4L10Qfx/p7ASH3gXCdIUtPbbIuegCoUJf3TMSFekjw= +k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0 h1:XRvcwJozkgZ1UQJmfMGpvRthQHOvihEhYtDfAaxMz/A= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6 h1:+WnxoVtG8TMiudHBSEtrVL1egv36TkkJm+bA8AxicmQ= +k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= +k8s.io/utils v0.0.0-20200729134348-d5654de09c73 h1:uJmqzgNWG7XyClnU/mLPBWwfKKF1K8Hf8whTseBgJcg= +k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0= +sigs.k8s.io/structured-merge-diff/v4 v4.0.1 h1:YXTMot5Qz/X1iBRJhAt+vI+HVttY0WkSqqhKxQ0xVbA= +sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/pkg/generated/clientset/versioned/fake/register.go b/pkg/generated/clientset/versioned/fake/register.go index 5363e8cc4..c5a24f7da 100644 --- a/pkg/generated/clientset/versioned/fake/register.go +++ b/pkg/generated/clientset/versioned/fake/register.go @@ -35,7 +35,7 @@ import ( var scheme = runtime.NewScheme() var codecs = serializer.NewCodecFactory(scheme) -var parameterCodec = runtime.NewParameterCodec(scheme) + var localSchemeBuilder = runtime.SchemeBuilder{ acidv1.AddToScheme, } diff --git a/pkg/generated/listers/acid.zalan.do/v1/postgresql.go b/pkg/generated/listers/acid.zalan.do/v1/postgresql.go index 9a60c8281..ee3efbdfe 100644 --- a/pkg/generated/listers/acid.zalan.do/v1/postgresql.go +++ b/pkg/generated/listers/acid.zalan.do/v1/postgresql.go @@ -32,8 +32,10 @@ import ( ) // PostgresqlLister helps list Postgresqls. +// All objects returned here must be treated as read-only. type PostgresqlLister interface { // List lists all Postgresqls in the indexer. + // Objects returned here must be treated as read-only. List(selector labels.Selector) (ret []*v1.Postgresql, err error) // Postgresqls returns an object that can list and get Postgresqls. Postgresqls(namespace string) PostgresqlNamespaceLister @@ -64,10 +66,13 @@ func (s *postgresqlLister) Postgresqls(namespace string) PostgresqlNamespaceList } // PostgresqlNamespaceLister helps list and get Postgresqls. +// All objects returned here must be treated as read-only. type PostgresqlNamespaceLister interface { // List lists all Postgresqls in the indexer for a given namespace. + // Objects returned here must be treated as read-only. List(selector labels.Selector) (ret []*v1.Postgresql, err error) // Get retrieves the Postgresql from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. Get(name string) (*v1.Postgresql, error) PostgresqlNamespaceListerExpansion } From 7730ecfdeceb6e447232801a4efccff5b60fa3be Mon Sep 17 00:00:00 2001 From: arminfelder Date: Wed, 28 Oct 2020 09:33:52 +0100 Subject: [PATCH 101/168] fixed case where, no ready label is defined, but node is unscheduable (#1162) * fixed case where, no ready label is defined, but node is unscheduable --- pkg/controller/node.go | 2 +- pkg/controller/node_test.go | 52 +++++++++++++++++++++++++++++-------- 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/pkg/controller/node.go b/pkg/controller/node.go index be41b79ab..4ffe7e26c 100644 --- a/pkg/controller/node.go +++ b/pkg/controller/node.go @@ -76,7 +76,7 @@ func (c *Controller) nodeUpdate(prev, cur interface{}) { } func (c *Controller) nodeIsReady(node *v1.Node) bool { - return (!node.Spec.Unschedulable || util.MapContains(node.Labels, c.opConfig.NodeReadinessLabel) || + return (!node.Spec.Unschedulable || (len(c.opConfig.NodeReadinessLabel) > 0 && util.MapContains(node.Labels, c.opConfig.NodeReadinessLabel)) || util.MapContains(node.Labels, map[string]string{"master": "true"})) } diff --git a/pkg/controller/node_test.go b/pkg/controller/node_test.go index 28e178bfb..919f30f39 100644 --- a/pkg/controller/node_test.go +++ b/pkg/controller/node_test.go @@ -15,7 +15,6 @@ const ( func newNodeTestController() *Controller { var controller = NewController(&spec.ControllerConfig{}, "node-test") - controller.opConfig.NodeReadinessLabel = map[string]string{readyLabel: readyValue} return controller } @@ -36,27 +35,58 @@ var nodeTestController = newNodeTestController() func TestNodeIsReady(t *testing.T) { testName := "TestNodeIsReady" var testTable = []struct { - in *v1.Node - out bool + in *v1.Node + out bool + readinessLabel map[string]string }{ { - in: makeNode(map[string]string{"foo": "bar"}, true), - out: true, + in: makeNode(map[string]string{"foo": "bar"}, true), + out: true, + readinessLabel: map[string]string{readyLabel: readyValue}, }, { - in: makeNode(map[string]string{"foo": "bar"}, false), - out: false, + in: makeNode(map[string]string{"foo": "bar"}, false), + out: false, + readinessLabel: map[string]string{readyLabel: readyValue}, }, { - in: makeNode(map[string]string{readyLabel: readyValue}, false), - out: true, + in: makeNode(map[string]string{readyLabel: readyValue}, false), + out: true, + readinessLabel: map[string]string{readyLabel: readyValue}, }, { - in: makeNode(map[string]string{"foo": "bar", "master": "true"}, false), - out: true, + in: makeNode(map[string]string{"foo": "bar", "master": "true"}, false), + out: true, + readinessLabel: map[string]string{readyLabel: readyValue}, + }, + { + in: makeNode(map[string]string{"foo": "bar", "master": "true"}, false), + out: true, + readinessLabel: map[string]string{readyLabel: readyValue}, + }, + { + in: makeNode(map[string]string{"foo": "bar"}, true), + out: true, + readinessLabel: map[string]string{}, + }, + { + in: makeNode(map[string]string{"foo": "bar"}, false), + out: false, + readinessLabel: map[string]string{}, + }, + { + in: makeNode(map[string]string{readyLabel: readyValue}, false), + out: false, + readinessLabel: map[string]string{}, + }, + { + in: makeNode(map[string]string{"foo": "bar", "master": "true"}, false), + out: true, + readinessLabel: map[string]string{}, }, } for _, tt := range testTable { + nodeTestController.opConfig.NodeReadinessLabel = tt.readinessLabel if isReady := nodeTestController.nodeIsReady(tt.in); isReady != tt.out { t.Errorf("%s: expected response %t doesn't match the actual %t for the node %#v", testName, tt.out, isReady, tt.in) From 3a86dfc8bbab3bf286e77ce225459adc71dc919d Mon Sep 17 00:00:00 2001 From: Jan Mussler Date: Wed, 28 Oct 2020 10:04:33 +0100 Subject: [PATCH 102/168] End 2 End tests speedup (#1180) * Improving end 2 end tests, especially speed of execution and error, by implementing proper eventual asserts and timeouts. * Add documentation for running individual tests * Fixed String encoding in Patorni state check and error case * Printing config as multi log line entity, makes it readable and grepable on startup * Cosmetic changes to logs. Removed quotes from diff. Move all object diffs to text diff. Enabled padding for log level. * Mount script with tools for easy logaccess and watching objects. * Set proper update strategy for Postgres operator deployment. * Move long running test to end. Move pooler test to new functions. * Remove quote from valid K8s identifiers. --- Makefile | 10 +- e2e/Dockerfile | 6 +- e2e/README.md | 44 ++ e2e/exec_into_env.sh | 14 + e2e/run.sh | 43 +- e2e/scripts/cleanup.sh | 7 + e2e/scripts/get_logs.sh | 2 + e2e/scripts/watch_objects.sh | 19 + e2e/tests/k8s_api.py | 522 ++++++++++++++++++ e2e/tests/test_e2e.py | 880 +++++++++++++------------------ manifests/postgres-operator.yaml | 2 + pkg/cluster/cluster.go | 30 +- pkg/cluster/database.go | 2 +- pkg/cluster/k8sres.go | 7 +- pkg/cluster/pod.go | 9 +- pkg/cluster/resources.go | 9 +- pkg/cluster/sync.go | 16 +- pkg/cluster/util.go | 41 +- pkg/controller/controller.go | 39 +- pkg/controller/node.go | 2 +- pkg/controller/postgresql.go | 2 +- pkg/util/config/config.go | 2 +- pkg/util/nicediff/diff.go | 191 +++++++ pkg/util/util_test.go | 10 + 24 files changed, 1317 insertions(+), 592 deletions(-) create mode 100755 e2e/exec_into_env.sh create mode 100755 e2e/scripts/cleanup.sh create mode 100755 e2e/scripts/get_logs.sh create mode 100755 e2e/scripts/watch_objects.sh create mode 100644 e2e/tests/k8s_api.py create mode 100644 pkg/util/nicediff/diff.go diff --git a/Makefile b/Makefile index 1a676adad..2b2d2668f 100644 --- a/Makefile +++ b/Makefile @@ -24,12 +24,16 @@ PKG := `go list ./... | grep -v /vendor/` ifeq ($(DEBUG),1) DOCKERFILE = DebugDockerfile - DEBUG_POSTFIX := -debug + DEBUG_POSTFIX := -debug-$(shell date hhmmss) BUILD_FLAGS += -gcflags "-N -l" else DOCKERFILE = Dockerfile endif +ifeq ($(FRESH),1) + DEBUG_FRESH=$(shell date +"%H-%M-%S") +endif + ifdef CDP_PULL_REQUEST_NUMBER CDP_TAG := -${CDP_BUILD_VERSION} endif @@ -66,7 +70,7 @@ docker: ${DOCKERDIR}/${DOCKERFILE} docker-context echo "Version ${VERSION}" echo "CDP tag ${CDP_TAG}" echo "git describe $(shell git describe --tags --always --dirty)" - cd "${DOCKERDIR}" && docker build --rm -t "$(IMAGE):$(TAG)$(CDP_TAG)$(DEBUG_POSTFIX)" -f "${DOCKERFILE}" . + cd "${DOCKERDIR}" && docker build --rm -t "$(IMAGE):$(TAG)$(CDP_TAG)$(DEBUG_FRESH)$(DEBUG_POSTFIX)" -f "${DOCKERFILE}" . indocker-race: docker run --rm -v "${GOPATH}":"${GOPATH}" -e GOPATH="${GOPATH}" -e RACE=1 -w ${PWD} golang:1.8.1 bash -c "make linux" @@ -97,4 +101,4 @@ test: GO111MODULE=on go test ./... e2e: docker # build operator image to be tested - cd e2e; make e2etest + cd e2e; make e2etest \ No newline at end of file diff --git a/e2e/Dockerfile b/e2e/Dockerfile index 70e6f0a84..3eb8c9d70 100644 --- a/e2e/Dockerfile +++ b/e2e/Dockerfile @@ -14,6 +14,7 @@ RUN apt-get update \ python3-setuptools \ python3-pip \ curl \ + vim \ && pip3 install --no-cache-dir -r requirements.txt \ && curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl \ && chmod +x ./kubectl \ @@ -21,4 +22,7 @@ RUN apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -ENTRYPOINT ["python3", "-m", "unittest", "discover", "--start-directory", ".", "-v"] +# working line +# python3 -m unittest discover -v --failfast -k test_e2e.EndToEndTestCase.test_lazy_spilo_upgrade --start-directory tests +ENTRYPOINT ["python3", "-m", "unittest"] +CMD ["discover","-v","--failfast","--start-directory","/tests"] \ No newline at end of file diff --git a/e2e/README.md b/e2e/README.md index f1bc5f9ed..92a1fc731 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -12,6 +12,10 @@ Docker. Docker Go +# Notice + +The `manifest` folder in e2e tests folder is not commited to git, it comes from `/manifests` + ## Build test runner In the directory of the cloned Postgres Operator repository change to the e2e @@ -35,6 +39,46 @@ In the e2e folder you can invoke tests either with `make test` or with: To run both the build and test step you can invoke `make e2e` from the parent directory. +To run the end 2 end test and keep the kind state execute: +```bash +NOCLEANUP=True ./run.sh +``` + +## Run indidual test + +After having executed a normal E2E run with `NOCLEANUP=True` Kind still continues to run, allowing you subsequent test runs. + +To run an individual test, run the following command in the `e2e` directory + +```bash +NOCLEANUP=True ./run.sh main tests.test_e2e.EndToEndTestCase.test_lazy_spilo_upgrade +``` + +## Inspecting Kind + +If you want to inspect Kind/Kubernetes cluster, use the following script to exec into the K8s setup and then use `kubectl` + +```bash +./exec_into_env.sh + +# use kube ctl +kubectl get pods + +# watch relevant objects +./scripts/watch_objects.sh + +# get operator logs +./scripts/get_logs.sh +``` + +## Cleaning up Kind + +To cleanup kind and start fresh + +```bash +e2e/run.sh cleanup +``` + ## Covered use cases The current tests are all bundled in [`test_e2e.py`](tests/test_e2e.py): diff --git a/e2e/exec_into_env.sh b/e2e/exec_into_env.sh new file mode 100755 index 000000000..ef12ba18a --- /dev/null +++ b/e2e/exec_into_env.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +export cluster_name="postgres-operator-e2e-tests" +export kubeconfig_path="/tmp/kind-config-${cluster_name}" +export operator_image="registry.opensource.zalan.do/acid/postgres-operator:latest" +export e2e_test_runner_image="registry.opensource.zalan.do/acid/postgres-operator-e2e-tests-runner:0.3" + +docker run -it --entrypoint /bin/bash --network=host -e "TERM=xterm-256color" \ + --mount type=bind,source="$(readlink -f ${kubeconfig_path})",target=/root/.kube/config \ + --mount type=bind,source="$(readlink -f manifests)",target=/manifests \ + --mount type=bind,source="$(readlink -f tests)",target=/tests \ + --mount type=bind,source="$(readlink -f exec.sh)",target=/exec.sh \ + --mount type=bind,source="$(readlink -f scripts)",target=/scripts \ + -e OPERATOR_IMAGE="${operator_image}" "${e2e_test_runner_image}" diff --git a/e2e/run.sh b/e2e/run.sh index 74d842879..0024a2569 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -9,6 +9,10 @@ IFS=$'\n\t' readonly cluster_name="postgres-operator-e2e-tests" readonly kubeconfig_path="/tmp/kind-config-${cluster_name}" readonly spilo_image="registry.opensource.zalan.do/acid/spilo-12:1.6-p5" +readonly e2e_test_runner_image="registry.opensource.zalan.do/acid/postgres-operator-e2e-tests-runner:0.3" + +export GOPATH=${GOPATH-~/go} +export PATH=${GOPATH}/bin:$PATH echo "Clustername: ${cluster_name}" echo "Kubeconfig path: ${kubeconfig_path}" @@ -19,12 +23,7 @@ function pull_images(){ then docker pull registry.opensource.zalan.do/acid/postgres-operator:latest fi - operator_image=$(docker images --filter=reference="registry.opensource.zalan.do/acid/postgres-operator" --format "{{.Repository}}:{{.Tag}}" | head -1) - - # this image does not contain the tests; a container mounts them from a local "./tests" dir at start time - e2e_test_runner_image="registry.opensource.zalan.do/acid/postgres-operator-e2e-tests-runner:latest" - docker pull ${e2e_test_runner_image} } function start_kind(){ @@ -36,12 +35,17 @@ function start_kind(){ fi export KUBECONFIG="${kubeconfig_path}" - kind create cluster --name ${cluster_name} --config kind-cluster-postgres-operator-e2e-tests.yaml - kind load docker-image "${operator_image}" --name ${cluster_name} + kind create cluster --name ${cluster_name} --config kind-cluster-postgres-operator-e2e-tests.yaml docker pull "${spilo_image}" kind load docker-image "${spilo_image}" --name ${cluster_name} } +function load_operator_image() { + echo "Loading operator image" + export KUBECONFIG="${kubeconfig_path}" + kind load docker-image "${operator_image}" --name ${cluster_name} +} + function set_kind_api_server_ip(){ echo "Setting up kind API server ip" # use the actual kubeconfig to connect to the 'kind' API server @@ -52,8 +56,7 @@ function set_kind_api_server_ip(){ } function run_tests(){ - echo "Running tests..." - + echo "Running tests... image: ${e2e_test_runner_image}" # tests modify files in ./manifests, so we mount a copy of this directory done by the e2e Makefile docker run --rm --network=host -e "TERM=xterm-256color" \ @@ -61,11 +64,11 @@ function run_tests(){ --mount type=bind,source="$(readlink -f manifests)",target=/manifests \ --mount type=bind,source="$(readlink -f tests)",target=/tests \ --mount type=bind,source="$(readlink -f exec.sh)",target=/exec.sh \ - -e OPERATOR_IMAGE="${operator_image}" "${e2e_test_runner_image}" - + --mount type=bind,source="$(readlink -f scripts)",target=/scripts \ + -e OPERATOR_IMAGE="${operator_image}" "${e2e_test_runner_image}" ${E2E_TEST_CASE-} $@ } -function clean_up(){ +function cleanup(){ echo "Executing cleanup" unset KUBECONFIG kind delete cluster --name ${cluster_name} @@ -73,14 +76,16 @@ function clean_up(){ } function main(){ + echo "Entering main function..." + [[ -z ${NOCLEANUP-} ]] && trap "cleanup" QUIT TERM EXIT + pull_images + [[ ! -f ${kubeconfig_path} ]] && start_kind + load_operator_image + set_kind_api_server_ip - trap "clean_up" QUIT TERM EXIT - - time pull_images - time start_kind - time set_kind_api_server_ip - run_tests + shift + run_tests $@ exit 0 } -"$@" +"$1" $@ diff --git a/e2e/scripts/cleanup.sh b/e2e/scripts/cleanup.sh new file mode 100755 index 000000000..2c82388ae --- /dev/null +++ b/e2e/scripts/cleanup.sh @@ -0,0 +1,7 @@ +#!/bin/bash +kubectl delete postgresql acid-minimal-cluster +kubectl delete deployments -l application=db-connection-pooler,cluster-name=acid-minimal-cluster +kubectl delete statefulsets -l application=spilo,cluster-name=acid-minimal-cluster +kubectl delete services -l application=spilo,cluster-name=acid-minimal-cluster +kubectl delete configmap postgres-operator +kubectl delete deployment postgres-operator \ No newline at end of file diff --git a/e2e/scripts/get_logs.sh b/e2e/scripts/get_logs.sh new file mode 100755 index 000000000..1639f3995 --- /dev/null +++ b/e2e/scripts/get_logs.sh @@ -0,0 +1,2 @@ +#!/bin/bash +kubectl logs $(kubectl get pods -l name=postgres-operator --field-selector status.phase=Running -o jsonpath='{.items..metadata.name}') diff --git a/e2e/scripts/watch_objects.sh b/e2e/scripts/watch_objects.sh new file mode 100755 index 000000000..c866fbd45 --- /dev/null +++ b/e2e/scripts/watch_objects.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +watch -c " +kubectl get postgresql +echo +echo -n 'Rolling upgrade pending: ' +kubectl get statefulset -o jsonpath='{.items..metadata.annotations.zalando-postgres-operator-rolling-update-required}' +echo +echo +kubectl get pods -o wide +echo +kubectl get statefulsets +echo +kubectl get deployments +echo +kubectl get pods -l name=postgres-operator -o jsonpath='{.items..metadata.annotations.step}' +echo +kubectl get pods -l application=spilo -o jsonpath='{.items..spec.containers..image}' +" \ No newline at end of file diff --git a/e2e/tests/k8s_api.py b/e2e/tests/k8s_api.py new file mode 100644 index 000000000..371fa8e0d --- /dev/null +++ b/e2e/tests/k8s_api.py @@ -0,0 +1,522 @@ +import json +import unittest +import time +import timeout_decorator +import subprocess +import warnings +import os +import yaml + +from datetime import datetime +from kubernetes import client, config +from kubernetes.client.rest import ApiException + +def to_selector(labels): + return ",".join(["=".join(l) for l in labels.items()]) + +class K8sApi: + + def __init__(self): + + # https://github.com/kubernetes-client/python/issues/309 + warnings.simplefilter("ignore", ResourceWarning) + + self.config = config.load_kube_config() + self.k8s_client = client.ApiClient() + + self.core_v1 = client.CoreV1Api() + self.apps_v1 = client.AppsV1Api() + self.batch_v1_beta1 = client.BatchV1beta1Api() + self.custom_objects_api = client.CustomObjectsApi() + self.policy_v1_beta1 = client.PolicyV1beta1Api() + self.storage_v1_api = client.StorageV1Api() + + +class K8s: + ''' + Wraps around K8s api client and helper methods. + ''' + + RETRY_TIMEOUT_SEC = 1 + + def __init__(self, labels='x=y', namespace='default'): + self.api = K8sApi() + self.labels=labels + self.namespace=namespace + + def get_pg_nodes(self, pg_cluster_name, namespace='default'): + master_pod_node = '' + replica_pod_nodes = [] + podsList = self.api.core_v1.list_namespaced_pod(namespace, label_selector=pg_cluster_name) + for pod in podsList.items: + if pod.metadata.labels.get('spilo-role') == 'master': + master_pod_node = pod.spec.node_name + elif pod.metadata.labels.get('spilo-role') == 'replica': + replica_pod_nodes.append(pod.spec.node_name) + + return master_pod_node, replica_pod_nodes + + def get_cluster_nodes(self, cluster_labels='cluster-name=acid-minimal-cluster', namespace='default'): + m = [] + r = [] + podsList = self.api.core_v1.list_namespaced_pod(namespace, label_selector=cluster_labels) + for pod in podsList.items: + if pod.metadata.labels.get('spilo-role') == 'master' and pod.status.phase == 'Running': + m.append(pod.spec.node_name) + elif pod.metadata.labels.get('spilo-role') == 'replica' and pod.status.phase == 'Running': + r.append(pod.spec.node_name) + + return m, r + + def wait_for_operator_pod_start(self): + self.wait_for_pod_start("name=postgres-operator") + # give operator time to subscribe to objects + time.sleep(1) + return True + + def get_operator_pod(self): + pods = self.api.core_v1.list_namespaced_pod( + 'default', label_selector='name=postgres-operator' + ).items + + pods = list(filter(lambda x: x.status.phase=='Running', pods)) + + if len(pods): + return pods[0] + + return None + + def get_operator_log(self): + operator_pod = self.get_operator_pod() + pod_name = operator_pod.metadata.name + return self.api.core_v1.read_namespaced_pod_log( + name=pod_name, + namespace='default' + ) + + def pg_get_status(self, name="acid-minimal-cluster", namespace="default"): + pg = self.api.custom_objects_api.get_namespaced_custom_object( + "acid.zalan.do", "v1", namespace, "postgresqls", name) + return pg.get("status", {}).get("PostgresClusterStatus", None) + + def wait_for_pod_start(self, pod_labels, namespace='default'): + pod_phase = 'No pod running' + while pod_phase != 'Running': + pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=pod_labels).items + if pods: + pod_phase = pods[0].status.phase + + time.sleep(self.RETRY_TIMEOUT_SEC) + + + def get_service_type(self, svc_labels, namespace='default'): + svc_type = '' + svcs = self.api.core_v1.list_namespaced_service(namespace, label_selector=svc_labels, limit=1).items + for svc in svcs: + svc_type = svc.spec.type + return svc_type + + def check_service_annotations(self, svc_labels, annotations, namespace='default'): + svcs = self.api.core_v1.list_namespaced_service(namespace, label_selector=svc_labels, limit=1).items + for svc in svcs: + for key, value in annotations.items(): + if not svc.metadata.annotations or key not in svc.metadata.annotations or svc.metadata.annotations[key] != value: + print("Expected key {} not found in annotations {}".format(key, svc.metadata.annotations)) + return False + return True + + def check_statefulset_annotations(self, sset_labels, annotations, namespace='default'): + ssets = self.api.apps_v1.list_namespaced_stateful_set(namespace, label_selector=sset_labels, limit=1).items + for sset in ssets: + for key, value in annotations.items(): + if key not in sset.metadata.annotations or sset.metadata.annotations[key] != value: + print("Expected key {} not found in annotations {}".format(key, sset.metadata.annotations)) + return False + return True + + def scale_cluster(self, number_of_instances, name="acid-minimal-cluster", namespace="default"): + body = { + "spec": { + "numberOfInstances": number_of_instances + } + } + self.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", namespace, "postgresqls", name, body) + + def wait_for_running_pods(self, labels, number, namespace=''): + while self.count_pods_with_label(labels) != number: + time.sleep(self.RETRY_TIMEOUT_SEC) + + def wait_for_pods_to_stop(self, labels, namespace=''): + while self.count_pods_with_label(labels) != 0: + time.sleep(self.RETRY_TIMEOUT_SEC) + + def wait_for_service(self, labels, namespace='default'): + def get_services(): + return self.api.core_v1.list_namespaced_service( + namespace, label_selector=labels + ).items + + while not get_services(): + time.sleep(self.RETRY_TIMEOUT_SEC) + + def count_pods_with_label(self, labels, namespace='default'): + return len(self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items) + + def count_services_with_label(self, labels, namespace='default'): + return len(self.api.core_v1.list_namespaced_service(namespace, label_selector=labels).items) + + def count_endpoints_with_label(self, labels, namespace='default'): + return len(self.api.core_v1.list_namespaced_endpoints(namespace, label_selector=labels).items) + + def count_secrets_with_label(self, labels, namespace='default'): + return len(self.api.core_v1.list_namespaced_secret(namespace, label_selector=labels).items) + + def count_statefulsets_with_label(self, labels, namespace='default'): + return len(self.api.apps_v1.list_namespaced_stateful_set(namespace, label_selector=labels).items) + + def count_deployments_with_label(self, labels, namespace='default'): + return len(self.api.apps_v1.list_namespaced_deployment(namespace, label_selector=labels).items) + + def count_pdbs_with_label(self, labels, namespace='default'): + return len(self.api.policy_v1_beta1.list_namespaced_pod_disruption_budget( + namespace, label_selector=labels).items) + + def count_running_pods(self, labels='application=spilo,cluster-name=acid-minimal-cluster', namespace='default'): + pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items + return len(list(filter(lambda x: x.status.phase=='Running', pods))) + + def wait_for_pod_failover(self, failover_targets, labels, namespace='default'): + pod_phase = 'Failing over' + new_pod_node = '' + + while (pod_phase != 'Running') or (new_pod_node not in failover_targets): + pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items + if pods: + new_pod_node = pods[0].spec.node_name + pod_phase = pods[0].status.phase + time.sleep(self.RETRY_TIMEOUT_SEC) + + def get_logical_backup_job(self, namespace='default'): + return self.api.batch_v1_beta1.list_namespaced_cron_job(namespace, label_selector="application=spilo") + + def wait_for_logical_backup_job(self, expected_num_of_jobs): + while (len(self.get_logical_backup_job().items) != expected_num_of_jobs): + time.sleep(self.RETRY_TIMEOUT_SEC) + + def wait_for_logical_backup_job_deletion(self): + self.wait_for_logical_backup_job(expected_num_of_jobs=0) + + def wait_for_logical_backup_job_creation(self): + self.wait_for_logical_backup_job(expected_num_of_jobs=1) + + def delete_operator_pod(self, step="Delete operator deplyment"): + operator_pod = self.api.core_v1.list_namespaced_pod('default', label_selector="name=postgres-operator").items[0].metadata.name + self.api.apps_v1.patch_namespaced_deployment("postgres-operator","default", {"spec":{"template":{"metadata":{"annotations":{"step":"{}-{}".format(step, time.time())}}}}}) + self.wait_for_operator_pod_start() + + def update_config(self, config_map_patch, step="Updating operator deployment"): + self.api.core_v1.patch_namespaced_config_map("postgres-operator", "default", config_map_patch) + self.delete_operator_pod(step=step) + + def patch_statefulset(self, data, name="acid-minimal-cluster", namespace="default"): + self.api.apps_v1.patch_namespaced_stateful_set(name, namespace, data) + + def create_with_kubectl(self, path): + return subprocess.run( + ["kubectl", "apply", "-f", path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + def exec_with_kubectl(self, pod, cmd): + return subprocess.run(["./exec.sh", pod, cmd], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + def get_patroni_state(self, pod): + r = self.exec_with_kubectl(pod, "patronictl list -f json") + if not r.returncode == 0 or not r.stdout.decode()[0:1]=="[": + return [] + return json.loads(r.stdout.decode()) + + def get_patroni_running_members(self, pod="acid-minimal-cluster-0"): + result = self.get_patroni_state(pod) + return list(filter(lambda x: "State" in x and x["State"] == "running", result)) + + def get_deployment_replica_count(self, name="acid-minimal-cluster-pooler", namespace="default"): + try: + deployment = self.api.apps_v1.read_namespaced_deployment(name, namespace) + return deployment.spec.replicas + except ApiException as e: + return None + + def get_statefulset_image(self, label_selector="application=spilo,cluster-name=acid-minimal-cluster", namespace='default'): + ssets = self.api.apps_v1.list_namespaced_stateful_set(namespace, label_selector=label_selector, limit=1) + if len(ssets.items) == 0: + return None + return ssets.items[0].spec.template.spec.containers[0].image + + def get_effective_pod_image(self, pod_name, namespace='default'): + ''' + Get the Spilo image pod currently uses. In case of lazy rolling updates + it may differ from the one specified in the stateful set. + ''' + pod = self.api.core_v1.list_namespaced_pod( + namespace, label_selector="statefulset.kubernetes.io/pod-name=" + pod_name) + + if len(pod.items) == 0: + return None + return pod.items[0].spec.containers[0].image + + def get_cluster_leader_pod(self, pg_cluster_name, namespace='default'): + labels = { + 'application': 'spilo', + 'cluster-name': pg_cluster_name, + 'spilo-role': 'master', + } + + pods = self.api.core_v1.list_namespaced_pod( + namespace, label_selector=to_selector(labels)).items + + if pods: + return pods[0] + + +class K8sBase: + ''' + K8s basic API wrapper class supposed to be inherited by other more specific classes for e2e tests + ''' + + RETRY_TIMEOUT_SEC = 1 + + def __init__(self, labels='x=y', namespace='default'): + self.api = K8sApi() + self.labels=labels + self.namespace=namespace + + def get_pg_nodes(self, pg_cluster_labels='cluster-name=acid-minimal-cluster', namespace='default'): + master_pod_node = '' + replica_pod_nodes = [] + podsList = self.api.core_v1.list_namespaced_pod(namespace, label_selector=pg_cluster_labels) + for pod in podsList.items: + if pod.metadata.labels.get('spilo-role') == 'master': + master_pod_node = pod.spec.node_name + elif pod.metadata.labels.get('spilo-role') == 'replica': + replica_pod_nodes.append(pod.spec.node_name) + + return master_pod_node, replica_pod_nodes + + def get_cluster_nodes(self, cluster_labels='cluster-name=acid-minimal-cluster', namespace='default'): + m = [] + r = [] + podsList = self.api.core_v1.list_namespaced_pod(namespace, label_selector=cluster_labels) + for pod in podsList.items: + if pod.metadata.labels.get('spilo-role') == 'master' and pod.status.phase == 'Running': + m.append(pod.spec.node_name) + elif pod.metadata.labels.get('spilo-role') == 'replica' and pod.status.phase == 'Running': + r.append(pod.spec.node_name) + + return m, r + + def wait_for_operator_pod_start(self): + self.wait_for_pod_start("name=postgres-operator") + + def get_operator_pod(self): + pods = self.api.core_v1.list_namespaced_pod( + 'default', label_selector='name=postgres-operator' + ).items + + if pods: + return pods[0] + + return None + + def get_operator_log(self): + operator_pod = self.get_operator_pod() + pod_name = operator_pod.metadata.name + return self.api.core_v1.read_namespaced_pod_log( + name=pod_name, + namespace='default' + ) + + def wait_for_pod_start(self, pod_labels, namespace='default'): + pod_phase = 'No pod running' + while pod_phase != 'Running': + pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=pod_labels).items + if pods: + pod_phase = pods[0].status.phase + + time.sleep(self.RETRY_TIMEOUT_SEC) + + def get_service_type(self, svc_labels, namespace='default'): + svc_type = '' + svcs = self.api.core_v1.list_namespaced_service(namespace, label_selector=svc_labels, limit=1).items + for svc in svcs: + svc_type = svc.spec.type + return svc_type + + def check_service_annotations(self, svc_labels, annotations, namespace='default'): + svcs = self.api.core_v1.list_namespaced_service(namespace, label_selector=svc_labels, limit=1).items + for svc in svcs: + for key, value in annotations.items(): + if key not in svc.metadata.annotations or svc.metadata.annotations[key] != value: + print("Expected key {} not found in annotations {}".format(key, svc.metadata.annotation)) + return False + return True + + def check_statefulset_annotations(self, sset_labels, annotations, namespace='default'): + ssets = self.api.apps_v1.list_namespaced_stateful_set(namespace, label_selector=sset_labels, limit=1).items + for sset in ssets: + for key, value in annotations.items(): + if key not in sset.metadata.annotations or sset.metadata.annotations[key] != value: + print("Expected key {} not found in annotations {}".format(key, sset.metadata.annotation)) + return False + return True + + def scale_cluster(self, number_of_instances, name="acid-minimal-cluster", namespace="default"): + body = { + "spec": { + "numberOfInstances": number_of_instances + } + } + self.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", namespace, "postgresqls", name, body) + + def wait_for_running_pods(self, labels, number, namespace=''): + while self.count_pods_with_label(labels) != number: + time.sleep(self.RETRY_TIMEOUT_SEC) + + def wait_for_pods_to_stop(self, labels, namespace=''): + while self.count_pods_with_label(labels) != 0: + time.sleep(self.RETRY_TIMEOUT_SEC) + + def wait_for_service(self, labels, namespace='default'): + def get_services(): + return self.api.core_v1.list_namespaced_service( + namespace, label_selector=labels + ).items + + while not get_services(): + time.sleep(self.RETRY_TIMEOUT_SEC) + + def count_pods_with_label(self, labels, namespace='default'): + return len(self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items) + + def count_services_with_label(self, labels, namespace='default'): + return len(self.api.core_v1.list_namespaced_service(namespace, label_selector=labels).items) + + def count_endpoints_with_label(self, labels, namespace='default'): + return len(self.api.core_v1.list_namespaced_endpoints(namespace, label_selector=labels).items) + + def count_secrets_with_label(self, labels, namespace='default'): + return len(self.api.core_v1.list_namespaced_secret(namespace, label_selector=labels).items) + + def count_statefulsets_with_label(self, labels, namespace='default'): + return len(self.api.apps_v1.list_namespaced_stateful_set(namespace, label_selector=labels).items) + + def count_deployments_with_label(self, labels, namespace='default'): + return len(self.api.apps_v1.list_namespaced_deployment(namespace, label_selector=labels).items) + + def count_pdbs_with_label(self, labels, namespace='default'): + return len(self.api.policy_v1_beta1.list_namespaced_pod_disruption_budget( + namespace, label_selector=labels).items) + + def count_running_pods(self, labels='application=spilo,cluster-name=acid-minimal-cluster', namespace='default'): + pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items + return len(list(filter(lambda x: x.status.phase=='Running', pods))) + + def wait_for_pod_failover(self, failover_targets, labels, namespace='default'): + pod_phase = 'Failing over' + new_pod_node = '' + + while (pod_phase != 'Running') or (new_pod_node not in failover_targets): + pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items + if pods: + new_pod_node = pods[0].spec.node_name + pod_phase = pods[0].status.phase + time.sleep(self.RETRY_TIMEOUT_SEC) + + def get_logical_backup_job(self, namespace='default'): + return self.api.batch_v1_beta1.list_namespaced_cron_job(namespace, label_selector="application=spilo") + + def wait_for_logical_backup_job(self, expected_num_of_jobs): + while (len(self.get_logical_backup_job().items) != expected_num_of_jobs): + time.sleep(self.RETRY_TIMEOUT_SEC) + + def wait_for_logical_backup_job_deletion(self): + self.wait_for_logical_backup_job(expected_num_of_jobs=0) + + def wait_for_logical_backup_job_creation(self): + self.wait_for_logical_backup_job(expected_num_of_jobs=1) + + def delete_operator_pod(self, step="Delete operator deplyment"): + operator_pod = self.api.core_v1.list_namespaced_pod('default', label_selector="name=postgres-operator").items[0].metadata.name + self.api.apps_v1.patch_namespaced_deployment("postgres-operator","default", {"spec":{"template":{"metadata":{"annotations":{"step":"{}-{}".format(step, time.time())}}}}}) + self.wait_for_operator_pod_start() + + def update_config(self, config_map_patch, step="Updating operator deployment"): + self.api.core_v1.patch_namespaced_config_map("postgres-operator", "default", config_map_patch) + self.delete_operator_pod(step=step) + + def create_with_kubectl(self, path): + return subprocess.run( + ["kubectl", "apply", "-f", path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + def exec_with_kubectl(self, pod, cmd): + return subprocess.run(["./exec.sh", pod, cmd], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + def get_patroni_state(self, pod): + r = self.exec_with_kubectl(pod, "patronictl list -f json") + if not r.returncode == 0 or not r.stdout.decode()[0:1]=="[": + return [] + return json.loads(r.stdout.decode()) + + def get_patroni_running_members(self, pod): + result = self.get_patroni_state(pod) + return list(filter(lambda x: x["State"]=="running", result)) + + def get_statefulset_image(self, label_selector="application=spilo,cluster-name=acid-minimal-cluster", namespace='default'): + ssets = self.api.apps_v1.list_namespaced_stateful_set(namespace, label_selector=label_selector, limit=1) + if len(ssets.items) == 0: + return None + return ssets.items[0].spec.template.spec.containers[0].image + + def get_effective_pod_image(self, pod_name, namespace='default'): + ''' + Get the Spilo image pod currently uses. In case of lazy rolling updates + it may differ from the one specified in the stateful set. + ''' + pod = self.api.core_v1.list_namespaced_pod( + namespace, label_selector="statefulset.kubernetes.io/pod-name=" + pod_name) + + if len(pod.items) == 0: + return None + return pod.items[0].spec.containers[0].image + + +""" + Inspiriational classes towards easier writing of end to end tests with one cluster per test case +""" +class K8sOperator(K8sBase): + def __init__(self, labels="name=postgres-operator", namespace="default"): + super().__init__(labels, namespace) + +class K8sPostgres(K8sBase): + def __init__(self, labels="cluster-name=acid-minimal-cluster", namespace="default"): + super().__init__(labels, namespace) + + def get_pg_nodes(self): + master_pod_node = '' + replica_pod_nodes = [] + podsList = self.api.core_v1.list_namespaced_pod(self.namespace, label_selector=self.labels) + for pod in podsList.items: + if pod.metadata.labels.get('spilo-role') == 'master': + master_pod_node = pod.spec.node_name + elif pod.metadata.labels.get('spilo-role') == 'replica': + replica_pod_nodes.append(pod.spec.node_name) + + return master_pod_node, replica_pod_nodes \ No newline at end of file diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index fc251c430..888fc2eaa 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -10,6 +10,10 @@ import yaml from datetime import datetime from kubernetes import client, config +from tests.k8s_api import K8s + +SPILO_CURRENT = "registry.opensource.zalan.do/acid/spilo-12:1.6-p5" +SPILO_LAZY = "registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p114" def to_selector(labels): return ",".join(["=".join(l) for l in labels.items()]) @@ -31,6 +35,41 @@ class EndToEndTestCase(unittest.TestCase): # `kind` pods may stuck in the `Terminating` phase for a few minutes; hence high test timeout TEST_TIMEOUT_SEC = 600 + def eventuallyEqual(self, f, x, m, retries=60, interval=2): + while True: + try: + y = f() + self.assertEqual(y, x, m.format(y)) + return True + except AssertionError: + retries = retries -1 + if not retries > 0: + raise + time.sleep(interval) + + def eventuallyNotEqual(self, f, x, m, retries=60, interval=2): + while True: + try: + y = f() + self.assertNotEqual(y, x, m.format(y)) + return True + except AssertionError: + retries = retries -1 + if not retries > 0: + raise + time.sleep(interval) + + def eventuallyTrue(self, f, m, retries=60, interval=2): + while True: + try: + self.assertTrue(f(), m) + return True + except AssertionError: + retries = retries -1 + if not retries > 0: + raise + time.sleep(interval) + @classmethod @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def setUpClass(cls): @@ -48,18 +87,26 @@ class EndToEndTestCase(unittest.TestCase): k8s = cls.k8s = K8s() # remove existing local storage class and create hostpath class - k8s.api.storage_v1_api.delete_storage_class("standard") + try: + k8s.api.storage_v1_api.delete_storage_class("standard") + except: + print("Storage class has already been remove") # operator deploys pod service account there on start up # needed for test_multi_namespace_support() cls.namespace = "test" - v1_namespace = client.V1Namespace(metadata=client.V1ObjectMeta(name=cls.namespace)) - k8s.api.core_v1.create_namespace(v1_namespace) + try: + v1_namespace = client.V1Namespace(metadata=client.V1ObjectMeta(name=cls.namespace)) + k8s.api.core_v1.create_namespace(v1_namespace) + except: + print("Namespace already present") # submit the most recent operator image built on the Docker host with open("manifests/postgres-operator.yaml", 'r+') as f: operator_deployment = yaml.safe_load(f) operator_deployment["spec"]["template"]["spec"]["containers"][0]["image"] = os.environ['OPERATOR_IMAGE'] + + with open("manifests/postgres-operator.yaml", 'w') as f: yaml.dump(operator_deployment, f, Dumper=yaml.Dumper) for filename in ["operator-service-account-rbac.yaml", @@ -73,6 +120,18 @@ class EndToEndTestCase(unittest.TestCase): k8s.wait_for_operator_pod_start() + # reset taints and tolerations + k8s.api.core_v1.patch_node("postgres-operator-e2e-tests-worker",{"spec":{"taints":[]}}) + k8s.api.core_v1.patch_node("postgres-operator-e2e-tests-worker2",{"spec":{"taints":[]}}) + + # make sure we start a new operator on every new run, + # this tackles the problem when kind is reused + # and the Docker image is infact changed (dirty one) + + # patch resync period, this can catch some problems with hanging e2e tests + # k8s.update_config({"data": {"resync_period":"30s"}},step="TestSuite setup") + k8s.update_config({}, step="TestSuite Startup") + actual_operator_image = k8s.api.core_v1.list_namespaced_pod( 'default', label_selector='name=postgres-operator').items[0].spec.containers[0].image print("Tested operator image: {}".format(actual_operator_image)) # shows up after tests finish @@ -105,124 +164,122 @@ class EndToEndTestCase(unittest.TestCase): pod_selector = to_selector(pod_labels) service_selector = to_selector(service_labels) - try: - # enable connection pooler - k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresqls', 'acid-minimal-cluster', - { - 'spec': { - 'enableConnectionPooler': True, - } - }) - k8s.wait_for_pod_start(pod_selector) + # enable connection pooler + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'enableConnectionPooler': True, + } + }) - pods = k8s.api.core_v1.list_namespaced_pod( - 'default', label_selector=pod_selector - ).items + self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(), 2, "Deployment replicas is 2 default") + self.eventuallyEqual(lambda: k8s.count_running_pods("connection-pooler=acid-minimal-cluster-pooler"), 2, "No pooler pods found") + self.eventuallyEqual(lambda: k8s.count_services_with_label('application=db-connection-pooler,cluster-name=acid-minimal-cluster'), 1, "No pooler service found") - self.assertTrue(pods, 'No connection pooler pods') + # scale up connection pooler deployment + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'connectionPooler': { + 'numberOfInstances': 3, + }, + } + }) - k8s.wait_for_service(service_selector) - services = k8s.api.core_v1.list_namespaced_service( - 'default', label_selector=service_selector - ).items - services = [ - s for s in services - if s.metadata.name.endswith('pooler') - ] + self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(), 3, "Deployment replicas is scaled to 3") + self.eventuallyEqual(lambda: k8s.count_running_pods("connection-pooler=acid-minimal-cluster-pooler"), 3, "Scale up of pooler pods does not work") - self.assertTrue(services, 'No connection pooler service') + # turn it off, keeping config should be overwritten by false + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'enableConnectionPooler': False + } + }) - # scale up connection pooler deployment - k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresqls', 'acid-minimal-cluster', - { - 'spec': { - 'connectionPooler': { - 'numberOfInstances': 2, - }, - } - }) + self.eventuallyEqual(lambda: k8s.count_running_pods("connection-pooler=acid-minimal-cluster-pooler"), 0, "Pooler pods not scaled down") + self.eventuallyEqual(lambda: k8s.count_services_with_label('application=db-connection-pooler,cluster-name=acid-minimal-cluster'), 0, "Pooler service not removed") - k8s.wait_for_running_pods(pod_selector, 2) + # Verify that all the databases have pooler schema installed. + # Do this via psql, since otherwise we need to deal with + # credentials. + dbList = [] - # Verify that all the databases have pooler schema installed. - # Do this via psql, since otherwise we need to deal with - # credentials. - dbList = [] + leader = k8s.get_cluster_leader_pod('acid-minimal-cluster') + dbListQuery = "select datname from pg_database" + schemasQuery = """ + select schema_name + from information_schema.schemata + where schema_name = 'pooler' + """ + exec_query = r"psql -tAq -c \"{}\" -d {}" - leader = k8s.get_cluster_leader_pod('acid-minimal-cluster') - dbListQuery = "select datname from pg_database" - schemasQuery = """ - select schema_name - from information_schema.schemata - where schema_name = 'pooler' - """ - exec_query = r"psql -tAq -c \"{}\" -d {}" + if leader: + try: + q = exec_query.format(dbListQuery, "postgres") + q = "su postgres -c \"{}\"".format(q) + print('Get databases: {}'.format(q)) + result = k8s.exec_with_kubectl(leader.metadata.name, q) + dbList = clean_list(result.stdout.split(b'\n')) + print('dbList: {}, stdout: {}, stderr {}'.format( + dbList, result.stdout, result.stderr + )) + except Exception as ex: + print('Could not get databases: {}'.format(ex)) + print('Stdout: {}'.format(result.stdout)) + print('Stderr: {}'.format(result.stderr)) - if leader: + for db in dbList: + if db in ('template0', 'template1'): + continue + + schemas = [] try: - q = exec_query.format(dbListQuery, "postgres") + q = exec_query.format(schemasQuery, db) q = "su postgres -c \"{}\"".format(q) - print('Get databases: {}'.format(q)) + print('Get schemas: {}'.format(q)) result = k8s.exec_with_kubectl(leader.metadata.name, q) - dbList = clean_list(result.stdout.split(b'\n')) - print('dbList: {}, stdout: {}, stderr {}'.format( - dbList, result.stdout, result.stderr + schemas = clean_list(result.stdout.split(b'\n')) + print('schemas: {}, stdout: {}, stderr {}'.format( + schemas, result.stdout, result.stderr )) except Exception as ex: print('Could not get databases: {}'.format(ex)) print('Stdout: {}'.format(result.stdout)) print('Stderr: {}'.format(result.stderr)) - for db in dbList: - if db in ('template0', 'template1'): - continue + self.assertNotEqual(len(schemas), 0) + else: + print('Could not find leader pod') - schemas = [] - try: - q = exec_query.format(schemasQuery, db) - q = "su postgres -c \"{}\"".format(q) - print('Get schemas: {}'.format(q)) - result = k8s.exec_with_kubectl(leader.metadata.name, q) - schemas = clean_list(result.stdout.split(b'\n')) - print('schemas: {}, stdout: {}, stderr {}'.format( - schemas, result.stdout, result.stderr - )) - except Exception as ex: - print('Could not get databases: {}'.format(ex)) - print('Stdout: {}'.format(result.stdout)) - print('Stderr: {}'.format(result.stderr)) - - self.assertNotEqual(len(schemas), 0) - else: - print('Could not find leader pod') - - # turn it off, keeping configuration section - k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresqls', 'acid-minimal-cluster', - { - 'spec': { - 'enableConnectionPooler': False, - } - }) - k8s.wait_for_pods_to_stop(pod_selector) - - except timeout_decorator.TimeoutError: - print('Operator log: {}'.format(k8s.get_operator_log())) - raise + # remove config section to make test work next time + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'connectionPooler': None + } + }) @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_enable_load_balancer(self): ''' - Test if services are updated when enabling/disabling load balancers + Test if services are updated when enabling/disabling load balancers in Postgres manifest ''' k8s = self.k8s - cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster,spilo-role={}' + + self.eventuallyEqual(lambda: k8s.get_service_type(cluster_label.format("master")), + 'ClusterIP', + "Expected ClusterIP type initially, found {}") try: # enable load balancer services @@ -234,16 +291,14 @@ class EndToEndTestCase(unittest.TestCase): } k8s.api.custom_objects_api.patch_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_lbs) - # wait for service recreation - time.sleep(60) + + self.eventuallyEqual(lambda: k8s.get_service_type(cluster_label.format("master")), + 'LoadBalancer', + "Expected LoadBalancer service type for master, found {}") - master_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=master') - self.assertEqual(master_svc_type, 'LoadBalancer', - "Expected LoadBalancer service type for master, found {}".format(master_svc_type)) - - repl_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=replica') - self.assertEqual(repl_svc_type, 'LoadBalancer', - "Expected LoadBalancer service type for replica, found {}".format(repl_svc_type)) + self.eventuallyEqual(lambda: k8s.get_service_type(cluster_label.format("replica")), + 'LoadBalancer', + "Expected LoadBalancer service type for master, found {}") # disable load balancer services again pg_patch_disable_lbs = { @@ -254,16 +309,14 @@ class EndToEndTestCase(unittest.TestCase): } k8s.api.custom_objects_api.patch_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_lbs) - # wait for service recreation - time.sleep(60) + + self.eventuallyEqual(lambda: k8s.get_service_type(cluster_label.format("master")), + 'ClusterIP', + "Expected LoadBalancer service type for master, found {}") - master_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=master') - self.assertEqual(master_svc_type, 'ClusterIP', - "Expected ClusterIP service type for master, found {}".format(master_svc_type)) - - repl_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=replica') - self.assertEqual(repl_svc_type, 'ClusterIP', - "Expected ClusterIP service type for replica, found {}".format(repl_svc_type)) + self.eventuallyEqual(lambda: k8s.get_service_type(cluster_label.format("replica")), + 'ClusterIP', + "Expected LoadBalancer service type for master, found {}") except timeout_decorator.TimeoutError: print('Operator log: {}'.format(k8s.get_operator_log())) @@ -277,8 +330,7 @@ class EndToEndTestCase(unittest.TestCase): k8s = self.k8s # update infrastructure roles description secret_name = "postgresql-infrastructure-roles" - roles = "secretname: postgresql-infrastructure-roles-new, \ - userkey: user, rolekey: memberof, passwordkey: password, defaultrolevalue: robot_zmon" + roles = "secretname: postgresql-infrastructure-roles-new, userkey: user, rolekey: memberof, passwordkey: password, defaultrolevalue: robot_zmon" patch_infrastructure_roles = { "data": { "infrastructure_roles_secret_name": secret_name, @@ -287,33 +339,41 @@ class EndToEndTestCase(unittest.TestCase): } k8s.update_config(patch_infrastructure_roles) - # wait a little before proceeding - time.sleep(30) - try: # check that new roles are represented in the config by requesting the # operator configuration via API - operator_pod = k8s.get_operator_pod() - get_config_cmd = "wget --quiet -O - localhost:8080/config" - result = k8s.exec_with_kubectl( - operator_pod.metadata.name, - get_config_cmd, - ) - roles_dict = (json.loads(result.stdout) - .get("controller", {}) - .get("InfrastructureRoles")) - self.assertTrue("robot_zmon_acid_monitoring_new" in roles_dict) - role = roles_dict["robot_zmon_acid_monitoring_new"] - role.pop("Password", None) - self.assertDictEqual(role, { - "Name": "robot_zmon_acid_monitoring_new", - "Flags": None, - "MemberOf": ["robot_zmon"], - "Parameters": None, - "AdminRole": "", - "Origin": 2, - }) + def verify_role(): + try: + operator_pod = k8s.get_operator_pod() + get_config_cmd = "wget --quiet -O - localhost:8080/config" + result = k8s.exec_with_kubectl(operator_pod.metadata.name, get_config_cmd) + try: + roles_dict = (json.loads(result.stdout) + .get("controller", {}) + .get("InfrastructureRoles")) + except: + return False + + if "robot_zmon_acid_monitoring_new" in roles_dict: + role = roles_dict["robot_zmon_acid_monitoring_new"] + role.pop("Password", None) + self.assertDictEqual(role, { + "Name": "robot_zmon_acid_monitoring_new", + "Flags": None, + "MemberOf": ["robot_zmon"], + "Parameters": None, + "AdminRole": "", + "Origin": 2, + }) + return True + except: + pass + + return False + + self.eventuallyTrue(verify_role, "infrastructure role setup is not loaded") + except timeout_decorator.TimeoutError: print('Operator log: {}'.format(k8s.get_operator_log())) @@ -333,33 +393,47 @@ class EndToEndTestCase(unittest.TestCase): k8s = self.k8s + pod0 = 'acid-minimal-cluster-0' + pod1 = 'acid-minimal-cluster-1' + + self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No 2 pods running") + self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members(pod0)), 2, "Postgres status did not enter running") + + patch_lazy_spilo_upgrade = { + "data": { + "docker_image": SPILO_CURRENT, + "enable_lazy_spilo_upgrade": "false" + } + } + k8s.update_config(patch_lazy_spilo_upgrade, step="Init baseline image version") + + self.eventuallyEqual(lambda: k8s.get_statefulset_image(), SPILO_CURRENT, "Stagefulset not updated initially") + self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No 2 pods running") + self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members(pod0)), 2, "Postgres status did not enter running") + + self.eventuallyEqual(lambda: k8s.get_effective_pod_image(pod0), SPILO_CURRENT, "Rolling upgrade was not executed") + self.eventuallyEqual(lambda: k8s.get_effective_pod_image(pod1), SPILO_CURRENT, "Rolling upgrade was not executed") + # update docker image in config and enable the lazy upgrade - conf_image = "registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p114" + conf_image = SPILO_LAZY patch_lazy_spilo_upgrade = { "data": { "docker_image": conf_image, "enable_lazy_spilo_upgrade": "true" } } - k8s.update_config(patch_lazy_spilo_upgrade) - - pod0 = 'acid-minimal-cluster-0' - pod1 = 'acid-minimal-cluster-1' + k8s.update_config(patch_lazy_spilo_upgrade,step="patch image and lazy upgrade") + self.eventuallyEqual(lambda: k8s.get_statefulset_image(), conf_image, "Statefulset not updated to next Docker image") try: # restart the pod to get a container with the new image - k8s.api.core_v1.delete_namespaced_pod(pod0, 'default') - time.sleep(60) - - # lazy update works if the restarted pod and older pods run different Spilo versions - new_image = k8s.get_effective_pod_image(pod0) - old_image = k8s.get_effective_pod_image(pod1) - self.assertNotEqual(new_image, old_image, - "Lazy updated failed: pods have the same image {}".format(new_image)) - - # sanity check - assert_msg = "Image {} of a new pod differs from {} in operator conf".format(new_image, conf_image) - self.assertEqual(new_image, conf_image, assert_msg) + k8s.api.core_v1.delete_namespaced_pod(pod0, 'default') + + # verify only pod-0 which was deleted got new image from statefulset + self.eventuallyEqual(lambda: k8s.get_effective_pod_image(pod0), conf_image, "Delete pod-0 did not get new spilo image") + self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No two pods running after lazy rolling upgrade") + self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members(pod0)), 2, "Postgres status did not enter running") + self.assertNotEqual(lambda: k8s.get_effective_pod_image(pod1), SPILO_CURRENT, "pod-1 should not have change Docker image to {}".format(SPILO_CURRENT)) # clean up unpatch_lazy_spilo_upgrade = { @@ -367,20 +441,12 @@ class EndToEndTestCase(unittest.TestCase): "enable_lazy_spilo_upgrade": "false", } } - k8s.update_config(unpatch_lazy_spilo_upgrade) + k8s.update_config(unpatch_lazy_spilo_upgrade, step="patch lazy upgrade") # at this point operator will complete the normal rolling upgrade # so we additonally test if disabling the lazy upgrade - forcing the normal rolling upgrade - works - - # XXX there is no easy way to wait until the end of Sync() - time.sleep(60) - - image0 = k8s.get_effective_pod_image(pod0) - image1 = k8s.get_effective_pod_image(pod1) - - assert_msg = "Disabling lazy upgrade failed: pods still have different \ - images {} and {}".format(image0, image1) - self.assertEqual(image0, image1, assert_msg) + self.eventuallyEqual(lambda: k8s.get_effective_pod_image(pod0), conf_image, "Rolling upgrade was not executed", 50, 3) + self.eventuallyEqual(lambda: k8s.get_effective_pod_image(pod1), conf_image, "Rolling upgrade was not executed", 50, 3) except timeout_decorator.TimeoutError: print('Operator log: {}'.format(k8s.get_operator_log())) @@ -412,12 +478,9 @@ class EndToEndTestCase(unittest.TestCase): "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_backup) try: - k8s.wait_for_logical_backup_job_creation() + self.eventuallyEqual(lambda: len(k8s.get_logical_backup_job().items), 1, "failed to create logical backup job") - jobs = k8s.get_logical_backup_job().items - self.assertEqual(1, len(jobs), "Expected 1 logical backup job, found {}".format(len(jobs))) - - job = jobs[0] + job = k8s.get_logical_backup_job().items[0] self.assertEqual(job.metadata.name, "logical-backup-acid-minimal-cluster", "Expected job name {}, found {}" .format("logical-backup-acid-minimal-cluster", job.metadata.name)) @@ -432,12 +495,14 @@ class EndToEndTestCase(unittest.TestCase): "logical_backup_docker_image": image, } } - k8s.update_config(patch_logical_backup_image) + k8s.update_config(patch_logical_backup_image, step="patch logical backup image") - jobs = k8s.get_logical_backup_job().items - actual_image = jobs[0].spec.job_template.spec.template.spec.containers[0].image - self.assertEqual(actual_image, image, - "Expected job image {}, found {}".format(image, actual_image)) + def get_docker_image(): + jobs = k8s.get_logical_backup_job().items + return jobs[0].spec.job_template.spec.template.spec.containers[0].image + + self.eventuallyEqual(get_docker_image, image, + "Expected job image {}, found {}".format(image, "{}")) # delete the logical backup cron job pg_patch_disable_backup = { @@ -447,10 +512,8 @@ class EndToEndTestCase(unittest.TestCase): } k8s.api.custom_objects_api.patch_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_backup) - k8s.wait_for_logical_backup_job_deletion() - jobs = k8s.get_logical_backup_job().items - self.assertEqual(0, len(jobs), - "Expected 0 logical backup jobs, found {}".format(len(jobs))) + + self.eventuallyEqual(lambda: len(k8s.get_logical_backup_job().items), 0, "failed to create logical backup job") except timeout_decorator.TimeoutError: print('Operator log: {}'.format(k8s.get_operator_log())) @@ -462,20 +525,18 @@ class EndToEndTestCase(unittest.TestCase): Lower resource limits below configured minimum and let operator fix it ''' k8s = self.k8s - cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' - labels = 'spilo-role=master,' + cluster_label - _, failover_targets = k8s.get_pg_nodes(cluster_label) + # self.eventuallyEqual(lambda: k8s.pg_get_status(), "Running", "Cluster not healthy at start") # configure minimum boundaries for CPU and memory limits - minCPULimit = '500m' - minMemoryLimit = '500Mi' + minCPULimit = '503m' + minMemoryLimit = '502Mi' + patch_min_resource_limits = { "data": { "min_cpu_limit": minCPULimit, "min_memory_limit": minMemoryLimit } } - k8s.update_config(patch_min_resource_limits) # lower resource limits below minimum pg_patch_resources = { @@ -494,26 +555,31 @@ class EndToEndTestCase(unittest.TestCase): } k8s.api.custom_objects_api.patch_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_resources) + + k8s.patch_statefulset({"metadata":{"annotations":{"zalando-postgres-operator-rolling-update-required": "False"}}}) + k8s.update_config(patch_min_resource_limits, "Minimum resource test") - try: - k8s.wait_for_pod_failover(failover_targets, labels) - k8s.wait_for_pod_start('spilo-role=replica') + self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No two pods running after lazy rolling upgrade") + self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members()), 2, "Postgres status did not enter running") + + def verify_pod_limits(): + pods = k8s.api.core_v1.list_namespaced_pod('default', label_selector="cluster-name=acid-minimal-cluster,application=spilo").items + if len(pods)<2: + return False - pods = k8s.api.core_v1.list_namespaced_pod( - 'default', label_selector=labels).items - self.assert_master_is_unique() - masterPod = pods[0] + r = pods[0].spec.containers[0].resources.limits['memory']==minMemoryLimit + r = r and pods[0].spec.containers[0].resources.limits['cpu'] == minCPULimit + r = r and pods[1].spec.containers[0].resources.limits['memory']==minMemoryLimit + r = r and pods[1].spec.containers[0].resources.limits['cpu'] == minCPULimit + return r - self.assertEqual(masterPod.spec.containers[0].resources.limits['cpu'], minCPULimit, - "Expected CPU limit {}, found {}" - .format(minCPULimit, masterPod.spec.containers[0].resources.limits['cpu'])) - self.assertEqual(masterPod.spec.containers[0].resources.limits['memory'], minMemoryLimit, - "Expected memory limit {}, found {}" - .format(minMemoryLimit, masterPod.spec.containers[0].resources.limits['memory'])) + self.eventuallyTrue(verify_pod_limits, "Pod limits where not adjusted") - except timeout_decorator.TimeoutError: - print('Operator log: {}'.format(k8s.get_operator_log())) - raise + @classmethod + def setUp(cls): + # cls.k8s.update_config({}, step="Setup") + cls.k8s.patch_statefulset({"meta":{"annotations":{"zalando-postgres-operator-rolling-update-required": False}}}) + pass @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_multi_namespace_support(self): @@ -537,7 +603,7 @@ class EndToEndTestCase(unittest.TestCase): raise @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_node_readiness_label(self): + def test_zz_node_readiness_label(self): ''' Remove node readiness label from master node. This must cause a failover. ''' @@ -560,6 +626,7 @@ class EndToEndTestCase(unittest.TestCase): } } } + self.assertTrue(len(failover_targets)>0, "No failover targets available") for failover_target in failover_targets: k8s.api.core_v1.patch_node(failover_target, patch_readiness_label) @@ -569,18 +636,15 @@ class EndToEndTestCase(unittest.TestCase): "node_readiness_label": readiness_label + ':' + readiness_value, } } - k8s.update_config(patch_readiness_label_config) + k8s.update_config(patch_readiness_label_config, "setting readiness label") new_master_node, new_replica_nodes = self.assert_failover( current_master_node, num_replicas, failover_targets, cluster_label) # patch also node where master ran before k8s.api.core_v1.patch_node(current_master_node, patch_readiness_label) - # wait a little before proceeding with the pod distribution test - time.sleep(30) - # toggle pod anti affinity to move replica away from master node - self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) + self.eventuallyTrue(lambda: self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label), "Pods are redistributed") except timeout_decorator.TimeoutError: print('Operator log: {}'.format(k8s.get_operator_log())) @@ -592,25 +656,20 @@ class EndToEndTestCase(unittest.TestCase): Scale up from 2 to 3 and back to 2 pods by updating the Postgres manifest at runtime. ''' k8s = self.k8s - labels = "application=spilo,cluster-name=acid-minimal-cluster" + pod="acid-minimal-cluster-0" - try: - k8s.wait_for_pg_to_scale(3) - self.assertEqual(3, k8s.count_pods_with_label(labels)) - self.assert_master_is_unique() - - k8s.wait_for_pg_to_scale(2) - self.assertEqual(2, k8s.count_pods_with_label(labels)) - self.assert_master_is_unique() - - except timeout_decorator.TimeoutError: - print('Operator log: {}'.format(k8s.get_operator_log())) - raise + k8s.scale_cluster(3) + self.eventuallyEqual(lambda: k8s.count_running_pods(), 3, "Scale up to 3 failed") + self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members(pod)), 3, "Not all 3 nodes healthy") + + k8s.scale_cluster(2) + self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "Scale down to 2 failed") + self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members(pod)), 2, "Not all members 2 healthy") @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_service_annotations(self): ''' - Create a Postgres cluster with service annotations and check them. + Create a Postgres cluster with service annotations and check them. ''' k8s = self.k8s patch_custom_service_annotations = { @@ -620,32 +679,25 @@ class EndToEndTestCase(unittest.TestCase): } k8s.update_config(patch_custom_service_annotations) - try: - pg_patch_custom_annotations = { - "spec": { - "serviceAnnotations": { - "annotation.key": "value", - "foo": "bar", - } + pg_patch_custom_annotations = { + "spec": { + "serviceAnnotations": { + "annotation.key": "value", + "alice": "bob", } } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_custom_annotations) + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_custom_annotations) - # wait a little before proceeding - time.sleep(30) - annotations = { - "annotation.key": "value", - "foo": "bar", - } - self.assertTrue(k8s.check_service_annotations( - "cluster-name=acid-minimal-cluster,spilo-role=master", annotations)) - self.assertTrue(k8s.check_service_annotations( - "cluster-name=acid-minimal-cluster,spilo-role=replica", annotations)) + annotations = { + "annotation.key": "value", + "foo": "bar", + "alice": "bob" + } - except timeout_decorator.TimeoutError: - print('Operator log: {}'.format(k8s.get_operator_log())) - raise + self.eventuallyTrue(lambda: k8s.check_service_annotations("cluster-name=acid-minimal-cluster,spilo-role=master", annotations), "Wrong annotations") + self.eventuallyTrue(lambda: k8s.check_service_annotations("cluster-name=acid-minimal-cluster,spilo-role=replica", annotations), "Wrong annotations") # clean up unpatch_custom_service_annotations = { @@ -670,42 +722,43 @@ class EndToEndTestCase(unittest.TestCase): } k8s.update_config(patch_sset_propagate_annotations) - try: - pg_crd_annotations = { - "metadata": { - "annotations": { - "deployment-time": "2020-04-30 12:00:00", - "downscaler/downtime_replicas": "0", - }, - } + pg_crd_annotations = { + "metadata": { + "annotations": { + "deployment-time": "2020-04-30 12:00:00", + "downscaler/downtime_replicas": "0", + }, } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_crd_annotations) + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_crd_annotations) + + annotations = { + "deployment-time": "2020-04-30 12:00:00", + "downscaler/downtime_replicas": "0", + } + + self.eventuallyTrue(lambda: k8s.check_statefulset_annotations(cluster_label, annotations), "Annotations missing") - # wait a little before proceeding - time.sleep(60) - annotations = { - "deployment-time": "2020-04-30 12:00:00", - "downscaler/downtime_replicas": "0", - } - self.assertTrue(k8s.check_statefulset_annotations(cluster_label, annotations)) - - except timeout_decorator.TimeoutError: - print('Operator log: {}'.format(k8s.get_operator_log())) - raise @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_taint_based_eviction(self): + @unittest.skip("Skipping this test until fixed") + def test_zzz_taint_based_eviction(self): ''' Add taint "postgres=:NoExecute" to node with master. This must cause a failover. ''' k8s = self.k8s cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + # verify we are in good state from potential previous tests + self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No 2 pods running") + self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members("acid-minimal-cluster-0")), 2, "Postgres status did not enter running") + # get nodes of master and replica(s) (expected target of new master) - current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label) - num_replicas = len(current_replica_nodes) - failover_targets = self.get_failover_targets(current_master_node, current_replica_nodes) + master_nodes, replica_nodes = k8s.get_cluster_nodes() + + self.assertNotEqual(master_nodes, []) + self.assertNotEqual(replica_nodes, []) # taint node with postgres=:NoExecute to force failover body = { @@ -719,32 +772,29 @@ class EndToEndTestCase(unittest.TestCase): } } - try: - # patch node and test if master is failing over to one of the expected nodes - k8s.api.core_v1.patch_node(current_master_node, body) - new_master_node, new_replica_nodes = self.assert_failover( - current_master_node, num_replicas, failover_targets, cluster_label) + k8s.api.core_v1.patch_node(master_nodes[0], body) + self.eventuallyTrue(lambda: k8s.get_cluster_nodes()[0], replica_nodes) + self.assertNotEqual(lambda: k8s.get_cluster_nodes()[0], master_nodes) - # add toleration to pods - patch_toleration_config = { - "data": { - "toleration": "key:postgres,operator:Exists,effect:NoExecute" - } + # add toleration to pods + patch_toleration_config = { + "data": { + "toleration": "key:postgres,operator:Exists,effect:NoExecute" } - k8s.update_config(patch_toleration_config) + } + + k8s.update_config(patch_toleration_config, step="allow tainted nodes") - # wait a little before proceeding with the pod distribution test - time.sleep(30) + self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No 2 pods running") + self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members("acid-minimal-cluster-0")), 2, "Postgres status did not enter running") - # toggle pod anti affinity to move replica away from master node - self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) - - except timeout_decorator.TimeoutError: - print('Operator log: {}'.format(k8s.get_operator_log())) - raise + # toggle pod anti affinity to move replica away from master node + nm, new_replica_nodes = k8s.get_cluster_nodes() + new_master_node = nm[0] + self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_x_cluster_deletion(self): + def test_zzzz_cluster_deletion(self): ''' Test deletion with configured protection ''' @@ -764,6 +814,7 @@ class EndToEndTestCase(unittest.TestCase): # this delete attempt should be omitted because of missing annotations k8s.api.custom_objects_api.delete_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster") + time.sleep(5) # check that pods and services are still there k8s.wait_for_running_pods(cluster_label, 2) @@ -789,7 +840,7 @@ class EndToEndTestCase(unittest.TestCase): "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_delete_annotations) # wait a little before proceeding - time.sleep(10) + time.sleep(20) k8s.wait_for_running_pods(cluster_label, 2) k8s.wait_for_service(cluster_label) @@ -797,22 +848,31 @@ class EndToEndTestCase(unittest.TestCase): k8s.api.custom_objects_api.delete_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster") - # wait until cluster is deleted - time.sleep(120) + self.eventuallyEqual(lambda: len(k8s.api.custom_objects_api.list_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", label_selector="cluster-name=acid-minimal-cluster")["items"]), 0, "Manifest not deleted") # check if everything has been deleted - self.assertEqual(0, k8s.count_pods_with_label(cluster_label)) - self.assertEqual(0, k8s.count_services_with_label(cluster_label)) - self.assertEqual(0, k8s.count_endpoints_with_label(cluster_label)) - self.assertEqual(0, k8s.count_statefulsets_with_label(cluster_label)) - self.assertEqual(0, k8s.count_deployments_with_label(cluster_label)) - self.assertEqual(0, k8s.count_pdbs_with_label(cluster_label)) - self.assertEqual(0, k8s.count_secrets_with_label(cluster_label)) + self.eventuallyEqual(lambda: k8s.count_pods_with_label(cluster_label), 0, "Pods not deleted") + self.eventuallyEqual(lambda: k8s.count_services_with_label(cluster_label), 0, "Service not deleted") + self.eventuallyEqual(lambda: k8s.count_endpoints_with_label(cluster_label), 0, "Endpoints not deleted") + self.eventuallyEqual(lambda: k8s.count_statefulsets_with_label(cluster_label), 0, "Statefulset not deleted") + self.eventuallyEqual(lambda: k8s.count_deployments_with_label(cluster_label), 0, "Deployments not deleted") + self.eventuallyEqual(lambda: k8s.count_pdbs_with_label(cluster_label), 0, "Pod disruption budget not deleted") + self.eventuallyEqual(lambda: k8s.count_secrets_with_label(cluster_label), 0, "Secrets not deleted") except timeout_decorator.TimeoutError: print('Operator log: {}'.format(k8s.get_operator_log())) raise + #reset configmap + patch_delete_annotations = { + "data": { + "delete_annotation_date_key": "", + "delete_annotation_name_key": "" + } + } + k8s.update_config(patch_delete_annotations) + def get_failover_targets(self, master_node, replica_nodes): ''' If all pods live on the same node, failover will happen to other worker(s) @@ -871,7 +931,7 @@ class EndToEndTestCase(unittest.TestCase): "enable_pod_antiaffinity": "true" } } - k8s.update_config(patch_enable_antiaffinity) + k8s.update_config(patch_enable_antiaffinity, "enable antiaffinity") self.assert_failover(master_node, len(replica_nodes), failover_targets, cluster_label) # now disable pod anti affintiy again which will cause yet another failover @@ -880,229 +940,11 @@ class EndToEndTestCase(unittest.TestCase): "enable_pod_antiaffinity": "false" } } - k8s.update_config(patch_disable_antiaffinity) + k8s.update_config(patch_disable_antiaffinity, "disalbe antiaffinity") k8s.wait_for_pod_start('spilo-role=master') k8s.wait_for_pod_start('spilo-role=replica') - - -class K8sApi: - - def __init__(self): - - # https://github.com/kubernetes-client/python/issues/309 - warnings.simplefilter("ignore", ResourceWarning) - - self.config = config.load_kube_config() - self.k8s_client = client.ApiClient() - - self.core_v1 = client.CoreV1Api() - self.apps_v1 = client.AppsV1Api() - self.batch_v1_beta1 = client.BatchV1beta1Api() - self.custom_objects_api = client.CustomObjectsApi() - self.policy_v1_beta1 = client.PolicyV1beta1Api() - self.storage_v1_api = client.StorageV1Api() - - -class K8s: - ''' - Wraps around K8s api client and helper methods. - ''' - - RETRY_TIMEOUT_SEC = 10 - - def __init__(self): - self.api = K8sApi() - - def get_pg_nodes(self, pg_cluster_name, namespace='default'): - master_pod_node = '' - replica_pod_nodes = [] - podsList = self.api.core_v1.list_namespaced_pod(namespace, label_selector=pg_cluster_name) - for pod in podsList.items: - if pod.metadata.labels.get('spilo-role') == 'master': - master_pod_node = pod.spec.node_name - elif pod.metadata.labels.get('spilo-role') == 'replica': - replica_pod_nodes.append(pod.spec.node_name) - - return master_pod_node, replica_pod_nodes - - def get_cluster_leader_pod(self, pg_cluster_name, namespace='default'): - labels = { - 'application': 'spilo', - 'cluster-name': pg_cluster_name, - 'spilo-role': 'master', - } - - pods = self.api.core_v1.list_namespaced_pod( - namespace, label_selector=to_selector(labels)).items - - if pods: - return pods[0] - - def wait_for_operator_pod_start(self): - self. wait_for_pod_start("name=postgres-operator") - # HACK operator must register CRD and/or Sync existing PG clusters after start up - # for local execution ~ 10 seconds suffices - time.sleep(60) - - def get_operator_pod(self): - pods = self.api.core_v1.list_namespaced_pod( - 'default', label_selector='name=postgres-operator' - ).items - - if pods: - return pods[0] - - return None - - def get_operator_log(self): - operator_pod = self.get_operator_pod() - pod_name = operator_pod.metadata.name - return self.api.core_v1.read_namespaced_pod_log( - name=pod_name, - namespace='default' - ) - - def wait_for_pod_start(self, pod_labels, namespace='default'): - pod_phase = 'No pod running' - while pod_phase != 'Running': - pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=pod_labels).items - if pods: - pod_phase = pods[0].status.phase - - time.sleep(self.RETRY_TIMEOUT_SEC) - - def get_service_type(self, svc_labels, namespace='default'): - svc_type = '' - svcs = self.api.core_v1.list_namespaced_service(namespace, label_selector=svc_labels, limit=1).items - for svc in svcs: - svc_type = svc.spec.type - return svc_type - - def check_service_annotations(self, svc_labels, annotations, namespace='default'): - svcs = self.api.core_v1.list_namespaced_service(namespace, label_selector=svc_labels, limit=1).items - for svc in svcs: - for key, value in annotations.items(): - if key not in svc.metadata.annotations or svc.metadata.annotations[key] != value: - print("Expected key {} not found in annotations {}".format(key, svc.metadata.annotation)) - return False return True - def check_statefulset_annotations(self, sset_labels, annotations, namespace='default'): - ssets = self.api.apps_v1.list_namespaced_stateful_set(namespace, label_selector=sset_labels, limit=1).items - for sset in ssets: - for key, value in annotations.items(): - if key not in sset.metadata.annotations or sset.metadata.annotations[key] != value: - print("Expected key {} not found in annotations {}".format(key, sset.metadata.annotation)) - return False - return True - - def wait_for_pg_to_scale(self, number_of_instances, namespace='default'): - - body = { - "spec": { - "numberOfInstances": number_of_instances - } - } - _ = self.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", namespace, "postgresqls", "acid-minimal-cluster", body) - - labels = 'application=spilo,cluster-name=acid-minimal-cluster' - while self.count_pods_with_label(labels) != number_of_instances: - time.sleep(self.RETRY_TIMEOUT_SEC) - - def wait_for_running_pods(self, labels, number, namespace=''): - while self.count_pods_with_label(labels) != number: - time.sleep(self.RETRY_TIMEOUT_SEC) - - def wait_for_pods_to_stop(self, labels, namespace=''): - while self.count_pods_with_label(labels) != 0: - time.sleep(self.RETRY_TIMEOUT_SEC) - - def wait_for_service(self, labels, namespace='default'): - def get_services(): - return self.api.core_v1.list_namespaced_service( - namespace, label_selector=labels - ).items - - while not get_services(): - time.sleep(self.RETRY_TIMEOUT_SEC) - - def count_pods_with_label(self, labels, namespace='default'): - return len(self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items) - - def count_services_with_label(self, labels, namespace='default'): - return len(self.api.core_v1.list_namespaced_service(namespace, label_selector=labels).items) - - def count_endpoints_with_label(self, labels, namespace='default'): - return len(self.api.core_v1.list_namespaced_endpoints(namespace, label_selector=labels).items) - - def count_secrets_with_label(self, labels, namespace='default'): - return len(self.api.core_v1.list_namespaced_secret(namespace, label_selector=labels).items) - - def count_statefulsets_with_label(self, labels, namespace='default'): - return len(self.api.apps_v1.list_namespaced_stateful_set(namespace, label_selector=labels).items) - - def count_deployments_with_label(self, labels, namespace='default'): - return len(self.api.apps_v1.list_namespaced_deployment(namespace, label_selector=labels).items) - - def count_pdbs_with_label(self, labels, namespace='default'): - return len(self.api.policy_v1_beta1.list_namespaced_pod_disruption_budget( - namespace, label_selector=labels).items) - - def wait_for_pod_failover(self, failover_targets, labels, namespace='default'): - pod_phase = 'Failing over' - new_pod_node = '' - - while (pod_phase != 'Running') or (new_pod_node not in failover_targets): - pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items - if pods: - new_pod_node = pods[0].spec.node_name - pod_phase = pods[0].status.phase - time.sleep(self.RETRY_TIMEOUT_SEC) - - def get_logical_backup_job(self, namespace='default'): - return self.api.batch_v1_beta1.list_namespaced_cron_job(namespace, label_selector="application=spilo") - - def wait_for_logical_backup_job(self, expected_num_of_jobs): - while (len(self.get_logical_backup_job().items) != expected_num_of_jobs): - time.sleep(self.RETRY_TIMEOUT_SEC) - - def wait_for_logical_backup_job_deletion(self): - self.wait_for_logical_backup_job(expected_num_of_jobs=0) - - def wait_for_logical_backup_job_creation(self): - self.wait_for_logical_backup_job(expected_num_of_jobs=1) - - def delete_operator_pod(self): - operator_pod = self.api.core_v1.list_namespaced_pod( - 'default', label_selector="name=postgres-operator").items[0].metadata.name - self.api.core_v1.delete_namespaced_pod(operator_pod, "default") # restart reloads the conf - self.wait_for_operator_pod_start() - - def update_config(self, config_map_patch): - self.api.core_v1.patch_namespaced_config_map("postgres-operator", "default", config_map_patch) - self.delete_operator_pod() - - def create_with_kubectl(self, path): - return subprocess.run( - ["kubectl", "create", "-f", path], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - - def exec_with_kubectl(self, pod, cmd): - return subprocess.run(["./exec.sh", pod, cmd], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - - def get_effective_pod_image(self, pod_name, namespace='default'): - ''' - Get the Spilo image pod currently uses. In case of lazy rolling updates - it may differ from the one specified in the stateful set. - ''' - pod = self.api.core_v1.list_namespaced_pod( - namespace, label_selector="statefulset.kubernetes.io/pod-name=" + pod_name) - return pod.items[0].spec.containers[0].image - if __name__ == '__main__': unittest.main() diff --git a/manifests/postgres-operator.yaml b/manifests/postgres-operator.yaml index e7a604a2d..16719a5d9 100644 --- a/manifests/postgres-operator.yaml +++ b/manifests/postgres-operator.yaml @@ -4,6 +4,8 @@ metadata: name: postgres-operator spec: replicas: 1 + strategy: + type: "Recreate" selector: matchLabels: name: postgres-operator diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 6aa1f6fa4..8636083c2 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -371,11 +371,11 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa //TODO: improve me if *c.Statefulset.Spec.Replicas != *statefulSet.Spec.Replicas { match = false - reasons = append(reasons, "new statefulset's number of replicas doesn't match the current one") + reasons = append(reasons, "new statefulset's number of replicas does not match the current one") } if !reflect.DeepEqual(c.Statefulset.Annotations, statefulSet.Annotations) { match = false - reasons = append(reasons, "new statefulset's annotations doesn't match the current one") + reasons = append(reasons, "new statefulset's annotations does not match the current one") } needsRollUpdate, reasons = c.compareContainers("initContainers", c.Statefulset.Spec.Template.Spec.InitContainers, statefulSet.Spec.Template.Spec.InitContainers, needsRollUpdate, reasons) @@ -392,24 +392,24 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa if c.Statefulset.Spec.Template.Spec.ServiceAccountName != statefulSet.Spec.Template.Spec.ServiceAccountName { needsReplace = true needsRollUpdate = true - reasons = append(reasons, "new statefulset's serviceAccountName service account name doesn't match the current one") + reasons = append(reasons, "new statefulset's serviceAccountName service account name does not match the current one") } if *c.Statefulset.Spec.Template.Spec.TerminationGracePeriodSeconds != *statefulSet.Spec.Template.Spec.TerminationGracePeriodSeconds { needsReplace = true needsRollUpdate = true - reasons = append(reasons, "new statefulset's terminationGracePeriodSeconds doesn't match the current one") + reasons = append(reasons, "new statefulset's terminationGracePeriodSeconds does not match the current one") } if !reflect.DeepEqual(c.Statefulset.Spec.Template.Spec.Affinity, statefulSet.Spec.Template.Spec.Affinity) { needsReplace = true needsRollUpdate = true - reasons = append(reasons, "new statefulset's pod affinity doesn't match the current one") + reasons = append(reasons, "new statefulset's pod affinity does not match the current one") } // Some generated fields like creationTimestamp make it not possible to use DeepCompare on Spec.Template.ObjectMeta if !reflect.DeepEqual(c.Statefulset.Spec.Template.Labels, statefulSet.Spec.Template.Labels) { needsReplace = true needsRollUpdate = true - reasons = append(reasons, "new statefulset's metadata labels doesn't match the current one") + reasons = append(reasons, "new statefulset's metadata labels does not match the current one") } if (c.Statefulset.Spec.Selector != nil) && (statefulSet.Spec.Selector != nil) { if !reflect.DeepEqual(c.Statefulset.Spec.Selector.MatchLabels, statefulSet.Spec.Selector.MatchLabels) { @@ -420,7 +420,7 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa return &compareStatefulsetResult{} } needsReplace = true - reasons = append(reasons, "new statefulset's selector doesn't match the current one") + reasons = append(reasons, "new statefulset's selector does not match the current one") } } @@ -434,7 +434,7 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa match = false needsReplace = true needsRollUpdate = true - reasons = append(reasons, "new statefulset's pod template security context in spec doesn't match the current one") + reasons = append(reasons, "new statefulset's pod template security context in spec does not match the current one") } if len(c.Statefulset.Spec.VolumeClaimTemplates) != len(statefulSet.Spec.VolumeClaimTemplates) { needsReplace = true @@ -445,17 +445,17 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa // Some generated fields like creationTimestamp make it not possible to use DeepCompare on ObjectMeta if name != statefulSet.Spec.VolumeClaimTemplates[i].Name { needsReplace = true - reasons = append(reasons, fmt.Sprintf("new statefulset's name for volume %d doesn't match the current one", i)) + reasons = append(reasons, fmt.Sprintf("new statefulset's name for volume %d does not match the current one", i)) continue } if !reflect.DeepEqual(c.Statefulset.Spec.VolumeClaimTemplates[i].Annotations, statefulSet.Spec.VolumeClaimTemplates[i].Annotations) { needsReplace = true - reasons = append(reasons, fmt.Sprintf("new statefulset's annotations for volume %q doesn't match the current one", name)) + reasons = append(reasons, fmt.Sprintf("new statefulset's annotations for volume %q does not match the current one", name)) } if !reflect.DeepEqual(c.Statefulset.Spec.VolumeClaimTemplates[i].Spec, statefulSet.Spec.VolumeClaimTemplates[i].Spec) { name := c.Statefulset.Spec.VolumeClaimTemplates[i].Name needsReplace = true - reasons = append(reasons, fmt.Sprintf("new statefulset's volumeClaimTemplates specification for volume %q doesn't match the current one", name)) + reasons = append(reasons, fmt.Sprintf("new statefulset's volumeClaimTemplates specification for volume %q does not match the current one", name)) } } @@ -465,14 +465,14 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa match = false needsReplace = true needsRollUpdate = true - reasons = append(reasons, "new statefulset's pod priority class in spec doesn't match the current one") + reasons = append(reasons, "new statefulset's pod priority class in spec does not match the current one") } // lazy Spilo update: modify the image in the statefulset itself but let its pods run with the old image // until they are re-created for other reasons, for example node rotation if c.OpConfig.EnableLazySpiloUpgrade && !reflect.DeepEqual(c.Statefulset.Spec.Template.Spec.Containers[0].Image, statefulSet.Spec.Template.Spec.Containers[0].Image) { needsReplace = true - reasons = append(reasons, "lazy Spilo update: new statefulset's pod image doesn't match the current one") + reasons = append(reasons, "lazy Spilo update: new statefulset's pod image does not match the current one") } if needsRollUpdate || needsReplace { @@ -582,7 +582,7 @@ func (c *Cluster) enforceMinResourceLimits(spec *acidv1.PostgresSpec) error { return fmt.Errorf("could not compare defined CPU limit %s with configured minimum value %s: %v", cpuLimit, minCPULimit, err) } if isSmaller { - c.logger.Warningf("defined CPU limit %s is below required minimum %s and will be set to it", cpuLimit, minCPULimit) + c.logger.Warningf("defined CPU limit %s is below required minimum %s and will be increased", cpuLimit, minCPULimit) c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "ResourceLimits", "defined CPU limit %s is below required minimum %s and will be set to it", cpuLimit, minCPULimit) spec.Resources.ResourceLimits.CPU = minCPULimit } @@ -595,7 +595,7 @@ func (c *Cluster) enforceMinResourceLimits(spec *acidv1.PostgresSpec) error { return fmt.Errorf("could not compare defined memory limit %s with configured minimum value %s: %v", memoryLimit, minMemoryLimit, err) } if isSmaller { - c.logger.Warningf("defined memory limit %s is below required minimum %s and will be set to it", memoryLimit, minMemoryLimit) + c.logger.Warningf("defined memory limit %s is below required minimum %s and will be increased", memoryLimit, minMemoryLimit) c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "ResourceLimits", "defined memory limit %s is below required minimum %s and will be set to it", memoryLimit, minMemoryLimit) spec.Resources.ResourceLimits.Memory = minMemoryLimit } diff --git a/pkg/cluster/database.go b/pkg/cluster/database.go index f51b58a89..57758c6aa 100644 --- a/pkg/cluster/database.go +++ b/pkg/cluster/database.go @@ -527,7 +527,7 @@ func (c *Cluster) installLookupFunction(poolerSchema, poolerUser string) error { continue } - c.logger.Infof("Install pooler lookup function into %s", dbname) + c.logger.Infof("install pooler lookup function into database '%s'", dbname) // golang sql will do retries couple of times if pq driver reports // connections issues (driver.ErrBadConn), but since our query is diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index ba22f24c3..9a328b7df 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1157,7 +1157,9 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef } // generate the spilo container - c.logger.Debugf("Generating Spilo container, environment variables: %v", spiloEnvVars) + c.logger.Debugf("Generating Spilo container, environment variables") + c.logger.Debugf("%v", spiloEnvVars) + spiloContainer := generateContainer(c.containerName(), &effectiveDockerImage, resourceRequirements, @@ -2055,7 +2057,8 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar { envVars = append(envVars, v1.EnvVar{Name: "AWS_SECRET_ACCESS_KEY", Value: c.OpConfig.LogicalBackup.LogicalBackupS3SecretAccessKey}) } - c.logger.Debugf("Generated logical backup env vars %v", envVars) + c.logger.Debugf("Generated logical backup env vars") + c.logger.Debugf("%v", envVars) return envVars } diff --git a/pkg/cluster/pod.go b/pkg/cluster/pod.go index 44b2222e0..a13eb479c 100644 --- a/pkg/cluster/pod.go +++ b/pkg/cluster/pod.go @@ -304,9 +304,16 @@ func (c *Cluster) isSafeToRecreatePods(pods *v1.PodList) bool { after this check succeeds but before a pod is re-created */ + for _, pod := range pods.Items { + c.logger.Debugf("name=%s phase=%s ip=%s", pod.Name, pod.Status.Phase, pod.Status.PodIP) + } + for _, pod := range pods.Items { state, err := c.patroni.GetPatroniMemberState(&pod) - if err != nil || state == "creating replica" { + if err != nil { + c.logger.Errorf("failed to get Patroni state for pod: %s", err) + return false + } else if state == "creating replica" { c.logger.Warningf("cannot re-create replica %s: it is currently being initialized", pod.Name) return false } diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index a9d13c124..4fb2c13c6 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -293,7 +293,7 @@ func (c *Cluster) preScaleDown(newStatefulSet *appsv1.StatefulSet) error { // setRollingUpdateFlagForStatefulSet sets the indicator or the rolling update requirement // in the StatefulSet annotation. -func (c *Cluster) setRollingUpdateFlagForStatefulSet(sset *appsv1.StatefulSet, val bool) { +func (c *Cluster) setRollingUpdateFlagForStatefulSet(sset *appsv1.StatefulSet, val bool, msg string) { anno := sset.GetAnnotations() if anno == nil { anno = make(map[string]string) @@ -301,13 +301,13 @@ func (c *Cluster) setRollingUpdateFlagForStatefulSet(sset *appsv1.StatefulSet, v anno[rollingUpdateStatefulsetAnnotationKey] = strconv.FormatBool(val) sset.SetAnnotations(anno) - c.logger.Debugf("statefulset's rolling update annotation has been set to %t", val) + c.logger.Debugf("set statefulset's rolling update annotation to %t: caller/reason %s", val, msg) } // applyRollingUpdateFlagforStatefulSet sets the rolling update flag for the cluster's StatefulSet // and applies that setting to the actual running cluster. func (c *Cluster) applyRollingUpdateFlagforStatefulSet(val bool) error { - c.setRollingUpdateFlagForStatefulSet(c.Statefulset, val) + c.setRollingUpdateFlagForStatefulSet(c.Statefulset, val, "applyRollingUpdateFlag") sset, err := c.updateStatefulSetAnnotations(c.Statefulset.GetAnnotations()) if err != nil { return err @@ -359,14 +359,13 @@ func (c *Cluster) mergeRollingUpdateFlagUsingCache(runningStatefulSet *appsv1.St podsRollingUpdateRequired = false } else { c.logger.Infof("found a statefulset with an unfinished rolling update of the pods") - } } return podsRollingUpdateRequired } func (c *Cluster) updateStatefulSetAnnotations(annotations map[string]string) (*appsv1.StatefulSet, error) { - c.logger.Debugf("updating statefulset annotations") + c.logger.Debugf("patching statefulset annotations") patchData, err := metaAnnotationsPatch(annotations) if err != nil { return nil, fmt.Errorf("could not form patch for the statefulset metadata: %v", err) diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 2a3959b1a..dced69461 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -348,13 +348,13 @@ func (c *Cluster) syncStatefulSet() error { if err != nil { return fmt.Errorf("could not generate statefulset: %v", err) } - c.setRollingUpdateFlagForStatefulSet(desiredSS, podsRollingUpdateRequired) + c.setRollingUpdateFlagForStatefulSet(desiredSS, podsRollingUpdateRequired, "from cache") cmp := c.compareStatefulSetWith(desiredSS) if !cmp.match { if cmp.rollingUpdate && !podsRollingUpdateRequired { podsRollingUpdateRequired = true - c.setRollingUpdateFlagForStatefulSet(desiredSS, podsRollingUpdateRequired) + c.setRollingUpdateFlagForStatefulSet(desiredSS, podsRollingUpdateRequired, "statefulset changes") } c.logStatefulSetChanges(c.Statefulset, desiredSS, false, cmp.reasons) @@ -497,11 +497,11 @@ func (c *Cluster) syncSecrets() error { return fmt.Errorf("could not get current secret: %v", err) } if secretUsername != string(secret.Data["username"]) { - c.logger.Warningf("secret %q does not contain the role %q", secretSpec.Name, secretUsername) + c.logger.Warningf("secret %s does not contain the role %q", secretSpec.Name, secretUsername) continue } c.Secrets[secret.UID] = secret - c.logger.Debugf("secret %q already exists, fetching its password", util.NameFromMeta(secret.ObjectMeta)) + c.logger.Debugf("secret %s already exists, fetching its password", util.NameFromMeta(secret.ObjectMeta)) if secretUsername == c.systemUsers[constants.SuperuserKeyName].Name { secretUsername = constants.SuperuserKeyName userMap = c.systemUsers @@ -804,7 +804,7 @@ func (c *Cluster) syncLogicalBackupJob() error { return fmt.Errorf("could not generate the desired logical backup job state: %v", err) } if match, reason := k8sutil.SameLogicalBackupJob(job, desiredJob); !match { - c.logger.Infof("logical job %q is not in the desired state and needs to be updated", + c.logger.Infof("logical job %s is not in the desired state and needs to be updated", c.getLogicalBackupJobName(), ) if reason != "" { @@ -825,12 +825,12 @@ func (c *Cluster) syncLogicalBackupJob() error { c.logger.Info("could not find the cluster's logical backup job") if err = c.createLogicalBackupJob(); err == nil { - c.logger.Infof("created missing logical backup job %q", jobName) + c.logger.Infof("created missing logical backup job %s", jobName) } else { if !k8sutil.ResourceAlreadyExists(err) { return fmt.Errorf("could not create missing logical backup job: %v", err) } - c.logger.Infof("logical backup job %q already exists", jobName) + c.logger.Infof("logical backup job %s already exists", jobName) if _, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Get(context.TODO(), jobName, metav1.GetOptions{}); err != nil { return fmt.Errorf("could not fetch existing logical backup job: %v", err) } @@ -975,7 +975,7 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql newConnectionPooler = &acidv1.ConnectionPooler{} } - c.logger.Infof("Old: %+v, New %+v", oldConnectionPooler, newConnectionPooler) + logNiceDiff(c.logger, oldConnectionPooler, newConnectionPooler) specSync, specReason := c.needSyncConnectionPoolerSpecs(oldConnectionPooler, newConnectionPooler) defaultsSync, defaultsReason := c.needSyncConnectionPoolerDefaults(newConnectionPooler, deployment) diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go index 7559ce3d4..d227ce155 100644 --- a/pkg/cluster/util.go +++ b/pkg/cluster/util.go @@ -18,12 +18,14 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "github.com/sirupsen/logrus" acidzalando "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" + "github.com/zalando/postgres-operator/pkg/util/nicediff" "github.com/zalando/postgres-operator/pkg/util/retryutil" ) @@ -166,40 +168,59 @@ func (c *Cluster) logPDBChanges(old, new *policybeta1.PodDisruptionBudget, isUpd ) } - c.logger.Debugf("diff\n%s\n", util.PrettyDiff(old.Spec, new.Spec)) + logNiceDiff(c.logger, old.Spec, new.Spec) +} + +func logNiceDiff(log *logrus.Entry, old, new interface{}) { + o, erro := json.MarshalIndent(old, "", " ") + n, errn := json.MarshalIndent(new, "", " ") + + if erro != nil || errn != nil { + panic("could not marshal API objects, should not happen") + } + + nice := nicediff.Diff(string(o), string(n), true) + for _, s := range strings.Split(nice, "\n") { + // " is not needed in the value to understand + log.Debugf(strings.ReplaceAll(s, "\"", "")) + } } func (c *Cluster) logStatefulSetChanges(old, new *appsv1.StatefulSet, isUpdate bool, reasons []string) { if isUpdate { - c.logger.Infof("statefulset %q has been changed", util.NameFromMeta(old.ObjectMeta)) + c.logger.Infof("statefulset %s has been changed", util.NameFromMeta(old.ObjectMeta)) } else { - c.logger.Infof("statefulset %q is not in the desired state and needs to be updated", + c.logger.Infof("statefulset %s is not in the desired state and needs to be updated", util.NameFromMeta(old.ObjectMeta), ) } + + logNiceDiff(c.logger, old.Spec, new.Spec) + if !reflect.DeepEqual(old.Annotations, new.Annotations) { - c.logger.Debugf("metadata.annotation diff\n%s\n", util.PrettyDiff(old.Annotations, new.Annotations)) + c.logger.Debugf("metadata.annotation are different") + logNiceDiff(c.logger, old.Annotations, new.Annotations) } - c.logger.Debugf("spec diff between old and new statefulsets: \n%s\n", util.PrettyDiff(old.Spec, new.Spec)) if len(reasons) > 0 { for _, reason := range reasons { - c.logger.Infof("reason: %q", reason) + c.logger.Infof("reason: %s", reason) } } } func (c *Cluster) logServiceChanges(role PostgresRole, old, new *v1.Service, isUpdate bool, reason string) { if isUpdate { - c.logger.Infof("%s service %q has been changed", + c.logger.Infof("%s service %s has been changed", role, util.NameFromMeta(old.ObjectMeta), ) } else { - c.logger.Infof("%s service %q is not in the desired state and needs to be updated", + c.logger.Infof("%s service %s is not in the desired state and needs to be updated", role, util.NameFromMeta(old.ObjectMeta), ) } - c.logger.Debugf("diff\n%s\n", util.PrettyDiff(old.Spec, new.Spec)) + + logNiceDiff(c.logger, old.Spec, new.Spec) if reason != "" { c.logger.Infof("reason: %s", reason) @@ -208,7 +229,7 @@ func (c *Cluster) logServiceChanges(role PostgresRole, old, new *v1.Service, isU func (c *Cluster) logVolumeChanges(old, new acidv1.Volume) { c.logger.Infof("volume specification has been changed") - c.logger.Debugf("diff\n%s\n", util.PrettyDiff(old, new)) + logNiceDiff(c.logger, old, new) } func (c *Cluster) getTeamMembers(teamID string) ([]string, error) { diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index cc08f1587..3442bfcfe 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -1,9 +1,12 @@ package controller import ( + "bytes" "context" + "encoding/json" "fmt" "os" + "strings" "sync" "time" @@ -73,6 +76,10 @@ func NewController(controllerConfig *spec.ControllerConfig, controllerId string) logger := logrus.New() if controllerConfig.EnableJsonLogging { logger.SetFormatter(&logrus.JSONFormatter{}) + } else { + if os.Getenv("LOG_NOQUOTE") != "" { + logger.SetFormatter(&logrus.TextFormatter{PadLevelText: true, DisableQuote: true}) + } } var myComponentName = "postgres-operator" @@ -81,7 +88,10 @@ func NewController(controllerConfig *spec.ControllerConfig, controllerId string) } eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(logger.Infof) + + // disabling the sending of events also to the logoutput + // the operator currently duplicates a lot of log entries with this setup + // eventBroadcaster.StartLogging(logger.Infof) recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: myComponentName}) c := &Controller{ @@ -190,10 +200,18 @@ func (c *Controller) warnOnDeprecatedOperatorParameters() { } } +func compactValue(v string) string { + var compact bytes.Buffer + if err := json.Compact(&compact, []byte(v)); err != nil { + panic("Hard coded json strings broken!") + } + return compact.String() +} + func (c *Controller) initPodServiceAccount() { if c.opConfig.PodServiceAccountDefinition == "" { - c.opConfig.PodServiceAccountDefinition = ` + stringValue := ` { "apiVersion": "v1", "kind": "ServiceAccount", @@ -201,6 +219,9 @@ func (c *Controller) initPodServiceAccount() { "name": "postgres-pod" } }` + + c.opConfig.PodServiceAccountDefinition = compactValue(stringValue) + } // re-uses k8s internal parsing. See k8s client-go issue #193 for explanation @@ -230,7 +251,7 @@ func (c *Controller) initRoleBinding() { // operator binds it to the cluster role with sufficient privileges // we assume the role is created by the k8s administrator if c.opConfig.PodServiceAccountRoleBindingDefinition == "" { - c.opConfig.PodServiceAccountRoleBindingDefinition = fmt.Sprintf(` + stringValue := fmt.Sprintf(` { "apiVersion": "rbac.authorization.k8s.io/v1", "kind": "RoleBinding", @@ -249,6 +270,7 @@ func (c *Controller) initRoleBinding() { } ] }`, c.PodServiceAccount.Name, c.PodServiceAccount.Name, c.PodServiceAccount.Name) + c.opConfig.PodServiceAccountRoleBindingDefinition = compactValue(stringValue) } c.logger.Info("Parse role bindings") // re-uses k8s internal parsing. See k8s client-go issue #193 for explanation @@ -267,7 +289,14 @@ func (c *Controller) initRoleBinding() { } - // actual roles bindings are deployed at the time of Postgres/Spilo cluster creation + // actual roles bindings ar*logrus.Entrye deployed at the time of Postgres/Spilo cluster creation +} + +func logMultiLineConfig(log *logrus.Entry, config string) { + lines := strings.Split(config, "\n") + for _, l := range lines { + log.Infof("%s", l) + } } func (c *Controller) initController() { @@ -301,7 +330,7 @@ func (c *Controller) initController() { c.logger.Logger.Level = logrus.DebugLevel } - c.logger.Infof("config: %s", c.opConfig.MustMarshal()) + logMultiLineConfig(c.logger, c.opConfig.MustMarshal()) roleDefs := c.getInfrastructureRoleDefinitions() if infraRoles, err := c.getInfrastructureRoles(roleDefs); err != nil { diff --git a/pkg/controller/node.go b/pkg/controller/node.go index 4ffe7e26c..2836b4f7f 100644 --- a/pkg/controller/node.go +++ b/pkg/controller/node.go @@ -42,7 +42,7 @@ func (c *Controller) nodeAdd(obj interface{}) { return } - c.logger.Debugf("new node has been added: %q (%s)", util.NameFromMeta(node.ObjectMeta), node.Spec.ProviderID) + c.logger.Debugf("new node has been added: %s (%s)", util.NameFromMeta(node.ObjectMeta), node.Spec.ProviderID) // check if the node became not ready while the operator was down (otherwise we would have caught it in nodeUpdate) if !c.nodeIsReady(node) { diff --git a/pkg/controller/postgresql.go b/pkg/controller/postgresql.go index c7074c7e4..23e9356e6 100644 --- a/pkg/controller/postgresql.go +++ b/pkg/controller/postgresql.go @@ -225,7 +225,7 @@ func (c *Controller) processEvent(event ClusterEvent) { switch event.EventType { case EventAdd: if clusterFound { - lg.Debugf("cluster already exists") + lg.Debugf("Recieved add event for existing cluster") return } diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 7a1ae8a41..35991248b 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -199,7 +199,7 @@ type Config struct { // MustMarshal marshals the config or panics func (c Config) MustMarshal() string { - b, err := json.MarshalIndent(c, "", "\t") + b, err := json.MarshalIndent(c, "", " ") if err != nil { panic(err) } diff --git a/pkg/util/nicediff/diff.go b/pkg/util/nicediff/diff.go new file mode 100644 index 000000000..e2793f2c7 --- /dev/null +++ b/pkg/util/nicediff/diff.go @@ -0,0 +1,191 @@ +// Copyright 2013 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package diff implements a linewise diff algorithm. +package nicediff + +import ( + "fmt" + "strings" +) + +// Chunk represents a piece of the diff. A chunk will not have both added and +// deleted lines. Equal lines are always after any added or deleted lines. +// A Chunk may or may not have any lines in it, especially for the first or last +// chunk in a computation. +type Chunk struct { + Added []string + Deleted []string + Equal []string +} + +func (c *Chunk) empty() bool { + return len(c.Added) == 0 && len(c.Deleted) == 0 && len(c.Equal) == 0 +} + +// Diff returns a string containing a line-by-line unified diff of the linewise +// changes required to make A into B. Each line is prefixed with '+', '-', or +// ' ' to indicate if it should be added, removed, or is correct respectively. +func Diff(A, B string, skipEqual bool) string { + aLines := strings.Split(A, "\n") + bLines := strings.Split(B, "\n") + return Render(DiffChunks(aLines, bLines), skipEqual) +} + +// Render renders the slice of chunks into a representation that prefixes +// the lines with '+', '-', or ' ' depending on whether the line was added, +// removed, or equal (respectively). +func Render(chunks []Chunk, skipEqual bool) string { + buf := new(strings.Builder) + for _, c := range chunks { + for _, line := range c.Added { + fmt.Fprintf(buf, "+%s\n", line) + } + for _, line := range c.Deleted { + fmt.Fprintf(buf, "-%s\n", line) + } + if !skipEqual { + for _, line := range c.Equal { + fmt.Fprintf(buf, " %s\n", line) + } + } + } + return strings.TrimRight(buf.String(), "\n") +} + +// DiffChunks uses an O(D(N+M)) shortest-edit-script algorithm +// to compute the edits required from A to B and returns the +// edit chunks. +func DiffChunks(a, b []string) []Chunk { + // algorithm: http://www.xmailserver.org/diff2.pdf + + // We'll need these quantities a lot. + alen, blen := len(a), len(b) // M, N + + // At most, it will require len(a) deletions and len(b) additions + // to transform a into b. + maxPath := alen + blen // MAX + if maxPath == 0 { + // degenerate case: two empty lists are the same + return nil + } + + // Store the endpoint of the path for diagonals. + // We store only the a index, because the b index on any diagonal + // (which we know during the loop below) is aidx-diag. + // endpoint[maxPath] represents the 0 diagonal. + // + // Stated differently: + // endpoint[d] contains the aidx of a furthest reaching path in diagonal d + endpoint := make([]int, 2*maxPath+1) // V + + saved := make([][]int, 0, 8) // Vs + save := func() { + dup := make([]int, len(endpoint)) + copy(dup, endpoint) + saved = append(saved, dup) + } + + var editDistance int // D +dLoop: + for editDistance = 0; editDistance <= maxPath; editDistance++ { + // The 0 diag(onal) represents equality of a and b. Each diagonal to + // the left is numbered one lower, to the right is one higher, from + // -alen to +blen. Negative diagonals favor differences from a, + // positive diagonals favor differences from b. The edit distance to a + // diagonal d cannot be shorter than d itself. + // + // The iterations of this loop cover either odds or evens, but not both, + // If odd indices are inputs, even indices are outputs and vice versa. + for diag := -editDistance; diag <= editDistance; diag += 2 { // k + var aidx int // x + switch { + case diag == -editDistance: + // This is a new diagonal; copy from previous iter + aidx = endpoint[maxPath-editDistance+1] + 0 + case diag == editDistance: + // This is a new diagonal; copy from previous iter + aidx = endpoint[maxPath+editDistance-1] + 1 + case endpoint[maxPath+diag+1] > endpoint[maxPath+diag-1]: + // diagonal d+1 was farther along, so use that + aidx = endpoint[maxPath+diag+1] + 0 + default: + // diagonal d-1 was farther (or the same), so use that + aidx = endpoint[maxPath+diag-1] + 1 + } + // On diagonal d, we can compute bidx from aidx. + bidx := aidx - diag // y + // See how far we can go on this diagonal before we find a difference. + for aidx < alen && bidx < blen && a[aidx] == b[bidx] { + aidx++ + bidx++ + } + // Store the end of the current edit chain. + endpoint[maxPath+diag] = aidx + // If we've found the end of both inputs, we're done! + if aidx >= alen && bidx >= blen { + save() // save the final path + break dLoop + } + } + save() // save the current path + } + if editDistance == 0 { + return nil + } + chunks := make([]Chunk, editDistance+1) + + x, y := alen, blen + for d := editDistance; d > 0; d-- { + endpoint := saved[d] + diag := x - y + insert := diag == -d || (diag != d && endpoint[maxPath+diag-1] < endpoint[maxPath+diag+1]) + + x1 := endpoint[maxPath+diag] + var x0, xM, kk int + if insert { + kk = diag + 1 + x0 = endpoint[maxPath+kk] + xM = x0 + } else { + kk = diag - 1 + x0 = endpoint[maxPath+kk] + xM = x0 + 1 + } + y0 := x0 - kk + + var c Chunk + if insert { + c.Added = b[y0:][:1] + } else { + c.Deleted = a[x0:][:1] + } + if xM < x1 { + c.Equal = a[xM:][:x1-xM] + } + + x, y = x0, y0 + chunks[d] = c + } + if x > 0 { + chunks[0].Equal = a[:x] + } + if chunks[0].empty() { + chunks = chunks[1:] + } + if len(chunks) == 0 { + return nil + } + return chunks +} diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index a9d25112b..ee5b1ac39 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -180,3 +180,13 @@ func TestIsSmallerQuantity(t *testing.T) { } } } + +/* +func TestNiceDiff(t *testing.T) { + o := "a\nb\nc\n" + n := "b\nd\n" + d := nicediff.Diff(o, n, true) + t.Log(d) + // t.Errorf("Lets see output") +} +*/ From d658b9672ea3c13c6e3c3a0fcc94820c46451a84 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 28 Oct 2020 10:40:10 +0100 Subject: [PATCH 103/168] PostgresTeam CRD for advanced team management (#1165) * PostgresTeamCRD for advanced team management * rework internal structure to be closer to CRD * superusers instead of admin * add more util functions and unit tests * fix initHumanUsers * check for superusers when creating normal teams * polishing and fixes * adding the essential missing pieces * add documentation and update rbac * reflect some feedback * reflect more feedback * fixing debug logs and raise QueueResyncPeriodTPR * add two more flags to disable CRD and its superuser support * fix chart * update go modules * move to client 1.19.3 and update codegen --- .../crds/operatorconfigurations.yaml | 4 + .../postgres-operator/crds/postgresteams.yaml | 67 ++++++ .../templates/clusterrole.yaml | 9 + charts/postgres-operator/values-crd.yaml | 5 + charts/postgres-operator/values.yaml | 7 +- docs/administrator.md | 9 +- docs/reference/operator_parameters.md | 14 +- docs/user.md | 61 ++++++ manifests/configmap.yaml | 2 + manifests/custom-team-membership.yaml | 13 ++ manifests/operator-service-account-rbac.yaml | 9 + manifests/operatorconfiguration.crd.yaml | 4 + ...gresql-operator-default-configuration.yaml | 2 + manifests/postgresteam.crd.yaml | 63 ++++++ pkg/apis/acid.zalan.do/v1/crds.go | 6 + .../v1/operator_configuration_type.go | 22 +- .../acid.zalan.do/v1/postgres_team_type.go | 33 +++ pkg/apis/acid.zalan.do/v1/register.go | 5 +- .../acid.zalan.do/v1/zz_generated.deepcopy.go | 126 ++++++++++++ pkg/cluster/cluster.go | 54 +++-- pkg/cluster/util.go | 27 ++- pkg/controller/controller.go | 46 ++++- pkg/controller/operator_config.go | 2 + pkg/controller/util.go | 33 +++ .../acid.zalan.do/v1/acid.zalan.do_client.go | 5 + .../v1/fake/fake_acid.zalan.do_client.go | 4 + .../v1/fake/fake_postgresteam.go | 136 ++++++++++++ .../acid.zalan.do/v1/generated_expansion.go | 2 + .../typed/acid.zalan.do/v1/postgresteam.go | 184 +++++++++++++++++ .../acid.zalan.do/v1/interface.go | 7 + .../acid.zalan.do/v1/postgresteam.go | 96 +++++++++ .../informers/externalversions/generic.go | 2 + .../acid.zalan.do/v1/expansion_generated.go | 8 + .../listers/acid.zalan.do/v1/postgresteam.go | 105 ++++++++++ pkg/teams/postgres_team.go | 118 +++++++++++ pkg/teams/postgres_team_test.go | 194 ++++++++++++++++++ pkg/util/config/config.go | 2 + pkg/util/util.go | 31 +++ pkg/util/util_test.go | 39 ++++ 39 files changed, 1509 insertions(+), 47 deletions(-) create mode 100644 charts/postgres-operator/crds/postgresteams.yaml create mode 100644 manifests/custom-team-membership.yaml create mode 100644 manifests/postgresteam.crd.yaml create mode 100644 pkg/apis/acid.zalan.do/v1/postgres_team_type.go create mode 100644 pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresteam.go create mode 100644 pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresteam.go create mode 100644 pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresteam.go create mode 100644 pkg/generated/listers/acid.zalan.do/v1/postgresteam.go create mode 100644 pkg/teams/postgres_team.go create mode 100644 pkg/teams/postgres_team_test.go diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 8b576822c..05b5090c2 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -319,6 +319,10 @@ spec: properties: enable_admin_role_for_users: type: boolean + enable_postgres_team_crd: + type: boolean + enable_postgres_team_crd_superusers: + type: boolean enable_team_superuser: type: boolean enable_teams_api: diff --git a/charts/postgres-operator/crds/postgresteams.yaml b/charts/postgres-operator/crds/postgresteams.yaml new file mode 100644 index 000000000..4f2e74034 --- /dev/null +++ b/charts/postgres-operator/crds/postgresteams.yaml @@ -0,0 +1,67 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: postgresteams.acid.zalan.do + labels: + app.kubernetes.io/name: postgres-operator + annotations: + "helm.sh/hook": crd-install +spec: + group: acid.zalan.do + names: + kind: PostgresTeam + listKind: PostgresTeamList + plural: postgresteams + singular: postgresteam + shortNames: + - pgteam + scope: Namespaced + subresources: + status: {} + version: v1 + validation: + openAPIV3Schema: + type: object + required: + - kind + - apiVersion + - spec + properties: + kind: + type: string + enum: + - PostgresTeam + apiVersion: + type: string + enum: + - acid.zalan.do/v1 + spec: + type: object + properties: + additionalSuperuserTeams: + type: object + description: "Map for teamId and associated additional superuser teams" + additionalProperties: + type: array + nullable: true + description: "List of teams to become Postgres superusers" + items: + type: string + additionalTeams: + type: object + description: "Map for teamId and associated additional teams" + additionalProperties: + type: array + nullable: true + description: "List of teams whose members will also be added to the Postgres cluster" + items: + type: string + additionalMembers: + type: object + description: "Map for teamId and associated additional users" + additionalProperties: + type: array + nullable: true + description: "List of users who will also be added to the Postgres cluster" + items: + type: string diff --git a/charts/postgres-operator/templates/clusterrole.yaml b/charts/postgres-operator/templates/clusterrole.yaml index bd34e803e..84da313d9 100644 --- a/charts/postgres-operator/templates/clusterrole.yaml +++ b/charts/postgres-operator/templates/clusterrole.yaml @@ -25,6 +25,15 @@ rules: - patch - update - watch +# operator only reads PostgresTeams +- apiGroups: + - acid.zalan.do + resources: + - postgresteams + verbs: + - get + - list + - watch # to create or get/update CRDs when starting up - apiGroups: - apiextensions.k8s.io diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index ffa8b7f51..52892c22c 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -256,6 +256,11 @@ configTeamsApi: # team_admin_role will have the rights to grant roles coming from PG manifests # enable_admin_role_for_users: true + # operator watches for PostgresTeam CRs to assign additional teams and members to clusters + enable_postgres_team_crd: true + # toogle to create additional superuser teams from PostgresTeam CRs + # enable_postgres_team_crd_superusers: "false" + # toggle to grant superuser to team members created from the Teams API enable_team_superuser: false # toggles usage of the Teams API by the operator diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index d4acfe1aa..ba5c7458c 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -1,7 +1,7 @@ image: registry: registry.opensource.zalan.do repository: acid/postgres-operator - tag: v1.5.0 + tag: v1.5.0-61-ged2b3239-dirty pullPolicy: "IfNotPresent" # Optionally specify an array of imagePullSecrets. @@ -248,6 +248,11 @@ configTeamsApi: # team_admin_role will have the rights to grant roles coming from PG manifests # enable_admin_role_for_users: "true" + # operator watches for PostgresTeam CRs to assign additional teams and members to clusters + enable_postgres_team_crd: "true" + # toogle to create additional superuser teams from PostgresTeam CRs + # enable_postgres_team_crd_superusers: "false" + # toggle to grant superuser to team members created from the Teams API # enable_team_superuser: "false" diff --git a/docs/administrator.md b/docs/administrator.md index 1a1b5e8f9..5357ddb74 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -561,9 +561,12 @@ database. * **Human users** originate from the [Teams API](user.md#teams-api-roles) that returns a list of the team members given a team id. The operator differentiates between (a) product teams that own a particular Postgres cluster and are granted -admin rights to maintain it, and (b) Postgres superuser teams that get the -superuser access to all Postgres databases running in a K8s cluster for the -purposes of maintaining and troubleshooting. +admin rights to maintain it, (b) Postgres superuser teams that get superuser +access to all Postgres databases running in a K8s cluster for the purposes of +maintaining and troubleshooting, and (c) additional teams, superuser teams or +members associated with the owning team. The latter is managed via the +[PostgresTeam CRD](user.md#additional-teams-and-members-per-cluster). + ## Understanding rolling update of Spilo pods diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 465465432..bd12eb922 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -598,8 +598,8 @@ key. The default is `"log_statement:all"` * **enable_team_superuser** - whether to grant superuser to team members created from the Teams API. - The default is `false`. + whether to grant superuser to members of the cluster's owning team created + from the Teams API. The default is `false`. * **team_admin_role** role name to grant to team members created from the Teams API. The default is @@ -632,6 +632,16 @@ key. cluster to administer Postgres and maintain infrastructure built around it. The default is empty. +* **enable_postgres_team_crd** + toggle to make the operator watch for created or updated `PostgresTeam` CRDs + and create roles for specified additional teams and members. + The default is `true`. + +* **enable_postgres_team_crd_superusers** + in a `PostgresTeam` CRD additional superuser teams can assigned to teams that + own clusters. With this flag set to `false`, it will be ignored. + The default is `false`. + ## Logging and REST API Parameters affecting logging and REST API listener. In the CRD-based diff --git a/docs/user.md b/docs/user.md index 9a9e01b9a..db107dccb 100644 --- a/docs/user.md +++ b/docs/user.md @@ -269,6 +269,67 @@ to choose superusers, group roles, [PAM configuration](https://github.com/CyberD etc. An OAuth2 token can be passed to the Teams API via a secret. The name for this secret is configurable with the `oauth_token_secret_name` parameter. +### Additional teams and members per cluster + +Postgres clusters are associated with one team by providing the `teamID` in +the manifest. Additional superuser teams can be configured as mentioned in +the previous paragraph. However, this is a global setting. To assign +additional teams, superuser teams and single users to clusters of a given +team, use the [PostgresTeam CRD](../manifests/postgresteam.yaml). It provides +a simple mapping structure. + + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: PostgresTeam +metadata: + name: custom-team-membership +spec: + additionalSuperuserTeams: + acid: + - "postgres_superusers" + additionalTeams: + acid: [] + additionalMembers: + acid: + - "elephant" +``` + +One `PostgresTeam` resource could contain mappings of multiple teams but you +can choose to create separate CRDs, alternatively. On each CRD creation or +update the operator will gather all mappings to create additional human users +in databases the next time they are synced. Additional teams are resolved +transitively, meaning you will also add users for their `additionalTeams` +or (not and) `additionalSuperuserTeams`. + +For each additional team the Teams API would be queried. Additional members +will be added either way. There can be "virtual teams" that do not exists in +your Teams API but users of associated teams as well as members will get +created. With `PostgresTeams` it's also easy to cover team name changes. Just +add the mapping between old and new team name and the rest can stay the same. + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: PostgresTeam +metadata: + name: virtualteam-membership +spec: + additionalSuperuserTeams: + acid: + - "virtual_superusers" + virtual_superusers: + - "real_teamA" + - "real_teamB" + real_teamA: + - "real_teamA_renamed" + additionalTeams: + real_teamA: + - "real_teamA_renamed" + additionalMembers: + virtual_superusers: + - "foo" +``` + ## Prepared databases with roles and default privileges The `users` section in the manifests only allows for creating database roles diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 970f845bf..ce20dfa58 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -41,6 +41,8 @@ data: enable_master_load_balancer: "false" # enable_pod_antiaffinity: "false" # enable_pod_disruption_budget: "true" + # enable_postgres_team_crd: "true" + # enable_postgres_team_crd_superusers: "false" enable_replica_load_balancer: "false" # enable_shm_volume: "true" # enable_sidecars: "true" diff --git a/manifests/custom-team-membership.yaml b/manifests/custom-team-membership.yaml new file mode 100644 index 000000000..9af153962 --- /dev/null +++ b/manifests/custom-team-membership.yaml @@ -0,0 +1,13 @@ +apiVersion: "acid.zalan.do/v1" +kind: PostgresTeam +metadata: + name: custom-team-membership +spec: + additionalSuperuserTeams: + acid: + - "postgres_superusers" + additionalTeams: + acid: [] + additionalMembers: + acid: + - "elephant" diff --git a/manifests/operator-service-account-rbac.yaml b/manifests/operator-service-account-rbac.yaml index 266df30c5..15ed7f53b 100644 --- a/manifests/operator-service-account-rbac.yaml +++ b/manifests/operator-service-account-rbac.yaml @@ -26,6 +26,15 @@ rules: - patch - update - watch +# operator only reads PostgresTeams +- apiGroups: + - acid.zalan.do + resources: + - postgresteams + verbs: + - get + - list + - watch # to create or get/update CRDs when starting up - apiGroups: - apiextensions.k8s.io diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 515f87438..d0f020f52 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -325,6 +325,10 @@ spec: properties: enable_admin_role_for_users: type: boolean + enable_postgres_team_crd: + type: boolean + enable_postgres_team_crd_superusers: + type: boolean enable_team_superuser: type: boolean enable_teams_api: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 5fb77bf76..71408ac43 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -122,6 +122,8 @@ configuration: enable_database_access: true teams_api: # enable_admin_role_for_users: true + # enable_postgres_team_crd: true + # enable_postgres_team_crd_superusers: false enable_team_superuser: false enable_teams_api: false # pam_configuration: "" diff --git a/manifests/postgresteam.crd.yaml b/manifests/postgresteam.crd.yaml new file mode 100644 index 000000000..5f55bdfcb --- /dev/null +++ b/manifests/postgresteam.crd.yaml @@ -0,0 +1,63 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: postgresteams.acid.zalan.do +spec: + group: acid.zalan.do + names: + kind: PostgresTeam + listKind: PostgresTeamList + plural: postgresteams + singular: postgresteam + shortNames: + - pgteam + scope: Namespaced + subresources: + status: {} + version: v1 + validation: + openAPIV3Schema: + type: object + required: + - kind + - apiVersion + - spec + properties: + kind: + type: string + enum: + - PostgresTeam + apiVersion: + type: string + enum: + - acid.zalan.do/v1 + spec: + type: object + properties: + additionalSuperuserTeams: + type: object + description: "Map for teamId and associated additional superuser teams" + additionalProperties: + type: array + nullable: true + description: "List of teams to become Postgres superusers" + items: + type: string + additionalTeams: + type: object + description: "Map for teamId and associated additional teams" + additionalProperties: + type: array + nullable: true + description: "List of teams whose members will also be added to the Postgres cluster" + items: + type: string + additionalMembers: + type: object + description: "Map for teamId and associated additional users" + additionalProperties: + type: array + nullable: true + description: "List of users who will also be added to the Postgres cluster" + items: + type: string diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index a7d9bccf0..0dca0c94b 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -1235,6 +1235,12 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation "enable_admin_role_for_users": { Type: "boolean", }, + "enable_postgres_team_crd": { + Type: "boolean", + }, + "enable_postgres_team_crd_superusers": { + Type: "boolean", + }, "enable_team_superuser": { Type: "boolean", }, diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 179b7e751..9dae0089b 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -135,16 +135,18 @@ type OperatorDebugConfiguration struct { // TeamsAPIConfiguration defines the configuration of TeamsAPI type TeamsAPIConfiguration struct { - EnableTeamsAPI bool `json:"enable_teams_api,omitempty"` - TeamsAPIUrl string `json:"teams_api_url,omitempty"` - TeamAPIRoleConfiguration map[string]string `json:"team_api_role_configuration,omitempty"` - EnableTeamSuperuser bool `json:"enable_team_superuser,omitempty"` - EnableAdminRoleForUsers bool `json:"enable_admin_role_for_users,omitempty"` - TeamAdminRole string `json:"team_admin_role,omitempty"` - PamRoleName string `json:"pam_role_name,omitempty"` - PamConfiguration string `json:"pam_configuration,omitempty"` - ProtectedRoles []string `json:"protected_role_names,omitempty"` - PostgresSuperuserTeams []string `json:"postgres_superuser_teams,omitempty"` + EnableTeamsAPI bool `json:"enable_teams_api,omitempty"` + TeamsAPIUrl string `json:"teams_api_url,omitempty"` + TeamAPIRoleConfiguration map[string]string `json:"team_api_role_configuration,omitempty"` + EnableTeamSuperuser bool `json:"enable_team_superuser,omitempty"` + EnableAdminRoleForUsers bool `json:"enable_admin_role_for_users,omitempty"` + TeamAdminRole string `json:"team_admin_role,omitempty"` + PamRoleName string `json:"pam_role_name,omitempty"` + PamConfiguration string `json:"pam_configuration,omitempty"` + ProtectedRoles []string `json:"protected_role_names,omitempty"` + PostgresSuperuserTeams []string `json:"postgres_superuser_teams,omitempty"` + EnablePostgresTeamCRD *bool `json:"enable_postgres_team_crd,omitempty"` + EnablePostgresTeamCRDSuperusers bool `json:"enable_postgres_team_crd_superusers,omitempty"` } // LoggingRESTAPIConfiguration defines Logging API conf diff --git a/pkg/apis/acid.zalan.do/v1/postgres_team_type.go b/pkg/apis/acid.zalan.do/v1/postgres_team_type.go new file mode 100644 index 000000000..5697c193e --- /dev/null +++ b/pkg/apis/acid.zalan.do/v1/postgres_team_type.go @@ -0,0 +1,33 @@ +package v1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// PostgresTeam defines Custom Resource Definition Object for team management. +type PostgresTeam struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec PostgresTeamSpec `json:"spec"` +} + +// PostgresTeamSpec defines the specification for the PostgresTeam TPR. +type PostgresTeamSpec struct { + AdditionalSuperuserTeams map[string][]string `json:"additionalSuperuserTeams,omitempty"` + AdditionalTeams map[string][]string `json:"additionalTeams,omitempty"` + AdditionalMembers map[string][]string `json:"additionalMembers,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// PostgresTeamList defines a list of PostgresTeam definitions. +type PostgresTeamList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + + Items []PostgresTeam `json:"items"` +} diff --git a/pkg/apis/acid.zalan.do/v1/register.go b/pkg/apis/acid.zalan.do/v1/register.go index 1c30e35fb..9dcbf2baf 100644 --- a/pkg/apis/acid.zalan.do/v1/register.go +++ b/pkg/apis/acid.zalan.do/v1/register.go @@ -1,11 +1,10 @@ package v1 import ( + acidzalando "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - - "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do" ) // APIVersion of the `postgresql` and `operator` CRDs @@ -44,6 +43,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { // TODO: User uppercase CRDResourceKind of our types in the next major API version scheme.AddKnownTypeWithName(SchemeGroupVersion.WithKind("postgresql"), &Postgresql{}) scheme.AddKnownTypeWithName(SchemeGroupVersion.WithKind("postgresqlList"), &PostgresqlList{}) + scheme.AddKnownTypeWithName(SchemeGroupVersion.WithKind("PostgresTeam"), &PostgresTeam{}) + scheme.AddKnownTypeWithName(SchemeGroupVersion.WithKind("PostgresTeamList"), &PostgresTeamList{}) scheme.AddKnownTypeWithName(SchemeGroupVersion.WithKind("OperatorConfiguration"), &OperatorConfiguration{}) scheme.AddKnownTypeWithName(SchemeGroupVersion.WithKind("OperatorConfigurationList"), diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 34e6b46e8..80a00f491 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -711,6 +711,127 @@ func (in *PostgresStatus) DeepCopy() *PostgresStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PostgresTeam) DeepCopyInto(out *PostgresTeam) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PostgresTeam. +func (in *PostgresTeam) DeepCopy() *PostgresTeam { + if in == nil { + return nil + } + out := new(PostgresTeam) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *PostgresTeam) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PostgresTeamList) DeepCopyInto(out *PostgresTeamList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]PostgresTeam, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PostgresTeamList. +func (in *PostgresTeamList) DeepCopy() *PostgresTeamList { + if in == nil { + return nil + } + out := new(PostgresTeamList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *PostgresTeamList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PostgresTeamSpec) DeepCopyInto(out *PostgresTeamSpec) { + *out = *in + if in.AdditionalSuperuserTeams != nil { + in, out := &in.AdditionalSuperuserTeams, &out.AdditionalSuperuserTeams + *out = make(map[string][]string, len(*in)) + for key, val := range *in { + var outVal []string + if val == nil { + (*out)[key] = nil + } else { + in, out := &val, &outVal + *out = make([]string, len(*in)) + copy(*out, *in) + } + (*out)[key] = outVal + } + } + if in.AdditionalTeams != nil { + in, out := &in.AdditionalTeams, &out.AdditionalTeams + *out = make(map[string][]string, len(*in)) + for key, val := range *in { + var outVal []string + if val == nil { + (*out)[key] = nil + } else { + in, out := &val, &outVal + *out = make([]string, len(*in)) + copy(*out, *in) + } + (*out)[key] = outVal + } + } + if in.AdditionalMembers != nil { + in, out := &in.AdditionalMembers, &out.AdditionalMembers + *out = make(map[string][]string, len(*in)) + for key, val := range *in { + var outVal []string + if val == nil { + (*out)[key] = nil + } else { + in, out := &val, &outVal + *out = make([]string, len(*in)) + copy(*out, *in) + } + (*out)[key] = outVal + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PostgresTeamSpec. +func (in *PostgresTeamSpec) DeepCopy() *PostgresTeamSpec { + if in == nil { + return nil + } + out := new(PostgresTeamSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PostgresUsersConfiguration) DeepCopyInto(out *PostgresUsersConfiguration) { *out = *in @@ -993,6 +1114,11 @@ func (in *TeamsAPIConfiguration) DeepCopyInto(out *TeamsAPIConfiguration) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.EnablePostgresTeamCRD != nil { + in, out := &in.EnablePostgresTeamCRD, &out.EnablePostgresTeamCRD + *out = new(bool) + **out = **in + } return } diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 8636083c2..06388d731 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -14,19 +14,10 @@ import ( "github.com/r3labs/diff" "github.com/sirupsen/logrus" - appsv1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - policybeta1 "k8s.io/api/policy/v1beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/cache" - "k8s.io/client-go/tools/record" - "k8s.io/client-go/tools/reference" - acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/scheme" "github.com/zalando/postgres-operator/pkg/spec" + pgteams "github.com/zalando/postgres-operator/pkg/teams" "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/constants" @@ -34,7 +25,16 @@ import ( "github.com/zalando/postgres-operator/pkg/util/patroni" "github.com/zalando/postgres-operator/pkg/util/teams" "github.com/zalando/postgres-operator/pkg/util/users" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + policybeta1 "k8s.io/api/policy/v1beta1" rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/tools/reference" ) var ( @@ -48,6 +48,7 @@ var ( type Config struct { OpConfig config.Config RestConfig *rest.Config + PgTeamMap pgteams.PostgresTeamMap InfrastructureRoles map[string]spec.PgUser // inherited from the controller PodServiceAccount *v1.ServiceAccount PodServiceAccountRoleBinding *rbacv1.RoleBinding @@ -1107,7 +1108,7 @@ func (c *Cluster) initTeamMembers(teamID string, isPostgresSuperuserTeam bool) e if c.shouldAvoidProtectedOrSystemRole(username, "API role") { continue } - if c.OpConfig.EnableTeamSuperuser || isPostgresSuperuserTeam { + if (c.OpConfig.EnableTeamSuperuser && teamID == c.Spec.TeamID) || isPostgresSuperuserTeam { flags = append(flags, constants.RoleFlagSuperuser) } else { if c.OpConfig.TeamAdminRole != "" { @@ -1136,17 +1137,38 @@ func (c *Cluster) initTeamMembers(teamID string, isPostgresSuperuserTeam bool) e func (c *Cluster) initHumanUsers() error { var clusterIsOwnedBySuperuserTeam bool + superuserTeams := []string{} + + if c.OpConfig.EnablePostgresTeamCRDSuperusers { + superuserTeams = c.PgTeamMap.GetAdditionalSuperuserTeams(c.Spec.TeamID, true) + } for _, postgresSuperuserTeam := range c.OpConfig.PostgresSuperuserTeams { - err := c.initTeamMembers(postgresSuperuserTeam, true) - if err != nil { - return fmt.Errorf("Cannot create a team %q of Postgres superusers: %v", postgresSuperuserTeam, err) + if !(util.SliceContains(superuserTeams, postgresSuperuserTeam)) { + superuserTeams = append(superuserTeams, postgresSuperuserTeam) } - if postgresSuperuserTeam == c.Spec.TeamID { + } + + for _, superuserTeam := range superuserTeams { + err := c.initTeamMembers(superuserTeam, true) + if err != nil { + return fmt.Errorf("Cannot initialize members for team %q of Postgres superusers: %v", superuserTeam, err) + } + if superuserTeam == c.Spec.TeamID { clusterIsOwnedBySuperuserTeam = true } } + additionalTeams := c.PgTeamMap.GetAdditionalTeams(c.Spec.TeamID, true) + for _, additionalTeam := range additionalTeams { + if !(util.SliceContains(superuserTeams, additionalTeam)) { + err := c.initTeamMembers(additionalTeam, false) + if err != nil { + return fmt.Errorf("Cannot initialize members for additional team %q for cluster owned by %q: %v", additionalTeam, c.Spec.TeamID, err) + } + } + } + if clusterIsOwnedBySuperuserTeam { c.logger.Infof("Team %q owning the cluster is also a team of superusers. Created superuser roles for its members instead of admin roles.", c.Spec.TeamID) return nil @@ -1154,7 +1176,7 @@ func (c *Cluster) initHumanUsers() error { err := c.initTeamMembers(c.Spec.TeamID, false) if err != nil { - return fmt.Errorf("Cannot create a team %q of admins owning the PG cluster: %v", c.Spec.TeamID, err) + return fmt.Errorf("Cannot initialize members for team %q who owns the Postgres cluster: %v", c.Spec.TeamID, err) } return nil diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go index d227ce155..b8ddb7087 100644 --- a/pkg/cluster/util.go +++ b/pkg/cluster/util.go @@ -238,24 +238,37 @@ func (c *Cluster) getTeamMembers(teamID string) ([]string, error) { return nil, fmt.Errorf("no teamId specified") } + c.logger.Debugf("fetching possible additional team members for team %q", teamID) + members := []string{} + additionalMembers := c.PgTeamMap[c.Spec.TeamID].AdditionalMembers + for _, member := range additionalMembers { + members = append(members, member) + } + if !c.OpConfig.EnableTeamsAPI { - c.logger.Debugf("team API is disabled, returning empty list of members for team %q", teamID) - return []string{}, nil + c.logger.Debugf("team API is disabled, only returning %d members for team %q", len(members), teamID) + return members, nil } token, err := c.oauthTokenGetter.getOAuthToken() if err != nil { - c.logger.Warnf("could not get oauth token to authenticate to team service API, returning empty list of team members: %v", err) - return []string{}, nil + c.logger.Warnf("could not get oauth token to authenticate to team service API, only returning %d members for team %q: %v", len(members), teamID, err) + return members, nil } teamInfo, err := c.teamsAPIClient.TeamInfo(teamID, token) if err != nil { - c.logger.Warnf("could not get team info for team %q, returning empty list of team members: %v", teamID, err) - return []string{}, nil + c.logger.Warnf("could not get team info for team %q, only returning %d members: %v", teamID, len(members), err) + return members, nil } - return teamInfo.Members, nil + for _, member := range teamInfo.Members { + if !(util.SliceContains(members, member)) { + members = append(members, member) + } + } + + return members, nil } func (c *Cluster) waitForPodLabel(podEvents chan PodEvent, stopChan chan struct{}, role *PostgresRole) (*v1.Pod, error) { diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 3442bfcfe..2169beb76 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -16,6 +16,7 @@ import ( "github.com/zalando/postgres-operator/pkg/cluster" acidv1informer "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/spec" + "github.com/zalando/postgres-operator/pkg/teams" "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/constants" @@ -34,8 +35,9 @@ import ( // Controller represents operator controller type Controller struct { - config spec.ControllerConfig - opConfig *config.Config + config spec.ControllerConfig + opConfig *config.Config + pgTeamMap teams.PostgresTeamMap logger *logrus.Entry KubeClient k8sutil.KubernetesClient @@ -56,10 +58,11 @@ type Controller struct { clusterHistory map[spec.NamespacedName]ringlog.RingLogger // history of the cluster changes teamClusters map[string][]spec.NamespacedName - postgresqlInformer cache.SharedIndexInformer - podInformer cache.SharedIndexInformer - nodesInformer cache.SharedIndexInformer - podCh chan cluster.PodEvent + postgresqlInformer cache.SharedIndexInformer + postgresTeamInformer cache.SharedIndexInformer + podInformer cache.SharedIndexInformer + nodesInformer cache.SharedIndexInformer + podCh chan cluster.PodEvent clusterEventQueues []*cache.FIFO // [workerID]Queue lastClusterSyncTime int64 @@ -326,6 +329,12 @@ func (c *Controller) initController() { c.initSharedInformers() + if c.opConfig.EnablePostgresTeamCRD != nil && *c.opConfig.EnablePostgresTeamCRD { + c.loadPostgresTeams() + } else { + c.pgTeamMap = teams.PostgresTeamMap{} + } + if c.opConfig.DebugLogging { c.logger.Logger.Level = logrus.DebugLevel } @@ -357,6 +366,7 @@ func (c *Controller) initController() { func (c *Controller) initSharedInformers() { + // Postgresqls c.postgresqlInformer = acidv1informer.NewPostgresqlInformer( c.KubeClient.AcidV1ClientSet, c.opConfig.WatchedNamespace, @@ -369,6 +379,20 @@ func (c *Controller) initSharedInformers() { DeleteFunc: c.postgresqlDelete, }) + // PostgresTeams + if c.opConfig.EnablePostgresTeamCRD != nil && *c.opConfig.EnablePostgresTeamCRD { + c.postgresTeamInformer = acidv1informer.NewPostgresTeamInformer( + c.KubeClient.AcidV1ClientSet, + c.opConfig.WatchedNamespace, + constants.QueueResyncPeriodTPR*6, // 30 min + cache.Indexers{}) + + c.postgresTeamInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: c.postgresTeamAdd, + UpdateFunc: c.postgresTeamUpdate, + }) + } + // Pods podLw := &cache.ListWatch{ ListFunc: c.podListFunc, @@ -429,6 +453,10 @@ func (c *Controller) Run(stopCh <-chan struct{}, wg *sync.WaitGroup) { go c.apiserver.Run(stopCh, wg) go c.kubeNodesInformer(stopCh, wg) + if c.opConfig.EnablePostgresTeamCRD != nil && *c.opConfig.EnablePostgresTeamCRD { + go c.runPostgresTeamInformer(stopCh, wg) + } + c.logger.Info("started working in background") } @@ -444,6 +472,12 @@ func (c *Controller) runPostgresqlInformer(stopCh <-chan struct{}, wg *sync.Wait c.postgresqlInformer.Run(stopCh) } +func (c *Controller) runPostgresTeamInformer(stopCh <-chan struct{}, wg *sync.WaitGroup) { + defer wg.Done() + + c.postgresTeamInformer.Run(stopCh) +} + func queueClusterKey(eventType EventType, uid types.UID) string { return fmt.Sprintf("%s-%s", eventType, uid) } diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 7e4880712..3ad09ad28 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -163,6 +163,8 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.PamConfiguration = util.Coalesce(fromCRD.TeamsAPI.PamConfiguration, "https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees") result.ProtectedRoles = util.CoalesceStrArr(fromCRD.TeamsAPI.ProtectedRoles, []string{"admin"}) result.PostgresSuperuserTeams = fromCRD.TeamsAPI.PostgresSuperuserTeams + result.EnablePostgresTeamCRD = util.CoalesceBool(fromCRD.TeamsAPI.EnablePostgresTeamCRD, util.True()) + result.EnablePostgresTeamCRDSuperusers = fromCRD.TeamsAPI.EnablePostgresTeamCRDSuperusers // logging REST API config result.APIPort = util.CoalesceInt(fromCRD.LoggingRESTAPI.APIPort, 8080) diff --git a/pkg/controller/util.go b/pkg/controller/util.go index e460db2a5..57196d371 100644 --- a/pkg/controller/util.go +++ b/pkg/controller/util.go @@ -15,6 +15,7 @@ import ( acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/cluster" "github.com/zalando/postgres-operator/pkg/spec" + "github.com/zalando/postgres-operator/pkg/teams" "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/k8sutil" @@ -30,6 +31,7 @@ func (c *Controller) makeClusterConfig() cluster.Config { return cluster.Config{ RestConfig: c.config.RestConfig, OpConfig: config.Copy(c.opConfig), + PgTeamMap: c.pgTeamMap, InfrastructureRoles: infrastructureRoles, PodServiceAccount: c.PodServiceAccount, } @@ -394,6 +396,37 @@ func (c *Controller) getInfrastructureRole( return roles, nil } +func (c *Controller) loadPostgresTeams() { + // reset team map + c.pgTeamMap = teams.PostgresTeamMap{} + + pgTeams, err := c.KubeClient.AcidV1ClientSet.AcidV1().PostgresTeams(c.opConfig.WatchedNamespace).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + c.logger.Errorf("could not list postgres team objects: %v", err) + } + + c.pgTeamMap.Load(pgTeams) + c.logger.Debugf("Internal Postgres Team Cache: %#v", c.pgTeamMap) +} + +func (c *Controller) postgresTeamAdd(obj interface{}) { + pgTeam, ok := obj.(*acidv1.PostgresTeam) + if !ok { + c.logger.Errorf("could not cast to PostgresTeam spec") + } + c.logger.Debugf("PostgreTeam %q added. Reloading postgres team CRDs and overwriting cached map", pgTeam.Name) + c.loadPostgresTeams() +} + +func (c *Controller) postgresTeamUpdate(prev, obj interface{}) { + pgTeam, ok := obj.(*acidv1.PostgresTeam) + if !ok { + c.logger.Errorf("could not cast to PostgresTeam spec") + } + c.logger.Debugf("PostgreTeam %q updated. Reloading postgres team CRDs and overwriting cached map", pgTeam.Name) + c.loadPostgresTeams() +} + func (c *Controller) podClusterName(pod *v1.Pod) spec.NamespacedName { if name, ok := pod.Labels[c.opConfig.ClusterNameLabel]; ok { return spec.NamespacedName{ diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/acid.zalan.do_client.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/acid.zalan.do_client.go index 1879b9514..e48e2d2a7 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/acid.zalan.do_client.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/acid.zalan.do_client.go @@ -33,6 +33,7 @@ import ( type AcidV1Interface interface { RESTClient() rest.Interface OperatorConfigurationsGetter + PostgresTeamsGetter PostgresqlsGetter } @@ -45,6 +46,10 @@ func (c *AcidV1Client) OperatorConfigurations(namespace string) OperatorConfigur return newOperatorConfigurations(c, namespace) } +func (c *AcidV1Client) PostgresTeams(namespace string) PostgresTeamInterface { + return newPostgresTeams(c, namespace) +} + func (c *AcidV1Client) Postgresqls(namespace string) PostgresqlInterface { return newPostgresqls(c, namespace) } diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go index 8cd4dc9da..9e31f5192 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go @@ -38,6 +38,10 @@ func (c *FakeAcidV1) OperatorConfigurations(namespace string) v1.OperatorConfigu return &FakeOperatorConfigurations{c, namespace} } +func (c *FakeAcidV1) PostgresTeams(namespace string) v1.PostgresTeamInterface { + return &FakePostgresTeams{c, namespace} +} + func (c *FakeAcidV1) Postgresqls(namespace string) v1.PostgresqlInterface { return &FakePostgresqls{c, namespace} } diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresteam.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresteam.go new file mode 100644 index 000000000..20c8ec809 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresteam.go @@ -0,0 +1,136 @@ +/* +Copyright 2020 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + acidzalandov1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakePostgresTeams implements PostgresTeamInterface +type FakePostgresTeams struct { + Fake *FakeAcidV1 + ns string +} + +var postgresteamsResource = schema.GroupVersionResource{Group: "acid.zalan.do", Version: "v1", Resource: "postgresteams"} + +var postgresteamsKind = schema.GroupVersionKind{Group: "acid.zalan.do", Version: "v1", Kind: "PostgresTeam"} + +// Get takes name of the postgresTeam, and returns the corresponding postgresTeam object, and an error if there is any. +func (c *FakePostgresTeams) Get(ctx context.Context, name string, options v1.GetOptions) (result *acidzalandov1.PostgresTeam, err error) { + obj, err := c.Fake. + Invokes(testing.NewGetAction(postgresteamsResource, c.ns, name), &acidzalandov1.PostgresTeam{}) + + if obj == nil { + return nil, err + } + return obj.(*acidzalandov1.PostgresTeam), err +} + +// List takes label and field selectors, and returns the list of PostgresTeams that match those selectors. +func (c *FakePostgresTeams) List(ctx context.Context, opts v1.ListOptions) (result *acidzalandov1.PostgresTeamList, err error) { + obj, err := c.Fake. + Invokes(testing.NewListAction(postgresteamsResource, postgresteamsKind, c.ns, opts), &acidzalandov1.PostgresTeamList{}) + + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &acidzalandov1.PostgresTeamList{ListMeta: obj.(*acidzalandov1.PostgresTeamList).ListMeta} + for _, item := range obj.(*acidzalandov1.PostgresTeamList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested postgresTeams. +func (c *FakePostgresTeams) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchAction(postgresteamsResource, c.ns, opts)) + +} + +// Create takes the representation of a postgresTeam and creates it. Returns the server's representation of the postgresTeam, and an error, if there is any. +func (c *FakePostgresTeams) Create(ctx context.Context, postgresTeam *acidzalandov1.PostgresTeam, opts v1.CreateOptions) (result *acidzalandov1.PostgresTeam, err error) { + obj, err := c.Fake. + Invokes(testing.NewCreateAction(postgresteamsResource, c.ns, postgresTeam), &acidzalandov1.PostgresTeam{}) + + if obj == nil { + return nil, err + } + return obj.(*acidzalandov1.PostgresTeam), err +} + +// Update takes the representation of a postgresTeam and updates it. Returns the server's representation of the postgresTeam, and an error, if there is any. +func (c *FakePostgresTeams) Update(ctx context.Context, postgresTeam *acidzalandov1.PostgresTeam, opts v1.UpdateOptions) (result *acidzalandov1.PostgresTeam, err error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateAction(postgresteamsResource, c.ns, postgresTeam), &acidzalandov1.PostgresTeam{}) + + if obj == nil { + return nil, err + } + return obj.(*acidzalandov1.PostgresTeam), err +} + +// Delete takes name of the postgresTeam and deletes it. Returns an error if one occurs. +func (c *FakePostgresTeams) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteAction(postgresteamsResource, c.ns, name), &acidzalandov1.PostgresTeam{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakePostgresTeams) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionAction(postgresteamsResource, c.ns, listOpts) + + _, err := c.Fake.Invokes(action, &acidzalandov1.PostgresTeamList{}) + return err +} + +// Patch applies the patch and returns the patched postgresTeam. +func (c *FakePostgresTeams) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *acidzalandov1.PostgresTeam, err error) { + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceAction(postgresteamsResource, c.ns, name, pt, data, subresources...), &acidzalandov1.PostgresTeam{}) + + if obj == nil { + return nil, err + } + return obj.(*acidzalandov1.PostgresTeam), err +} diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/generated_expansion.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/generated_expansion.go index fd5707c75..bcd80f922 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/generated_expansion.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/generated_expansion.go @@ -26,4 +26,6 @@ package v1 type OperatorConfigurationExpansion interface{} +type PostgresTeamExpansion interface{} + type PostgresqlExpansion interface{} diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresteam.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresteam.go new file mode 100644 index 000000000..82157dceb --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresteam.go @@ -0,0 +1,184 @@ +/* +Copyright 2020 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1 + +import ( + "context" + "time" + + v1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + scheme "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/scheme" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// PostgresTeamsGetter has a method to return a PostgresTeamInterface. +// A group's client should implement this interface. +type PostgresTeamsGetter interface { + PostgresTeams(namespace string) PostgresTeamInterface +} + +// PostgresTeamInterface has methods to work with PostgresTeam resources. +type PostgresTeamInterface interface { + Create(ctx context.Context, postgresTeam *v1.PostgresTeam, opts metav1.CreateOptions) (*v1.PostgresTeam, error) + Update(ctx context.Context, postgresTeam *v1.PostgresTeam, opts metav1.UpdateOptions) (*v1.PostgresTeam, error) + Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error + Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.PostgresTeam, error) + List(ctx context.Context, opts metav1.ListOptions) (*v1.PostgresTeamList, error) + Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.PostgresTeam, err error) + PostgresTeamExpansion +} + +// postgresTeams implements PostgresTeamInterface +type postgresTeams struct { + client rest.Interface + ns string +} + +// newPostgresTeams returns a PostgresTeams +func newPostgresTeams(c *AcidV1Client, namespace string) *postgresTeams { + return &postgresTeams{ + client: c.RESTClient(), + ns: namespace, + } +} + +// Get takes name of the postgresTeam, and returns the corresponding postgresTeam object, and an error if there is any. +func (c *postgresTeams) Get(ctx context.Context, name string, options metav1.GetOptions) (result *v1.PostgresTeam, err error) { + result = &v1.PostgresTeam{} + err = c.client.Get(). + Namespace(c.ns). + Resource("postgresteams"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of PostgresTeams that match those selectors. +func (c *postgresTeams) List(ctx context.Context, opts metav1.ListOptions) (result *v1.PostgresTeamList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v1.PostgresTeamList{} + err = c.client.Get(). + Namespace(c.ns). + Resource("postgresteams"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(ctx). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested postgresTeams. +func (c *postgresTeams) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Namespace(c.ns). + Resource("postgresteams"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch(ctx) +} + +// Create takes the representation of a postgresTeam and creates it. Returns the server's representation of the postgresTeam, and an error, if there is any. +func (c *postgresTeams) Create(ctx context.Context, postgresTeam *v1.PostgresTeam, opts metav1.CreateOptions) (result *v1.PostgresTeam, err error) { + result = &v1.PostgresTeam{} + err = c.client.Post(). + Namespace(c.ns). + Resource("postgresteams"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(postgresTeam). + Do(ctx). + Into(result) + return +} + +// Update takes the representation of a postgresTeam and updates it. Returns the server's representation of the postgresTeam, and an error, if there is any. +func (c *postgresTeams) Update(ctx context.Context, postgresTeam *v1.PostgresTeam, opts metav1.UpdateOptions) (result *v1.PostgresTeam, err error) { + result = &v1.PostgresTeam{} + err = c.client.Put(). + Namespace(c.ns). + Resource("postgresteams"). + Name(postgresTeam.Name). + VersionedParams(&opts, scheme.ParameterCodec). + Body(postgresTeam). + Do(ctx). + Into(result) + return +} + +// Delete takes name of the postgresTeam and deletes it. Returns an error if one occurs. +func (c *postgresTeams) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error { + return c.client.Delete(). + Namespace(c.ns). + Resource("postgresteams"). + Name(name). + Body(&opts). + Do(ctx). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *postgresTeams) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error { + var timeout time.Duration + if listOpts.TimeoutSeconds != nil { + timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second + } + return c.client.Delete(). + Namespace(c.ns). + Resource("postgresteams"). + VersionedParams(&listOpts, scheme.ParameterCodec). + Timeout(timeout). + Body(&opts). + Do(ctx). + Error() +} + +// Patch applies the patch and returns the patched postgresTeam. +func (c *postgresTeams) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.PostgresTeam, err error) { + result = &v1.PostgresTeam{} + err = c.client.Patch(pt). + Namespace(c.ns). + Resource("postgresteams"). + Name(name). + SubResource(subresources...). + VersionedParams(&opts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} diff --git a/pkg/generated/informers/externalversions/acid.zalan.do/v1/interface.go b/pkg/generated/informers/externalversions/acid.zalan.do/v1/interface.go index 30090afee..b83d4d0f0 100644 --- a/pkg/generated/informers/externalversions/acid.zalan.do/v1/interface.go +++ b/pkg/generated/informers/externalversions/acid.zalan.do/v1/interface.go @@ -30,6 +30,8 @@ import ( // Interface provides access to all the informers in this group version. type Interface interface { + // PostgresTeams returns a PostgresTeamInformer. + PostgresTeams() PostgresTeamInformer // Postgresqls returns a PostgresqlInformer. Postgresqls() PostgresqlInformer } @@ -45,6 +47,11 @@ func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakList return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} } +// PostgresTeams returns a PostgresTeamInformer. +func (v *version) PostgresTeams() PostgresTeamInformer { + return &postgresTeamInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} + // Postgresqls returns a PostgresqlInformer. func (v *version) Postgresqls() PostgresqlInformer { return &postgresqlInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} diff --git a/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresteam.go b/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresteam.go new file mode 100644 index 000000000..7ae532cbd --- /dev/null +++ b/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresteam.go @@ -0,0 +1,96 @@ +/* +Copyright 2020 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + "context" + time "time" + + acidzalandov1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + versioned "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned" + internalinterfaces "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/internalinterfaces" + v1 "github.com/zalando/postgres-operator/pkg/generated/listers/acid.zalan.do/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// PostgresTeamInformer provides access to a shared informer and lister for +// PostgresTeams. +type PostgresTeamInformer interface { + Informer() cache.SharedIndexInformer + Lister() v1.PostgresTeamLister +} + +type postgresTeamInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewPostgresTeamInformer constructs a new informer for PostgresTeam type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewPostgresTeamInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredPostgresTeamInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredPostgresTeamInformer constructs a new informer for PostgresTeam type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredPostgresTeamInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AcidV1().PostgresTeams(namespace).List(context.TODO(), options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AcidV1().PostgresTeams(namespace).Watch(context.TODO(), options) + }, + }, + &acidzalandov1.PostgresTeam{}, + resyncPeriod, + indexers, + ) +} + +func (f *postgresTeamInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredPostgresTeamInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *postgresTeamInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&acidzalandov1.PostgresTeam{}, f.defaultInformer) +} + +func (f *postgresTeamInformer) Lister() v1.PostgresTeamLister { + return v1.NewPostgresTeamLister(f.Informer().GetIndexer()) +} diff --git a/pkg/generated/informers/externalversions/generic.go b/pkg/generated/informers/externalversions/generic.go index 562dec419..7dff3e4e5 100644 --- a/pkg/generated/informers/externalversions/generic.go +++ b/pkg/generated/informers/externalversions/generic.go @@ -59,6 +59,8 @@ func (f *genericInformer) Lister() cache.GenericLister { func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) { switch resource { // Group=acid.zalan.do, Version=v1 + case v1.SchemeGroupVersion.WithResource("postgresteams"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Acid().V1().PostgresTeams().Informer()}, nil case v1.SchemeGroupVersion.WithResource("postgresqls"): return &genericInformer{resource: resource.GroupResource(), informer: f.Acid().V1().Postgresqls().Informer()}, nil diff --git a/pkg/generated/listers/acid.zalan.do/v1/expansion_generated.go b/pkg/generated/listers/acid.zalan.do/v1/expansion_generated.go index 1b96a7c76..81e829926 100644 --- a/pkg/generated/listers/acid.zalan.do/v1/expansion_generated.go +++ b/pkg/generated/listers/acid.zalan.do/v1/expansion_generated.go @@ -24,6 +24,14 @@ SOFTWARE. package v1 +// PostgresTeamListerExpansion allows custom methods to be added to +// PostgresTeamLister. +type PostgresTeamListerExpansion interface{} + +// PostgresTeamNamespaceListerExpansion allows custom methods to be added to +// PostgresTeamNamespaceLister. +type PostgresTeamNamespaceListerExpansion interface{} + // PostgresqlListerExpansion allows custom methods to be added to // PostgresqlLister. type PostgresqlListerExpansion interface{} diff --git a/pkg/generated/listers/acid.zalan.do/v1/postgresteam.go b/pkg/generated/listers/acid.zalan.do/v1/postgresteam.go new file mode 100644 index 000000000..102dae832 --- /dev/null +++ b/pkg/generated/listers/acid.zalan.do/v1/postgresteam.go @@ -0,0 +1,105 @@ +/* +Copyright 2020 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1 + +import ( + v1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/cache" +) + +// PostgresTeamLister helps list PostgresTeams. +// All objects returned here must be treated as read-only. +type PostgresTeamLister interface { + // List lists all PostgresTeams in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1.PostgresTeam, err error) + // PostgresTeams returns an object that can list and get PostgresTeams. + PostgresTeams(namespace string) PostgresTeamNamespaceLister + PostgresTeamListerExpansion +} + +// postgresTeamLister implements the PostgresTeamLister interface. +type postgresTeamLister struct { + indexer cache.Indexer +} + +// NewPostgresTeamLister returns a new PostgresTeamLister. +func NewPostgresTeamLister(indexer cache.Indexer) PostgresTeamLister { + return &postgresTeamLister{indexer: indexer} +} + +// List lists all PostgresTeams in the indexer. +func (s *postgresTeamLister) List(selector labels.Selector) (ret []*v1.PostgresTeam, err error) { + err = cache.ListAll(s.indexer, selector, func(m interface{}) { + ret = append(ret, m.(*v1.PostgresTeam)) + }) + return ret, err +} + +// PostgresTeams returns an object that can list and get PostgresTeams. +func (s *postgresTeamLister) PostgresTeams(namespace string) PostgresTeamNamespaceLister { + return postgresTeamNamespaceLister{indexer: s.indexer, namespace: namespace} +} + +// PostgresTeamNamespaceLister helps list and get PostgresTeams. +// All objects returned here must be treated as read-only. +type PostgresTeamNamespaceLister interface { + // List lists all PostgresTeams in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1.PostgresTeam, err error) + // Get retrieves the PostgresTeam from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*v1.PostgresTeam, error) + PostgresTeamNamespaceListerExpansion +} + +// postgresTeamNamespaceLister implements the PostgresTeamNamespaceLister +// interface. +type postgresTeamNamespaceLister struct { + indexer cache.Indexer + namespace string +} + +// List lists all PostgresTeams in the indexer for a given namespace. +func (s postgresTeamNamespaceLister) List(selector labels.Selector) (ret []*v1.PostgresTeam, err error) { + err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { + ret = append(ret, m.(*v1.PostgresTeam)) + }) + return ret, err +} + +// Get retrieves the PostgresTeam from the indexer for a given namespace and name. +func (s postgresTeamNamespaceLister) Get(name string) (*v1.PostgresTeam, error) { + obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(v1.Resource("postgresteam"), name) + } + return obj.(*v1.PostgresTeam), nil +} diff --git a/pkg/teams/postgres_team.go b/pkg/teams/postgres_team.go new file mode 100644 index 000000000..7fb725765 --- /dev/null +++ b/pkg/teams/postgres_team.go @@ -0,0 +1,118 @@ +package teams + +import ( + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + "github.com/zalando/postgres-operator/pkg/util" +) + +// PostgresTeamMap is the operator's internal representation of all PostgresTeam CRDs +type PostgresTeamMap map[string]postgresTeamMembership + +type postgresTeamMembership struct { + AdditionalSuperuserTeams []string + AdditionalTeams []string + AdditionalMembers []string +} + +type teamHashSet map[string]map[string]struct{} + +func (ths *teamHashSet) has(team string) bool { + _, ok := (*ths)[team] + return ok +} + +func (ths *teamHashSet) add(newTeam string, newSet []string) { + set := make(map[string]struct{}) + if ths.has(newTeam) { + set = (*ths)[newTeam] + } + for _, t := range newSet { + set[t] = struct{}{} + } + (*ths)[newTeam] = set +} + +func (ths *teamHashSet) toMap() map[string][]string { + newTeamMap := make(map[string][]string) + for team, items := range *ths { + list := []string{} + for item := range items { + list = append(list, item) + } + newTeamMap[team] = list + } + return newTeamMap +} + +func (ths *teamHashSet) mergeCrdMap(crdTeamMap map[string][]string) { + for t, at := range crdTeamMap { + ths.add(t, at) + } +} + +func fetchTeams(teamset *map[string]struct{}, set teamHashSet) { + for key := range set { + (*teamset)[key] = struct{}{} + } +} + +func (ptm *PostgresTeamMap) fetchAdditionalTeams(team string, superuserTeams bool, transitive bool, exclude []string) []string { + + var teams []string + + if superuserTeams { + teams = (*ptm)[team].AdditionalSuperuserTeams + } else { + teams = (*ptm)[team].AdditionalTeams + } + if transitive { + exclude = append(exclude, team) + for _, additionalTeam := range teams { + if !(util.SliceContains(exclude, additionalTeam)) { + transitiveTeams := (*ptm).fetchAdditionalTeams(additionalTeam, superuserTeams, transitive, exclude) + for _, transitiveTeam := range transitiveTeams { + if !(util.SliceContains(exclude, transitiveTeam)) && !(util.SliceContains(teams, transitiveTeam)) { + teams = append(teams, transitiveTeam) + } + } + } + } + } + + return teams +} + +// GetAdditionalTeams function to retrieve list of additional teams +func (ptm *PostgresTeamMap) GetAdditionalTeams(team string, transitive bool) []string { + return ptm.fetchAdditionalTeams(team, false, transitive, []string{}) +} + +// GetAdditionalSuperuserTeams function to retrieve list of additional superuser teams +func (ptm *PostgresTeamMap) GetAdditionalSuperuserTeams(team string, transitive bool) []string { + return ptm.fetchAdditionalTeams(team, true, transitive, []string{}) +} + +// Load function to import data from PostgresTeam CRD +func (ptm *PostgresTeamMap) Load(pgTeams *acidv1.PostgresTeamList) { + superuserTeamSet := teamHashSet{} + teamSet := teamHashSet{} + teamMemberSet := teamHashSet{} + teamIDs := make(map[string]struct{}) + + for _, pgTeam := range pgTeams.Items { + superuserTeamSet.mergeCrdMap(pgTeam.Spec.AdditionalSuperuserTeams) + teamSet.mergeCrdMap(pgTeam.Spec.AdditionalTeams) + teamMemberSet.mergeCrdMap(pgTeam.Spec.AdditionalMembers) + } + fetchTeams(&teamIDs, superuserTeamSet) + fetchTeams(&teamIDs, teamSet) + fetchTeams(&teamIDs, teamMemberSet) + + for teamID := range teamIDs { + (*ptm)[teamID] = postgresTeamMembership{ + AdditionalSuperuserTeams: util.CoalesceStrArr(superuserTeamSet.toMap()[teamID], []string{}), + AdditionalTeams: util.CoalesceStrArr(teamSet.toMap()[teamID], []string{}), + AdditionalMembers: util.CoalesceStrArr(teamMemberSet.toMap()[teamID], []string{}), + } + } +} diff --git a/pkg/teams/postgres_team_test.go b/pkg/teams/postgres_team_test.go new file mode 100644 index 000000000..f8c3a21d8 --- /dev/null +++ b/pkg/teams/postgres_team_test.go @@ -0,0 +1,194 @@ +package teams + +import ( + "testing" + + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + "github.com/zalando/postgres-operator/pkg/util" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var ( + True = true + False = false + pgTeamList = acidv1.PostgresTeamList{ + TypeMeta: metav1.TypeMeta{ + Kind: "List", + APIVersion: "v1", + }, + Items: []acidv1.PostgresTeam{ + { + TypeMeta: metav1.TypeMeta{ + Kind: "PostgresTeam", + APIVersion: "acid.zalan.do/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "teamAB", + }, + Spec: acidv1.PostgresTeamSpec{ + AdditionalSuperuserTeams: map[string][]string{"teamA": []string{"teamB", "team24x7"}, "teamB": []string{"teamA", "teamC", "team24x7"}}, + AdditionalTeams: map[string][]string{"teamA": []string{"teamC"}, "teamB": []string{}}, + AdditionalMembers: map[string][]string{"team24x7": []string{"optimusprime"}, "teamB": []string{"drno"}}, + }, + }, { + TypeMeta: metav1.TypeMeta{ + Kind: "PostgresTeam", + APIVersion: "acid.zalan.do/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "teamC", + }, + Spec: acidv1.PostgresTeamSpec{ + AdditionalSuperuserTeams: map[string][]string{"teamC": []string{"team24x7"}}, + AdditionalTeams: map[string][]string{"teamA": []string{"teamC"}, "teamC": []string{"teamA", "teamB", "acid"}}, + AdditionalMembers: map[string][]string{"acid": []string{"batman"}}, + }, + }, + }, + } +) + +// PostgresTeamMap is the operator's internal representation of all PostgresTeam CRDs +func TestLoadingPostgresTeamCRD(t *testing.T) { + tests := []struct { + name string + crd acidv1.PostgresTeamList + ptm PostgresTeamMap + error string + }{ + { + "Check that CRD is imported correctly into the internal format", + pgTeamList, + PostgresTeamMap{ + "teamA": { + AdditionalSuperuserTeams: []string{"teamB", "team24x7"}, + AdditionalTeams: []string{"teamC"}, + AdditionalMembers: []string{}, + }, + "teamB": { + AdditionalSuperuserTeams: []string{"teamA", "teamC", "team24x7"}, + AdditionalTeams: []string{}, + AdditionalMembers: []string{"drno"}, + }, + "teamC": { + AdditionalSuperuserTeams: []string{"team24x7"}, + AdditionalTeams: []string{"teamA", "teamB", "acid"}, + AdditionalMembers: []string{}, + }, + "team24x7": { + AdditionalSuperuserTeams: []string{}, + AdditionalTeams: []string{}, + AdditionalMembers: []string{"optimusprime"}, + }, + "acid": { + AdditionalSuperuserTeams: []string{}, + AdditionalTeams: []string{}, + AdditionalMembers: []string{"batman"}, + }, + }, + "Mismatch between PostgresTeam CRD and internal map", + }, + } + + for _, tt := range tests { + postgresTeamMap := PostgresTeamMap{} + postgresTeamMap.Load(&tt.crd) + for team, ptmeamMembership := range postgresTeamMap { + if !util.IsEqualIgnoreOrder(ptmeamMembership.AdditionalSuperuserTeams, tt.ptm[team].AdditionalSuperuserTeams) { + t.Errorf("%s: %v: expected additional members %#v, got %#v", tt.name, tt.error, tt.ptm, postgresTeamMap) + } + if !util.IsEqualIgnoreOrder(ptmeamMembership.AdditionalTeams, tt.ptm[team].AdditionalTeams) { + t.Errorf("%s: %v: expected additional teams %#v, got %#v", tt.name, tt.error, tt.ptm, postgresTeamMap) + } + if !util.IsEqualIgnoreOrder(ptmeamMembership.AdditionalMembers, tt.ptm[team].AdditionalMembers) { + t.Errorf("%s: %v: expected additional superuser teams %#v, got %#v", tt.name, tt.error, tt.ptm, postgresTeamMap) + } + } + } +} + +// TestGetAdditionalTeams if returns teams with and without transitive dependencies +func TestGetAdditionalTeams(t *testing.T) { + tests := []struct { + name string + team string + transitive bool + teams []string + error string + }{ + { + "Check that additional teams are returned", + "teamA", + false, + []string{"teamC"}, + "GetAdditionalTeams returns wrong list", + }, + { + "Check that additional teams are returned incl. transitive teams", + "teamA", + true, + []string{"teamC", "teamB", "acid"}, + "GetAdditionalTeams returns wrong list", + }, + { + "Check that empty list is returned", + "teamB", + false, + []string{}, + "GetAdditionalTeams returns wrong list", + }, + } + + postgresTeamMap := PostgresTeamMap{} + postgresTeamMap.Load(&pgTeamList) + + for _, tt := range tests { + additionalTeams := postgresTeamMap.GetAdditionalTeams(tt.team, tt.transitive) + if !util.IsEqualIgnoreOrder(additionalTeams, tt.teams) { + t.Errorf("%s: %v: expected additional teams %#v, got %#v", tt.name, tt.error, tt.teams, additionalTeams) + } + } +} + +// TestGetAdditionalSuperuserTeams if returns teams with and without transitive dependencies +func TestGetAdditionalSuperuserTeams(t *testing.T) { + tests := []struct { + name string + team string + transitive bool + teams []string + error string + }{ + { + "Check that additional superuser teams are returned", + "teamA", + false, + []string{"teamB", "team24x7"}, + "GetAdditionalSuperuserTeams returns wrong list", + }, + { + "Check that additional superuser teams are returned incl. transitive superuser teams", + "teamA", + true, + []string{"teamB", "teamC", "team24x7"}, + "GetAdditionalSuperuserTeams returns wrong list", + }, + { + "Check that empty list is returned", + "team24x7", + false, + []string{}, + "GetAdditionalSuperuserTeams returns wrong list", + }, + } + + postgresTeamMap := PostgresTeamMap{} + postgresTeamMap.Load(&pgTeamList) + + for _, tt := range tests { + additionalTeams := postgresTeamMap.GetAdditionalSuperuserTeams(tt.team, tt.transitive) + if !util.IsEqualIgnoreOrder(additionalTeams, tt.teams) { + t.Errorf("%s: %v: expected additional teams %#v, got %#v", tt.name, tt.error, tt.teams, additionalTeams) + } + } +} diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 35991248b..b6c583399 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -169,6 +169,8 @@ type Config struct { EnableTeamSuperuser bool `name:"enable_team_superuser" default:"false"` TeamAdminRole string `name:"team_admin_role" default:"admin"` EnableAdminRoleForUsers bool `name:"enable_admin_role_for_users" default:"true"` + EnablePostgresTeamCRD *bool `name:"enable_postgres_team_crd" default:"true"` + EnablePostgresTeamCRDSuperusers bool `name:"enable_postgres_team_crd_superusers" default:"false"` EnableMasterLoadBalancer bool `name:"enable_master_load_balancer" default:"true"` EnableReplicaLoadBalancer bool `name:"enable_replica_load_balancer" default:"false"` CustomServiceAnnotations map[string]string `name:"custom_service_annotations"` diff --git a/pkg/util/util.go b/pkg/util/util.go index abb9be01f..20e2915ba 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -10,7 +10,9 @@ import ( "fmt" "math/big" "math/rand" + "reflect" "regexp" + "sort" "strings" "time" @@ -134,6 +136,21 @@ func PrettyDiff(a, b interface{}) string { return strings.Join(Diff(a, b), "\n") } +// Compare two string slices while ignoring the order of elements +func IsEqualIgnoreOrder(a, b []string) bool { + if len(a) != len(b) { + return false + } + a_copy := make([]string, len(a)) + b_copy := make([]string, len(b)) + copy(a_copy, a) + copy(b_copy, b) + sort.Strings(a_copy) + sort.Strings(b_copy) + + return reflect.DeepEqual(a_copy, b_copy) +} + // SubstractStringSlices finds elements in a that are not in b and return them as a result slice. func SubstractStringSlices(a []string, b []string) (result []string, equal bool) { // Slices are assumed to contain unique elements only @@ -176,6 +193,20 @@ func FindNamedStringSubmatch(r *regexp.Regexp, s string) map[string]string { return res } +// SliceContains +func SliceContains(slice interface{}, item interface{}) bool { + s := reflect.ValueOf(slice) + if s.Kind() != reflect.Slice { + panic("Invalid data-type") + } + for i := 0; i < s.Len(); i++ { + if s.Index(i).Interface() == item { + return true + } + } + return false +} + // MapContains returns true if and only if haystack contains all the keys from the needle with matching corresponding values func MapContains(haystack, needle map[string]string) bool { if len(haystack) < len(needle) { diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index ee5b1ac39..c02d2c075 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -43,6 +43,17 @@ var prettyDiffTest = []struct { {[]int{1, 2, 3, 4}, []int{1, 2, 3, 4}, ""}, } +var isEqualIgnoreOrderTest = []struct { + inA []string + inB []string + outEqual bool +}{ + {[]string{"a", "b", "c"}, []string{"a", "b", "c"}, true}, + {[]string{"a", "b", "c"}, []string{"a", "c", "b"}, true}, + {[]string{"a", "b"}, []string{"a", "c", "b"}, false}, + {[]string{"a", "b", "c"}, []string{"a", "d", "c"}, false}, +} + var substractTest = []struct { inA []string inB []string @@ -53,6 +64,16 @@ var substractTest = []struct { {[]string{"a", "b", "c", "d"}, []string{"a", "bb", "c", "d"}, []string{"b"}, false}, } +var sliceContaintsTest = []struct { + slice []string + item string + out bool +}{ + {[]string{"a", "b", "c"}, "a", true}, + {[]string{"a", "b", "c"}, "d", false}, + {[]string{}, "d", false}, +} + var mapContaintsTest = []struct { inA map[string]string inB map[string]string @@ -136,6 +157,15 @@ func TestPrettyDiff(t *testing.T) { } } +func TestIsEqualIgnoreOrder(t *testing.T) { + for _, tt := range isEqualIgnoreOrderTest { + actualEqual := IsEqualIgnoreOrder(tt.inA, tt.inB) + if actualEqual != tt.outEqual { + t.Errorf("IsEqualIgnoreOrder expected: %t, got: %t", tt.outEqual, actualEqual) + } + } +} + func TestSubstractSlices(t *testing.T) { for _, tt := range substractTest { actualRes, actualEqual := SubstractStringSlices(tt.inA, tt.inB) @@ -160,6 +190,15 @@ func TestFindNamedStringSubmatch(t *testing.T) { } } +func TestSliceContains(t *testing.T) { + for _, tt := range sliceContaintsTest { + res := SliceContains(tt.slice, tt.item) + if res != tt.out { + t.Errorf("SliceContains expected: %#v, got: %#v", tt.out, res) + } + } +} + func TestMapContains(t *testing.T) { for _, tt := range mapContaintsTest { res := MapContains(tt.inA, tt.inB) From e10e0fec9e41d4f6c25aa6ef3f06f8115c82d14d Mon Sep 17 00:00:00 2001 From: Jakub Warczarek Date: Wed, 28 Oct 2020 10:56:50 +0100 Subject: [PATCH 104/168] Add support in UI for custom S3 endpoints for backups (#1152) * Support custom S3 endpoint for backups * Log info about AWS S3 endpoint during start up --- ui/operator_ui/main.py | 3 +++ ui/operator_ui/spiloutils.py | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ui/operator_ui/main.py b/ui/operator_ui/main.py index dc2450b9f..d159bee2d 100644 --- a/ui/operator_ui/main.py +++ b/ui/operator_ui/main.py @@ -104,6 +104,8 @@ USE_AWS_INSTANCE_PROFILE = ( getenv('USE_AWS_INSTANCE_PROFILE', 'false').lower() != 'false' ) +AWS_ENDPOINT = getenv('AWS_ENDPOINT') + tokens.configure() tokens.manage('read-only') tokens.start() @@ -1055,6 +1057,7 @@ def main(port, secret_key, debug, clusters: list): logger.info(f'Tokeninfo URL: {TOKENINFO_URL}') logger.info(f'Use AWS instance_profile: {USE_AWS_INSTANCE_PROFILE}') logger.info(f'WAL-E S3 endpoint: {WALE_S3_ENDPOINT}') + logger.info(f'AWS S3 endpoint: {AWS_ENDPOINT}') if TARGET_NAMESPACE is None: @on_exception( diff --git a/ui/operator_ui/spiloutils.py b/ui/operator_ui/spiloutils.py index ea347a84d..7a71f6dab 100644 --- a/ui/operator_ui/spiloutils.py +++ b/ui/operator_ui/spiloutils.py @@ -16,6 +16,8 @@ logger = getLogger(__name__) session = Session() +AWS_ENDPOINT = getenv('AWS_ENDPOINT') + OPERATOR_CLUSTER_NAME_LABEL = getenv('OPERATOR_CLUSTER_NAME_LABEL', 'cluster-name') COMMON_CLUSTER_LABEL = getenv('COMMON_CLUSTER_LABEL', '{"application":"spilo"}') @@ -266,7 +268,7 @@ def read_stored_clusters(bucket, prefix, delimiter='/'): return [ prefix['Prefix'].split('/')[-2] for prefix in these( - client('s3').list_objects( + client('s3', endpoint_url=AWS_ENDPOINT).list_objects( Bucket=bucket, Delimiter=delimiter, Prefix=prefix, @@ -287,7 +289,7 @@ def read_versions( return [ 'base' if uid == 'wal' else uid for prefix in these( - client('s3').list_objects( + client('s3', endpoint_url=AWS_ENDPOINT).list_objects( Bucket=bucket, Delimiter=delimiter, Prefix=prefix + pg_cluster + delimiter, From 9a11e85d57392916ad2b9b80aabcda02ad7c7a09 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 28 Oct 2020 17:51:37 +0100 Subject: [PATCH 105/168] disable PostgresTeam by default (#1186) * disable PostgresTeam by default * fix version in chart --- charts/postgres-operator/values-crd.yaml | 2 +- charts/postgres-operator/values.yaml | 4 ++-- docs/reference/operator_parameters.md | 2 +- docs/user.md | 5 +++++ manifests/configmap.yaml | 2 +- manifests/postgresql-operator-default-configuration.yaml | 2 +- pkg/apis/acid.zalan.do/v1/operator_configuration_type.go | 2 +- pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go | 5 ----- pkg/controller/controller.go | 6 +++--- pkg/controller/operator_config.go | 2 +- pkg/util/config/config.go | 2 +- 11 files changed, 17 insertions(+), 17 deletions(-) diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 52892c22c..6196c6fb2 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -257,7 +257,7 @@ configTeamsApi: # enable_admin_role_for_users: true # operator watches for PostgresTeam CRs to assign additional teams and members to clusters - enable_postgres_team_crd: true + enable_postgres_team_crd: false # toogle to create additional superuser teams from PostgresTeam CRs # enable_postgres_team_crd_superusers: "false" diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index ba5c7458c..231b0b9ac 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -1,7 +1,7 @@ image: registry: registry.opensource.zalan.do repository: acid/postgres-operator - tag: v1.5.0-61-ged2b3239-dirty + tag: v1.5.0 pullPolicy: "IfNotPresent" # Optionally specify an array of imagePullSecrets. @@ -249,7 +249,7 @@ configTeamsApi: # enable_admin_role_for_users: "true" # operator watches for PostgresTeam CRs to assign additional teams and members to clusters - enable_postgres_team_crd: "true" + enable_postgres_team_crd: "false" # toogle to create additional superuser teams from PostgresTeam CRs # enable_postgres_team_crd_superusers: "false" diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index bd12eb922..bd8c80d9c 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -635,7 +635,7 @@ key. * **enable_postgres_team_crd** toggle to make the operator watch for created or updated `PostgresTeam` CRDs and create roles for specified additional teams and members. - The default is `true`. + The default is `false`. * **enable_postgres_team_crd_superusers** in a `PostgresTeam` CRD additional superuser teams can assigned to teams that diff --git a/docs/user.md b/docs/user.md index db107dccb..8cacad0e8 100644 --- a/docs/user.md +++ b/docs/user.md @@ -330,6 +330,11 @@ spec: - "foo" ``` +Note, by default the `PostgresTeam` support is disabled in the configuration. +Switch `enable_postgres_team_crd` flag to `true` and the operator will start to +watch for this CRD. Make sure, the cluster role is up to date and contains a +section for [PostgresTeam](../manifests/operator-service-account-rbac.yaml#L30). + ## Prepared databases with roles and default privileges The `users` section in the manifests only allows for creating database roles diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index ce20dfa58..e59bfcea0 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -41,7 +41,7 @@ data: enable_master_load_balancer: "false" # enable_pod_antiaffinity: "false" # enable_pod_disruption_budget: "true" - # enable_postgres_team_crd: "true" + # enable_postgres_team_crd: "false" # enable_postgres_team_crd_superusers: "false" enable_replica_load_balancer: "false" # enable_shm_volume: "true" diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 71408ac43..14acc4356 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -122,7 +122,7 @@ configuration: enable_database_access: true teams_api: # enable_admin_role_for_users: true - # enable_postgres_team_crd: true + # enable_postgres_team_crd: false # enable_postgres_team_crd_superusers: false enable_team_superuser: false enable_teams_api: false diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 9dae0089b..a9abcf0ee 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -145,7 +145,7 @@ type TeamsAPIConfiguration struct { PamConfiguration string `json:"pam_configuration,omitempty"` ProtectedRoles []string `json:"protected_role_names,omitempty"` PostgresSuperuserTeams []string `json:"postgres_superuser_teams,omitempty"` - EnablePostgresTeamCRD *bool `json:"enable_postgres_team_crd,omitempty"` + EnablePostgresTeamCRD bool `json:"enable_postgres_team_crd,omitempty"` EnablePostgresTeamCRDSuperusers bool `json:"enable_postgres_team_crd_superusers,omitempty"` } diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 80a00f491..364b3e161 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -1114,11 +1114,6 @@ func (in *TeamsAPIConfiguration) DeepCopyInto(out *TeamsAPIConfiguration) { *out = make([]string, len(*in)) copy(*out, *in) } - if in.EnablePostgresTeamCRD != nil { - in, out := &in.EnablePostgresTeamCRD, &out.EnablePostgresTeamCRD - *out = new(bool) - **out = **in - } return } diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 2169beb76..0c29275e6 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -329,7 +329,7 @@ func (c *Controller) initController() { c.initSharedInformers() - if c.opConfig.EnablePostgresTeamCRD != nil && *c.opConfig.EnablePostgresTeamCRD { + if c.opConfig.EnablePostgresTeamCRD { c.loadPostgresTeams() } else { c.pgTeamMap = teams.PostgresTeamMap{} @@ -380,7 +380,7 @@ func (c *Controller) initSharedInformers() { }) // PostgresTeams - if c.opConfig.EnablePostgresTeamCRD != nil && *c.opConfig.EnablePostgresTeamCRD { + if c.opConfig.EnablePostgresTeamCRD { c.postgresTeamInformer = acidv1informer.NewPostgresTeamInformer( c.KubeClient.AcidV1ClientSet, c.opConfig.WatchedNamespace, @@ -453,7 +453,7 @@ func (c *Controller) Run(stopCh <-chan struct{}, wg *sync.WaitGroup) { go c.apiserver.Run(stopCh, wg) go c.kubeNodesInformer(stopCh, wg) - if c.opConfig.EnablePostgresTeamCRD != nil && *c.opConfig.EnablePostgresTeamCRD { + if c.opConfig.EnablePostgresTeamCRD { go c.runPostgresTeamInformer(stopCh, wg) } diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 3ad09ad28..9b2713da8 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -163,7 +163,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.PamConfiguration = util.Coalesce(fromCRD.TeamsAPI.PamConfiguration, "https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees") result.ProtectedRoles = util.CoalesceStrArr(fromCRD.TeamsAPI.ProtectedRoles, []string{"admin"}) result.PostgresSuperuserTeams = fromCRD.TeamsAPI.PostgresSuperuserTeams - result.EnablePostgresTeamCRD = util.CoalesceBool(fromCRD.TeamsAPI.EnablePostgresTeamCRD, util.True()) + result.EnablePostgresTeamCRD = fromCRD.TeamsAPI.EnablePostgresTeamCRD result.EnablePostgresTeamCRDSuperusers = fromCRD.TeamsAPI.EnablePostgresTeamCRDSuperusers // logging REST API config diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index b6c583399..47a120227 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -169,7 +169,7 @@ type Config struct { EnableTeamSuperuser bool `name:"enable_team_superuser" default:"false"` TeamAdminRole string `name:"team_admin_role" default:"admin"` EnableAdminRoleForUsers bool `name:"enable_admin_role_for_users" default:"true"` - EnablePostgresTeamCRD *bool `name:"enable_postgres_team_crd" default:"true"` + EnablePostgresTeamCRD bool `name:"enable_postgres_team_crd" default:"false"` EnablePostgresTeamCRDSuperusers bool `name:"enable_postgres_team_crd_superusers" default:"false"` EnableMasterLoadBalancer bool `name:"enable_master_load_balancer" default:"true"` EnableReplicaLoadBalancer bool `name:"enable_replica_load_balancer" default:"false"` From c694a72352dc1f76d93a460b6013e3b16835303e Mon Sep 17 00:00:00 2001 From: Jan Mussler Date: Thu, 29 Oct 2020 13:12:25 +0100 Subject: [PATCH 106/168] Make failure in retry a warning not an error. (#1188) --- pkg/cluster/database.go | 10 +++++----- pkg/controller/postgresql.go | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/cluster/database.go b/pkg/cluster/database.go index 57758c6aa..5e05f443a 100644 --- a/pkg/cluster/database.go +++ b/pkg/cluster/database.go @@ -131,12 +131,12 @@ func (c *Cluster) initDbConnWithName(dbname string) error { } if _, ok := err.(*net.OpError); ok { - c.logger.Errorf("could not connect to PostgreSQL database: %v", err) + c.logger.Warningf("could not connect to Postgres database: %v", err) return false, nil } if err2 := conn.Close(); err2 != nil { - c.logger.Errorf("error when closing PostgreSQL connection after another error: %v", err) + c.logger.Errorf("error when closing Postgres connection after another error: %v", err) return false, err2 } @@ -151,7 +151,7 @@ func (c *Cluster) initDbConnWithName(dbname string) error { conn.SetMaxIdleConns(-1) if c.pgDb != nil { - msg := "Closing an existing connection before opening a new one to %s" + msg := "closing an existing connection before opening a new one to %s" c.logger.Warningf(msg, dbname) c.closeDbConn() } @@ -166,7 +166,7 @@ func (c *Cluster) connectionIsClosed() bool { } func (c *Cluster) closeDbConn() (err error) { - c.setProcessName("closing db connection") + c.setProcessName("closing database connection") if c.pgDb != nil { c.logger.Debug("closing database connection") if err = c.pgDb.Close(); err != nil { @@ -181,7 +181,7 @@ func (c *Cluster) closeDbConn() (err error) { } func (c *Cluster) readPgUsersFromDatabase(userNames []string) (users spec.PgUserMap, err error) { - c.setProcessName("reading users from the db") + c.setProcessName("reading users from the database") var rows *sql.Rows users = make(spec.PgUserMap) if rows, err = c.pgDb.Query(getUserSQL, pq.Array(userNames)); err != nil { diff --git a/pkg/controller/postgresql.go b/pkg/controller/postgresql.go index 23e9356e6..09ed83dc0 100644 --- a/pkg/controller/postgresql.go +++ b/pkg/controller/postgresql.go @@ -225,11 +225,11 @@ func (c *Controller) processEvent(event ClusterEvent) { switch event.EventType { case EventAdd: if clusterFound { - lg.Debugf("Recieved add event for existing cluster") + lg.Infof("Recieved add event for already existing Postgres cluster") return } - lg.Infof("creation of the cluster started") + lg.Infof("creating a new Postgres cluster") cl = c.addCluster(lg, clusterName, event.NewSpec) From 7f7beba66b93501c47af9e3687902d6e69d31fe6 Mon Sep 17 00:00:00 2001 From: Jan Mussler Date: Thu, 29 Oct 2020 13:59:22 +0100 Subject: [PATCH 107/168] Improving e2e more (#1185) * Add curl to operator image. * Wait for idle operator in delete. --- docker/Dockerfile | 1 + e2e/Makefile | 3 +++ e2e/README.md | 4 ++-- e2e/scripts/watch_objects.sh | 10 +++++++++- e2e/tests/k8s_api.py | 13 +++++++++++++ e2e/tests/test_e2e.py | 18 ++++++++++++++++++ 6 files changed, 46 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 520fd2d07..4c47eb346 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,6 +2,7 @@ FROM alpine MAINTAINER Team ACID @ Zalando # We need root certificates to deal with teams api over https +RUN apk --no-cache add curl RUN apk --no-cache add ca-certificates COPY build/* / diff --git a/e2e/Makefile b/e2e/Makefile index a72c6bef0..b904ad763 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -51,3 +51,6 @@ tools: e2etest: tools copy clean ./run.sh main + +cleanup: clean + ./run.sh cleanup \ No newline at end of file diff --git a/e2e/README.md b/e2e/README.md index 92a1fc731..ce8931f62 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -33,7 +33,7 @@ runtime. In the e2e folder you can invoke tests either with `make test` or with: ```bash -./run.sh +./run.sh main ``` To run both the build and test step you can invoke `make e2e` from the parent @@ -41,7 +41,7 @@ directory. To run the end 2 end test and keep the kind state execute: ```bash -NOCLEANUP=True ./run.sh +NOCLEANUP=True ./run.sh main ``` ## Run indidual test diff --git a/e2e/scripts/watch_objects.sh b/e2e/scripts/watch_objects.sh index c866fbd45..9dde54f5e 100755 --- a/e2e/scripts/watch_objects.sh +++ b/e2e/scripts/watch_objects.sh @@ -13,7 +13,15 @@ kubectl get statefulsets echo kubectl get deployments echo +echo +echo 'Step from operator deployment' kubectl get pods -l name=postgres-operator -o jsonpath='{.items..metadata.annotations.step}' echo +echo +echo 'Spilo Image in statefulset' kubectl get pods -l application=spilo -o jsonpath='{.items..spec.containers..image}' -" \ No newline at end of file +echo +echo +echo 'Queue Status' +kubectl exec -it \$(kubectl get pods -l name=postgres-operator -o jsonpath='{.items..metadata.name}') -- curl localhost:8080/workers/all/status/ +echo" \ No newline at end of file diff --git a/e2e/tests/k8s_api.py b/e2e/tests/k8s_api.py index 371fa8e0d..f2abd8e0c 100644 --- a/e2e/tests/k8s_api.py +++ b/e2e/tests/k8s_api.py @@ -239,6 +239,19 @@ class K8s: return [] return json.loads(r.stdout.decode()) + def get_operator_state(self): + pod = self.get_operator_pod() + if pod == None: + return None + pod = pod.metadata.name + + r = self.exec_with_kubectl(pod, "curl localhost:8080/workers/all/status/") + if not r.returncode == 0 or not r.stdout.decode()[0:1]=="{": + return None + + return json.loads(r.stdout.decode()) + + def get_patroni_running_members(self, pod="acid-minimal-cluster-0"): result = self.get_patroni_state(pod) return list(filter(lambda x: "State" in x and x["State"] == "running", result)) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 888fc2eaa..05cc09a70 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -109,9 +109,18 @@ class EndToEndTestCase(unittest.TestCase): with open("manifests/postgres-operator.yaml", 'w') as f: yaml.dump(operator_deployment, f, Dumper=yaml.Dumper) + with open("manifests/configmap.yaml", 'r+') as f: + configmap = yaml.safe_load(f) + configmap["data"]["workers"] = "1" + + with open("manifests/configmap.yaml", 'w') as f: + yaml.dump(configmap, f, Dumper=yaml.Dumper) + for filename in ["operator-service-account-rbac.yaml", + "postgresteam.crd.yaml", "configmap.yaml", "postgres-operator.yaml", + "api-service.yaml", "infrastructure-roles.yaml", "infrastructure-roles-new.yaml", "e2e-storage-class.yaml"]: @@ -338,6 +347,7 @@ class EndToEndTestCase(unittest.TestCase): }, } k8s.update_config(patch_infrastructure_roles) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0":"idle"}, "Operator does not get in sync") try: # check that new roles are represented in the config by requesting the @@ -447,6 +457,7 @@ class EndToEndTestCase(unittest.TestCase): # so we additonally test if disabling the lazy upgrade - forcing the normal rolling upgrade - works self.eventuallyEqual(lambda: k8s.get_effective_pod_image(pod0), conf_image, "Rolling upgrade was not executed", 50, 3) self.eventuallyEqual(lambda: k8s.get_effective_pod_image(pod1), conf_image, "Rolling upgrade was not executed", 50, 3) + self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members(pod0)), 2, "Postgres status did not enter running") except timeout_decorator.TimeoutError: print('Operator log: {}'.format(k8s.get_operator_log())) @@ -519,6 +530,9 @@ class EndToEndTestCase(unittest.TestCase): print('Operator log: {}'.format(k8s.get_operator_log())) raise + # ensure cluster is healthy after tests + self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members("acid-minimal-cluster-0")), 2, "Postgres status did not enter running") + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_min_resource_limits(self): ''' @@ -809,12 +823,14 @@ class EndToEndTestCase(unittest.TestCase): } } k8s.update_config(patch_delete_annotations) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0":"idle"}, "Operator does not get in sync") try: # this delete attempt should be omitted because of missing annotations k8s.api.custom_objects_api.delete_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster") time.sleep(5) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0":"idle"}, "Operator does not get in sync") # check that pods and services are still there k8s.wait_for_running_pods(cluster_label, 2) @@ -825,6 +841,7 @@ class EndToEndTestCase(unittest.TestCase): # wait a little before proceeding time.sleep(10) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0":"idle"}, "Operator does not get in sync") # add annotations to manifest delete_date = datetime.today().strftime('%Y-%m-%d') @@ -838,6 +855,7 @@ class EndToEndTestCase(unittest.TestCase): } k8s.api.custom_objects_api.patch_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_delete_annotations) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0":"idle"}, "Operator does not get in sync") # wait a little before proceeding time.sleep(20) From d76419565b9a511c515e30e5d7cf0e5769fbcd80 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 2 Nov 2020 10:49:29 +0100 Subject: [PATCH 108/168] move to apiextensions from v1beta1 to v1 (#746) * move to apiextensions from v1beta1 to v1 * remove metadata from CRD validation * some forgotten change --- .../crds/operatorconfigurations.yaml | 949 ++++++++++------- .../postgres-operator/crds/postgresqls.yaml | 957 +++++++++-------- .../postgres-operator/crds/postgresteams.yaml | 103 +- kubectl-pg/cmd/check.go | 13 +- manifests/fake-teams-api.yaml | 2 +- manifests/operatorconfiguration.crd.yaml | 959 +++++++++-------- manifests/postgresql.crd.yaml | 961 ++++++++++-------- manifests/postgresteam.crd.yaml | 103 +- pkg/apis/acid.zalan.do/v1/crds.go | 309 +++--- pkg/controller/util.go | 12 +- pkg/util/k8sutil/k8sutil.go | 6 +- 11 files changed, 2459 insertions(+), 1915 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 05b5090c2..59671dc19 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -1,4 +1,4 @@ -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: operatorconfigurations.acid.zalan.do @@ -15,410 +15,575 @@ spec: singular: operatorconfiguration shortNames: - opconfig - additionalPrinterColumns: - - name: Image - type: string - description: Spilo image to be used for Pods - JSONPath: .configuration.docker_image - - name: Cluster-Label - type: string - description: Label for K8s resources created by operator - JSONPath: .configuration.kubernetes.cluster_name_label - - name: Service-Account - type: string - description: Name of service account to be used - JSONPath: .configuration.kubernetes.pod_service_account_name - - name: Min-Instances - type: integer - description: Minimum number of instances per Postgres cluster - JSONPath: .configuration.min_instances - - name: Age - type: date - JSONPath: .metadata.creationTimestamp scope: Namespaced - subresources: - status: {} - version: v1 - validation: - openAPIV3Schema: - type: object - required: - - kind - - apiVersion - - configuration - properties: - kind: - type: string - enum: - - OperatorConfiguration - apiVersion: - type: string - enum: - - acid.zalan.do/v1 - configuration: - type: object + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - name: Image + type: string + description: Spilo image to be used for Pods + JSONPath: .configuration.docker_image + - name: Cluster-Label + type: string + description: Label for K8s resources created by operator + JSONPath: .configuration.kubernetes.cluster_name_label + - name: Service-Account + type: string + description: Name of service account to be used + JSONPath: .configuration.kubernetes.pod_service_account_name + - name: Min-Instances + type: integer + description: Minimum number of instances per Postgres cluster + JSONPath: .configuration.min_instances + - name: Age + type: date + JSONPath: .metadata.creationTimestamp + schema: + openAPIV3Schema: + type: object + required: + - kind + - apiVersion + - configuration properties: - docker_image: + kind: type: string - enable_crd_validation: - type: boolean - enable_lazy_spilo_upgrade: - type: boolean - enable_shm_volume: - type: boolean - etcd_host: + enum: + - OperatorConfiguration + apiVersion: type: string - kubernetes_use_configmaps: - type: boolean - max_instances: - type: integer - minimum: -1 # -1 = disabled - min_instances: - type: integer - minimum: -1 # -1 = disabled - resync_period: - type: string - repair_period: - type: string - set_memory_request_to_limit: - type: boolean - sidecar_docker_images: - type: object - additionalProperties: + enum: + - acid.zalan.do/v1 + configuration: + type: object + properties: + docker_image: type: string - sidecars: - type: array - nullable: true - items: + enable_crd_validation: + type: boolean + enable_lazy_spilo_upgrade: + type: boolean + enable_shm_volume: + type: boolean + etcd_host: + type: string + kubernetes_use_configmaps: + type: boolean + max_instances: + type: integer + minimum: -1 # -1 = disabled + min_instances: + type: integer + minimum: -1 # -1 = disabled + resync_period: + type: string + repair_period: + type: string + set_memory_request_to_limit: + type: boolean + sidecar_docker_images: type: object - additionalProperties: true - workers: - type: integer - minimum: 1 - users: - type: object - properties: - replication_username: - type: string - super_username: - type: string - kubernetes: - type: object - properties: - cluster_domain: + additionalProperties: type: string - cluster_labels: + sidecars: + type: array + nullable: true + items: type: object - additionalProperties: + additionalProperties: true + workers: + type: integer + minimum: 1 + users: + type: object + properties: + replication_username: + type: string + super_username: + type: string + kubernetes: + type: object + properties: + cluster_domain: type: string - cluster_name_label: - type: string - custom_pod_annotations: - type: object - additionalProperties: - type: string - delete_annotation_date_key: - type: string - delete_annotation_name_key: - type: string - downscaler_annotations: - type: array - items: - type: string - enable_init_containers: - type: boolean - enable_pod_antiaffinity: - type: boolean - enable_pod_disruption_budget: - type: boolean - enable_sidecars: - type: boolean - infrastructure_roles_secret_name: - type: string - infrastructure_roles_secrets: - type: array - nullable: true - items: + cluster_labels: type: object - required: - - secretname - - userkey - - passwordkey - properties: - secretname: - type: string - userkey: - type: string - passwordkey: - type: string - rolekey: - type: string - defaultuservalue: - type: string - defaultrolevalue: - type: string - details: - type: string - template: - type: boolean - inherited_labels: - type: array - items: + additionalProperties: + type: string + cluster_name_label: type: string - master_pod_move_timeout: - type: string - node_readiness_label: + custom_pod_annotations: + type: object + additionalProperties: + type: string + delete_annotation_date_key: + type: string + delete_annotation_name_key: + type: string + downscaler_annotations: + type: array + items: + type: string + enable_init_containers: + type: boolean + enable_pod_antiaffinity: + type: boolean + enable_pod_disruption_budget: + type: boolean + enable_sidecars: + type: boolean + infrastructure_roles_secret_name: + type: string + infrastructure_roles_secrets: + type: array + nullable: true + items: + type: object + required: + - secretname + - userkey + - passwordkey + properties: + secretname: + type: string + userkey: + type: string + passwordkey: + type: string + rolekey: + type: string + defaultuservalue: + type: string + defaultrolevalue: + type: string + details: + type: string + template: + type: boolean + inherited_labels: + type: array + items: + type: string + master_pod_move_timeout: + type: string + node_readiness_label: + type: object + additionalProperties: + type: string + oauth_token_secret_name: + type: string + pdb_name_format: + type: string + pod_antiaffinity_topology_key: + type: string + pod_environment_configmap: + type: string + pod_environment_secret: + type: string + pod_management_policy: + type: string + enum: + - "ordered_ready" + - "parallel" + pod_priority_class_name: + type: string + pod_role_label: + type: string + pod_service_account_definition: + type: string + pod_service_account_name: + type: string + pod_service_account_role_binding_definition: + type: string + pod_terminate_grace_period: + type: string + secret_name_template: + type: string + spilo_runasuser: + type: integer + spilo_runasgroup: + type: integer + spilo_fsgroup: + type: integer + spilo_privileged: + type: boolean + storage_resize_mode: + type: string + enum: + - "ebs" + - "pvc" + - "off" + toleration: + type: object + additionalProperties: + type: string + watched_namespace: + type: string + postgres_pod_resources: + type: object + properties: + default_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + default_cpu_request: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + default_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + default_memory_request: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + min_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + min_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + timeouts: + type: object + properties: + pod_label_wait_timeout: + type: string + pod_deletion_wait_timeout: + type: string + ready_wait_interval: + type: string + ready_wait_timeout: + type: string + resource_check_interval: + type: string + resource_check_timeout: + type: string + load_balancer: + type: object + properties: + custom_service_annotations: + type: object + additionalProperties: + type: string + db_hosted_zone: + type: string + enable_master_load_balancer: + type: boolean + enable_replica_load_balancer: + type: boolean + master_dns_name_format: + type: string + replica_dns_name_format: + type: string + aws_or_gcp: + type: object + properties: + additional_secret_mount: + type: string + additional_secret_mount_path: + type: string + aws_region: + type: string + gcp_credentials: + type: string + kube_iam_role: + type: string + log_s3_bucket: + type: string + wal_gs_bucket: + type: string + wal_s3_bucket: + type: string + logical_backup: + type: object + properties: + logical_backup_docker_image: + type: string + logical_backup_s3_access_key_id: + type: string + logical_backup_s3_bucket: + type: string + logical_backup_s3_endpoint: + type: string + logical_backup_s3_region: + type: string + logical_backup_s3_secret_access_key: + type: string + logical_backup_s3_sse: + type: string + logical_backup_schedule: + type: string + pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' + debug: + type: object + properties: + debug_logging: + type: boolean + enable_database_access: + type: boolean + teams_api: + type: object + properties: + enable_admin_role_for_users: + type: boolean + enable_team_superuser: + type: boolean + enable_teams_api: + type: boolean + pam_configuration: + type: string + pam_role_name: + type: string + postgres_superuser_teams: + type: array + items: + type: string + pod_service_account_name: + type: string + pod_terminate_grace_period: + type: string + secret_name_template: + type: string + spilo_fsgroup: + type: integer + spilo_privileged: + type: boolean + toleration: + type: object + additionalProperties: + type: string + watched_namespace: + type: string + postgres_pod_resources: type: object - additionalProperties: - type: string - oauth_token_secret_name: - type: string - pdb_name_format: - type: string - pod_antiaffinity_topology_key: - type: string - pod_environment_configmap: - type: string - pod_environment_secret: - type: string - pod_management_policy: - type: string - enum: - - "ordered_ready" - - "parallel" - pod_priority_class_name: - type: string - pod_role_label: - type: string - pod_service_account_definition: - type: string - pod_service_account_name: - type: string - pod_service_account_role_binding_definition: - type: string - pod_terminate_grace_period: - type: string - secret_name_template: - type: string - spilo_runasuser: - type: integer - spilo_runasgroup: - type: integer - spilo_fsgroup: - type: integer - spilo_privileged: - type: boolean - toleration: + properties: + default_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + default_cpu_request: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + default_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + default_memory_request: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + timeouts: type: object - additionalProperties: - type: string - watched_namespace: - type: string - postgres_pod_resources: - type: object - properties: - default_cpu_limit: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - default_cpu_request: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - default_memory_limit: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - default_memory_request: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - min_cpu_limit: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - min_memory_limit: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - timeouts: - type: object - properties: - pod_label_wait_timeout: - type: string - pod_deletion_wait_timeout: - type: string - ready_wait_interval: - type: string - ready_wait_timeout: - type: string - resource_check_interval: - type: string - resource_check_timeout: - type: string - load_balancer: - type: object - properties: - custom_service_annotations: + properties: + pod_label_wait_timeout: + type: string + pod_deletion_wait_timeout: + type: string + ready_wait_interval: + type: string + ready_wait_timeout: + type: string + resource_check_interval: + type: string + resource_check_timeout: + type: string + load_balancer: type: object - additionalProperties: - type: string - db_hosted_zone: - type: string - enable_master_load_balancer: - type: boolean - enable_replica_load_balancer: - type: boolean - external_traffic_policy: - type: string - enum: - - "Cluster" - - "Local" - master_dns_name_format: - type: string - replica_dns_name_format: - type: string - aws_or_gcp: - type: object - properties: - additional_secret_mount: - type: string - additional_secret_mount_path: - type: string - aws_region: - type: string - kube_iam_role: - type: string - log_s3_bucket: - type: string - wal_s3_bucket: - type: string - logical_backup: - type: object - properties: - logical_backup_docker_image: - type: string - logical_backup_s3_access_key_id: - type: string - logical_backup_s3_bucket: - type: string - logical_backup_s3_endpoint: - type: string - logical_backup_s3_region: - type: string - logical_backup_s3_secret_access_key: - type: string - logical_backup_s3_sse: - type: string - logical_backup_schedule: - type: string - pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' - debug: - type: object - properties: - debug_logging: - type: boolean - enable_database_access: - type: boolean - teams_api: - type: object - properties: - enable_admin_role_for_users: - type: boolean - enable_postgres_team_crd: - type: boolean - enable_postgres_team_crd_superusers: - type: boolean - enable_team_superuser: - type: boolean - enable_teams_api: - type: boolean - pam_configuration: - type: string - pam_role_name: - type: string - postgres_superuser_teams: - type: array - items: - type: string - protected_role_names: - type: array - items: - type: string - team_admin_role: - type: string - team_api_role_configuration: + properties: + custom_service_annotations: + type: object + additionalProperties: + type: string + db_hosted_zone: + type: string + enable_master_load_balancer: + type: boolean + enable_replica_load_balancer: + type: boolean + external_traffic_policy: + type: string + enum: + - "Cluster" + - "Local" + master_dns_name_format: + type: string + replica_dns_name_format: + type: string + aws_or_gcp: type: object - additionalProperties: + properties: + additional_secret_mount: + type: string + additional_secret_mount_path: + type: string + aws_region: + type: string + kube_iam_role: + type: string + log_s3_bucket: + type: string + wal_s3_bucket: + type: string + logical_backup: + type: object + properties: + logical_backup_schedule: + type: string + pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' + logical_backup_docker_image: + type: string + logical_backup_s3_bucket: + type: string + logical_backup_s3_endpoint: + type: string + logical_backup_s3_sse: + type: string + logical_backup_s3_access_key_id: + type: string + logical_backup_s3_secret_access_key: + type: string + debug: + type: object + properties: + debug_logging: + type: boolean + enable_database_access: + type: boolean + teams_api: + type: object + properties: + enable_admin_role_for_users: + type: boolean + enable_postgres_team_crd: + type: boolean + enable_postgres_team_crd_superusers: + type: boolean + enable_team_superuser: + type: boolean + enable_teams_api: + type: boolean + pam_configuration: + type: string + pam_role_name: + type: string + postgres_superuser_teams: + type: array + items: + type: string + protected_role_names: + type: array + items: + type: string + team_admin_role: + type: string + team_api_role_configuration: + type: object + additionalProperties: + type: string + teams_api_url: + type: string + logging_rest_api: + type: object + properties: + api_port: + type: integer + cluster_history_entries: + type: integer + ring_log_lines: + type: integer + scalyr: + type: object + properties: + scalyr_api_key: + type: string + scalyr_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + scalyr_cpu_request: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + scalyr_image: + type: string + scalyr_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + scalyr_memory_request: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + scalyr_server_url: + type: string + teams_api_url: type: string - teams_api_url: - type: string - logging_rest_api: - type: object - properties: - api_port: - type: integer - cluster_history_entries: - type: integer - ring_log_lines: - type: integer - scalyr: # deprecated - type: object - properties: - scalyr_api_key: - type: string - scalyr_cpu_limit: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - scalyr_cpu_request: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - scalyr_image: - type: string - scalyr_memory_limit: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - scalyr_memory_request: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - scalyr_server_url: - type: string - connection_pooler: - type: object - properties: - connection_pooler_schema: - type: string - #default: "pooler" - connection_pooler_user: - type: string - #default: "pooler" - connection_pooler_image: - type: string - #default: "registry.opensource.zalan.do/acid/pgbouncer" - connection_pooler_max_db_connections: - type: integer - #default: 60 - connection_pooler_mode: - type: string - enum: - - "session" - - "transaction" - #default: "transaction" - connection_pooler_number_of_instances: - type: integer - minimum: 2 - #default: 2 - connection_pooler_default_cpu_limit: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - #default: "1" - connection_pooler_default_cpu_request: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - #default: "500m" - connection_pooler_default_memory_limit: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - #default: "100Mi" - connection_pooler_default_memory_request: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - #default: "100Mi" - status: - type: object - additionalProperties: - type: string + logging_rest_api: + type: object + properties: + api_port: + type: integer + cluster_history_entries: + type: integer + ring_log_lines: + type: integer + scalyr: # deprecated + type: object + properties: + scalyr_api_key: + type: string + scalyr_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + scalyr_cpu_request: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + scalyr_image: + type: string + scalyr_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + scalyr_memory_request: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + scalyr_server_url: + type: string + connection_pooler: + type: object + properties: + connection_pooler_schema: + type: string + #default: "pooler" + connection_pooler_user: + type: string + #default: "pooler" + connection_pooler_image: + type: string + #default: "registry.opensource.zalan.do/acid/pgbouncer" + connection_pooler_max_db_connections: + type: integer + #default: 60 + connection_pooler_mode: + type: string + enum: + - "session" + - "transaction" + #default: "transaction" + connection_pooler_number_of_instances: + type: integer + minimum: 2 + #default: 2 + connection_pooler_default_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + #default: "1" + connection_pooler_default_cpu_request: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + #default: "500m" + connection_pooler_default_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + #default: "100Mi" + connection_pooler_default_memory_request: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + #default: "100Mi" + status: + type: object + additionalProperties: + type: string diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 488f17c2b..dc7fe0d05 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -1,4 +1,4 @@ -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: postgresqls.acid.zalan.do @@ -15,144 +15,315 @@ spec: singular: postgresql shortNames: - pg - additionalPrinterColumns: - - name: Team - type: string - description: Team responsible for Postgres CLuster - JSONPath: .spec.teamId - - name: Version - type: string - description: PostgreSQL version - JSONPath: .spec.postgresql.version - - name: Pods - type: integer - description: Number of Pods per Postgres cluster - JSONPath: .spec.numberOfInstances - - name: Volume - type: string - description: Size of the bound volume - JSONPath: .spec.volume.size - - name: CPU-Request - type: string - description: Requested CPU for Postgres containers - JSONPath: .spec.resources.requests.cpu - - name: Memory-Request - type: string - description: Requested memory for Postgres containers - JSONPath: .spec.resources.requests.memory - - name: Age - type: date - JSONPath: .metadata.creationTimestamp - - name: Status - type: string - description: Current sync status of postgresql resource - JSONPath: .status.PostgresClusterStatus scope: Namespaced - subresources: - status: {} - version: v1 - validation: - openAPIV3Schema: - type: object - required: - - kind - - apiVersion - - metadata - - spec - properties: - kind: - type: string - enum: - - postgresql - apiVersion: - type: string - enum: - - acid.zalan.do/v1 - metadata: - type: object - required: - - name - properties: - name: - type: string - maxLength: 53 - spec: - type: object - required: - - numberOfInstances - - teamId - - postgresql - - volume - properties: - additionalVolumes: - type: array - items: + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - name: Team + type: string + description: Team responsible for Postgres CLuster + JSONPath: .spec.teamId + - name: Version + type: string + description: PostgreSQL version + JSONPath: .spec.postgresql.version + - name: Pods + type: integer + description: Number of Pods per Postgres cluster + JSONPath: .spec.numberOfInstances + - name: Volume + type: string + description: Size of the bound volume + JSONPath: .spec.volume.size + - name: CPU-Request + type: string + description: Requested CPU for Postgres containers + JSONPath: .spec.resources.requests.cpu + - name: Memory-Request + type: string + description: Requested memory for Postgres containers + JSONPath: .spec.resources.requests.memory + - name: Age + type: date + JSONPath: .metadata.creationTimestamp + - name: Status + type: string + description: Current sync status of postgresql resource + JSONPath: .status.PostgresClusterStatus + schema: + openAPIV3Schema: + type: object + required: + - kind + - apiVersion + - spec + properties: + kind: + type: string + enum: + - postgresql + apiVersion: + type: string + enum: + - acid.zalan.do/v1 + spec: + type: object + required: + - numberOfInstances + - teamId + - postgresql + - volume + properties: + additionalVolumes: + type: array + items: + type: object + required: + - name + - mountPath + - volumeSource + properties: + name: + type: string + mountPath: + type: string + targetContainers: + type: array + nullable: true + items: + type: string + volumeSource: + type: object + subPath: + type: string + allowedSourceRanges: + type: array + nullable: true + items: + type: string + pattern: '^(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\/(\d|[1-2]\d|3[0-2])$' + clone: type: object required: - - name - - mountPath - - volumeSource + - cluster properties: - name: + cluster: type: string - mountPath: + s3_endpoint: type: string - targetContainers: - type: array - nullable: true - items: - type: string - volumeSource: + s3_access_key_id: + type: string + s3_secret_access_key: + type: string + s3_force_path_style: + type: boolean + s3_wal_path: + type: string + timestamp: + type: string + pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$' + # The regexp matches the date-time format (RFC 3339 Section 5.6) that specifies a timezone as an offset relative to UTC + # Example: 1996-12-19T16:39:57-08:00 + # Note: this field requires a timezone + uid: + format: uuid + type: string + connectionPooler: + type: object + properties: + dockerImage: + type: string + maxDBConnections: + type: integer + mode: + type: string + enum: + - "session" + - "transaction" + numberOfInstances: + type: integer + minimum: 2 + resources: type: object - subPath: + required: + - requests + - limits + properties: + limits: + type: object + required: + - cpu + - memory + properties: + cpu: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + memory: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + requests: + type: object + required: + - cpu + - memory + properties: + cpu: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + memory: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + schema: type: string - allowedSourceRanges: - type: array - nullable: true - items: + user: + type: string + databases: + type: object + additionalProperties: + type: string + # Note: usernames specified here as database owners must be declared in the users key of the spec key. + dockerImage: type: string - pattern: '^(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\/(\d|[1-2]\d|3[0-2])$' - clone: - type: object - required: - - cluster - properties: - cluster: - type: string - s3_endpoint: - type: string - s3_access_key_id: - type: string - s3_secret_access_key: - type: string - s3_force_path_style: - type: boolean - s3_wal_path: - type: string - timestamp: - type: string - pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$' - # The regexp matches the date-time format (RFC 3339 Section 5.6) that specifies a timezone as an offset relative to UTC - # Example: 1996-12-19T16:39:57-08:00 - # Note: this field requires a timezone - uid: - format: uuid - type: string - connectionPooler: - type: object - properties: + enableConnectionPooler: + type: boolean + enableLogicalBackup: + type: boolean + enableMasterLoadBalancer: + type: boolean + enableReplicaLoadBalancer: + type: boolean + enableShmVolume: + type: boolean + init_containers: # deprecated + type: array + nullable: true + items: + type: object + required: + - cluster + properties: + cluster: + type: string + s3_endpoint: + type: string + s3_access_key_id: + type: string + s3_secret_access_key: + type: string + s3_force_path_style: + type: string + s3_wal_path: + type: string + timestamp: + type: string + pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$' + # The regexp matches the date-time format (RFC 3339 Section 5.6) that specifies a timezone as an offset relative to UTC + # Example: 1996-12-19T16:39:57-08:00 + # Note: this field requires a timezone + uid: + format: uuid + type: string + databases: + type: object + additionalProperties: + type: string + # Note: usernames specified here as database owners must be declared in the users key of the spec key. dockerImage: type: string - maxDBConnections: - type: integer - mode: + enableLogicalBackup: + type: boolean + enableMasterLoadBalancer: + type: boolean + enableReplicaLoadBalancer: + type: boolean + enableShmVolume: + type: boolean + init_containers: # deprecated + type: array + nullable: true + items: + type: object + additionalProperties: true + initContainers: + type: array + nullable: true + items: + type: object + additionalProperties: true + logicalBackupSchedule: type: string - enum: - - "session" - - "transaction" + pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' + maintenanceWindows: + type: array + items: + type: string + pattern: '^\ *((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))-((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))\ *$' numberOfInstances: type: integer - minimum: 2 + minimum: 0 + patroni: + type: object + properties: + initdb: + type: object + additionalProperties: + type: string + ttl: + type: integer + loop_wait: + type: integer + retry_timeout: + type: integer + maximum_lag_on_failover: + type: integer + synchronous_mode: + type: boolean + synchronous_mode_strict: + type: boolean + podAnnotations: + type: object + additionalProperties: + type: string + pod_priority_class_name: # deprecated + type: string + podPriorityClassName: + type: string + postgresql: + type: object + required: + - version + properties: + version: + type: string + pod_priority_class_name: # deprecated + type: string + podPriorityClassName: + type: string + postgresql: + type: object + required: + - version + properties: + version: + type: string + enum: + - "9.3" + - "9.4" + - "9.5" + - "9.6" + - "10" + - "11" + - "12" + parameters: + type: object + additionalProperties: + type: string + replicaLoadBalancer: # deprecated + type: boolean resources: type: object required: @@ -167,10 +338,29 @@ spec: properties: cpu: type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + # Decimal natural followed by m, or decimal natural followed by + # dot followed by up to three decimal digits. + # + # This is because the Kubernetes CPU resource has millis as the + # maximum precision. The actual values are checked in code + # because the regular expression would be huge and horrible and + # not very helpful in validation error messages; this one checks + # only the format of the given number. + # + # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + pattern: '^(\d+m|\d+\.\d{1,3})$' + # Note: the value specified here must not be zero or be lower + # than the corresponding request. memory: type: string + # You can express memory as a plain integer or as a fixed-point + # integer using one of these suffixes: E, P, T, G, M, k. You can + # also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki + # + # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + # Note: the value specified here must not be zero or be lower + # than the corresponding request. requests: type: object required: @@ -179,315 +369,238 @@ spec: properties: cpu: type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + # Decimal natural followed by m, or decimal natural followed by + # dot followed by up to three decimal digits. + # + # This is because the Kubernetes CPU resource has millis as the + # maximum precision. The actual values are checked in code + # because the regular expression would be huge and horrible and + # not very helpful in validation error messages; this one checks + # only the format of the given number. + # + # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + pattern: '^(\d+m|\d+\.\d{1,3})$' + # Note: the value specified here must not be zero or be higher + # than the corresponding limit. memory: type: string + # You can express memory as a plain integer or as a fixed-point + # integer using one of these suffixes: E, P, T, G, M, k. You can + # also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki + # + # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - schema: - type: string - user: - type: string - databases: - type: object - additionalProperties: - type: string - # Note: usernames specified here as database owners must be declared in the users key of the spec key. - dockerImage: - type: string - enableConnectionPooler: - type: boolean - enableLogicalBackup: - type: boolean - enableMasterLoadBalancer: - type: boolean - enableReplicaLoadBalancer: - type: boolean - enableShmVolume: - type: boolean - init_containers: # deprecated - type: array - nullable: true - items: - type: object - additionalProperties: true - initContainers: - type: array - nullable: true - items: - type: object - additionalProperties: true - logicalBackupSchedule: - type: string - pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' - maintenanceWindows: - type: array - items: - type: string - pattern: '^\ *((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))-((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))\ *$' - numberOfInstances: - type: integer - minimum: 0 - patroni: - type: object - properties: - initdb: - type: object - additionalProperties: - type: string - pg_hba: + # Note: the value specified here must not be zero or be higher + # than the corresponding limit. + sidecars: type: array + nullable: true items: - type: string - slots: - type: object - additionalProperties: type: object - additionalProperties: - type: string - ttl: + additionalProperties: true + spiloFSGroup: type: integer - loop_wait: - type: integer - retry_timeout: - type: integer - synchronous_mode: - type: boolean - synchronous_mode_strict: - type: boolean - maximum_lag_on_failover: - type: integer - podAnnotations: - type: object - additionalProperties: - type: string - pod_priority_class_name: # deprecated - type: string - podPriorityClassName: - type: string - postgresql: - type: object - required: - - version - properties: - version: - type: string - enum: - - "9.3" - - "9.4" - - "9.5" - - "9.6" - - "10" - - "11" - - "12" - parameters: + standby: type: object - additionalProperties: - type: string - preparedDatabases: - type: object - additionalProperties: + required: + - s3_wal_path + properties: + s3_wal_path: + type: string + preparedDatabases: type: object - properties: - defaultUsers: - type: boolean - extensions: - type: object - additionalProperties: - type: string - schemas: - type: object - additionalProperties: + additionalProperties: + type: object + properties: + defaultUsers: + type: boolean + extensions: type: object - properties: - defaultUsers: - type: boolean - defaultRoles: - type: boolean - replicaLoadBalancer: # deprecated - type: boolean - resources: - type: object - required: - - requests - - limits - properties: - limits: - type: object - required: - - cpu - - memory - properties: - cpu: - type: string - # Decimal natural followed by m, or decimal natural followed by - # dot followed by up to three decimal digits. - # - # This is because the Kubernetes CPU resource has millis as the - # maximum precision. The actual values are checked in code - # because the regular expression would be huge and horrible and - # not very helpful in validation error messages; this one checks - # only the format of the given number. - # - # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - # Note: the value specified here must not be zero or be lower - # than the corresponding request. - memory: - type: string - # You can express memory as a plain integer or as a fixed-point - # integer using one of these suffixes: E, P, T, G, M, k. You can - # also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki - # - # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - # Note: the value specified here must not be zero or be lower - # than the corresponding request. - requests: - type: object - required: - - cpu - - memory - properties: - cpu: - type: string - # Decimal natural followed by m, or decimal natural followed by - # dot followed by up to three decimal digits. - # - # This is because the Kubernetes CPU resource has millis as the - # maximum precision. The actual values are checked in code - # because the regular expression would be huge and horrible and - # not very helpful in validation error messages; this one checks - # only the format of the given number. - # - # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - # Note: the value specified here must not be zero or be higher - # than the corresponding limit. - memory: - type: string - # You can express memory as a plain integer or as a fixed-point - # integer using one of these suffixes: E, P, T, G, M, k. You can - # also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki - # - # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - # Note: the value specified here must not be zero or be higher - # than the corresponding limit. - serviceAnnotations: - type: object - additionalProperties: - type: string - sidecars: - type: array - nullable: true - items: - type: object - additionalProperties: true - spiloRunAsUser: - type: integer - spiloRunAsGroup: - type: integer - spiloFSGroup: - type: integer - standby: - type: object - required: - - s3_wal_path - properties: - s3_wal_path: - type: string - teamId: - type: string - tls: - type: object - required: - - secretName - properties: - secretName: - type: string - certificateFile: - type: string - privateKeyFile: - type: string - caFile: - type: string - caSecretName: - type: string - tolerations: - type: array - items: + additionalProperties: + type: string + schemas: + type: object + additionalProperties: + type: object + properties: + defaultUsers: + type: boolean + defaultRoles: + type: boolean + replicaLoadBalancer: # deprecated + type: boolean + resources: type: object required: - - key - - operator - - effect + - requests + - limits properties: - key: - type: string - operator: - type: string - enum: - - Equal - - Exists - value: - type: string - effect: - type: string - enum: - - NoExecute - - NoSchedule - - PreferNoSchedule - tolerationSeconds: - type: integer - useLoadBalancer: # deprecated - type: boolean - users: - type: object - additionalProperties: + limits: + type: object + required: + - key + - operator + - effect + properties: + key: + type: string + # Decimal natural followed by m, or decimal natural followed by + # dot followed by up to three decimal digits. + # + # This is because the Kubernetes CPU resource has millis as the + # maximum precision. The actual values are checked in code + # because the regular expression would be huge and horrible and + # not very helpful in validation error messages; this one checks + # only the format of the given number. + # + # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + # Note: the value specified here must not be zero or be lower + # than the corresponding request. + memory: + type: string + enum: + - Equal + - Exists + value: + type: string + # Decimal natural followed by m, or decimal natural followed by + # dot followed by up to three decimal digits. + # + # This is because the Kubernetes CPU resource has millis as the + # maximum precision. The actual values are checked in code + # because the regular expression would be huge and horrible and + # not very helpful in validation error messages; this one checks + # only the format of the given number. + # + # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + # Note: the value specified here must not be zero or be higher + # than the corresponding limit. + memory: + type: string + # You can express memory as a plain integer or as a fixed-point + # integer using one of these suffixes: E, P, T, G, M, k. You can + # also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki + # + # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + # Note: the value specified here must not be zero or be higher + # than the corresponding limit. + serviceAnnotations: + type: object + additionalProperties: + type: string + sidecars: type: array nullable: true - description: "Role flags specified here must not contradict each other" items: - type: string - enum: - - bypassrls - - BYPASSRLS - - nobypassrls - - NOBYPASSRLS - - createdb - - CREATEDB - - nocreatedb - - NOCREATEDB - - createrole - - CREATEROLE - - nocreaterole - - NOCREATEROLE - - inherit - - INHERIT - - noinherit - - NOINHERIT - - login - - LOGIN - - nologin - - NOLOGIN - - replication - - REPLICATION - - noreplication - - NOREPLICATION - - superuser - - SUPERUSER - - nosuperuser - - NOSUPERUSER - volume: - type: object - required: - - size - properties: - size: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - # Note: the value specified here must not be zero. - storageClass: - type: string - subPath: - type: string + type: object + additionalProperties: true + spiloRunAsUser: + type: integer + spiloRunAsGroup: + type: integer + spiloFSGroup: + type: integer + standby: + type: object + required: + - s3_wal_path + properties: + s3_wal_path: + type: string + teamId: + type: string + tls: + type: object + required: + - secretName + properties: + secretName: + type: string + certificateFile: + type: string + privateKeyFile: + type: string + caFile: + type: string + caSecretName: + type: string + tolerations: + type: array + items: + type: object + required: + - size + properties: + size: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + # Note: the value specified here must not be zero. + storageClass: + type: string + subPath: + type: string + enum: + - NoExecute + - NoSchedule + - PreferNoSchedule + tolerationSeconds: + type: integer + useLoadBalancer: # deprecated + type: boolean + users: + type: object + additionalProperties: + type: array + nullable: true + description: "Role flags specified here must not contradict each other" + items: + type: string + enum: + - bypassrls + - BYPASSRLS + - nobypassrls + - NOBYPASSRLS + - createdb + - CREATEDB + - nocreatedb + - NOCREATEDB + - createrole + - CREATEROLE + - nocreaterole + - NOCREATEROLE + - inherit + - INHERIT + - noinherit + - NOINHERIT + - login + - LOGIN + - nologin + - NOLOGIN + - replication + - REPLICATION + - noreplication + - NOREPLICATION + - superuser + - SUPERUSER + - nosuperuser + - NOSUPERUSER + volume: + type: object + required: + - size + properties: + size: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + # Note: the value specified here must not be zero. + storageClass: + type: string + subPath: + type: string + status: + type: object + additionalProperties: + type: string diff --git a/charts/postgres-operator/crds/postgresteams.yaml b/charts/postgres-operator/crds/postgresteams.yaml index 4f2e74034..81c5e1eaf 100644 --- a/charts/postgres-operator/crds/postgresteams.yaml +++ b/charts/postgres-operator/crds/postgresteams.yaml @@ -1,4 +1,4 @@ -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: postgresteams.acid.zalan.do @@ -16,52 +16,55 @@ spec: shortNames: - pgteam scope: Namespaced - subresources: - status: {} - version: v1 - validation: - openAPIV3Schema: - type: object - required: - - kind - - apiVersion - - spec - properties: - kind: - type: string - enum: - - PostgresTeam - apiVersion: - type: string - enum: - - acid.zalan.do/v1 - spec: - type: object - properties: - additionalSuperuserTeams: - type: object - description: "Map for teamId and associated additional superuser teams" - additionalProperties: - type: array - nullable: true - description: "List of teams to become Postgres superusers" - items: - type: string - additionalTeams: - type: object - description: "Map for teamId and associated additional teams" - additionalProperties: - type: array - nullable: true - description: "List of teams whose members will also be added to the Postgres cluster" - items: - type: string - additionalMembers: - type: object - description: "Map for teamId and associated additional users" - additionalProperties: - type: array - nullable: true - description: "List of users who will also be added to the Postgres cluster" - items: - type: string + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + required: + - kind + - apiVersion + - spec + properties: + kind: + type: string + enum: + - PostgresTeam + apiVersion: + type: string + enum: + - acid.zalan.do/v1 + spec: + type: object + properties: + additionalSuperuserTeams: + type: object + description: "Map for teamId and associated additional superuser teams" + additionalProperties: + type: array + nullable: true + description: "List of teams to become Postgres superusers" + items: + type: string + additionalTeams: + type: object + description: "Map for teamId and associated additional teams" + additionalProperties: + type: array + nullable: true + description: "List of teams whose members will also be added to the Postgres cluster" + items: + type: string + additionalMembers: + type: object + description: "Map for teamId and associated additional users" + additionalProperties: + type: array + nullable: true + description: "List of users who will also be added to the Postgres cluster" + items: + type: string diff --git a/kubectl-pg/cmd/check.go b/kubectl-pg/cmd/check.go index 266047cf0..4f88e7efa 100644 --- a/kubectl-pg/cmd/check.go +++ b/kubectl-pg/cmd/check.go @@ -24,19 +24,20 @@ package cmd import ( "fmt" + "log" + "github.com/spf13/cobra" postgresConstants "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" - "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - apiextbeta1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1" + v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "log" ) // checkCmd represent kubectl pg check. var checkCmd = &cobra.Command{ Use: "check", Short: "Checks the Postgres operator is installed in the k8s cluster", - Long: `Checks that the Postgres CRD is registered in a k8s cluster. + Long: `Checks that the Postgres CRD is registered in a k8s cluster. This means that the operator pod was able to start normally.`, Run: func(cmd *cobra.Command, args []string) { check() @@ -47,9 +48,9 @@ kubectl pg check } // check validates postgresql CRD registered or not. -func check() *v1beta1.CustomResourceDefinition { +func check() *v1.CustomResourceDefinition { config := getConfig() - apiExtClient, err := apiextbeta1.NewForConfig(config) + apiExtClient, err := apiextv1.NewForConfig(config) if err != nil { log.Fatal(err) } diff --git a/manifests/fake-teams-api.yaml b/manifests/fake-teams-api.yaml index 97d1b2a98..15f7c7576 100644 --- a/manifests/fake-teams-api.yaml +++ b/manifests/fake-teams-api.yaml @@ -1,4 +1,4 @@ -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: name: fake-teams-api diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index d0f020f52..808e3acb0 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -1,4 +1,4 @@ -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: operatorconfigurations.acid.zalan.do @@ -11,420 +11,575 @@ spec: singular: operatorconfiguration shortNames: - opconfig - additionalPrinterColumns: - - name: Image - type: string - description: Spilo image to be used for Pods - JSONPath: .configuration.docker_image - - name: Cluster-Label - type: string - description: Label for K8s resources created by operator - JSONPath: .configuration.kubernetes.cluster_name_label - - name: Service-Account - type: string - description: Name of service account to be used - JSONPath: .configuration.kubernetes.pod_service_account_name - - name: Min-Instances - type: integer - description: Minimum number of instances per Postgres cluster - JSONPath: .configuration.min_instances - - name: Age - type: date - JSONPath: .metadata.creationTimestamp scope: Namespaced - subresources: - status: {} - version: v1 - validation: - openAPIV3Schema: - type: object - required: - - kind - - apiVersion - - configuration - properties: - kind: - type: string - enum: - - OperatorConfiguration - apiVersion: - type: string - enum: - - acid.zalan.do/v1 - configuration: - type: object + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - name: Image + type: string + description: Spilo image to be used for Pods + JSONPath: .configuration.docker_image + - name: Cluster-Label + type: string + description: Label for K8s resources created by operator + JSONPath: .configuration.kubernetes.cluster_name_label + - name: Service-Account + type: string + description: Name of service account to be used + JSONPath: .configuration.kubernetes.pod_service_account_name + - name: Min-Instances + type: integer + description: Minimum number of instances per Postgres cluster + JSONPath: .configuration.min_instances + - name: Age + type: date + JSONPath: .metadata.creationTimestamp + schema: + openAPIV3Schema: + type: object + required: + - kind + - apiVersion + - configuration properties: - docker_image: + kind: type: string - enable_crd_validation: - type: boolean - enable_lazy_spilo_upgrade: - type: boolean - enable_shm_volume: - type: boolean - etcd_host: + enum: + - OperatorConfiguration + apiVersion: type: string - kubernetes_use_configmaps: - type: boolean - max_instances: - type: integer - minimum: -1 # -1 = disabled - min_instances: - type: integer - minimum: -1 # -1 = disabled - resync_period: - type: string - repair_period: - type: string - set_memory_request_to_limit: - type: boolean - sidecar_docker_images: - type: object - additionalProperties: + enum: + - acid.zalan.do/v1 + configuration: + type: object + properties: + docker_image: type: string - sidecars: - type: array - nullable: true - items: + enable_crd_validation: + type: boolean + enable_lazy_spilo_upgrade: + type: boolean + enable_shm_volume: + type: boolean + etcd_host: + type: string + kubernetes_use_configmaps: + type: boolean + max_instances: + type: integer + minimum: -1 # -1 = disabled + min_instances: + type: integer + minimum: -1 # -1 = disabled + resync_period: + type: string + repair_period: + type: string + set_memory_request_to_limit: + type: boolean + sidecar_docker_images: type: object - additionalProperties: true - workers: - type: integer - minimum: 1 - users: - type: object - properties: - replication_username: - type: string - super_username: - type: string - kubernetes: - type: object - properties: - cluster_domain: + additionalProperties: type: string - cluster_labels: + sidecars: + type: array + nullable: true + items: type: object - additionalProperties: + additionalProperties: true + workers: + type: integer + minimum: 1 + users: + type: object + properties: + replication_username: + type: string + super_username: + type: string + kubernetes: + type: object + properties: + cluster_domain: type: string - cluster_name_label: - type: string - custom_pod_annotations: - type: object - additionalProperties: - type: string - delete_annotation_date_key: - type: string - delete_annotation_name_key: - type: string - downscaler_annotations: - type: array - items: - type: string - enable_init_containers: - type: boolean - enable_pod_antiaffinity: - type: boolean - enable_pod_disruption_budget: - type: boolean - enable_sidecars: - type: boolean - infrastructure_roles_secret_name: - type: string - infrastructure_roles_secrets: - type: array - nullable: true - items: + cluster_labels: type: object - required: - - secretname - - userkey - - passwordkey - properties: - secretname: - type: string - userkey: - type: string - passwordkey: - type: string - rolekey: - type: string - defaultuservalue: - type: string - defaultrolevalue: - type: string - details: - type: string - template: - type: boolean - inherited_labels: - type: array - items: + additionalProperties: + type: string + cluster_name_label: type: string - master_pod_move_timeout: - type: string - node_readiness_label: + custom_pod_annotations: + type: object + additionalProperties: + type: string + delete_annotation_date_key: + type: string + delete_annotation_name_key: + type: string + downscaler_annotations: + type: array + items: + type: string + enable_init_containers: + type: boolean + enable_pod_antiaffinity: + type: boolean + enable_pod_disruption_budget: + type: boolean + enable_sidecars: + type: boolean + infrastructure_roles_secret_name: + type: string + infrastructure_roles_secrets: + type: array + nullable: true + items: + type: object + required: + - secretname + - userkey + - passwordkey + properties: + secretname: + type: string + userkey: + type: string + passwordkey: + type: string + rolekey: + type: string + defaultuservalue: + type: string + defaultrolevalue: + type: string + details: + type: string + template: + type: boolean + inherited_labels: + type: array + items: + type: string + master_pod_move_timeout: + type: string + node_readiness_label: + type: object + additionalProperties: + type: string + oauth_token_secret_name: + type: string + pdb_name_format: + type: string + pod_antiaffinity_topology_key: + type: string + pod_environment_configmap: + type: string + pod_environment_secret: + type: string + pod_management_policy: + type: string + enum: + - "ordered_ready" + - "parallel" + pod_priority_class_name: + type: string + pod_role_label: + type: string + pod_service_account_definition: + type: string + pod_service_account_name: + type: string + pod_service_account_role_binding_definition: + type: string + pod_terminate_grace_period: + type: string + secret_name_template: + type: string + spilo_runasuser: + type: integer + spilo_runasgroup: + type: integer + spilo_fsgroup: + type: integer + spilo_privileged: + type: boolean + storage_resize_mode: + type: string + enum: + - "ebs" + - "pvc" + - "off" + toleration: + type: object + additionalProperties: + type: string + watched_namespace: + type: string + postgres_pod_resources: + type: object + properties: + default_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + default_cpu_request: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + default_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + default_memory_request: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + min_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + min_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + timeouts: + type: object + properties: + pod_label_wait_timeout: + type: string + pod_deletion_wait_timeout: + type: string + ready_wait_interval: + type: string + ready_wait_timeout: + type: string + resource_check_interval: + type: string + resource_check_timeout: + type: string + load_balancer: + type: object + properties: + custom_service_annotations: + type: object + additionalProperties: + type: string + db_hosted_zone: + type: string + enable_master_load_balancer: + type: boolean + enable_replica_load_balancer: + type: boolean + master_dns_name_format: + type: string + replica_dns_name_format: + type: string + aws_or_gcp: + type: object + properties: + additional_secret_mount: + type: string + additional_secret_mount_path: + type: string + aws_region: + type: string + gcp_credentials: + type: string + kube_iam_role: + type: string + log_s3_bucket: + type: string + wal_gs_bucket: + type: string + wal_s3_bucket: + type: string + logical_backup: + type: object + properties: + logical_backup_docker_image: + type: string + logical_backup_s3_access_key_id: + type: string + logical_backup_s3_bucket: + type: string + logical_backup_s3_endpoint: + type: string + logical_backup_s3_region: + type: string + logical_backup_s3_secret_access_key: + type: string + logical_backup_s3_sse: + type: string + logical_backup_schedule: + type: string + pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' + debug: + type: object + properties: + debug_logging: + type: boolean + enable_database_access: + type: boolean + teams_api: + type: object + properties: + enable_admin_role_for_users: + type: boolean + enable_team_superuser: + type: boolean + enable_teams_api: + type: boolean + pam_configuration: + type: string + pam_role_name: + type: string + postgres_superuser_teams: + type: array + items: + type: string + pod_service_account_name: + type: string + pod_terminate_grace_period: + type: string + secret_name_template: + type: string + spilo_fsgroup: + type: integer + spilo_privileged: + type: boolean + toleration: + type: object + additionalProperties: + type: string + watched_namespace: + type: string + postgres_pod_resources: type: object - additionalProperties: - type: string - oauth_token_secret_name: - type: string - pdb_name_format: - type: string - pod_antiaffinity_topology_key: - type: string - pod_environment_configmap: - type: string - pod_environment_secret: - type: string - pod_management_policy: - type: string - enum: - - "ordered_ready" - - "parallel" - pod_priority_class_name: - type: string - pod_role_label: - type: string - pod_service_account_definition: - type: string - pod_service_account_name: - type: string - pod_service_account_role_binding_definition: - type: string - pod_terminate_grace_period: - type: string - secret_name_template: - type: string - spilo_runasuser: - type: integer - spilo_runasgroup: - type: integer - spilo_fsgroup: - type: integer - spilo_privileged: - type: boolean - storage_resize_mode: - type: string - enum: - - "ebs" - - "pvc" - - "off" - toleration: + properties: + default_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + default_cpu_request: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + default_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + default_memory_request: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + timeouts: type: object - additionalProperties: - type: string - watched_namespace: - type: string - postgres_pod_resources: - type: object - properties: - default_cpu_limit: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - default_cpu_request: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - default_memory_limit: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - default_memory_request: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - min_cpu_limit: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - min_memory_limit: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - timeouts: - type: object - properties: - pod_label_wait_timeout: - type: string - pod_deletion_wait_timeout: - type: string - ready_wait_interval: - type: string - ready_wait_timeout: - type: string - resource_check_interval: - type: string - resource_check_timeout: - type: string - load_balancer: - type: object - properties: - custom_service_annotations: + properties: + pod_label_wait_timeout: + type: string + pod_deletion_wait_timeout: + type: string + ready_wait_interval: + type: string + ready_wait_timeout: + type: string + resource_check_interval: + type: string + resource_check_timeout: + type: string + load_balancer: type: object - additionalProperties: - type: string - db_hosted_zone: - type: string - enable_master_load_balancer: - type: boolean - enable_replica_load_balancer: - type: boolean - external_traffic_policy: - type: string - enum: - - "Cluster" - - "Local" - master_dns_name_format: - type: string - replica_dns_name_format: - type: string - aws_or_gcp: - type: object - properties: - additional_secret_mount: - type: string - additional_secret_mount_path: - type: string - aws_region: - type: string - gcp_credentials: - type: string - kube_iam_role: - type: string - log_s3_bucket: - type: string - wal_gs_bucket: - type: string - wal_s3_bucket: - type: string - logical_backup: - type: object - properties: - logical_backup_docker_image: - type: string - logical_backup_s3_access_key_id: - type: string - logical_backup_s3_bucket: - type: string - logical_backup_s3_endpoint: - type: string - logical_backup_s3_region: - type: string - logical_backup_s3_secret_access_key: - type: string - logical_backup_s3_sse: - type: string - logical_backup_schedule: - type: string - pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' - debug: - type: object - properties: - debug_logging: - type: boolean - enable_database_access: - type: boolean - teams_api: - type: object - properties: - enable_admin_role_for_users: - type: boolean - enable_postgres_team_crd: - type: boolean - enable_postgres_team_crd_superusers: - type: boolean - enable_team_superuser: - type: boolean - enable_teams_api: - type: boolean - pam_configuration: - type: string - pam_role_name: - type: string - postgres_superuser_teams: - type: array - items: - type: string - protected_role_names: - type: array - items: - type: string - team_admin_role: - type: string - team_api_role_configuration: + properties: + custom_service_annotations: + type: object + additionalProperties: + type: string + db_hosted_zone: + type: string + enable_master_load_balancer: + type: boolean + enable_replica_load_balancer: + type: boolean + external_traffic_policy: + type: string + enum: + - "Cluster" + - "Local" + master_dns_name_format: + type: string + replica_dns_name_format: + type: string + aws_or_gcp: type: object - additionalProperties: + properties: + additional_secret_mount: + type: string + additional_secret_mount_path: + type: string + aws_region: + type: string + kube_iam_role: + type: string + log_s3_bucket: + type: string + wal_s3_bucket: + type: string + logical_backup: + type: object + properties: + logical_backup_schedule: + type: string + pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' + logical_backup_docker_image: + type: string + logical_backup_s3_bucket: + type: string + logical_backup_s3_endpoint: + type: string + logical_backup_s3_sse: + type: string + logical_backup_s3_access_key_id: + type: string + logical_backup_s3_secret_access_key: + type: string + debug: + type: object + properties: + debug_logging: + type: boolean + enable_database_access: + type: boolean + teams_api: + type: object + properties: + enable_admin_role_for_users: + type: boolean + enable_postgres_team_crd: + type: boolean + enable_postgres_team_crd_superusers: + type: boolean + enable_team_superuser: + type: boolean + enable_teams_api: + type: boolean + pam_configuration: + type: string + pam_role_name: + type: string + postgres_superuser_teams: + type: array + items: + type: string + protected_role_names: + type: array + items: + type: string + team_admin_role: + type: string + team_api_role_configuration: + type: object + additionalProperties: + type: string + teams_api_url: + type: string + logging_rest_api: + type: object + properties: + api_port: + type: integer + cluster_history_entries: + type: integer + ring_log_lines: + type: integer + scalyr: + type: object + properties: + scalyr_api_key: + type: string + scalyr_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + scalyr_cpu_request: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + scalyr_image: + type: string + scalyr_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + scalyr_memory_request: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + scalyr_server_url: + type: string + teams_api_url: type: string - teams_api_url: - type: string - logging_rest_api: - type: object - properties: - api_port: - type: integer - cluster_history_entries: - type: integer - ring_log_lines: - type: integer - scalyr: # deprecated - type: object - properties: - scalyr_api_key: - type: string - scalyr_cpu_limit: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - scalyr_cpu_request: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - scalyr_image: - type: string - scalyr_memory_limit: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - scalyr_memory_request: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - scalyr_server_url: - type: string - connection_pooler: - type: object - properties: - connection_pooler_schema: - type: string - #default: "pooler" - connection_pooler_user: - type: string - #default: "pooler" - connection_pooler_image: - type: string - #default: "registry.opensource.zalan.do/acid/pgbouncer" - connection_pooler_max_db_connections: - type: integer - #default: 60 - connection_pooler_mode: - type: string - enum: - - "session" - - "transaction" - #default: "transaction" - connection_pooler_number_of_instances: - type: integer - minimum: 2 - #default: 2 - connection_pooler_default_cpu_limit: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - #default: "1" - connection_pooler_default_cpu_request: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - #default: "500m" - connection_pooler_default_memory_limit: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - #default: "100Mi" - connection_pooler_default_memory_request: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - #default: "100Mi" - status: - type: object - additionalProperties: - type: string + logging_rest_api: + type: object + properties: + api_port: + type: integer + cluster_history_entries: + type: integer + ring_log_lines: + type: integer + scalyr: # deprecated + type: object + properties: + scalyr_api_key: + type: string + scalyr_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + scalyr_cpu_request: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + scalyr_image: + type: string + scalyr_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + scalyr_memory_request: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + scalyr_server_url: + type: string + connection_pooler: + type: object + properties: + connection_pooler_schema: + type: string + #default: "pooler" + connection_pooler_user: + type: string + #default: "pooler" + connection_pooler_image: + type: string + #default: "registry.opensource.zalan.do/acid/pgbouncer" + connection_pooler_max_db_connections: + type: integer + #default: 60 + connection_pooler_mode: + type: string + enum: + - "session" + - "transaction" + #default: "transaction" + connection_pooler_number_of_instances: + type: integer + minimum: 2 + #default: 2 + connection_pooler_default_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + #default: "1" + connection_pooler_default_cpu_request: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + #default: "500m" + connection_pooler_default_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + #default: "100Mi" + connection_pooler_default_memory_request: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + #default: "100Mi" + status: + type: object + additionalProperties: + type: string diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 56c010739..ffcf49056 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -1,4 +1,4 @@ -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: postgresqls.acid.zalan.do @@ -11,144 +11,315 @@ spec: singular: postgresql shortNames: - pg - additionalPrinterColumns: - - name: Team - type: string - description: Team responsible for Postgres CLuster - JSONPath: .spec.teamId - - name: Version - type: string - description: PostgreSQL version - JSONPath: .spec.postgresql.version - - name: Pods - type: integer - description: Number of Pods per Postgres cluster - JSONPath: .spec.numberOfInstances - - name: Volume - type: string - description: Size of the bound volume - JSONPath: .spec.volume.size - - name: CPU-Request - type: string - description: Requested CPU for Postgres containers - JSONPath: .spec.resources.requests.cpu - - name: Memory-Request - type: string - description: Requested memory for Postgres containers - JSONPath: .spec.resources.requests.memory - - name: Age - type: date - JSONPath: .metadata.creationTimestamp - - name: Status - type: string - description: Current sync status of postgresql resource - JSONPath: .status.PostgresClusterStatus scope: Namespaced - subresources: - status: {} - version: v1 - validation: - openAPIV3Schema: - type: object - required: - - kind - - apiVersion - - metadata - - spec - properties: - kind: - type: string - enum: - - postgresql - apiVersion: - type: string - enum: - - acid.zalan.do/v1 - metadata: - type: object - required: - - name - properties: - name: - type: string - maxLength: 53 - spec: - type: object - required: - - numberOfInstances - - teamId - - postgresql - - volume - properties: - additionalVolumes: - type: array - items: + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - name: Team + type: string + description: Team responsible for Postgres CLuster + JSONPath: .spec.teamId + - name: Version + type: string + description: PostgreSQL version + JSONPath: .spec.postgresql.version + - name: Pods + type: integer + description: Number of Pods per Postgres cluster + JSONPath: .spec.numberOfInstances + - name: Volume + type: string + description: Size of the bound volume + JSONPath: .spec.volume.size + - name: CPU-Request + type: string + description: Requested CPU for Postgres containers + JSONPath: .spec.resources.requests.cpu + - name: Memory-Request + type: string + description: Requested memory for Postgres containers + JSONPath: .spec.resources.requests.memory + - name: Age + type: date + JSONPath: .metadata.creationTimestamp + - name: Status + type: string + description: Current sync status of postgresql resource + JSONPath: .status.PostgresClusterStatus + schema: + openAPIV3Schema: + type: object + required: + - kind + - apiVersion + - spec + properties: + kind: + type: string + enum: + - postgresql + apiVersion: + type: string + enum: + - acid.zalan.do/v1 + spec: + type: object + required: + - numberOfInstances + - teamId + - postgresql + - volume + properties: + additionalVolumes: + type: array + items: + type: object + required: + - name + - mountPath + - volumeSource + properties: + name: + type: string + mountPath: + type: string + targetContainers: + type: array + nullable: true + items: + type: string + volumeSource: + type: object + subPath: + type: string + allowedSourceRanges: + type: array + nullable: true + items: + type: string + pattern: '^(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\/(\d|[1-2]\d|3[0-2])$' + clone: type: object required: - - name - - mountPath - - volumeSource + - cluster properties: - name: + cluster: type: string - mountPath: + s3_endpoint: type: string - targetContainers: - type: array - nullable: true - items: - type: string - volumeSource: + s3_access_key_id: + type: string + s3_secret_access_key: + type: string + s3_force_path_style: + type: boolean + s3_wal_path: + type: string + timestamp: + type: string + pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$' + # The regexp matches the date-time format (RFC 3339 Section 5.6) that specifies a timezone as an offset relative to UTC + # Example: 1996-12-19T16:39:57-08:00 + # Note: this field requires a timezone + uid: + format: uuid + type: string + connectionPooler: + type: object + properties: + dockerImage: + type: string + maxDBConnections: + type: integer + mode: + type: string + enum: + - "session" + - "transaction" + numberOfInstances: + type: integer + minimum: 2 + resources: type: object - subPath: + required: + - requests + - limits + properties: + limits: + type: object + required: + - cpu + - memory + properties: + cpu: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + memory: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + requests: + type: object + required: + - cpu + - memory + properties: + cpu: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + memory: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + schema: type: string - allowedSourceRanges: - type: array - nullable: true - items: + user: + type: string + databases: + type: object + additionalProperties: + type: string + # Note: usernames specified here as database owners must be declared in the users key of the spec key. + dockerImage: type: string - pattern: '^(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\/(\d|[1-2]\d|3[0-2])$' - clone: - type: object - required: - - cluster - properties: - cluster: - type: string - s3_endpoint: - type: string - s3_access_key_id: - type: string - s3_secret_access_key: - type: string - s3_force_path_style: - type: boolean - s3_wal_path: - type: string - timestamp: - type: string - pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$' - # The regexp matches the date-time format (RFC 3339 Section 5.6) that specifies a timezone as an offset relative to UTC - # Example: 1996-12-19T16:39:57-08:00 - # Note: this field requires a timezone - uid: - format: uuid - type: string - connectionPooler: - type: object - properties: + enableConnectionPooler: + type: boolean + enableLogicalBackup: + type: boolean + enableMasterLoadBalancer: + type: boolean + enableReplicaLoadBalancer: + type: boolean + enableShmVolume: + type: boolean + init_containers: # deprecated + type: array + nullable: true + items: + type: object + required: + - cluster + properties: + cluster: + type: string + s3_endpoint: + type: string + s3_access_key_id: + type: string + s3_secret_access_key: + type: string + s3_force_path_style: + type: string + s3_wal_path: + type: string + timestamp: + type: string + pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$' + # The regexp matches the date-time format (RFC 3339 Section 5.6) that specifies a timezone as an offset relative to UTC + # Example: 1996-12-19T16:39:57-08:00 + # Note: this field requires a timezone + uid: + format: uuid + type: string + databases: + type: object + additionalProperties: + type: string + # Note: usernames specified here as database owners must be declared in the users key of the spec key. dockerImage: type: string - maxDBConnections: - type: integer - mode: + enableLogicalBackup: + type: boolean + enableMasterLoadBalancer: + type: boolean + enableReplicaLoadBalancer: + type: boolean + enableShmVolume: + type: boolean + init_containers: # deprecated + type: array + nullable: true + items: + type: object + additionalProperties: true + initContainers: + type: array + nullable: true + items: + type: object + additionalProperties: true + logicalBackupSchedule: type: string - enum: - - "session" - - "transaction" + pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' + maintenanceWindows: + type: array + items: + type: string + pattern: '^\ *((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))-((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))\ *$' numberOfInstances: type: integer - minimum: 2 + minimum: 0 + patroni: + type: object + properties: + initdb: + type: object + additionalProperties: + type: string + ttl: + type: integer + loop_wait: + type: integer + retry_timeout: + type: integer + maximum_lag_on_failover: + type: integer + synchronous_mode: + type: boolean + synchronous_mode_strict: + type: boolean + podAnnotations: + type: object + additionalProperties: + type: string + pod_priority_class_name: # deprecated + type: string + podPriorityClassName: + type: string + postgresql: + type: object + required: + - version + properties: + version: + type: string + pod_priority_class_name: # deprecated + type: string + podPriorityClassName: + type: string + postgresql: + type: object + required: + - version + properties: + version: + type: string + enum: + - "9.3" + - "9.4" + - "9.5" + - "9.6" + - "10" + - "11" + - "12" + parameters: + type: object + additionalProperties: + type: string + replicaLoadBalancer: # deprecated + type: boolean resources: type: object required: @@ -163,10 +334,29 @@ spec: properties: cpu: type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + # Decimal natural followed by m, or decimal natural followed by + # dot followed by up to three decimal digits. + # + # This is because the Kubernetes CPU resource has millis as the + # maximum precision. The actual values are checked in code + # because the regular expression would be huge and horrible and + # not very helpful in validation error messages; this one checks + # only the format of the given number. + # + # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + pattern: '^(\d+m|\d+\.\d{1,3})$' + # Note: the value specified here must not be zero or be lower + # than the corresponding request. memory: type: string + # You can express memory as a plain integer or as a fixed-point + # integer using one of these suffixes: E, P, T, G, M, k. You can + # also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki + # + # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + # Note: the value specified here must not be zero or be lower + # than the corresponding request. requests: type: object required: @@ -175,319 +365,238 @@ spec: properties: cpu: type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + # Decimal natural followed by m, or decimal natural followed by + # dot followed by up to three decimal digits. + # + # This is because the Kubernetes CPU resource has millis as the + # maximum precision. The actual values are checked in code + # because the regular expression would be huge and horrible and + # not very helpful in validation error messages; this one checks + # only the format of the given number. + # + # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + pattern: '^(\d+m|\d+\.\d{1,3})$' + # Note: the value specified here must not be zero or be higher + # than the corresponding limit. memory: type: string + # You can express memory as a plain integer or as a fixed-point + # integer using one of these suffixes: E, P, T, G, M, k. You can + # also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki + # + # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - schema: - type: string - user: - type: string - databases: - type: object - additionalProperties: - type: string - # Note: usernames specified here as database owners must be declared in the users key of the spec key. - dockerImage: - type: string - enableConnectionPooler: - type: boolean - enableLogicalBackup: - type: boolean - enableMasterLoadBalancer: - type: boolean - enableReplicaLoadBalancer: - type: boolean - enableShmVolume: - type: boolean - init_containers: # deprecated - type: array - nullable: true - items: - type: object - additionalProperties: true - initContainers: - type: array - nullable: true - items: - type: object - additionalProperties: true - logicalBackupSchedule: - type: string - pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' - maintenanceWindows: - type: array - items: - type: string - pattern: '^\ *((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))-((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))\ *$' - numberOfInstances: - type: integer - minimum: 0 - patroni: - type: object - properties: - initdb: - type: object - additionalProperties: - type: string - pg_hba: + # Note: the value specified here must not be zero or be higher + # than the corresponding limit. + sidecars: type: array + nullable: true items: - type: string - slots: - type: object - additionalProperties: type: object - additionalProperties: - type: string - ttl: + additionalProperties: true + spiloFSGroup: type: integer - loop_wait: - type: integer - retry_timeout: - type: integer - maximum_lag_on_failover: - type: integer - synchronous_mode: - type: boolean - synchronous_mode_strict: - type: boolean - podAnnotations: - type: object - additionalProperties: - type: string - pod_priority_class_name: # deprecated - type: string - podPriorityClassName: - type: string - postgresql: - type: object - required: - - version - properties: - version: - type: string - enum: - - "9.3" - - "9.4" - - "9.5" - - "9.6" - - "10" - - "11" - - "12" - parameters: + standby: type: object - additionalProperties: - type: string - preparedDatabases: - type: object - additionalProperties: + required: + - s3_wal_path + properties: + s3_wal_path: + type: string + preparedDatabases: type: object - properties: - defaultUsers: - type: boolean - extensions: - type: object - additionalProperties: - type: string - schemas: - type: object - additionalProperties: + additionalProperties: + type: object + properties: + defaultUsers: + type: boolean + extensions: type: object - properties: - defaultUsers: - type: boolean - defaultRoles: - type: boolean - replicaLoadBalancer: # deprecated - type: boolean - resources: - type: object - required: - - requests - - limits - properties: - limits: - type: object - required: - - cpu - - memory - properties: - cpu: - type: string - # Decimal natural followed by m, or decimal natural followed by - # dot followed by up to three decimal digits. - # - # This is because the Kubernetes CPU resource has millis as the - # maximum precision. The actual values are checked in code - # because the regular expression would be huge and horrible and - # not very helpful in validation error messages; this one checks - # only the format of the given number. - # - # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - # Note: the value specified here must not be zero or be lower - # than the corresponding request. - memory: - type: string - # You can express memory as a plain integer or as a fixed-point - # integer using one of these suffixes: E, P, T, G, M, k. You can - # also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki - # - # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - # Note: the value specified here must not be zero or be lower - # than the corresponding request. - requests: - type: object - required: - - cpu - - memory - properties: - cpu: - type: string - # Decimal natural followed by m, or decimal natural followed by - # dot followed by up to three decimal digits. - # - # This is because the Kubernetes CPU resource has millis as the - # maximum precision. The actual values are checked in code - # because the regular expression would be huge and horrible and - # not very helpful in validation error messages; this one checks - # only the format of the given number. - # - # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - # Note: the value specified here must not be zero or be higher - # than the corresponding limit. - memory: - type: string - # You can express memory as a plain integer or as a fixed-point - # integer using one of these suffixes: E, P, T, G, M, k. You can - # also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki - # - # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - # Note: the value specified here must not be zero or be higher - # than the corresponding limit. - serviceAnnotations: - type: object - additionalProperties: - type: string - sidecars: - type: array - nullable: true - items: - type: object - additionalProperties: true - spiloRunAsUser: - type: integer - spiloRunAsGroup: - type: integer - spiloFSGroup: - type: integer - standby: - type: object - required: - - s3_wal_path - properties: - s3_wal_path: - type: string - teamId: - type: string - tls: - type: object - required: - - secretName - properties: - secretName: - type: string - certificateFile: - type: string - privateKeyFile: - type: string - caFile: - type: string - caSecretName: - type: string - tolerations: - type: array - items: + additionalProperties: + type: string + schemas: + type: object + additionalProperties: + type: object + properties: + defaultUsers: + type: boolean + defaultRoles: + type: boolean + replicaLoadBalancer: # deprecated + type: boolean + resources: type: object required: - - key - - operator - - effect + - requests + - limits properties: - key: - type: string - operator: - type: string - enum: - - Equal - - Exists - value: - type: string - effect: - type: string - enum: - - NoExecute - - NoSchedule - - PreferNoSchedule - tolerationSeconds: - type: integer - useLoadBalancer: # deprecated - type: boolean - users: - type: object - additionalProperties: + limits: + type: object + required: + - key + - operator + - effect + properties: + key: + type: string + # Decimal natural followed by m, or decimal natural followed by + # dot followed by up to three decimal digits. + # + # This is because the Kubernetes CPU resource has millis as the + # maximum precision. The actual values are checked in code + # because the regular expression would be huge and horrible and + # not very helpful in validation error messages; this one checks + # only the format of the given number. + # + # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + # Note: the value specified here must not be zero or be lower + # than the corresponding request. + memory: + type: string + enum: + - Equal + - Exists + value: + type: string + # Decimal natural followed by m, or decimal natural followed by + # dot followed by up to three decimal digits. + # + # This is because the Kubernetes CPU resource has millis as the + # maximum precision. The actual values are checked in code + # because the regular expression would be huge and horrible and + # not very helpful in validation error messages; this one checks + # only the format of the given number. + # + # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + # Note: the value specified here must not be zero or be higher + # than the corresponding limit. + memory: + type: string + # You can express memory as a plain integer or as a fixed-point + # integer using one of these suffixes: E, P, T, G, M, k. You can + # also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki + # + # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + # Note: the value specified here must not be zero or be higher + # than the corresponding limit. + serviceAnnotations: + type: object + additionalProperties: + type: string + sidecars: type: array nullable: true - description: "Role flags specified here must not contradict each other" items: - type: string - enum: - - bypassrls - - BYPASSRLS - - nobypassrls - - NOBYPASSRLS - - createdb - - CREATEDB - - nocreatedb - - NOCREATEDB - - createrole - - CREATEROLE - - nocreaterole - - NOCREATEROLE - - inherit - - INHERIT - - noinherit - - NOINHERIT - - login - - LOGIN - - nologin - - NOLOGIN - - replication - - REPLICATION - - noreplication - - NOREPLICATION - - superuser - - SUPERUSER - - nosuperuser - - NOSUPERUSER - volume: - type: object - required: - - size - properties: - size: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - # Note: the value specified here must not be zero. - storageClass: - type: string - subPath: - type: string - status: - type: object - additionalProperties: - type: string + type: object + additionalProperties: true + spiloRunAsUser: + type: integer + spiloRunAsGroup: + type: integer + spiloFSGroup: + type: integer + standby: + type: object + required: + - s3_wal_path + properties: + s3_wal_path: + type: string + teamId: + type: string + tls: + type: object + required: + - secretName + properties: + secretName: + type: string + certificateFile: + type: string + privateKeyFile: + type: string + caFile: + type: string + caSecretName: + type: string + tolerations: + type: array + items: + type: object + required: + - size + properties: + size: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + # Note: the value specified here must not be zero. + storageClass: + type: string + subPath: + type: string + enum: + - NoExecute + - NoSchedule + - PreferNoSchedule + tolerationSeconds: + type: integer + useLoadBalancer: # deprecated + type: boolean + users: + type: object + additionalProperties: + type: array + nullable: true + description: "Role flags specified here must not contradict each other" + items: + type: string + enum: + - bypassrls + - BYPASSRLS + - nobypassrls + - NOBYPASSRLS + - createdb + - CREATEDB + - nocreatedb + - NOCREATEDB + - createrole + - CREATEROLE + - nocreaterole + - NOCREATEROLE + - inherit + - INHERIT + - noinherit + - NOINHERIT + - login + - LOGIN + - nologin + - NOLOGIN + - replication + - REPLICATION + - noreplication + - NOREPLICATION + - superuser + - SUPERUSER + - nosuperuser + - NOSUPERUSER + volume: + type: object + required: + - size + properties: + size: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + # Note: the value specified here must not be zero. + storageClass: + type: string + subPath: + type: string + status: + type: object + additionalProperties: + type: string diff --git a/manifests/postgresteam.crd.yaml b/manifests/postgresteam.crd.yaml index 5f55bdfcb..645c8848d 100644 --- a/manifests/postgresteam.crd.yaml +++ b/manifests/postgresteam.crd.yaml @@ -1,4 +1,4 @@ -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: postgresteams.acid.zalan.do @@ -12,52 +12,55 @@ spec: shortNames: - pgteam scope: Namespaced - subresources: - status: {} - version: v1 - validation: - openAPIV3Schema: - type: object - required: - - kind - - apiVersion - - spec - properties: - kind: - type: string - enum: - - PostgresTeam - apiVersion: - type: string - enum: - - acid.zalan.do/v1 - spec: - type: object - properties: - additionalSuperuserTeams: - type: object - description: "Map for teamId and associated additional superuser teams" - additionalProperties: - type: array - nullable: true - description: "List of teams to become Postgres superusers" - items: - type: string - additionalTeams: - type: object - description: "Map for teamId and associated additional teams" - additionalProperties: - type: array - nullable: true - description: "List of teams whose members will also be added to the Postgres cluster" - items: - type: string - additionalMembers: - type: object - description: "Map for teamId and associated additional users" - additionalProperties: - type: array - nullable: true - description: "List of users who will also be added to the Postgres cluster" - items: - type: string + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + required: + - kind + - apiVersion + - spec + properties: + kind: + type: string + enum: + - PostgresTeam + apiVersion: + type: string + enum: + - acid.zalan.do/v1 + spec: + type: object + properties: + additionalSuperuserTeams: + type: object + description: "Map for teamId and associated additional superuser teams" + additionalProperties: + type: array + nullable: true + description: "List of teams to become Postgres superusers" + items: + type: string + additionalTeams: + type: object + description: "Map for teamId and associated additional teams" + additionalProperties: + type: array + nullable: true + description: "List of teams whose members will also be added to the Postgres cluster" + items: + type: string + additionalMembers: + type: object + description: "Map for teamId and associated additional users" + additionalProperties: + type: array + nullable: true + description: "List of users who will also be added to the Postgres cluster" + items: + type: string diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 0dca0c94b..92b904bae 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -2,7 +2,7 @@ package v1 import ( acidzalando "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do" - apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -20,7 +20,7 @@ const ( ) // PostgresCRDResourceColumns definition of AdditionalPrinterColumns for postgresql CRD -var PostgresCRDResourceColumns = []apiextv1beta1.CustomResourceColumnDefinition{ +var PostgresCRDResourceColumns = []apiextv1.CustomResourceColumnDefinition{ { Name: "Team", Type: "string", @@ -71,7 +71,7 @@ var PostgresCRDResourceColumns = []apiextv1beta1.CustomResourceColumnDefinition{ } // OperatorConfigCRDResourceColumns definition of AdditionalPrinterColumns for OperatorConfiguration CRD -var OperatorConfigCRDResourceColumns = []apiextv1beta1.CustomResourceColumnDefinition{ +var OperatorConfigCRDResourceColumns = []apiextv1.CustomResourceColumnDefinition{ { Name: "Image", Type: "string", @@ -107,17 +107,16 @@ var min0 = 0.0 var min1 = 1.0 var min2 = 2.0 var minDisable = -1.0 -var maxLength = int64(53) // PostgresCRDResourceValidation to check applied manifest parameters -var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ - OpenAPIV3Schema: &apiextv1beta1.JSONSchemaProps{ +var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextv1.JSONSchemaProps{ Type: "object", - Required: []string{"kind", "apiVersion", "metadata", "spec"}, - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Required: []string{"kind", "apiVersion", "spec"}, + Properties: map[string]apiextv1.JSONSchemaProps{ "kind": { Type: "string", - Enum: []apiextv1beta1.JSON{ + Enum: []apiextv1.JSON{ { Raw: []byte(`"postgresql"`), }, @@ -125,31 +124,21 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, "apiVersion": { Type: "string", - Enum: []apiextv1beta1.JSON{ + Enum: []apiextv1.JSON{ { Raw: []byte(`"acid.zalan.do/v1"`), }, }, }, - "metadata": { - Type: "object", - Required: []string{"name"}, - Properties: map[string]apiextv1beta1.JSONSchemaProps{ - "name": { - Type: "string", - MaxLength: &maxLength, - }, - }, - }, "spec": { Type: "object", Required: []string{"numberOfInstances", "teamId", "postgresql", "volume"}, - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "allowedSourceRanges": { Type: "array", Nullable: true, - Items: &apiextv1beta1.JSONSchemaPropsOrArray{ - Schema: &apiextv1beta1.JSONSchemaProps{ + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ Type: "string", Pattern: "^(\\d|[1-9]\\d|1\\d\\d|2[0-4]\\d|25[0-5])\\.(\\d|[1-9]\\d|1\\d\\d|2[0-4]\\d|25[0-5])\\.(\\d|[1-9]\\d|1\\d\\d|2[0-4]\\d|25[0-5])\\.(\\d|[1-9]\\d|1\\d\\d|2[0-4]\\d|25[0-5])\\/(\\d|[1-2]\\d|3[0-2])$", }, @@ -158,7 +147,7 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ "clone": { Type: "object", Required: []string{"cluster"}, - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "cluster": { Type: "string", }, @@ -190,7 +179,7 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, "connectionPooler": { Type: "object", - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "dockerImage": { Type: "string", }, @@ -199,7 +188,7 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, "mode": { Type: "string", - Enum: []apiextv1beta1.JSON{ + Enum: []apiextv1.JSON{ { Raw: []byte(`"session"`), }, @@ -215,11 +204,11 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ "resources": { Type: "object", Required: []string{"requests", "limits"}, - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "limits": { Type: "object", Required: []string{"cpu", "memory"}, - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "cpu": { Type: "string", Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0", @@ -235,7 +224,7 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ "requests": { Type: "object", Required: []string{"cpu", "memory"}, - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "cpu": { Type: "string", Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0", @@ -260,8 +249,8 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, "databases": { Type: "object", - AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ - Schema: &apiextv1beta1.JSONSchemaProps{ + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ + Schema: &apiextv1.JSONSchemaProps{ Type: "string", Description: "User names specified here as database owners must be declared in the users key of the spec key", }, @@ -288,10 +277,10 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ "init_containers": { Type: "array", Description: "Deprecated", - Items: &apiextv1beta1.JSONSchemaPropsOrArray{ - Schema: &apiextv1beta1.JSONSchemaProps{ + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ Type: "object", - AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ Allows: true, }, }, @@ -299,10 +288,10 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, "initContainers": { Type: "array", - Items: &apiextv1beta1.JSONSchemaPropsOrArray{ - Schema: &apiextv1beta1.JSONSchemaProps{ + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ Type: "object", - AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ Allows: true, }, }, @@ -314,8 +303,8 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, "maintenanceWindows": { Type: "array", - Items: &apiextv1beta1.JSONSchemaPropsOrArray{ - Schema: &apiextv1beta1.JSONSchemaProps{ + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ Type: "string", Pattern: "^\\ *((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\\d):([0-5]?\\d)|(2[0-3]|[01]?\\d):([0-5]?\\d))-((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\\d):([0-5]?\\d)|(2[0-3]|[01]?\\d):([0-5]?\\d))\\ *$", }, @@ -327,30 +316,30 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, "patroni": { Type: "object", - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "initdb": { Type: "object", - AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ - Schema: &apiextv1beta1.JSONSchemaProps{ + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ + Schema: &apiextv1.JSONSchemaProps{ Type: "string", }, }, }, "pg_hba": { Type: "array", - Items: &apiextv1beta1.JSONSchemaPropsOrArray{ - Schema: &apiextv1beta1.JSONSchemaProps{ + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ Type: "string", }, }, }, "slots": { Type: "object", - AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ - Schema: &apiextv1beta1.JSONSchemaProps{ + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ + Schema: &apiextv1.JSONSchemaProps{ Type: "object", - AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ - Schema: &apiextv1beta1.JSONSchemaProps{ + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ + Schema: &apiextv1.JSONSchemaProps{ Type: "string", }, }, @@ -379,8 +368,8 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, "podAnnotations": { Type: "object", - AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ - Schema: &apiextv1beta1.JSONSchemaProps{ + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ + Schema: &apiextv1.JSONSchemaProps{ Type: "string", }, }, @@ -395,10 +384,10 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ "postgresql": { Type: "object", Required: []string{"version"}, - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "version": { Type: "string", - Enum: []apiextv1beta1.JSON{ + Enum: []apiextv1.JSON{ { Raw: []byte(`"9.3"`), }, @@ -424,8 +413,8 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, "parameters": { Type: "object", - AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ - Schema: &apiextv1beta1.JSONSchemaProps{ + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ + Schema: &apiextv1.JSONSchemaProps{ Type: "string", }, }, @@ -434,27 +423,27 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, "preparedDatabases": { Type: "object", - AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ - Schema: &apiextv1beta1.JSONSchemaProps{ + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ + Schema: &apiextv1.JSONSchemaProps{ Type: "object", - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "defaultUsers": { Type: "boolean", }, "extensions": { Type: "object", - AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ - Schema: &apiextv1beta1.JSONSchemaProps{ + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ + Schema: &apiextv1.JSONSchemaProps{ Type: "string", }, }, }, "schemas": { Type: "object", - AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ - Schema: &apiextv1beta1.JSONSchemaProps{ + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ + Schema: &apiextv1.JSONSchemaProps{ Type: "object", - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "defaultUsers": { Type: "boolean", }, @@ -476,11 +465,11 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ "resources": { Type: "object", Required: []string{"requests", "limits"}, - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "limits": { Type: "object", Required: []string{"cpu", "memory"}, - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "cpu": { Type: "string", Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0", @@ -496,7 +485,7 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ "requests": { Type: "object", Required: []string{"cpu", "memory"}, - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "cpu": { Type: "string", Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0", @@ -513,18 +502,18 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, "serviceAnnotations": { Type: "object", - AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ - Schema: &apiextv1beta1.JSONSchemaProps{ + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ + Schema: &apiextv1.JSONSchemaProps{ Type: "string", }, }, }, "sidecars": { Type: "array", - Items: &apiextv1beta1.JSONSchemaPropsOrArray{ - Schema: &apiextv1beta1.JSONSchemaProps{ + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ Type: "object", - AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ Allows: true, }, }, @@ -542,7 +531,7 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ "standby": { Type: "object", Required: []string{"s3_wal_path"}, - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "s3_wal_path": { Type: "string", }, @@ -554,7 +543,7 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ "tls": { Type: "object", Required: []string{"secretName"}, - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "secretName": { Type: "string", }, @@ -574,17 +563,17 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, "tolerations": { Type: "array", - Items: &apiextv1beta1.JSONSchemaPropsOrArray{ - Schema: &apiextv1beta1.JSONSchemaProps{ + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ Type: "object", Required: []string{"key", "operator", "effect"}, - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "key": { Type: "string", }, "operator": { Type: "string", - Enum: []apiextv1beta1.JSON{ + Enum: []apiextv1.JSON{ { Raw: []byte(`"Equal"`), }, @@ -598,7 +587,7 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, "effect": { Type: "string", - Enum: []apiextv1beta1.JSON{ + Enum: []apiextv1.JSON{ { Raw: []byte(`"NoExecute"`), }, @@ -623,15 +612,15 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, "users": { Type: "object", - AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ - Schema: &apiextv1beta1.JSONSchemaProps{ + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ + Schema: &apiextv1.JSONSchemaProps{ Type: "array", Description: "Role flags specified here must not contradict each other", Nullable: true, - Items: &apiextv1beta1.JSONSchemaPropsOrArray{ - Schema: &apiextv1beta1.JSONSchemaProps{ + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ Type: "string", - Enum: []apiextv1beta1.JSON{ + Enum: []apiextv1.JSON{ { Raw: []byte(`"bypassrls"`), }, @@ -725,7 +714,7 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ "volume": { Type: "object", Required: []string{"size"}, - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "size": { Type: "string", Description: "Value must not be zero", @@ -741,11 +730,11 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, "additionalVolumes": { Type: "array", - Items: &apiextv1beta1.JSONSchemaPropsOrArray{ - Schema: &apiextv1beta1.JSONSchemaProps{ + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ Type: "object", Required: []string{"name", "mountPath", "volumeSource"}, - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "name": { Type: "string", }, @@ -754,8 +743,8 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, "targetContainers": { Type: "array", - Items: &apiextv1beta1.JSONSchemaPropsOrArray{ - Schema: &apiextv1beta1.JSONSchemaProps{ + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ Type: "string", }, }, @@ -774,8 +763,8 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, "status": { Type: "object", - AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ - Schema: &apiextv1beta1.JSONSchemaProps{ + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ + Schema: &apiextv1.JSONSchemaProps{ Type: "string", }, }, @@ -785,14 +774,14 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ } // OperatorConfigCRDResourceValidation to check applied manifest parameters -var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ - OpenAPIV3Schema: &apiextv1beta1.JSONSchemaProps{ +var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextv1.JSONSchemaProps{ Type: "object", Required: []string{"kind", "apiVersion", "configuration"}, - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "kind": { Type: "string", - Enum: []apiextv1beta1.JSON{ + Enum: []apiextv1.JSON{ { Raw: []byte(`"OperatorConfiguration"`), }, @@ -800,7 +789,7 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, "apiVersion": { Type: "string", - Enum: []apiextv1beta1.JSON{ + Enum: []apiextv1.JSON{ { Raw: []byte(`"acid.zalan.do/v1"`), }, @@ -808,7 +797,7 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, "configuration": { Type: "object", - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "docker_image": { Type: "string", }, @@ -848,18 +837,18 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, "sidecar_docker_images": { Type: "object", - AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ - Schema: &apiextv1beta1.JSONSchemaProps{ + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ + Schema: &apiextv1.JSONSchemaProps{ Type: "string", }, }, }, "sidecars": { Type: "array", - Items: &apiextv1beta1.JSONSchemaPropsOrArray{ - Schema: &apiextv1beta1.JSONSchemaProps{ + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ Type: "object", - AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ Allows: true, }, }, @@ -871,7 +860,7 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, "users": { Type: "object", - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "replication_username": { Type: "string", }, @@ -882,14 +871,14 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, "kubernetes": { Type: "object", - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "cluster_domain": { Type: "string", }, "cluster_labels": { Type: "object", - AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ - Schema: &apiextv1beta1.JSONSchemaProps{ + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ + Schema: &apiextv1.JSONSchemaProps{ Type: "string", }, }, @@ -899,8 +888,8 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, "custom_pod_annotations": { Type: "object", - AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ - Schema: &apiextv1beta1.JSONSchemaProps{ + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ + Schema: &apiextv1.JSONSchemaProps{ Type: "string", }, }, @@ -913,8 +902,8 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, "downscaler_annotations": { Type: "array", - Items: &apiextv1beta1.JSONSchemaPropsOrArray{ - Schema: &apiextv1beta1.JSONSchemaProps{ + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ Type: "string", }, }, @@ -936,11 +925,11 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, "infrastructure_roles_secrets": { Type: "array", - Items: &apiextv1beta1.JSONSchemaPropsOrArray{ - Schema: &apiextv1beta1.JSONSchemaProps{ + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ Type: "object", Required: []string{"secretname", "userkey", "passwordkey"}, - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "secretname": { Type: "string", }, @@ -971,8 +960,8 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, "inherited_labels": { Type: "array", - Items: &apiextv1beta1.JSONSchemaPropsOrArray{ - Schema: &apiextv1beta1.JSONSchemaProps{ + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ Type: "string", }, }, @@ -982,8 +971,8 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, "node_readiness_label": { Type: "object", - AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ - Schema: &apiextv1beta1.JSONSchemaProps{ + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ + Schema: &apiextv1.JSONSchemaProps{ Type: "string", }, }, @@ -1005,7 +994,7 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, "pod_management_policy": { Type: "string", - Enum: []apiextv1beta1.JSON{ + Enum: []apiextv1.JSON{ { Raw: []byte(`"ordered_ready"`), }, @@ -1049,7 +1038,7 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, "storage_resize_mode": { Type: "string", - Enum: []apiextv1beta1.JSON{ + Enum: []apiextv1.JSON{ { Raw: []byte(`"ebs"`), }, @@ -1063,8 +1052,8 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, "toleration": { Type: "object", - AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ - Schema: &apiextv1beta1.JSONSchemaProps{ + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ + Schema: &apiextv1.JSONSchemaProps{ Type: "string", }, }, @@ -1076,7 +1065,7 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, "postgres_pod_resources": { Type: "object", - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "default_cpu_limit": { Type: "string", Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", @@ -1105,7 +1094,7 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, "timeouts": { Type: "object", - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "pod_label_wait_timeout": { Type: "string", }, @@ -1128,11 +1117,11 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, "load_balancer": { Type: "object", - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "custom_service_annotations": { Type: "object", - AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ - Schema: &apiextv1beta1.JSONSchemaProps{ + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ + Schema: &apiextv1.JSONSchemaProps{ Type: "string", }, }, @@ -1148,7 +1137,7 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, "external_traffic_policy": { Type: "string", - Enum: []apiextv1beta1.JSON{ + Enum: []apiextv1.JSON{ { Raw: []byte(`"Cluster"`), }, @@ -1167,7 +1156,7 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, "aws_or_gcp": { Type: "object", - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "additional_secret_mount": { Type: "string", }, @@ -1190,7 +1179,7 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, "logical_backup": { Type: "object", - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "logical_backup_docker_image": { Type: "string", }, @@ -1220,7 +1209,7 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, "debug": { Type: "object", - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "debug_logging": { Type: "boolean", }, @@ -1231,7 +1220,7 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, "teams_api": { Type: "object", - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "enable_admin_role_for_users": { Type: "boolean", }, @@ -1255,16 +1244,16 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, "postgres_superuser_teams": { Type: "array", - Items: &apiextv1beta1.JSONSchemaPropsOrArray{ - Schema: &apiextv1beta1.JSONSchemaProps{ + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ Type: "string", }, }, }, "protected_role_names": { Type: "array", - Items: &apiextv1beta1.JSONSchemaPropsOrArray{ - Schema: &apiextv1beta1.JSONSchemaProps{ + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ Type: "string", }, }, @@ -1274,8 +1263,8 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, "team_api_role_configuration": { Type: "object", - AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ - Schema: &apiextv1beta1.JSONSchemaProps{ + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ + Schema: &apiextv1.JSONSchemaProps{ Type: "string", }, }, @@ -1287,7 +1276,7 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, "logging_rest_api": { Type: "object", - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "api_port": { Type: "integer", }, @@ -1301,7 +1290,7 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, "scalyr": { Type: "object", - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "scalyr_api_key": { Type: "string", }, @@ -1331,7 +1320,7 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, "connection_pooler": { Type: "object", - Properties: map[string]apiextv1beta1.JSONSchemaProps{ + Properties: map[string]apiextv1.JSONSchemaProps{ "connection_pooler_default_cpu_limit": { Type: "string", Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", @@ -1356,7 +1345,7 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, "connection_pooler_mode": { Type: "string", - Enum: []apiextv1beta1.JSON{ + Enum: []apiextv1.JSON{ { Raw: []byte(`"session"`), }, @@ -1381,8 +1370,8 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, "status": { Type: "object", - AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ - Schema: &apiextv1beta1.JSONSchemaProps{ + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ + Schema: &apiextv1.JSONSchemaProps{ Type: "string", }, }, @@ -1391,32 +1380,38 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, } -func buildCRD(name, kind, plural, short string, columns []apiextv1beta1.CustomResourceColumnDefinition, validation apiextv1beta1.CustomResourceValidation) *apiextv1beta1.CustomResourceDefinition { - return &apiextv1beta1.CustomResourceDefinition{ +func buildCRD(name, kind, plural, short string, columns []apiextv1.CustomResourceColumnDefinition, validation apiextv1.CustomResourceValidation) *apiextv1.CustomResourceDefinition { + return &apiextv1.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{ Name: name, }, - Spec: apiextv1beta1.CustomResourceDefinitionSpec{ - Group: SchemeGroupVersion.Group, - Version: SchemeGroupVersion.Version, - Names: apiextv1beta1.CustomResourceDefinitionNames{ + Spec: apiextv1.CustomResourceDefinitionSpec{ + Group: SchemeGroupVersion.Group, + Names: apiextv1.CustomResourceDefinitionNames{ Plural: plural, ShortNames: []string{short}, Kind: kind, }, - Scope: apiextv1beta1.NamespaceScoped, - Subresources: &apiextv1beta1.CustomResourceSubresources{ - Status: &apiextv1beta1.CustomResourceSubresourceStatus{}, + Scope: apiextv1.NamespaceScoped, + Versions: []apiextv1.CustomResourceDefinitionVersion{ + apiextv1.CustomResourceDefinitionVersion{ + Name: SchemeGroupVersion.Version, + Served: true, + Storage: true, + Subresources: &apiextv1.CustomResourceSubresources{ + Status: &apiextv1.CustomResourceSubresourceStatus{}, + }, + AdditionalPrinterColumns: columns, + Schema: &validation, + }, }, - AdditionalPrinterColumns: columns, - Validation: &validation, }, } } // PostgresCRD returns CustomResourceDefinition built from PostgresCRDResource -func PostgresCRD(enableValidation *bool) *apiextv1beta1.CustomResourceDefinition { - postgresCRDvalidation := apiextv1beta1.CustomResourceValidation{} +func PostgresCRD(enableValidation *bool) *apiextv1.CustomResourceDefinition { + postgresCRDvalidation := apiextv1.CustomResourceValidation{} if enableValidation != nil && *enableValidation { postgresCRDvalidation = PostgresCRDResourceValidation @@ -1431,8 +1426,8 @@ func PostgresCRD(enableValidation *bool) *apiextv1beta1.CustomResourceDefinition } // ConfigurationCRD returns CustomResourceDefinition built from OperatorConfigCRDResource -func ConfigurationCRD(enableValidation *bool) *apiextv1beta1.CustomResourceDefinition { - opconfigCRDvalidation := apiextv1beta1.CustomResourceValidation{} +func ConfigurationCRD(enableValidation *bool) *apiextv1.CustomResourceDefinition { + opconfigCRDvalidation := apiextv1.CustomResourceValidation{} if enableValidation != nil && *enableValidation { opconfigCRDvalidation = OperatorConfigCRDResourceValidation diff --git a/pkg/controller/util.go b/pkg/controller/util.go index 57196d371..2adc0bea1 100644 --- a/pkg/controller/util.go +++ b/pkg/controller/util.go @@ -7,7 +7,7 @@ import ( "strings" v1 "k8s.io/api/core/v1" - apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" @@ -54,7 +54,7 @@ func (c *Controller) clusterWorkerID(clusterName spec.NamespacedName) uint32 { return c.clusterWorkers[clusterName] } -func (c *Controller) createOperatorCRD(crd *apiextv1beta1.CustomResourceDefinition) error { +func (c *Controller) createOperatorCRD(crd *apiextv1.CustomResourceDefinition) error { if _, err := c.KubeClient.CustomResourceDefinitions().Create(context.TODO(), crd, metav1.CreateOptions{}); err != nil { if k8sutil.ResourceAlreadyExists(err) { c.logger.Infof("customResourceDefinition %q is already registered and will only be updated", crd.Name) @@ -82,12 +82,12 @@ func (c *Controller) createOperatorCRD(crd *apiextv1beta1.CustomResourceDefiniti for _, cond := range c.Status.Conditions { switch cond.Type { - case apiextv1beta1.Established: - if cond.Status == apiextv1beta1.ConditionTrue { + case apiextv1.Established: + if cond.Status == apiextv1.ConditionTrue { return true, err } - case apiextv1beta1.NamesAccepted: - if cond.Status == apiextv1beta1.ConditionFalse { + case apiextv1.NamesAccepted: + if cond.Status == apiextv1.ConditionFalse { return false, fmt.Errorf("name conflict: %v", cond.Reason) } } diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index 1234ef74a..19f95d9f1 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -17,7 +17,7 @@ import ( v1 "k8s.io/api/core/v1" policybeta1 "k8s.io/api/policy/v1beta1" apiextclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" - apiextbeta1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" @@ -53,7 +53,7 @@ type KubernetesClient struct { appsv1.DeploymentsGetter rbacv1.RoleBindingsGetter policyv1beta1.PodDisruptionBudgetsGetter - apiextbeta1.CustomResourceDefinitionsGetter + apiextv1.CustomResourceDefinitionsGetter clientbatchv1beta1.CronJobsGetter RESTClient rest.Interface @@ -153,7 +153,7 @@ func NewFromConfig(cfg *rest.Config) (KubernetesClient, error) { return kubeClient, fmt.Errorf("could not create api client:%v", err) } - kubeClient.CustomResourceDefinitionsGetter = apiextClient.ApiextensionsV1beta1() + kubeClient.CustomResourceDefinitionsGetter = apiextClient.ApiextensionsV1() kubeClient.AcidV1ClientSet = acidv1client.NewForConfigOrDie(cfg) return kubeClient, nil From 4f3bb6aa8cbecfbc68cfad1fee2a4c5c2b5d6632 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Mon, 2 Nov 2020 16:49:29 +0100 Subject: [PATCH 109/168] Remove operator checks that prevent PG major version upgrade (#1160) * remove checks that prevent major version upgrade Co-authored-by: Sergey Dudoladov --- docs/administrator.md | 12 +++-- docs/user.md | 4 ++ pkg/cluster/cluster.go | 7 ++- pkg/cluster/k8sres.go | 35 --------------- pkg/cluster/k8sres_test.go | 89 -------------------------------------- pkg/cluster/sync.go | 12 ----- 6 files changed, 14 insertions(+), 145 deletions(-) diff --git a/docs/administrator.md b/docs/administrator.md index 5357ddb74..5c1831cfe 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -11,17 +11,15 @@ switchover (planned failover) of the master to the Pod with new minor version. The switch should usually take less than 5 seconds, still clients have to reconnect. -Major version upgrades are supported via [cloning](user.md#how-to-clone-an-existing-postgresql-cluster). -The new cluster manifest must have a higher `version` string than the source +Major version upgrades are supported either via [cloning](user.md#how-to-clone-an-existing-postgresql-cluster)or in-place. + +With cloning, the new cluster manifest must have a higher `version` string than the source cluster and will be created from a basebackup. Depending of the cluster size, downtime in this case can be significant as writes to the database should be stopped and all WAL files should be archived first before cloning is started. -Note, that simply changing the version string in the `postgresql` manifest does -not work at present and leads to errors. Neither Patroni nor Postgres Operator -can do in place `pg_upgrade`. Still, it can be executed manually in the Postgres -container, which is tricky (i.e. systems need to be stopped, replicas have to be -synced) but of course faster than cloning. +Starting with Spilo 13, Postgres Operator can do in-place major version upgrade, which should be faster than cloning. To trigger the upgrade, simply increase the version in the cluster manifest. As the very last step of +processing the manifest update event, the operator will call the `inplace_upgrade.py` script in Spilo. The upgrade is usually fast, well under one minute for most DBs. Note the changes become irrevertible once `pg_upgrade` is called. To understand the upgrade procedure, refer to the [corresponding PR in Spilo](https://github.com/zalando/spilo/pull/488). ## CRD Validation diff --git a/docs/user.md b/docs/user.md index 8cacad0e8..f834c788a 100644 --- a/docs/user.md +++ b/docs/user.md @@ -542,6 +542,10 @@ section in the spec. There are two options here: Note, that cloning can also be used for [major version upgrades](administrator.md#minor-and-major-version-upgrade) of PostgreSQL. +## In-place major version upgrade + +Starting with Spilo 13, operator supports in-place major version upgrade to a higher major version (e.g. from PG 10 to PG 12). To trigger the upgrade, simply increase the version in the manifest. It is your responsibility to test your applications against the new version before the upgrade; downgrading is not supported. The easiest way to do so is to try the upgrade on the cloned cluster first. For details of how Spilo does the upgrade [see here](https://github.com/zalando/spilo/pull/488), operator implementation is described [in the admin docs](administrator.md#minor-and-major-version-upgrade). + ### Clone from S3 Cloning from S3 has the advantage that there is no impact on your production diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 06388d731..18778fd41 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -626,13 +626,16 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { } }() - if oldSpec.Spec.PostgresqlParam.PgVersion != newSpec.Spec.PostgresqlParam.PgVersion { // PG versions comparison + if oldSpec.Spec.PostgresqlParam.PgVersion >= newSpec.Spec.PostgresqlParam.PgVersion { c.logger.Warningf("postgresql version change(%q -> %q) has no effect", oldSpec.Spec.PostgresqlParam.PgVersion, newSpec.Spec.PostgresqlParam.PgVersion) c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "PostgreSQL", "postgresql version change(%q -> %q) has no effect", oldSpec.Spec.PostgresqlParam.PgVersion, newSpec.Spec.PostgresqlParam.PgVersion) - //we need that hack to generate statefulset with the old version + // we need that hack to generate statefulset with the old version newSpec.Spec.PostgresqlParam.PgVersion = oldSpec.Spec.PostgresqlParam.PgVersion + } else { + c.logger.Infof("postgresql version increased (%q -> %q), major version upgrade can be done manually after StatefulSet Sync", + oldSpec.Spec.PostgresqlParam.PgVersion, newSpec.Spec.PostgresqlParam.PgVersion) } // Service diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 9a328b7df..c07c1d212 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -909,41 +909,6 @@ func extractPgVersionFromBinPath(binPath string, template string) (string, error return fmt.Sprintf("%v", pgVersion), nil } -func (c *Cluster) getNewPgVersion(container v1.Container, newPgVersion string) (string, error) { - var ( - spiloConfiguration spiloConfiguration - runningPgVersion string - err error - ) - - for _, env := range container.Env { - if env.Name != "SPILO_CONFIGURATION" { - continue - } - err = json.Unmarshal([]byte(env.Value), &spiloConfiguration) - if err != nil { - return newPgVersion, err - } - } - - if len(spiloConfiguration.PgLocalConfiguration) > 0 { - currentBinPath := fmt.Sprintf("%v", spiloConfiguration.PgLocalConfiguration[patroniPGBinariesParameterName]) - runningPgVersion, err = extractPgVersionFromBinPath(currentBinPath, pgBinariesLocationTemplate) - if err != nil { - return "", fmt.Errorf("could not extract Postgres version from %v in SPILO_CONFIGURATION", currentBinPath) - } - } else { - return "", fmt.Errorf("could not find %q setting in SPILO_CONFIGURATION", patroniPGBinariesParameterName) - } - - if runningPgVersion != newPgVersion { - c.logger.Warningf("postgresql version change(%q -> %q) has no effect", runningPgVersion, newPgVersion) - newPgVersion = runningPgVersion - } - - return newPgVersion, nil -} - func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.StatefulSet, error) { var ( diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index f44b071bb..f1c0e968b 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -557,95 +557,6 @@ func TestExtractPgVersionFromBinPath(t *testing.T) { } } -func TestGetPgVersion(t *testing.T) { - testName := "TestGetPgVersion" - tests := []struct { - subTest string - pgContainer v1.Container - currentPgVersion string - newPgVersion string - }{ - { - subTest: "new version with decimal point differs from current SPILO_CONFIGURATION", - pgContainer: v1.Container{ - Name: "postgres", - Env: []v1.EnvVar{ - { - Name: "SPILO_CONFIGURATION", - Value: "{\"postgresql\": {\"bin_dir\": \"/usr/lib/postgresql/9.6/bin\"}}", - }, - }, - }, - currentPgVersion: "9.6", - newPgVersion: "12", - }, - { - subTest: "new version differs from current SPILO_CONFIGURATION", - pgContainer: v1.Container{ - Name: "postgres", - Env: []v1.EnvVar{ - { - Name: "SPILO_CONFIGURATION", - Value: "{\"postgresql\": {\"bin_dir\": \"/usr/lib/postgresql/11/bin\"}}", - }, - }, - }, - currentPgVersion: "11", - newPgVersion: "12", - }, - { - subTest: "new version is lower than the one found in current SPILO_CONFIGURATION", - pgContainer: v1.Container{ - Name: "postgres", - Env: []v1.EnvVar{ - { - Name: "SPILO_CONFIGURATION", - Value: "{\"postgresql\": {\"bin_dir\": \"/usr/lib/postgresql/12/bin\"}}", - }, - }, - }, - currentPgVersion: "12", - newPgVersion: "11", - }, - { - subTest: "new version is the same like in the current SPILO_CONFIGURATION", - pgContainer: v1.Container{ - Name: "postgres", - Env: []v1.EnvVar{ - { - Name: "SPILO_CONFIGURATION", - Value: "{\"postgresql\": {\"bin_dir\": \"/usr/lib/postgresql/12/bin\"}}", - }, - }, - }, - currentPgVersion: "12", - newPgVersion: "12", - }, - } - - var cluster = New( - Config{ - OpConfig: config.Config{ - ProtectedRoles: []string{"admin"}, - Auth: config.Auth{ - SuperUsername: superUserName, - ReplicationUsername: replicationUserName, - }, - }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) - - for _, tt := range tests { - pgVersion, err := cluster.getNewPgVersion(tt.pgContainer, tt.newPgVersion) - if err != nil { - t.Errorf("Unexpected error: %v", err) - } - if pgVersion != tt.currentPgVersion { - t.Errorf("%s %s: Expected version %s, have %s instead", - testName, tt.subTest, tt.currentPgVersion, pgVersion) - } - } -} - func TestSecretVolume(t *testing.T) { testName := "TestSecretVolume" tests := []struct { diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index dced69461..ee7672a5b 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -332,18 +332,6 @@ func (c *Cluster) syncStatefulSet() error { // statefulset is already there, make sure we use its definition in order to compare with the spec. c.Statefulset = sset - // check if there is no Postgres version mismatch - for _, container := range c.Statefulset.Spec.Template.Spec.Containers { - if container.Name != "postgres" { - continue - } - pgVersion, err := c.getNewPgVersion(container, c.Spec.PostgresqlParam.PgVersion) - if err != nil { - return fmt.Errorf("could not parse current Postgres version: %v", err) - } - c.Spec.PostgresqlParam.PgVersion = pgVersion - } - desiredSS, err := c.generateStatefulSet(&c.Spec) if err != nil { return fmt.Errorf("could not generate statefulset: %v", err) From db0d089e758aceb4435b77c028c651004a5e8b9f Mon Sep 17 00:00:00 2001 From: Pavel Tumik Date: Tue, 3 Nov 2020 06:05:44 -0800 Subject: [PATCH 110/168] Fix cloning from GCS (#1176) * Fix clone from gcs * pass google credentials env var if using GS bucket * remove requirement for timezone as GCS returns timestamp in local time to the region it is in * Revert "remove requirement for timezone as GCS returns timestamp in local time to the region it is in" This reverts commit ac4eb350d94a8272dd3c8a0efb49f3d0cf5d20fd. * update GCS documentation * remove sentence about logical backups * reword pod environment configmap section * fix documentation --- docs/administrator.md | 26 ++++++++++++++++++++++++++ pkg/cluster/k8sres.go | 28 ++++++++++++++++++++++++---- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/docs/administrator.md b/docs/administrator.md index 5c1831cfe..64bb10b68 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -686,6 +686,32 @@ aws_or_gcp: ... ``` +### Setup pod environment configmap + +To make postgres-operator work with GCS, use following configmap: +```yml +apiVersion: v1 +kind: ConfigMap +metadata: + name: pod-env-overrides + namespace: postgres-operator-system +data: + # Any env variable used by spilo can be added + USE_WALG_BACKUP: "true" + USE_WALG_RESTORE: "true" + CLONE_USE_WALG_RESTORE: "true" +``` +This configmap will instruct operator to use WAL-G, instead of WAL-E, for backup and restore. + +Then provide this configmap in postgres-operator settings: +```yml +... +# namespaced name of the ConfigMap with environment variables to populate on every pod +pod_environment_configmap: "postgres-operator-system/pod-env-overrides" +... +``` + + ## Sidecars for Postgres clusters A list of sidecars is added to each cluster created by the operator. The default diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index c07c1d212..2ca4ad4a8 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1719,11 +1719,31 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription) msg := "Figure out which S3 bucket to use from env" c.logger.Info(msg, description.S3WalPath) + if c.OpConfig.WALES3Bucket != "" { + envs := []v1.EnvVar{ + { + Name: "CLONE_WAL_S3_BUCKET", + Value: c.OpConfig.WALES3Bucket, + }, + } + result = append(result, envs...) + } else if c.OpConfig.WALGSBucket != "" { + envs := []v1.EnvVar{ + { + Name: "CLONE_WAL_GS_BUCKET", + Value: c.OpConfig.WALGSBucket, + }, + { + Name: "CLONE_GOOGLE_APPLICATION_CREDENTIALS", + Value: c.OpConfig.GCPCredentials, + }, + } + result = append(result, envs...) + } else { + c.logger.Error("Cannot figure out S3 or GS bucket. Both are empty.") + } + envs := []v1.EnvVar{ - { - Name: "CLONE_WAL_S3_BUCKET", - Value: c.OpConfig.WALES3Bucket, - }, { Name: "CLONE_WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(description.UID), From 90799d7e7f4a7b886ebafc18c8ac386ab2adab5c Mon Sep 17 00:00:00 2001 From: Jan Mussler Date: Wed, 4 Nov 2020 16:40:15 +0100 Subject: [PATCH 111/168] More output from test watch script. All namespaces and deployments. (#1193) --- e2e/scripts/watch_objects.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/e2e/scripts/watch_objects.sh b/e2e/scripts/watch_objects.sh index 9dde54f5e..dbd98ffc6 100755 --- a/e2e/scripts/watch_objects.sh +++ b/e2e/scripts/watch_objects.sh @@ -1,17 +1,20 @@ #!/bin/bash watch -c " -kubectl get postgresql +kubectl get postgresql --all-namespaces echo echo -n 'Rolling upgrade pending: ' kubectl get statefulset -o jsonpath='{.items..metadata.annotations.zalando-postgres-operator-rolling-update-required}' echo echo -kubectl get pods -o wide +echo 'Pods' +kubectl get pods -l application=spilo -l name=postgres-operator -l application=db-connection-pooler -o wide --all-namespaces echo -kubectl get statefulsets +echo 'Statefulsets' +kubectl get statefulsets --all-namespaces echo -kubectl get deployments +echo 'Deployments' +kubectl get deployments --all-namespaces -l application=db-connection-pooler -l name=postgres-operator echo echo echo 'Step from operator deployment' From 9a824c38f46c1769a771b164a38790111cc6bde2 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 5 Nov 2020 11:49:24 +0100 Subject: [PATCH 112/168] fix identation in operatorconfiguration CRD and jsonPath case (#1195) * fix identation in operatorconfiguration CRD * fix jsonPath field case --- .../crds/operatorconfigurations.yaml | 28 +++++++++---------- .../postgres-operator/crds/postgresqls.yaml | 16 +++++------ manifests/operatorconfiguration.crd.yaml | 28 +++++++++---------- manifests/postgresql.crd.yaml | 16 +++++------ 4 files changed, 44 insertions(+), 44 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 59671dc19..28b8f28ca 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -26,22 +26,22 @@ spec: - name: Image type: string description: Spilo image to be used for Pods - JSONPath: .configuration.docker_image + jsonPath: .configuration.docker_image - name: Cluster-Label type: string description: Label for K8s resources created by operator - JSONPath: .configuration.kubernetes.cluster_name_label + jsonPath: .configuration.kubernetes.cluster_name_label - name: Service-Account type: string description: Name of service account to be used - JSONPath: .configuration.kubernetes.pod_service_account_name + jsonPath: .configuration.kubernetes.pod_service_account_name - name: Min-Instances type: integer description: Minimum number of instances per Postgres cluster - JSONPath: .configuration.min_instances + jsonPath: .configuration.min_instances - name: Age type: date - JSONPath: .metadata.creationTimestamp + jsonPath: .metadata.creationTimestamp schema: openAPIV3Schema: type: object @@ -49,15 +49,15 @@ spec: - kind - apiVersion - configuration - properties: - kind: - type: string - enum: - - OperatorConfiguration - apiVersion: - type: string - enum: - - acid.zalan.do/v1 + properties: + kind: + type: string + enum: + - OperatorConfiguration + apiVersion: + type: string + enum: + - acid.zalan.do/v1 configuration: type: object properties: diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index dc7fe0d05..74c8f74b8 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -26,34 +26,34 @@ spec: - name: Team type: string description: Team responsible for Postgres CLuster - JSONPath: .spec.teamId + jsonPath: .spec.teamId - name: Version type: string description: PostgreSQL version - JSONPath: .spec.postgresql.version + jsonPath: .spec.postgresql.version - name: Pods type: integer description: Number of Pods per Postgres cluster - JSONPath: .spec.numberOfInstances + jsonPath: .spec.numberOfInstances - name: Volume type: string description: Size of the bound volume - JSONPath: .spec.volume.size + jsonPath: .spec.volume.size - name: CPU-Request type: string description: Requested CPU for Postgres containers - JSONPath: .spec.resources.requests.cpu + jsonPath: .spec.resources.requests.cpu - name: Memory-Request type: string description: Requested memory for Postgres containers - JSONPath: .spec.resources.requests.memory + jsonPath: .spec.resources.requests.memory - name: Age type: date - JSONPath: .metadata.creationTimestamp + jsonPath: .metadata.creationTimestamp - name: Status type: string description: Current sync status of postgresql resource - JSONPath: .status.PostgresClusterStatus + jsonPath: .status.PostgresClusterStatus schema: openAPIV3Schema: type: object diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 808e3acb0..0987467aa 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -22,22 +22,22 @@ spec: - name: Image type: string description: Spilo image to be used for Pods - JSONPath: .configuration.docker_image + jsonPath: .configuration.docker_image - name: Cluster-Label type: string description: Label for K8s resources created by operator - JSONPath: .configuration.kubernetes.cluster_name_label + jsonPath: .configuration.kubernetes.cluster_name_label - name: Service-Account type: string description: Name of service account to be used - JSONPath: .configuration.kubernetes.pod_service_account_name + jsonPath: .configuration.kubernetes.pod_service_account_name - name: Min-Instances type: integer description: Minimum number of instances per Postgres cluster - JSONPath: .configuration.min_instances + jsonPath: .configuration.min_instances - name: Age type: date - JSONPath: .metadata.creationTimestamp + jsonPath: .metadata.creationTimestamp schema: openAPIV3Schema: type: object @@ -45,15 +45,15 @@ spec: - kind - apiVersion - configuration - properties: - kind: - type: string - enum: - - OperatorConfiguration - apiVersion: - type: string - enum: - - acid.zalan.do/v1 + properties: + kind: + type: string + enum: + - OperatorConfiguration + apiVersion: + type: string + enum: + - acid.zalan.do/v1 configuration: type: object properties: diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index ffcf49056..16bdac564 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -22,34 +22,34 @@ spec: - name: Team type: string description: Team responsible for Postgres CLuster - JSONPath: .spec.teamId + jsonPath: .spec.teamId - name: Version type: string description: PostgreSQL version - JSONPath: .spec.postgresql.version + jsonPath: .spec.postgresql.version - name: Pods type: integer description: Number of Pods per Postgres cluster - JSONPath: .spec.numberOfInstances + jsonPath: .spec.numberOfInstances - name: Volume type: string description: Size of the bound volume - JSONPath: .spec.volume.size + jsonPath: .spec.volume.size - name: CPU-Request type: string description: Requested CPU for Postgres containers - JSONPath: .spec.resources.requests.cpu + jsonPath: .spec.resources.requests.cpu - name: Memory-Request type: string description: Requested memory for Postgres containers - JSONPath: .spec.resources.requests.memory + jsonPath: .spec.resources.requests.memory - name: Age type: date - JSONPath: .metadata.creationTimestamp + jsonPath: .metadata.creationTimestamp - name: Status type: string description: Current sync status of postgresql resource - JSONPath: .status.PostgresClusterStatus + jsonPath: .status.PostgresClusterStatus schema: openAPIV3Schema: type: object From b379db20edc05fa946c6cc50ec13161b80a89c5d Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 5 Nov 2020 12:04:51 +0100 Subject: [PATCH 113/168] fix redundant appending of infrastructure roles (#1192) --- pkg/controller/util.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/controller/util.go b/pkg/controller/util.go index 2adc0bea1..7f87de97d 100644 --- a/pkg/controller/util.go +++ b/pkg/controller/util.go @@ -341,9 +341,7 @@ func (c *Controller) getInfrastructureRole( util.Coalesce(string(secretData[infraRole.RoleKey]), infraRole.DefaultRoleValue)) } - if roleDescr.Valid() { - roles = append(roles, *roleDescr) - } else { + if !roleDescr.Valid() { msg := "infrastructure role %q is not complete and ignored" c.logger.Warningf(msg, roleDescr) From e779eab22fca8f805d9bf2e1735ecf5d2ccfec25 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Wed, 11 Nov 2020 10:21:46 +0100 Subject: [PATCH 114/168] Update e2e pipeline (#1202) * clean up after test_multi_namespace test * see the PR description for complete list of changes Co-authored-by: Sergey Dudoladov --- e2e/README.md | 30 +++++++++++++++++++++-- e2e/tests/k8s_api.py | 24 +++++++++---------- e2e/tests/test_e2e.py | 54 ++++++++++++++++++++++-------------------- pkg/cluster/cluster.go | 4 ++-- 4 files changed, 70 insertions(+), 42 deletions(-) diff --git a/e2e/README.md b/e2e/README.md index ce8931f62..3bba6ccc3 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -56,12 +56,24 @@ NOCLEANUP=True ./run.sh main tests.test_e2e.EndToEndTestCase.test_lazy_spilo_upg ## Inspecting Kind -If you want to inspect Kind/Kubernetes cluster, use the following script to exec into the K8s setup and then use `kubectl` +If you want to inspect Kind/Kubernetes cluster, switch `kubeconfig` file and context +```bash +# save the old config in case you have it +export KUBECONFIG_SAVED=$KUBECONFIG + +# use the one created by e2e tests +export KUBECONFIG=/tmp/kind-config-postgres-operator-e2e-tests + +# this kubeconfig defines a single context +kubectl config use-context kind-postgres-operator-e2e-tests +``` + +or use the following script to exec into the K8s setup and then use `kubectl` ```bash ./exec_into_env.sh -# use kube ctl +# use kubectl kubectl get pods # watch relevant objects @@ -71,6 +83,14 @@ kubectl get pods ./scripts/get_logs.sh ``` +If you want to inspect the state of the `kind` cluster manually with a single command, add a `context` flag +```bash +kubectl get pods --context kind-kind +``` +or set the context for a few commands at once + + + ## Cleaning up Kind To cleanup kind and start fresh @@ -79,6 +99,12 @@ To cleanup kind and start fresh e2e/run.sh cleanup ``` +That also helps in case you see the +``` +ERROR: no nodes found for cluster "postgres-operator-e2e-tests" +``` +that happens when the `kind` cluster was deleted manually but its configuraiton file was not. + ## Covered use cases The current tests are all bundled in [`test_e2e.py`](tests/test_e2e.py): diff --git a/e2e/tests/k8s_api.py b/e2e/tests/k8s_api.py index f2abd8e0c..93280dd53 100644 --- a/e2e/tests/k8s_api.py +++ b/e2e/tests/k8s_api.py @@ -11,9 +11,11 @@ from datetime import datetime from kubernetes import client, config from kubernetes.client.rest import ApiException + def to_selector(labels): return ",".join(["=".join(l) for l in labels.items()]) + class K8sApi: def __init__(self): @@ -181,10 +183,10 @@ class K8s: def count_pdbs_with_label(self, labels, namespace='default'): return len(self.api.policy_v1_beta1.list_namespaced_pod_disruption_budget( namespace, label_selector=labels).items) - + def count_running_pods(self, labels='application=spilo,cluster-name=acid-minimal-cluster', namespace='default'): pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items - return len(list(filter(lambda x: x.status.phase=='Running', pods))) + return len(list(filter(lambda x: x.status.phase == 'Running', pods))) def wait_for_pod_failover(self, failover_targets, labels, namespace='default'): pod_phase = 'Failing over' @@ -210,9 +212,9 @@ class K8s: def wait_for_logical_backup_job_creation(self): self.wait_for_logical_backup_job(expected_num_of_jobs=1) - def delete_operator_pod(self, step="Delete operator deplyment"): - operator_pod = self.api.core_v1.list_namespaced_pod('default', label_selector="name=postgres-operator").items[0].metadata.name - self.api.apps_v1.patch_namespaced_deployment("postgres-operator","default", {"spec":{"template":{"metadata":{"annotations":{"step":"{}-{}".format(step, time.time())}}}}}) + def delete_operator_pod(self, step="Delete operator pod"): + # patching the pod template in the deployment restarts the operator pod + self.api.apps_v1.patch_namespaced_deployment("postgres-operator","default", {"spec":{"template":{"metadata":{"annotations":{"step":"{}-{}".format(step, datetime.fromtimestamp(time.time()))}}}}}) self.wait_for_operator_pod_start() def update_config(self, config_map_patch, step="Updating operator deployment"): @@ -241,7 +243,7 @@ class K8s: def get_operator_state(self): pod = self.get_operator_pod() - if pod == None: + if pod is None: return None pod = pod.metadata.name @@ -251,7 +253,6 @@ class K8s: return json.loads(r.stdout.decode()) - def get_patroni_running_members(self, pod="acid-minimal-cluster-0"): result = self.get_patroni_state(pod) return list(filter(lambda x: "State" in x and x["State"] == "running", result)) @@ -260,9 +261,9 @@ class K8s: try: deployment = self.api.apps_v1.read_namespaced_deployment(name, namespace) return deployment.spec.replicas - except ApiException as e: + except ApiException: return None - + def get_statefulset_image(self, label_selector="application=spilo,cluster-name=acid-minimal-cluster", namespace='default'): ssets = self.api.apps_v1.list_namespaced_stateful_set(namespace, label_selector=label_selector, limit=1) if len(ssets.items) == 0: @@ -463,7 +464,6 @@ class K8sBase: self.wait_for_logical_backup_job(expected_num_of_jobs=1) def delete_operator_pod(self, step="Delete operator deplyment"): - operator_pod = self.api.core_v1.list_namespaced_pod('default', label_selector="name=postgres-operator").items[0].metadata.name self.api.apps_v1.patch_namespaced_deployment("postgres-operator","default", {"spec":{"template":{"metadata":{"annotations":{"step":"{}-{}".format(step, time.time())}}}}}) self.wait_for_operator_pod_start() @@ -521,7 +521,7 @@ class K8sOperator(K8sBase): class K8sPostgres(K8sBase): def __init__(self, labels="cluster-name=acid-minimal-cluster", namespace="default"): super().__init__(labels, namespace) - + def get_pg_nodes(self): master_pod_node = '' replica_pod_nodes = [] @@ -532,4 +532,4 @@ class K8sPostgres(K8sBase): elif pod.metadata.labels.get('spilo-role') == 'replica': replica_pod_nodes.append(pod.spec.node_name) - return master_pod_node, replica_pod_nodes \ No newline at end of file + return master_pod_node, replica_pod_nodes diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 05cc09a70..0fc60bf42 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -2,15 +2,14 @@ import json import unittest import time import timeout_decorator -import subprocess -import warnings import os import yaml from datetime import datetime -from kubernetes import client, config +from kubernetes import client from tests.k8s_api import K8s +from kubernetes.client.rest import ApiException SPILO_CURRENT = "registry.opensource.zalan.do/acid/spilo-12:1.6-p5" SPILO_LAZY = "registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p114" @@ -89,17 +88,17 @@ class EndToEndTestCase(unittest.TestCase): # remove existing local storage class and create hostpath class try: k8s.api.storage_v1_api.delete_storage_class("standard") - except: - print("Storage class has already been remove") + except ApiException as e: + print("Failed to delete the 'standard' storage class: {0}".format(e)) # operator deploys pod service account there on start up # needed for test_multi_namespace_support() - cls.namespace = "test" + cls.test_namespace = "test" try: - v1_namespace = client.V1Namespace(metadata=client.V1ObjectMeta(name=cls.namespace)) + v1_namespace = client.V1Namespace(metadata=client.V1ObjectMeta(name=cls.test_namespace)) k8s.api.core_v1.create_namespace(v1_namespace) - except: - print("Namespace already present") + except ApiException as e: + print("Failed to create the '{0}' namespace: {1}".format(cls.test_namespace, e)) # submit the most recent operator image built on the Docker host with open("manifests/postgres-operator.yaml", 'r+') as f: @@ -135,10 +134,8 @@ class EndToEndTestCase(unittest.TestCase): # make sure we start a new operator on every new run, # this tackles the problem when kind is reused - # and the Docker image is infact changed (dirty one) + # and the Docker image is in fact changed (dirty one) - # patch resync period, this can catch some problems with hanging e2e tests - # k8s.update_config({"data": {"resync_period":"30s"}},step="TestSuite setup") k8s.update_config({}, step="TestSuite Startup") actual_operator_image = k8s.api.core_v1.list_namespaced_pod( @@ -170,9 +167,6 @@ class EndToEndTestCase(unittest.TestCase): 'connection-pooler': 'acid-minimal-cluster-pooler', }) - pod_selector = to_selector(pod_labels) - service_selector = to_selector(service_labels) - # enable connection pooler k8s.api.custom_objects_api.patch_namespaced_custom_object( 'acid.zalan.do', 'v1', 'default', @@ -347,7 +341,7 @@ class EndToEndTestCase(unittest.TestCase): }, } k8s.update_config(patch_infrastructure_roles) - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0":"idle"}, "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") try: # check that new roles are represented in the config by requesting the @@ -604,17 +598,25 @@ class EndToEndTestCase(unittest.TestCase): with open("manifests/complete-postgres-manifest.yaml", 'r+') as f: pg_manifest = yaml.safe_load(f) - pg_manifest["metadata"]["namespace"] = self.namespace + pg_manifest["metadata"]["namespace"] = self.test_namespace yaml.dump(pg_manifest, f, Dumper=yaml.Dumper) try: k8s.create_with_kubectl("manifests/complete-postgres-manifest.yaml") - k8s.wait_for_pod_start("spilo-role=master", self.namespace) - self.assert_master_is_unique(self.namespace, "acid-test-cluster") + k8s.wait_for_pod_start("spilo-role=master", self.test_namespace) + self.assert_master_is_unique(self.test_namespace, "acid-test-cluster") except timeout_decorator.TimeoutError: print('Operator log: {}'.format(k8s.get_operator_log())) raise + finally: + # delete the new cluster so that the k8s_api.get_operator_state works correctly in subsequent tests + # ideally we should delete the 'test' namespace here but + # the pods inside the namespace stuck in the Terminating state making the test time out + k8s.api.custom_objects_api.delete_namespaced_custom_object( + "acid.zalan.do", "v1", self.test_namespace, "postgresqls", "acid-test-cluster") + time.sleep(5) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_zz_node_readiness_label(self): @@ -746,12 +748,12 @@ class EndToEndTestCase(unittest.TestCase): } k8s.api.custom_objects_api.patch_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_crd_annotations) - + annotations = { "deployment-time": "2020-04-30 12:00:00", "downscaler/downtime_replicas": "0", } - + self.eventuallyTrue(lambda: k8s.check_statefulset_annotations(cluster_label, annotations), "Annotations missing") @@ -823,14 +825,14 @@ class EndToEndTestCase(unittest.TestCase): } } k8s.update_config(patch_delete_annotations) - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0":"idle"}, "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") try: # this delete attempt should be omitted because of missing annotations k8s.api.custom_objects_api.delete_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster") time.sleep(5) - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0":"idle"}, "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") # check that pods and services are still there k8s.wait_for_running_pods(cluster_label, 2) @@ -841,7 +843,7 @@ class EndToEndTestCase(unittest.TestCase): # wait a little before proceeding time.sleep(10) - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0":"idle"}, "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") # add annotations to manifest delete_date = datetime.today().strftime('%Y-%m-%d') @@ -855,7 +857,7 @@ class EndToEndTestCase(unittest.TestCase): } k8s.api.custom_objects_api.patch_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_delete_annotations) - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0":"idle"}, "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") # wait a little before proceeding time.sleep(20) @@ -882,7 +884,7 @@ class EndToEndTestCase(unittest.TestCase): print('Operator log: {}'.format(k8s.get_operator_log())) raise - #reset configmap + # reset configmap patch_delete_annotations = { "data": { "delete_annotation_date_key": "", diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 18778fd41..1764ea726 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -626,14 +626,14 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { } }() - if oldSpec.Spec.PostgresqlParam.PgVersion >= newSpec.Spec.PostgresqlParam.PgVersion { + if oldSpec.Spec.PostgresqlParam.PgVersion > newSpec.Spec.PostgresqlParam.PgVersion { c.logger.Warningf("postgresql version change(%q -> %q) has no effect", oldSpec.Spec.PostgresqlParam.PgVersion, newSpec.Spec.PostgresqlParam.PgVersion) c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "PostgreSQL", "postgresql version change(%q -> %q) has no effect", oldSpec.Spec.PostgresqlParam.PgVersion, newSpec.Spec.PostgresqlParam.PgVersion) // we need that hack to generate statefulset with the old version newSpec.Spec.PostgresqlParam.PgVersion = oldSpec.Spec.PostgresqlParam.PgVersion - } else { + } else if oldSpec.Spec.PostgresqlParam.PgVersion < newSpec.Spec.PostgresqlParam.PgVersion { c.logger.Infof("postgresql version increased (%q -> %q), major version upgrade can be done manually after StatefulSet Sync", oldSpec.Spec.PostgresqlParam.PgVersion, newSpec.Spec.PostgresqlParam.PgVersion) } From 3fed565328c111cf8e878039d6ca4a3ea570abd1 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 11 Nov 2020 13:22:43 +0100 Subject: [PATCH 115/168] check resize mode on update events (#1194) * check resize mode on update events * add unit test for PVC resizing * set resize mode to pvc in charts and manifests * add test for quantityToGigabyte * just one debug line for syncing volumes * extend test and update log msg --- .../templates/clusterrole.yaml | 6 + charts/postgres-operator/values-crd.yaml | 2 +- charts/postgres-operator/values.yaml | 2 +- manifests/configmap.yaml | 2 +- manifests/operator-service-account-rbac.yaml | 2 + ...gresql-operator-default-configuration.yaml | 2 +- pkg/cluster/cluster.go | 20 +- pkg/cluster/sync.go | 3 +- pkg/cluster/volumes.go | 2 +- pkg/cluster/volumes_test.go | 171 ++++++++++++++++++ 10 files changed, 199 insertions(+), 13 deletions(-) create mode 100644 pkg/cluster/volumes_test.go diff --git a/charts/postgres-operator/templates/clusterrole.yaml b/charts/postgres-operator/templates/clusterrole.yaml index 84da313d9..00ee776f5 100644 --- a/charts/postgres-operator/templates/clusterrole.yaml +++ b/charts/postgres-operator/templates/clusterrole.yaml @@ -105,6 +105,10 @@ rules: - delete - get - list +{{- if toString .Values.configKubernetes.storage_resize_mode | eq "pvc" }} + - patch + - update +{{- end }} # to read existing PVs. Creation should be done via dynamic provisioning - apiGroups: - "" @@ -113,7 +117,9 @@ rules: verbs: - get - list +{{- if toString .Values.configKubernetes.storage_resize_mode | eq "ebs" }} - update # only for resizing AWS volumes +{{- end }} # to watch Spilo pods and do rolling updates. Creation via StatefulSet - apiGroups: - "" diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 6196c6fb2..71c2d5bb1 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -136,7 +136,7 @@ configKubernetes: # whether the Spilo container should run in privileged mode spilo_privileged: false # storage resize strategy, available options are: ebs, pvc, off - storage_resize_mode: ebs + storage_resize_mode: pvc # operator watches for postgres objects in the given namespace watched_namespace: "*" # listen to all namespaces diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 231b0b9ac..95865503d 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -130,7 +130,7 @@ configKubernetes: # whether the Spilo container should run in privileged mode spilo_privileged: "false" # storage resize strategy, available options are: ebs, pvc, off - storage_resize_mode: ebs + storage_resize_mode: pvc # operator watches for postgres objects in the given namespace watched_namespace: "*" # listen to all namespaces diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index e59bfcea0..59283fd6e 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -107,7 +107,7 @@ data: # spilo_runasgroup: 103 # spilo_fsgroup: 103 spilo_privileged: "false" - # storage_resize_mode: "off" + storage_resize_mode: "pvc" super_username: postgres # team_admin_role: "admin" # team_api_role_configuration: "log_statement:all" diff --git a/manifests/operator-service-account-rbac.yaml b/manifests/operator-service-account-rbac.yaml index 15ed7f53b..1ba5b4d23 100644 --- a/manifests/operator-service-account-rbac.yaml +++ b/manifests/operator-service-account-rbac.yaml @@ -106,6 +106,8 @@ rules: - delete - get - list + - patch + - update # to read existing PVs. Creation should be done via dynamic provisioning - apiGroups: - "" diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 14acc4356..84537e06a 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -72,7 +72,7 @@ configuration: # spilo_runasgroup: 103 # spilo_fsgroup: 103 spilo_privileged: false - storage_resize_mode: ebs + storage_resize_mode: pvc # toleration: {} # watched_namespace: "" postgres_pod_resources: diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 1764ea726..7ec7be176 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -140,7 +140,7 @@ func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgres Secrets: make(map[types.UID]*v1.Secret), Services: make(map[PostgresRole]*v1.Service), Endpoints: make(map[PostgresRole]*v1.Endpoints)}, - userSyncStrategy: users.DefaultUserSyncStrategy{password_encryption}, + userSyncStrategy: users.DefaultUserSyncStrategy{PasswordEncryption: password_encryption}, deleteOptions: metav1.DeleteOptions{PropagationPolicy: &deletePropagationPolicy}, podEventsQueue: podEventsQueue, KubeClient: kubeClient, @@ -671,13 +671,21 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { // Volume if oldSpec.Spec.Size != newSpec.Spec.Size { - c.logger.Debugf("syncing persistent volumes") c.logVolumeChanges(oldSpec.Spec.Volume, newSpec.Spec.Volume) - - if err := c.syncVolumes(); err != nil { - c.logger.Errorf("could not sync persistent volumes: %v", err) - updateFailed = true + c.logger.Debugf("syncing volumes using %q storage resize mode", c.OpConfig.StorageResizeMode) + if c.OpConfig.StorageResizeMode == "pvc" { + if err := c.syncVolumeClaims(); err != nil { + c.logger.Errorf("could not sync persistent volume claims: %v", err) + updateFailed = true + } + } else if c.OpConfig.StorageResizeMode == "ebs" { + if err := c.syncVolumes(); err != nil { + c.logger.Errorf("could not sync persistent volumes: %v", err) + updateFailed = true + } } + } else { + c.logger.Infof("Storage resize is disabled (storage_resize_mode is off). Skipping volume sync.") } // Statefulset diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index ee7672a5b..61be7919d 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -57,8 +57,8 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { return err } + c.logger.Debugf("syncing volumes using %q storage resize mode", c.OpConfig.StorageResizeMode) if c.OpConfig.StorageResizeMode == "pvc" { - c.logger.Debugf("syncing persistent volume claims") if err = c.syncVolumeClaims(); err != nil { err = fmt.Errorf("could not sync persistent volume claims: %v", err) return err @@ -70,7 +70,6 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { // TODO: handle the case of the cluster that is downsized and enlarged again // (there will be a volume from the old pod for which we can't act before the // the statefulset modification is concluded) - c.logger.Debugf("syncing persistent volumes") if err = c.syncVolumes(); err != nil { err = fmt.Errorf("could not sync persistent volumes: %v", err) return err diff --git a/pkg/cluster/volumes.go b/pkg/cluster/volumes.go index d5c08c2e2..44b85663f 100644 --- a/pkg/cluster/volumes.go +++ b/pkg/cluster/volumes.go @@ -62,7 +62,7 @@ func (c *Cluster) resizeVolumeClaims(newVolume acidv1.Volume) error { if err != nil { return fmt.Errorf("could not parse volume size: %v", err) } - _, newSize, err := c.listVolumesWithManifestSize(newVolume) + newSize := quantityToGigabyte(newQuantity) for _, pvc := range pvcs { volumeSize := quantityToGigabyte(pvc.Spec.Resources.Requests[v1.ResourceStorage]) if volumeSize >= newSize { diff --git a/pkg/cluster/volumes_test.go b/pkg/cluster/volumes_test.go new file mode 100644 index 000000000..49fbbd228 --- /dev/null +++ b/pkg/cluster/volumes_test.go @@ -0,0 +1,171 @@ +package cluster + +import ( + "testing" + + "context" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + + "github.com/stretchr/testify/assert" + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + "github.com/zalando/postgres-operator/pkg/util/config" + "github.com/zalando/postgres-operator/pkg/util/constants" + "github.com/zalando/postgres-operator/pkg/util/k8sutil" + "k8s.io/client-go/kubernetes/fake" +) + +func NewFakeKubernetesClient() (k8sutil.KubernetesClient, *fake.Clientset) { + clientSet := fake.NewSimpleClientset() + + return k8sutil.KubernetesClient{ + PersistentVolumeClaimsGetter: clientSet.CoreV1(), + }, clientSet +} + +func TestResizeVolumeClaim(t *testing.T) { + testName := "test resizing of persistent volume claims" + client, _ := NewFakeKubernetesClient() + clusterName := "acid-test-cluster" + namespace := "default" + newVolumeSize := "2Gi" + + // new cluster with pvc storage resize mode and configured labels + var cluster = New( + Config{ + OpConfig: config.Config{ + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: "cluster-name", + }, + StorageResizeMode: "pvc", + }, + }, client, acidv1.Postgresql{}, logger, eventRecorder) + + // set metadata, so that labels will get correct values + cluster.Name = clusterName + cluster.Namespace = namespace + filterLabels := cluster.labelsSet(false) + + // define and create PVCs for 1Gi volumes + storage1Gi, err := resource.ParseQuantity("1Gi") + assert.NoError(t, err) + + pvcList := &v1.PersistentVolumeClaimList{ + Items: []v1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: constants.DataVolumeName + "-" + clusterName + "-0", + Namespace: namespace, + Labels: filterLabels, + }, + Spec: v1.PersistentVolumeClaimSpec{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: storage1Gi, + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: constants.DataVolumeName + "-" + clusterName + "-1", + Namespace: namespace, + Labels: filterLabels, + }, + Spec: v1.PersistentVolumeClaimSpec{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: storage1Gi, + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: constants.DataVolumeName + "-" + clusterName + "-2-0", + Namespace: namespace, + Labels: labels.Set{}, + }, + Spec: v1.PersistentVolumeClaimSpec{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: storage1Gi, + }, + }, + }, + }, + }, + } + + for _, pvc := range pvcList.Items { + cluster.KubeClient.PersistentVolumeClaims(namespace).Create(context.TODO(), &pvc, metav1.CreateOptions{}) + } + + // test resizing + cluster.resizeVolumeClaims(acidv1.Volume{Size: newVolumeSize}) + + pvcs, err := cluster.listPersistentVolumeClaims() + assert.NoError(t, err) + + // check if listPersistentVolumeClaims returns only the PVCs matching the filter + if len(pvcs) != len(pvcList.Items)-1 { + t.Errorf("%s: could not find all PVCs, got %v, expected %v", testName, len(pvcs), len(pvcList.Items)-1) + } + + // check if PVCs were correctly resized + for _, pvc := range pvcs { + newStorageSize := quantityToGigabyte(pvc.Spec.Resources.Requests[v1.ResourceStorage]) + expectedQuantity, err := resource.ParseQuantity(newVolumeSize) + assert.NoError(t, err) + expectedSize := quantityToGigabyte(expectedQuantity) + if newStorageSize != expectedSize { + t.Errorf("%s: resizing failed, got %v, expected %v", testName, newStorageSize, expectedSize) + } + } + + // check if other PVC was not resized + pvc2, err := cluster.KubeClient.PersistentVolumeClaims(namespace).Get(context.TODO(), constants.DataVolumeName+"-"+clusterName+"-2-0", metav1.GetOptions{}) + assert.NoError(t, err) + unchangedSize := quantityToGigabyte(pvc2.Spec.Resources.Requests[v1.ResourceStorage]) + expectedSize := quantityToGigabyte(storage1Gi) + if unchangedSize != expectedSize { + t.Errorf("%s: volume size changed, got %v, expected %v", testName, unchangedSize, expectedSize) + } +} + +func TestQuantityToGigabyte(t *testing.T) { + tests := []struct { + name string + quantityStr string + expected int64 + }{ + { + "test with 1Gi", + "1Gi", + 1, + }, + { + "test with float", + "1.5Gi", + int64(1), + }, + { + "test with 1000Mi", + "1000Mi", + int64(0), + }, + } + + for _, tt := range tests { + quantity, err := resource.ParseQuantity(tt.quantityStr) + assert.NoError(t, err) + gigabyte := quantityToGigabyte(quantity) + if gigabyte != tt.expected { + t.Errorf("%s: got %v, expected %v", tt.name, gigabyte, tt.expected) + } + } +} From 49158ecb6843181dd984b6ff4f4390c068bec062 Mon Sep 17 00:00:00 2001 From: Rafia Sabih Date: Fri, 13 Nov 2020 14:52:21 +0100 Subject: [PATCH 116/168] Connection pooler for replica (#1127) * Enable connection pooler for replica * Refactor code for connection pooler - Move all the relevant code to a separate file - Move all the related tests to a separate file - Avoid using cluster where not required - Simplify the logic in sync and other methods - Cleanup of duplicated or unused code * Fix labels for the replica pods * Update deleteConnectionPooler to include role * Adding test cases and other changes - Fix unit test and delete secret when required only - Make sure we use empty fresh cluster for every test case. * enhance e2e test * Disable pooler in complete manifest as this is source for e2e too an creates unnecessary pooler setups. Co-authored-by: Rafia Sabih Co-authored-by: Jan Mussler --- .../postgres-operator/crds/postgresqls.yaml | 4 +- docs/reference/cluster_manifest.md | 19 +- docs/user.md | 8 +- e2e/scripts/watch_objects.sh | 4 +- e2e/tests/k8s_api.py | 43 +- e2e/tests/test_e2e.py | 239 +++-- manifests/complete-postgres-manifest.yaml | 28 +- manifests/postgresql.crd.yaml | 2 + pkg/apis/acid.zalan.do/v1/crds.go | 3 + pkg/apis/acid.zalan.do/v1/postgresql_type.go | 5 +- pkg/cluster/cluster.go | 183 +--- pkg/cluster/connection_pooler.go | 905 +++++++++++++++++ pkg/cluster/connection_pooler_new_test.go | 45 + pkg/cluster/connection_pooler_test.go | 956 ++++++++++++++++++ pkg/cluster/database.go | 7 +- pkg/cluster/k8sres.go | 306 ------ pkg/cluster/k8sres_test.go | 318 +----- pkg/cluster/resources.go | 198 ---- pkg/cluster/resources_test.go | 127 --- pkg/cluster/sync.go | 206 +--- pkg/cluster/sync_test.go | 264 ----- pkg/cluster/types.go | 2 +- pkg/cluster/util.go | 34 - pkg/controller/postgresql.go | 4 +- ui/operator_ui/main.py | 11 + 25 files changed, 2185 insertions(+), 1736 deletions(-) create mode 100644 pkg/cluster/connection_pooler.go create mode 100644 pkg/cluster/connection_pooler_new_test.go create mode 100644 pkg/cluster/connection_pooler_test.go delete mode 100644 pkg/cluster/resources_test.go delete mode 100644 pkg/cluster/sync_test.go diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 74c8f74b8..9127fa86e 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -190,6 +190,8 @@ spec: type: string enableConnectionPooler: type: boolean + enableReplicaConnectionPooler: + type: boolean enableLogicalBackup: type: boolean enableMasterLoadBalancer: @@ -603,4 +605,4 @@ spec: status: type: object additionalProperties: - type: string + type: string \ No newline at end of file diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 70ab14855..f7ddb6ff1 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -151,10 +151,15 @@ These parameters are grouped directly under the `spec` key in the manifest. configured (so you can override the operator configuration). Optional. * **enableConnectionPooler** - Tells the operator to create a connection pooler with a database. If this - field is true, a connection pooler deployment will be created even if + Tells the operator to create a connection pooler with a database for the master + service. If this field is true, a connection pooler deployment will be created even if `connectionPooler` section is empty. Optional, not set by default. +* **enableReplicaConnectionPooler** + Tells the operator to create a connection pooler with a database for the replica + service. If this field is true, a connection pooler deployment for replica + will be created even if `connectionPooler` section is empty. Optional, not set by default. + * **enableLogicalBackup** Determines if the logical backup of this cluster should be taken and uploaded to S3. Default: false. Optional. @@ -241,10 +246,10 @@ explanation of `ttl` and `loop_wait` parameters. * **synchronous_mode** Patroni `synchronous_mode` parameter value. The default is set to `false`. Optional. - + * **synchronous_mode_strict** Patroni `synchronous_mode_strict` parameter value. Can be used in addition to `synchronous_mode`. The default is set to `false`. Optional. - + ## Postgres container resources Those parameters define [CPU and memory requests and limits](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) @@ -397,8 +402,10 @@ CPU and memory limits for the sidecar container. Parameters are grouped under the `connectionPooler` top-level key and specify configuration for connection pooler. If this section is not empty, a connection -pooler will be created for a database even if `enableConnectionPooler` is not -present. +pooler will be created for master service only even if `enableConnectionPooler` +is not present. But if this section is present then it defines the configuration +for both master and replica pooler services (if `enableReplicaConnectionPooler` + is enabled). * **numberOfInstances** How many instances of connection pooler to create. diff --git a/docs/user.md b/docs/user.md index f834c788a..8723a01e4 100644 --- a/docs/user.md +++ b/docs/user.md @@ -807,11 +807,17 @@ manifest: ```yaml spec: enableConnectionPooler: true + enableReplicaConnectionPooler: true ``` This will tell the operator to create a connection pooler with default configuration, through which one can access the master via a separate service -`{cluster-name}-pooler`. In most of the cases the +`{cluster-name}-pooler`. With the first option, connection pooler for master service +is created and with the second option, connection pooler for replica is created. +Note that both of these flags are independent of each other and user can set or +unset any of them as per their requirements without any effect on the other. + +In most of the cases the [default configuration](reference/operator_parameters.md#connection-pooler-configuration) should be good enough. To configure a new connection pooler individually for each Postgres cluster, specify: diff --git a/e2e/scripts/watch_objects.sh b/e2e/scripts/watch_objects.sh index dbd98ffc6..52364f247 100755 --- a/e2e/scripts/watch_objects.sh +++ b/e2e/scripts/watch_objects.sh @@ -8,7 +8,9 @@ kubectl get statefulset -o jsonpath='{.items..metadata.annotations.zalando-postg echo echo echo 'Pods' -kubectl get pods -l application=spilo -l name=postgres-operator -l application=db-connection-pooler -o wide --all-namespaces +kubectl get pods -l application=spilo -o wide --all-namespaces +echo +kubectl get pods -l application=db-connection-pooler -o wide --all-namespaces echo echo 'Statefulsets' kubectl get statefulsets --all-namespaces diff --git a/e2e/tests/k8s_api.py b/e2e/tests/k8s_api.py index 93280dd53..30165e6a0 100644 --- a/e2e/tests/k8s_api.py +++ b/e2e/tests/k8s_api.py @@ -1,19 +1,14 @@ import json -import unittest import time -import timeout_decorator import subprocess import warnings -import os -import yaml -from datetime import datetime from kubernetes import client, config from kubernetes.client.rest import ApiException def to_selector(labels): - return ",".join(["=".join(l) for l in labels.items()]) + return ",".join(["=".join(lbl) for lbl in labels.items()]) class K8sApi: @@ -43,8 +38,8 @@ class K8s: def __init__(self, labels='x=y', namespace='default'): self.api = K8sApi() - self.labels=labels - self.namespace=namespace + self.labels = labels + self.namespace = namespace def get_pg_nodes(self, pg_cluster_name, namespace='default'): master_pod_node = '' @@ -81,7 +76,7 @@ class K8s: 'default', label_selector='name=postgres-operator' ).items - pods = list(filter(lambda x: x.status.phase=='Running', pods)) + pods = list(filter(lambda x: x.status.phase == 'Running', pods)) if len(pods): return pods[0] @@ -110,7 +105,6 @@ class K8s: time.sleep(self.RETRY_TIMEOUT_SEC) - def get_service_type(self, svc_labels, namespace='default'): svc_type = '' svcs = self.api.core_v1.list_namespaced_service(namespace, label_selector=svc_labels, limit=1).items @@ -213,8 +207,8 @@ class K8s: self.wait_for_logical_backup_job(expected_num_of_jobs=1) def delete_operator_pod(self, step="Delete operator pod"): - # patching the pod template in the deployment restarts the operator pod - self.api.apps_v1.patch_namespaced_deployment("postgres-operator","default", {"spec":{"template":{"metadata":{"annotations":{"step":"{}-{}".format(step, datetime.fromtimestamp(time.time()))}}}}}) + # patching the pod template in the deployment restarts the operator pod + self.api.apps_v1.patch_namespaced_deployment("postgres-operator", "default", {"spec": {"template": {"metadata": {"annotations": {"step": "{}-{}".format(step, time.time())}}}}}) self.wait_for_operator_pod_start() def update_config(self, config_map_patch, step="Updating operator deployment"): @@ -237,7 +231,7 @@ class K8s: def get_patroni_state(self, pod): r = self.exec_with_kubectl(pod, "patronictl list -f json") - if not r.returncode == 0 or not r.stdout.decode()[0:1]=="[": + if not r.returncode == 0 or not r.stdout.decode()[0:1] == "[": return [] return json.loads(r.stdout.decode()) @@ -248,7 +242,7 @@ class K8s: pod = pod.metadata.name r = self.exec_with_kubectl(pod, "curl localhost:8080/workers/all/status/") - if not r.returncode == 0 or not r.stdout.decode()[0:1]=="{": + if not r.returncode == 0 or not r.stdout.decode()[0:1] == "{": return None return json.loads(r.stdout.decode()) @@ -277,7 +271,7 @@ class K8s: ''' pod = self.api.core_v1.list_namespaced_pod( namespace, label_selector="statefulset.kubernetes.io/pod-name=" + pod_name) - + if len(pod.items) == 0: return None return pod.items[0].spec.containers[0].image @@ -305,8 +299,8 @@ class K8sBase: def __init__(self, labels='x=y', namespace='default'): self.api = K8sApi() - self.labels=labels - self.namespace=namespace + self.labels = labels + self.namespace = namespace def get_pg_nodes(self, pg_cluster_labels='cluster-name=acid-minimal-cluster', namespace='default'): master_pod_node = '' @@ -434,10 +428,10 @@ class K8sBase: def count_pdbs_with_label(self, labels, namespace='default'): return len(self.api.policy_v1_beta1.list_namespaced_pod_disruption_budget( namespace, label_selector=labels).items) - + def count_running_pods(self, labels='application=spilo,cluster-name=acid-minimal-cluster', namespace='default'): pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items - return len(list(filter(lambda x: x.status.phase=='Running', pods))) + return len(list(filter(lambda x: x.status.phase == 'Running', pods))) def wait_for_pod_failover(self, failover_targets, labels, namespace='default'): pod_phase = 'Failing over' @@ -484,14 +478,14 @@ class K8sBase: def get_patroni_state(self, pod): r = self.exec_with_kubectl(pod, "patronictl list -f json") - if not r.returncode == 0 or not r.stdout.decode()[0:1]=="[": + if not r.returncode == 0 or not r.stdout.decode()[0:1] == "[": return [] return json.loads(r.stdout.decode()) def get_patroni_running_members(self, pod): result = self.get_patroni_state(pod) - return list(filter(lambda x: x["State"]=="running", result)) - + return list(filter(lambda x: x["State"] == "running", result)) + def get_statefulset_image(self, label_selector="application=spilo,cluster-name=acid-minimal-cluster", namespace='default'): ssets = self.api.apps_v1.list_namespaced_stateful_set(namespace, label_selector=label_selector, limit=1) if len(ssets.items) == 0: @@ -505,7 +499,7 @@ class K8sBase: ''' pod = self.api.core_v1.list_namespaced_pod( namespace, label_selector="statefulset.kubernetes.io/pod-name=" + pod_name) - + if len(pod.items) == 0: return None return pod.items[0].spec.containers[0].image @@ -514,10 +508,13 @@ class K8sBase: """ Inspiriational classes towards easier writing of end to end tests with one cluster per test case """ + + class K8sOperator(K8sBase): def __init__(self, labels="name=postgres-operator", namespace="default"): super().__init__(labels, namespace) + class K8sPostgres(K8sBase): def __init__(self, labels="cluster-name=acid-minimal-cluster", namespace="default"): super().__init__(labels, namespace) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 0fc60bf42..f863123bd 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -14,8 +14,9 @@ from kubernetes.client.rest import ApiException SPILO_CURRENT = "registry.opensource.zalan.do/acid/spilo-12:1.6-p5" SPILO_LAZY = "registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p114" + def to_selector(labels): - return ",".join(["=".join(l) for l in labels.items()]) + return ",".join(["=".join(lbl) for lbl in labels.items()]) def clean_list(values): @@ -41,7 +42,7 @@ class EndToEndTestCase(unittest.TestCase): self.assertEqual(y, x, m.format(y)) return True except AssertionError: - retries = retries -1 + retries = retries - 1 if not retries > 0: raise time.sleep(interval) @@ -53,7 +54,7 @@ class EndToEndTestCase(unittest.TestCase): self.assertNotEqual(y, x, m.format(y)) return True except AssertionError: - retries = retries -1 + retries = retries - 1 if not retries > 0: raise time.sleep(interval) @@ -64,7 +65,7 @@ class EndToEndTestCase(unittest.TestCase): self.assertTrue(f(), m) return True except AssertionError: - retries = retries -1 + retries = retries - 1 if not retries > 0: raise time.sleep(interval) @@ -104,13 +105,13 @@ class EndToEndTestCase(unittest.TestCase): with open("manifests/postgres-operator.yaml", 'r+') as f: operator_deployment = yaml.safe_load(f) operator_deployment["spec"]["template"]["spec"]["containers"][0]["image"] = os.environ['OPERATOR_IMAGE'] - + with open("manifests/postgres-operator.yaml", 'w') as f: yaml.dump(operator_deployment, f, Dumper=yaml.Dumper) with open("manifests/configmap.yaml", 'r+') as f: - configmap = yaml.safe_load(f) - configmap["data"]["workers"] = "1" + configmap = yaml.safe_load(f) + configmap["data"]["workers"] = "1" with open("manifests/configmap.yaml", 'w') as f: yaml.dump(configmap, f, Dumper=yaml.Dumper) @@ -129,8 +130,8 @@ class EndToEndTestCase(unittest.TestCase): k8s.wait_for_operator_pod_start() # reset taints and tolerations - k8s.api.core_v1.patch_node("postgres-operator-e2e-tests-worker",{"spec":{"taints":[]}}) - k8s.api.core_v1.patch_node("postgres-operator-e2e-tests-worker2",{"spec":{"taints":[]}}) + k8s.api.core_v1.patch_node("postgres-operator-e2e-tests-worker", {"spec": {"taints": []}}) + k8s.api.core_v1.patch_node("postgres-operator-e2e-tests-worker2", {"spec": {"taints": []}}) # make sure we start a new operator on every new run, # this tackles the problem when kind is reused @@ -160,26 +161,76 @@ class EndToEndTestCase(unittest.TestCase): the end turn connection pooler off to not interfere with other tests. ''' k8s = self.k8s - service_labels = { - 'cluster-name': 'acid-minimal-cluster', - } - pod_labels = dict({ - 'connection-pooler': 'acid-minimal-cluster-pooler', - }) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - # enable connection pooler k8s.api.custom_objects_api.patch_namespaced_custom_object( 'acid.zalan.do', 'v1', 'default', 'postgresqls', 'acid-minimal-cluster', { 'spec': { 'enableConnectionPooler': True, + 'enableReplicaConnectionPooler': True, } }) - self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(), 2, "Deployment replicas is 2 default") - self.eventuallyEqual(lambda: k8s.count_running_pods("connection-pooler=acid-minimal-cluster-pooler"), 2, "No pooler pods found") - self.eventuallyEqual(lambda: k8s.count_services_with_label('application=db-connection-pooler,cluster-name=acid-minimal-cluster'), 1, "No pooler service found") + self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(), 2, + "Deployment replicas is 2 default") + self.eventuallyEqual(lambda: k8s.count_running_pods( + "connection-pooler=acid-minimal-cluster-pooler"), + 2, "No pooler pods found") + self.eventuallyEqual(lambda: k8s.count_running_pods( + "connection-pooler=acid-minimal-cluster-pooler-repl"), + 2, "No pooler replica pods found") + self.eventuallyEqual(lambda: k8s.count_services_with_label( + 'application=db-connection-pooler,cluster-name=acid-minimal-cluster'), + 2, "No pooler service found") + + # Turn off only master connection pooler + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'enableConnectionPooler': False, + 'enableReplicaConnectionPooler': True, + } + }) + + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, + "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(name="acid-minimal-cluster-pooler-repl"), 2, + "Deployment replicas is 2 default") + self.eventuallyEqual(lambda: k8s.count_running_pods( + "connection-pooler=acid-minimal-cluster-pooler"), + 0, "Master pooler pods not deleted") + self.eventuallyEqual(lambda: k8s.count_running_pods( + "connection-pooler=acid-minimal-cluster-pooler-repl"), + 2, "Pooler replica pods not found") + self.eventuallyEqual(lambda: k8s.count_services_with_label( + 'application=db-connection-pooler,cluster-name=acid-minimal-cluster'), + 1, "No pooler service found") + + # Turn off only replica connection pooler + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'enableConnectionPooler': True, + 'enableReplicaConnectionPooler': False, + } + }) + + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, + "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(), 2, + "Deployment replicas is 2 default") + self.eventuallyEqual(lambda: k8s.count_running_pods("connection-pooler=acid-minimal-cluster-pooler"), + 2, "Master pooler pods not found") + self.eventuallyEqual(lambda: k8s.count_running_pods("connection-pooler=acid-minimal-cluster-pooler-repl"), + 0, "Pooler replica pods not deleted") + self.eventuallyEqual(lambda: k8s.count_services_with_label('application=db-connection-pooler,cluster-name=acid-minimal-cluster'), + 1, "No pooler service found") # scale up connection pooler deployment k8s.api.custom_objects_api.patch_namespaced_custom_object( @@ -193,8 +244,10 @@ class EndToEndTestCase(unittest.TestCase): } }) - self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(), 3, "Deployment replicas is scaled to 3") - self.eventuallyEqual(lambda: k8s.count_running_pods("connection-pooler=acid-minimal-cluster-pooler"), 3, "Scale up of pooler pods does not work") + self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(), 3, + "Deployment replicas is scaled to 3") + self.eventuallyEqual(lambda: k8s.count_running_pods("connection-pooler=acid-minimal-cluster-pooler"), + 3, "Scale up of pooler pods does not work") # turn it off, keeping config should be overwritten by false k8s.api.custom_objects_api.patch_namespaced_custom_object( @@ -202,12 +255,15 @@ class EndToEndTestCase(unittest.TestCase): 'postgresqls', 'acid-minimal-cluster', { 'spec': { - 'enableConnectionPooler': False + 'enableConnectionPooler': False, + 'enableReplicaConnectionPooler': False, } }) - self.eventuallyEqual(lambda: k8s.count_running_pods("connection-pooler=acid-minimal-cluster-pooler"), 0, "Pooler pods not scaled down") - self.eventuallyEqual(lambda: k8s.count_services_with_label('application=db-connection-pooler,cluster-name=acid-minimal-cluster'), 0, "Pooler service not removed") + self.eventuallyEqual(lambda: k8s.count_running_pods("connection-pooler=acid-minimal-cluster-pooler"), + 0, "Pooler pods not scaled down") + self.eventuallyEqual(lambda: k8s.count_services_with_label('application=db-connection-pooler,cluster-name=acid-minimal-cluster'), + 0, "Pooler service not removed") # Verify that all the databases have pooler schema installed. # Do this via psql, since otherwise we need to deal with @@ -267,7 +323,8 @@ class EndToEndTestCase(unittest.TestCase): 'postgresqls', 'acid-minimal-cluster', { 'spec': { - 'connectionPooler': None + 'connectionPooler': None, + 'EnableReplicaConnectionPooler': False, } }) @@ -281,8 +338,8 @@ class EndToEndTestCase(unittest.TestCase): cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster,spilo-role={}' self.eventuallyEqual(lambda: k8s.get_service_type(cluster_label.format("master")), - 'ClusterIP', - "Expected ClusterIP type initially, found {}") + 'ClusterIP', + "Expected ClusterIP type initially, found {}") try: # enable load balancer services @@ -294,14 +351,14 @@ class EndToEndTestCase(unittest.TestCase): } k8s.api.custom_objects_api.patch_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_lbs) - + self.eventuallyEqual(lambda: k8s.get_service_type(cluster_label.format("master")), 'LoadBalancer', - "Expected LoadBalancer service type for master, found {}") + "Expected LoadBalancer service type for master, found {}") self.eventuallyEqual(lambda: k8s.get_service_type(cluster_label.format("replica")), 'LoadBalancer', - "Expected LoadBalancer service type for master, found {}") + "Expected LoadBalancer service type for master, found {}") # disable load balancer services again pg_patch_disable_lbs = { @@ -312,14 +369,14 @@ class EndToEndTestCase(unittest.TestCase): } k8s.api.custom_objects_api.patch_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_lbs) - + self.eventuallyEqual(lambda: k8s.get_service_type(cluster_label.format("master")), 'ClusterIP', - "Expected LoadBalancer service type for master, found {}") + "Expected LoadBalancer service type for master, found {}") self.eventuallyEqual(lambda: k8s.get_service_type(cluster_label.format("replica")), 'ClusterIP', - "Expected LoadBalancer service type for master, found {}") + "Expected LoadBalancer service type for master, found {}") except timeout_decorator.TimeoutError: print('Operator log: {}'.format(k8s.get_operator_log())) @@ -333,7 +390,8 @@ class EndToEndTestCase(unittest.TestCase): k8s = self.k8s # update infrastructure roles description secret_name = "postgresql-infrastructure-roles" - roles = "secretname: postgresql-infrastructure-roles-new, userkey: user, rolekey: memberof, passwordkey: password, defaultrolevalue: robot_zmon" + roles = "secretname: postgresql-infrastructure-roles-new, userkey: user,"\ + "rolekey: memberof, passwordkey: password, defaultrolevalue: robot_zmon" patch_infrastructure_roles = { "data": { "infrastructure_roles_secret_name": secret_name, @@ -341,7 +399,8 @@ class EndToEndTestCase(unittest.TestCase): }, } k8s.update_config(patch_infrastructure_roles) - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, + "Operator does not get in sync") try: # check that new roles are represented in the config by requesting the @@ -351,11 +410,12 @@ class EndToEndTestCase(unittest.TestCase): try: operator_pod = k8s.get_operator_pod() get_config_cmd = "wget --quiet -O - localhost:8080/config" - result = k8s.exec_with_kubectl(operator_pod.metadata.name, get_config_cmd) + result = k8s.exec_with_kubectl(operator_pod.metadata.name, + get_config_cmd) try: roles_dict = (json.loads(result.stdout) - .get("controller", {}) - .get("InfrastructureRoles")) + .get("controller", {}) + .get("InfrastructureRoles")) except: return False @@ -377,7 +437,6 @@ class EndToEndTestCase(unittest.TestCase): return False self.eventuallyTrue(verify_role, "infrastructure role setup is not loaded") - except timeout_decorator.TimeoutError: print('Operator log: {}'.format(k8s.get_operator_log())) @@ -386,13 +445,15 @@ class EndToEndTestCase(unittest.TestCase): @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_lazy_spilo_upgrade(self): ''' - Test lazy upgrade for the Spilo image: operator changes a stateful set but lets pods run with the old image - until they are recreated for reasons other than operator's activity. That works because the operator configures - stateful sets to use "onDelete" pod update policy. + Test lazy upgrade for the Spilo image: operator changes a stateful set + but lets pods run with the old image until they are recreated for + reasons other than operator's activity. That works because the operator + configures stateful sets to use "onDelete" pod update policy. The test covers: 1) enabling lazy upgrade in existing operator deployment - 2) forcing the normal rolling upgrade by changing the operator configmap and restarting its pod + 2) forcing the normal rolling upgrade by changing the operator + configmap and restarting its pod ''' k8s = self.k8s @@ -400,8 +461,10 @@ class EndToEndTestCase(unittest.TestCase): pod0 = 'acid-minimal-cluster-0' pod1 = 'acid-minimal-cluster-1' - self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No 2 pods running") - self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members(pod0)), 2, "Postgres status did not enter running") + self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, + "No 2 pods running") + self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members(pod0)), + 2, "Postgres status did not enter running") patch_lazy_spilo_upgrade = { "data": { @@ -409,14 +472,20 @@ class EndToEndTestCase(unittest.TestCase): "enable_lazy_spilo_upgrade": "false" } } - k8s.update_config(patch_lazy_spilo_upgrade, step="Init baseline image version") + k8s.update_config(patch_lazy_spilo_upgrade, + step="Init baseline image version") - self.eventuallyEqual(lambda: k8s.get_statefulset_image(), SPILO_CURRENT, "Stagefulset not updated initially") - self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No 2 pods running") - self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members(pod0)), 2, "Postgres status did not enter running") + self.eventuallyEqual(lambda: k8s.get_statefulset_image(), SPILO_CURRENT, + "Statefulset not updated initially") + self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, + "No 2 pods running") + self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members(pod0)), + 2, "Postgres status did not enter running") - self.eventuallyEqual(lambda: k8s.get_effective_pod_image(pod0), SPILO_CURRENT, "Rolling upgrade was not executed") - self.eventuallyEqual(lambda: k8s.get_effective_pod_image(pod1), SPILO_CURRENT, "Rolling upgrade was not executed") + self.eventuallyEqual(lambda: k8s.get_effective_pod_image(pod0), + SPILO_CURRENT, "Rolling upgrade was not executed") + self.eventuallyEqual(lambda: k8s.get_effective_pod_image(pod1), + SPILO_CURRENT, "Rolling upgrade was not executed") # update docker image in config and enable the lazy upgrade conf_image = SPILO_LAZY @@ -426,18 +495,25 @@ class EndToEndTestCase(unittest.TestCase): "enable_lazy_spilo_upgrade": "true" } } - k8s.update_config(patch_lazy_spilo_upgrade,step="patch image and lazy upgrade") - self.eventuallyEqual(lambda: k8s.get_statefulset_image(), conf_image, "Statefulset not updated to next Docker image") + k8s.update_config(patch_lazy_spilo_upgrade, + step="patch image and lazy upgrade") + self.eventuallyEqual(lambda: k8s.get_statefulset_image(), conf_image, + "Statefulset not updated to next Docker image") try: # restart the pod to get a container with the new image - k8s.api.core_v1.delete_namespaced_pod(pod0, 'default') - + k8s.api.core_v1.delete_namespaced_pod(pod0, 'default') + # verify only pod-0 which was deleted got new image from statefulset - self.eventuallyEqual(lambda: k8s.get_effective_pod_image(pod0), conf_image, "Delete pod-0 did not get new spilo image") - self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No two pods running after lazy rolling upgrade") - self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members(pod0)), 2, "Postgres status did not enter running") - self.assertNotEqual(lambda: k8s.get_effective_pod_image(pod1), SPILO_CURRENT, "pod-1 should not have change Docker image to {}".format(SPILO_CURRENT)) + self.eventuallyEqual(lambda: k8s.get_effective_pod_image(pod0), + conf_image, "Delete pod-0 did not get new spilo image") + self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, + "No two pods running after lazy rolling upgrade") + self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members(pod0)), + 2, "Postgres status did not enter running") + self.assertNotEqual(lambda: k8s.get_effective_pod_image(pod1), + SPILO_CURRENT, + "pod-1 should not have change Docker image to {}".format(SPILO_CURRENT)) # clean up unpatch_lazy_spilo_upgrade = { @@ -449,9 +525,14 @@ class EndToEndTestCase(unittest.TestCase): # at this point operator will complete the normal rolling upgrade # so we additonally test if disabling the lazy upgrade - forcing the normal rolling upgrade - works - self.eventuallyEqual(lambda: k8s.get_effective_pod_image(pod0), conf_image, "Rolling upgrade was not executed", 50, 3) - self.eventuallyEqual(lambda: k8s.get_effective_pod_image(pod1), conf_image, "Rolling upgrade was not executed", 50, 3) - self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members(pod0)), 2, "Postgres status did not enter running") + self.eventuallyEqual(lambda: k8s.get_effective_pod_image(pod0), + conf_image, "Rolling upgrade was not executed", + 50, 3) + self.eventuallyEqual(lambda: k8s.get_effective_pod_image(pod1), + conf_image, "Rolling upgrade was not executed", + 50, 3) + self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members(pod0)), + 2, "Postgres status did not enter running") except timeout_decorator.TimeoutError: print('Operator log: {}'.format(k8s.get_operator_log())) @@ -505,9 +586,9 @@ class EndToEndTestCase(unittest.TestCase): def get_docker_image(): jobs = k8s.get_logical_backup_job().items return jobs[0].spec.job_template.spec.template.spec.containers[0].image - + self.eventuallyEqual(get_docker_image, image, - "Expected job image {}, found {}".format(image, "{}")) + "Expected job image {}, found {}".format(image, "{}")) # delete the logical backup cron job pg_patch_disable_backup = { @@ -517,7 +598,7 @@ class EndToEndTestCase(unittest.TestCase): } k8s.api.custom_objects_api.patch_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_backup) - + self.eventuallyEqual(lambda: len(k8s.get_logical_backup_job().items), 0, "failed to create logical backup job") except timeout_decorator.TimeoutError: @@ -563,21 +644,21 @@ class EndToEndTestCase(unittest.TestCase): } k8s.api.custom_objects_api.patch_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_resources) - - k8s.patch_statefulset({"metadata":{"annotations":{"zalando-postgres-operator-rolling-update-required": "False"}}}) + + k8s.patch_statefulset({"metadata": {"annotations": {"zalando-postgres-operator-rolling-update-required": "False"}}}) k8s.update_config(patch_min_resource_limits, "Minimum resource test") self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No two pods running after lazy rolling upgrade") self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members()), 2, "Postgres status did not enter running") - + def verify_pod_limits(): pods = k8s.api.core_v1.list_namespaced_pod('default', label_selector="cluster-name=acid-minimal-cluster,application=spilo").items - if len(pods)<2: + if len(pods) < 2: return False - r = pods[0].spec.containers[0].resources.limits['memory']==minMemoryLimit + r = pods[0].spec.containers[0].resources.limits['memory'] == minMemoryLimit r = r and pods[0].spec.containers[0].resources.limits['cpu'] == minCPULimit - r = r and pods[1].spec.containers[0].resources.limits['memory']==minMemoryLimit + r = r and pods[1].spec.containers[0].resources.limits['memory'] == minMemoryLimit r = r and pods[1].spec.containers[0].resources.limits['cpu'] == minCPULimit return r @@ -586,7 +667,7 @@ class EndToEndTestCase(unittest.TestCase): @classmethod def setUp(cls): # cls.k8s.update_config({}, step="Setup") - cls.k8s.patch_statefulset({"meta":{"annotations":{"zalando-postgres-operator-rolling-update-required": False}}}) + cls.k8s.patch_statefulset({"meta": {"annotations": {"zalando-postgres-operator-rolling-update-required": False}}}) pass @timeout_decorator.timeout(TEST_TIMEOUT_SEC) @@ -642,7 +723,7 @@ class EndToEndTestCase(unittest.TestCase): } } } - self.assertTrue(len(failover_targets)>0, "No failover targets available") + self.assertTrue(len(failover_targets) > 0, "No failover targets available") for failover_target in failover_targets: k8s.api.core_v1.patch_node(failover_target, patch_readiness_label) @@ -672,12 +753,12 @@ class EndToEndTestCase(unittest.TestCase): Scale up from 2 to 3 and back to 2 pods by updating the Postgres manifest at runtime. ''' k8s = self.k8s - pod="acid-minimal-cluster-0" + pod = "acid-minimal-cluster-0" - k8s.scale_cluster(3) + k8s.scale_cluster(3) self.eventuallyEqual(lambda: k8s.count_running_pods(), 3, "Scale up to 3 failed") self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members(pod)), 3, "Not all 3 nodes healthy") - + k8s.scale_cluster(2) self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "Scale down to 2 failed") self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members(pod)), 2, "Not all members 2 healthy") @@ -756,6 +837,7 @@ class EndToEndTestCase(unittest.TestCase): self.eventuallyTrue(lambda: k8s.check_statefulset_annotations(cluster_label, annotations), "Annotations missing") + self.eventuallyTrue(lambda: k8s.check_statefulset_annotations(cluster_label, annotations), "Annotations missing") @timeout_decorator.timeout(TEST_TIMEOUT_SEC) @unittest.skip("Skipping this test until fixed") @@ -798,7 +880,7 @@ class EndToEndTestCase(unittest.TestCase): "toleration": "key:postgres,operator:Exists,effect:NoExecute" } } - + k8s.update_config(patch_toleration_config, step="allow tainted nodes") self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No 2 pods running") @@ -825,13 +907,14 @@ class EndToEndTestCase(unittest.TestCase): } } k8s.update_config(patch_delete_annotations) + time.sleep(25) self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") try: # this delete attempt should be omitted because of missing annotations k8s.api.custom_objects_api.delete_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster") - time.sleep(5) + time.sleep(15) self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") # check that pods and services are still there diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 79d1251e6..e6fb9a43c 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -18,7 +18,8 @@ spec: - createdb enableMasterLoadBalancer: false enableReplicaLoadBalancer: false -# enableConnectionPooler: true # not needed when connectionPooler section is present (see below) + enableConnectionPooler: false # enable/disable connection pooler deployment + enableReplicaConnectionPooler: false # set to enable connectionPooler for replica service allowedSourceRanges: # load balancers' source ranges for both master and replica services - 127.0.0.1/32 databases: @@ -126,18 +127,19 @@ spec: # - 01:00-06:00 #UTC # - Sat:00:00-04:00 - connectionPooler: - numberOfInstances: 2 - mode: "transaction" - schema: "pooler" - user: "pooler" - resources: - requests: - cpu: 300m - memory: 100Mi - limits: - cpu: "1" - memory: 100Mi +# overwrite custom properties for connection pooler deployments +# connectionPooler: +# numberOfInstances: 2 +# mode: "transaction" +# schema: "pooler" +# user: "pooler" +# resources: +# requests: +# cpu: 300m +# memory: 100Mi +# limits: +# cpu: "1" +# memory: 100Mi initContainers: - name: date diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 16bdac564..5ee05f444 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -186,6 +186,8 @@ spec: type: string enableConnectionPooler: type: boolean + enableReplicaConnectionPooler: + type: boolean enableLogicalBackup: type: boolean enableMasterLoadBalancer: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 92b904bae..2ed0d6b01 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -262,6 +262,9 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ "enableConnectionPooler": { Type: "boolean", }, + "enableReplicaConnectionPooler": { + Type: "boolean", + }, "enableLogicalBackup": { Type: "boolean", }, diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 499a4cfda..a3dc490b5 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -29,8 +29,9 @@ type PostgresSpec struct { Patroni `json:"patroni,omitempty"` Resources `json:"resources,omitempty"` - EnableConnectionPooler *bool `json:"enableConnectionPooler,omitempty"` - ConnectionPooler *ConnectionPooler `json:"connectionPooler,omitempty"` + EnableConnectionPooler *bool `json:"enableConnectionPooler,omitempty"` + EnableReplicaConnectionPooler *bool `json:"enableReplicaConnectionPooler,omitempty"` + ConnectionPooler *ConnectionPooler `json:"connectionPooler,omitempty"` TeamID string `json:"teamId"` DockerImage string `json:"dockerImage,omitempty"` diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 7ec7be176..ee5c44bc9 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -12,9 +12,9 @@ import ( "sync" "time" - "github.com/r3labs/diff" "github.com/sirupsen/logrus" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/scheme" "github.com/zalando/postgres-operator/pkg/spec" pgteams "github.com/zalando/postgres-operator/pkg/teams" @@ -54,26 +54,11 @@ type Config struct { PodServiceAccountRoleBinding *rbacv1.RoleBinding } -// K8S objects that are belongs to a connection pooler -type ConnectionPoolerObjects struct { - Deployment *appsv1.Deployment - Service *v1.Service - - // It could happen that a connection pooler was enabled, but the operator - // was not able to properly process a corresponding event or was restarted. - // In this case we will miss missing/require situation and a lookup function - // will not be installed. To avoid synchronizing it all the time to prevent - // this, we can remember the result in memory at least until the next - // restart. - LookupFunction bool -} - type kubeResources struct { Services map[PostgresRole]*v1.Service Endpoints map[PostgresRole]*v1.Endpoints Secrets map[types.UID]*v1.Secret Statefulset *appsv1.StatefulSet - ConnectionPooler *ConnectionPoolerObjects PodDisruptionBudget *policybeta1.PodDisruptionBudget //Pods are treated separately //PVCs are treated separately @@ -103,9 +88,8 @@ type Cluster struct { currentProcess Process processMu sync.RWMutex // protects the current operation for reporting, no need to hold the master mutex specMu sync.RWMutex // protects the spec for reporting, no need to hold the master mutex - + ConnectionPooler map[PostgresRole]*ConnectionPoolerObjects } - type compareStatefulsetResult struct { match bool replace bool @@ -347,19 +331,7 @@ func (c *Cluster) Create() error { // // Do not consider connection pooler as a strict requirement, and if // something fails, report warning - if c.needConnectionPooler() { - if c.ConnectionPooler != nil { - c.logger.Warning("Connection pooler already exists in the cluster") - return nil - } - connectionPooler, err := c.createConnectionPooler(c.installLookupFunction) - if err != nil { - c.logger.Warningf("could not create connection pooler: %v", err) - return nil - } - c.logger.Infof("connection pooler %q has been successfully created", - util.NameFromMeta(connectionPooler.Deployment.ObjectMeta)) - } + c.createConnectionPooler(c.installLookupFunction) return nil } @@ -626,6 +598,8 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { } }() + logNiceDiff(c.logger, oldSpec, newSpec) + if oldSpec.Spec.PostgresqlParam.PgVersion > newSpec.Spec.PostgresqlParam.PgVersion { c.logger.Warningf("postgresql version change(%q -> %q) has no effect", oldSpec.Spec.PostgresqlParam.PgVersion, newSpec.Spec.PostgresqlParam.PgVersion) @@ -641,7 +615,6 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { // Service if !reflect.DeepEqual(c.generateService(Master, &oldSpec.Spec), c.generateService(Master, &newSpec.Spec)) || !reflect.DeepEqual(c.generateService(Replica, &oldSpec.Spec), c.generateService(Replica, &newSpec.Spec)) { - c.logger.Debugf("syncing services") if err := c.syncServices(); err != nil { c.logger.Errorf("could not sync services: %v", err) updateFailed = true @@ -652,7 +625,8 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { // initUsers. Check if it needs to be called. sameUsers := reflect.DeepEqual(oldSpec.Spec.Users, newSpec.Spec.Users) && reflect.DeepEqual(oldSpec.Spec.PreparedDatabases, newSpec.Spec.PreparedDatabases) - needConnectionPooler := c.needConnectionPoolerWorker(&newSpec.Spec) + needConnectionPooler := needMasterConnectionPoolerWorker(&newSpec.Spec) || + needReplicaConnectionPoolerWorker(&newSpec.Spec) if !sameUsers || needConnectionPooler { c.logger.Debugf("syncing secrets") if err := c.initUsers(); err != nil { @@ -797,10 +771,8 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { // need to process. In the future we may want to do this more careful and // check which databases we need to process, but even repeating the whole // installation process should be good enough. - c.ConnectionPooler.LookupFunction = false - if _, err := c.syncConnectionPooler(oldSpec, newSpec, - c.installLookupFunction); err != nil { + if _, err := c.syncConnectionPooler(oldSpec, newSpec, c.installLookupFunction); err != nil { c.logger.Errorf("could not sync connection pooler: %v", err) updateFailed = true } @@ -808,6 +780,20 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { return nil } +func syncResources(a, b *v1.ResourceRequirements) bool { + for _, res := range []v1.ResourceName{ + v1.ResourceCPU, + v1.ResourceMemory, + } { + if !a.Limits[res].Equal(b.Limits[res]) || + !a.Requests[res].Equal(b.Requests[res]) { + return true + } + } + + return false +} + // Delete deletes the cluster and cleans up all objects associated with it (including statefulsets). // The deletion order here is somewhat significant, because Patroni, when running with the Kubernetes // DCS, reuses the master's endpoint to store the leader related metadata. If we remove the endpoint @@ -856,9 +842,12 @@ func (c *Cluster) Delete() { // Delete connection pooler objects anyway, even if it's not mentioned in the // manifest, just to not keep orphaned components in case if something went // wrong - if err := c.deleteConnectionPooler(); err != nil { - c.logger.Warningf("could not remove connection pooler: %v", err) + for _, role := range [2]PostgresRole{Master, Replica} { + if err := c.deleteConnectionPooler(role); err != nil { + c.logger.Warningf("could not remove connection pooler: %v", err) + } } + } //NeedsRepair returns true if the cluster should be included in the repair scan (based on its in-memory status). @@ -928,7 +917,7 @@ func (c *Cluster) initSystemUsers() { // Connection pooler user is an exception, if requested it's going to be // created by operator as a normal pgUser - if c.needConnectionPooler() { + if needConnectionPooler(&c.Spec) { // initialize empty connection pooler if not done yet if c.Spec.ConnectionPooler == nil { c.Spec.ConnectionPooler = &acidv1.ConnectionPooler{} @@ -1423,119 +1412,3 @@ func (c *Cluster) deletePatroniClusterConfigMaps() error { return c.deleteClusterObject(get, deleteConfigMapFn, "configmap") } - -// Test if two connection pooler configuration needs to be synced. For simplicity -// compare not the actual K8S objects, but the configuration itself and request -// sync if there is any difference. -func (c *Cluster) needSyncConnectionPoolerSpecs(oldSpec, newSpec *acidv1.ConnectionPooler) (sync bool, reasons []string) { - reasons = []string{} - sync = false - - changelog, err := diff.Diff(oldSpec, newSpec) - if err != nil { - c.logger.Infof("Cannot get diff, do not do anything, %+v", err) - return false, reasons - } - - if len(changelog) > 0 { - sync = true - } - - for _, change := range changelog { - msg := fmt.Sprintf("%s %+v from '%+v' to '%+v'", - change.Type, change.Path, change.From, change.To) - reasons = append(reasons, msg) - } - - return sync, reasons -} - -func syncResources(a, b *v1.ResourceRequirements) bool { - for _, res := range []v1.ResourceName{ - v1.ResourceCPU, - v1.ResourceMemory, - } { - if !a.Limits[res].Equal(b.Limits[res]) || - !a.Requests[res].Equal(b.Requests[res]) { - return true - } - } - - return false -} - -// Check if we need to synchronize connection pooler deployment due to new -// defaults, that are different from what we see in the DeploymentSpec -func (c *Cluster) needSyncConnectionPoolerDefaults( - spec *acidv1.ConnectionPooler, - deployment *appsv1.Deployment) (sync bool, reasons []string) { - - reasons = []string{} - sync = false - - config := c.OpConfig.ConnectionPooler - podTemplate := deployment.Spec.Template - poolerContainer := podTemplate.Spec.Containers[constants.ConnectionPoolerContainer] - - if spec == nil { - spec = &acidv1.ConnectionPooler{} - } - - if spec.NumberOfInstances == nil && - *deployment.Spec.Replicas != *config.NumberOfInstances { - - sync = true - msg := fmt.Sprintf("NumberOfInstances is different (having %d, required %d)", - *deployment.Spec.Replicas, *config.NumberOfInstances) - reasons = append(reasons, msg) - } - - if spec.DockerImage == "" && - poolerContainer.Image != config.Image { - - sync = true - msg := fmt.Sprintf("DockerImage is different (having %s, required %s)", - poolerContainer.Image, config.Image) - reasons = append(reasons, msg) - } - - expectedResources, err := generateResourceRequirements(spec.Resources, - c.makeDefaultConnectionPoolerResources()) - - // An error to generate expected resources means something is not quite - // right, but for the purpose of robustness do not panic here, just report - // and ignore resources comparison (in the worst case there will be no - // updates for new resource values). - if err == nil && syncResources(&poolerContainer.Resources, expectedResources) { - sync = true - msg := fmt.Sprintf("Resources are different (having %+v, required %+v)", - poolerContainer.Resources, expectedResources) - reasons = append(reasons, msg) - } - - if err != nil { - c.logger.Warningf("Cannot generate expected resources, %v", err) - } - - for _, env := range poolerContainer.Env { - if spec.User == "" && env.Name == "PGUSER" { - ref := env.ValueFrom.SecretKeyRef.LocalObjectReference - - if ref.Name != c.credentialSecretName(config.User) { - sync = true - msg := fmt.Sprintf("pooler user is different (having %s, required %s)", - ref.Name, config.User) - reasons = append(reasons, msg) - } - } - - if spec.Schema == "" && env.Name == "PGSCHEMA" && env.Value != config.Schema { - sync = true - msg := fmt.Sprintf("pooler schema is different (having %s, required %s)", - env.Value, config.Schema) - reasons = append(reasons, msg) - } - } - - return sync, reasons -} diff --git a/pkg/cluster/connection_pooler.go b/pkg/cluster/connection_pooler.go new file mode 100644 index 000000000..0d9171b87 --- /dev/null +++ b/pkg/cluster/connection_pooler.go @@ -0,0 +1,905 @@ +package cluster + +import ( + "context" + "fmt" + "strings" + + "github.com/r3labs/diff" + "github.com/sirupsen/logrus" + acidzalando "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do" + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + + "github.com/zalando/postgres-operator/pkg/util" + "github.com/zalando/postgres-operator/pkg/util/config" + "github.com/zalando/postgres-operator/pkg/util/constants" + "github.com/zalando/postgres-operator/pkg/util/k8sutil" +) + +// K8S objects that are belong to connection pooler +type ConnectionPoolerObjects struct { + Deployment *appsv1.Deployment + Service *v1.Service + Name string + ClusterName string + Namespace string + Role PostgresRole + // It could happen that a connection pooler was enabled, but the operator + // was not able to properly process a corresponding event or was restarted. + // In this case we will miss missing/require situation and a lookup function + // will not be installed. To avoid synchronizing it all the time to prevent + // this, we can remember the result in memory at least until the next + // restart. + LookupFunction bool + // Careful with referencing cluster.spec this object pointer changes + // during runtime and lifetime of cluster +} + +func (c *Cluster) connectionPoolerName(role PostgresRole) string { + name := c.Name + "-pooler" + if role == Replica { + name = name + "-repl" + } + return name +} + +// isConnectionPoolerEnabled +func needConnectionPooler(spec *acidv1.PostgresSpec) bool { + return needMasterConnectionPoolerWorker(spec) || + needReplicaConnectionPoolerWorker(spec) +} + +func needMasterConnectionPooler(spec *acidv1.PostgresSpec) bool { + return needMasterConnectionPoolerWorker(spec) +} + +func needMasterConnectionPoolerWorker(spec *acidv1.PostgresSpec) bool { + return (nil != spec.EnableConnectionPooler && *spec.EnableConnectionPooler) || + (spec.ConnectionPooler != nil && spec.EnableConnectionPooler == nil) +} + +func needReplicaConnectionPooler(spec *acidv1.PostgresSpec) bool { + return needReplicaConnectionPoolerWorker(spec) +} + +func needReplicaConnectionPoolerWorker(spec *acidv1.PostgresSpec) bool { + return spec.EnableReplicaConnectionPooler != nil && + *spec.EnableReplicaConnectionPooler +} + +// Return connection pooler labels selector, which should from one point of view +// inherit most of the labels from the cluster itself, but at the same time +// have e.g. different `application` label, so that recreatePod operation will +// not interfere with it (it lists all the pods via labels, and if there would +// be no difference, it will recreate also pooler pods). +func (c *Cluster) connectionPoolerLabelsSelector(role PostgresRole) *metav1.LabelSelector { + connectionPoolerLabels := labels.Set(map[string]string{}) + + extraLabels := labels.Set(map[string]string{ + "connection-pooler": c.connectionPoolerName(role), + "application": "db-connection-pooler", + "spilo-role": string(role), + "cluster-name": c.Name, + "Namespace": c.Namespace, + }) + + connectionPoolerLabels = labels.Merge(connectionPoolerLabels, c.labelsSet(false)) + connectionPoolerLabels = labels.Merge(connectionPoolerLabels, extraLabels) + + return &metav1.LabelSelector{ + MatchLabels: connectionPoolerLabels, + MatchExpressions: nil, + } +} + +// Prepare the database for connection pooler to be used, i.e. install lookup +// function (do it first, because it should be fast and if it didn't succeed, +// it doesn't makes sense to create more K8S objects. At this moment we assume +// that necessary connection pooler user exists. +// +// After that create all the objects for connection pooler, namely a deployment +// with a chosen pooler and a service to expose it. + +// have connectionpooler name in the cp object to have it immutable name +// add these cp related functions to a new cp file +// opConfig, cluster, and database name +func (c *Cluster) createConnectionPooler(LookupFunction InstallFunction) (SyncReason, error) { + var reason SyncReason + c.setProcessName("creating connection pooler") + + //this is essentially sync with nil as oldSpec + if reason, err := c.syncConnectionPooler(nil, &c.Postgresql, LookupFunction); err != nil { + return reason, err + } + return reason, nil +} + +// +// Generate pool size related environment variables. +// +// MAX_DB_CONN would specify the global maximum for connections to a target +// database. +// +// MAX_CLIENT_CONN is not configurable at the moment, just set it high enough. +// +// DEFAULT_SIZE is a pool size per db/user (having in mind the use case when +// most of the queries coming through a connection pooler are from the same +// user to the same db). In case if we want to spin up more connection pooler +// instances, take this into account and maintain the same number of +// connections. +// +// MIN_SIZE is a pool's minimal size, to prevent situation when sudden workload +// have to wait for spinning up a new connections. +// +// RESERVE_SIZE is how many additional connections to allow for a pooler. +func (c *Cluster) getConnectionPoolerEnvVars() []v1.EnvVar { + spec := &c.Spec + effectiveMode := util.Coalesce( + spec.ConnectionPooler.Mode, + c.OpConfig.ConnectionPooler.Mode) + + numberOfInstances := spec.ConnectionPooler.NumberOfInstances + if numberOfInstances == nil { + numberOfInstances = util.CoalesceInt32( + c.OpConfig.ConnectionPooler.NumberOfInstances, + k8sutil.Int32ToPointer(1)) + } + + effectiveMaxDBConn := util.CoalesceInt32( + spec.ConnectionPooler.MaxDBConnections, + c.OpConfig.ConnectionPooler.MaxDBConnections) + + if effectiveMaxDBConn == nil { + effectiveMaxDBConn = k8sutil.Int32ToPointer( + constants.ConnectionPoolerMaxDBConnections) + } + + maxDBConn := *effectiveMaxDBConn / *numberOfInstances + + defaultSize := maxDBConn / 2 + minSize := defaultSize / 2 + reserveSize := minSize + + return []v1.EnvVar{ + { + Name: "CONNECTION_POOLER_PORT", + Value: fmt.Sprint(pgPort), + }, + { + Name: "CONNECTION_POOLER_MODE", + Value: effectiveMode, + }, + { + Name: "CONNECTION_POOLER_DEFAULT_SIZE", + Value: fmt.Sprint(defaultSize), + }, + { + Name: "CONNECTION_POOLER_MIN_SIZE", + Value: fmt.Sprint(minSize), + }, + { + Name: "CONNECTION_POOLER_RESERVE_SIZE", + Value: fmt.Sprint(reserveSize), + }, + { + Name: "CONNECTION_POOLER_MAX_CLIENT_CONN", + Value: fmt.Sprint(constants.ConnectionPoolerMaxClientConnections), + }, + { + Name: "CONNECTION_POOLER_MAX_DB_CONN", + Value: fmt.Sprint(maxDBConn), + }, + } +} + +func (c *Cluster) generateConnectionPoolerPodTemplate(role PostgresRole) ( + *v1.PodTemplateSpec, error) { + spec := &c.Spec + gracePeriod := int64(c.OpConfig.PodTerminateGracePeriod.Seconds()) + resources, err := generateResourceRequirements( + spec.ConnectionPooler.Resources, + makeDefaultConnectionPoolerResources(&c.OpConfig)) + + effectiveDockerImage := util.Coalesce( + spec.ConnectionPooler.DockerImage, + c.OpConfig.ConnectionPooler.Image) + + effectiveSchema := util.Coalesce( + spec.ConnectionPooler.Schema, + c.OpConfig.ConnectionPooler.Schema) + + if err != nil { + return nil, fmt.Errorf("could not generate resource requirements: %v", err) + } + + secretSelector := func(key string) *v1.SecretKeySelector { + effectiveUser := util.Coalesce( + spec.ConnectionPooler.User, + c.OpConfig.ConnectionPooler.User) + + return &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: c.credentialSecretName(effectiveUser), + }, + Key: key, + } + } + + envVars := []v1.EnvVar{ + { + Name: "PGHOST", + Value: c.serviceAddress(role), + }, + { + Name: "PGPORT", + Value: c.servicePort(role), + }, + { + Name: "PGUSER", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: secretSelector("username"), + }, + }, + // the convention is to use the same schema name as + // connection pooler username + { + Name: "PGSCHEMA", + Value: effectiveSchema, + }, + { + Name: "PGPASSWORD", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: secretSelector("password"), + }, + }, + } + envVars = append(envVars, c.getConnectionPoolerEnvVars()...) + + poolerContainer := v1.Container{ + Name: connectionPoolerContainer, + Image: effectiveDockerImage, + ImagePullPolicy: v1.PullIfNotPresent, + Resources: *resources, + Ports: []v1.ContainerPort{ + { + ContainerPort: pgPort, + Protocol: v1.ProtocolTCP, + }, + }, + Env: envVars, + ReadinessProbe: &v1.Probe{ + Handler: v1.Handler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.IntOrString{IntVal: pgPort}, + }, + }, + }, + } + + podTemplate := &v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: c.connectionPoolerLabelsSelector(role).MatchLabels, + Namespace: c.Namespace, + Annotations: c.generatePodAnnotations(spec), + }, + Spec: v1.PodSpec{ + ServiceAccountName: c.OpConfig.PodServiceAccountName, + TerminationGracePeriodSeconds: &gracePeriod, + Containers: []v1.Container{poolerContainer}, + // TODO: add tolerations to scheduler pooler on the same node + // as database + //Tolerations: *tolerationsSpec, + }, + } + + return podTemplate, nil +} + +func (c *Cluster) generateConnectionPoolerDeployment(connectionPooler *ConnectionPoolerObjects) ( + *appsv1.Deployment, error) { + spec := &c.Spec + + // there are two ways to enable connection pooler, either to specify a + // connectionPooler section or enableConnectionPooler. In the second case + // spec.connectionPooler will be nil, so to make it easier to calculate + // default values, initialize it to an empty structure. It could be done + // anywhere, but here is the earliest common entry point between sync and + // create code, so init here. + if spec.ConnectionPooler == nil { + spec.ConnectionPooler = &acidv1.ConnectionPooler{} + } + podTemplate, err := c.generateConnectionPoolerPodTemplate(connectionPooler.Role) + + numberOfInstances := spec.ConnectionPooler.NumberOfInstances + if numberOfInstances == nil { + numberOfInstances = util.CoalesceInt32( + c.OpConfig.ConnectionPooler.NumberOfInstances, + k8sutil.Int32ToPointer(1)) + } + + if *numberOfInstances < constants.ConnectionPoolerMinInstances { + msg := "Adjusted number of connection pooler instances from %d to %d" + c.logger.Warningf(msg, numberOfInstances, constants.ConnectionPoolerMinInstances) + + *numberOfInstances = constants.ConnectionPoolerMinInstances + } + + if err != nil { + return nil, err + } + + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: connectionPooler.Name, + Namespace: connectionPooler.Namespace, + Labels: c.connectionPoolerLabelsSelector(connectionPooler.Role).MatchLabels, + Annotations: map[string]string{}, + // make StatefulSet object its owner to represent the dependency. + // By itself StatefulSet is being deleted with "Orphaned" + // propagation policy, which means that it's deletion will not + // clean up this deployment, but there is a hope that this object + // will be garbage collected if something went wrong and operator + // didn't deleted it. + OwnerReferences: c.ownerReferences(), + }, + Spec: appsv1.DeploymentSpec{ + Replicas: numberOfInstances, + Selector: c.connectionPoolerLabelsSelector(connectionPooler.Role), + Template: *podTemplate, + }, + } + + return deployment, nil +} + +func (c *Cluster) generateConnectionPoolerService(connectionPooler *ConnectionPoolerObjects) *v1.Service { + + spec := &c.Spec + // there are two ways to enable connection pooler, either to specify a + // connectionPooler section or enableConnectionPooler. In the second case + // spec.connectionPooler will be nil, so to make it easier to calculate + // default values, initialize it to an empty structure. It could be done + // anywhere, but here is the earliest common entry point between sync and + // create code, so init here. + if spec.ConnectionPooler == nil { + spec.ConnectionPooler = &acidv1.ConnectionPooler{} + } + + serviceSpec := v1.ServiceSpec{ + Ports: []v1.ServicePort{ + { + Name: connectionPooler.Name, + Port: pgPort, + TargetPort: intstr.IntOrString{StrVal: c.servicePort(connectionPooler.Role)}, + }, + }, + Type: v1.ServiceTypeClusterIP, + Selector: map[string]string{ + "connection-pooler": c.connectionPoolerName(connectionPooler.Role), + }, + } + + service := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: connectionPooler.Name, + Namespace: connectionPooler.Namespace, + Labels: c.connectionPoolerLabelsSelector(connectionPooler.Role).MatchLabels, + Annotations: map[string]string{}, + // make StatefulSet object its owner to represent the dependency. + // By itself StatefulSet is being deleted with "Orphaned" + // propagation policy, which means that it's deletion will not + // clean up this service, but there is a hope that this object will + // be garbage collected if something went wrong and operator didn't + // deleted it. + OwnerReferences: c.ownerReferences(), + }, + Spec: serviceSpec, + } + + return service +} + +//delete connection pooler +func (c *Cluster) deleteConnectionPooler(role PostgresRole) (err error) { + c.logger.Infof("deleting connection pooler spilo-role=%s", role) + + // Lack of connection pooler objects is not a fatal error, just log it if + // it was present before in the manifest + if c.ConnectionPooler[role] == nil || role == "" { + c.logger.Debugf("no connection pooler to delete") + return nil + } + + // Clean up the deployment object. If deployment resource we've remembered + // is somehow empty, try to delete based on what would we generate + var deployment *appsv1.Deployment + deployment = c.ConnectionPooler[role].Deployment + + policy := metav1.DeletePropagationForeground + options := metav1.DeleteOptions{PropagationPolicy: &policy} + + if deployment != nil { + + // set delete propagation policy to foreground, so that replica set will be + // also deleted. + + err = c.KubeClient. + Deployments(c.Namespace). + Delete(context.TODO(), deployment.Name, options) + + if k8sutil.ResourceNotFound(err) { + c.logger.Debugf("connection pooler deployment was already deleted") + } else if err != nil { + return fmt.Errorf("could not delete connection pooler deployment: %v", err) + } + + c.logger.Infof("connection pooler deployment %s has been deleted for role %s", deployment.Name, role) + } + + // Repeat the same for the service object + var service *v1.Service + service = c.ConnectionPooler[role].Service + if service == nil { + c.logger.Debugf("no connection pooler service object to delete") + } else { + + err = c.KubeClient. + Services(c.Namespace). + Delete(context.TODO(), service.Name, options) + + if k8sutil.ResourceNotFound(err) { + c.logger.Debugf("connection pooler service was already deleted") + } else if err != nil { + return fmt.Errorf("could not delete connection pooler service: %v", err) + } + + c.logger.Infof("connection pooler service %s has been deleted for role %s", service.Name, role) + } + + c.ConnectionPooler[role].Deployment = nil + c.ConnectionPooler[role].Service = nil + return nil +} + +//delete connection pooler +func (c *Cluster) deleteConnectionPoolerSecret() (err error) { + // Repeat the same for the secret object + secretName := c.credentialSecretName(c.OpConfig.ConnectionPooler.User) + + secret, err := c.KubeClient. + Secrets(c.Namespace). + Get(context.TODO(), secretName, metav1.GetOptions{}) + + if err != nil { + c.logger.Debugf("could not get connection pooler secret %s: %v", secretName, err) + } else { + if err = c.deleteSecret(secret.UID, *secret); err != nil { + return fmt.Errorf("could not delete pooler secret: %v", err) + } + } + return nil +} + +// Perform actual patching of a connection pooler deployment, assuming that all +// the check were already done before. +func updateConnectionPoolerDeployment(KubeClient k8sutil.KubernetesClient, newDeployment *appsv1.Deployment) (*appsv1.Deployment, error) { + if newDeployment == nil { + return nil, fmt.Errorf("there is no connection pooler in the cluster") + } + + patchData, err := specPatch(newDeployment.Spec) + if err != nil { + return nil, fmt.Errorf("could not form patch for the connection pooler deployment: %v", err) + } + + // An update probably requires RetryOnConflict, but since only one operator + // worker at one time will try to update it chances of conflicts are + // minimal. + deployment, err := KubeClient. + Deployments(newDeployment.Namespace).Patch( + context.TODO(), + newDeployment.Name, + types.MergePatchType, + patchData, + metav1.PatchOptions{}, + "") + if err != nil { + return nil, fmt.Errorf("could not patch connection pooler deployment: %v", err) + } + + return deployment, nil +} + +//updateConnectionPoolerAnnotations updates the annotations of connection pooler deployment +func updateConnectionPoolerAnnotations(KubeClient k8sutil.KubernetesClient, deployment *appsv1.Deployment, annotations map[string]string) (*appsv1.Deployment, error) { + patchData, err := metaAnnotationsPatch(annotations) + if err != nil { + return nil, fmt.Errorf("could not form patch for the connection pooler deployment metadata: %v", err) + } + result, err := KubeClient.Deployments(deployment.Namespace).Patch( + context.TODO(), + deployment.Name, + types.MergePatchType, + []byte(patchData), + metav1.PatchOptions{}, + "") + if err != nil { + return nil, fmt.Errorf("could not patch connection pooler annotations %q: %v", patchData, err) + } + return result, nil + +} + +// Test if two connection pooler configuration needs to be synced. For simplicity +// compare not the actual K8S objects, but the configuration itself and request +// sync if there is any difference. +func needSyncConnectionPoolerSpecs(oldSpec, newSpec *acidv1.ConnectionPooler) (sync bool, reasons []string) { + reasons = []string{} + sync = false + + changelog, err := diff.Diff(oldSpec, newSpec) + if err != nil { + //c.logger.Infof("Cannot get diff, do not do anything, %+v", err) + return false, reasons + } + + if len(changelog) > 0 { + sync = true + } + + for _, change := range changelog { + msg := fmt.Sprintf("%s %+v from '%+v' to '%+v'", + change.Type, change.Path, change.From, change.To) + reasons = append(reasons, msg) + } + + return sync, reasons +} + +// Check if we need to synchronize connection pooler deployment due to new +// defaults, that are different from what we see in the DeploymentSpec +func needSyncConnectionPoolerDefaults(Config *Config, spec *acidv1.ConnectionPooler, deployment *appsv1.Deployment) (sync bool, reasons []string) { + + reasons = []string{} + sync = false + + config := Config.OpConfig.ConnectionPooler + podTemplate := deployment.Spec.Template + poolerContainer := podTemplate.Spec.Containers[constants.ConnectionPoolerContainer] + + if spec == nil { + spec = &acidv1.ConnectionPooler{} + } + if spec.NumberOfInstances == nil && + *deployment.Spec.Replicas != *config.NumberOfInstances { + + sync = true + msg := fmt.Sprintf("NumberOfInstances is different (having %d, required %d)", + *deployment.Spec.Replicas, *config.NumberOfInstances) + reasons = append(reasons, msg) + } + + if spec.DockerImage == "" && + poolerContainer.Image != config.Image { + + sync = true + msg := fmt.Sprintf("DockerImage is different (having %s, required %s)", + poolerContainer.Image, config.Image) + reasons = append(reasons, msg) + } + + expectedResources, err := generateResourceRequirements(spec.Resources, + makeDefaultConnectionPoolerResources(&Config.OpConfig)) + + // An error to generate expected resources means something is not quite + // right, but for the purpose of robustness do not panic here, just report + // and ignore resources comparison (in the worst case there will be no + // updates for new resource values). + if err == nil && syncResources(&poolerContainer.Resources, expectedResources) { + sync = true + msg := fmt.Sprintf("Resources are different (having %+v, required %+v)", + poolerContainer.Resources, expectedResources) + reasons = append(reasons, msg) + } + + if err != nil { + return false, reasons + } + + for _, env := range poolerContainer.Env { + if spec.User == "" && env.Name == "PGUSER" { + ref := env.ValueFrom.SecretKeyRef.LocalObjectReference + secretName := Config.OpConfig.SecretNameTemplate.Format( + "username", strings.Replace(config.User, "_", "-", -1), + "cluster", deployment.ClusterName, + "tprkind", acidv1.PostgresCRDResourceKind, + "tprgroup", acidzalando.GroupName) + + if ref.Name != secretName { + sync = true + msg := fmt.Sprintf("pooler user is different (having %s, required %s)", + ref.Name, config.User) + reasons = append(reasons, msg) + } + } + + if spec.Schema == "" && env.Name == "PGSCHEMA" && env.Value != config.Schema { + sync = true + msg := fmt.Sprintf("pooler schema is different (having %s, required %s)", + env.Value, config.Schema) + reasons = append(reasons, msg) + } + } + + return sync, reasons +} + +// Generate default resource section for connection pooler deployment, to be +// used if nothing custom is specified in the manifest +func makeDefaultConnectionPoolerResources(config *config.Config) acidv1.Resources { + + defaultRequests := acidv1.ResourceDescription{ + CPU: config.ConnectionPooler.ConnectionPoolerDefaultCPURequest, + Memory: config.ConnectionPooler.ConnectionPoolerDefaultMemoryRequest, + } + defaultLimits := acidv1.ResourceDescription{ + CPU: config.ConnectionPooler.ConnectionPoolerDefaultCPULimit, + Memory: config.ConnectionPooler.ConnectionPoolerDefaultMemoryLimit, + } + + return acidv1.Resources{ + ResourceRequests: defaultRequests, + ResourceLimits: defaultLimits, + } +} + +func logPoolerEssentials(log *logrus.Entry, oldSpec, newSpec *acidv1.Postgresql) { + var v []string + + var input []*bool + if oldSpec == nil { + input = []*bool{nil, nil, newSpec.Spec.EnableConnectionPooler, newSpec.Spec.EnableReplicaConnectionPooler} + } else { + input = []*bool{oldSpec.Spec.EnableConnectionPooler, oldSpec.Spec.EnableReplicaConnectionPooler, newSpec.Spec.EnableConnectionPooler, newSpec.Spec.EnableReplicaConnectionPooler} + } + + for _, b := range input { + if b == nil { + v = append(v, "nil") + } else { + v = append(v, fmt.Sprintf("%v", *b)) + } + } + + log.Debugf("syncing connection pooler from (%v, %v) to (%v, %v)", v[0], v[1], v[2], v[3]) +} + +func (c *Cluster) syncConnectionPooler(oldSpec, newSpec *acidv1.Postgresql, LookupFunction InstallFunction) (SyncReason, error) { + logPoolerEssentials(c.logger, oldSpec, newSpec) + + var reason SyncReason + var err error + var newNeedConnectionPooler, oldNeedConnectionPooler bool + oldNeedConnectionPooler = false + + // Check and perform the sync requirements for each of the roles. + for _, role := range [2]PostgresRole{Master, Replica} { + + if role == Master { + newNeedConnectionPooler = needMasterConnectionPoolerWorker(&newSpec.Spec) + if oldSpec != nil { + oldNeedConnectionPooler = needMasterConnectionPoolerWorker(&oldSpec.Spec) + } + } else { + newNeedConnectionPooler = needReplicaConnectionPoolerWorker(&newSpec.Spec) + if oldSpec != nil { + oldNeedConnectionPooler = needReplicaConnectionPoolerWorker(&oldSpec.Spec) + } + } + + // if the call is via createConnectionPooler, then it is required to initialize + // the structure + if c.ConnectionPooler == nil { + c.ConnectionPooler = map[PostgresRole]*ConnectionPoolerObjects{} + } + if c.ConnectionPooler[role] == nil { + c.ConnectionPooler[role] = &ConnectionPoolerObjects{ + Deployment: nil, + Service: nil, + Name: c.connectionPoolerName(role), + ClusterName: c.ClusterName, + Namespace: c.Namespace, + LookupFunction: false, + Role: role, + } + } + + if newNeedConnectionPooler { + // Try to sync in any case. If we didn't needed connection pooler before, + // it means we want to create it. If it was already present, still sync + // since it could happen that there is no difference in specs, and all + // the resources are remembered, but the deployment was manually deleted + // in between + + // in this case also do not forget to install lookup function as for + // creating cluster + if !oldNeedConnectionPooler || !c.ConnectionPooler[role].LookupFunction { + newConnectionPooler := newSpec.Spec.ConnectionPooler + + specSchema := "" + specUser := "" + + if newConnectionPooler != nil { + specSchema = newConnectionPooler.Schema + specUser = newConnectionPooler.User + } + + schema := util.Coalesce( + specSchema, + c.OpConfig.ConnectionPooler.Schema) + + user := util.Coalesce( + specUser, + c.OpConfig.ConnectionPooler.User) + + if err = LookupFunction(schema, user, role); err != nil { + return NoSync, err + } + } + + if reason, err = c.syncConnectionPoolerWorker(oldSpec, newSpec, role); err != nil { + c.logger.Errorf("could not sync connection pooler: %v", err) + return reason, err + } + } else { + // delete and cleanup resources if they are already detected + if c.ConnectionPooler[role] != nil && + (c.ConnectionPooler[role].Deployment != nil || + c.ConnectionPooler[role].Service != nil) { + + if err = c.deleteConnectionPooler(role); err != nil { + c.logger.Warningf("could not remove connection pooler: %v", err) + } + } + } + } + if !needMasterConnectionPoolerWorker(&newSpec.Spec) && + !needReplicaConnectionPoolerWorker(&newSpec.Spec) { + if err = c.deleteConnectionPoolerSecret(); err != nil { + c.logger.Warningf("could not remove connection pooler secret: %v", err) + } + } + + return reason, nil +} + +// Synchronize connection pooler resources. Effectively we're interested only in +// synchronizing the corresponding deployment, but in case of deployment or +// service is missing, create it. After checking, also remember an object for +// the future references. +func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql, role PostgresRole) ( + SyncReason, error) { + + deployment, err := c.KubeClient. + Deployments(c.Namespace). + Get(context.TODO(), c.connectionPoolerName(role), metav1.GetOptions{}) + + if err != nil && k8sutil.ResourceNotFound(err) { + msg := "deployment %s for connection pooler synchronization is not found, create it" + c.logger.Warningf(msg, c.connectionPoolerName(role)) + + deploymentSpec, err := c.generateConnectionPoolerDeployment(c.ConnectionPooler[role]) + if err != nil { + msg = "could not generate deployment for connection pooler: %v" + return NoSync, fmt.Errorf(msg, err) + } + + deployment, err := c.KubeClient. + Deployments(deploymentSpec.Namespace). + Create(context.TODO(), deploymentSpec, metav1.CreateOptions{}) + + if err != nil { + return NoSync, err + } + c.ConnectionPooler[role].Deployment = deployment + } else if err != nil { + msg := "could not get connection pooler deployment to sync: %v" + return NoSync, fmt.Errorf(msg, err) + } else { + c.ConnectionPooler[role].Deployment = deployment + // actual synchronization + + var oldConnectionPooler *acidv1.ConnectionPooler + + if oldSpec != nil { + oldConnectionPooler = oldSpec.Spec.ConnectionPooler + } + + newConnectionPooler := newSpec.Spec.ConnectionPooler + // sync implementation below assumes that both old and new specs are + // not nil, but it can happen. To avoid any confusion like updating a + // deployment because the specification changed from nil to an empty + // struct (that was initialized somewhere before) replace any nil with + // an empty spec. + if oldConnectionPooler == nil { + oldConnectionPooler = &acidv1.ConnectionPooler{} + } + + if newConnectionPooler == nil { + newConnectionPooler = &acidv1.ConnectionPooler{} + } + + c.logger.Infof("old: %+v, new %+v", oldConnectionPooler, newConnectionPooler) + + var specSync bool + var specReason []string + + if oldSpec != nil { + specSync, specReason = needSyncConnectionPoolerSpecs(oldConnectionPooler, newConnectionPooler) + } + + defaultsSync, defaultsReason := needSyncConnectionPoolerDefaults(&c.Config, newConnectionPooler, deployment) + reason := append(specReason, defaultsReason...) + + if specSync || defaultsSync { + c.logger.Infof("Update connection pooler deployment %s, reason: %+v", + c.connectionPoolerName(role), reason) + newDeploymentSpec, err := c.generateConnectionPoolerDeployment(c.ConnectionPooler[role]) + if err != nil { + msg := "could not generate deployment for connection pooler: %v" + return reason, fmt.Errorf(msg, err) + } + + deployment, err := updateConnectionPoolerDeployment(c.KubeClient, + newDeploymentSpec) + + if err != nil { + return reason, err + } + c.ConnectionPooler[role].Deployment = deployment + } + } + + newAnnotations := c.AnnotationsToPropagate(c.ConnectionPooler[role].Deployment.Annotations) + if newAnnotations != nil { + deployment, err = updateConnectionPoolerAnnotations(c.KubeClient, c.ConnectionPooler[role].Deployment, newAnnotations) + if err != nil { + return nil, err + } + c.ConnectionPooler[role].Deployment = deployment + } + + service, err := c.KubeClient. + Services(c.Namespace). + Get(context.TODO(), c.connectionPoolerName(role), metav1.GetOptions{}) + + if err != nil && k8sutil.ResourceNotFound(err) { + msg := "Service %s for connection pooler synchronization is not found, create it" + c.logger.Warningf(msg, c.connectionPoolerName(role)) + + serviceSpec := c.generateConnectionPoolerService(c.ConnectionPooler[role]) + service, err := c.KubeClient. + Services(serviceSpec.Namespace). + Create(context.TODO(), serviceSpec, metav1.CreateOptions{}) + + if err != nil { + return NoSync, err + } + c.ConnectionPooler[role].Service = service + + } else if err != nil { + msg := "could not get connection pooler service to sync: %v" + return NoSync, fmt.Errorf(msg, err) + } else { + // Service updates are not supported and probably not that useful anyway + c.ConnectionPooler[role].Service = service + } + + return NoSync, nil +} diff --git a/pkg/cluster/connection_pooler_new_test.go b/pkg/cluster/connection_pooler_new_test.go new file mode 100644 index 000000000..72b3408e3 --- /dev/null +++ b/pkg/cluster/connection_pooler_new_test.go @@ -0,0 +1,45 @@ +package cluster + +import ( + "testing" + + "context" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "k8s.io/apimachinery/pkg/labels" + + "k8s.io/client-go/kubernetes/fake" +) + +func TestFakeClient(t *testing.T) { + clientSet := fake.NewSimpleClientset() + namespace := "default" + + l := labels.Set(map[string]string{ + "application": "spilo", + }) + + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-deployment1", + Namespace: namespace, + Labels: l, + }, + } + + clientSet.AppsV1().Deployments(namespace).Create(context.TODO(), deployment, metav1.CreateOptions{}) + + deployment2, _ := clientSet.AppsV1().Deployments(namespace).Get(context.TODO(), "my-deployment1", metav1.GetOptions{}) + + if deployment.ObjectMeta.Name != deployment2.ObjectMeta.Name { + t.Errorf("Deployments are not equal") + } + + deployments, _ := clientSet.AppsV1().Deployments(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: "application=spilo"}) + + if len(deployments.Items) != 1 { + t.Errorf("Label search does not work") + } +} diff --git a/pkg/cluster/connection_pooler_test.go b/pkg/cluster/connection_pooler_test.go new file mode 100644 index 000000000..b795fe14f --- /dev/null +++ b/pkg/cluster/connection_pooler_test.go @@ -0,0 +1,956 @@ +package cluster + +import ( + "errors" + "fmt" + "strings" + "testing" + + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + "github.com/zalando/postgres-operator/pkg/util/config" + "github.com/zalando/postgres-operator/pkg/util/k8sutil" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func mockInstallLookupFunction(schema string, user string, role PostgresRole) error { + return nil +} + +func boolToPointer(value bool) *bool { + return &value +} + +func int32ToPointer(value int32) *int32 { + return &value +} + +func TestConnectionPoolerCreationAndDeletion(t *testing.T) { + testName := "Test connection pooler creation" + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", + NumberOfInstances: int32ToPointer(1), + }, + }, + }, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger, eventRecorder) + + cluster.Statefulset = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sts", + }, + } + + cluster.Spec = acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + EnableReplicaConnectionPooler: boolToPointer(true), + } + + reason, err := cluster.createConnectionPooler(mockInstallLookupFunction) + + if err != nil { + t.Errorf("%s: Cannot create connection pooler, %s, %+v", + testName, err, reason) + } + for _, role := range [2]PostgresRole{Master, Replica} { + if cluster.ConnectionPooler[role] != nil { + if cluster.ConnectionPooler[role].Deployment == nil { + t.Errorf("%s: Connection pooler deployment is empty for role %s", testName, role) + } + + if cluster.ConnectionPooler[role].Service == nil { + t.Errorf("%s: Connection pooler service is empty for role %s", testName, role) + } + } + } + oldSpec := &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + EnableConnectionPooler: boolToPointer(true), + EnableReplicaConnectionPooler: boolToPointer(true), + }, + } + newSpec := &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + EnableConnectionPooler: boolToPointer(false), + EnableReplicaConnectionPooler: boolToPointer(false), + }, + } + + // Delete connection pooler via sync + _, err = cluster.syncConnectionPooler(oldSpec, newSpec, mockInstallLookupFunction) + if err != nil { + t.Errorf("%s: Cannot sync connection pooler, %s", testName, err) + } + + for _, role := range [2]PostgresRole{Master, Replica} { + err = cluster.deleteConnectionPooler(role) + if err != nil { + t.Errorf("%s: Cannot delete connection pooler, %s", testName, err) + } + } +} + +func TestNeedConnectionPooler(t *testing.T) { + testName := "Test how connection pooler can be enabled" + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", + }, + }, + }, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger, eventRecorder) + + cluster.Spec = acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + } + + if !needMasterConnectionPooler(&cluster.Spec) { + t.Errorf("%s: Connection pooler is not enabled with full definition", + testName) + } + + cluster.Spec = acidv1.PostgresSpec{ + EnableConnectionPooler: boolToPointer(true), + } + + if !needMasterConnectionPooler(&cluster.Spec) { + t.Errorf("%s: Connection pooler is not enabled with flag", + testName) + } + + cluster.Spec = acidv1.PostgresSpec{ + EnableConnectionPooler: boolToPointer(false), + ConnectionPooler: &acidv1.ConnectionPooler{}, + } + + if needMasterConnectionPooler(&cluster.Spec) { + t.Errorf("%s: Connection pooler is still enabled with flag being false", + testName) + } + + cluster.Spec = acidv1.PostgresSpec{ + EnableConnectionPooler: boolToPointer(true), + ConnectionPooler: &acidv1.ConnectionPooler{}, + } + + if !needMasterConnectionPooler(&cluster.Spec) { + t.Errorf("%s: Connection pooler is not enabled with flag and full", + testName) + } + + cluster.Spec = acidv1.PostgresSpec{ + EnableConnectionPooler: boolToPointer(false), + EnableReplicaConnectionPooler: boolToPointer(false), + ConnectionPooler: nil, + } + + if needMasterConnectionPooler(&cluster.Spec) { + t.Errorf("%s: Connection pooler is enabled with flag false and nil", + testName) + } + + // Test for replica connection pooler + cluster.Spec = acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + } + + if needReplicaConnectionPooler(&cluster.Spec) { + t.Errorf("%s: Replica Connection pooler is not enabled with full definition", + testName) + } + + cluster.Spec = acidv1.PostgresSpec{ + EnableReplicaConnectionPooler: boolToPointer(true), + } + + if !needReplicaConnectionPooler(&cluster.Spec) { + t.Errorf("%s: Replica Connection pooler is not enabled with flag", + testName) + } + + cluster.Spec = acidv1.PostgresSpec{ + EnableReplicaConnectionPooler: boolToPointer(false), + ConnectionPooler: &acidv1.ConnectionPooler{}, + } + + if needReplicaConnectionPooler(&cluster.Spec) { + t.Errorf("%s: Replica Connection pooler is still enabled with flag being false", + testName) + } + + cluster.Spec = acidv1.PostgresSpec{ + EnableReplicaConnectionPooler: boolToPointer(true), + ConnectionPooler: &acidv1.ConnectionPooler{}, + } + + if !needReplicaConnectionPooler(&cluster.Spec) { + t.Errorf("%s: Replica Connection pooler is not enabled with flag and full", + testName) + } +} + +func deploymentUpdated(cluster *Cluster, err error, reason SyncReason) error { + for _, role := range [2]PostgresRole{Master, Replica} { + if cluster.ConnectionPooler[role] != nil && cluster.ConnectionPooler[role].Deployment != nil && + (cluster.ConnectionPooler[role].Deployment.Spec.Replicas == nil || + *cluster.ConnectionPooler[role].Deployment.Spec.Replicas != 2) { + return fmt.Errorf("Wrong number of instances") + } + } + return nil +} + +func objectsAreSaved(cluster *Cluster, err error, reason SyncReason) error { + if cluster.ConnectionPooler == nil { + return fmt.Errorf("Connection pooler resources are empty") + } + + for _, role := range []PostgresRole{Master, Replica} { + if cluster.ConnectionPooler[role].Deployment == nil { + return fmt.Errorf("Deployment was not saved %s", role) + } + + if cluster.ConnectionPooler[role].Service == nil { + return fmt.Errorf("Service was not saved %s", role) + } + } + + return nil +} + +func MasterobjectsAreSaved(cluster *Cluster, err error, reason SyncReason) error { + if cluster.ConnectionPooler == nil { + return fmt.Errorf("Connection pooler resources are empty") + } + + if cluster.ConnectionPooler[Master].Deployment == nil { + return fmt.Errorf("Deployment was not saved") + } + + if cluster.ConnectionPooler[Master].Service == nil { + return fmt.Errorf("Service was not saved") + } + + return nil +} + +func ReplicaobjectsAreSaved(cluster *Cluster, err error, reason SyncReason) error { + if cluster.ConnectionPooler == nil { + return fmt.Errorf("Connection pooler resources are empty") + } + + if cluster.ConnectionPooler[Replica].Deployment == nil { + return fmt.Errorf("Deployment was not saved") + } + + if cluster.ConnectionPooler[Replica].Service == nil { + return fmt.Errorf("Service was not saved") + } + + return nil +} + +func objectsAreDeleted(cluster *Cluster, err error, reason SyncReason) error { + for _, role := range [2]PostgresRole{Master, Replica} { + if cluster.ConnectionPooler[role] != nil && + (cluster.ConnectionPooler[role].Deployment != nil || cluster.ConnectionPooler[role].Service != nil) { + return fmt.Errorf("Connection pooler was not deleted for role %v", role) + } + } + + return nil +} + +func OnlyMasterDeleted(cluster *Cluster, err error, reason SyncReason) error { + + if cluster.ConnectionPooler[Master] != nil && + (cluster.ConnectionPooler[Master].Deployment != nil || cluster.ConnectionPooler[Master].Service != nil) { + return fmt.Errorf("Connection pooler master was not deleted") + } + return nil +} + +func OnlyReplicaDeleted(cluster *Cluster, err error, reason SyncReason) error { + + if cluster.ConnectionPooler[Replica] != nil && + (cluster.ConnectionPooler[Replica].Deployment != nil || cluster.ConnectionPooler[Replica].Service != nil) { + return fmt.Errorf("Connection pooler replica was not deleted") + } + return nil +} + +func noEmptySync(cluster *Cluster, err error, reason SyncReason) error { + for _, msg := range reason { + if strings.HasPrefix(msg, "update [] from '' to '") { + return fmt.Errorf("There is an empty reason, %s", msg) + } + } + + return nil +} + +func TestConnectionPoolerSynchronization(t *testing.T) { + testName := "Test connection pooler synchronization" + newCluster := func(client k8sutil.KubernetesClient) *Cluster { + return New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", + NumberOfInstances: int32ToPointer(1), + }, + }, + }, client, acidv1.Postgresql{}, logger, eventRecorder) + } + cluster := newCluster(k8sutil.KubernetesClient{}) + + cluster.Statefulset = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sts", + }, + } + + tests := []struct { + subTest string + oldSpec *acidv1.Postgresql + newSpec *acidv1.Postgresql + cluster *Cluster + defaultImage string + defaultInstances int32 + check func(cluster *Cluster, err error, reason SyncReason) error + }{ + { + subTest: "create from scratch", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{}, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + }, + }, + cluster: newCluster(k8sutil.ClientMissingObjects()), + defaultImage: "pooler:1.0", + defaultInstances: 1, + check: MasterobjectsAreSaved, + }, + { + subTest: "create if doesn't exist", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + }, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + }, + }, + cluster: newCluster(k8sutil.ClientMissingObjects()), + defaultImage: "pooler:1.0", + defaultInstances: 1, + check: MasterobjectsAreSaved, + }, + { + subTest: "create if doesn't exist with a flag", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{}, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + EnableConnectionPooler: boolToPointer(true), + }, + }, + cluster: newCluster(k8sutil.ClientMissingObjects()), + defaultImage: "pooler:1.0", + defaultInstances: 1, + check: MasterobjectsAreSaved, + }, + { + subTest: "create no replica with flag", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{}, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + EnableReplicaConnectionPooler: boolToPointer(false), + }, + }, + cluster: newCluster(k8sutil.NewMockKubernetesClient()), + defaultImage: "pooler:1.0", + defaultInstances: 1, + check: objectsAreDeleted, + }, + { + subTest: "create replica if doesn't exist with a flag", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{}, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + EnableReplicaConnectionPooler: boolToPointer(true), + }, + }, + cluster: newCluster(k8sutil.NewMockKubernetesClient()), + defaultImage: "pooler:1.0", + defaultInstances: 1, + check: ReplicaobjectsAreSaved, + }, + { + subTest: "create both master and replica", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{}, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + EnableReplicaConnectionPooler: boolToPointer(true), + EnableConnectionPooler: boolToPointer(true), + }, + }, + cluster: newCluster(k8sutil.ClientMissingObjects()), + defaultImage: "pooler:1.0", + defaultInstances: 1, + check: objectsAreSaved, + }, + { + subTest: "delete only replica if not needed", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + EnableReplicaConnectionPooler: boolToPointer(true), + }, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + }, + }, + cluster: newCluster(k8sutil.NewMockKubernetesClient()), + defaultImage: "pooler:1.0", + defaultInstances: 1, + check: OnlyReplicaDeleted, + }, + { + subTest: "delete only master if not needed", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + EnableConnectionPooler: boolToPointer(true), + }, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + EnableReplicaConnectionPooler: boolToPointer(true), + }, + }, + cluster: newCluster(k8sutil.NewMockKubernetesClient()), + defaultImage: "pooler:1.0", + defaultInstances: 1, + check: OnlyMasterDeleted, + }, + { + subTest: "delete if not needed", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + }, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{}, + }, + cluster: newCluster(k8sutil.NewMockKubernetesClient()), + defaultImage: "pooler:1.0", + defaultInstances: 1, + check: objectsAreDeleted, + }, + { + subTest: "cleanup if still there", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{}, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{}, + }, + cluster: newCluster(k8sutil.NewMockKubernetesClient()), + defaultImage: "pooler:1.0", + defaultInstances: 1, + check: objectsAreDeleted, + }, + { + subTest: "update deployment", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{ + NumberOfInstances: int32ToPointer(1), + }, + }, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{ + NumberOfInstances: int32ToPointer(2), + }, + }, + }, + cluster: newCluster(k8sutil.NewMockKubernetesClient()), + defaultImage: "pooler:1.0", + defaultInstances: 1, + check: deploymentUpdated, + }, + { + subTest: "update deployment", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{ + NumberOfInstances: int32ToPointer(1), + }, + }, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{ + NumberOfInstances: int32ToPointer(2), + }, + }, + }, + cluster: newCluster(k8sutil.NewMockKubernetesClient()), + defaultImage: "pooler:1.0", + defaultInstances: 1, + check: deploymentUpdated, + }, + { + subTest: "update image from changed defaults", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + }, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + }, + }, + cluster: newCluster(k8sutil.NewMockKubernetesClient()), + defaultImage: "pooler:2.0", + defaultInstances: 2, + check: deploymentUpdated, + }, + { + subTest: "there is no sync from nil to an empty spec", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + EnableConnectionPooler: boolToPointer(true), + ConnectionPooler: nil, + }, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + EnableConnectionPooler: boolToPointer(true), + ConnectionPooler: &acidv1.ConnectionPooler{}, + }, + }, + cluster: newCluster(k8sutil.NewMockKubernetesClient()), + defaultImage: "pooler:1.0", + defaultInstances: 1, + check: noEmptySync, + }, + } + for _, tt := range tests { + tt.cluster.OpConfig.ConnectionPooler.Image = tt.defaultImage + tt.cluster.OpConfig.ConnectionPooler.NumberOfInstances = + int32ToPointer(tt.defaultInstances) + + reason, err := tt.cluster.syncConnectionPooler(tt.oldSpec, + tt.newSpec, mockInstallLookupFunction) + + if err := tt.check(tt.cluster, err, reason); err != nil { + t.Errorf("%s [%s]: Could not synchronize, %+v", + testName, tt.subTest, err) + } + } +} + +func TestConnectionPoolerPodSpec(t *testing.T) { + testName := "Test connection pooler pod template generation" + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPooler: config.ConnectionPooler{ + MaxDBConnections: int32ToPointer(60), + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) + + cluster.Spec = acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + EnableReplicaConnectionPooler: boolToPointer(true), + } + var clusterNoDefaultRes = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPooler: config.ConnectionPooler{}, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) + + clusterNoDefaultRes.Spec = acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + EnableReplicaConnectionPooler: boolToPointer(true), + } + + noCheck := func(cluster *Cluster, podSpec *v1.PodTemplateSpec, role PostgresRole) error { return nil } + + tests := []struct { + subTest string + spec *acidv1.PostgresSpec + expected error + cluster *Cluster + check func(cluster *Cluster, podSpec *v1.PodTemplateSpec, role PostgresRole) error + }{ + { + subTest: "default configuration", + spec: &acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + }, + expected: nil, + cluster: cluster, + check: noCheck, + }, + { + subTest: "no default resources", + spec: &acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + }, + expected: errors.New(`could not generate resource requirements: could not fill resource requests: could not parse default CPU quantity: quantities must match the regular expression '^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$'`), + cluster: clusterNoDefaultRes, + check: noCheck, + }, + { + subTest: "default resources are set", + spec: &acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + }, + expected: nil, + cluster: cluster, + check: testResources, + }, + { + subTest: "labels for service", + spec: &acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + EnableReplicaConnectionPooler: boolToPointer(true), + }, + expected: nil, + cluster: cluster, + check: testLabels, + }, + { + subTest: "required envs", + spec: &acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + }, + expected: nil, + cluster: cluster, + check: testEnvs, + }, + } + for _, role := range [2]PostgresRole{Master, Replica} { + for _, tt := range tests { + podSpec, err := tt.cluster.generateConnectionPoolerPodTemplate(role) + + if err != tt.expected && err.Error() != tt.expected.Error() { + t.Errorf("%s [%s]: Could not generate pod template,\n %+v, expected\n %+v", + testName, tt.subTest, err, tt.expected) + } + + err = tt.check(cluster, podSpec, role) + if err != nil { + t.Errorf("%s [%s]: Pod spec is incorrect, %+v", + testName, tt.subTest, err) + } + } + } +} + +func TestConnectionPoolerDeploymentSpec(t *testing.T) { + testName := "Test connection pooler deployment spec generation" + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) + cluster.Statefulset = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sts", + }, + } + cluster.ConnectionPooler = map[PostgresRole]*ConnectionPoolerObjects{ + Master: { + Deployment: nil, + Service: nil, + LookupFunction: true, + Name: "", + Role: Master, + }, + } + + noCheck := func(cluster *Cluster, deployment *appsv1.Deployment) error { + return nil + } + + tests := []struct { + subTest string + spec *acidv1.PostgresSpec + expected error + cluster *Cluster + check func(cluster *Cluster, deployment *appsv1.Deployment) error + }{ + { + subTest: "default configuration", + spec: &acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + EnableReplicaConnectionPooler: boolToPointer(true), + }, + expected: nil, + cluster: cluster, + check: noCheck, + }, + { + subTest: "owner reference", + spec: &acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + EnableReplicaConnectionPooler: boolToPointer(true), + }, + expected: nil, + cluster: cluster, + check: testDeploymentOwnwerReference, + }, + { + subTest: "selector", + spec: &acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + EnableReplicaConnectionPooler: boolToPointer(true), + }, + expected: nil, + cluster: cluster, + check: testSelector, + }, + } + for _, tt := range tests { + deployment, err := tt.cluster.generateConnectionPoolerDeployment(cluster.ConnectionPooler[Master]) + + if err != tt.expected && err.Error() != tt.expected.Error() { + t.Errorf("%s [%s]: Could not generate deployment spec,\n %+v, expected\n %+v", + testName, tt.subTest, err, tt.expected) + } + + err = tt.check(cluster, deployment) + if err != nil { + t.Errorf("%s [%s]: Deployment spec is incorrect, %+v", + testName, tt.subTest, err) + } + } +} + +func testResources(cluster *Cluster, podSpec *v1.PodTemplateSpec, role PostgresRole) error { + cpuReq := podSpec.Spec.Containers[0].Resources.Requests["cpu"] + if cpuReq.String() != cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultCPURequest { + return fmt.Errorf("CPU request doesn't match, got %s, expected %s", + cpuReq.String(), cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultCPURequest) + } + + memReq := podSpec.Spec.Containers[0].Resources.Requests["memory"] + if memReq.String() != cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultMemoryRequest { + return fmt.Errorf("Memory request doesn't match, got %s, expected %s", + memReq.String(), cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultMemoryRequest) + } + + cpuLim := podSpec.Spec.Containers[0].Resources.Limits["cpu"] + if cpuLim.String() != cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultCPULimit { + return fmt.Errorf("CPU limit doesn't match, got %s, expected %s", + cpuLim.String(), cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultCPULimit) + } + + memLim := podSpec.Spec.Containers[0].Resources.Limits["memory"] + if memLim.String() != cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultMemoryLimit { + return fmt.Errorf("Memory limit doesn't match, got %s, expected %s", + memLim.String(), cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultMemoryLimit) + } + + return nil +} + +func testLabels(cluster *Cluster, podSpec *v1.PodTemplateSpec, role PostgresRole) error { + poolerLabels := podSpec.ObjectMeta.Labels["connection-pooler"] + + if poolerLabels != cluster.connectionPoolerLabelsSelector(role).MatchLabels["connection-pooler"] { + return fmt.Errorf("Pod labels do not match, got %+v, expected %+v", + podSpec.ObjectMeta.Labels, cluster.connectionPoolerLabelsSelector(role).MatchLabels) + } + + return nil +} + +func testSelector(cluster *Cluster, deployment *appsv1.Deployment) error { + labels := deployment.Spec.Selector.MatchLabels + expected := cluster.connectionPoolerLabelsSelector(Master).MatchLabels + + if labels["connection-pooler"] != expected["connection-pooler"] { + return fmt.Errorf("Labels are incorrect, got %+v, expected %+v", + labels, expected) + } + + return nil +} + +func testServiceSelector(cluster *Cluster, service *v1.Service, role PostgresRole) error { + selector := service.Spec.Selector + + if selector["connection-pooler"] != cluster.connectionPoolerName(role) { + return fmt.Errorf("Selector is incorrect, got %s, expected %s", + selector["connection-pooler"], cluster.connectionPoolerName(role)) + } + + return nil +} + +func TestConnectionPoolerServiceSpec(t *testing.T) { + testName := "Test connection pooler service spec generation" + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) + cluster.Statefulset = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sts", + }, + } + cluster.ConnectionPooler = map[PostgresRole]*ConnectionPoolerObjects{ + Master: { + Deployment: nil, + Service: nil, + LookupFunction: false, + Role: Master, + }, + Replica: { + Deployment: nil, + Service: nil, + LookupFunction: false, + Role: Replica, + }, + } + + noCheck := func(cluster *Cluster, deployment *v1.Service, role PostgresRole) error { + return nil + } + + tests := []struct { + subTest string + spec *acidv1.PostgresSpec + cluster *Cluster + check func(cluster *Cluster, deployment *v1.Service, role PostgresRole) error + }{ + { + subTest: "default configuration", + spec: &acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + }, + cluster: cluster, + check: noCheck, + }, + { + subTest: "owner reference", + spec: &acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + }, + cluster: cluster, + check: testServiceOwnwerReference, + }, + { + subTest: "selector", + spec: &acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + EnableReplicaConnectionPooler: boolToPointer(true), + }, + cluster: cluster, + check: testServiceSelector, + }, + } + for _, role := range [2]PostgresRole{Master, Replica} { + for _, tt := range tests { + service := tt.cluster.generateConnectionPoolerService(tt.cluster.ConnectionPooler[role]) + + if err := tt.check(cluster, service, role); err != nil { + t.Errorf("%s [%s]: Service spec is incorrect, %+v", + testName, tt.subTest, err) + } + } + } +} diff --git a/pkg/cluster/database.go b/pkg/cluster/database.go index 5e05f443a..760b68d72 100644 --- a/pkg/cluster/database.go +++ b/pkg/cluster/database.go @@ -473,8 +473,8 @@ func (c *Cluster) execCreateOrAlterExtension(extName, schemaName, statement, doi } // Creates a connection pool credentials lookup function in every database to -// perform remote authentification. -func (c *Cluster) installLookupFunction(poolerSchema, poolerUser string) error { +// perform remote authentication. +func (c *Cluster) installLookupFunction(poolerSchema, poolerUser string, role PostgresRole) error { var stmtBytes bytes.Buffer c.logger.Info("Installing lookup function") @@ -567,12 +567,11 @@ func (c *Cluster) installLookupFunction(poolerSchema, poolerUser string) error { failedDatabases = append(failedDatabases, dbname) continue } - c.logger.Infof("pooler lookup function installed into %s", dbname) } if len(failedDatabases) == 0 { - c.ConnectionPooler.LookupFunction = true + c.ConnectionPooler[role].LookupFunction = true } return nil diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 2ca4ad4a8..50957e22a 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -75,10 +75,6 @@ func (c *Cluster) statefulSetName() string { return c.Name } -func (c *Cluster) connectionPoolerName() string { - return c.Name + "-pooler" -} - func (c *Cluster) endpointName(role PostgresRole) string { name := c.Name if role == Replica { @@ -142,26 +138,6 @@ func (c *Cluster) makeDefaultResources() acidv1.Resources { } } -// Generate default resource section for connection pooler deployment, to be -// used if nothing custom is specified in the manifest -func (c *Cluster) makeDefaultConnectionPoolerResources() acidv1.Resources { - config := c.OpConfig - - defaultRequests := acidv1.ResourceDescription{ - CPU: config.ConnectionPooler.ConnectionPoolerDefaultCPURequest, - Memory: config.ConnectionPooler.ConnectionPoolerDefaultMemoryRequest, - } - defaultLimits := acidv1.ResourceDescription{ - CPU: config.ConnectionPooler.ConnectionPoolerDefaultCPULimit, - Memory: config.ConnectionPooler.ConnectionPoolerDefaultMemoryLimit, - } - - return acidv1.Resources{ - ResourceRequests: defaultRequests, - ResourceLimits: defaultLimits, - } -} - func generateResourceRequirements(resources acidv1.Resources, defaultResources acidv1.Resources) (*v1.ResourceRequirements, error) { var err error @@ -2052,186 +2028,6 @@ func (c *Cluster) getLogicalBackupJobName() (jobName string) { return "logical-backup-" + c.clusterName().Name } -// Generate pool size related environment variables. -// -// MAX_DB_CONN would specify the global maximum for connections to a target -// database. -// -// MAX_CLIENT_CONN is not configurable at the moment, just set it high enough. -// -// DEFAULT_SIZE is a pool size per db/user (having in mind the use case when -// most of the queries coming through a connection pooler are from the same -// user to the same db). In case if we want to spin up more connection pooler -// instances, take this into account and maintain the same number of -// connections. -// -// MIN_SIZE is a pool's minimal size, to prevent situation when sudden workload -// have to wait for spinning up a new connections. -// -// RESERVE_SIZE is how many additional connections to allow for a pooler. -func (c *Cluster) getConnectionPoolerEnvVars(spec *acidv1.PostgresSpec) []v1.EnvVar { - effectiveMode := util.Coalesce( - spec.ConnectionPooler.Mode, - c.OpConfig.ConnectionPooler.Mode) - - numberOfInstances := spec.ConnectionPooler.NumberOfInstances - if numberOfInstances == nil { - numberOfInstances = util.CoalesceInt32( - c.OpConfig.ConnectionPooler.NumberOfInstances, - k8sutil.Int32ToPointer(1)) - } - - effectiveMaxDBConn := util.CoalesceInt32( - spec.ConnectionPooler.MaxDBConnections, - c.OpConfig.ConnectionPooler.MaxDBConnections) - - if effectiveMaxDBConn == nil { - effectiveMaxDBConn = k8sutil.Int32ToPointer( - constants.ConnectionPoolerMaxDBConnections) - } - - maxDBConn := *effectiveMaxDBConn / *numberOfInstances - - defaultSize := maxDBConn / 2 - minSize := defaultSize / 2 - reserveSize := minSize - - return []v1.EnvVar{ - { - Name: "CONNECTION_POOLER_PORT", - Value: fmt.Sprint(pgPort), - }, - { - Name: "CONNECTION_POOLER_MODE", - Value: effectiveMode, - }, - { - Name: "CONNECTION_POOLER_DEFAULT_SIZE", - Value: fmt.Sprint(defaultSize), - }, - { - Name: "CONNECTION_POOLER_MIN_SIZE", - Value: fmt.Sprint(minSize), - }, - { - Name: "CONNECTION_POOLER_RESERVE_SIZE", - Value: fmt.Sprint(reserveSize), - }, - { - Name: "CONNECTION_POOLER_MAX_CLIENT_CONN", - Value: fmt.Sprint(constants.ConnectionPoolerMaxClientConnections), - }, - { - Name: "CONNECTION_POOLER_MAX_DB_CONN", - Value: fmt.Sprint(maxDBConn), - }, - } -} - -func (c *Cluster) generateConnectionPoolerPodTemplate(spec *acidv1.PostgresSpec) ( - *v1.PodTemplateSpec, error) { - - gracePeriod := int64(c.OpConfig.PodTerminateGracePeriod.Seconds()) - resources, err := generateResourceRequirements( - spec.ConnectionPooler.Resources, - c.makeDefaultConnectionPoolerResources()) - - effectiveDockerImage := util.Coalesce( - spec.ConnectionPooler.DockerImage, - c.OpConfig.ConnectionPooler.Image) - - effectiveSchema := util.Coalesce( - spec.ConnectionPooler.Schema, - c.OpConfig.ConnectionPooler.Schema) - - if err != nil { - return nil, fmt.Errorf("could not generate resource requirements: %v", err) - } - - secretSelector := func(key string) *v1.SecretKeySelector { - effectiveUser := util.Coalesce( - spec.ConnectionPooler.User, - c.OpConfig.ConnectionPooler.User) - - return &v1.SecretKeySelector{ - LocalObjectReference: v1.LocalObjectReference{ - Name: c.credentialSecretName(effectiveUser), - }, - Key: key, - } - } - - envVars := []v1.EnvVar{ - { - Name: "PGHOST", - Value: c.serviceAddress(Master), - }, - { - Name: "PGPORT", - Value: c.servicePort(Master), - }, - { - Name: "PGUSER", - ValueFrom: &v1.EnvVarSource{ - SecretKeyRef: secretSelector("username"), - }, - }, - // the convention is to use the same schema name as - // connection pooler username - { - Name: "PGSCHEMA", - Value: effectiveSchema, - }, - { - Name: "PGPASSWORD", - ValueFrom: &v1.EnvVarSource{ - SecretKeyRef: secretSelector("password"), - }, - }, - } - - envVars = append(envVars, c.getConnectionPoolerEnvVars(spec)...) - - poolerContainer := v1.Container{ - Name: connectionPoolerContainer, - Image: effectiveDockerImage, - ImagePullPolicy: v1.PullIfNotPresent, - Resources: *resources, - Ports: []v1.ContainerPort{ - { - ContainerPort: pgPort, - Protocol: v1.ProtocolTCP, - }, - }, - Env: envVars, - ReadinessProbe: &v1.Probe{ - Handler: v1.Handler{ - TCPSocket: &v1.TCPSocketAction{ - Port: intstr.IntOrString{IntVal: pgPort}, - }, - }, - }, - } - - podTemplate := &v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: c.connectionPoolerLabelsSelector().MatchLabels, - Namespace: c.Namespace, - Annotations: c.generatePodAnnotations(spec), - }, - Spec: v1.PodSpec{ - ServiceAccountName: c.OpConfig.PodServiceAccountName, - TerminationGracePeriodSeconds: &gracePeriod, - Containers: []v1.Container{poolerContainer}, - // TODO: add tolerations to scheduler pooler on the same node - // as database - //Tolerations: *tolerationsSpec, - }, - } - - return podTemplate, nil -} - // Return an array of ownerReferences to make an arbitraty object dependent on // the StatefulSet. Dependency is made on StatefulSet instead of PostgreSQL CRD // while the former is represent the actual state, and only it's deletion means @@ -2257,108 +2053,6 @@ func (c *Cluster) ownerReferences() []metav1.OwnerReference { } } -func (c *Cluster) generateConnectionPoolerDeployment(spec *acidv1.PostgresSpec) ( - *appsv1.Deployment, error) { - - // there are two ways to enable connection pooler, either to specify a - // connectionPooler section or enableConnectionPooler. In the second case - // spec.connectionPooler will be nil, so to make it easier to calculate - // default values, initialize it to an empty structure. It could be done - // anywhere, but here is the earliest common entry point between sync and - // create code, so init here. - if spec.ConnectionPooler == nil { - spec.ConnectionPooler = &acidv1.ConnectionPooler{} - } - - podTemplate, err := c.generateConnectionPoolerPodTemplate(spec) - numberOfInstances := spec.ConnectionPooler.NumberOfInstances - if numberOfInstances == nil { - numberOfInstances = util.CoalesceInt32( - c.OpConfig.ConnectionPooler.NumberOfInstances, - k8sutil.Int32ToPointer(1)) - } - - if *numberOfInstances < constants.ConnectionPoolerMinInstances { - msg := "Adjusted number of connection pooler instances from %d to %d" - c.logger.Warningf(msg, numberOfInstances, constants.ConnectionPoolerMinInstances) - - *numberOfInstances = constants.ConnectionPoolerMinInstances - } - - if err != nil { - return nil, err - } - - deployment := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: c.connectionPoolerName(), - Namespace: c.Namespace, - Labels: c.connectionPoolerLabelsSelector().MatchLabels, - Annotations: map[string]string{}, - // make StatefulSet object its owner to represent the dependency. - // By itself StatefulSet is being deleted with "Orphaned" - // propagation policy, which means that it's deletion will not - // clean up this deployment, but there is a hope that this object - // will be garbage collected if something went wrong and operator - // didn't deleted it. - OwnerReferences: c.ownerReferences(), - }, - Spec: appsv1.DeploymentSpec{ - Replicas: numberOfInstances, - Selector: c.connectionPoolerLabelsSelector(), - Template: *podTemplate, - }, - } - - return deployment, nil -} - -func (c *Cluster) generateConnectionPoolerService(spec *acidv1.PostgresSpec) *v1.Service { - - // there are two ways to enable connection pooler, either to specify a - // connectionPooler section or enableConnectionPooler. In the second case - // spec.connectionPooler will be nil, so to make it easier to calculate - // default values, initialize it to an empty structure. It could be done - // anywhere, but here is the earliest common entry point between sync and - // create code, so init here. - if spec.ConnectionPooler == nil { - spec.ConnectionPooler = &acidv1.ConnectionPooler{} - } - - serviceSpec := v1.ServiceSpec{ - Ports: []v1.ServicePort{ - { - Name: c.connectionPoolerName(), - Port: pgPort, - TargetPort: intstr.IntOrString{StrVal: c.servicePort(Master)}, - }, - }, - Type: v1.ServiceTypeClusterIP, - Selector: map[string]string{ - "connection-pooler": c.connectionPoolerName(), - }, - } - - service := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: c.connectionPoolerName(), - Namespace: c.Namespace, - Labels: c.connectionPoolerLabelsSelector().MatchLabels, - Annotations: map[string]string{}, - // make StatefulSet object its owner to represent the dependency. - // By itself StatefulSet is being deleted with "Orphaned" - // propagation policy, which means that it's deletion will not - // clean up this service, but there is a hope that this object will - // be garbage collected if something went wrong and operator didn't - // deleted it. - OwnerReferences: c.ownerReferences(), - }, - Spec: serviceSpec, - } - - return service -} - func ensurePath(file string, defaultDir string, defaultFile string) string { if file == "" { return path.Join(defaultDir, defaultFile) diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index f1c0e968b..52c10cb8b 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -2,7 +2,6 @@ package cluster import ( "context" - "errors" "fmt" "reflect" "sort" @@ -840,46 +839,7 @@ func TestPodEnvironmentSecretVariables(t *testing.T) { } -func testResources(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { - cpuReq := podSpec.Spec.Containers[0].Resources.Requests["cpu"] - if cpuReq.String() != cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultCPURequest { - return fmt.Errorf("CPU request doesn't match, got %s, expected %s", - cpuReq.String(), cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultCPURequest) - } - - memReq := podSpec.Spec.Containers[0].Resources.Requests["memory"] - if memReq.String() != cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultMemoryRequest { - return fmt.Errorf("Memory request doesn't match, got %s, expected %s", - memReq.String(), cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultMemoryRequest) - } - - cpuLim := podSpec.Spec.Containers[0].Resources.Limits["cpu"] - if cpuLim.String() != cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultCPULimit { - return fmt.Errorf("CPU limit doesn't match, got %s, expected %s", - cpuLim.String(), cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultCPULimit) - } - - memLim := podSpec.Spec.Containers[0].Resources.Limits["memory"] - if memLim.String() != cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultMemoryLimit { - return fmt.Errorf("Memory limit doesn't match, got %s, expected %s", - memLim.String(), cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultMemoryLimit) - } - - return nil -} - -func testLabels(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { - poolerLabels := podSpec.ObjectMeta.Labels["connection-pooler"] - - if poolerLabels != cluster.connectionPoolerLabelsSelector().MatchLabels["connection-pooler"] { - return fmt.Errorf("Pod labels do not match, got %+v, expected %+v", - podSpec.ObjectMeta.Labels, cluster.connectionPoolerLabelsSelector().MatchLabels) - } - - return nil -} - -func testEnvs(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { +func testEnvs(cluster *Cluster, podSpec *v1.PodTemplateSpec, role PostgresRole) error { required := map[string]bool{ "PGHOST": false, "PGPORT": false, @@ -913,109 +873,6 @@ func testCustomPodTemplate(cluster *Cluster, podSpec *v1.PodTemplateSpec) error return nil } -func TestConnectionPoolerPodSpec(t *testing.T) { - testName := "Test connection pooler pod template generation" - var cluster = New( - Config{ - OpConfig: config.Config{ - ProtectedRoles: []string{"admin"}, - Auth: config.Auth{ - SuperUsername: superUserName, - ReplicationUsername: replicationUserName, - }, - ConnectionPooler: config.ConnectionPooler{ - MaxDBConnections: int32ToPointer(60), - ConnectionPoolerDefaultCPURequest: "100m", - ConnectionPoolerDefaultCPULimit: "100m", - ConnectionPoolerDefaultMemoryRequest: "100Mi", - ConnectionPoolerDefaultMemoryLimit: "100Mi", - }, - }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) - - var clusterNoDefaultRes = New( - Config{ - OpConfig: config.Config{ - ProtectedRoles: []string{"admin"}, - Auth: config.Auth{ - SuperUsername: superUserName, - ReplicationUsername: replicationUserName, - }, - ConnectionPooler: config.ConnectionPooler{}, - }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) - - noCheck := func(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { return nil } - - tests := []struct { - subTest string - spec *acidv1.PostgresSpec - expected error - cluster *Cluster - check func(cluster *Cluster, podSpec *v1.PodTemplateSpec) error - }{ - { - subTest: "default configuration", - spec: &acidv1.PostgresSpec{ - ConnectionPooler: &acidv1.ConnectionPooler{}, - }, - expected: nil, - cluster: cluster, - check: noCheck, - }, - { - subTest: "no default resources", - spec: &acidv1.PostgresSpec{ - ConnectionPooler: &acidv1.ConnectionPooler{}, - }, - expected: errors.New(`could not generate resource requirements: could not fill resource requests: could not parse default CPU quantity: quantities must match the regular expression '^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$'`), - cluster: clusterNoDefaultRes, - check: noCheck, - }, - { - subTest: "default resources are set", - spec: &acidv1.PostgresSpec{ - ConnectionPooler: &acidv1.ConnectionPooler{}, - }, - expected: nil, - cluster: cluster, - check: testResources, - }, - { - subTest: "labels for service", - spec: &acidv1.PostgresSpec{ - ConnectionPooler: &acidv1.ConnectionPooler{}, - }, - expected: nil, - cluster: cluster, - check: testLabels, - }, - { - subTest: "required envs", - spec: &acidv1.PostgresSpec{ - ConnectionPooler: &acidv1.ConnectionPooler{}, - }, - expected: nil, - cluster: cluster, - check: testEnvs, - }, - } - for _, tt := range tests { - podSpec, err := tt.cluster.generateConnectionPoolerPodTemplate(tt.spec) - - if err != tt.expected && err.Error() != tt.expected.Error() { - t.Errorf("%s [%s]: Could not generate pod template,\n %+v, expected\n %+v", - testName, tt.subTest, err, tt.expected) - } - - err = tt.check(cluster, podSpec) - if err != nil { - t.Errorf("%s [%s]: Pod spec is incorrect, %+v", - testName, tt.subTest, err) - } - } -} - func testDeploymentOwnwerReference(cluster *Cluster, deployment *appsv1.Deployment) error { owner := deployment.ObjectMeta.OwnerReferences[0] @@ -1027,98 +884,7 @@ func testDeploymentOwnwerReference(cluster *Cluster, deployment *appsv1.Deployme return nil } -func testSelector(cluster *Cluster, deployment *appsv1.Deployment) error { - labels := deployment.Spec.Selector.MatchLabels - expected := cluster.connectionPoolerLabelsSelector().MatchLabels - - if labels["connection-pooler"] != expected["connection-pooler"] { - return fmt.Errorf("Labels are incorrect, got %+v, expected %+v", - labels, expected) - } - - return nil -} - -func TestConnectionPoolerDeploymentSpec(t *testing.T) { - testName := "Test connection pooler deployment spec generation" - var cluster = New( - Config{ - OpConfig: config.Config{ - ProtectedRoles: []string{"admin"}, - Auth: config.Auth{ - SuperUsername: superUserName, - ReplicationUsername: replicationUserName, - }, - ConnectionPooler: config.ConnectionPooler{ - ConnectionPoolerDefaultCPURequest: "100m", - ConnectionPoolerDefaultCPULimit: "100m", - ConnectionPoolerDefaultMemoryRequest: "100Mi", - ConnectionPoolerDefaultMemoryLimit: "100Mi", - }, - }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) - cluster.Statefulset = &appsv1.StatefulSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-sts", - }, - } - - noCheck := func(cluster *Cluster, deployment *appsv1.Deployment) error { - return nil - } - - tests := []struct { - subTest string - spec *acidv1.PostgresSpec - expected error - cluster *Cluster - check func(cluster *Cluster, deployment *appsv1.Deployment) error - }{ - { - subTest: "default configuration", - spec: &acidv1.PostgresSpec{ - ConnectionPooler: &acidv1.ConnectionPooler{}, - }, - expected: nil, - cluster: cluster, - check: noCheck, - }, - { - subTest: "owner reference", - spec: &acidv1.PostgresSpec{ - ConnectionPooler: &acidv1.ConnectionPooler{}, - }, - expected: nil, - cluster: cluster, - check: testDeploymentOwnwerReference, - }, - { - subTest: "selector", - spec: &acidv1.PostgresSpec{ - ConnectionPooler: &acidv1.ConnectionPooler{}, - }, - expected: nil, - cluster: cluster, - check: testSelector, - }, - } - for _, tt := range tests { - deployment, err := tt.cluster.generateConnectionPoolerDeployment(tt.spec) - - if err != tt.expected && err.Error() != tt.expected.Error() { - t.Errorf("%s [%s]: Could not generate deployment spec,\n %+v, expected\n %+v", - testName, tt.subTest, err, tt.expected) - } - - err = tt.check(cluster, deployment) - if err != nil { - t.Errorf("%s [%s]: Deployment spec is incorrect, %+v", - testName, tt.subTest, err) - } - } -} - -func testServiceOwnwerReference(cluster *Cluster, service *v1.Service) error { +func testServiceOwnwerReference(cluster *Cluster, service *v1.Service, role PostgresRole) error { owner := service.ObjectMeta.OwnerReferences[0] if owner.Name != cluster.Statefulset.ObjectMeta.Name { @@ -1129,86 +895,6 @@ func testServiceOwnwerReference(cluster *Cluster, service *v1.Service) error { return nil } -func testServiceSelector(cluster *Cluster, service *v1.Service) error { - selector := service.Spec.Selector - - if selector["connection-pooler"] != cluster.connectionPoolerName() { - return fmt.Errorf("Selector is incorrect, got %s, expected %s", - selector["connection-pooler"], cluster.connectionPoolerName()) - } - - return nil -} - -func TestConnectionPoolerServiceSpec(t *testing.T) { - testName := "Test connection pooler service spec generation" - var cluster = New( - Config{ - OpConfig: config.Config{ - ProtectedRoles: []string{"admin"}, - Auth: config.Auth{ - SuperUsername: superUserName, - ReplicationUsername: replicationUserName, - }, - ConnectionPooler: config.ConnectionPooler{ - ConnectionPoolerDefaultCPURequest: "100m", - ConnectionPoolerDefaultCPULimit: "100m", - ConnectionPoolerDefaultMemoryRequest: "100Mi", - ConnectionPoolerDefaultMemoryLimit: "100Mi", - }, - }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) - cluster.Statefulset = &appsv1.StatefulSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-sts", - }, - } - - noCheck := func(cluster *Cluster, deployment *v1.Service) error { - return nil - } - - tests := []struct { - subTest string - spec *acidv1.PostgresSpec - cluster *Cluster - check func(cluster *Cluster, deployment *v1.Service) error - }{ - { - subTest: "default configuration", - spec: &acidv1.PostgresSpec{ - ConnectionPooler: &acidv1.ConnectionPooler{}, - }, - cluster: cluster, - check: noCheck, - }, - { - subTest: "owner reference", - spec: &acidv1.PostgresSpec{ - ConnectionPooler: &acidv1.ConnectionPooler{}, - }, - cluster: cluster, - check: testServiceOwnwerReference, - }, - { - subTest: "selector", - spec: &acidv1.PostgresSpec{ - ConnectionPooler: &acidv1.ConnectionPooler{}, - }, - cluster: cluster, - check: testServiceSelector, - }, - } - for _, tt := range tests { - service := tt.cluster.generateConnectionPoolerService(tt.spec) - - if err := tt.check(cluster, service); err != nil { - t.Errorf("%s [%s]: Service spec is incorrect, %+v", - testName, tt.subTest, err) - } - } -} - func TestTLS(t *testing.T) { var err error var spec acidv1.PostgresSpec diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index 4fb2c13c6..bcc568adc 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -94,150 +94,6 @@ func (c *Cluster) createStatefulSet() (*appsv1.StatefulSet, error) { return statefulSet, nil } -// Prepare the database for connection pooler to be used, i.e. install lookup -// function (do it first, because it should be fast and if it didn't succeed, -// it doesn't makes sense to create more K8S objects. At this moment we assume -// that necessary connection pooler user exists. -// -// After that create all the objects for connection pooler, namely a deployment -// with a chosen pooler and a service to expose it. -func (c *Cluster) createConnectionPooler(lookup InstallFunction) (*ConnectionPoolerObjects, error) { - var msg string - c.setProcessName("creating connection pooler") - - if c.ConnectionPooler == nil { - c.ConnectionPooler = &ConnectionPoolerObjects{} - } - - schema := c.Spec.ConnectionPooler.Schema - - if schema == "" { - schema = c.OpConfig.ConnectionPooler.Schema - } - - user := c.Spec.ConnectionPooler.User - if user == "" { - user = c.OpConfig.ConnectionPooler.User - } - - err := lookup(schema, user) - - if err != nil { - msg = "could not prepare database for connection pooler: %v" - return nil, fmt.Errorf(msg, err) - } - - deploymentSpec, err := c.generateConnectionPoolerDeployment(&c.Spec) - if err != nil { - msg = "could not generate deployment for connection pooler: %v" - return nil, fmt.Errorf(msg, err) - } - - // client-go does retry 10 times (with NoBackoff by default) when the API - // believe a request can be retried and returns Retry-After header. This - // should be good enough to not think about it here. - deployment, err := c.KubeClient. - Deployments(deploymentSpec.Namespace). - Create(context.TODO(), deploymentSpec, metav1.CreateOptions{}) - - if err != nil { - return nil, err - } - - serviceSpec := c.generateConnectionPoolerService(&c.Spec) - service, err := c.KubeClient. - Services(serviceSpec.Namespace). - Create(context.TODO(), serviceSpec, metav1.CreateOptions{}) - - if err != nil { - return nil, err - } - - c.ConnectionPooler = &ConnectionPoolerObjects{ - Deployment: deployment, - Service: service, - } - c.logger.Debugf("created new connection pooler %q, uid: %q", - util.NameFromMeta(deployment.ObjectMeta), deployment.UID) - - return c.ConnectionPooler, nil -} - -func (c *Cluster) deleteConnectionPooler() (err error) { - c.setProcessName("deleting connection pooler") - c.logger.Debugln("deleting connection pooler") - - // Lack of connection pooler objects is not a fatal error, just log it if - // it was present before in the manifest - if c.ConnectionPooler == nil { - c.logger.Infof("No connection pooler to delete") - return nil - } - - // Clean up the deployment object. If deployment resource we've remembered - // is somehow empty, try to delete based on what would we generate - deploymentName := c.connectionPoolerName() - deployment := c.ConnectionPooler.Deployment - - if deployment != nil { - deploymentName = deployment.Name - } - - // set delete propagation policy to foreground, so that replica set will be - // also deleted. - policy := metav1.DeletePropagationForeground - options := metav1.DeleteOptions{PropagationPolicy: &policy} - err = c.KubeClient. - Deployments(c.Namespace). - Delete(context.TODO(), deploymentName, options) - - if k8sutil.ResourceNotFound(err) { - c.logger.Debugf("Connection pooler deployment was already deleted") - } else if err != nil { - return fmt.Errorf("could not delete deployment: %v", err) - } - - c.logger.Infof("Connection pooler deployment %q has been deleted", deploymentName) - - // Repeat the same for the service object - service := c.ConnectionPooler.Service - serviceName := c.connectionPoolerName() - - if service != nil { - serviceName = service.Name - } - - err = c.KubeClient. - Services(c.Namespace). - Delete(context.TODO(), serviceName, options) - - if k8sutil.ResourceNotFound(err) { - c.logger.Debugf("Connection pooler service was already deleted") - } else if err != nil { - return fmt.Errorf("could not delete service: %v", err) - } - - c.logger.Infof("Connection pooler service %q has been deleted", serviceName) - - // Repeat the same for the secret object - secretName := c.credentialSecretName(c.OpConfig.ConnectionPooler.User) - - secret, err := c.KubeClient. - Secrets(c.Namespace). - Get(context.TODO(), secretName, metav1.GetOptions{}) - - if err != nil { - c.logger.Debugf("could not get connection pooler secret %q: %v", secretName, err) - } else { - if err = c.deleteSecret(secret.UID, *secret); err != nil { - return fmt.Errorf("could not delete pooler secret: %v", err) - } - } - - c.ConnectionPooler = nil - return nil -} - func getPodIndex(podName string) (int32, error) { parts := strings.Split(podName, "-") if len(parts) == 0 { @@ -852,57 +708,3 @@ func (c *Cluster) GetStatefulSet() *appsv1.StatefulSet { func (c *Cluster) GetPodDisruptionBudget() *policybeta1.PodDisruptionBudget { return c.PodDisruptionBudget } - -// Perform actual patching of a connection pooler deployment, assuming that all -// the check were already done before. -func (c *Cluster) updateConnectionPoolerDeployment(oldDeploymentSpec, newDeployment *appsv1.Deployment) (*appsv1.Deployment, error) { - c.setProcessName("updating connection pooler") - if c.ConnectionPooler == nil || c.ConnectionPooler.Deployment == nil { - return nil, fmt.Errorf("there is no connection pooler in the cluster") - } - - patchData, err := specPatch(newDeployment.Spec) - if err != nil { - return nil, fmt.Errorf("could not form patch for the deployment: %v", err) - } - - // An update probably requires RetryOnConflict, but since only one operator - // worker at one time will try to update it chances of conflicts are - // minimal. - deployment, err := c.KubeClient. - Deployments(c.ConnectionPooler.Deployment.Namespace).Patch( - context.TODO(), - c.ConnectionPooler.Deployment.Name, - types.MergePatchType, - patchData, - metav1.PatchOptions{}, - "") - if err != nil { - return nil, fmt.Errorf("could not patch deployment: %v", err) - } - - c.ConnectionPooler.Deployment = deployment - - return deployment, nil -} - -//updateConnectionPoolerAnnotations updates the annotations of connection pooler deployment -func (c *Cluster) updateConnectionPoolerAnnotations(annotations map[string]string) (*appsv1.Deployment, error) { - c.logger.Debugf("updating connection pooler annotations") - patchData, err := metaAnnotationsPatch(annotations) - if err != nil { - return nil, fmt.Errorf("could not form patch for the deployment metadata: %v", err) - } - result, err := c.KubeClient.Deployments(c.ConnectionPooler.Deployment.Namespace).Patch( - context.TODO(), - c.ConnectionPooler.Deployment.Name, - types.MergePatchType, - []byte(patchData), - metav1.PatchOptions{}, - "") - if err != nil { - return nil, fmt.Errorf("could not patch connection pooler annotations %q: %v", patchData, err) - } - return result, nil - -} diff --git a/pkg/cluster/resources_test.go b/pkg/cluster/resources_test.go deleted file mode 100644 index 9739cc354..000000000 --- a/pkg/cluster/resources_test.go +++ /dev/null @@ -1,127 +0,0 @@ -package cluster - -import ( - "testing" - - acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" - "github.com/zalando/postgres-operator/pkg/util/config" - "github.com/zalando/postgres-operator/pkg/util/k8sutil" - - appsv1 "k8s.io/api/apps/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func mockInstallLookupFunction(schema string, user string) error { - return nil -} - -func boolToPointer(value bool) *bool { - return &value -} - -func TestConnectionPoolerCreationAndDeletion(t *testing.T) { - testName := "Test connection pooler creation" - var cluster = New( - Config{ - OpConfig: config.Config{ - ProtectedRoles: []string{"admin"}, - Auth: config.Auth{ - SuperUsername: superUserName, - ReplicationUsername: replicationUserName, - }, - ConnectionPooler: config.ConnectionPooler{ - ConnectionPoolerDefaultCPURequest: "100m", - ConnectionPoolerDefaultCPULimit: "100m", - ConnectionPoolerDefaultMemoryRequest: "100Mi", - ConnectionPoolerDefaultMemoryLimit: "100Mi", - }, - }, - }, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger, eventRecorder) - - cluster.Statefulset = &appsv1.StatefulSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-sts", - }, - } - - cluster.Spec = acidv1.PostgresSpec{ - ConnectionPooler: &acidv1.ConnectionPooler{}, - } - poolerResources, err := cluster.createConnectionPooler(mockInstallLookupFunction) - - if err != nil { - t.Errorf("%s: Cannot create connection pooler, %s, %+v", - testName, err, poolerResources) - } - - if poolerResources.Deployment == nil { - t.Errorf("%s: Connection pooler deployment is empty", testName) - } - - if poolerResources.Service == nil { - t.Errorf("%s: Connection pooler service is empty", testName) - } - - err = cluster.deleteConnectionPooler() - if err != nil { - t.Errorf("%s: Cannot delete connection pooler, %s", testName, err) - } -} - -func TestNeedConnectionPooler(t *testing.T) { - testName := "Test how connection pooler can be enabled" - var cluster = New( - Config{ - OpConfig: config.Config{ - ProtectedRoles: []string{"admin"}, - Auth: config.Auth{ - SuperUsername: superUserName, - ReplicationUsername: replicationUserName, - }, - ConnectionPooler: config.ConnectionPooler{ - ConnectionPoolerDefaultCPURequest: "100m", - ConnectionPoolerDefaultCPULimit: "100m", - ConnectionPoolerDefaultMemoryRequest: "100Mi", - ConnectionPoolerDefaultMemoryLimit: "100Mi", - }, - }, - }, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger, eventRecorder) - - cluster.Spec = acidv1.PostgresSpec{ - ConnectionPooler: &acidv1.ConnectionPooler{}, - } - - if !cluster.needConnectionPooler() { - t.Errorf("%s: Connection pooler is not enabled with full definition", - testName) - } - - cluster.Spec = acidv1.PostgresSpec{ - EnableConnectionPooler: boolToPointer(true), - } - - if !cluster.needConnectionPooler() { - t.Errorf("%s: Connection pooler is not enabled with flag", - testName) - } - - cluster.Spec = acidv1.PostgresSpec{ - EnableConnectionPooler: boolToPointer(false), - ConnectionPooler: &acidv1.ConnectionPooler{}, - } - - if cluster.needConnectionPooler() { - t.Errorf("%s: Connection pooler is still enabled with flag being false", - testName) - } - - cluster.Spec = acidv1.PostgresSpec{ - EnableConnectionPooler: boolToPointer(true), - ConnectionPooler: &acidv1.ConnectionPooler{}, - } - - if !cluster.needConnectionPooler() { - t.Errorf("%s: Connection pooler is not enabled with flag and full", - testName) - } -} diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 61be7919d..e91adf757 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -43,15 +43,12 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { return err } - c.logger.Debugf("syncing secrets") - //TODO: mind the secrets of the deleted/new users if err = c.syncSecrets(); err != nil { err = fmt.Errorf("could not sync secrets: %v", err) return err } - c.logger.Debugf("syncing services") if err = c.syncServices(); err != nil { err = fmt.Errorf("could not sync services: %v", err) return err @@ -469,6 +466,7 @@ func (c *Cluster) syncSecrets() error { err error secret *v1.Secret ) + c.logger.Info("syncing secrets") c.setProcessName("syncing secrets") secrets := c.generateUserSecrets() @@ -547,7 +545,7 @@ func (c *Cluster) syncRoles() (err error) { userNames = append(userNames, u.Name) } - if c.needConnectionPooler() { + if needMasterConnectionPooler(&c.Spec) || needReplicaConnectionPooler(&c.Spec) { connectionPoolerUser := c.systemUsers[constants.ConnectionPoolerUserKeyName] userNames = append(userNames, connectionPoolerUser.Name) @@ -825,203 +823,3 @@ func (c *Cluster) syncLogicalBackupJob() error { return nil } - -func (c *Cluster) syncConnectionPooler(oldSpec, - newSpec *acidv1.Postgresql, - lookup InstallFunction) (SyncReason, error) { - - var reason SyncReason - var err error - - if c.ConnectionPooler == nil { - c.ConnectionPooler = &ConnectionPoolerObjects{ - LookupFunction: false, - } - } - - newNeedConnectionPooler := c.needConnectionPoolerWorker(&newSpec.Spec) - oldNeedConnectionPooler := c.needConnectionPoolerWorker(&oldSpec.Spec) - - if newNeedConnectionPooler { - // Try to sync in any case. If we didn't needed connection pooler before, - // it means we want to create it. If it was already present, still sync - // since it could happen that there is no difference in specs, and all - // the resources are remembered, but the deployment was manually deleted - // in between - c.logger.Debug("syncing connection pooler") - - // in this case also do not forget to install lookup function as for - // creating cluster - if !oldNeedConnectionPooler || !c.ConnectionPooler.LookupFunction { - newConnectionPooler := newSpec.Spec.ConnectionPooler - - specSchema := "" - specUser := "" - - if newConnectionPooler != nil { - specSchema = newConnectionPooler.Schema - specUser = newConnectionPooler.User - } - - schema := util.Coalesce( - specSchema, - c.OpConfig.ConnectionPooler.Schema) - - user := util.Coalesce( - specUser, - c.OpConfig.ConnectionPooler.User) - - if err = lookup(schema, user); err != nil { - return NoSync, err - } - } else { - // Lookup function installation seems to be a fragile point, so - // let's log for debugging if we skip it - msg := "Skip lookup function installation, old: %d, already installed %d" - c.logger.Debug(msg, oldNeedConnectionPooler, c.ConnectionPooler.LookupFunction) - } - - if reason, err = c.syncConnectionPoolerWorker(oldSpec, newSpec); err != nil { - c.logger.Errorf("could not sync connection pooler: %v", err) - return reason, err - } - } - - if oldNeedConnectionPooler && !newNeedConnectionPooler { - // delete and cleanup resources - if err = c.deleteConnectionPooler(); err != nil { - c.logger.Warningf("could not remove connection pooler: %v", err) - } - } - - if !oldNeedConnectionPooler && !newNeedConnectionPooler { - // delete and cleanup resources if not empty - if c.ConnectionPooler != nil && - (c.ConnectionPooler.Deployment != nil || - c.ConnectionPooler.Service != nil) { - - if err = c.deleteConnectionPooler(); err != nil { - c.logger.Warningf("could not remove connection pooler: %v", err) - } - } - } - - return reason, nil -} - -// Synchronize connection pooler resources. Effectively we're interested only in -// synchronizing the corresponding deployment, but in case of deployment or -// service is missing, create it. After checking, also remember an object for -// the future references. -func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql) ( - SyncReason, error) { - - deployment, err := c.KubeClient. - Deployments(c.Namespace). - Get(context.TODO(), c.connectionPoolerName(), metav1.GetOptions{}) - - if err != nil && k8sutil.ResourceNotFound(err) { - msg := "Deployment %s for connection pooler synchronization is not found, create it" - c.logger.Warningf(msg, c.connectionPoolerName()) - - deploymentSpec, err := c.generateConnectionPoolerDeployment(&newSpec.Spec) - if err != nil { - msg = "could not generate deployment for connection pooler: %v" - return NoSync, fmt.Errorf(msg, err) - } - - deployment, err := c.KubeClient. - Deployments(deploymentSpec.Namespace). - Create(context.TODO(), deploymentSpec, metav1.CreateOptions{}) - - if err != nil { - return NoSync, err - } - - c.ConnectionPooler.Deployment = deployment - } else if err != nil { - msg := "could not get connection pooler deployment to sync: %v" - return NoSync, fmt.Errorf(msg, err) - } else { - c.ConnectionPooler.Deployment = deployment - - // actual synchronization - oldConnectionPooler := oldSpec.Spec.ConnectionPooler - newConnectionPooler := newSpec.Spec.ConnectionPooler - - // sync implementation below assumes that both old and new specs are - // not nil, but it can happen. To avoid any confusion like updating a - // deployment because the specification changed from nil to an empty - // struct (that was initialized somewhere before) replace any nil with - // an empty spec. - if oldConnectionPooler == nil { - oldConnectionPooler = &acidv1.ConnectionPooler{} - } - - if newConnectionPooler == nil { - newConnectionPooler = &acidv1.ConnectionPooler{} - } - - logNiceDiff(c.logger, oldConnectionPooler, newConnectionPooler) - - specSync, specReason := c.needSyncConnectionPoolerSpecs(oldConnectionPooler, newConnectionPooler) - defaultsSync, defaultsReason := c.needSyncConnectionPoolerDefaults(newConnectionPooler, deployment) - reason := append(specReason, defaultsReason...) - if specSync || defaultsSync { - c.logger.Infof("Update connection pooler deployment %s, reason: %+v", - c.connectionPoolerName(), reason) - - newDeploymentSpec, err := c.generateConnectionPoolerDeployment(&newSpec.Spec) - if err != nil { - msg := "could not generate deployment for connection pooler: %v" - return reason, fmt.Errorf(msg, err) - } - - oldDeploymentSpec := c.ConnectionPooler.Deployment - - deployment, err := c.updateConnectionPoolerDeployment( - oldDeploymentSpec, - newDeploymentSpec) - - if err != nil { - return reason, err - } - - c.ConnectionPooler.Deployment = deployment - return reason, nil - } - } - - newAnnotations := c.AnnotationsToPropagate(c.ConnectionPooler.Deployment.Annotations) - if newAnnotations != nil { - c.updateConnectionPoolerAnnotations(newAnnotations) - } - - service, err := c.KubeClient. - Services(c.Namespace). - Get(context.TODO(), c.connectionPoolerName(), metav1.GetOptions{}) - - if err != nil && k8sutil.ResourceNotFound(err) { - msg := "Service %s for connection pooler synchronization is not found, create it" - c.logger.Warningf(msg, c.connectionPoolerName()) - - serviceSpec := c.generateConnectionPoolerService(&newSpec.Spec) - service, err := c.KubeClient. - Services(serviceSpec.Namespace). - Create(context.TODO(), serviceSpec, metav1.CreateOptions{}) - - if err != nil { - return NoSync, err - } - - c.ConnectionPooler.Service = service - } else if err != nil { - msg := "could not get connection pooler service to sync: %v" - return NoSync, fmt.Errorf(msg, err) - } else { - // Service updates are not supported and probably not that useful anyway - c.ConnectionPooler.Service = service - } - - return NoSync, nil -} diff --git a/pkg/cluster/sync_test.go b/pkg/cluster/sync_test.go deleted file mode 100644 index d9248ae33..000000000 --- a/pkg/cluster/sync_test.go +++ /dev/null @@ -1,264 +0,0 @@ -package cluster - -import ( - "fmt" - "strings" - "testing" - - acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" - "github.com/zalando/postgres-operator/pkg/util/config" - "github.com/zalando/postgres-operator/pkg/util/k8sutil" - - appsv1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func int32ToPointer(value int32) *int32 { - return &value -} - -func deploymentUpdated(cluster *Cluster, err error, reason SyncReason) error { - if cluster.ConnectionPooler.Deployment.Spec.Replicas == nil || - *cluster.ConnectionPooler.Deployment.Spec.Replicas != 2 { - return fmt.Errorf("Wrong nubmer of instances") - } - - return nil -} - -func objectsAreSaved(cluster *Cluster, err error, reason SyncReason) error { - if cluster.ConnectionPooler == nil { - return fmt.Errorf("Connection pooler resources are empty") - } - - if cluster.ConnectionPooler.Deployment == nil { - return fmt.Errorf("Deployment was not saved") - } - - if cluster.ConnectionPooler.Service == nil { - return fmt.Errorf("Service was not saved") - } - - return nil -} - -func objectsAreDeleted(cluster *Cluster, err error, reason SyncReason) error { - if cluster.ConnectionPooler != nil { - return fmt.Errorf("Connection pooler was not deleted") - } - - return nil -} - -func noEmptySync(cluster *Cluster, err error, reason SyncReason) error { - for _, msg := range reason { - if strings.HasPrefix(msg, "update [] from '' to '") { - return fmt.Errorf("There is an empty reason, %s", msg) - } - } - - return nil -} - -func TestConnectionPoolerSynchronization(t *testing.T) { - testName := "Test connection pooler synchronization" - newCluster := func() *Cluster { - return New( - Config{ - OpConfig: config.Config{ - ProtectedRoles: []string{"admin"}, - Auth: config.Auth{ - SuperUsername: superUserName, - ReplicationUsername: replicationUserName, - }, - ConnectionPooler: config.ConnectionPooler{ - ConnectionPoolerDefaultCPURequest: "100m", - ConnectionPoolerDefaultCPULimit: "100m", - ConnectionPoolerDefaultMemoryRequest: "100Mi", - ConnectionPoolerDefaultMemoryLimit: "100Mi", - NumberOfInstances: int32ToPointer(1), - }, - }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) - } - cluster := newCluster() - - cluster.Statefulset = &appsv1.StatefulSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-sts", - }, - } - - clusterMissingObjects := newCluster() - clusterMissingObjects.KubeClient = k8sutil.ClientMissingObjects() - - clusterMock := newCluster() - clusterMock.KubeClient = k8sutil.NewMockKubernetesClient() - - clusterDirtyMock := newCluster() - clusterDirtyMock.KubeClient = k8sutil.NewMockKubernetesClient() - clusterDirtyMock.ConnectionPooler = &ConnectionPoolerObjects{ - Deployment: &appsv1.Deployment{}, - Service: &v1.Service{}, - } - - clusterNewDefaultsMock := newCluster() - clusterNewDefaultsMock.KubeClient = k8sutil.NewMockKubernetesClient() - - tests := []struct { - subTest string - oldSpec *acidv1.Postgresql - newSpec *acidv1.Postgresql - cluster *Cluster - defaultImage string - defaultInstances int32 - check func(cluster *Cluster, err error, reason SyncReason) error - }{ - { - subTest: "create if doesn't exist", - oldSpec: &acidv1.Postgresql{ - Spec: acidv1.PostgresSpec{ - ConnectionPooler: &acidv1.ConnectionPooler{}, - }, - }, - newSpec: &acidv1.Postgresql{ - Spec: acidv1.PostgresSpec{ - ConnectionPooler: &acidv1.ConnectionPooler{}, - }, - }, - cluster: clusterMissingObjects, - defaultImage: "pooler:1.0", - defaultInstances: 1, - check: objectsAreSaved, - }, - { - subTest: "create if doesn't exist with a flag", - oldSpec: &acidv1.Postgresql{ - Spec: acidv1.PostgresSpec{}, - }, - newSpec: &acidv1.Postgresql{ - Spec: acidv1.PostgresSpec{ - EnableConnectionPooler: boolToPointer(true), - }, - }, - cluster: clusterMissingObjects, - defaultImage: "pooler:1.0", - defaultInstances: 1, - check: objectsAreSaved, - }, - { - subTest: "create from scratch", - oldSpec: &acidv1.Postgresql{ - Spec: acidv1.PostgresSpec{}, - }, - newSpec: &acidv1.Postgresql{ - Spec: acidv1.PostgresSpec{ - ConnectionPooler: &acidv1.ConnectionPooler{}, - }, - }, - cluster: clusterMissingObjects, - defaultImage: "pooler:1.0", - defaultInstances: 1, - check: objectsAreSaved, - }, - { - subTest: "delete if not needed", - oldSpec: &acidv1.Postgresql{ - Spec: acidv1.PostgresSpec{ - ConnectionPooler: &acidv1.ConnectionPooler{}, - }, - }, - newSpec: &acidv1.Postgresql{ - Spec: acidv1.PostgresSpec{}, - }, - cluster: clusterMock, - defaultImage: "pooler:1.0", - defaultInstances: 1, - check: objectsAreDeleted, - }, - { - subTest: "cleanup if still there", - oldSpec: &acidv1.Postgresql{ - Spec: acidv1.PostgresSpec{}, - }, - newSpec: &acidv1.Postgresql{ - Spec: acidv1.PostgresSpec{}, - }, - cluster: clusterDirtyMock, - defaultImage: "pooler:1.0", - defaultInstances: 1, - check: objectsAreDeleted, - }, - { - subTest: "update deployment", - oldSpec: &acidv1.Postgresql{ - Spec: acidv1.PostgresSpec{ - ConnectionPooler: &acidv1.ConnectionPooler{ - NumberOfInstances: int32ToPointer(1), - }, - }, - }, - newSpec: &acidv1.Postgresql{ - Spec: acidv1.PostgresSpec{ - ConnectionPooler: &acidv1.ConnectionPooler{ - NumberOfInstances: int32ToPointer(2), - }, - }, - }, - cluster: clusterMock, - defaultImage: "pooler:1.0", - defaultInstances: 1, - check: deploymentUpdated, - }, - { - subTest: "update image from changed defaults", - oldSpec: &acidv1.Postgresql{ - Spec: acidv1.PostgresSpec{ - ConnectionPooler: &acidv1.ConnectionPooler{}, - }, - }, - newSpec: &acidv1.Postgresql{ - Spec: acidv1.PostgresSpec{ - ConnectionPooler: &acidv1.ConnectionPooler{}, - }, - }, - cluster: clusterNewDefaultsMock, - defaultImage: "pooler:2.0", - defaultInstances: 2, - check: deploymentUpdated, - }, - { - subTest: "there is no sync from nil to an empty spec", - oldSpec: &acidv1.Postgresql{ - Spec: acidv1.PostgresSpec{ - EnableConnectionPooler: boolToPointer(true), - ConnectionPooler: nil, - }, - }, - newSpec: &acidv1.Postgresql{ - Spec: acidv1.PostgresSpec{ - EnableConnectionPooler: boolToPointer(true), - ConnectionPooler: &acidv1.ConnectionPooler{}, - }, - }, - cluster: clusterMock, - defaultImage: "pooler:1.0", - defaultInstances: 1, - check: noEmptySync, - }, - } - for _, tt := range tests { - tt.cluster.OpConfig.ConnectionPooler.Image = tt.defaultImage - tt.cluster.OpConfig.ConnectionPooler.NumberOfInstances = - int32ToPointer(tt.defaultInstances) - - reason, err := tt.cluster.syncConnectionPooler(tt.oldSpec, - tt.newSpec, mockInstallLookupFunction) - - if err := tt.check(tt.cluster, err, reason); err != nil { - t.Errorf("%s [%s]: Could not synchronize, %+v", - testName, tt.subTest, err) - } - } -} diff --git a/pkg/cluster/types.go b/pkg/cluster/types.go index 199914ccc..8aa519817 100644 --- a/pkg/cluster/types.go +++ b/pkg/cluster/types.go @@ -72,7 +72,7 @@ type ClusterStatus struct { type TemplateParams map[string]interface{} -type InstallFunction func(schema string, user string) error +type InstallFunction func(schema string, user string, role PostgresRole) error type SyncReason []string diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go index b8ddb7087..a2fdcb08e 100644 --- a/pkg/cluster/util.go +++ b/pkg/cluster/util.go @@ -449,28 +449,6 @@ func (c *Cluster) labelsSelector() *metav1.LabelSelector { } } -// Return connection pooler labels selector, which should from one point of view -// inherit most of the labels from the cluster itself, but at the same time -// have e.g. different `application` label, so that recreatePod operation will -// not interfere with it (it lists all the pods via labels, and if there would -// be no difference, it will recreate also pooler pods). -func (c *Cluster) connectionPoolerLabelsSelector() *metav1.LabelSelector { - connectionPoolerLabels := labels.Set(map[string]string{}) - - extraLabels := labels.Set(map[string]string{ - "connection-pooler": c.connectionPoolerName(), - "application": "db-connection-pooler", - }) - - connectionPoolerLabels = labels.Merge(connectionPoolerLabels, c.labelsSet(false)) - connectionPoolerLabels = labels.Merge(connectionPoolerLabels, extraLabels) - - return &metav1.LabelSelector{ - MatchLabels: connectionPoolerLabels, - MatchExpressions: nil, - } -} - func (c *Cluster) roleLabelsSet(shouldAddExtraLabels bool, role PostgresRole) labels.Set { lbls := c.labelsSet(shouldAddExtraLabels) lbls[c.OpConfig.PodRoleLabel] = string(role) @@ -553,18 +531,6 @@ func (c *Cluster) patroniKubernetesUseConfigMaps() bool { return c.OpConfig.KubernetesUseConfigMaps } -func (c *Cluster) needConnectionPoolerWorker(spec *acidv1.PostgresSpec) bool { - if spec.EnableConnectionPooler == nil { - return spec.ConnectionPooler != nil - } else { - return *spec.EnableConnectionPooler - } -} - -func (c *Cluster) needConnectionPooler() bool { - return c.needConnectionPoolerWorker(&c.Spec) -} - // Earlier arguments take priority func mergeContainers(containers ...[]v1.Container) ([]v1.Container, []string) { containerNameTaken := map[string]bool{} diff --git a/pkg/controller/postgresql.go b/pkg/controller/postgresql.go index 09ed83dc0..4b5d68fe5 100644 --- a/pkg/controller/postgresql.go +++ b/pkg/controller/postgresql.go @@ -473,7 +473,7 @@ func (c *Controller) queueClusterEvent(informerOldSpec, informerNewSpec *acidv1. if err := c.clusterEventQueues[workerID].Add(clusterEvent); err != nil { lg.Errorf("error while queueing cluster event: %v", clusterEvent) } - lg.Infof("%q event has been queued", eventType) + lg.Infof("%s event has been queued", eventType) if eventType != EventDelete { return @@ -494,7 +494,7 @@ func (c *Controller) queueClusterEvent(informerOldSpec, informerNewSpec *acidv1. if err != nil { lg.Warningf("could not delete event from the queue: %v", err) } else { - lg.Debugf("event %q has been discarded for the cluster", evType) + lg.Debugf("event %s has been discarded for the cluster", evType) } } } diff --git a/ui/operator_ui/main.py b/ui/operator_ui/main.py index d159bee2d..f1242fa37 100644 --- a/ui/operator_ui/main.py +++ b/ui/operator_ui/main.py @@ -620,6 +620,17 @@ def update_postgresql(namespace: str, cluster: str): if 'enableConnectionPooler' in o['spec']: del o['spec']['enableConnectionPooler'] + if 'enableReplicaConnectionPooler' in postgresql['spec']: + cp = postgresql['spec']['enableReplicaConnectionPooler'] + if not cp: + if 'enableReplicaConnectionPooler' in o['spec']: + del o['spec']['enableReplicaConnectionPooler'] + else: + spec['enableReplicaConnectionPooler'] = True + else: + if 'enableReplicaConnectionPooler' in o['spec']: + del o['spec']['enableReplicaConnectionPooler'] + if 'enableReplicaLoadBalancer' in postgresql['spec']: rlb = postgresql['spec']['enableReplicaLoadBalancer'] if not rlb: From a7f453352a722cf7ba0070199cbd1c50577ccdb8 Mon Sep 17 00:00:00 2001 From: Jan Mussler Date: Mon, 16 Nov 2020 10:15:47 +0100 Subject: [PATCH 117/168] Use Github.com actions to run tests and e2e tests. (#1215) * Use GH action to run tests and end 2 end tests. * Remove travis. --- .github/workflows/run_e2e.yaml | 24 ++++++++++++++++++++++++ .travis.yml | 23 ----------------------- delivery.yaml | 2 +- 3 files changed, 25 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/run_e2e.yaml delete mode 100644 .travis.yml diff --git a/.github/workflows/run_e2e.yaml b/.github/workflows/run_e2e.yaml new file mode 100644 index 000000000..9f9849284 --- /dev/null +++ b/.github/workflows/run_e2e.yaml @@ -0,0 +1,24 @@ +name: ubuntu + +on: + pull_request: + push: + branches: + - master + +jobs: + tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-go@v2 + with: + go-version: "^1.15.5" + - name: Make dependencies + run: make deps + - name: Compile + run: make linux + - name: Run unit tests + run: go test ./... + - name: Run end-2-end tests + run: make e2e diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index a52769c91..000000000 --- a/.travis.yml +++ /dev/null @@ -1,23 +0,0 @@ -dist: trusty -sudo: false - -branches: - only: - - master - -language: go - -go: - - "1.14.x" - -before_install: - - go get github.com/mattn/goveralls - -install: - - make deps - -script: - - hack/verify-codegen.sh - - travis_wait 20 go test -race -covermode atomic -coverprofile=profile.cov ./pkg/... -v - - goveralls -coverprofile=profile.cov -service=travis-ci -v - - make e2e diff --git a/delivery.yaml b/delivery.yaml index d1eec8a2b..94c94b246 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -16,7 +16,7 @@ pipeline: - desc: 'Install go' cmd: | cd /tmp - wget -q https://storage.googleapis.com/golang/go1.14.7.linux-amd64.tar.gz -O go.tar.gz + wget -q https://storage.googleapis.com/golang/go1.15.5.linux-amd64.tar.gz -O go.tar.gz tar -xf go.tar.gz mv go /usr/local ln -s /usr/local/go/bin/go /usr/bin/go From 3e42e8a8968504a92caf072335593affa7e4fc9b Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 16 Nov 2020 10:29:01 +0100 Subject: [PATCH 118/168] CRD: preserve unknown fields and add to all category (#1212) * CRD: preserve unknown fields and add to all category * allow Pg13 * left over --- .../crds/operatorconfigurations.yaml | 3 +++ charts/postgres-operator/crds/postgresqls.yaml | 4 ++++ charts/postgres-operator/crds/postgresteams.yaml | 3 +++ manifests/operatorconfiguration.crd.yaml | 3 +++ manifests/postgresql.crd.yaml | 4 ++++ manifests/postgresteam.crd.yaml | 3 +++ pkg/apis/acid.zalan.do/v1/crds.go | 15 +++++++++++---- 7 files changed, 31 insertions(+), 4 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 28b8f28ca..57424374b 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -15,6 +15,8 @@ spec: singular: operatorconfiguration shortNames: - opconfig + categories: + - all scope: Namespaced versions: - name: v1 @@ -45,6 +47,7 @@ spec: schema: openAPIV3Schema: type: object + x-preserve-unknown-fields: true required: - kind - apiVersion diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 9127fa86e..c5dcfd247 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -15,6 +15,8 @@ spec: singular: postgresql shortNames: - pg + categories: + - all scope: Namespaced versions: - name: v1 @@ -57,6 +59,7 @@ spec: schema: openAPIV3Schema: type: object + x-preserve-unknown-fields: true required: - kind - apiVersion @@ -320,6 +323,7 @@ spec: - "10" - "11" - "12" + - "13" parameters: type: object additionalProperties: diff --git a/charts/postgres-operator/crds/postgresteams.yaml b/charts/postgres-operator/crds/postgresteams.yaml index 81c5e1eaf..8f40fc661 100644 --- a/charts/postgres-operator/crds/postgresteams.yaml +++ b/charts/postgres-operator/crds/postgresteams.yaml @@ -15,6 +15,8 @@ spec: singular: postgresteam shortNames: - pgteam + categories: + - all scope: Namespaced versions: - name: v1 @@ -25,6 +27,7 @@ spec: schema: openAPIV3Schema: type: object + x-preserve-unknown-fields: true required: - kind - apiVersion diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 0987467aa..c42f4a1c2 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -11,6 +11,8 @@ spec: singular: operatorconfiguration shortNames: - opconfig + categories: + - all scope: Namespaced versions: - name: v1 @@ -41,6 +43,7 @@ spec: schema: openAPIV3Schema: type: object + x-preserve-unknown-fields: true required: - kind - apiVersion diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 5ee05f444..0926c21e6 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -11,6 +11,8 @@ spec: singular: postgresql shortNames: - pg + categories: + - all scope: Namespaced versions: - name: v1 @@ -53,6 +55,7 @@ spec: schema: openAPIV3Schema: type: object + x-preserve-unknown-fields: true required: - kind - apiVersion @@ -316,6 +319,7 @@ spec: - "10" - "11" - "12" + - "13" parameters: type: object additionalProperties: diff --git a/manifests/postgresteam.crd.yaml b/manifests/postgresteam.crd.yaml index 645c8848d..845414aea 100644 --- a/manifests/postgresteam.crd.yaml +++ b/manifests/postgresteam.crd.yaml @@ -11,6 +11,8 @@ spec: singular: postgresteam shortNames: - pgteam + categories: + - all scope: Namespaced versions: - name: v1 @@ -21,6 +23,7 @@ spec: schema: openAPIV3Schema: type: object + x-preserve-unknown-fields: true required: - kind - apiVersion diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 2ed0d6b01..79332d597 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -2,6 +2,7 @@ package v1 import ( acidzalando "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do" + "github.com/zalando/postgres-operator/pkg/util" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -111,8 +112,9 @@ var minDisable = -1.0 // PostgresCRDResourceValidation to check applied manifest parameters var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ OpenAPIV3Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - Required: []string{"kind", "apiVersion", "spec"}, + Type: "object", + XPreserveUnknownFields: util.True(), + Required: []string{"kind", "apiVersion", "spec"}, Properties: map[string]apiextv1.JSONSchemaProps{ "kind": { Type: "string", @@ -412,6 +414,9 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ { Raw: []byte(`"12"`), }, + { + Raw: []byte(`"13"`), + }, }, }, "parameters": { @@ -779,8 +784,9 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ // OperatorConfigCRDResourceValidation to check applied manifest parameters var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{ OpenAPIV3Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - Required: []string{"kind", "apiVersion", "configuration"}, + Type: "object", + XPreserveUnknownFields: util.True(), + Required: []string{"kind", "apiVersion", "configuration"}, Properties: map[string]apiextv1.JSONSchemaProps{ "kind": { Type: "string", @@ -1394,6 +1400,7 @@ func buildCRD(name, kind, plural, short string, columns []apiextv1.CustomResourc Plural: plural, ShortNames: []string{short}, Kind: kind, + Categories: []string{"all"}, }, Scope: apiextv1.NamespaceScoped, Versions: []apiextv1.CustomResourceDefinitionVersion{ From 65d1a71cc9a3cbb2f465c359488626825909558b Mon Sep 17 00:00:00 2001 From: Jan Mussler Date: Mon, 16 Nov 2020 10:30:18 +0100 Subject: [PATCH 119/168] Update README.md (#1217) Removed all the badges, they don't work anyways and are off. --- README.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/README.md b/README.md index 28532e246..98d902354 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,5 @@ # Postgres Operator -[![Build Status](https://travis-ci.org/zalando/postgres-operator.svg?branch=master)](https://travis-ci.org/zalando/postgres-operator) -[![Coverage Status](https://coveralls.io/repos/github/zalando/postgres-operator/badge.svg)](https://coveralls.io/github/zalando/postgres-operator) -[![Go Report Card](https://goreportcard.com/badge/github.com/zalando/postgres-operator)](https://goreportcard.com/report/github.com/zalando/postgres-operator) -[![GoDoc](https://godoc.org/github.com/zalando/postgres-operator?status.svg)](https://godoc.org/github.com/zalando/postgres-operator) -[![golangci](https://golangci.com/badges/github.com/zalando/postgres-operator.svg)](https://golangci.com/r/github.com/zalando/postgres-operator) - The Postgres Operator delivers an easy to run highly-available [PostgreSQL](https://www.postgresql.org/) From 67d1b4b16709b46bdd0d39efdacdad824087376d Mon Sep 17 00:00:00 2001 From: Jan Mussler Date: Mon, 16 Nov 2020 12:08:28 +0100 Subject: [PATCH 120/168] compile coverage report and add badges. (#1218) Add GH badges, and added coveralls support again. --- .github/workflows/run_e2e.yaml | 3 ++- .github/workflows/run_tests.yaml | 30 ++++++++++++++++++++++++++++++ README.md | 4 ++++ 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/run_tests.yaml diff --git a/.github/workflows/run_e2e.yaml b/.github/workflows/run_e2e.yaml index 9f9849284..b241f89f1 100644 --- a/.github/workflows/run_e2e.yaml +++ b/.github/workflows/run_e2e.yaml @@ -1,4 +1,4 @@ -name: ubuntu +name: operator-e2e-tests on: pull_request: @@ -8,6 +8,7 @@ on: jobs: tests: + name: End-2-End tests runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml new file mode 100644 index 000000000..4f9b3a5bb --- /dev/null +++ b/.github/workflows/run_tests.yaml @@ -0,0 +1,30 @@ +name: operator-tests + +on: + pull_request: + push: + branches: + - master + +jobs: + tests: + name: Unit tests and coverage + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-go@v2 + with: + go-version: "^1.15.5" + - name: Make dependencies + run: make deps + - name: Compile + run: make linux + - name: Run unit tests + run: go test -race -covermode atomic -coverprofile=coverage.out ./... + - name: Convert coverage to lcov + uses: jandelgado/gcov2lcov-action@v1.0.5 + - name: Coveralls + uses: coverallsapp/github-action@master + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + path-to-lcov: coverage.lcov diff --git a/README.md b/README.md index 98d902354..5e00dba8b 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # Postgres Operator +![Tests](https://github.com/zalando/postgres-operator/workflows/operator-tests/badge.svg) +![E2E Tests](https://github.com/zalando/postgres-operator/workflows/operator-e2e-tests/badge.svg) +[![Coverage Status](https://coveralls.io/repos/github/zalando/postgres-operator/badge.svg?branch=master)](https://coveralls.io/github/zalando/postgres-operator?branch=master) + The Postgres Operator delivers an easy to run highly-available [PostgreSQL](https://www.postgresql.org/) From 580883bc591de72375f914a08a443e9bae1dd752 Mon Sep 17 00:00:00 2001 From: Thunderbolt Date: Tue, 17 Nov 2020 10:47:35 +0100 Subject: [PATCH 121/168] add operator ui helm chart value imagePullSecret (#1211) --- charts/postgres-operator-ui/templates/deployment.yaml | 4 ++++ charts/postgres-operator-ui/values.yaml | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/charts/postgres-operator-ui/templates/deployment.yaml b/charts/postgres-operator-ui/templates/deployment.yaml index 4c6d46689..5733a1c35 100644 --- a/charts/postgres-operator-ui/templates/deployment.yaml +++ b/charts/postgres-operator-ui/templates/deployment.yaml @@ -21,6 +21,10 @@ spec: team: "acid" # Parameterize? spec: serviceAccountName: {{ include "postgres-operator-ui.serviceAccountName" . }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{ toYaml .Values.imagePullSecrets | indent 8 }} + {{- end }} containers: - name: "service" image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}" diff --git a/charts/postgres-operator-ui/values.yaml b/charts/postgres-operator-ui/values.yaml index 2fdb8f894..55c1d2452 100644 --- a/charts/postgres-operator-ui/values.yaml +++ b/charts/postgres-operator-ui/values.yaml @@ -11,6 +11,12 @@ image: tag: v1.5.0-dirty pullPolicy: "IfNotPresent" +# Optionally specify an array of imagePullSecrets. +# Secrets must be manually created in the namespace. +# ref: https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod +# imagePullSecrets: +# - name: + rbac: # Specifies whether RBAC resources should be created create: true From c4ae11629b2f84aba8cabd62afa28b09af1c6241 Mon Sep 17 00:00:00 2001 From: Jan Mussler Date: Mon, 23 Nov 2020 17:18:18 +0100 Subject: [PATCH 122/168] Fix connection pooler deployment selectors (#1213) Stick with the existing pooler deployment selector labels to make it compatible with existing deployments. Make the use of additional labels clear and avoid where not needed. Deployment Selector and Service Selector now do not use extra labels, pod spec does. --- e2e/scripts/watch_objects.sh | 3 +- e2e/tests/test_e2e.py | 34 ++++++++++++++++++ manifests/minimal-fake-pooler-deployment.yaml | 35 +++++++++++++++++++ manifests/postgres-operator.yaml | 2 ++ pkg/cluster/connection_pooler.go | 32 ++++++++--------- pkg/cluster/connection_pooler_test.go | 6 ++-- 6 files changed, 92 insertions(+), 20 deletions(-) create mode 100644 manifests/minimal-fake-pooler-deployment.yaml diff --git a/e2e/scripts/watch_objects.sh b/e2e/scripts/watch_objects.sh index 52364f247..4c9b82404 100755 --- a/e2e/scripts/watch_objects.sh +++ b/e2e/scripts/watch_objects.sh @@ -16,7 +16,8 @@ echo 'Statefulsets' kubectl get statefulsets --all-namespaces echo echo 'Deployments' -kubectl get deployments --all-namespaces -l application=db-connection-pooler -l name=postgres-operator +kubectl get deployments --all-namespaces -l application=db-connection-pooler +kubectl get deployments --all-namespaces -l application=postgres-operator echo echo echo 'Step from operator deployment' diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index f863123bd..95c7748cb 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -152,6 +152,40 @@ class EndToEndTestCase(unittest.TestCase): print('Operator log: {}'.format(k8s.get_operator_log())) raise + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_overwrite_pooler_deployment(self): + self.k8s.create_with_kubectl("manifests/minimal-fake-pooler-deployment.yaml") + self.eventuallyEqual(lambda: self.k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + self.eventuallyEqual(lambda: self.k8s.get_deployment_replica_count(name="acid-minimal-cluster-pooler"), 1, + "Initial broken deplyment not rolled out") + + self.k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'enableConnectionPooler': True + } + }) + + self.eventuallyEqual(lambda: self.k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + self.eventuallyEqual(lambda: self.k8s.get_deployment_replica_count(name="acid-minimal-cluster-pooler"), 2, + "Operator did not succeed in overwriting labels") + + self.k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'enableConnectionPooler': False + } + }) + + self.eventuallyEqual(lambda: self.k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + self.eventuallyEqual(lambda: self.k8s.count_running_pods("connection-pooler=acid-minimal-cluster-pooler"), + 0, "Pooler pods not scaled down") + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_enable_disable_connection_pooler(self): ''' diff --git a/manifests/minimal-fake-pooler-deployment.yaml b/manifests/minimal-fake-pooler-deployment.yaml new file mode 100644 index 000000000..0406b195a --- /dev/null +++ b/manifests/minimal-fake-pooler-deployment.yaml @@ -0,0 +1,35 @@ +# will not run but is good enough for tests to fail +apiVersion: apps/v1 +kind: Deployment +metadata: + name: acid-minimal-cluster-pooler + labels: + application: db-connection-pooler + connection-pooler: acid-minimal-cluster-pooler +spec: + replicas: 1 + selector: + matchLabels: + application: db-connection-pooler + connection-pooler: acid-minimal-cluster-pooler + cluster-name: acid-minimal-cluster + template: + metadata: + labels: + application: db-connection-pooler + connection-pooler: acid-minimal-cluster-pooler + cluster-name: acid-minimal-cluster + spec: + serviceAccountName: postgres-operator + containers: + - name: postgres-operator + image: registry.opensource.zalan.do/acid/pgbouncer:master-11 + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 100m + memory: 250Mi + limits: + cpu: 500m + memory: 500Mi + env: [] diff --git a/manifests/postgres-operator.yaml b/manifests/postgres-operator.yaml index 16719a5d9..ac319631b 100644 --- a/manifests/postgres-operator.yaml +++ b/manifests/postgres-operator.yaml @@ -2,6 +2,8 @@ apiVersion: apps/v1 kind: Deployment metadata: name: postgres-operator + labels: + application: postgres-operator spec: replicas: 1 strategy: diff --git a/pkg/cluster/connection_pooler.go b/pkg/cluster/connection_pooler.go index 0d9171b87..10354ca32 100644 --- a/pkg/cluster/connection_pooler.go +++ b/pkg/cluster/connection_pooler.go @@ -78,22 +78,22 @@ func needReplicaConnectionPoolerWorker(spec *acidv1.PostgresSpec) bool { // have e.g. different `application` label, so that recreatePod operation will // not interfere with it (it lists all the pods via labels, and if there would // be no difference, it will recreate also pooler pods). -func (c *Cluster) connectionPoolerLabelsSelector(role PostgresRole) *metav1.LabelSelector { - connectionPoolerLabels := labels.Set(map[string]string{}) +func (c *Cluster) connectionPoolerLabels(role PostgresRole, addExtraLabels bool) *metav1.LabelSelector { + poolerLabels := c.labelsSet(addExtraLabels) - extraLabels := labels.Set(map[string]string{ - "connection-pooler": c.connectionPoolerName(role), - "application": "db-connection-pooler", - "spilo-role": string(role), - "cluster-name": c.Name, - "Namespace": c.Namespace, - }) + // TODO should be config values + poolerLabels["application"] = "db-connection-pooler" + poolerLabels["connection-pooler"] = c.connectionPoolerName(role) - connectionPoolerLabels = labels.Merge(connectionPoolerLabels, c.labelsSet(false)) - connectionPoolerLabels = labels.Merge(connectionPoolerLabels, extraLabels) + if addExtraLabels { + extraLabels := map[string]string{} + extraLabels["spilo-role"] = string(role) + + poolerLabels = labels.Merge(poolerLabels, extraLabels) + } return &metav1.LabelSelector{ - MatchLabels: connectionPoolerLabels, + MatchLabels: poolerLabels, MatchExpressions: nil, } } @@ -284,7 +284,7 @@ func (c *Cluster) generateConnectionPoolerPodTemplate(role PostgresRole) ( podTemplate := &v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: c.connectionPoolerLabelsSelector(role).MatchLabels, + Labels: c.connectionPoolerLabels(role, true).MatchLabels, Namespace: c.Namespace, Annotations: c.generatePodAnnotations(spec), }, @@ -338,7 +338,7 @@ func (c *Cluster) generateConnectionPoolerDeployment(connectionPooler *Connectio ObjectMeta: metav1.ObjectMeta{ Name: connectionPooler.Name, Namespace: connectionPooler.Namespace, - Labels: c.connectionPoolerLabelsSelector(connectionPooler.Role).MatchLabels, + Labels: c.connectionPoolerLabels(connectionPooler.Role, true).MatchLabels, Annotations: map[string]string{}, // make StatefulSet object its owner to represent the dependency. // By itself StatefulSet is being deleted with "Orphaned" @@ -350,7 +350,7 @@ func (c *Cluster) generateConnectionPoolerDeployment(connectionPooler *Connectio }, Spec: appsv1.DeploymentSpec{ Replicas: numberOfInstances, - Selector: c.connectionPoolerLabelsSelector(connectionPooler.Role), + Selector: c.connectionPoolerLabels(connectionPooler.Role, false), Template: *podTemplate, }, } @@ -389,7 +389,7 @@ func (c *Cluster) generateConnectionPoolerService(connectionPooler *ConnectionPo ObjectMeta: metav1.ObjectMeta{ Name: connectionPooler.Name, Namespace: connectionPooler.Namespace, - Labels: c.connectionPoolerLabelsSelector(connectionPooler.Role).MatchLabels, + Labels: c.connectionPoolerLabels(connectionPooler.Role, false).MatchLabels, Annotations: map[string]string{}, // make StatefulSet object its owner to represent the dependency. // By itself StatefulSet is being deleted with "Orphaned" diff --git a/pkg/cluster/connection_pooler_test.go b/pkg/cluster/connection_pooler_test.go index b795fe14f..2528460f5 100644 --- a/pkg/cluster/connection_pooler_test.go +++ b/pkg/cluster/connection_pooler_test.go @@ -838,9 +838,9 @@ func testResources(cluster *Cluster, podSpec *v1.PodTemplateSpec, role PostgresR func testLabels(cluster *Cluster, podSpec *v1.PodTemplateSpec, role PostgresRole) error { poolerLabels := podSpec.ObjectMeta.Labels["connection-pooler"] - if poolerLabels != cluster.connectionPoolerLabelsSelector(role).MatchLabels["connection-pooler"] { + if poolerLabels != cluster.connectionPoolerLabels(role, true).MatchLabels["connection-pooler"] { return fmt.Errorf("Pod labels do not match, got %+v, expected %+v", - podSpec.ObjectMeta.Labels, cluster.connectionPoolerLabelsSelector(role).MatchLabels) + podSpec.ObjectMeta.Labels, cluster.connectionPoolerLabels(role, true).MatchLabels) } return nil @@ -848,7 +848,7 @@ func testLabels(cluster *Cluster, podSpec *v1.PodTemplateSpec, role PostgresRole func testSelector(cluster *Cluster, deployment *appsv1.Deployment) error { labels := deployment.Spec.Selector.MatchLabels - expected := cluster.connectionPoolerLabelsSelector(Master).MatchLabels + expected := cluster.connectionPoolerLabels(Master, true).MatchLabels if labels["connection-pooler"] != expected["connection-pooler"] { return fmt.Errorf("Labels are incorrect, got %+v, expected %+v", From cfd83e33c8eee153960d469111cb1eeab5efa382 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 24 Nov 2020 16:23:22 +0100 Subject: [PATCH 123/168] preserving fields only when using k8s specs (#1228) * preserving fields when k8s specs are used with x-kubernetes-preserve-unknown-fields flag * cleaning up merge errors in postgresql and operatorconfiguration CRD * add operatorconfiguration CRD and sample manifests in setUpClass of e2e tests * update generated code and go modules --- .../crds/operatorconfigurations.yaml | 189 ++---------- .../postgres-operator/crds/postgresqls.yaml | 286 +++++------------- .../postgres-operator/crds/postgresteams.yaml | 1 - e2e/tests/test_e2e.py | 3 + go.mod | 2 +- go.sum | 4 +- manifests/operatorconfiguration.crd.yaml | 189 ++---------- manifests/postgresql.crd.yaml | 286 +++++------------- manifests/postgresteam.crd.yaml | 1 - pkg/apis/acid.zalan.do/v1/crds.go | 115 ++++--- .../acid.zalan.do/v1/zz_generated.deepcopy.go | 5 + 11 files changed, 253 insertions(+), 828 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 57424374b..4f85d1642 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -47,7 +47,6 @@ spec: schema: openAPIV3Schema: type: object - x-preserve-unknown-fields: true required: - kind - apiVersion @@ -97,7 +96,7 @@ spec: nullable: true items: type: object - additionalProperties: true + x-kubernetes-preserve-unknown-fields: true workers: type: integer minimum: 1 @@ -275,6 +274,11 @@ spec: type: boolean enable_replica_load_balancer: type: boolean + external_traffic_policy: + type: string + enum: + - "Cluster" + - "Local" master_dns_name_format: type: string replica_dns_name_format: @@ -330,6 +334,10 @@ spec: properties: enable_admin_role_for_users: type: boolean + enable_postgres_team_crd: + type: boolean + enable_postgres_team_crd_superusers: + type: boolean enable_team_superuser: type: boolean enable_teams_api: @@ -342,176 +350,15 @@ spec: type: array items: type: string - pod_service_account_name: + protected_role_names: + type: array + items: type: string - pod_terminate_grace_period: - type: string - secret_name_template: - type: string - spilo_fsgroup: - type: integer - spilo_privileged: - type: boolean - toleration: - type: object - additionalProperties: - type: string - watched_namespace: - type: string - postgres_pod_resources: - type: object - properties: - default_cpu_limit: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - default_cpu_request: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - default_memory_limit: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - default_memory_request: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - timeouts: - type: object - properties: - pod_label_wait_timeout: - type: string - pod_deletion_wait_timeout: - type: string - ready_wait_interval: - type: string - ready_wait_timeout: - type: string - resource_check_interval: - type: string - resource_check_timeout: - type: string - load_balancer: - type: object - properties: - custom_service_annotations: - type: object - additionalProperties: - type: string - db_hosted_zone: - type: string - enable_master_load_balancer: - type: boolean - enable_replica_load_balancer: - type: boolean - external_traffic_policy: - type: string - enum: - - "Cluster" - - "Local" - master_dns_name_format: - type: string - replica_dns_name_format: - type: string - aws_or_gcp: - type: object - properties: - additional_secret_mount: - type: string - additional_secret_mount_path: - type: string - aws_region: - type: string - kube_iam_role: - type: string - log_s3_bucket: - type: string - wal_s3_bucket: - type: string - logical_backup: - type: object - properties: - logical_backup_schedule: - type: string - pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' - logical_backup_docker_image: - type: string - logical_backup_s3_bucket: - type: string - logical_backup_s3_endpoint: - type: string - logical_backup_s3_sse: - type: string - logical_backup_s3_access_key_id: - type: string - logical_backup_s3_secret_access_key: - type: string - debug: - type: object - properties: - debug_logging: - type: boolean - enable_database_access: - type: boolean - teams_api: - type: object - properties: - enable_admin_role_for_users: - type: boolean - enable_postgres_team_crd: - type: boolean - enable_postgres_team_crd_superusers: - type: boolean - enable_team_superuser: - type: boolean - enable_teams_api: - type: boolean - pam_configuration: - type: string - pam_role_name: - type: string - postgres_superuser_teams: - type: array - items: - type: string - protected_role_names: - type: array - items: - type: string - team_admin_role: - type: string - team_api_role_configuration: - type: object - additionalProperties: - type: string - teams_api_url: - type: string - logging_rest_api: - type: object - properties: - api_port: - type: integer - cluster_history_entries: - type: integer - ring_log_lines: - type: integer - scalyr: - type: object - properties: - scalyr_api_key: - type: string - scalyr_cpu_limit: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - scalyr_cpu_request: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - scalyr_image: - type: string - scalyr_memory_limit: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - scalyr_memory_request: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - scalyr_server_url: + team_admin_role: + type: string + team_api_role_configuration: + type: object + additionalProperties: type: string teams_api_url: type: string diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index c5dcfd247..323a2b1bb 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -59,7 +59,6 @@ spec: schema: openAPIV3Schema: type: object - x-preserve-unknown-fields: true required: - kind - apiVersion @@ -101,6 +100,7 @@ spec: type: string volumeSource: type: object + x-kubernetes-preserve-unknown-fields: true subPath: type: string allowedSourceRanges: @@ -208,87 +208,53 @@ spec: nullable: true items: type: object - required: - - cluster - properties: - cluster: - type: string - s3_endpoint: - type: string - s3_access_key_id: - type: string - s3_secret_access_key: - type: string - s3_force_path_style: - type: string - s3_wal_path: - type: string - timestamp: - type: string - pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$' - # The regexp matches the date-time format (RFC 3339 Section 5.6) that specifies a timezone as an offset relative to UTC - # Example: 1996-12-19T16:39:57-08:00 - # Note: this field requires a timezone - uid: - format: uuid - type: string - databases: + x-kubernetes-preserve-unknown-fields: true + initContainers: + type: array + nullable: true + items: type: object - additionalProperties: - type: string - # Note: usernames specified here as database owners must be declared in the users key of the spec key. - dockerImage: + x-kubernetes-preserve-unknown-fields: true + logicalBackupSchedule: + type: string + pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' + maintenanceWindows: + type: array + items: type: string - enableLogicalBackup: - type: boolean - enableMasterLoadBalancer: - type: boolean - enableReplicaLoadBalancer: - type: boolean - enableShmVolume: - type: boolean - init_containers: # deprecated - type: array - nullable: true - items: + pattern: '^\ *((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))-((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))\ *$' + numberOfInstances: + type: integer + minimum: 0 + patroni: + type: object + properties: + initdb: type: object - additionalProperties: true - initContainers: - type: array - nullable: true - items: - type: object - additionalProperties: true - logicalBackupSchedule: - type: string - pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' - maintenanceWindows: - type: array - items: - type: string - pattern: '^\ *((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))-((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))\ *$' - numberOfInstances: - type: integer - minimum: 0 - patroni: - type: object - properties: - initdb: - type: object - additionalProperties: - type: string - ttl: - type: integer + additionalProperties: + type: string loop_wait: type: integer - retry_timeout: - type: integer maximum_lag_on_failover: type: integer + pg_hba: + type: array + items: + type: string + retry_timeout: + type: integer + slots: + type: object + additionalProperties: + type: object + additionalProperties: + type: string synchronous_mode: type: boolean synchronous_mode_strict: type: boolean + ttl: + type: integer podAnnotations: type: object additionalProperties: @@ -304,114 +270,18 @@ spec: properties: version: type: string - pod_priority_class_name: # deprecated - type: string - podPriorityClassName: - type: string - postgresql: - type: object - required: - - version - properties: - version: - type: string - enum: - - "9.3" - - "9.4" - - "9.5" - - "9.6" - - "10" - - "11" - - "12" - - "13" - parameters: - type: object - additionalProperties: - type: string - replicaLoadBalancer: # deprecated - type: boolean - resources: - type: object - required: - - requests - - limits - properties: - limits: - type: object - required: - - cpu - - memory - properties: - cpu: - type: string - # Decimal natural followed by m, or decimal natural followed by - # dot followed by up to three decimal digits. - # - # This is because the Kubernetes CPU resource has millis as the - # maximum precision. The actual values are checked in code - # because the regular expression would be huge and horrible and - # not very helpful in validation error messages; this one checks - # only the format of the given number. - # - # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu - pattern: '^(\d+m|\d+\.\d{1,3})$' - # Note: the value specified here must not be zero or be lower - # than the corresponding request. - memory: - type: string - # You can express memory as a plain integer or as a fixed-point - # integer using one of these suffixes: E, P, T, G, M, k. You can - # also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki - # - # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - # Note: the value specified here must not be zero or be lower - # than the corresponding request. - requests: - type: object - required: - - cpu - - memory - properties: - cpu: - type: string - # Decimal natural followed by m, or decimal natural followed by - # dot followed by up to three decimal digits. - # - # This is because the Kubernetes CPU resource has millis as the - # maximum precision. The actual values are checked in code - # because the regular expression would be huge and horrible and - # not very helpful in validation error messages; this one checks - # only the format of the given number. - # - # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu - pattern: '^(\d+m|\d+\.\d{1,3})$' - # Note: the value specified here must not be zero or be higher - # than the corresponding limit. - memory: - type: string - # You can express memory as a plain integer or as a fixed-point - # integer using one of these suffixes: E, P, T, G, M, k. You can - # also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki - # - # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - # Note: the value specified here must not be zero or be higher - # than the corresponding limit. - sidecars: - type: array - nullable: true - items: + enum: + - "9.3" + - "9.4" + - "9.5" + - "9.6" + - "10" + - "11" + - "12" + - "13" + parameters: type: object - additionalProperties: true - spiloFSGroup: - type: integer - standby: - type: object - required: - - s3_wal_path - properties: - s3_wal_path: + additionalProperties: type: string preparedDatabases: type: object @@ -444,11 +314,10 @@ spec: limits: type: object required: - - key - - operator - - effect + - cpu + - memory properties: - key: + cpu: type: string # Decimal natural followed by m, or decimal natural followed by # dot followed by up to three decimal digits. @@ -463,26 +332,6 @@ spec: pattern: '^(\d+m|\d+(\.\d{1,3})?)$' # Note: the value specified here must not be zero or be lower # than the corresponding request. - memory: - type: string - enum: - - Equal - - Exists - value: - type: string - # Decimal natural followed by m, or decimal natural followed by - # dot followed by up to three decimal digits. - # - # This is because the Kubernetes CPU resource has millis as the - # maximum precision. The actual values are checked in code - # because the regular expression would be huge and horrible and - # not very helpful in validation error messages; this one checks - # only the format of the given number. - # - # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - # Note: the value specified here must not be zero or be higher - # than the corresponding limit. memory: type: string # You can express memory as a plain integer or as a fixed-point @@ -493,6 +342,18 @@ spec: pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' # Note: the value specified here must not be zero or be higher # than the corresponding limit. + requests: + type: object + required: + - cpu + - memory + properties: + cpu: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + memory: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' serviceAnnotations: type: object additionalProperties: @@ -502,7 +363,7 @@ spec: nullable: true items: type: object - additionalProperties: true + x-kubernetes-preserve-unknown-fields: true spiloRunAsUser: type: integer spiloRunAsGroup: @@ -538,15 +399,20 @@ spec: items: type: object required: - - size + - key + - operator + - effect properties: - size: + key: type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - # Note: the value specified here must not be zero. - storageClass: + operator: type: string - subPath: + enum: + - Equal + - Exists + value: + type: string + effect: type: string enum: - NoExecute @@ -609,4 +475,4 @@ spec: status: type: object additionalProperties: - type: string \ No newline at end of file + type: string diff --git a/charts/postgres-operator/crds/postgresteams.yaml b/charts/postgres-operator/crds/postgresteams.yaml index 8f40fc661..fbf873b84 100644 --- a/charts/postgres-operator/crds/postgresteams.yaml +++ b/charts/postgres-operator/crds/postgresteams.yaml @@ -27,7 +27,6 @@ spec: schema: openAPIV3Schema: type: object - x-preserve-unknown-fields: true required: - kind - apiVersion diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 95c7748cb..aac056ed4 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -117,8 +117,11 @@ class EndToEndTestCase(unittest.TestCase): yaml.dump(configmap, f, Dumper=yaml.Dumper) for filename in ["operator-service-account-rbac.yaml", + "postgresql.crd.yaml", + "operatorconfiguration.crd.yaml", "postgresteam.crd.yaml", "configmap.yaml", + "postgresql-operator-default-configuration.yaml", "postgres-operator.yaml", "api-service.yaml", "infrastructure-roles.yaml", diff --git a/go.mod b/go.mod index 341af771c..0ce0dad93 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/sirupsen/logrus v1.7.0 github.com/stretchr/testify v1.5.1 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 - golang.org/x/tools v0.0.0-20201026223136-e84cfc6dd5ca // indirect + golang.org/x/tools v0.0.0-20201121010211-780cb80bd7fb // indirect gopkg.in/yaml.v2 v2.2.8 k8s.io/api v0.19.3 k8s.io/apiextensions-apiserver v0.19.3 diff --git a/go.sum b/go.sum index 1f2e5f1d8..4e5704525 100644 --- a/go.sum +++ b/go.sum @@ -505,8 +505,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20201026223136-e84cfc6dd5ca h1:vL6Mv8VrSxz8azdgLrH/zO/Rd1Bzdk89ZfMVW39gD0Q= -golang.org/x/tools v0.0.0-20201026223136-e84cfc6dd5ca/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201121010211-780cb80bd7fb h1:z5+u0pkAUPUWd3taoTialQ2JAMo4Wo1Z3L25U4ZV9r0= +golang.org/x/tools v0.0.0-20201121010211-780cb80bd7fb/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index c42f4a1c2..f529d3353 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -43,7 +43,6 @@ spec: schema: openAPIV3Schema: type: object - x-preserve-unknown-fields: true required: - kind - apiVersion @@ -93,7 +92,7 @@ spec: nullable: true items: type: object - additionalProperties: true + x-kubernetes-preserve-unknown-fields: true workers: type: integer minimum: 1 @@ -271,6 +270,11 @@ spec: type: boolean enable_replica_load_balancer: type: boolean + external_traffic_policy: + type: string + enum: + - "Cluster" + - "Local" master_dns_name_format: type: string replica_dns_name_format: @@ -326,6 +330,10 @@ spec: properties: enable_admin_role_for_users: type: boolean + enable_postgres_team_crd: + type: boolean + enable_postgres_team_crd_superusers: + type: boolean enable_team_superuser: type: boolean enable_teams_api: @@ -338,176 +346,15 @@ spec: type: array items: type: string - pod_service_account_name: + protected_role_names: + type: array + items: type: string - pod_terminate_grace_period: - type: string - secret_name_template: - type: string - spilo_fsgroup: - type: integer - spilo_privileged: - type: boolean - toleration: - type: object - additionalProperties: - type: string - watched_namespace: - type: string - postgres_pod_resources: - type: object - properties: - default_cpu_limit: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - default_cpu_request: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - default_memory_limit: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - default_memory_request: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - timeouts: - type: object - properties: - pod_label_wait_timeout: - type: string - pod_deletion_wait_timeout: - type: string - ready_wait_interval: - type: string - ready_wait_timeout: - type: string - resource_check_interval: - type: string - resource_check_timeout: - type: string - load_balancer: - type: object - properties: - custom_service_annotations: - type: object - additionalProperties: - type: string - db_hosted_zone: - type: string - enable_master_load_balancer: - type: boolean - enable_replica_load_balancer: - type: boolean - external_traffic_policy: - type: string - enum: - - "Cluster" - - "Local" - master_dns_name_format: - type: string - replica_dns_name_format: - type: string - aws_or_gcp: - type: object - properties: - additional_secret_mount: - type: string - additional_secret_mount_path: - type: string - aws_region: - type: string - kube_iam_role: - type: string - log_s3_bucket: - type: string - wal_s3_bucket: - type: string - logical_backup: - type: object - properties: - logical_backup_schedule: - type: string - pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' - logical_backup_docker_image: - type: string - logical_backup_s3_bucket: - type: string - logical_backup_s3_endpoint: - type: string - logical_backup_s3_sse: - type: string - logical_backup_s3_access_key_id: - type: string - logical_backup_s3_secret_access_key: - type: string - debug: - type: object - properties: - debug_logging: - type: boolean - enable_database_access: - type: boolean - teams_api: - type: object - properties: - enable_admin_role_for_users: - type: boolean - enable_postgres_team_crd: - type: boolean - enable_postgres_team_crd_superusers: - type: boolean - enable_team_superuser: - type: boolean - enable_teams_api: - type: boolean - pam_configuration: - type: string - pam_role_name: - type: string - postgres_superuser_teams: - type: array - items: - type: string - protected_role_names: - type: array - items: - type: string - team_admin_role: - type: string - team_api_role_configuration: - type: object - additionalProperties: - type: string - teams_api_url: - type: string - logging_rest_api: - type: object - properties: - api_port: - type: integer - cluster_history_entries: - type: integer - ring_log_lines: - type: integer - scalyr: - type: object - properties: - scalyr_api_key: - type: string - scalyr_cpu_limit: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - scalyr_cpu_request: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - scalyr_image: - type: string - scalyr_memory_limit: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - scalyr_memory_request: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - scalyr_server_url: + team_admin_role: + type: string + team_api_role_configuration: + type: object + additionalProperties: type: string teams_api_url: type: string diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 0926c21e6..208dbf948 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -55,7 +55,6 @@ spec: schema: openAPIV3Schema: type: object - x-preserve-unknown-fields: true required: - kind - apiVersion @@ -97,6 +96,7 @@ spec: type: string volumeSource: type: object + x-kubernetes-preserve-unknown-fields: true subPath: type: string allowedSourceRanges: @@ -190,7 +190,7 @@ spec: enableConnectionPooler: type: boolean enableReplicaConnectionPooler: - type: boolean + type: boolean enableLogicalBackup: type: boolean enableMasterLoadBalancer: @@ -204,87 +204,53 @@ spec: nullable: true items: type: object - required: - - cluster - properties: - cluster: - type: string - s3_endpoint: - type: string - s3_access_key_id: - type: string - s3_secret_access_key: - type: string - s3_force_path_style: - type: string - s3_wal_path: - type: string - timestamp: - type: string - pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$' - # The regexp matches the date-time format (RFC 3339 Section 5.6) that specifies a timezone as an offset relative to UTC - # Example: 1996-12-19T16:39:57-08:00 - # Note: this field requires a timezone - uid: - format: uuid - type: string - databases: + x-kubernetes-preserve-unknown-fields: true + initContainers: + type: array + nullable: true + items: type: object - additionalProperties: - type: string - # Note: usernames specified here as database owners must be declared in the users key of the spec key. - dockerImage: + x-kubernetes-preserve-unknown-fields: true + logicalBackupSchedule: + type: string + pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' + maintenanceWindows: + type: array + items: type: string - enableLogicalBackup: - type: boolean - enableMasterLoadBalancer: - type: boolean - enableReplicaLoadBalancer: - type: boolean - enableShmVolume: - type: boolean - init_containers: # deprecated - type: array - nullable: true - items: + pattern: '^\ *((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))-((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))\ *$' + numberOfInstances: + type: integer + minimum: 0 + patroni: + type: object + properties: + initdb: type: object - additionalProperties: true - initContainers: - type: array - nullable: true - items: - type: object - additionalProperties: true - logicalBackupSchedule: - type: string - pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' - maintenanceWindows: - type: array - items: - type: string - pattern: '^\ *((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))-((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))\ *$' - numberOfInstances: - type: integer - minimum: 0 - patroni: - type: object - properties: - initdb: - type: object - additionalProperties: - type: string - ttl: - type: integer + additionalProperties: + type: string loop_wait: type: integer - retry_timeout: - type: integer maximum_lag_on_failover: type: integer + pg_hba: + type: array + items: + type: string + retry_timeout: + type: integer + slots: + type: object + additionalProperties: + type: object + additionalProperties: + type: string synchronous_mode: type: boolean synchronous_mode_strict: type: boolean + ttl: + type: integer podAnnotations: type: object additionalProperties: @@ -300,114 +266,18 @@ spec: properties: version: type: string - pod_priority_class_name: # deprecated - type: string - podPriorityClassName: - type: string - postgresql: - type: object - required: - - version - properties: - version: - type: string - enum: - - "9.3" - - "9.4" - - "9.5" - - "9.6" - - "10" - - "11" - - "12" - - "13" - parameters: - type: object - additionalProperties: - type: string - replicaLoadBalancer: # deprecated - type: boolean - resources: - type: object - required: - - requests - - limits - properties: - limits: - type: object - required: - - cpu - - memory - properties: - cpu: - type: string - # Decimal natural followed by m, or decimal natural followed by - # dot followed by up to three decimal digits. - # - # This is because the Kubernetes CPU resource has millis as the - # maximum precision. The actual values are checked in code - # because the regular expression would be huge and horrible and - # not very helpful in validation error messages; this one checks - # only the format of the given number. - # - # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu - pattern: '^(\d+m|\d+\.\d{1,3})$' - # Note: the value specified here must not be zero or be lower - # than the corresponding request. - memory: - type: string - # You can express memory as a plain integer or as a fixed-point - # integer using one of these suffixes: E, P, T, G, M, k. You can - # also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki - # - # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - # Note: the value specified here must not be zero or be lower - # than the corresponding request. - requests: - type: object - required: - - cpu - - memory - properties: - cpu: - type: string - # Decimal natural followed by m, or decimal natural followed by - # dot followed by up to three decimal digits. - # - # This is because the Kubernetes CPU resource has millis as the - # maximum precision. The actual values are checked in code - # because the regular expression would be huge and horrible and - # not very helpful in validation error messages; this one checks - # only the format of the given number. - # - # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu - pattern: '^(\d+m|\d+\.\d{1,3})$' - # Note: the value specified here must not be zero or be higher - # than the corresponding limit. - memory: - type: string - # You can express memory as a plain integer or as a fixed-point - # integer using one of these suffixes: E, P, T, G, M, k. You can - # also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki - # - # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - # Note: the value specified here must not be zero or be higher - # than the corresponding limit. - sidecars: - type: array - nullable: true - items: + enum: + - "9.3" + - "9.4" + - "9.5" + - "9.6" + - "10" + - "11" + - "12" + - "13" + parameters: type: object - additionalProperties: true - spiloFSGroup: - type: integer - standby: - type: object - required: - - s3_wal_path - properties: - s3_wal_path: + additionalProperties: type: string preparedDatabases: type: object @@ -440,11 +310,10 @@ spec: limits: type: object required: - - key - - operator - - effect + - cpu + - memory properties: - key: + cpu: type: string # Decimal natural followed by m, or decimal natural followed by # dot followed by up to three decimal digits. @@ -459,26 +328,6 @@ spec: pattern: '^(\d+m|\d+(\.\d{1,3})?)$' # Note: the value specified here must not be zero or be lower # than the corresponding request. - memory: - type: string - enum: - - Equal - - Exists - value: - type: string - # Decimal natural followed by m, or decimal natural followed by - # dot followed by up to three decimal digits. - # - # This is because the Kubernetes CPU resource has millis as the - # maximum precision. The actual values are checked in code - # because the regular expression would be huge and horrible and - # not very helpful in validation error messages; this one checks - # only the format of the given number. - # - # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - # Note: the value specified here must not be zero or be higher - # than the corresponding limit. memory: type: string # You can express memory as a plain integer or as a fixed-point @@ -489,6 +338,18 @@ spec: pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' # Note: the value specified here must not be zero or be higher # than the corresponding limit. + requests: + type: object + required: + - cpu + - memory + properties: + cpu: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + memory: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' serviceAnnotations: type: object additionalProperties: @@ -498,7 +359,7 @@ spec: nullable: true items: type: object - additionalProperties: true + x-kubernetes-preserve-unknown-fields: true spiloRunAsUser: type: integer spiloRunAsGroup: @@ -534,15 +395,20 @@ spec: items: type: object required: - - size + - key + - operator + - effect properties: - size: + key: type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - # Note: the value specified here must not be zero. - storageClass: + operator: type: string - subPath: + enum: + - Equal + - Exists + value: + type: string + effect: type: string enum: - NoExecute diff --git a/manifests/postgresteam.crd.yaml b/manifests/postgresteam.crd.yaml index 845414aea..2588e53b1 100644 --- a/manifests/postgresteam.crd.yaml +++ b/manifests/postgresteam.crd.yaml @@ -23,7 +23,6 @@ spec: schema: openAPIV3Schema: type: object - x-preserve-unknown-fields: true required: - kind - apiVersion diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 79332d597..eab9286c1 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -112,9 +112,8 @@ var minDisable = -1.0 // PostgresCRDResourceValidation to check applied manifest parameters var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ OpenAPIV3Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - XPreserveUnknownFields: util.True(), - Required: []string{"kind", "apiVersion", "spec"}, + Type: "object", + Required: []string{"kind", "apiVersion", "spec"}, Properties: map[string]apiextv1.JSONSchemaProps{ "kind": { Type: "string", @@ -136,6 +135,38 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ Type: "object", Required: []string{"numberOfInstances", "teamId", "postgresql", "volume"}, Properties: map[string]apiextv1.JSONSchemaProps{ + "additionalVolumes": { + Type: "array", + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ + Type: "object", + Required: []string{"name", "mountPath", "volumeSource"}, + Properties: map[string]apiextv1.JSONSchemaProps{ + "name": { + Type: "string", + }, + "mountPath": { + Type: "string", + }, + "targetContainers": { + Type: "array", + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ + Type: "string", + }, + }, + }, + "volumeSource": { + Type: "object", + XPreserveUnknownFields: util.True(), + }, + "subPath": { + Type: "string", + }, + }, + }, + }, + }, "allowedSourceRanges": { Type: "array", Nullable: true, @@ -284,10 +315,8 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ Description: "Deprecated", Items: &apiextv1.JSONSchemaPropsOrArray{ Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Allows: true, - }, + Type: "object", + XPreserveUnknownFields: util.True(), }, }, }, @@ -295,10 +324,8 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ Type: "array", Items: &apiextv1.JSONSchemaPropsOrArray{ Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Allows: true, - }, + Type: "object", + XPreserveUnknownFields: util.True(), }, }, }, @@ -330,6 +357,12 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ }, }, }, + "loop_wait": { + Type: "integer", + }, + "maximum_lag_on_failover": { + Type: "integer", + }, "pg_hba": { Type: "array", Items: &apiextv1.JSONSchemaPropsOrArray{ @@ -338,6 +371,9 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ }, }, }, + "retry_timeout": { + Type: "integer", + }, "slots": { Type: "object", AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ @@ -351,24 +387,15 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ }, }, }, - "ttl": { - Type: "integer", - }, - "loop_wait": { - Type: "integer", - }, - "retry_timeout": { - Type: "integer", - }, - "maximum_lag_on_failover": { - Type: "integer", - }, "synchronous_mode": { Type: "boolean", }, "synchronous_mode_strict": { Type: "boolean", }, + "ttl": { + Type: "integer", + }, }, }, "podAnnotations": { @@ -736,37 +763,6 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ }, }, }, - "additionalVolumes": { - Type: "array", - Items: &apiextv1.JSONSchemaPropsOrArray{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - Required: []string{"name", "mountPath", "volumeSource"}, - Properties: map[string]apiextv1.JSONSchemaProps{ - "name": { - Type: "string", - }, - "mountPath": { - Type: "string", - }, - "targetContainers": { - Type: "array", - Items: &apiextv1.JSONSchemaPropsOrArray{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "string", - }, - }, - }, - "volumeSource": { - Type: "object", - }, - "subPath": { - Type: "string", - }, - }, - }, - }, - }, }, }, "status": { @@ -784,9 +780,8 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ // OperatorConfigCRDResourceValidation to check applied manifest parameters var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{ OpenAPIV3Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - XPreserveUnknownFields: util.True(), - Required: []string{"kind", "apiVersion", "configuration"}, + Type: "object", + Required: []string{"kind", "apiVersion", "configuration"}, Properties: map[string]apiextv1.JSONSchemaProps{ "kind": { Type: "string", @@ -856,10 +851,8 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{ Type: "array", Items: &apiextv1.JSONSchemaPropsOrArray{ Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Allows: true, - }, + Type: "object", + XPreserveUnknownFields: util.True(), }, }, }, diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 364b3e161..bdca06547 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -532,6 +532,11 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { *out = new(bool) **out = **in } + if in.EnableReplicaConnectionPooler != nil { + in, out := &in.EnableReplicaConnectionPooler, &out.EnableReplicaConnectionPooler + *out = new(bool) + **out = **in + } if in.ConnectionPooler != nil { in, out := &in.ConnectionPooler, &out.ConnectionPooler *out = new(ConnectionPooler) From 85d1a72cd69a437df0b4b34892546b5455c64e44 Mon Sep 17 00:00:00 2001 From: Boyan Bonev Date: Wed, 25 Nov 2020 11:55:05 +0200 Subject: [PATCH 124/168] Add scheduler name support - [Update #990] (#1226) * Add ability to specify alternative schedulers via schedulerName. Co-authored-by: micah.coletti@gmail.com --- charts/postgres-operator/crds/postgresqls.yaml | 2 ++ docs/reference/cluster_manifest.md | 4 ++++ manifests/postgresql.crd.yaml | 2 ++ pkg/apis/acid.zalan.do/v1/crds.go | 3 +++ pkg/apis/acid.zalan.do/v1/postgresql_type.go | 1 + pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go | 5 +++++ pkg/cluster/k8sres.go | 7 +++++++ 7 files changed, 24 insertions(+) diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 323a2b1bb..fe54fb600 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -354,6 +354,8 @@ spec: memory: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + schedulerName: + type: string serviceAnnotations: type: object additionalProperties: diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index f7ddb6ff1..589921bc5 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -65,6 +65,10 @@ These parameters are grouped directly under the `spec` key in the manifest. custom Docker image that overrides the **docker_image** operator parameter. It should be a [Spilo](https://github.com/zalando/spilo) image. Optional. +* **schedulerName** + specifies the scheduling profile for database pods. If no value is provided + K8s' `default-scheduler` will be used. Optional. + * **spiloRunAsUser** sets the user ID which should be used in the container to run the process. This must be set to run the container without root. By default the container diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 208dbf948..80046a0f2 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -350,6 +350,8 @@ spec: memory: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + schedulerName: + type: string serviceAnnotations: type: object additionalProperties: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index eab9286c1..63d486dad 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -535,6 +535,9 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ }, }, }, + "schedulerName": { + Type: "string", + }, "serviceAnnotations": { Type: "object", AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index a3dc490b5..665da7d0c 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -60,6 +60,7 @@ type PostgresSpec struct { ClusterName string `json:"-"` Databases map[string]string `json:"databases,omitempty"` PreparedDatabases map[string]PreparedDatabase `json:"preparedDatabases,omitempty"` + SchedulerName *string `json:"schedulerName,omitempty"` Tolerations []v1.Toleration `json:"tolerations,omitempty"` Sidecars []Sidecar `json:"sidecars,omitempty"` InitContainers []v1.Container `json:"initContainers,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index bdca06547..de260dc53 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -687,6 +687,11 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.SchedulerName != nil { + in, out := &in.SchedulerName, &out.SchedulerName + *out = new(string) + **out = **in + } return } diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 50957e22a..798e163dd 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -537,6 +537,7 @@ func (c *Cluster) generatePodTemplate( spiloRunAsGroup *int64, spiloFSGroup *int64, nodeAffinity *v1.Affinity, + schedulerName *string, terminateGracePeriod int64, podServiceAccountName string, kubeIAMRole string, @@ -575,6 +576,10 @@ func (c *Cluster) generatePodTemplate( SecurityContext: &securityContext, } + if schedulerName != nil { + podSpec.SchedulerName = *schedulerName + } + if shmVolume != nil && *shmVolume { addShmVolume(&podSpec) } @@ -1184,6 +1189,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef effectiveRunAsGroup, effectiveFSGroup, nodeAffinity(c.OpConfig.NodeReadinessLabel), + spec.SchedulerName, int64(c.OpConfig.PodTerminateGracePeriod.Seconds()), c.OpConfig.PodServiceAccountName, c.OpConfig.KubeIAMRole, @@ -1885,6 +1891,7 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) { nil, nil, nodeAffinity(c.OpConfig.NodeReadinessLabel), + nil, int64(c.OpConfig.PodTerminateGracePeriod.Seconds()), c.OpConfig.PodServiceAccountName, c.OpConfig.KubeIAMRole, From 2b5382edf37b73d14e34583e97f7552a93fcd505 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 27 Nov 2020 15:40:35 +0100 Subject: [PATCH 125/168] Add PR template (#1234) * Add PR template * move to github folder and rename file * add headlines --- .../postgres-operator-pull-request-template.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/PULL_REQUEST_TEMPLATE/postgres-operator-pull-request-template.md diff --git a/.github/PULL_REQUEST_TEMPLATE/postgres-operator-pull-request-template.md b/.github/PULL_REQUEST_TEMPLATE/postgres-operator-pull-request-template.md new file mode 100644 index 000000000..78ebc4993 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE/postgres-operator-pull-request-template.md @@ -0,0 +1,18 @@ +## Problem description + + + +## Linked issues + + + +## Checklist + +Thanks for submitting a pull request to the Postgres Operator project. +Please, ensure your contribution matches the following items: + +- [ ] Your go code is [formatted](https://blog.golang.org/gofmt). Your IDE should do it automatically for you. +- [ ] You have updated [generated code](https://github.com/zalando/postgres-operator/blob/master/docs/developer.md#code-generation) when introducing new fields to the `acid.zalan.do` api package. +- [ ] New [configuration options](https://github.com/zalando/postgres-operator/blob/master/docs/developer.md#introduce-additional-configuration-parameters) are reflected in CRD validation, helm charts and sample manifests. +- [ ] New functionality is covered by [unit](https://github.com/zalando/postgres-operator/blob/master/docs/developer.md#unit-tests) and/or [e2e](https://github.com/zalando/postgres-operator/blob/master/docs/developer.md#end-to-end-tests) tests. +- [ ] You have checked existing open PRs for possible overlay and referenced them. From 6f5751fe553fe7fda4719e8b038386c2c9bb0b4c Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 27 Nov 2020 18:47:50 +0100 Subject: [PATCH 126/168] raise log level for malformed secrets (#1235) Co-authored-by: Sergey Dudoladov --- pkg/cluster/sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index e91adf757..625c67313 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -482,7 +482,7 @@ func (c *Cluster) syncSecrets() error { return fmt.Errorf("could not get current secret: %v", err) } if secretUsername != string(secret.Data["username"]) { - c.logger.Warningf("secret %s does not contain the role %q", secretSpec.Name, secretUsername) + c.logger.Errorf("secret %s does not contain the role %q", secretSpec.Name, secretUsername) continue } c.Secrets[secret.UID] = secret From dc9a5b1e61cae6581495b651e42b231893a388a9 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 27 Nov 2020 18:49:49 +0100 Subject: [PATCH 127/168] Introduce PGVERSION (#1172) * introduce PGVERSION Co-authored-by: Sergey Dudoladov --- charts/postgres-operator/values-crd.yaml | 2 ++ charts/postgres-operator/values.yaml | 2 ++ docs/reference/operator_parameters.md | 3 +++ e2e/tests/test_e2e.py | 2 +- manifests/configmap.yaml | 1 + manifests/operatorconfiguration.crd.yaml | 2 ++ .../v1/operator_configuration_type.go | 3 ++- pkg/cluster/k8sres.go | 16 +++++++++++++--- pkg/cluster/k8sres_test.go | 2 +- pkg/controller/operator_config.go | 1 + pkg/util/config/config.go | 1 + 11 files changed, 29 insertions(+), 6 deletions(-) diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 71c2d5bb1..70ae3d53c 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -21,6 +21,8 @@ configGeneral: enable_crd_validation: true # update only the statefulsets without immediately doing the rolling update enable_lazy_spilo_upgrade: false + # set the PGVERSION env var instead of providing the version via postgresql.bin_dir in SPILO_CONFIGURATION + enable_pgversion_env_var: "false" # start any new database pod without limitations on shm memory enable_shm_volume: true # etcd connection string for Patroni. Empty uses K8s-native DCS. diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 95865503d..2e831b142 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -24,6 +24,8 @@ configGeneral: enable_crd_validation: "true" # update only the statefulsets without immediately doing the rolling update enable_lazy_spilo_upgrade: "false" + # set the PGVERSION env var instead of providing the version via postgresql.bin_dir in SPILO_CONFIGURATION + enable_pgversion_env_var: "false" # start any new database pod without limitations on shm memory enable_shm_volume: "true" # etcd connection string for Patroni. Empty uses K8s-native DCS. diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index bd8c80d9c..5a5a7edc0 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -118,6 +118,9 @@ Those are top-level keys, containing both leaf keys and groups. This option is global for an operator object, and can be overwritten by `enableShmVolume` parameter from Postgres manifest. The default is `true`. +* **enable_pgversion_env_var** + With newer versions of Spilo, it is preferable to use `PGVERSION` pod environment variable instead of the setting `postgresql.bin_dir` in the `SPILO_CONFIGURATION` env variable. When this option is true, the operator sets `PGVERSION` and omits `postgresql.bin_dir` from `SPILO_CONFIGURATION`. When false, the `postgresql.bin_dir` is set. This setting takes precedence over `PGVERSION`; see PR 222 in Spilo. The default is `false`. + * **workers** number of working routines the operator spawns to process requests to create/update/delete/sync clusters concurrently. The default is `4`. diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index aac056ed4..b98d0d956 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -1080,7 +1080,7 @@ class EndToEndTestCase(unittest.TestCase): "enable_pod_antiaffinity": "false" } } - k8s.update_config(patch_disable_antiaffinity, "disalbe antiaffinity") + k8s.update_config(patch_disable_antiaffinity, "disable antiaffinity") k8s.wait_for_pod_start('spilo-role=master') k8s.wait_for_pod_start('spilo-role=replica') return True diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 59283fd6e..dbefbebcf 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -45,6 +45,7 @@ data: # enable_postgres_team_crd_superusers: "false" enable_replica_load_balancer: "false" # enable_shm_volume: "true" + # enable_pgversion_env_var: "false" # enable_sidecars: "true" # enable_team_superuser: "false" enable_teams_api: "false" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index f529d3353..9370c1500 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -65,6 +65,8 @@ spec: type: boolean enable_lazy_spilo_upgrade: type: boolean + enable_pgversion_env_var: + type: boolean enable_shm_volume: type: boolean etcd_host: diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index a9abcf0ee..56f808159 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -167,7 +167,7 @@ type ScalyrConfiguration struct { ScalyrMemoryLimit string `json:"scalyr_memory_limit,omitempty"` } -// Defines default configuration for connection pooler +// ConnectionPoolerConfiguration defines default configuration for connection pooler type ConnectionPoolerConfiguration struct { NumberOfInstances *int32 `json:"connection_pooler_number_of_instances,omitempty"` Schema string `json:"connection_pooler_schema,omitempty"` @@ -197,6 +197,7 @@ type OperatorLogicalBackupConfiguration struct { type OperatorConfigurationData struct { EnableCRDValidation *bool `json:"enable_crd_validation,omitempty"` EnableLazySpiloUpgrade bool `json:"enable_lazy_spilo_upgrade,omitempty"` + EnablePgVersionEnvVar bool `json:"enable_pgversion_env_var,omitempty"` EtcdHost string `json:"etcd_host,omitempty"` KubernetesUseConfigMaps bool `json:"kubernetes_use_configmaps,omitempty"` DockerImage string `json:"docker_image,omitempty"` diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 798e163dd..28d711a33 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -189,7 +189,7 @@ func fillResourceList(spec acidv1.ResourceDescription, defaults acidv1.ResourceD return requests, nil } -func generateSpiloJSONConfiguration(pg *acidv1.PostgresqlParam, patroni *acidv1.Patroni, pamRoleName string, logger *logrus.Entry) (string, error) { +func generateSpiloJSONConfiguration(pg *acidv1.PostgresqlParam, patroni *acidv1.Patroni, pamRoleName string, EnablePgVersionEnvVar bool, logger *logrus.Entry) (string, error) { config := spiloConfiguration{} config.Bootstrap = pgBootstrap{} @@ -270,7 +270,14 @@ PatroniInitDBParams: } config.PgLocalConfiguration = make(map[string]interface{}) - config.PgLocalConfiguration[patroniPGBinariesParameterName] = fmt.Sprintf(pgBinariesLocationTemplate, pg.PgVersion) + + // the newer and preferred way to specify the PG version is to use the `PGVERSION` env variable + // setting postgresq.bin_dir in the SPILO_CONFIGURATION still works and takes precedence over PGVERSION + // so we add postgresq.bin_dir only if PGVERSION is unused + // see PR 222 in Spilo + if !EnablePgVersionEnvVar { + config.PgLocalConfiguration[patroniPGBinariesParameterName] = fmt.Sprintf(pgBinariesLocationTemplate, pg.PgVersion) + } if len(pg.Parameters) > 0 { local, bootstrap := getLocalAndBoostrapPostgreSQLParameters(pg.Parameters) @@ -696,6 +703,9 @@ func (c *Cluster) generateSpiloPodEnvVars(uid types.UID, spiloConfiguration stri Value: c.OpConfig.PamRoleName, }, } + if c.OpConfig.EnablePgVersionEnvVar { + envVars = append(envVars, v1.EnvVar{Name: "PGVERSION", Value: c.Spec.PgVersion}) + } // Spilo expects cluster labels as JSON if clusterLabels, err := json.Marshal(labels.Set(c.OpConfig.ClusterLabels)); err != nil { envVars = append(envVars, v1.EnvVar{Name: "KUBERNETES_LABELS", Value: labels.Set(c.OpConfig.ClusterLabels).String()}) @@ -1012,7 +1022,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef } } - spiloConfiguration, err := generateSpiloJSONConfiguration(&spec.PostgresqlParam, &spec.Patroni, c.OpConfig.PamRoleName, c.logger) + spiloConfiguration, err := generateSpiloJSONConfiguration(&spec.PostgresqlParam, &spec.Patroni, c.OpConfig.PamRoleName, c.OpConfig.EnablePgVersionEnvVar, c.logger) if err != nil { return nil, fmt.Errorf("could not generate Spilo JSON configuration: %v", err) } diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 52c10cb8b..2f2b353ab 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -93,7 +93,7 @@ func TestGenerateSpiloJSONConfiguration(t *testing.T) { } for _, tt := range tests { cluster.OpConfig = tt.opConfig - result, err := generateSpiloJSONConfiguration(tt.pgParam, tt.patroni, tt.role, logger) + result, err := generateSpiloJSONConfiguration(tt.pgParam, tt.patroni, tt.role, false, logger) if err != nil { t.Errorf("Unexpected error: %v", err) } diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 9b2713da8..8fb951a80 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -35,6 +35,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur // general config result.EnableCRDValidation = util.CoalesceBool(fromCRD.EnableCRDValidation, util.True()) result.EnableLazySpiloUpgrade = fromCRD.EnableLazySpiloUpgrade + result.EnablePgVersionEnvVar = fromCRD.EnablePgVersionEnvVar result.EtcdHost = fromCRD.EtcdHost result.KubernetesUseConfigMaps = fromCRD.KubernetesUseConfigMaps result.DockerImage = util.Coalesce(fromCRD.DockerImage, "registry.opensource.zalan.do/acid/spilo-12:1.6-p3") diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 47a120227..7f9c66ea4 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -197,6 +197,7 @@ type Config struct { PostgresSuperuserTeams []string `name:"postgres_superuser_teams" default:""` SetMemoryRequestToLimit bool `name:"set_memory_request_to_limit" default:"false"` EnableLazySpiloUpgrade bool `name:"enable_lazy_spilo_upgrade" default:"false"` + EnablePgVersionEnvVar bool `name:"enable_pgversion_env_var" default:"false"` } // MustMarshal marshals the config or panics From 5a6da7275fb27cb724fac1af49866553315c65e1 Mon Sep 17 00:00:00 2001 From: Rafia Sabih Date: Wed, 9 Dec 2020 13:00:06 +0100 Subject: [PATCH 128/168] avoid hard-codeed spilo-role (#1246) Co-authored-by: Rafia Sabih --- pkg/cluster/connection_pooler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cluster/connection_pooler.go b/pkg/cluster/connection_pooler.go index 10354ca32..36c75bd91 100644 --- a/pkg/cluster/connection_pooler.go +++ b/pkg/cluster/connection_pooler.go @@ -87,7 +87,7 @@ func (c *Cluster) connectionPoolerLabels(role PostgresRole, addExtraLabels bool) if addExtraLabels { extraLabels := map[string]string{} - extraLabels["spilo-role"] = string(role) + extraLabels[c.OpConfig.PodRoleLabel] = string(role) poolerLabels = labels.Merge(poolerLabels, extraLabels) } From 598c05b64b516475e887df3789cbc241660426e5 Mon Sep 17 00:00:00 2001 From: Thunderbolt Date: Thu, 10 Dec 2020 14:04:03 +0100 Subject: [PATCH 129/168] add to postgresql_type.go omitempty annotation (#1223) * add to postgresql_type.go omitempty annotation * add postgresql_type.go additional omitempty * remove postgresql.go defaults --- pkg/apis/acid.zalan.do/v1/postgresql_type.go | 26 ++++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 665da7d0c..8a64462b2 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -54,7 +54,7 @@ type PostgresSpec struct { AllowedSourceRanges []string `json:"allowedSourceRanges"` NumberOfInstances int32 `json:"numberOfInstances"` - Users map[string]UserFlags `json:"users"` + Users map[string]UserFlags `json:"users,omitempty"` MaintenanceWindows []MaintenanceWindow `json:"maintenanceWindows,omitempty"` Clone *CloneDescription `json:"clone,omitempty"` ClusterName string `json:"-"` @@ -113,14 +113,14 @@ type MaintenanceWindow struct { // Volume describes a single volume in the manifest. type Volume struct { Size string `json:"size"` - StorageClass string `json:"storageClass"` + StorageClass string `json:"storageClass,omitempty"` SubPath string `json:"subPath,omitempty"` } type AdditionalVolume struct { Name string `json:"name"` MountPath string `json:"mountPath"` - SubPath string `json:"subPath"` + SubPath string `json:"subPath,omitempty"` TargetContainers []string `json:"targetContainers"` VolumeSource v1.VolumeSource `json:"volumeSource"` } @@ -128,7 +128,7 @@ type AdditionalVolume struct { // PostgresqlParam describes PostgreSQL version and pairs of configuration parameter name - values. type PostgresqlParam struct { PgVersion string `json:"version"` - Parameters map[string]string `json:"parameters"` + Parameters map[string]string `json:"parameters,omitempty"` } // ResourceDescription describes CPU and memory resources defined for a cluster. @@ -145,15 +145,15 @@ type Resources struct { // Patroni contains Patroni-specific configuration type Patroni struct { - InitDB map[string]string `json:"initdb"` - PgHba []string `json:"pg_hba"` - TTL uint32 `json:"ttl"` - LoopWait uint32 `json:"loop_wait"` - RetryTimeout uint32 `json:"retry_timeout"` - MaximumLagOnFailover float32 `json:"maximum_lag_on_failover"` // float32 because https://github.com/kubernetes/kubernetes/issues/30213 - Slots map[string]map[string]string `json:"slots"` - SynchronousMode bool `json:"synchronous_mode"` - SynchronousModeStrict bool `json:"synchronous_mode_strict"` + InitDB map[string]string `json:"initdb,omitempty"` + PgHba []string `json:"pg_hba,omitempty"` + TTL uint32 `json:"ttl,omitempty"` + LoopWait uint32 `json:"loop_wait,omitempty"` + RetryTimeout uint32 `json:"retry_timeout,omitempty"` + MaximumLagOnFailover float32 `json:"maximum_lag_on_failover,omitempty"` // float32 because https://github.com/kubernetes/kubernetes/issues/30213 + Slots map[string]map[string]string `json:"slots,omitempty"` + SynchronousMode bool `json:"synchronous_mode,omitempty"` + SynchronousModeStrict bool `json:"synchronous_mode_strict,omitempty"` } //StandbyCluster From b9ef88f842f4776c737f31fe369b4fe7fa995cdf Mon Sep 17 00:00:00 2001 From: Igor Yanchenko <1504692+yanchenko-igor@users.noreply.github.com> Date: Thu, 10 Dec 2020 17:27:03 +0200 Subject: [PATCH 130/168] replace AdditionalProperties with XPreserveUnknownFields FIXES #1206 (#1248) --- pkg/apis/acid.zalan.do/v1/crds.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 63d486dad..0d098835b 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -550,10 +550,8 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ Type: "array", Items: &apiextv1.JSONSchemaPropsOrArray{ Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Allows: true, - }, + Type: "object", + XPreserveUnknownFields: util.True(), }, }, }, From 549f71bb490c7469979518ee6e6eea523260bb71 Mon Sep 17 00:00:00 2001 From: Jan Mussler Date: Fri, 11 Dec 2020 15:52:32 +0100 Subject: [PATCH 131/168] Support EBS gp2 to gp3 migration on sync for below 1tb volumes (#1242) * initial commit for gp3 migration. * Default volume migration done. * Added Gomock and one test case with mock. * Dep update. * more changes for code gen. * push fake package. * Rename var. * Changes to Makefile and return value. * Macke mocks phony due to overlap in foldername. * Learning as one goes. Initialize map. * Wrong toggle. * Expect modify call. * Fix mapping of ids in test. * Fix volume id. * volume ids. * Fixing test setup. Late night... * create all pvs. * Fix test case config. * store volumes and compare. * More logs. * Logging of migration action. * Ensure to log errors. * Log warning if modify failed, e.g. due to ebs volume state. * Add more output. * Skip local e2e tests. * Reflect k8s volume id in test data. Extract aws volume id from k8s value. * Finalizing ebs migration. * More logs. describe fails. * Fix non existing fields in gp2 discovery. * Remove nothing to do flag for migration. * Final commit for migration. * add new options to all places Co-authored-by: Felix Kunde --- .github/workflows/run_e2e.yaml | 2 +- .github/workflows/run_tests.yaml | 2 +- .gitignore | 2 + Makefile | 7 +- .../crds/operatorconfigurations.yaml | 4 + charts/postgres-operator/values-crd.yaml | 5 + charts/postgres-operator/values.yaml | 11 +- delivery.yaml | 2 +- docs/reference/operator_parameters.md | 16 +- go.mod | 22 +- go.sum | 81 ++++++-- manifests/configmap.yaml | 2 + manifests/operatorconfiguration.crd.yaml | 4 + ...gresql-operator-default-configuration.yaml | 2 + mocks/mocks.go | 1 + pkg/apis/acid.zalan.do/v1/crds.go | 9 + .../v1/operator_configuration_type.go | 18 +- pkg/apis/acid.zalan.do/v1/postgresql_type.go | 8 +- .../acid.zalan.do/v1/zz_generated.deepcopy.go | 22 +- pkg/cluster/cluster.go | 11 + pkg/cluster/sync.go | 22 +- pkg/cluster/volumes.go | 146 ++++++++++---- pkg/cluster/volumes_test.go | 190 +++++++++++++----- pkg/controller/operator_config.go | 2 + pkg/util/config/config.go | 2 + pkg/util/volumes/ebs.go | 125 ++++++++++-- pkg/util/volumes/volumes.go | 18 +- 27 files changed, 568 insertions(+), 168 deletions(-) create mode 100644 mocks/mocks.go diff --git a/.github/workflows/run_e2e.yaml b/.github/workflows/run_e2e.yaml index b241f89f1..6eef5c8f2 100644 --- a/.github/workflows/run_e2e.yaml +++ b/.github/workflows/run_e2e.yaml @@ -16,7 +16,7 @@ jobs: with: go-version: "^1.15.5" - name: Make dependencies - run: make deps + run: make deps mocks - name: Compile run: make linux - name: Run unit tests diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml index 4f9b3a5bb..56d147990 100644 --- a/.github/workflows/run_tests.yaml +++ b/.github/workflows/run_tests.yaml @@ -16,7 +16,7 @@ jobs: with: go-version: "^1.15.5" - name: Make dependencies - run: make deps + run: make deps mocks - name: Compile run: make linux - name: Run unit tests diff --git a/.gitignore b/.gitignore index b9a730ad8..e062f8479 100644 --- a/.gitignore +++ b/.gitignore @@ -98,3 +98,5 @@ e2e/manifests # Translations *.mo *.pot + +mocks diff --git a/Makefile b/Makefile index 2b2d2668f..fe2387670 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean local test linux macos docker push scm-source.json e2e +.PHONY: clean local test linux macos mocks docker push scm-source.json e2e BINARY ?= postgres-operator BUILD_FLAGS ?= -v @@ -81,9 +81,12 @@ push: scm-source.json: .git echo '{\n "url": "git:$(GITURL)",\n "revision": "$(GITHEAD)",\n "author": "$(USER)",\n "status": "$(GITSTATUS)"\n}' > scm-source.json +mocks: + GO111MODULE=on go generate ./... + tools: - GO111MODULE=on go get -u honnef.co/go/tools/cmd/staticcheck GO111MODULE=on go get k8s.io/client-go@kubernetes-1.19.3 + GO111MODULE=on go get github.com/golang/mock/mockgen@v1.4.4 GO111MODULE=on go mod tidy fmt: diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 4f85d1642..0e0efd6c1 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -292,6 +292,10 @@ spec: type: string aws_region: type: string + enable_ebs_gp3_migration: + type: boolean + enable_ebs_gp3_migration_max_size: + type: integer gcp_credentials: type: string kube_iam_role: diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 70ae3d53c..21292a13e 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -219,6 +219,11 @@ configAwsOrGcp: # AWS region used to store ESB volumes aws_region: eu-central-1 + # enable automatic migration on AWS from gp2 to gp3 volumes + enable_ebs_gp3_migration: false + # defines maximum volume size in GB until which auto migration happens + # enable_ebs_gp3_migration_max_size: 1000 + # GCP credentials that will be used by the operator / pods # gcp_credentials: "" diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 2e831b142..8a7776c54 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -211,6 +211,14 @@ configAwsOrGcp: # AWS region used to store ESB volumes aws_region: eu-central-1 + # enable automatic migration on AWS from gp2 to gp3 volumes + enable_ebs_gp3_migration: "false" + # defines maximum volume size in GB until which auto migration happens + # enable_ebs_gp3_migration_max_size: 1000 + + # GCP credentials for setting the GOOGLE_APPLICATION_CREDNETIALS environment variable + # gcp_credentials: "" + # AWS IAM role to supply in the iam.amazonaws.com/role annotation of Postgres pods # kube_iam_role: "" @@ -223,9 +231,6 @@ configAwsOrGcp: # GCS bucket to use for shipping WAL segments with WAL-E # wal_gs_bucket: "" - # GCP credentials for setting the GOOGLE_APPLICATION_CREDNETIALS environment variable - # gcp_credentials: "" - # configure K8s cron job managed by the operator configLogicalBackup: # image for pods of the logical backup job (example runs pg_dumpall) diff --git a/delivery.yaml b/delivery.yaml index 94c94b246..a4d42af7d 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -32,7 +32,7 @@ pipeline: IMAGE=registry-write.opensource.zalan.do/acid/postgres-operator-test fi export IMAGE - make deps docker + make deps mocks docker - desc: 'Run unit tests' cmd: | export PATH=$PATH:$HOME/go/bin diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 5a5a7edc0..63903cb81 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -518,10 +518,22 @@ yet officially supported. AWS region used to store EBS volumes. The default is `eu-central-1`. * **additional_secret_mount** - Additional Secret (aws or gcp credentials) to mount in the pod. The default is empty. + Additional Secret (aws or gcp credentials) to mount in the pod. + The default is empty. * **additional_secret_mount_path** - Path to mount the above Secret in the filesystem of the container(s). The default is empty. + Path to mount the above Secret in the filesystem of the container(s). + The default is empty. + +* **enable_ebs_gp3_migration** + enable automatic migration on AWS from gp2 to gp3 volumes, that are smaller + than the configured max size (see below). This ignores that EBS gp3 is by + default only 125 MB/sec vs 250 MB/sec for gp2 >= 333GB. + The default is `false`. + +* **enable_ebs_gp3_migration_max_size** + defines the maximum volume size in GB until which auto migration happens. + Default is 1000 (1TB) which matches 3000 IOPS. ## Logical backup diff --git a/go.mod b/go.mod index 0ce0dad93..4e9c8a742 100644 --- a/go.mod +++ b/go.mod @@ -1,20 +1,22 @@ module github.com/zalando/postgres-operator -go 1.14 +go 1.15 require ( - github.com/aws/aws-sdk-go v1.35.15 - github.com/lib/pq v1.8.0 + github.com/aws/aws-sdk-go v1.36.3 + github.com/golang/mock v1.4.4 + github.com/lib/pq v1.9.0 github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d github.com/r3labs/diff v1.1.0 github.com/sirupsen/logrus v1.7.0 - github.com/stretchr/testify v1.5.1 - golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 - golang.org/x/tools v0.0.0-20201121010211-780cb80bd7fb // indirect - gopkg.in/yaml.v2 v2.2.8 - k8s.io/api v0.19.3 + github.com/stretchr/testify v1.6.1 + golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c + golang.org/x/mod v0.4.0 // indirect + golang.org/x/tools v0.0.0-20201207204333-a835c872fcea // indirect + gopkg.in/yaml.v2 v2.4.0 + k8s.io/api v0.19.4 k8s.io/apiextensions-apiserver v0.19.3 - k8s.io/apimachinery v0.19.3 + k8s.io/apimachinery v0.19.4 k8s.io/client-go v0.19.3 - k8s.io/code-generator v0.19.3 + k8s.io/code-generator v0.19.4 ) diff --git a/go.sum b/go.sum index 4e5704525..d8df3fda4 100644 --- a/go.sum +++ b/go.sum @@ -23,9 +23,9 @@ github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxB github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46 h1:lsxEuwrXEAokXB9qhlbKWPpo3KMLZQ5WB5WLQRW1uq0= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -43,16 +43,21 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.35.15 h1:JdQNM8hJe+9N9xP53S54NDmX8GCaZn8CCJ4LBHfom4U= -github.com/aws/aws-sdk-go v1.35.15/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= +github.com/aws/aws-sdk-go v1.36.3 h1:KYpG5OegwW3xgOsMxy01nj/Td281yxi1Ha2lJQJs4tI= +github.com/aws/aws-sdk-go v1.36.3/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/blang/semver v3.5.0+incompatible h1:CGxCgetQ64DKk7rdZ++Vfnb1+ogGNnB17OJKJXD2Cfs= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -63,10 +68,13 @@ github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkE github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= @@ -111,9 +119,11 @@ github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70t github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= +github.com/go-openapi/analysis v0.19.5 h1:8b2ZgKfKIUTVQpTb77MoRDIMEIwvDVw40o3aOXdfYzI= github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU= github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= +github.com/go-openapi/errors v0.19.2 h1:a2kIyV3w+OS3S97zxUndRVD46+FhGOUBDFY7nmu4CsY= github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= @@ -131,9 +141,11 @@ github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= +github.com/go-openapi/loads v0.19.4 h1:5I4CCSqoWzT+82bBkNIvmLc0UOsoKKQ4Fz+3VxOB7SY= github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCsn0N0ZrQsk= github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= +github.com/go-openapi/runtime v0.19.4 h1:csnOgcgAiuGoM/Po7PEpKDoNulCcF3FGbSnbHfxgjMI= github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= @@ -144,6 +156,7 @@ github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8 github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= +github.com/go-openapi/strfmt v0.19.3 h1:eRfyY5SkaNJCAwmmMcADjY31ow9+N7MCLW7oRkbsINA= github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= @@ -153,7 +166,9 @@ github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tF github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= +github.com/go-openapi/validate v0.19.5 h1:QhCBKRYqZR+SKo4gl1lPhPahope8/RLt6EVgY8X80w0= github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -168,6 +183,8 @@ github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4er github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -207,6 +224,7 @@ github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoA github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= @@ -219,6 +237,7 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -245,8 +264,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= -github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.9.0 h1:L8nSXQQzAYByakOFMTwpjRoHsMJklur4Gi59b6VivR8= +github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -259,8 +278,10 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -272,6 +293,7 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d h1:LznySqW8MqVeFh+pW6rOkFdld9QQ7jRydBKKM6jyPVI= github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d/go.mod h1:u3hJ0kqCQu/cPpsu3RbCOPZ0d7V3IjPjv1adNRleM9I= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= @@ -297,18 +319,22 @@ github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prY github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/r3labs/diff v1.1.0 h1:V53xhrbTHrWFWq3gI4b94AjgEJOerO1+1l0xyHOBi8M= @@ -329,6 +355,7 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -339,13 +366,15 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -359,16 +388,21 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5 h1:Gqga3zA9tdAcfqobUGjSoCob5L3f8Dt5EuOp3ihNZko= go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.mongodb.org/mongo-driver v1.1.2 h1:jxcFYjlkl8xaERsgLo+RNquI0epW6zuy/ZRQs6jnrFA= go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -381,6 +415,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c h1:9HhBz5L/UjnK9XLtiZhYAdue5BVKep3PMmS2LuPDt8k= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -401,8 +437,9 @@ golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKG golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0 h1:8pl+sMODzuvGJkmj2W4kZihvVb5mKm8pB/X44PIQHv8= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -423,12 +460,12 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -440,6 +477,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -469,6 +507,8 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 h1:/ZHdbVpdR/jk3g30/d4yUL0JU9kksj8+F/bnQUVLGDM= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -505,11 +545,10 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20201121010211-780cb80bd7fb h1:z5+u0pkAUPUWd3taoTialQ2JAMo4Wo1Z3L25U4ZV9r0= -golang.org/x/tools v0.0.0-20201121010211-780cb80bd7fb/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201207204333-a835c872fcea h1:LgKM3cNs8xO6GK1ZVK0nasPn7IN39Sz9EBTwQLyishk= +golang.org/x/tools v0.0.0-20201207204333-a835c872fcea/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -533,6 +572,7 @@ google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -540,6 +580,7 @@ google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -562,6 +603,7 @@ gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= @@ -574,25 +616,31 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -k8s.io/api v0.19.3 h1:GN6ntFnv44Vptj/b+OnMW7FmzkpDoIDLZRvKX3XH9aU= k8s.io/api v0.19.3/go.mod h1:VF+5FT1B74Pw3KxMdKyinLo+zynBaMBiAfGMuldcNDs= +k8s.io/api v0.19.4 h1:I+1I4cgJYuCDgiLNjKx7SLmIbwgj9w7N7Zr5vSIdwpo= +k8s.io/api v0.19.4/go.mod h1:SbtJ2aHCItirzdJ36YslycFNzWADYH3tgOhvBEFtZAk= k8s.io/apiextensions-apiserver v0.19.3 h1:WZxBypSHW4SdXHbdPTS/Jy7L2la6Niggs8BuU5o+avo= k8s.io/apiextensions-apiserver v0.19.3/go.mod h1:igVEkrE9TzInc1tYE7qSqxaLg/rEAp6B5+k9Q7+IC8Q= -k8s.io/apimachinery v0.19.3 h1:bpIQXlKjB4cB/oNpnNnV+BybGPR7iP5oYpsOTEJ4hgc= k8s.io/apimachinery v0.19.3/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= +k8s.io/apimachinery v0.19.4 h1:+ZoddM7nbzrDCp0T3SWnyxqf8cbWPT2fkZImoyvHUG0= +k8s.io/apimachinery v0.19.4/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= k8s.io/apiserver v0.19.3/go.mod h1:bx6dMm+H6ifgKFpCQT/SAhPwhzoeIMlHIaibomUDec0= k8s.io/client-go v0.19.3 h1:ctqR1nQ52NUs6LpI0w+a5U+xjYwflFwA13OJKcicMxg= k8s.io/client-go v0.19.3/go.mod h1:+eEMktZM+MG0KO+PTkci8xnbCZHvj9TqR6Q1XDUIJOM= -k8s.io/code-generator v0.19.3 h1:fTrTpJ8PZog5oo6MmeZtveo89emjQZHiw0ieybz1RSs= k8s.io/code-generator v0.19.3/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= +k8s.io/code-generator v0.19.4 h1:c8IL7RgTgJaYgr2bYMgjN0WikHnohbBhEgajfIkuP5I= +k8s.io/code-generator v0.19.4/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= k8s.io/component-base v0.19.3/go.mod h1:WhLWSIefQn8W8jxSLl5WNiR6z8oyMe/8Zywg7alOkRc= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14 h1:t4L10Qfx/p7ASH3gXCdIUtPbbIuegCoUJf3TMSFekjw= @@ -605,6 +653,7 @@ k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H k8s.io/utils v0.0.0-20200729134348-d5654de09c73 h1:uJmqzgNWG7XyClnU/mLPBWwfKKF1K8Hf8whTseBgJcg= k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9 h1:rusRLrDhjBp6aYtl9sGEvQJr6faoHoDLd0YcUBTZguI= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0= sigs.k8s.io/structured-merge-diff/v4 v4.0.1 h1:YXTMot5Qz/X1iBRJhAt+vI+HVttY0WkSqqhKxQ0xVbA= sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index dbefbebcf..7b99f4f45 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -36,6 +36,8 @@ data: # enable_admin_role_for_users: "true" # enable_crd_validation: "true" # enable_database_access: "true" + enable_ebs_gp3_migration: "false" + # enable_ebs_gp3_migration_max_size: 1000 # enable_init_containers: "true" # enable_lazy_spilo_upgrade: "false" enable_master_load_balancer: "false" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 9370c1500..f1270d136 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -290,6 +290,10 @@ spec: type: string aws_region: type: string + enable_ebs_gp3_migration: + type: boolean + enable_ebs_gp3_migration_max_size: + type: integer gcp_credentials: type: string kube_iam_role: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 84537e06a..fdfe09096 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -103,6 +103,8 @@ configuration: # additional_secret_mount: "some-secret-name" # additional_secret_mount_path: "/some/dir" aws_region: eu-central-1 + enable_ebs_gp3_migration: false + # enable_ebs_gp3_migration_max_size: 1000 # gcp_credentials: "" # kube_iam_role: "" # log_s3_bucket: "" diff --git a/mocks/mocks.go b/mocks/mocks.go new file mode 100644 index 000000000..f726b26e5 --- /dev/null +++ b/mocks/mocks.go @@ -0,0 +1 @@ +package mocks diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 0d098835b..8bdf0cd1f 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -1169,6 +1169,15 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{ "aws_region": { Type: "string", }, + "enable_ebs_gp3_migration": { + Type: "boolean", + }, + "enable_ebs_gp3_migration_max_size": { + Type: "integer", + }, + "gcp_credentials": { + Type: "string", + }, "kube_iam_role": { Type: "string", }, diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 56f808159..6c7c7767b 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -117,14 +117,16 @@ type LoadBalancerConfiguration struct { // AWSGCPConfiguration defines the configuration for AWS // TODO complete Google Cloud Platform (GCP) configuration type AWSGCPConfiguration struct { - WALES3Bucket string `json:"wal_s3_bucket,omitempty"` - AWSRegion string `json:"aws_region,omitempty"` - WALGSBucket string `json:"wal_gs_bucket,omitempty"` - GCPCredentials string `json:"gcp_credentials,omitempty"` - LogS3Bucket string `json:"log_s3_bucket,omitempty"` - KubeIAMRole string `json:"kube_iam_role,omitempty"` - AdditionalSecretMount string `json:"additional_secret_mount,omitempty"` - AdditionalSecretMountPath string `json:"additional_secret_mount_path" default:"/meta/credentials"` + WALES3Bucket string `json:"wal_s3_bucket,omitempty"` + AWSRegion string `json:"aws_region,omitempty"` + WALGSBucket string `json:"wal_gs_bucket,omitempty"` + GCPCredentials string `json:"gcp_credentials,omitempty"` + LogS3Bucket string `json:"log_s3_bucket,omitempty"` + KubeIAMRole string `json:"kube_iam_role,omitempty"` + AdditionalSecretMount string `json:"additional_secret_mount,omitempty"` + AdditionalSecretMountPath string `json:"additional_secret_mount_path" default:"/meta/credentials"` + EnableEBSGp3Migration bool `json:"enable_ebs_gp3_migration" default:"false"` + EnableEBSGp3MigrationMaxSize int64 `json:"enable_ebs_gp3_migration_max_size" default:"1000"` } // OperatorDebugConfiguration defines options for the debug mode diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 8a64462b2..74a9057a5 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -115,8 +115,11 @@ type Volume struct { Size string `json:"size"` StorageClass string `json:"storageClass,omitempty"` SubPath string `json:"subPath,omitempty"` + Iops *int64 `json:"iops,omitempty"` + Throughput *int64 `json:"throughput,omitempty"` } +// AdditionalVolume specs additional optional volumes for statefulset type AdditionalVolume struct { Name string `json:"name"` MountPath string `json:"mountPath"` @@ -156,11 +159,12 @@ type Patroni struct { SynchronousModeStrict bool `json:"synchronous_mode_strict,omitempty"` } -//StandbyCluster +// StandbyDescription contains s3 wal path type StandbyDescription struct { S3WalPath string `json:"s3_wal_path,omitempty"` } +// TLSDescription specs TLS properties type TLSDescription struct { SecretName string `json:"secretName,omitempty"` CertificateFile string `json:"certificateFile,omitempty"` @@ -198,7 +202,7 @@ type PostgresStatus struct { PostgresClusterStatus string `json:"PostgresClusterStatus"` } -// Options for connection pooler +// ConnectionPooler Options for connection pooler // // TODO: prepared snippets of configuration, one can choose via type, e.g. // pgbouncer-large (with higher resources) or odyssey-small (with smaller diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index de260dc53..51d9861e4 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -524,7 +524,7 @@ func (in *PostgresPodResourcesDefaults) DeepCopy() *PostgresPodResourcesDefaults func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { *out = *in in.PostgresqlParam.DeepCopyInto(&out.PostgresqlParam) - out.Volume = in.Volume + in.Volume.DeepCopyInto(&out.Volume) in.Patroni.DeepCopyInto(&out.Patroni) out.Resources = in.Resources if in.EnableConnectionPooler != nil { @@ -623,6 +623,11 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { (*out)[key] = *val.DeepCopy() } } + if in.SchedulerName != nil { + in, out := &in.SchedulerName, &out.SchedulerName + *out = new(string) + **out = **in + } if in.Tolerations != nil { in, out := &in.Tolerations, &out.Tolerations *out = make([]corev1.Toleration, len(*in)) @@ -687,11 +692,6 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.SchedulerName != nil { - in, out := &in.SchedulerName, &out.SchedulerName - *out = new(string) - **out = **in - } return } @@ -1160,6 +1160,16 @@ func (in UserFlags) DeepCopy() UserFlags { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Volume) DeepCopyInto(out *Volume) { *out = *in + if in.Iops != nil { + in, out := &in.Iops, &out.Iops + *out = new(int64) + **out = **in + } + if in.Throughput != nil { + in, out := &in.Throughput, &out.Throughput + *out = new(int64) + **out = **in + } return } diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index ee5c44bc9..00c75f801 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -25,6 +25,7 @@ import ( "github.com/zalando/postgres-operator/pkg/util/patroni" "github.com/zalando/postgres-operator/pkg/util/teams" "github.com/zalando/postgres-operator/pkg/util/users" + "github.com/zalando/postgres-operator/pkg/util/volumes" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" policybeta1 "k8s.io/api/policy/v1beta1" @@ -89,7 +90,10 @@ type Cluster struct { processMu sync.RWMutex // protects the current operation for reporting, no need to hold the master mutex specMu sync.RWMutex // protects the spec for reporting, no need to hold the master mutex ConnectionPooler map[PostgresRole]*ConnectionPoolerObjects + EBSVolumes map[string]volumes.VolumeProperties + VolumeResizer volumes.VolumeResizer } + type compareStatefulsetResult struct { match bool replace bool @@ -134,6 +138,13 @@ func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgres cluster.oauthTokenGetter = newSecretOauthTokenGetter(&kubeClient, cfg.OpConfig.OAuthTokenSecretName) cluster.patroni = patroni.New(cluster.logger) cluster.eventRecorder = eventRecorder + + cluster.EBSVolumes = make(map[string]volumes.VolumeProperties) + if cfg.OpConfig.StorageResizeMode != "pvc" || cfg.OpConfig.EnableEBSGp3Migration { + cluster.VolumeResizer = &volumes.EBSVolumeResizer{AWSRegion: cfg.OpConfig.AWSRegion} + + } + return cluster } diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 625c67313..94736b531 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -11,7 +11,6 @@ import ( "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" - "github.com/zalando/postgres-operator/pkg/util/volumes" appsv1 "k8s.io/api/apps/v1" batchv1beta1 "k8s.io/api/batch/v1beta1" v1 "k8s.io/api/core/v1" @@ -55,7 +54,23 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { } c.logger.Debugf("syncing volumes using %q storage resize mode", c.OpConfig.StorageResizeMode) - if c.OpConfig.StorageResizeMode == "pvc" { + + if c.OpConfig.EnableEBSGp3Migration { + err = c.executeEBSMigration() + if nil != err { + return err + } + } + + if c.OpConfig.StorageResizeMode == "mixed" { + // mixed op uses AWS API to adjust size,throughput,iops and calls pvc chance for file system resize + + // resize pvc to adjust filesystem size until better K8s support + if err = c.syncVolumeClaims(); err != nil { + err = fmt.Errorf("could not sync persistent volume claims: %v", err) + return err + } + } else if c.OpConfig.StorageResizeMode == "pvc" { if err = c.syncVolumeClaims(); err != nil { err = fmt.Errorf("could not sync persistent volume claims: %v", err) return err @@ -599,7 +614,8 @@ func (c *Cluster) syncVolumes() error { if !act { return nil } - if err := c.resizeVolumes(c.Spec.Volume, []volumes.VolumeResizer{&volumes.EBSVolumeResizer{AWSRegion: c.OpConfig.AWSRegion}}); err != nil { + + if err := c.resizeVolumes(); err != nil { return fmt.Errorf("could not sync volumes: %v", err) } diff --git a/pkg/cluster/volumes.go b/pkg/cluster/volumes.go index 44b85663f..162d24075 100644 --- a/pkg/cluster/volumes.go +++ b/pkg/cluster/volumes.go @@ -15,7 +15,6 @@ import ( "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/filesystems" - "github.com/zalando/postgres-operator/pkg/util/volumes" ) func (c *Cluster) listPersistentVolumeClaims() ([]v1.PersistentVolumeClaim, error) { @@ -119,19 +118,26 @@ func (c *Cluster) listPersistentVolumes() ([]*v1.PersistentVolume, error) { } // resizeVolumes resize persistent volumes compatible with the given resizer interface -func (c *Cluster) resizeVolumes(newVolume acidv1.Volume, resizers []volumes.VolumeResizer) error { - c.setProcessName("resizing volumes") +func (c *Cluster) resizeVolumes() error { + if c.VolumeResizer == nil { + return fmt.Errorf("no volume resizer set for EBS volume handling") + } + c.setProcessName("resizing EBS volumes") + + resizer := c.VolumeResizer var totalIncompatible int - newQuantity, err := resource.ParseQuantity(newVolume.Size) + newQuantity, err := resource.ParseQuantity(c.Spec.Volume.Size) if err != nil { return fmt.Errorf("could not parse volume size: %v", err) } - pvs, newSize, err := c.listVolumesWithManifestSize(newVolume) + + pvs, newSize, err := c.listVolumesWithManifestSize(c.Spec.Volume) if err != nil { return fmt.Errorf("could not list persistent volumes: %v", err) } + for _, pv := range pvs { volumeSize := quantityToGigabyte(pv.Spec.Capacity[v1.ResourceStorage]) if volumeSize >= newSize { @@ -141,43 +147,43 @@ func (c *Cluster) resizeVolumes(newVolume acidv1.Volume, resizers []volumes.Volu continue } compatible := false - for _, resizer := range resizers { - if !resizer.VolumeBelongsToProvider(pv) { - continue - } - compatible = true - if !resizer.IsConnectedToProvider() { - err := resizer.ConnectToProvider() - if err != nil { - return fmt.Errorf("could not connect to the volume provider: %v", err) - } - defer func() { - if err := resizer.DisconnectFromProvider(); err != nil { - c.logger.Errorf("%v", err) - } - }() - } - awsVolumeID, err := resizer.GetProviderVolumeID(pv) - if err != nil { - return err - } - c.logger.Debugf("updating persistent volume %q to %d", pv.Name, newSize) - if err := resizer.ResizeVolume(awsVolumeID, newSize); err != nil { - return fmt.Errorf("could not resize EBS volume %q: %v", awsVolumeID, err) - } - c.logger.Debugf("resizing the filesystem on the volume %q", pv.Name) - podName := getPodNameFromPersistentVolume(pv) - if err := c.resizePostgresFilesystem(podName, []filesystems.FilesystemResizer{&filesystems.Ext234Resize{}}); err != nil { - return fmt.Errorf("could not resize the filesystem on pod %q: %v", podName, err) - } - c.logger.Debugf("filesystem resize successful on volume %q", pv.Name) - pv.Spec.Capacity[v1.ResourceStorage] = newQuantity - c.logger.Debugf("updating persistent volume definition for volume %q", pv.Name) - if _, err := c.KubeClient.PersistentVolumes().Update(context.TODO(), pv, metav1.UpdateOptions{}); err != nil { - return fmt.Errorf("could not update persistent volume: %q", err) - } - c.logger.Debugf("successfully updated persistent volume %q", pv.Name) + + if !resizer.VolumeBelongsToProvider(pv) { + continue } + compatible = true + if !resizer.IsConnectedToProvider() { + err := resizer.ConnectToProvider() + if err != nil { + return fmt.Errorf("could not connect to the volume provider: %v", err) + } + defer func() { + if err := resizer.DisconnectFromProvider(); err != nil { + c.logger.Errorf("%v", err) + } + }() + } + awsVolumeID, err := resizer.GetProviderVolumeID(pv) + if err != nil { + return err + } + c.logger.Debugf("updating persistent volume %q to %d", pv.Name, newSize) + if err := resizer.ResizeVolume(awsVolumeID, newSize); err != nil { + return fmt.Errorf("could not resize EBS volume %q: %v", awsVolumeID, err) + } + c.logger.Debugf("resizing the filesystem on the volume %q", pv.Name) + podName := getPodNameFromPersistentVolume(pv) + if err := c.resizePostgresFilesystem(podName, []filesystems.FilesystemResizer{&filesystems.Ext234Resize{}}); err != nil { + return fmt.Errorf("could not resize the filesystem on pod %q: %v", podName, err) + } + c.logger.Debugf("filesystem resize successful on volume %q", pv.Name) + pv.Spec.Capacity[v1.ResourceStorage] = newQuantity + c.logger.Debugf("updating persistent volume definition for volume %q", pv.Name) + if _, err := c.KubeClient.PersistentVolumes().Update(context.TODO(), pv, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("could not update persistent volume: %q", err) + } + c.logger.Debugf("successfully updated persistent volume %q", pv.Name) + if !compatible { c.logger.Warningf("volume %q is incompatible with all available resizing providers, consider switching storage_resize_mode to pvc or off", pv.Name) totalIncompatible++ @@ -245,3 +251,61 @@ func getPodNameFromPersistentVolume(pv *v1.PersistentVolume) *spec.NamespacedNam func quantityToGigabyte(q resource.Quantity) int64 { return q.ScaledValue(0) / (1 * constants.Gigabyte) } + +func (c *Cluster) executeEBSMigration() error { + if !c.OpConfig.EnableEBSGp3Migration { + return nil + } + c.logger.Infof("starting EBS gp2 to gp3 migration") + + pvs, _, err := c.listVolumesWithManifestSize(c.Spec.Volume) + if err != nil { + return fmt.Errorf("could not list persistent volumes: %v", err) + } + c.logger.Debugf("found %d volumes, size of known volumes %d", len(pvs), len(c.EBSVolumes)) + + volumeIds := []string{} + var volumeID string + for _, pv := range pvs { + volumeID, err = c.VolumeResizer.ExtractVolumeID(pv.Spec.AWSElasticBlockStore.VolumeID) + if err != nil { + continue + } + + volumeIds = append(volumeIds, volumeID) + } + + if len(volumeIds) == len(c.EBSVolumes) { + hasGp2 := false + for _, v := range c.EBSVolumes { + if v.VolumeType == "gp2" { + hasGp2 = true + } + } + + if !hasGp2 { + c.logger.Infof("no EBS gp2 volumes left to migrate") + return nil + } + } + + awsVolumes, err := c.VolumeResizer.DescribeVolumes(volumeIds) + if nil != err { + return err + } + + for _, volume := range awsVolumes { + if volume.VolumeType == "gp2" && volume.Size < c.OpConfig.EnableEBSGp3MigrationMaxSize { + c.logger.Infof("modifying EBS volume %s to type gp3 migration (%d)", volume.VolumeID, volume.Size) + err = c.VolumeResizer.ModifyVolume(volume.VolumeID, "gp3", volume.Size, 3000, 125) + if nil != err { + c.logger.Warningf("modifying volume %s failed: %v", volume.VolumeID, err) + } + } else { + c.logger.Debugf("skipping EBS volume %s to type gp3 migration (%d)", volume.VolumeID, volume.Size) + } + c.EBSVolumes[volume.VolumeID] = volume + } + + return nil +} diff --git a/pkg/cluster/volumes_test.go b/pkg/cluster/volumes_test.go index 49fbbd228..907b9959f 100644 --- a/pkg/cluster/volumes_test.go +++ b/pkg/cluster/volumes_test.go @@ -1,6 +1,7 @@ package cluster import ( + "fmt" "testing" "context" @@ -10,11 +11,14 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" + "github.com/zalando/postgres-operator/mocks" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" + "github.com/zalando/postgres-operator/pkg/util/volumes" "k8s.io/client-go/kubernetes/fake" ) @@ -23,6 +27,8 @@ func NewFakeKubernetesClient() (k8sutil.KubernetesClient, *fake.Clientset) { return k8sutil.KubernetesClient{ PersistentVolumeClaimsGetter: clientSet.CoreV1(), + PersistentVolumesGetter: clientSet.CoreV1(), + PodsGetter: clientSet.CoreV1(), }, clientSet } @@ -33,6 +39,9 @@ func TestResizeVolumeClaim(t *testing.T) { namespace := "default" newVolumeSize := "2Gi" + storage1Gi, err := resource.ParseQuantity("1Gi") + assert.NoError(t, err) + // new cluster with pvc storage resize mode and configured labels var cluster = New( Config{ @@ -51,55 +60,9 @@ func TestResizeVolumeClaim(t *testing.T) { filterLabels := cluster.labelsSet(false) // define and create PVCs for 1Gi volumes - storage1Gi, err := resource.ParseQuantity("1Gi") - assert.NoError(t, err) - - pvcList := &v1.PersistentVolumeClaimList{ - Items: []v1.PersistentVolumeClaim{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: constants.DataVolumeName + "-" + clusterName + "-0", - Namespace: namespace, - Labels: filterLabels, - }, - Spec: v1.PersistentVolumeClaimSpec{ - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceStorage: storage1Gi, - }, - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: constants.DataVolumeName + "-" + clusterName + "-1", - Namespace: namespace, - Labels: filterLabels, - }, - Spec: v1.PersistentVolumeClaimSpec{ - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceStorage: storage1Gi, - }, - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: constants.DataVolumeName + "-" + clusterName + "-2-0", - Namespace: namespace, - Labels: labels.Set{}, - }, - Spec: v1.PersistentVolumeClaimSpec{ - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceStorage: storage1Gi, - }, - }, - }, - }, - }, - } + pvcList := CreatePVCs(namespace, clusterName, filterLabels, 2, "1Gi") + // add another PVC with different cluster name + pvcList.Items = append(pvcList.Items, CreatePVCs(namespace, clusterName+"-2", labels.Set{}, 1, "1Gi").Items[0]) for _, pvc := range pvcList.Items { cluster.KubeClient.PersistentVolumeClaims(namespace).Create(context.TODO(), &pvc, metav1.CreateOptions{}) @@ -169,3 +132,132 @@ func TestQuantityToGigabyte(t *testing.T) { } } } + +func CreatePVCs(namespace string, clusterName string, labels labels.Set, n int, size string) v1.PersistentVolumeClaimList { + // define and create PVCs for 1Gi volumes + storage1Gi, _ := resource.ParseQuantity(size) + pvcList := v1.PersistentVolumeClaimList{ + Items: []v1.PersistentVolumeClaim{}, + } + + for i := 0; i < n; i++ { + pvc := v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s-%d", constants.DataVolumeName, clusterName, i), + Namespace: namespace, + Labels: labels, + }, + Spec: v1.PersistentVolumeClaimSpec{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: storage1Gi, + }, + }, + VolumeName: fmt.Sprintf("persistent-volume-%d", i), + }, + } + pvcList.Items = append(pvcList.Items, pvc) + } + + return pvcList +} + +func TestMigrateEBS(t *testing.T) { + client, _ := NewFakeKubernetesClient() + clusterName := "acid-test-cluster" + namespace := "default" + + // new cluster with pvc storage resize mode and configured labels + var cluster = New( + Config{ + OpConfig: config.Config{ + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: "cluster-name", + }, + StorageResizeMode: "pvc", + EnableEBSGp3Migration: true, + EnableEBSGp3MigrationMaxSize: 1000, + }, + }, client, acidv1.Postgresql{}, logger, eventRecorder) + cluster.Spec.Volume.Size = "1Gi" + + // set metadata, so that labels will get correct values + cluster.Name = clusterName + cluster.Namespace = namespace + filterLabels := cluster.labelsSet(false) + + pvcList := CreatePVCs(namespace, clusterName, filterLabels, 2, "1Gi") + + ps := v1.PersistentVolumeSpec{} + ps.AWSElasticBlockStore = &v1.AWSElasticBlockStoreVolumeSource{} + ps.AWSElasticBlockStore.VolumeID = "aws://eu-central-1b/ebs-volume-1" + + ps2 := v1.PersistentVolumeSpec{} + ps2.AWSElasticBlockStore = &v1.AWSElasticBlockStoreVolumeSource{} + ps2.AWSElasticBlockStore.VolumeID = "aws://eu-central-1b/ebs-volume-2" + + pvList := &v1.PersistentVolumeList{ + Items: []v1.PersistentVolume{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "persistent-volume-0", + }, + Spec: ps, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "persistent-volume-1", + }, + Spec: ps2, + }, + }, + } + + for _, pvc := range pvcList.Items { + cluster.KubeClient.PersistentVolumeClaims(namespace).Create(context.TODO(), &pvc, metav1.CreateOptions{}) + } + + for _, pv := range pvList.Items { + cluster.KubeClient.PersistentVolumes().Create(context.TODO(), &pv, metav1.CreateOptions{}) + } + + pod := v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName + "-0", + Labels: filterLabels, + }, + Spec: v1.PodSpec{}, + } + + cluster.KubeClient.Pods(namespace).Create(context.TODO(), &pod, metav1.CreateOptions{}) + + pod = v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName + "-1", + Labels: filterLabels, + }, + Spec: v1.PodSpec{}, + } + + cluster.KubeClient.Pods(namespace).Create(context.TODO(), &pod, metav1.CreateOptions{}) + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resizer := mocks.NewMockVolumeResizer(ctrl) + + resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-1")).Return("ebs-volume-1", nil) + resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-2")).Return("ebs-volume-2", nil) + + resizer.EXPECT().DescribeVolumes(gomock.Eq([]string{"ebs-volume-1", "ebs-volume-2"})).Return( + []volumes.VolumeProperties{ + {VolumeID: "ebs-volume-1", VolumeType: "gp2", Size: 100}, + {VolumeID: "ebs-volume-2", VolumeType: "gp3", Size: 100}}, nil) + + // expect only gp2 volume to be modified + resizer.EXPECT().ModifyVolume(gomock.Eq("ebs-volume-1"), gomock.Eq("gp3"), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + + cluster.VolumeResizer = resizer + cluster.executeEBSMigration() +} diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 8fb951a80..20fb0f0dc 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -138,6 +138,8 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.GCPCredentials = fromCRD.AWSGCP.GCPCredentials result.AdditionalSecretMount = fromCRD.AWSGCP.AdditionalSecretMount result.AdditionalSecretMountPath = util.Coalesce(fromCRD.AWSGCP.AdditionalSecretMountPath, "/meta/credentials") + result.EnableEBSGp3Migration = fromCRD.AWSGCP.EnableEBSGp3Migration + result.EnableEBSGp3MigrationMaxSize = fromCRD.AWSGCP.EnableEBSGp3MigrationMaxSize // logical backup config result.LogicalBackupSchedule = util.Coalesce(fromCRD.LogicalBackup.Schedule, "30 00 * * *") diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 7f9c66ea4..122c192a5 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -163,6 +163,8 @@ type Config struct { GCPCredentials string `name:"gcp_credentials"` AdditionalSecretMount string `name:"additional_secret_mount"` AdditionalSecretMountPath string `name:"additional_secret_mount_path" default:"/meta/credentials"` + EnableEBSGp3Migration bool `name:"enable_ebs_gp3_migration" default:"false"` + EnableEBSGp3MigrationMaxSize int64 `name:"enable_ebs_gp3_migration_max_size" default:"1000"` DebugLogging bool `name:"debug_logging" default:"true"` EnableDBAccess bool `name:"enable_database_access" default:"true"` EnableTeamsAPI bool `name:"enable_teams_api" default:"true"` diff --git a/pkg/util/volumes/ebs.go b/pkg/util/volumes/ebs.go index 666436a06..17016fb09 100644 --- a/pkg/util/volumes/ebs.go +++ b/pkg/util/volumes/ebs.go @@ -7,7 +7,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/retryutil" @@ -20,42 +20,80 @@ type EBSVolumeResizer struct { } // ConnectToProvider connects to AWS. -func (c *EBSVolumeResizer) ConnectToProvider() error { - sess, err := session.NewSession(&aws.Config{Region: aws.String(c.AWSRegion)}) +func (r *EBSVolumeResizer) ConnectToProvider() error { + sess, err := session.NewSession(&aws.Config{Region: aws.String(r.AWSRegion)}) if err != nil { return fmt.Errorf("could not establish AWS session: %v", err) } - c.connection = ec2.New(sess) + r.connection = ec2.New(sess) return nil } // IsConnectedToProvider checks if AWS connection is established. -func (c *EBSVolumeResizer) IsConnectedToProvider() bool { - return c.connection != nil +func (r *EBSVolumeResizer) IsConnectedToProvider() bool { + return r.connection != nil } // VolumeBelongsToProvider checks if the given persistent volume is backed by EBS. -func (c *EBSVolumeResizer) VolumeBelongsToProvider(pv *v1.PersistentVolume) bool { +func (r *EBSVolumeResizer) VolumeBelongsToProvider(pv *v1.PersistentVolume) bool { return pv.Spec.AWSElasticBlockStore != nil && pv.Annotations[constants.VolumeStorateProvisionerAnnotation] == constants.EBSProvisioner } -// GetProviderVolumeID converts aws://eu-central-1b/vol-00f93d4827217c629 to vol-00f93d4827217c629 for EBS volumes -func (c *EBSVolumeResizer) GetProviderVolumeID(pv *v1.PersistentVolume) (string, error) { - volumeID := pv.Spec.AWSElasticBlockStore.VolumeID - if volumeID == "" { - return "", fmt.Errorf("volume id is empty for volume %q", pv.Name) - } +// ExtractVolumeID extracts volumeID +func (r *EBSVolumeResizer) ExtractVolumeID(volumeID string) (string, error) { idx := strings.LastIndex(volumeID, constants.EBSVolumeIDStart) + 1 if idx == 0 { - return "", fmt.Errorf("malfored EBS volume id %q", volumeID) + return "", fmt.Errorf("malformed EBS volume id %q", volumeID) } return volumeID[idx:], nil } +// GetProviderVolumeID converts aws://eu-central-1b/vol-00f93d4827217c629 to vol-00f93d4827217c629 for EBS volumes +func (r *EBSVolumeResizer) GetProviderVolumeID(pv *v1.PersistentVolume) (string, error) { + volumeID := pv.Spec.AWSElasticBlockStore.VolumeID + if volumeID == "" { + return "", fmt.Errorf("got empty volume id for volume %v", pv) + } + + return r.ExtractVolumeID(volumeID) +} + +// DescribeVolumes ... +func (r *EBSVolumeResizer) DescribeVolumes(volumeIds []string) ([]VolumeProperties, error) { + if !r.IsConnectedToProvider() { + err := r.ConnectToProvider() + if err != nil { + return nil, err + } + } + + volumeOutput, err := r.connection.DescribeVolumes(&ec2.DescribeVolumesInput{VolumeIds: aws.StringSlice((volumeIds))}) + if err != nil { + return nil, err + } + + p := []VolumeProperties{} + if nil == volumeOutput.Volumes { + return p, nil + } + + for _, v := range volumeOutput.Volumes { + if *v.VolumeType == "gp3" { + p = append(p, VolumeProperties{VolumeID: *v.VolumeId, Size: *v.Size, VolumeType: *v.VolumeType, Iops: *v.Iops, Throughput: *v.Throughput}) + } else if *v.VolumeType == "gp2" { + p = append(p, VolumeProperties{VolumeID: *v.VolumeId, Size: *v.Size, VolumeType: *v.VolumeType}) + } else { + return nil, fmt.Errorf("Discovered unexpected volume type %s %s", *v.VolumeId, *v.VolumeType) + } + } + + return p, nil +} + // ResizeVolume actually calls AWS API to resize the EBS volume if necessary. -func (c *EBSVolumeResizer) ResizeVolume(volumeID string, newSize int64) error { +func (r *EBSVolumeResizer) ResizeVolume(volumeID string, newSize int64) error { /* first check if the volume is already of a requested size */ - volumeOutput, err := c.connection.DescribeVolumes(&ec2.DescribeVolumesInput{VolumeIds: []*string{&volumeID}}) + volumeOutput, err := r.connection.DescribeVolumes(&ec2.DescribeVolumesInput{VolumeIds: []*string{&volumeID}}) if err != nil { return fmt.Errorf("could not get information about the volume: %v", err) } @@ -68,7 +106,7 @@ func (c *EBSVolumeResizer) ResizeVolume(volumeID string, newSize int64) error { return nil } input := ec2.ModifyVolumeInput{Size: &newSize, VolumeId: &volumeID} - output, err := c.connection.ModifyVolume(&input) + output, err := r.connection.ModifyVolume(&input) if err != nil { return fmt.Errorf("could not modify persistent volume: %v", err) } @@ -87,7 +125,54 @@ func (c *EBSVolumeResizer) ResizeVolume(volumeID string, newSize int64) error { in := ec2.DescribeVolumesModificationsInput{VolumeIds: []*string{&volumeID}} return retryutil.Retry(constants.EBSVolumeResizeWaitInterval, constants.EBSVolumeResizeWaitTimeout, func() (bool, error) { - out, err := c.connection.DescribeVolumesModifications(&in) + out, err := r.connection.DescribeVolumesModifications(&in) + if err != nil { + return false, fmt.Errorf("could not describe volume modification: %v", err) + } + if len(out.VolumesModifications) != 1 { + return false, fmt.Errorf("describe volume modification didn't return one record for volume %q", volumeID) + } + if *out.VolumesModifications[0].VolumeId != volumeID { + return false, fmt.Errorf("non-matching volume id when describing modifications: %q is different from %q", + *out.VolumesModifications[0].VolumeId, volumeID) + } + return *out.VolumesModifications[0].ModificationState != constants.EBSVolumeStateModifying, nil + }) +} + +// ModifyVolume Modify EBS volume +func (r *EBSVolumeResizer) ModifyVolume(volumeID string, newType string, newSize int64, iops int64, throughput int64) error { + /* first check if the volume is already of a requested size */ + volumeOutput, err := r.connection.DescribeVolumes(&ec2.DescribeVolumesInput{VolumeIds: []*string{&volumeID}}) + if err != nil { + return fmt.Errorf("could not get information about the volume: %v", err) + } + vol := volumeOutput.Volumes[0] + if *vol.VolumeId != volumeID { + return fmt.Errorf("describe volume %q returned information about a non-matching volume %q", volumeID, *vol.VolumeId) + } + + input := ec2.ModifyVolumeInput{Size: &newSize, VolumeId: &volumeID, VolumeType: &newType, Iops: &iops, Throughput: &throughput} + output, err := r.connection.ModifyVolume(&input) + if err != nil { + return fmt.Errorf("could not modify persistent volume: %v", err) + } + + state := *output.VolumeModification.ModificationState + if state == constants.EBSVolumeStateFailed { + return fmt.Errorf("could not modify persistent volume %q: modification state failed", volumeID) + } + if state == "" { + return fmt.Errorf("received empty modification status") + } + if state == constants.EBSVolumeStateOptimizing || state == constants.EBSVolumeStateCompleted { + return nil + } + // wait until the volume reaches the "optimizing" or "completed" state + in := ec2.DescribeVolumesModificationsInput{VolumeIds: []*string{&volumeID}} + return retryutil.Retry(constants.EBSVolumeResizeWaitInterval, constants.EBSVolumeResizeWaitTimeout, + func() (bool, error) { + out, err := r.connection.DescribeVolumesModifications(&in) if err != nil { return false, fmt.Errorf("could not describe volume modification: %v", err) } @@ -103,7 +188,7 @@ func (c *EBSVolumeResizer) ResizeVolume(volumeID string, newSize int64) error { } // DisconnectFromProvider closes connection to the EC2 instance -func (c *EBSVolumeResizer) DisconnectFromProvider() error { - c.connection = nil +func (r *EBSVolumeResizer) DisconnectFromProvider() error { + r.connection = nil return nil } diff --git a/pkg/util/volumes/volumes.go b/pkg/util/volumes/volumes.go index 94c0fffc8..9b44c0d00 100644 --- a/pkg/util/volumes/volumes.go +++ b/pkg/util/volumes/volumes.go @@ -1,8 +1,17 @@ package volumes -import ( - "k8s.io/api/core/v1" -) +//go:generate mockgen -package mocks -destination=$PWD/mocks/$GOFILE -source=$GOFILE -build_flags=-mod=vendor + +import v1 "k8s.io/api/core/v1" + +// VolumeProperties ... +type VolumeProperties struct { + VolumeID string + VolumeType string + Size int64 + Iops int64 + Throughput int64 +} // VolumeResizer defines the set of methods used to implememnt provider-specific resizing of persistent volumes. type VolumeResizer interface { @@ -10,6 +19,9 @@ type VolumeResizer interface { IsConnectedToProvider() bool VolumeBelongsToProvider(pv *v1.PersistentVolume) bool GetProviderVolumeID(pv *v1.PersistentVolume) (string, error) + ExtractVolumeID(volumeID string) (string, error) ResizeVolume(providerVolumeID string, newSize int64) error + ModifyVolume(providerVolumeID string, newType string, newSize int64, iops int64, throughput int64) error DisconnectFromProvider() error + DescribeVolumes(providerVolumesID []string) ([]VolumeProperties, error) } From 6a97316a69e5a0b05646c3d4b1b4c348e0760395 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 11 Dec 2020 16:34:01 +0100 Subject: [PATCH 132/168] Support inherited annotations for all major objects (#1236) * add comments where inherited annotations could be added * add inheritedAnnotations feature * return nil if no annotations are set * minor changes * first downscaler then inherited annotations * add unit test for inherited annotations * add pvc to test + minor changes * missing comma * fix nil map assignment * set annotations in the same order it is done in other places * replace acidClientSet with acid getters in K8s client * more fixes on clientSet vs getters * minor changes * remove endpoints from annotation test * refine unit test - but deployment and sts are still empty * fix checkinng sts and deployment * make annotations setter one liners * no need for len check anymore Co-authored-by: Rafia Sabih --- .../crds/operatorconfigurations.yaml | 4 + charts/postgres-operator/values-crd.yaml | 6 +- charts/postgres-operator/values.yaml | 5 +- docs/reference/operator_parameters.md | 21 ++- e2e/tests/k8s_api.py | 4 +- e2e/tests/test_e2e.py | 7 +- manifests/configmap.yaml | 1 + manifests/operatorconfiguration.crd.yaml | 4 + ...gresql-operator-default-configuration.yaml | 2 + pkg/apis/acid.zalan.do/v1/crds.go | 10 +- .../v1/operator_configuration_type.go | 1 + .../acid.zalan.do/v1/zz_generated.deepcopy.go | 5 + pkg/cluster/connection_pooler.go | 10 +- pkg/cluster/k8sres.go | 34 +++-- pkg/cluster/sync.go | 22 ++- pkg/cluster/util.go | 27 ++++ pkg/cluster/util_test.go | 141 ++++++++++++++++++ pkg/cluster/volumes_test.go | 4 +- pkg/controller/operator_config.go | 3 +- pkg/controller/postgresql.go | 2 +- pkg/controller/util.go | 2 +- pkg/util/config/config.go | 1 + pkg/util/k8sutil/k8sutil.go | 27 +++- 23 files changed, 288 insertions(+), 55 deletions(-) create mode 100644 pkg/cluster/util_test.go diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 0e0efd6c1..a19d3cfd5 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -166,6 +166,10 @@ spec: type: string template: type: boolean + inherited_annotations: + type: array + items: + type: string inherited_labels: type: array items: diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 21292a13e..e553c9d6f 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -91,7 +91,11 @@ configKubernetes: # namespaced name of the secret containing infrastructure roles names and passwords # infrastructure_roles_secret_name: postgresql-infrastructure-roles - # list of labels that can be inherited from the cluster manifest + # list of annotation keys that can be inherited from the cluster manifest + # inherited_annotations: + # - owned-by + + # list of label keys that can be inherited from the cluster manifest # inherited_labels: # - application # - environment diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 8a7776c54..2f68c9d4b 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -88,7 +88,10 @@ configKubernetes: # namespaced name of the secret containing infrastructure roles names and passwords # infrastructure_roles_secret_name: postgresql-infrastructure-roles - # list of labels that can be inherited from the cluster manifest + # list of annotation keys that can be inherited from the cluster manifest + # inherited_annotations: owned-by + + # list of label keys that can be inherited from the cluster manifest # inherited_labels: application,environment # timeout for successful migration of master pods from unschedulable node diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 63903cb81..87841d1d5 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -274,6 +274,12 @@ configuration they are grouped under the `kubernetes` key. are extracted. For the ConfigMap this has to be a string which allows referencing only one infrastructure roles secret. The default is empty. +* **inherited_annotations** + list of annotation keys that can be inherited from the cluster manifest, and + added to each child objects (`Deployment`, `StatefulSet`, `Pod`, `PDB` and + `Services`) created by the operator incl. the ones from the connection + pooler deployment. The default is empty. + * **pod_role_label** name of the label assigned to the Postgres pods (and services/endpoints) by the operator. The default is `spilo-role`. @@ -283,15 +289,16 @@ configuration they are grouped under the `kubernetes` key. objects. The default is `application:spilo`. * **inherited_labels** - list of labels that can be inherited from the cluster manifest, and added to - each child objects (`StatefulSet`, `Pod`, `Service` and `Endpoints`) created - by the operator. Typical use case is to dynamically pass labels that are - specific to a given Postgres cluster, in order to implement `NetworkPolicy`. - The default is empty. + list of label keys that can be inherited from the cluster manifest, and + added to each child objects (`Deployment`, `StatefulSet`, `Pod`, `PVCs`, + `PDB`, `Service`, `Endpoints` and `Secrets`) created by the operator. + Typical use case is to dynamically pass labels that are specific to a + given Postgres cluster, in order to implement `NetworkPolicy`. The default + is empty. * **cluster_name_label** - name of the label assigned to Kubernetes objects created by the operator that - indicates which cluster a given object belongs to. The default is + name of the label assigned to Kubernetes objects created by the operator + that indicates which cluster a given object belongs to. The default is `cluster-name`. * **node_readiness_label** diff --git a/e2e/tests/k8s_api.py b/e2e/tests/k8s_api.py index 30165e6a0..95e1dc9ad 100644 --- a/e2e/tests/k8s_api.py +++ b/e2e/tests/k8s_api.py @@ -117,7 +117,7 @@ class K8s: for svc in svcs: for key, value in annotations.items(): if not svc.metadata.annotations or key not in svc.metadata.annotations or svc.metadata.annotations[key] != value: - print("Expected key {} not found in annotations {}".format(key, svc.metadata.annotations)) + print("Expected key {} not found in service annotations {}".format(key, svc.metadata.annotations)) return False return True @@ -126,7 +126,7 @@ class K8s: for sset in ssets: for key, value in annotations.items(): if key not in sset.metadata.annotations or sset.metadata.annotations[key] != value: - print("Expected key {} not found in annotations {}".format(key, sset.metadata.annotations)) + print("Expected key {} not found in statefulset annotations {}".format(key, sset.metadata.annotations)) return False return True diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index b98d0d956..d396da01b 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -852,6 +852,7 @@ class EndToEndTestCase(unittest.TestCase): patch_sset_propagate_annotations = { "data": { "downscaler_annotations": "deployment-time,downscaler/*", + "inherited_annotations": "owned-by", } } k8s.update_config(patch_sset_propagate_annotations) @@ -861,6 +862,7 @@ class EndToEndTestCase(unittest.TestCase): "annotations": { "deployment-time": "2020-04-30 12:00:00", "downscaler/downtime_replicas": "0", + "owned-by": "acid", }, } } @@ -870,10 +872,9 @@ class EndToEndTestCase(unittest.TestCase): annotations = { "deployment-time": "2020-04-30 12:00:00", "downscaler/downtime_replicas": "0", + "owned-by": "acid", } - - self.eventuallyTrue(lambda: k8s.check_statefulset_annotations(cluster_label, annotations), "Annotations missing") - + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") self.eventuallyTrue(lambda: k8s.check_statefulset_annotations(cluster_label, annotations), "Annotations missing") @timeout_decorator.timeout(TEST_TIMEOUT_SEC) diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 7b99f4f45..111701829 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -57,6 +57,7 @@ data: # kubernetes_use_configmaps: "false" # infrastructure_roles_secret_name: "postgresql-infrastructure-roles" # infrastructure_roles_secrets: "secretname:monitoring-roles,userkey:user,passwordkey:password,rolekey:inrole" + # inherited_annotations: owned-by # inherited_labels: application,environment # kube_iam_role: "" # log_s3_bucket: "" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index f1270d136..8fbc6f042 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -164,6 +164,10 @@ spec: type: string template: type: boolean + inherited_annotations: + type: array + items: + type: string inherited_labels: type: array items: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index fdfe09096..00b095c1b 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -49,6 +49,8 @@ configuration: # - secretname: "other-infrastructure-role" # userkey: "other-user-key" # passwordkey: "other-password-key" + # inherited_annotations: + # - owned-by # inherited_labels: # - application # - environment diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 8bdf0cd1f..938abf7fc 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -961,6 +961,14 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{ }, }, }, + "inherited_annotations": { + Type: "array", + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ + Type: "string", + }, + }, + }, "inherited_labels": { Type: "array", Items: &apiextv1.JSONSchemaPropsOrArray{ @@ -1407,7 +1415,7 @@ func buildCRD(name, kind, plural, short string, columns []apiextv1.CustomResourc }, Scope: apiextv1.NamespaceScoped, Versions: []apiextv1.CustomResourceDefinitionVersion{ - apiextv1.CustomResourceDefinitionVersion{ + { Name: SchemeGroupVersion.Version, Served: true, Storage: true, diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 6c7c7767b..e79405224 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -66,6 +66,7 @@ type KubernetesMetaConfiguration struct { PodRoleLabel string `json:"pod_role_label,omitempty"` ClusterLabels map[string]string `json:"cluster_labels,omitempty"` InheritedLabels []string `json:"inherited_labels,omitempty"` + InheritedAnnotations []string `json:"inherited_annotations,omitempty"` DownscalerAnnotations []string `json:"downscaler_annotations,omitempty"` ClusterNameLabel string `json:"cluster_name_label,omitempty"` DeleteAnnotationDateKey string `json:"delete_annotation_date_key,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 51d9861e4..f04a29490 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -202,6 +202,11 @@ func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfigura *out = make([]string, len(*in)) copy(*out, *in) } + if in.InheritedAnnotations != nil { + in, out := &in.InheritedAnnotations, &out.InheritedAnnotations + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.DownscalerAnnotations != nil { in, out := &in.DownscalerAnnotations, &out.DownscalerAnnotations *out = make([]string, len(*in)) diff --git a/pkg/cluster/connection_pooler.go b/pkg/cluster/connection_pooler.go index 36c75bd91..82b855bf2 100644 --- a/pkg/cluster/connection_pooler.go +++ b/pkg/cluster/connection_pooler.go @@ -286,7 +286,7 @@ func (c *Cluster) generateConnectionPoolerPodTemplate(role PostgresRole) ( ObjectMeta: metav1.ObjectMeta{ Labels: c.connectionPoolerLabels(role, true).MatchLabels, Namespace: c.Namespace, - Annotations: c.generatePodAnnotations(spec), + Annotations: c.annotationsSet(c.generatePodAnnotations(spec)), }, Spec: v1.PodSpec{ ServiceAccountName: c.OpConfig.PodServiceAccountName, @@ -325,7 +325,7 @@ func (c *Cluster) generateConnectionPoolerDeployment(connectionPooler *Connectio if *numberOfInstances < constants.ConnectionPoolerMinInstances { msg := "Adjusted number of connection pooler instances from %d to %d" - c.logger.Warningf(msg, numberOfInstances, constants.ConnectionPoolerMinInstances) + c.logger.Warningf(msg, *numberOfInstances, constants.ConnectionPoolerMinInstances) *numberOfInstances = constants.ConnectionPoolerMinInstances } @@ -339,7 +339,7 @@ func (c *Cluster) generateConnectionPoolerDeployment(connectionPooler *Connectio Name: connectionPooler.Name, Namespace: connectionPooler.Namespace, Labels: c.connectionPoolerLabels(connectionPooler.Role, true).MatchLabels, - Annotations: map[string]string{}, + Annotations: c.AnnotationsToPropagate(c.annotationsSet(nil)), // make StatefulSet object its owner to represent the dependency. // By itself StatefulSet is being deleted with "Orphaned" // propagation policy, which means that it's deletion will not @@ -390,7 +390,7 @@ func (c *Cluster) generateConnectionPoolerService(connectionPooler *ConnectionPo Name: connectionPooler.Name, Namespace: connectionPooler.Namespace, Labels: c.connectionPoolerLabels(connectionPooler.Role, false).MatchLabels, - Annotations: map[string]string{}, + Annotations: c.annotationsSet(c.generateServiceAnnotations(connectionPooler.Role, spec)), // make StatefulSet object its owner to represent the dependency. // By itself StatefulSet is being deleted with "Orphaned" // propagation policy, which means that it's deletion will not @@ -866,7 +866,7 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql } } - newAnnotations := c.AnnotationsToPropagate(c.ConnectionPooler[role].Deployment.Annotations) + newAnnotations := c.AnnotationsToPropagate(c.annotationsSet(c.ConnectionPooler[role].Deployment.Annotations)) if newAnnotations != nil { deployment, err = updateConnectionPoolerAnnotations(c.KubeClient, c.ConnectionPooler[role].Deployment, newAnnotations) if err != nil { diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 28d711a33..602695e0e 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1184,13 +1184,13 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef tolerationSpec := tolerations(&spec.Tolerations, c.OpConfig.PodToleration) effectivePodPriorityClassName := util.Coalesce(spec.PodPriorityClassName, c.OpConfig.PodPriorityClassName) - annotations := c.generatePodAnnotations(spec) + podAnnotations := c.generatePodAnnotations(spec) // generate pod template for the statefulset, based on the spilo container and sidecars podTemplate, err = c.generatePodTemplate( c.Namespace, c.labelsSet(true), - annotations, + c.annotationsSet(podAnnotations), spiloContainer, initContainers, sidecarContainers, @@ -1236,15 +1236,16 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef return nil, fmt.Errorf("could not set the pod management policy to the unknown value: %v", c.OpConfig.PodManagementPolicy) } - annotations = make(map[string]string) - annotations[rollingUpdateStatefulsetAnnotationKey] = strconv.FormatBool(false) + stsAnnotations := make(map[string]string) + stsAnnotations[rollingUpdateStatefulsetAnnotationKey] = strconv.FormatBool(false) + stsAnnotations = c.AnnotationsToPropagate(c.annotationsSet(nil)) statefulSet := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: c.statefulSetName(), Namespace: c.Namespace, Labels: c.labelsSet(true), - Annotations: c.AnnotationsToPropagate(annotations), + Annotations: stsAnnotations, }, Spec: appsv1.StatefulSetSpec{ Replicas: &numberOfInstances, @@ -1537,9 +1538,10 @@ func (c *Cluster) generateSingleUserSecret(namespace string, pgUser spec.PgUser) username := pgUser.Name secret := v1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: c.credentialSecretName(username), - Namespace: namespace, - Labels: c.labelsSet(true), + Name: c.credentialSecretName(username), + Namespace: namespace, + Labels: c.labelsSet(true), + Annotations: c.annotationsSet(nil), }, Type: v1.SecretTypeOpaque, Data: map[string][]byte{ @@ -1613,7 +1615,7 @@ func (c *Cluster) generateService(role PostgresRole, spec *acidv1.PostgresSpec) Name: c.serviceName(role), Namespace: c.Namespace, Labels: c.roleLabelsSet(true, role), - Annotations: c.generateServiceAnnotations(role, spec), + Annotations: c.annotationsSet(c.generateServiceAnnotations(role, spec)), }, Spec: serviceSpec, } @@ -1816,9 +1818,10 @@ func (c *Cluster) generatePodDisruptionBudget() *policybeta1.PodDisruptionBudget return &policybeta1.PodDisruptionBudget{ ObjectMeta: metav1.ObjectMeta{ - Name: c.podDisruptionBudgetName(), - Namespace: c.Namespace, - Labels: c.labelsSet(true), + Name: c.podDisruptionBudgetName(), + Namespace: c.Namespace, + Labels: c.labelsSet(true), + Annotations: c.annotationsSet(nil), }, Spec: policybeta1.PodDisruptionBudgetSpec{ MinAvailable: &minAvailable, @@ -1938,9 +1941,10 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) { cronJob := &batchv1beta1.CronJob{ ObjectMeta: metav1.ObjectMeta{ - Name: c.getLogicalBackupJobName(), - Namespace: c.Namespace, - Labels: c.labelsSet(true), + Name: c.getLogicalBackupJobName(), + Namespace: c.Namespace, + Labels: c.labelsSet(true), + Annotations: c.annotationsSet(nil), }, Spec: batchv1beta1.CronJobSpec{ Schedule: schedule, diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 94736b531..2781144b2 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -368,8 +368,8 @@ func (c *Cluster) syncStatefulSet() error { } } } - annotations := c.AnnotationsToPropagate(c.Statefulset.Annotations) - c.updateStatefulSetAnnotations(annotations) + + c.updateStatefulSetAnnotations(c.AnnotationsToPropagate(c.annotationsSet(c.Statefulset.Annotations))) if !podsRollingUpdateRequired && !c.OpConfig.EnableLazySpiloUpgrade { // even if desired and actual statefulsets match @@ -412,11 +412,15 @@ func (c *Cluster) syncStatefulSet() error { // AnnotationsToPropagate get the annotations to update if required // based on the annotations in postgres CRD func (c *Cluster) AnnotationsToPropagate(annotations map[string]string) map[string]string { - toPropagateAnnotations := c.OpConfig.DownscalerAnnotations - pgCRDAnnotations := c.Postgresql.ObjectMeta.GetAnnotations() - if toPropagateAnnotations != nil && pgCRDAnnotations != nil { - for _, anno := range toPropagateAnnotations { + if annotations == nil { + annotations = make(map[string]string) + } + + pgCRDAnnotations := c.ObjectMeta.Annotations + + if pgCRDAnnotations != nil { + for _, anno := range c.OpConfig.DownscalerAnnotations { for k, v := range pgCRDAnnotations { matched, err := regexp.MatchString(anno, k) if err != nil { @@ -430,7 +434,11 @@ func (c *Cluster) AnnotationsToPropagate(annotations map[string]string) map[stri } } - return annotations + if len(annotations) > 0 { + return annotations + } + + return nil } // checkAndSetGlobalPostgreSQLConfiguration checks whether cluster-wide API parameters diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go index a2fdcb08e..d5e887656 100644 --- a/pkg/cluster/util.go +++ b/pkg/cluster/util.go @@ -271,6 +271,33 @@ func (c *Cluster) getTeamMembers(teamID string) ([]string, error) { return members, nil } +// Returns annotations to be passed to child objects +func (c *Cluster) annotationsSet(annotations map[string]string) map[string]string { + + if annotations == nil { + annotations = make(map[string]string) + } + + pgCRDAnnotations := c.ObjectMeta.Annotations + + // allow to inherit certain labels from the 'postgres' object + if pgCRDAnnotations != nil { + for k, v := range pgCRDAnnotations { + for _, match := range c.OpConfig.InheritedAnnotations { + if k == match { + annotations[k] = v + } + } + } + } + + if len(annotations) > 0 { + return annotations + } + + return nil +} + func (c *Cluster) waitForPodLabel(podEvents chan PodEvent, stopChan chan struct{}, role *PostgresRole) (*v1.Pod, error) { timeout := time.After(c.OpConfig.PodLabelWaitTimeout) for { diff --git a/pkg/cluster/util_test.go b/pkg/cluster/util_test.go new file mode 100644 index 000000000..7afc59f28 --- /dev/null +++ b/pkg/cluster/util_test.go @@ -0,0 +1,141 @@ +package cluster + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + fakeacidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/fake" + "github.com/zalando/postgres-operator/pkg/util" + "github.com/zalando/postgres-operator/pkg/util/config" + "github.com/zalando/postgres-operator/pkg/util/k8sutil" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sFake "k8s.io/client-go/kubernetes/fake" +) + +func newFakeK8sAnnotationsClient() (k8sutil.KubernetesClient, *k8sFake.Clientset) { + clientSet := k8sFake.NewSimpleClientset() + acidClientSet := fakeacidv1.NewSimpleClientset() + + return k8sutil.KubernetesClient{ + PodDisruptionBudgetsGetter: clientSet.PolicyV1beta1(), + ServicesGetter: clientSet.CoreV1(), + StatefulSetsGetter: clientSet.AppsV1(), + PostgresqlsGetter: acidClientSet.AcidV1(), + }, clientSet +} + +func TestInheritedAnnotations(t *testing.T) { + testName := "test inheriting annotations from manifest" + client, _ := newFakeK8sAnnotationsClient() + clusterName := "acid-test-cluster" + namespace := "default" + annotationValue := "acid" + role := Master + + pg := acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Annotations: map[string]string{ + "owned-by": annotationValue, + }, + }, + Spec: acidv1.PostgresSpec{ + EnableReplicaConnectionPooler: boolToPointer(true), + Volume: acidv1.Volume{ + Size: "1Gi", + }, + }, + } + + var cluster = New( + Config{ + OpConfig: config.Config{ + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", + NumberOfInstances: int32ToPointer(1), + }, + PodManagementPolicy: "ordered_ready", + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: "cluster-name", + DefaultCPURequest: "300m", + DefaultCPULimit: "300m", + DefaultMemoryRequest: "300Mi", + DefaultMemoryLimit: "300Mi", + InheritedAnnotations: []string{"owned-by"}, + PodRoleLabel: "spilo-role", + }, + }, + }, client, pg, logger, eventRecorder) + + cluster.Name = clusterName + cluster.Namespace = namespace + + // test annotationsSet function + inheritedAnnotations := cluster.annotationsSet(nil) + + listOptions := metav1.ListOptions{ + LabelSelector: cluster.labelsSet(false).String(), + } + + // check statefulset annotations + _, err := cluster.createStatefulSet() + assert.NoError(t, err) + + stsList, err := client.StatefulSets(namespace).List(context.TODO(), listOptions) + assert.NoError(t, err) + for _, sts := range stsList.Items { + if !(util.MapContains(sts.ObjectMeta.Annotations, inheritedAnnotations)) { + t.Errorf("%s: StatefulSet %v not inherited annotations %#v, got %#v", testName, sts.ObjectMeta.Name, inheritedAnnotations, sts.ObjectMeta.Annotations) + } + // pod template + if !(util.MapContains(sts.Spec.Template.ObjectMeta.Annotations, inheritedAnnotations)) { + t.Errorf("%s: pod template %v not inherited annotations %#v, got %#v", testName, sts.ObjectMeta.Name, inheritedAnnotations, sts.ObjectMeta.Annotations) + } + // pvc template + if util.MapContains(sts.Spec.VolumeClaimTemplates[0].Annotations, inheritedAnnotations) { + t.Errorf("%s: PVC template %v not expected to have inherited annotations %#v, got %#v", testName, sts.ObjectMeta.Name, inheritedAnnotations, sts.ObjectMeta.Annotations) + } + } + + // check service annotations + cluster.createService(Master) + svcList, err := client.Services(namespace).List(context.TODO(), listOptions) + assert.NoError(t, err) + for _, svc := range svcList.Items { + if !(util.MapContains(svc.ObjectMeta.Annotations, inheritedAnnotations)) { + t.Errorf("%s: Service %v not inherited annotations %#v, got %#v", testName, svc.ObjectMeta.Name, inheritedAnnotations, svc.ObjectMeta.Annotations) + } + } + + // check pod disruption budget annotations + cluster.createPodDisruptionBudget() + pdbList, err := client.PodDisruptionBudgets(namespace).List(context.TODO(), listOptions) + assert.NoError(t, err) + for _, pdb := range pdbList.Items { + if !(util.MapContains(pdb.ObjectMeta.Annotations, inheritedAnnotations)) { + t.Errorf("%s: Pod Disruption Budget %v not inherited annotations %#v, got %#v", testName, pdb.ObjectMeta.Name, inheritedAnnotations, pdb.ObjectMeta.Annotations) + } + } + + // check pooler deployment annotations + cluster.ConnectionPooler = map[PostgresRole]*ConnectionPoolerObjects{} + cluster.ConnectionPooler[role] = &ConnectionPoolerObjects{ + Name: cluster.connectionPoolerName(role), + ClusterName: cluster.ClusterName, + Namespace: cluster.Namespace, + Role: role, + } + deploy, err := cluster.generateConnectionPoolerDeployment(cluster.ConnectionPooler[role]) + assert.NoError(t, err) + + if !(util.MapContains(deploy.ObjectMeta.Annotations, inheritedAnnotations)) { + t.Errorf("%s: Deployment %v not inherited annotations %#v, got %#v", testName, deploy.ObjectMeta.Name, inheritedAnnotations, deploy.ObjectMeta.Annotations) + } + +} diff --git a/pkg/cluster/volumes_test.go b/pkg/cluster/volumes_test.go index 907b9959f..4288cdfc4 100644 --- a/pkg/cluster/volumes_test.go +++ b/pkg/cluster/volumes_test.go @@ -22,7 +22,7 @@ import ( "k8s.io/client-go/kubernetes/fake" ) -func NewFakeKubernetesClient() (k8sutil.KubernetesClient, *fake.Clientset) { +func newFakeK8sPVCclient() (k8sutil.KubernetesClient, *fake.Clientset) { clientSet := fake.NewSimpleClientset() return k8sutil.KubernetesClient{ @@ -34,7 +34,7 @@ func NewFakeKubernetesClient() (k8sutil.KubernetesClient, *fake.Clientset) { func TestResizeVolumeClaim(t *testing.T) { testName := "test resizing of persistent volume claims" - client, _ := NewFakeKubernetesClient() + client, _ := newFakeK8sPVCclient() clusterName := "acid-test-cluster" namespace := "default" newVolumeSize := "2Gi" diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 20fb0f0dc..f5b8b3b51 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -15,7 +15,7 @@ import ( func (c *Controller) readOperatorConfigurationFromCRD(configObjectNamespace, configObjectName string) (*acidv1.OperatorConfiguration, error) { - config, err := c.KubeClient.AcidV1ClientSet.AcidV1().OperatorConfigurations(configObjectNamespace).Get( + config, err := c.KubeClient.OperatorConfigurationsGetter.OperatorConfigurations(configObjectNamespace).Get( context.TODO(), configObjectName, metav1.GetOptions{}) if err != nil { return nil, fmt.Errorf("could not get operator configuration object %q: %v", configObjectName, err) @@ -93,6 +93,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.PodRoleLabel = util.Coalesce(fromCRD.Kubernetes.PodRoleLabel, "spilo-role") result.ClusterLabels = util.CoalesceStrMap(fromCRD.Kubernetes.ClusterLabels, map[string]string{"application": "spilo"}) result.InheritedLabels = fromCRD.Kubernetes.InheritedLabels + result.InheritedAnnotations = fromCRD.Kubernetes.InheritedAnnotations result.DownscalerAnnotations = fromCRD.Kubernetes.DownscalerAnnotations result.ClusterNameLabel = util.Coalesce(fromCRD.Kubernetes.ClusterNameLabel, "cluster-name") result.DeleteAnnotationDateKey = fromCRD.Kubernetes.DeleteAnnotationDateKey diff --git a/pkg/controller/postgresql.go b/pkg/controller/postgresql.go index 4b5d68fe5..0fe0c1120 100644 --- a/pkg/controller/postgresql.go +++ b/pkg/controller/postgresql.go @@ -46,7 +46,7 @@ func (c *Controller) listClusters(options metav1.ListOptions) (*acidv1.Postgresq var pgList acidv1.PostgresqlList // TODO: use the SharedInformer cache instead of quering Kubernetes API directly. - list, err := c.KubeClient.AcidV1ClientSet.AcidV1().Postgresqls(c.opConfig.WatchedNamespace).List(context.TODO(), options) + list, err := c.KubeClient.PostgresqlsGetter.Postgresqls(c.opConfig.WatchedNamespace).List(context.TODO(), options) if err != nil { c.logger.Errorf("could not list postgresql objects: %v", err) } diff --git a/pkg/controller/util.go b/pkg/controller/util.go index 7f87de97d..815bc7b74 100644 --- a/pkg/controller/util.go +++ b/pkg/controller/util.go @@ -398,7 +398,7 @@ func (c *Controller) loadPostgresTeams() { // reset team map c.pgTeamMap = teams.PostgresTeamMap{} - pgTeams, err := c.KubeClient.AcidV1ClientSet.AcidV1().PostgresTeams(c.opConfig.WatchedNamespace).List(context.TODO(), metav1.ListOptions{}) + pgTeams, err := c.KubeClient.PostgresTeamsGetter.PostgresTeams(c.opConfig.WatchedNamespace).List(context.TODO(), metav1.ListOptions{}) if err != nil { c.logger.Errorf("could not list postgres team objects: %v", err) } diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 122c192a5..0b2941683 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -36,6 +36,7 @@ type Resources struct { SpiloPrivileged bool `name:"spilo_privileged" default:"false"` ClusterLabels map[string]string `name:"cluster_labels" default:"application:spilo"` InheritedLabels []string `name:"inherited_labels" default:""` + InheritedAnnotations []string `name:"inherited_annotations" default:""` DownscalerAnnotations []string `name:"downscaler_annotations"` ClusterNameLabel string `name:"cluster_name_label" default:"cluster-name"` DeleteAnnotationDateKey string `name:"delete_annotation_date_key"` diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index 19f95d9f1..a23c1f842 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -11,7 +11,9 @@ import ( batchv1beta1 "k8s.io/api/batch/v1beta1" clientbatchv1beta1 "k8s.io/client-go/kubernetes/typed/batch/v1beta1" - acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + apiacidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + acidv1client "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned" + acidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/spec" apiappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -19,6 +21,7 @@ import ( apiextclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apiextv1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" appsv1 "k8s.io/client-go/kubernetes/typed/apps/v1" @@ -27,9 +30,6 @@ import ( rbacv1 "k8s.io/client-go/kubernetes/typed/rbac/v1" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" - - acidv1client "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func Int32ToPointer(value int32) *int32 { @@ -55,6 +55,9 @@ type KubernetesClient struct { policyv1beta1.PodDisruptionBudgetsGetter apiextv1.CustomResourceDefinitionsGetter clientbatchv1beta1.CronJobsGetter + acidv1.OperatorConfigurationsGetter + acidv1.PostgresTeamsGetter + acidv1.PostgresqlsGetter RESTClient rest.Interface AcidV1ClientSet *acidv1client.Clientset @@ -154,15 +157,23 @@ func NewFromConfig(cfg *rest.Config) (KubernetesClient, error) { } kubeClient.CustomResourceDefinitionsGetter = apiextClient.ApiextensionsV1() + kubeClient.AcidV1ClientSet = acidv1client.NewForConfigOrDie(cfg) + if err != nil { + return kubeClient, fmt.Errorf("could not create acid.zalan.do clientset: %v", err) + } + + kubeClient.OperatorConfigurationsGetter = kubeClient.AcidV1ClientSet.AcidV1() + kubeClient.PostgresTeamsGetter = kubeClient.AcidV1ClientSet.AcidV1() + kubeClient.PostgresqlsGetter = kubeClient.AcidV1ClientSet.AcidV1() return kubeClient, nil } // SetPostgresCRDStatus of Postgres cluster -func (client *KubernetesClient) SetPostgresCRDStatus(clusterName spec.NamespacedName, status string) (*acidv1.Postgresql, error) { - var pg *acidv1.Postgresql - var pgStatus acidv1.PostgresStatus +func (client *KubernetesClient) SetPostgresCRDStatus(clusterName spec.NamespacedName, status string) (*apiacidv1.Postgresql, error) { + var pg *apiacidv1.Postgresql + var pgStatus apiacidv1.PostgresStatus pgStatus.PostgresClusterStatus = status patch, err := json.Marshal(struct { @@ -176,7 +187,7 @@ func (client *KubernetesClient) SetPostgresCRDStatus(clusterName spec.Namespaced // we cannot do a full scale update here without fetching the previous manifest (as the resourceVersion may differ), // however, we could do patch without it. In the future, once /status subresource is there (starting Kubernetes 1.11) // we should take advantage of it. - pg, err = client.AcidV1ClientSet.AcidV1().Postgresqls(clusterName.Namespace).Patch( + pg, err = client.PostgresqlsGetter.Postgresqls(clusterName.Namespace).Patch( context.TODO(), clusterName.Name, types.MergePatchType, patch, metav1.PatchOptions{}, "status") if err != nil { return pg, fmt.Errorf("could not update status: %v", err) From b88d8e34e188f75cd115ba9e9dc891401ac414b3 Mon Sep 17 00:00:00 2001 From: Jan Mussler Date: Sat, 12 Dec 2020 00:35:27 +0100 Subject: [PATCH 133/168] Fix function name in test (#1250) * Fix function name in test Error was somehow introduced in last 2 PRs merged. * Update volumes_test.go --- pkg/cluster/volumes_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cluster/volumes_test.go b/pkg/cluster/volumes_test.go index 4288cdfc4..17fb8a4af 100644 --- a/pkg/cluster/volumes_test.go +++ b/pkg/cluster/volumes_test.go @@ -163,7 +163,7 @@ func CreatePVCs(namespace string, clusterName string, labels labels.Set, n int, } func TestMigrateEBS(t *testing.T) { - client, _ := NewFakeKubernetesClient() + client, _ := newFakeK8sPVCclient() clusterName := "acid-test-cluster" namespace := "default" From 028f23eec748c9670fb37b0b0ea0c019af9ad698 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 14 Dec 2020 12:37:09 +0100 Subject: [PATCH 134/168] raise pooler image and fix pgversion config in chart (#1253) * raise pooler image and fix pgversion config in chart * enable_ebs_gp3_migration_max_size with quotes * set ConnectionPoolerMinInstances to 1 --- charts/postgres-operator/crds/operatorconfigurations.yaml | 2 ++ charts/postgres-operator/values-crd.yaml | 4 ++-- charts/postgres-operator/values.yaml | 2 +- manifests/configmap.yaml | 5 +++-- manifests/minimal-fake-pooler-deployment.yaml | 2 +- manifests/postgresql-operator-default-configuration.yaml | 1 + pkg/util/constants/pooler.go | 2 +- 7 files changed, 11 insertions(+), 7 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index a19d3cfd5..c2eab0cdc 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -69,6 +69,8 @@ spec: type: boolean enable_lazy_spilo_upgrade: type: boolean + enable_pgversion_env_var: + type: boolean enable_shm_volume: type: boolean etcd_host: diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index e553c9d6f..87a9820ae 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -22,7 +22,7 @@ configGeneral: # update only the statefulsets without immediately doing the rolling update enable_lazy_spilo_upgrade: false # set the PGVERSION env var instead of providing the version via postgresql.bin_dir in SPILO_CONFIGURATION - enable_pgversion_env_var: "false" + enable_pgversion_env_var: false # start any new database pod without limitations on shm memory enable_shm_volume: true # etcd connection string for Patroni. Empty uses K8s-native DCS. @@ -270,7 +270,7 @@ configTeamsApi: # operator watches for PostgresTeam CRs to assign additional teams and members to clusters enable_postgres_team_crd: false # toogle to create additional superuser teams from PostgresTeam CRs - # enable_postgres_team_crd_superusers: "false" + # enable_postgres_team_crd_superusers: false # toggle to grant superuser to team members created from the Teams API enable_team_superuser: false diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 2f68c9d4b..e1f9ddd98 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -217,7 +217,7 @@ configAwsOrGcp: # enable automatic migration on AWS from gp2 to gp3 volumes enable_ebs_gp3_migration: "false" # defines maximum volume size in GB until which auto migration happens - # enable_ebs_gp3_migration_max_size: 1000 + # enable_ebs_gp3_migration_max_size: "1000" # GCP credentials for setting the GOOGLE_APPLICATION_CREDNETIALS environment variable # gcp_credentials: "" diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 111701829..6fcd85862 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -15,7 +15,7 @@ data: # connection_pooler_default_cpu_request: "500m" # connection_pooler_default_memory_limit: 100Mi # connection_pooler_default_memory_request: 100Mi - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-11" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-12" # connection_pooler_max_db_connections: 60 # connection_pooler_mode: "transaction" # connection_pooler_number_of_instances: 2 @@ -37,10 +37,11 @@ data: # enable_crd_validation: "true" # enable_database_access: "true" enable_ebs_gp3_migration: "false" - # enable_ebs_gp3_migration_max_size: 1000 + # enable_ebs_gp3_migration_max_size: "1000" # enable_init_containers: "true" # enable_lazy_spilo_upgrade: "false" enable_master_load_balancer: "false" + # enable_pgversion_env_var: "false" # enable_pod_antiaffinity: "false" # enable_pod_disruption_budget: "true" # enable_postgres_team_crd: "false" diff --git a/manifests/minimal-fake-pooler-deployment.yaml b/manifests/minimal-fake-pooler-deployment.yaml index 0406b195a..5ee8cf05f 100644 --- a/manifests/minimal-fake-pooler-deployment.yaml +++ b/manifests/minimal-fake-pooler-deployment.yaml @@ -23,7 +23,7 @@ spec: serviceAccountName: postgres-operator containers: - name: postgres-operator - image: registry.opensource.zalan.do/acid/pgbouncer:master-11 + image: registry.opensource.zalan.do/acid/pgbouncer:master-12 imagePullPolicy: IfNotPresent resources: requests: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 00b095c1b..36705ce43 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -6,6 +6,7 @@ configuration: docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p3 # enable_crd_validation: true # enable_lazy_spilo_upgrade: false + # enable_pgversion_env_var: false # enable_shm_volume: true etcd_host: "" # kubernetes_use_configmaps: false diff --git a/pkg/util/constants/pooler.go b/pkg/util/constants/pooler.go index 52e47c9cd..ded795bbe 100644 --- a/pkg/util/constants/pooler.go +++ b/pkg/util/constants/pooler.go @@ -14,5 +14,5 @@ const ( ConnectionPoolerContainer = 0 ConnectionPoolerMaxDBConnections = 60 ConnectionPoolerMaxClientConnections = 10000 - ConnectionPoolerMinInstances = 2 + ConnectionPoolerMinInstances = 1 ) From 83fbccac5a0eca90d78c43cf9d37ef74eede5ca6 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 14 Dec 2020 18:43:53 +0100 Subject: [PATCH 135/168] new env var for backwards compatability between spilo 12 and 13 (#1254) --- .../crds/operatorconfigurations.yaml | 2 ++ charts/postgres-operator/values-crd.yaml | 2 ++ charts/postgres-operator/values.yaml | 2 ++ docs/reference/operator_parameters.md | 9 ++++++--- manifests/configmap.yaml | 1 + manifests/operatorconfiguration.crd.yaml | 2 ++ ...postgresql-operator-default-configuration.yaml | 1 + pkg/apis/acid.zalan.do/v1/crds.go | 3 +++ .../v1/operator_configuration_type.go | 1 + pkg/cluster/k8sres.go | 15 +++++++++++++-- pkg/controller/operator_config.go | 1 + pkg/util/config/config.go | 1 + 12 files changed, 35 insertions(+), 5 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index c2eab0cdc..2cbc2cfce 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -73,6 +73,8 @@ spec: type: boolean enable_shm_volume: type: boolean + enable_spilo_wal_path_compat: + type: boolean etcd_host: type: string kubernetes_use_configmaps: diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 87a9820ae..fc8e36ec9 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -25,6 +25,8 @@ configGeneral: enable_pgversion_env_var: false # start any new database pod without limitations on shm memory enable_shm_volume: true + # enables backwards compatible path between Spilo 12 and Spilo 13 images + enable_spilo_wal_path_compat: false # etcd connection string for Patroni. Empty uses K8s-native DCS. etcd_host: "" # Select if setup uses endpoints (default), or configmaps to manage leader (DCS=k8s) diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index e1f9ddd98..65340d1cd 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -28,6 +28,8 @@ configGeneral: enable_pgversion_env_var: "false" # start any new database pod without limitations on shm memory enable_shm_volume: "true" + # enables backwards compatible path between Spilo 12 and Spilo 13 images + enable_spilo_wal_path_compat: "false" # etcd connection string for Patroni. Empty uses K8s-native DCS. etcd_host: "" # Select if setup uses endpoints (default), or configmaps to manage leader (DCS=k8s) diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 87841d1d5..09158e536 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -79,6 +79,12 @@ Those are top-level keys, containing both leaf keys and groups. Instruct operator to update only the statefulsets with new images (Spilo and InitContainers) without immediately doing the rolling update. The assumption is pods will be re-started later with new images, for example due to the node rotation. The default is `false`. +* **enable_pgversion_env_var** + With newer versions of Spilo, it is preferable to use `PGVERSION` pod environment variable instead of the setting `postgresql.bin_dir` in the `SPILO_CONFIGURATION` env variable. When this option is true, the operator sets `PGVERSION` and omits `postgresql.bin_dir` from `SPILO_CONFIGURATION`. When false, the `postgresql.bin_dir` is set. This setting takes precedence over `PGVERSION`; see PR 222 in Spilo. The default is `false`. + +* **enable_spilo_wal_path_compat** + enables backwards compatible path between Spilo 12 and Spilo 13 images. The default is `false`. + * **etcd_host** Etcd connection string for Patroni defined as `host:port`. Not required when Patroni native Kubernetes support is used. The default is empty (use @@ -118,9 +124,6 @@ Those are top-level keys, containing both leaf keys and groups. This option is global for an operator object, and can be overwritten by `enableShmVolume` parameter from Postgres manifest. The default is `true`. -* **enable_pgversion_env_var** - With newer versions of Spilo, it is preferable to use `PGVERSION` pod environment variable instead of the setting `postgresql.bin_dir` in the `SPILO_CONFIGURATION` env variable. When this option is true, the operator sets `PGVERSION` and omits `postgresql.bin_dir` from `SPILO_CONFIGURATION`. When false, the `postgresql.bin_dir` is set. This setting takes precedence over `PGVERSION`; see PR 222 in Spilo. The default is `false`. - * **workers** number of working routines the operator spawns to process requests to create/update/delete/sync clusters concurrently. The default is `4`. diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 6fcd85862..293404e99 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -50,6 +50,7 @@ data: # enable_shm_volume: "true" # enable_pgversion_env_var: "false" # enable_sidecars: "true" + enable_spilo_wal_path_compat: "false" # enable_team_superuser: "false" enable_teams_api: "false" # etcd_host: "" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 8fbc6f042..5ac955913 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -69,6 +69,8 @@ spec: type: boolean enable_shm_volume: type: boolean + enable_spilo_wal_path_compat: + type: boolean etcd_host: type: string kubernetes_use_configmaps: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 36705ce43..7ab6c35a9 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -8,6 +8,7 @@ configuration: # enable_lazy_spilo_upgrade: false # enable_pgversion_env_var: false # enable_shm_volume: true + enable_spilo_wal_path_compat: false etcd_host: "" # kubernetes_use_configmaps: false max_instances: -1 diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 938abf7fc..cc79a3efe 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -815,6 +815,9 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{ "enable_shm_volume": { Type: "boolean", }, + "enable_spilo_wal_path_compat": { + Type: "boolean", + }, "etcd_host": { Type: "string", }, diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index e79405224..9578f8138 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -201,6 +201,7 @@ type OperatorConfigurationData struct { EnableCRDValidation *bool `json:"enable_crd_validation,omitempty"` EnableLazySpiloUpgrade bool `json:"enable_lazy_spilo_upgrade,omitempty"` EnablePgVersionEnvVar bool `json:"enable_pgversion_env_var,omitempty"` + EnableSpiloWalPathCompat bool `json:"enable_spilo_wal_path_compat,omitempty"` EtcdHost string `json:"etcd_host,omitempty"` KubernetesUseConfigMaps bool `json:"kubernetes_use_configmaps,omitempty"` DockerImage string `json:"docker_image,omitempty"` diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 602695e0e..5360f2668 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -824,7 +824,7 @@ func (c *Cluster) getPodEnvironmentConfigMapVariables() ([]v1.EnvVar, error) { return configMapPodEnvVarsList, nil } -// Return list of variables the pod recieved from the configured Secret +// Return list of variables the pod received from the configured Secret func (c *Cluster) getPodEnvironmentSecretVariables() ([]v1.EnvVar, error) { secretPodEnvVarsList := make([]v1.EnvVar, 0) @@ -979,6 +979,16 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef initContainers = spec.InitContainers } + spiloCompathWalPathList := make([]v1.EnvVar, 0) + if c.OpConfig.EnableSpiloWalPathCompat { + spiloCompathWalPathList = append(spiloCompathWalPathList, + v1.EnvVar{ + Name: "ENABLE_WAL_PATH_COMPAT", + Value: "true", + }, + ) + } + // fetch env vars from custom ConfigMap configMapEnvVarsList, err := c.getPodEnvironmentConfigMapVariables() if err != nil { @@ -992,7 +1002,8 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef } // concat all custom pod env vars and sort them - customPodEnvVarsList := append(configMapEnvVarsList, secretEnvVarsList...) + customPodEnvVarsList := append(spiloCompathWalPathList, configMapEnvVarsList...) + customPodEnvVarsList = append(customPodEnvVarsList, secretEnvVarsList...) sort.Slice(customPodEnvVarsList, func(i, j int) bool { return customPodEnvVarsList[i].Name < customPodEnvVarsList[j].Name }) diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index f5b8b3b51..9b0c6f194 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -36,6 +36,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.EnableCRDValidation = util.CoalesceBool(fromCRD.EnableCRDValidation, util.True()) result.EnableLazySpiloUpgrade = fromCRD.EnableLazySpiloUpgrade result.EnablePgVersionEnvVar = fromCRD.EnablePgVersionEnvVar + result.EnableSpiloWalPathCompat = fromCRD.EnableSpiloWalPathCompat result.EtcdHost = fromCRD.EtcdHost result.KubernetesUseConfigMaps = fromCRD.KubernetesUseConfigMaps result.DockerImage = util.Coalesce(fromCRD.DockerImage, "registry.opensource.zalan.do/acid/spilo-12:1.6-p3") diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 0b2941683..5e292dedd 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -201,6 +201,7 @@ type Config struct { SetMemoryRequestToLimit bool `name:"set_memory_request_to_limit" default:"false"` EnableLazySpiloUpgrade bool `name:"enable_lazy_spilo_upgrade" default:"false"` EnablePgVersionEnvVar bool `name:"enable_pgversion_env_var" default:"false"` + EnableSpiloWalPathCompat bool `name:"enable_spilo_wal_path_compat" default:"false"` } // MustMarshal marshals the config or panics From 929075814a69000da187949896be8c95a4e1f6e9 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 15 Dec 2020 10:06:53 +0100 Subject: [PATCH 136/168] diff SecurityContext of containers (#1255) * diff SecurityContext of containers * change log messages to use "does not" vs "doesn't" --- kubectl-pg/cmd/util.go | 15 ++++++++------- pkg/cluster/cluster.go | 18 ++++++++++-------- pkg/cluster/connection_pooler_test.go | 8 ++++---- pkg/cluster/sync.go | 2 +- pkg/controller/node_test.go | 2 +- pkg/util/k8sutil/k8sutil.go | 12 ++++++------ pkg/util/k8sutil/k8sutil_test.go | 24 ++++++++++++------------ 7 files changed, 42 insertions(+), 39 deletions(-) diff --git a/kubectl-pg/cmd/util.go b/kubectl-pg/cmd/util.go index 329f9a28f..a2a5c2073 100644 --- a/kubectl-pg/cmd/util.go +++ b/kubectl-pg/cmd/util.go @@ -25,6 +25,13 @@ package cmd import ( "flag" "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + PostgresqlLister "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1" v1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -32,12 +39,6 @@ import ( restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/util/homedir" - "log" - "os" - "os/exec" - "path/filepath" - "strconv" - "strings" ) const ( @@ -88,7 +89,7 @@ func confirmAction(clusterName string, namespace string) { } clusterDetails := strings.Split(confirmClusterDetails, "/") if clusterDetails[0] != namespace || clusterDetails[1] != clusterName { - fmt.Printf("cluster name or namespace doesn't match. Please re-enter %s/%s\nHint: Press (ctrl+c) to exit\n", namespace, clusterName) + fmt.Printf("cluster name or namespace does not match. Please re-enter %s/%s\nHint: Press (ctrl+c) to exit\n", namespace, clusterName) } else { return } diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 00c75f801..d374a7732 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -248,7 +248,7 @@ func (c *Cluster) Create() error { } if role == Master { // replica endpoint will be created by the replica service. Master endpoint needs to be created by us, - // since the corresponding master service doesn't define any selectors. + // since the corresponding master service does not define any selectors. ep, err = c.createEndpoint(role) if err != nil { return fmt.Errorf("could not create %s endpoint: %v", role, err) @@ -412,7 +412,7 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa match = false needsReplace = true needsRollUpdate = true - reasons = append(reasons, "new statefulset's pod template metadata annotations doesn't match the current one") + reasons = append(reasons, "new statefulset's pod template metadata annotations does not match the current one") } if !reflect.DeepEqual(c.Statefulset.Spec.Template.Spec.SecurityContext, statefulSet.Spec.Template.Spec.SecurityContext) { match = false @@ -488,20 +488,22 @@ func (c *Cluster) compareContainers(description string, setA, setB []v1.Containe } checks := []containerCheck{ - newCheck("new statefulset %s's %s (index %d) name doesn't match the current one", + newCheck("new statefulset %s's %s (index %d) name does not match the current one", func(a, b v1.Container) bool { return a.Name != b.Name }), - newCheck("new statefulset %s's %s (index %d) ports don't match the current one", + newCheck("new statefulset %s's %s (index %d) ports do not match the current one", func(a, b v1.Container) bool { return !reflect.DeepEqual(a.Ports, b.Ports) }), - newCheck("new statefulset %s's %s (index %d) resources don't match the current ones", + newCheck("new statefulset %s's %s (index %d) resources do not match the current ones", func(a, b v1.Container) bool { return !compareResources(&a.Resources, &b.Resources) }), - newCheck("new statefulset %s's %s (index %d) environment doesn't match the current one", + newCheck("new statefulset %s's %s (index %d) environment does not match the current one", func(a, b v1.Container) bool { return !reflect.DeepEqual(a.Env, b.Env) }), - newCheck("new statefulset %s's %s (index %d) environment sources don't match the current one", + newCheck("new statefulset %s's %s (index %d) environment sources do not match the current one", func(a, b v1.Container) bool { return !reflect.DeepEqual(a.EnvFrom, b.EnvFrom) }), + newCheck("new statefulset %s's %s (index %d) security context does not match the current one", + func(a, b v1.Container) bool { return !reflect.DeepEqual(a.SecurityContext, b.SecurityContext) }), } if !c.OpConfig.EnableLazySpiloUpgrade { - checks = append(checks, newCheck("new statefulset %s's %s (index %d) image doesn't match the current one", + checks = append(checks, newCheck("new statefulset %s's %s (index %d) image does not match the current one", func(a, b v1.Container) bool { return a.Image != b.Image })) } diff --git a/pkg/cluster/connection_pooler_test.go b/pkg/cluster/connection_pooler_test.go index 2528460f5..39e0ba9ba 100644 --- a/pkg/cluster/connection_pooler_test.go +++ b/pkg/cluster/connection_pooler_test.go @@ -810,25 +810,25 @@ func TestConnectionPoolerDeploymentSpec(t *testing.T) { func testResources(cluster *Cluster, podSpec *v1.PodTemplateSpec, role PostgresRole) error { cpuReq := podSpec.Spec.Containers[0].Resources.Requests["cpu"] if cpuReq.String() != cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultCPURequest { - return fmt.Errorf("CPU request doesn't match, got %s, expected %s", + return fmt.Errorf("CPU request does not match, got %s, expected %s", cpuReq.String(), cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultCPURequest) } memReq := podSpec.Spec.Containers[0].Resources.Requests["memory"] if memReq.String() != cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultMemoryRequest { - return fmt.Errorf("Memory request doesn't match, got %s, expected %s", + return fmt.Errorf("Memory request does not match, got %s, expected %s", memReq.String(), cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultMemoryRequest) } cpuLim := podSpec.Spec.Containers[0].Resources.Limits["cpu"] if cpuLim.String() != cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultCPULimit { - return fmt.Errorf("CPU limit doesn't match, got %s, expected %s", + return fmt.Errorf("CPU limit does not match, got %s, expected %s", cpuLim.String(), cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultCPULimit) } memLim := podSpec.Spec.Containers[0].Resources.Limits["memory"] if memLim.String() != cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultMemoryLimit { - return fmt.Errorf("Memory limit doesn't match, got %s, expected %s", + return fmt.Errorf("Memory limit does not match, got %s, expected %s", memLim.String(), cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultMemoryLimit) } diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 2781144b2..dc54ae8ee 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -599,7 +599,7 @@ func (c *Cluster) syncVolumeClaims() error { return fmt.Errorf("could not compare size of the volume claims: %v", err) } if !act { - c.logger.Infof("volume claims don't require changes") + c.logger.Infof("volume claims do not require changes") return nil } if err := c.resizeVolumeClaims(c.Spec.Volume); err != nil { diff --git a/pkg/controller/node_test.go b/pkg/controller/node_test.go index 919f30f39..a9616e256 100644 --- a/pkg/controller/node_test.go +++ b/pkg/controller/node_test.go @@ -88,7 +88,7 @@ func TestNodeIsReady(t *testing.T) { for _, tt := range testTable { nodeTestController.opConfig.NodeReadinessLabel = tt.readinessLabel if isReady := nodeTestController.nodeIsReady(tt.in); isReady != tt.out { - t.Errorf("%s: expected response %t doesn't match the actual %t for the node %#v", + t.Errorf("%s: expected response %t does not match the actual %t for the node %#v", testName, tt.out, isReady, tt.in) } } diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index a23c1f842..dd6ec1e8b 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -201,7 +201,7 @@ func (client *KubernetesClient) SetPostgresCRDStatus(clusterName spec.Namespaced func SameService(cur, new *v1.Service) (match bool, reason string) { //TODO: improve comparison if cur.Spec.Type != new.Spec.Type { - return false, fmt.Sprintf("new service's type %q doesn't match the current one %q", + return false, fmt.Sprintf("new service's type %q does not match the current one %q", new.Spec.Type, cur.Spec.Type) } @@ -211,13 +211,13 @@ func SameService(cur, new *v1.Service) (match bool, reason string) { /* work around Kubernetes 1.6 serializing [] as nil. See https://github.com/kubernetes/kubernetes/issues/43203 */ if (len(oldSourceRanges) != 0) || (len(newSourceRanges) != 0) { if !reflect.DeepEqual(oldSourceRanges, newSourceRanges) { - return false, "new service's LoadBalancerSourceRange doesn't match the current one" + return false, "new service's LoadBalancerSourceRange does not match the current one" } } match = true - reasonPrefix := "new service's annotations doesn't match the current one:" + reasonPrefix := "new service's annotations does not match the current one:" for ann := range cur.Annotations { if _, ok := new.Annotations[ann]; !ok { match = false @@ -253,7 +253,7 @@ func SamePDB(cur, new *policybeta1.PodDisruptionBudget) (match bool, reason stri //TODO: improve comparison match = reflect.DeepEqual(new.Spec, cur.Spec) if !match { - reason = "new PDB spec doesn't match the current one" + reason = "new PDB spec does not match the current one" } return @@ -267,14 +267,14 @@ func getJobImage(cronJob *batchv1beta1.CronJob) string { func SameLogicalBackupJob(cur, new *batchv1beta1.CronJob) (match bool, reason string) { if cur.Spec.Schedule != new.Spec.Schedule { - return false, fmt.Sprintf("new job's schedule %q doesn't match the current one %q", + return false, fmt.Sprintf("new job's schedule %q does not match the current one %q", new.Spec.Schedule, cur.Spec.Schedule) } newImage := getJobImage(new) curImage := getJobImage(cur) if newImage != curImage { - return false, fmt.Sprintf("new job's image %q doesn't match the current one %q", + return false, fmt.Sprintf("new job's image %q does not match the current one %q", newImage, curImage) } diff --git a/pkg/util/k8sutil/k8sutil_test.go b/pkg/util/k8sutil/k8sutil_test.go index 9b4f2eac3..b3e768501 100644 --- a/pkg/util/k8sutil/k8sutil_test.go +++ b/pkg/util/k8sutil/k8sutil_test.go @@ -63,7 +63,7 @@ func TestSameService(t *testing.T) { v1.ServiceTypeLoadBalancer, []string{"128.141.0.0/16", "137.138.0.0/16"}), match: false, - reason: `new service's type "LoadBalancer" doesn't match the current one "ClusterIP"`, + reason: `new service's type "LoadBalancer" does not match the current one "ClusterIP"`, }, { about: "services differ on lb source ranges", @@ -82,7 +82,7 @@ func TestSameService(t *testing.T) { v1.ServiceTypeLoadBalancer, []string{"185.249.56.0/22"}), match: false, - reason: `new service's LoadBalancerSourceRange doesn't match the current one`, + reason: `new service's LoadBalancerSourceRange does not match the current one`, }, { about: "new service doesn't have lb source ranges", @@ -101,7 +101,7 @@ func TestSameService(t *testing.T) { v1.ServiceTypeLoadBalancer, []string{}), match: false, - reason: `new service's LoadBalancerSourceRange doesn't match the current one`, + reason: `new service's LoadBalancerSourceRange does not match the current one`, }, { about: "services differ on DNS annotation", @@ -120,7 +120,7 @@ func TestSameService(t *testing.T) { v1.ServiceTypeLoadBalancer, []string{"128.141.0.0/16", "137.138.0.0/16"}), match: false, - reason: `new service's annotations doesn't match the current one: 'external-dns.alpha.kubernetes.io/hostname' changed from 'clstr.acid.zalan.do' to 'new_clstr.acid.zalan.do'.`, + reason: `new service's annotations does not match the current one: 'external-dns.alpha.kubernetes.io/hostname' changed from 'clstr.acid.zalan.do' to 'new_clstr.acid.zalan.do'.`, }, { about: "services differ on AWS ELB annotation", @@ -139,7 +139,7 @@ func TestSameService(t *testing.T) { v1.ServiceTypeLoadBalancer, []string{"128.141.0.0/16", "137.138.0.0/16"}), match: false, - reason: `new service's annotations doesn't match the current one: 'service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout' changed from '3600' to '1800'.`, + reason: `new service's annotations does not match the current one: 'service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout' changed from '3600' to '1800'.`, }, { about: "service changes existing annotation", @@ -160,7 +160,7 @@ func TestSameService(t *testing.T) { v1.ServiceTypeLoadBalancer, []string{"128.141.0.0/16", "137.138.0.0/16"}), match: false, - reason: `new service's annotations doesn't match the current one: 'foo' changed from 'bar' to 'baz'.`, + reason: `new service's annotations does not match the current one: 'foo' changed from 'bar' to 'baz'.`, }, { about: "service changes multiple existing annotations", @@ -184,7 +184,7 @@ func TestSameService(t *testing.T) { []string{"128.141.0.0/16", "137.138.0.0/16"}), match: false, // Test just the prefix to avoid flakiness and map sorting - reason: `new service's annotations doesn't match the current one:`, + reason: `new service's annotations does not match the current one:`, }, { about: "service adds a new custom annotation", @@ -204,7 +204,7 @@ func TestSameService(t *testing.T) { v1.ServiceTypeLoadBalancer, []string{"128.141.0.0/16", "137.138.0.0/16"}), match: false, - reason: `new service's annotations doesn't match the current one: Added 'foo' with value 'bar'.`, + reason: `new service's annotations does not match the current one: Added 'foo' with value 'bar'.`, }, { about: "service removes a custom annotation", @@ -224,7 +224,7 @@ func TestSameService(t *testing.T) { v1.ServiceTypeLoadBalancer, []string{"128.141.0.0/16", "137.138.0.0/16"}), match: false, - reason: `new service's annotations doesn't match the current one: Removed 'foo'.`, + reason: `new service's annotations does not match the current one: Removed 'foo'.`, }, { about: "service removes a custom annotation and adds a new one", @@ -245,7 +245,7 @@ func TestSameService(t *testing.T) { v1.ServiceTypeLoadBalancer, []string{"128.141.0.0/16", "137.138.0.0/16"}), match: false, - reason: `new service's annotations doesn't match the current one: Removed 'foo'. Added 'bar' with value 'foo'.`, + reason: `new service's annotations does not match the current one: Removed 'foo'. Added 'bar' with value 'foo'.`, }, { about: "service removes a custom annotation, adds a new one and change another", @@ -269,7 +269,7 @@ func TestSameService(t *testing.T) { []string{"128.141.0.0/16", "137.138.0.0/16"}), match: false, // Test just the prefix to avoid flakiness and map sorting - reason: `new service's annotations doesn't match the current one: Removed 'foo'.`, + reason: `new service's annotations does not match the current one: Removed 'foo'.`, }, { about: "service add annotations", @@ -286,7 +286,7 @@ func TestSameService(t *testing.T) { []string{"128.141.0.0/16", "137.138.0.0/16"}), match: false, // Test just the prefix to avoid flakiness and map sorting - reason: `new service's annotations doesn't match the current one: Added `, + reason: `new service's annotations does not match the current one: Added `, }, } for _, tt := range tests { From fbd04896c2416ea1d0ad7010e3b1bc47a75472a9 Mon Sep 17 00:00:00 2001 From: Pavel Tumik Date: Wed, 16 Dec 2020 01:41:08 -0800 Subject: [PATCH 137/168] Add ability to upload logical backup to gcs (#1173) Support logical backup provider/storage S3 and GCS equivalent --- docker/logical-backup/Dockerfile | 3 +++ docker/logical-backup/dump.sh | 19 ++++++++++++++++++- docs/reference/operator_parameters.md | 7 +++++++ .../v1/operator_configuration_type.go | 18 ++++++++++-------- pkg/cluster/k8sres.go | 8 ++++++++ pkg/controller/operator_config.go | 2 ++ pkg/util/config/config.go | 18 ++++++++++-------- 7 files changed, 58 insertions(+), 17 deletions(-) diff --git a/docker/logical-backup/Dockerfile b/docker/logical-backup/Dockerfile index 94c524381..6cc875c58 100644 --- a/docker/logical-backup/Dockerfile +++ b/docker/logical-backup/Dockerfile @@ -13,7 +13,10 @@ RUN apt-get update \ curl \ jq \ gnupg \ + gcc \ + libffi-dev \ && pip3 install --no-cache-dir awscli --upgrade \ + && pip3 install --no-cache-dir gsutil --upgrade \ && echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ && cat /etc/apt/sources.list.d/pgdg.list \ && curl --silent https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ diff --git a/docker/logical-backup/dump.sh b/docker/logical-backup/dump.sh index 2d9a39e02..50f7e6e4c 100755 --- a/docker/logical-backup/dump.sh +++ b/docker/logical-backup/dump.sh @@ -46,6 +46,23 @@ function aws_upload { aws s3 cp - "$PATH_TO_BACKUP" "${args[@]//\'/}" } +function gcs_upload { + PATH_TO_BACKUP=gs://$LOGICAL_BACKUP_S3_BUCKET"/spilo/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz + + gsutil -o Credentials:gs_service_key_file=$LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS cp - "$PATH_TO_BACKUP" +} + +function upload { + case $LOGICAL_BACKUP_PROVIDER in + "gcs") + gcs_upload + ;; + *) + aws_upload $(($(estimate_size) / DUMP_SIZE_COEFF)) + ;; + esac +} + function get_pods { declare -r SELECTOR="$1" @@ -93,7 +110,7 @@ for search in "${search_strategy[@]}"; do done set -x -dump | compress | aws_upload $(($(estimate_size) / DUMP_SIZE_COEFF)) +dump | compress | upload [[ ${PIPESTATUS[0]} != 0 || ${PIPESTATUS[1]} != 0 || ${PIPESTATUS[2]} != 0 ]] && (( ERRORCOUNT += 1 )) set +x diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 09158e536..cd1cf8782 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -563,6 +563,10 @@ grouped under the `logical_backup` key. The default image is the same image built with the Zalando-internal CI pipeline. Default: "registry.opensource.zalan.do/acid/logical-backup" +* **logical_backup_provider** + Specifies the storage provider to which the backup should be uploaded (`s3` or `gcs`). + Default: "s3" + * **logical_backup_s3_bucket** S3 bucket to store backup results. The bucket has to be present and accessible by Postgres pods. Default: empty. @@ -583,6 +587,9 @@ grouped under the `logical_backup` key. * **logical_backup_s3_secret_access_key** When set, value will be in AWS_SECRET_ACCESS_KEY env variable. The Default is empty. +* **logical_backup_google_application_credentials** + Specifies the path of the google cloud service account json file. Default is empty. + ## Debugging the operator Options to aid debugging of the operator itself. Grouped under the `debug` key. diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 9578f8138..9e5d01040 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -186,14 +186,16 @@ type ConnectionPoolerConfiguration struct { // OperatorLogicalBackupConfiguration defines configuration for logical backup type OperatorLogicalBackupConfiguration struct { - Schedule string `json:"logical_backup_schedule,omitempty"` - DockerImage string `json:"logical_backup_docker_image,omitempty"` - S3Bucket string `json:"logical_backup_s3_bucket,omitempty"` - S3Region string `json:"logical_backup_s3_region,omitempty"` - S3Endpoint string `json:"logical_backup_s3_endpoint,omitempty"` - S3AccessKeyID string `json:"logical_backup_s3_access_key_id,omitempty"` - S3SecretAccessKey string `json:"logical_backup_s3_secret_access_key,omitempty"` - S3SSE string `json:"logical_backup_s3_sse,omitempty"` + Schedule string `json:"logical_backup_schedule,omitempty"` + DockerImage string `json:"logical_backup_docker_image,omitempty"` + BackupProvider string `json:"logical_backup_provider,omitempty"` + S3Bucket string `json:"logical_backup_s3_bucket,omitempty"` + S3Region string `json:"logical_backup_s3_region,omitempty"` + S3Endpoint string `json:"logical_backup_s3_endpoint,omitempty"` + S3AccessKeyID string `json:"logical_backup_s3_access_key_id,omitempty"` + S3SecretAccessKey string `json:"logical_backup_s3_secret_access_key,omitempty"` + S3SSE string `json:"logical_backup_s3_sse,omitempty"` + GoogleApplicationCredentials string `json:"logical_backup_google_application_credentials,omitempty"` } // OperatorConfigurationData defines the operation config diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 5360f2668..9e5ed7050 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1988,6 +1988,10 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar { }, }, // Bucket env vars + { + Name: "LOGICAL_BACKUP_PROVIDER", + Value: c.OpConfig.LogicalBackup.LogicalBackupProvider, + }, { Name: "LOGICAL_BACKUP_S3_BUCKET", Value: c.OpConfig.LogicalBackup.LogicalBackupS3Bucket, @@ -2008,6 +2012,10 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar { Name: "LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(c.Postgresql.GetUID())), }, + { + Name: "LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS", + Value: c.OpConfig.LogicalBackup.LogicalBackupGoogleApplicationCredentials, + }, // Postgres env vars { Name: "PG_VERSION", diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 9b0c6f194..250705656 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -146,12 +146,14 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur // logical backup config result.LogicalBackupSchedule = util.Coalesce(fromCRD.LogicalBackup.Schedule, "30 00 * * *") result.LogicalBackupDockerImage = util.Coalesce(fromCRD.LogicalBackup.DockerImage, "registry.opensource.zalan.do/acid/logical-backup") + result.LogicalBackupProvider = util.Coalesce(fromCRD.LogicalBackup.BackupProvider, "s3") result.LogicalBackupS3Bucket = fromCRD.LogicalBackup.S3Bucket result.LogicalBackupS3Region = fromCRD.LogicalBackup.S3Region result.LogicalBackupS3Endpoint = fromCRD.LogicalBackup.S3Endpoint result.LogicalBackupS3AccessKeyID = fromCRD.LogicalBackup.S3AccessKeyID result.LogicalBackupS3SecretAccessKey = fromCRD.LogicalBackup.S3SecretAccessKey result.LogicalBackupS3SSE = fromCRD.LogicalBackup.S3SSE + result.LogicalBackupGoogleApplicationCredentials = fromCRD.LogicalBackup.GoogleApplicationCredentials // debug config result.DebugLogging = fromCRD.OperatorDebug.DebugLogging diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 5e292dedd..acc00b2e8 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -111,14 +111,16 @@ type Scalyr struct { // LogicalBackup defines configuration for logical backup type LogicalBackup struct { - LogicalBackupSchedule string `name:"logical_backup_schedule" default:"30 00 * * *"` - LogicalBackupDockerImage string `name:"logical_backup_docker_image" default:"registry.opensource.zalan.do/acid/logical-backup"` - LogicalBackupS3Bucket string `name:"logical_backup_s3_bucket" default:""` - LogicalBackupS3Region string `name:"logical_backup_s3_region" default:""` - LogicalBackupS3Endpoint string `name:"logical_backup_s3_endpoint" default:""` - LogicalBackupS3AccessKeyID string `name:"logical_backup_s3_access_key_id" default:""` - LogicalBackupS3SecretAccessKey string `name:"logical_backup_s3_secret_access_key" default:""` - LogicalBackupS3SSE string `name:"logical_backup_s3_sse" default:""` + LogicalBackupSchedule string `name:"logical_backup_schedule" default:"30 00 * * *"` + LogicalBackupDockerImage string `name:"logical_backup_docker_image" default:"registry.opensource.zalan.do/acid/logical-backup"` + LogicalBackupProvider string `name:"logical_backup_provider" default:"s3"` + LogicalBackupS3Bucket string `name:"logical_backup_s3_bucket" default:""` + LogicalBackupS3Region string `name:"logical_backup_s3_region" default:""` + LogicalBackupS3Endpoint string `name:"logical_backup_s3_endpoint" default:""` + LogicalBackupS3AccessKeyID string `name:"logical_backup_s3_access_key_id" default:""` + LogicalBackupS3SecretAccessKey string `name:"logical_backup_s3_secret_access_key" default:""` + LogicalBackupS3SSE string `name:"logical_backup_s3_sse" default:""` + LogicalBackupGoogleApplicationCredentials string `name:"logical_backup_google_application_credentials" default:""` } // Operator options for connection pooler From 4b90809ade33d2cec63a4a1a312cfaef5e376f99 Mon Sep 17 00:00:00 2001 From: Enno Boland Date: Wed, 16 Dec 2020 10:44:25 +0100 Subject: [PATCH 138/168] =?UTF-8?q?helm-chart:=20allow=20configmaps=20inst?= =?UTF-8?q?ead=20of=20endpoints=20for=20leader=20elections=E2=80=A6=20(#10?= =?UTF-8?q?37)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * helm-chart: allow configmaps instead of endpoints if leader elections uses the configmaps method * helm-chart: allow endpoints get even if config maps are used * helm-chart: allow configmaps instead of endpoints on the operator role too. Co-authored-by: Enno Boland --- .../templates/clusterrole-postgres-pod.yaml | 22 +++++++++++++++++++ .../templates/clusterrole.yaml | 6 ++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml b/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml index ef607ae3c..b3f9f08f5 100644 --- a/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml +++ b/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml @@ -10,6 +10,27 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} rules: # Patroni needs to watch and manage endpoints +{{- if toString .Values.configGeneral.kubernetes_use_configmaps | eq "true" }} +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - endpoints + verbs: + - get +{{- else }} - apiGroups: - "" resources: @@ -23,6 +44,7 @@ rules: - patch - update - watch +{{- end }} # Patroni needs to watch pods - apiGroups: - "" diff --git a/charts/postgres-operator/templates/clusterrole.yaml b/charts/postgres-operator/templates/clusterrole.yaml index 00ee776f5..46113c4f1 100644 --- a/charts/postgres-operator/templates/clusterrole.yaml +++ b/charts/postgres-operator/templates/clusterrole.yaml @@ -63,11 +63,15 @@ rules: - patch - update - watch -# to manage endpoints which are also used by Patroni +# to manage endpoints/configmaps which are also used by Patroni - apiGroups: - "" resources: +{{- if toString .Values.configGeneral.kubernetes_use_configmaps | eq "true" }} + - configmaps +{{- else }} - endpoints +{{- end }} verbs: - create - delete From 5076e669cb4d547e565400adf900d364ac066503 Mon Sep 17 00:00:00 2001 From: Pavel Tumik Date: Wed, 16 Dec 2020 02:17:08 -0800 Subject: [PATCH 139/168] Fix timestamp regex (#1178) --- charts/postgres-operator/crds/postgresqls.yaml | 2 +- manifests/postgresql.crd.yaml | 2 +- pkg/apis/acid.zalan.do/v1/crds.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index fe54fb600..784bb2a76 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -128,7 +128,7 @@ spec: type: string timestamp: type: string - pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$' + pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$' # The regexp matches the date-time format (RFC 3339 Section 5.6) that specifies a timezone as an offset relative to UTC # Example: 1996-12-19T16:39:57-08:00 # Note: this field requires a timezone diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 80046a0f2..7836d07e7 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -124,7 +124,7 @@ spec: type: string timestamp: type: string - pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$' + pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$' # The regexp matches the date-time format (RFC 3339 Section 5.6) that specifies a timezone as an offset relative to UTC # Example: 1996-12-19T16:39:57-08:00 # Note: this field requires a timezone diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index cc79a3efe..737346f5e 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -202,7 +202,7 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ "timestamp": { Type: "string", Description: "Date-time format that specifies a timezone as an offset relative to UTC e.g. 1996-12-19T16:39:57-08:00", - Pattern: "^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\\.[0-9]+)?(([Zz])|([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$", + Pattern: "^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\\.[0-9]+)?(([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$", }, "uid": { Type: "string", From f28706e940649faf4a2cd2a9be643a3a4dfb8316 Mon Sep 17 00:00:00 2001 From: Rafia Sabih Date: Wed, 16 Dec 2020 13:50:24 +0100 Subject: [PATCH 140/168] Sync sts at pgversion upgrade (#1256) When pgversion is updated to a higher major version number, sync statefulSets also. Co-authored-by: Rafia Sabih --- pkg/cluster/cluster.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index d374a7732..050aeb5b5 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -596,6 +596,7 @@ func (c *Cluster) enforceMinResourceLimits(spec *acidv1.PostgresSpec) error { // for a cluster that had no such job before. In this case a missing job is not an error. func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { updateFailed := false + syncStatetfulSet := false c.mu.Lock() defer c.mu.Unlock() @@ -623,6 +624,7 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { } else if oldSpec.Spec.PostgresqlParam.PgVersion < newSpec.Spec.PostgresqlParam.PgVersion { c.logger.Infof("postgresql version increased (%q -> %q), major version upgrade can be done manually after StatefulSet Sync", oldSpec.Spec.PostgresqlParam.PgVersion, newSpec.Spec.PostgresqlParam.PgVersion) + syncStatetfulSet = true } // Service @@ -699,8 +701,9 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { updateFailed = true return } - if !reflect.DeepEqual(oldSs, newSs) || !reflect.DeepEqual(oldSpec.Annotations, newSpec.Annotations) { + if syncStatetfulSet || !reflect.DeepEqual(oldSs, newSs) || !reflect.DeepEqual(oldSpec.Annotations, newSpec.Annotations) { c.logger.Debugf("syncing statefulsets") + syncStatetfulSet = false // TODO: avoid generating the StatefulSet object twice by passing it to syncStatefulSet if err := c.syncStatefulSet(); err != nil { c.logger.Errorf("could not sync statefulsets: %v", err) From 77252e316c97ca4c572485d2b0803503ddd675ea Mon Sep 17 00:00:00 2001 From: Pavel Tumik Date: Wed, 16 Dec 2020 05:56:28 -0800 Subject: [PATCH 141/168] Add node affinity support (#1166) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Adding nodeaffinity support alongside node_readiness_label * add documentation for node affinity * add node affinity e2e test * add unit test for node affinity Co-authored-by: Steffen Pøhner Henriksen Co-authored-by: Adrian Astley --- .../postgres-operator/crds/postgresqls.yaml | 91 +++++++++++++++ docs/user.md | 24 +++- e2e/tests/test_e2e.py | 106 ++++++++++++++++++ manifests/complete-postgres-manifest.yaml | 11 ++ manifests/postgresql.crd.yaml | 91 +++++++++++++++ pkg/apis/acid.zalan.do/v1/crds.go | 85 ++++++++++++++ pkg/apis/acid.zalan.do/v1/postgresql_type.go | 1 + .../acid.zalan.do/v1/zz_generated.deepcopy.go | 1 + pkg/cluster/k8sres.go | 46 +++++--- pkg/cluster/k8sres_test.go | 66 +++++++++++ 10 files changed, 505 insertions(+), 17 deletions(-) diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 784bb2a76..13811936d 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -396,6 +396,97 @@ spec: type: string caSecretName: type: string + nodeAffinity: + type: object + properties: + preferredDuringSchedulingIgnoredDuringExecution: + type: array + items: + type: object + required: + - weight + - preference + properties: + preference: + type: object + properties: + matchExpressions: + type: array + items: + type: object + required: + - key + - operator + properties: + key: + type: string + operator: + type: string + values: + type: array + items: + type: string + matchFields: + type: array + items: + type: object + required: + - key + - operator + properties: + key: + type: string + operator: + type: string + values: + type: array + items: + type: string + weight: + format: int32 + type: integer + requiredDuringSchedulingIgnoredDuringExecution: + type: object + required: + - nodeSelectorTerms + properties: + nodeSelectorTerms: + type: array + items: + type: object + properties: + matchExpressions: + type: array + items: + type: object + required: + - key + - operator + properties: + key: + type: string + operator: + type: string + values: + type: array + items: + type: string + matchFields: + type: array + items: + type: object + required: + - key + - operator + properties: + key: + type: string + operator: + type: string + values: + type: array + items: + type: string tolerations: type: array items: diff --git a/docs/user.md b/docs/user.md index 8723a01e4..7552dca9d 100644 --- a/docs/user.md +++ b/docs/user.md @@ -517,7 +517,7 @@ manifest the operator will raise the limits to the configured minimum values. If no resources are defined in the manifest they will be obtained from the configured [default requests](reference/operator_parameters.md#kubernetes-resource-requests). -## Use taints and tolerations for dedicated PostgreSQL nodes +## Use taints, tolerations and node affinity for dedicated PostgreSQL nodes To ensure Postgres pods are running on nodes without any other application pods, you can use [taints and tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) @@ -531,6 +531,28 @@ spec: effect: NoSchedule ``` +If you need the pods to be scheduled on specific nodes you may use [node affinity](https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes-using-node-affinity/) +to specify a set of label(s), of which a prospective host node must have at least one. This could be used to +place nodes with certain hardware capabilities (e.g. SSD drives) in certain environments or network segments, +e.g. for PCI compliance. + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: postgresql +metadata: + name: acid-minimal-cluster +spec: + teamId: "ACID" + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: environment + operator: In + values: + - pci +``` + ## How to clone an existing PostgreSQL cluster You can spin up a new cluster as a clone of the existing one, using a `clone` diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index d396da01b..fa889047c 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -929,6 +929,112 @@ class EndToEndTestCase(unittest.TestCase): new_master_node = nm[0] self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_node_affinity(self): + ''' + Add label to a node and update postgres cluster spec to deploy only on a node with that label + ''' + k8s = self.k8s + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + + # verify we are in good state from potential previous tests + self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No 2 pods running") + self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members("acid-minimal-cluster-0")), 2, "Postgres status did not enter running") + self.eventuallyEqual(lambda: self.k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + # get nodes of master and replica(s) + master_node, replica_nodes = k8s.get_pg_nodes(cluster_label) + + self.assertNotEqual(master_node, []) + self.assertNotEqual(replica_nodes, []) + + # label node with environment=postgres + node_label_body = { + "metadata": { + "labels": { + "node-affinity-test": "postgres" + } + } + } + + try: + # patch current master node with the label + print('patching master node: {}'.format(master_node)) + k8s.api.core_v1.patch_node(master_node, node_label_body) + + # add node affinity to cluster + patch_node_affinity_config = { + "spec": { + "nodeAffinity" : { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "node-affinity-test", + "operator": "In", + "values": [ + "postgres" + ] + } + ] + } + ] + } + } + } + } + + k8s.api.custom_objects_api.patch_namespaced_custom_object( + group="acid.zalan.do", + version="v1", + namespace="default", + plural="postgresqls", + name="acid-minimal-cluster", + body=patch_node_affinity_config) + self.eventuallyEqual(lambda: self.k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + # node affinity change should cause replica to relocate from replica node to master node due to node affinity requirement + k8s.wait_for_pod_failover(master_node, 'spilo-role=replica,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) + + podsList = k8s.api.core_v1.list_namespaced_pod('default', label_selector=cluster_label) + for pod in podsList.items: + if pod.metadata.labels.get('spilo-role') == 'replica': + self.assertEqual(master_node, pod.spec.node_name, + "Sanity check: expected replica to relocate to master node {}, but found on {}".format(master_node, pod.spec.node_name)) + + # check that pod has correct node affinity + key = pod.spec.affinity.node_affinity.required_during_scheduling_ignored_during_execution.node_selector_terms[0].match_expressions[0].key + value = pod.spec.affinity.node_affinity.required_during_scheduling_ignored_during_execution.node_selector_terms[0].match_expressions[0].values[0] + self.assertEqual("node-affinity-test", key, + "Sanity check: expect node selector key to be equal to 'node-affinity-test' but got {}".format(key)) + self.assertEqual("postgres", value, + "Sanity check: expect node selector value to be equal to 'postgres' but got {}".format(value)) + + patch_node_remove_affinity_config = { + "spec": { + "nodeAffinity" : None + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + group="acid.zalan.do", + version="v1", + namespace="default", + plural="postgresqls", + name="acid-minimal-cluster", + body=patch_node_remove_affinity_config) + self.eventuallyEqual(lambda: self.k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + # remove node affinity to move replica away from master node + nm, new_replica_nodes = k8s.get_cluster_nodes() + new_master_node = nm[0] + self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_zzzz_cluster_deletion(self): ''' diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index e6fb9a43c..2555ed450 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -172,3 +172,14 @@ spec: # When TLS is enabled, also set spiloFSGroup parameter above to the relevant value. # if unknown, set it to 103 which is the usual value in the default spilo images. # In Openshift, there is no need to set spiloFSGroup/spilo_fsgroup. + +# Add node affinity support by allowing postgres pods to schedule only on nodes that +# have label: "postgres-operator:enabled" set. +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchExpressions: +# - key: postgres-operator +# operator: In +# values: +# - enabled diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 7836d07e7..d5170e9d4 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -392,6 +392,97 @@ spec: type: string caSecretName: type: string + nodeAffinity: + type: object + properties: + preferredDuringSchedulingIgnoredDuringExecution: + type: array + items: + type: object + required: + - weight + - preference + properties: + preference: + type: object + properties: + matchExpressions: + type: array + items: + type: object + required: + - key + - operator + properties: + key: + type: string + operator: + type: string + values: + type: array + items: + type: string + matchFields: + type: array + items: + type: object + required: + - key + - operator + properties: + key: + type: string + operator: + type: string + values: + type: array + items: + type: string + weight: + format: int32 + type: integer + requiredDuringSchedulingIgnoredDuringExecution: + type: object + required: + - nodeSelectorTerms + properties: + nodeSelectorTerms: + type: array + items: + type: object + properties: + matchExpressions: + type: array + items: + type: object + required: + - key + - operator + properties: + key: + type: string + operator: + type: string + values: + type: array + items: + type: string + matchFields: + type: array + items: + type: object + required: + - key + - operator + properties: + key: + type: string + operator: + type: string + values: + type: array + items: + type: string tolerations: type: array items: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 737346f5e..852a2f961 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -597,6 +597,91 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ }, }, }, + "nodeAffinity": { + Type: "object", + Properties: map[string]apiextv1.JSONSchemaProps{ + "preferredDuringSchedulingIgnoredDuringExecution": { + Type: "array", + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ + Type: "object", + Required: []string{"preference, weight"}, + Properties: map[string]apiextv1.JSONSchemaProps{ + "preference": { + Type: "object", + Properties: map[string]apiextv1.JSONSchemaProps{ + "matchExpressions": { + Type: "array", + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ + Type: "object", + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ + Allows: true, + }, + }, + }, + }, + "matchFields": { + Type: "array", + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ + Type: "object", + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ + Allows: true, + }, + }, + }, + }, + }, + }, + "weight": { + Type: "integer", + Format: "int32", + }, + }, + }, + }, + }, + "requiredDuringSchedulingIgnoredDuringExecution": { + Type: "object", + Required: []string{"nodeSelectorTerms"}, + Properties: map[string]apiextv1.JSONSchemaProps{ + "nodeSelectorTerms": { + Type: "array", + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextv1.JSONSchemaProps{ + "matchExpressions": { + Type: "array", + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ + Type: "object", + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ + Allows: true, + }, + }, + }, + }, + "matchFields": { + Type: "array", + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ + Type: "object", + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ + Allows: true, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, "tolerations": { Type: "array", Items: &apiextv1.JSONSchemaPropsOrArray{ diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 74a9057a5..0c87f96f8 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -61,6 +61,7 @@ type PostgresSpec struct { Databases map[string]string `json:"databases,omitempty"` PreparedDatabases map[string]PreparedDatabase `json:"preparedDatabases,omitempty"` SchedulerName *string `json:"schedulerName,omitempty"` + NodeAffinity v1.NodeAffinity `json:"nodeAffinity,omitempty"` Tolerations []v1.Toleration `json:"tolerations,omitempty"` Sidecars []Sidecar `json:"sidecars,omitempty"` InitContainers []v1.Container `json:"initContainers,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index f04a29490..83cd18c40 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -633,6 +633,7 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { *out = new(string) **out = **in } + in.NodeAffinity.DeepCopyInto(&out.NodeAffinity) if in.Tolerations != nil { in, out := &in.Tolerations, &out.Tolerations *out = make([]corev1.Toleration, len(*in)) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 9e5ed7050..1650d38d3 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -320,25 +320,39 @@ func getLocalAndBoostrapPostgreSQLParameters(parameters map[string]string) (loca return } -func nodeAffinity(nodeReadinessLabel map[string]string) *v1.Affinity { - matchExpressions := make([]v1.NodeSelectorRequirement, 0) - if len(nodeReadinessLabel) == 0 { +func nodeAffinity(nodeReadinessLabel map[string]string, nodeAffinity *v1.NodeAffinity) *v1.Affinity { + if len(nodeReadinessLabel) == 0 && nodeAffinity == nil { return nil } - for k, v := range nodeReadinessLabel { - matchExpressions = append(matchExpressions, v1.NodeSelectorRequirement{ - Key: k, - Operator: v1.NodeSelectorOpIn, - Values: []string{v}, - }) + nodeAffinityCopy := *&v1.NodeAffinity{} + if nodeAffinity != nil { + nodeAffinityCopy = *nodeAffinity.DeepCopy() + } + if len(nodeReadinessLabel) > 0 { + matchExpressions := make([]v1.NodeSelectorRequirement, 0) + for k, v := range nodeReadinessLabel { + matchExpressions = append(matchExpressions, v1.NodeSelectorRequirement{ + Key: k, + Operator: v1.NodeSelectorOpIn, + Values: []string{v}, + }) + } + nodeReadinessSelectorTerm := v1.NodeSelectorTerm{MatchExpressions: matchExpressions} + if nodeAffinityCopy.RequiredDuringSchedulingIgnoredDuringExecution == nil { + nodeAffinityCopy.RequiredDuringSchedulingIgnoredDuringExecution = &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + nodeReadinessSelectorTerm, + }, + } + } else { + nodeAffinityCopy.RequiredDuringSchedulingIgnoredDuringExecution = &v1.NodeSelector{ + NodeSelectorTerms: append(nodeAffinityCopy.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, nodeReadinessSelectorTerm), + } + } } return &v1.Affinity{ - NodeAffinity: &v1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ - NodeSelectorTerms: []v1.NodeSelectorTerm{{MatchExpressions: matchExpressions}}, - }, - }, + NodeAffinity: &nodeAffinityCopy, } } @@ -1209,7 +1223,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef effectiveRunAsUser, effectiveRunAsGroup, effectiveFSGroup, - nodeAffinity(c.OpConfig.NodeReadinessLabel), + nodeAffinity(c.OpConfig.NodeReadinessLabel, &spec.NodeAffinity), spec.SchedulerName, int64(c.OpConfig.PodTerminateGracePeriod.Seconds()), c.OpConfig.PodServiceAccountName, @@ -1914,7 +1928,7 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) { nil, nil, nil, - nodeAffinity(c.OpConfig.NodeReadinessLabel), + nodeAffinity(c.OpConfig.NodeReadinessLabel, nil), nil, int64(c.OpConfig.PodTerminateGracePeriod.Seconds()), c.OpConfig.PodServiceAccountName, diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 2f2b353ab..8a5103cbe 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -864,6 +864,72 @@ func testEnvs(cluster *Cluster, podSpec *v1.PodTemplateSpec, role PostgresRole) return nil } +func TestNodeAffinity(t *testing.T) { + var err error + var spec acidv1.PostgresSpec + var cluster *Cluster + var spiloRunAsUser = int64(101) + var spiloRunAsGroup = int64(103) + var spiloFSGroup = int64(103) + + makeSpec := func(nodeAffinity *v1.NodeAffinity) acidv1.PostgresSpec { + return acidv1.PostgresSpec{ + TeamID: "myapp", NumberOfInstances: 1, + Resources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + }, + Volume: acidv1.Volume{ + Size: "1G", + }, + NodeAffinity: *nodeAffinity, + } + } + + cluster = New( + Config{ + OpConfig: config.Config{ + PodManagementPolicy: "ordered_ready", + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + Resources: config.Resources{ + SpiloRunAsUser: &spiloRunAsUser, + SpiloRunAsGroup: &spiloRunAsGroup, + SpiloFSGroup: &spiloFSGroup, + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) + + nodeAff := &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + v1.NodeSelectorTerm{ + MatchExpressions: []v1.NodeSelectorRequirement{ + v1.NodeSelectorRequirement{ + Key: "test-label", + Operator: v1.NodeSelectorOpIn, + Values: []string{ + "test-value", + }, + }, + }, + }, + }, + }, + } + spec = makeSpec(nodeAff) + s, err := cluster.generateStatefulSet(&spec) + if err != nil { + assert.NoError(t, err) + } + + assert.NotNil(t, s.Spec.Template.Spec.Affinity.NodeAffinity, "node affinity in statefulset shouldn't be nil") + assert.Equal(t, s.Spec.Template.Spec.Affinity.NodeAffinity, nodeAff, "cluster template has correct node affinity") +} + func testCustomPodTemplate(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { if podSpec.ObjectMeta.Name != "test-pod-template" { return fmt.Errorf("Custom pod template is not used, current spec %+v", From 636ba9b84624a3552182e5156b1c6d752b213c97 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 16 Dec 2020 15:23:06 +0100 Subject: [PATCH 142/168] [UI] reflect new backup paths and cluster status (#1260) * [UI] reflect new backup paths and cluster status --- ui/app/src/postgresql.tag.pug | 7 ++++++- ui/app/src/postgresqls.tag.pug | 8 +++----- ui/operator_ui/main.py | 7 ++++++- ui/operator_ui/spiloutils.py | 31 +++++++++++++++++++------------ 4 files changed, 34 insertions(+), 19 deletions(-) diff --git a/ui/app/src/postgresql.tag.pug b/ui/app/src/postgresql.tag.pug index 9edae99d3..c557e4da8 100644 --- a/ui/app/src/postgresql.tag.pug +++ b/ui/app/src/postgresql.tag.pug @@ -74,11 +74,13 @@ postgresql .alert.alert-info(if='{ !progress.requestStatus }') PostgreSQL cluster requested .alert.alert-danger(if='{ progress.requestStatus !== "OK" }') Create request failed - .alert.alert-success(if='{ progress.requestStatus === "OK" }') Create request successful ({ new Date(progress.createdTimestamp).toLocaleString() }) + .alert.alert-success(if='{ progress.requestStatus === "OK" }') Manifest creation successful ({ new Date(progress.createdTimestamp).toLocaleString() }) .alert.alert-info(if='{ !progress.postgresql }') PostgreSQL cluster manifest pending .alert.alert-success(if='{ progress.postgresql }') PostgreSQL cluster manifest created + .alert.alert-danger(if='{progress.status && progress.status.PostgresClusterStatus == "CreateFailed"}') Cluster creation failed: Check events and cluster name! + .alert.alert-info(if='{ !progress.statefulSet }') StatefulSet pending .alert.alert-success(if='{ progress.statefulSet }') StatefulSet created @@ -127,6 +129,8 @@ postgresql this.progress.pooler = false this.progress.postgresql = true this.progress.postgresqlManifest = data + // copy status as we delete later for edit + this.progress.status = data.status this.progress.createdTimestamp = data.metadata.creationTimestamp this.progress.poolerEnabled = data.spec.enableConnectionPooler this.uid = this.progress.postgresqlManifest.metadata.uid @@ -203,6 +207,7 @@ postgresql delete manifest.metadata.annotations[last_applied] } + delete manifest.metadata.managedFields delete manifest.metadata.creationTimestamp delete manifest.metadata.deletionGracePeriodSeconds delete manifest.metadata.deletionTimestamp diff --git a/ui/app/src/postgresqls.tag.pug b/ui/app/src/postgresqls.tag.pug index 250c175ec..38e5fcd9d 100644 --- a/ui/app/src/postgresqls.tag.pug +++ b/ui/app/src/postgresqls.tag.pug @@ -63,10 +63,8 @@ postgresqls td(style='white-space: pre') | { namespace } td - a( - href='/#/status/{ cluster_path(this) }' - ) - | { name } + a(href='/#/status/{ cluster_path(this) }') { name } + btn.btn-danger(if='{status.PostgresClusterStatus == "CreateFailed"}') Create Failed td { nodes } td { cpu } / { cpu_limit } td { memory } / { memory_limit } @@ -230,7 +228,7 @@ postgresqls ) const calcCosts = this.calcCosts = (nodes, cpu, memory, disk) => { - costs = nodes * (toCores(cpu) * opts.config.cost_core + toMemory(memory) * opts.config.cost_memory + toDisk(disk) * opts.config.cost_ebs) + costs = Math.max(nodes, opts.config.min_pods) * (toCores(cpu) * opts.config.cost_core + toMemory(memory) * opts.config.cost_memory + toDisk(disk) * opts.config.cost_ebs) return costs.toFixed(2) } diff --git a/ui/operator_ui/main.py b/ui/operator_ui/main.py index f1242fa37..893e45ef0 100644 --- a/ui/operator_ui/main.py +++ b/ui/operator_ui/main.py @@ -87,6 +87,7 @@ SPILO_S3_BACKUP_PREFIX = getenv('SPILO_S3_BACKUP_PREFIX', 'spilo/') SUPERUSER_TEAM = getenv('SUPERUSER_TEAM', 'acid') TARGET_NAMESPACE = getenv('TARGET_NAMESPACE') GOOGLE_ANALYTICS = getenv('GOOGLE_ANALYTICS', False) +MIN_PODS= getenv('MIN_PODS', 2) # storage pricing, i.e. https://aws.amazon.com/ebs/pricing/ COST_EBS = float(getenv('COST_EBS', 0.119)) # GB per month @@ -308,7 +309,8 @@ DEFAULT_UI_CONFIG = { 'static_network_whitelist': {}, 'cost_ebs': COST_EBS, 'cost_core': COST_CORE, - 'cost_memory': COST_MEMORY + 'cost_memory': COST_MEMORY, + 'min_pods': MIN_PODS } @@ -320,6 +322,7 @@ def get_config(): config['resources_visible'] = RESOURCES_VISIBLE config['superuser_team'] = SUPERUSER_TEAM config['target_namespace'] = TARGET_NAMESPACE + config['min_pods'] = MIN_PODS config['namespaces'] = ( [TARGET_NAMESPACE] @@ -493,6 +496,7 @@ def get_postgresqls(): 'uid': uid, 'namespaced_name': namespace + '/' + name, 'full_name': namespace + '/' + name + ('/' + uid if uid else ''), + 'status': status, } for cluster in these( read_postgresqls( @@ -506,6 +510,7 @@ def get_postgresqls(): 'items', ) for spec in [cluster.get('spec', {}) if cluster.get('spec', {}) is not None else {"error": "Invalid spec in manifest"}] + for status in [cluster.get('status', {})] for metadata in [cluster['metadata']] for namespace in [metadata['namespace']] for name in [metadata['name']] diff --git a/ui/operator_ui/spiloutils.py b/ui/operator_ui/spiloutils.py index 7a71f6dab..6b1d394a1 100644 --- a/ui/operator_ui/spiloutils.py +++ b/ui/operator_ui/spiloutils.py @@ -302,6 +302,7 @@ def read_versions( if uid == 'wal' or defaulting(lambda: UUID(uid)) ] +BACKUP_VERSION_PREFIXES = ['','9.5/', '9.6/', '10/','11/', '12/', '13/'] def read_basebackups( pg_cluster, @@ -314,18 +315,24 @@ def read_basebackups( ): environ['WALE_S3_ENDPOINT'] = s3_endpoint suffix = '' if uid == 'base' else '/' + uid - return [ - { - key: value - for key, value in basebackup.__dict__.items() - if isinstance(value, str) or isinstance(value, int) - } - for basebackup in Attrs.call( - f=configure_backup_cxt, - aws_instance_profile=use_aws_instance_profile, - s3_prefix=f's3://{bucket}/{prefix}{pg_cluster}{suffix}/wal/', - )._backup_list(detail=True)._backup_list(prefix=f"{prefix}{pg_cluster}{suffix}/wal/") - ] + backups = [] + + for vp in BACKUP_VERSION_PREFIXES: + + backups = backups + [ + { + key: value + for key, value in basebackup.__dict__.items() + if isinstance(value, str) or isinstance(value, int) + } + for basebackup in Attrs.call( + f=configure_backup_cxt, + aws_instance_profile=use_aws_instance_profile, + s3_prefix=f's3://{bucket}/{prefix}{pg_cluster}{suffix}/wal/{vp}', + )._backup_list(detail=True) + ] + + return backups def parse_time(s: str): From 5f3f6988bc18358293e53b59799faf05ff11644a Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 17 Dec 2020 12:07:40 +0100 Subject: [PATCH 143/168] add logical-backup build and push to delivery.yaml (#1259) * add logical-backup build and push to delivery.yaml * enable manual approval for UI and logical-backup --- delivery.yaml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/delivery.yaml b/delivery.yaml index a4d42af7d..fca5af7ea 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -55,7 +55,7 @@ pipeline: - id: build-operator-ui type: script - + requires_human_approval: true commands: - desc: 'Prepare environment' cmd: | @@ -80,3 +80,15 @@ pipeline: export IMAGE make docker make push + + - id: build-logical-backup + type: script + requires_human_approval: true + commands: + - desc: Build image + cmd: | + cd docker/logical-backup + export TAG=$(git describe --tags --always --dirty) + IMAGE="registry-write.opensource.zalan.do/acid/logical-backup" + docker build --rm -t "$IMAGE:$TAG$CDP_TAG" . + docker push "$IMAGE:$TAG$CDP_TAG" From a63ad49ef8101bb58725d892151279cd751df5c0 Mon Sep 17 00:00:00 2001 From: Jan Mussler Date: Thu, 17 Dec 2020 15:00:29 +0100 Subject: [PATCH 144/168] Initial commit for new 1.6 release with Postgres 13 support. (#1257) * Initial commit for new 1.6 release with Postgres 13 support. * Updating maintainers, Go version, Codeowners. * Use lazy upgrade image that contains pg13. * fix typo for ownerReference * fix clusterrole in helm chart * reflect GCP logical backup in validation * improve PostgresTeam docs * change defaults for enable_pgversion_env_var and storage_resize_mode * explain manual part of in-place upgrade * remove gsoc docs Co-authored-by: Felix Kunde --- .github/workflows/run_e2e.yaml | 2 +- .github/workflows/run_tests.yaml | 2 +- CODEOWNERS | 2 +- MAINTAINERS | 4 +- README.md | 26 ++- .../crds/operatorconfigurations.yaml | 4 + .../templates/clusterrole.yaml | 40 +++-- charts/postgres-operator/values-crd.yaml | 2 +- charts/postgres-operator/values.yaml | 2 +- delivery.yaml | 2 +- docs/administrator.md | 28 +++- docs/developer.md | 2 +- docs/gsoc-2019/ideas.md | 63 -------- docs/reference/operator_parameters.md | 4 +- docs/user.md | 148 +++++++++++++----- e2e/run.sh | 2 +- e2e/tests/test_e2e.py | 4 +- manifests/complete-postgres-manifest.yaml | 8 +- manifests/configmap.yaml | 10 +- manifests/minimal-postgres-manifest.yaml | 2 +- manifests/operatorconfiguration.crd.yaml | 4 + ...gresql-operator-default-configuration.yaml | 2 +- mkdocs.yml | 1 - pkg/apis/acid.zalan.do/v1/crds.go | 6 + pkg/cluster/connection_pooler_test.go | 4 +- pkg/cluster/k8sres_test.go | 4 +- pkg/controller/operator_config.go | 2 +- pkg/util/config/config.go | 4 +- ui/manifests/deployment.yaml | 6 +- ui/operator_ui/main.py | 2 +- 30 files changed, 233 insertions(+), 159 deletions(-) delete mode 100644 docs/gsoc-2019/ideas.md diff --git a/.github/workflows/run_e2e.yaml b/.github/workflows/run_e2e.yaml index 6eef5c8f2..cff0d49ef 100644 --- a/.github/workflows/run_e2e.yaml +++ b/.github/workflows/run_e2e.yaml @@ -14,7 +14,7 @@ jobs: - uses: actions/checkout@v1 - uses: actions/setup-go@v2 with: - go-version: "^1.15.5" + go-version: "^1.15.6" - name: Make dependencies run: make deps mocks - name: Compile diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml index 56d147990..ebcdfedf6 100644 --- a/.github/workflows/run_tests.yaml +++ b/.github/workflows/run_tests.yaml @@ -14,7 +14,7 @@ jobs: - uses: actions/checkout@v1 - uses: actions/setup-go@v2 with: - go-version: "^1.15.5" + go-version: "^1.15.6" - name: Make dependencies run: make deps mocks - name: Compile diff --git a/CODEOWNERS b/CODEOWNERS index 96fe74510..398856c66 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,2 +1,2 @@ # global owners -* @alexeyklyukin @erthalion @sdudoladov @Jan-M @CyberDem0n @avaczi @FxKu @RafiaSabih +* @erthalion @sdudoladov @Jan-M @CyberDem0n @avaczi @FxKu @RafiaSabih diff --git a/MAINTAINERS b/MAINTAINERS index 4f4ca87ba..572e6d971 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1,3 +1,5 @@ -Oleksii Kliukin Dmitrii Dolgov Sergey Dudoladov +Felix Kunde +Jan Mussler +Rafia Sabih \ No newline at end of file diff --git a/README.md b/README.md index 5e00dba8b..465e726d4 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ pipelines with no access to Kubernetes API directly, promoting infrastructure as ### Operator features * Rolling updates on Postgres cluster changes, incl. quick minor version updates -* Live volume resize without pod restarts (AWS EBS, others pending) +* Live volume resize without pod restarts (AWS EBS, PvC) * Database connection pooler with PGBouncer * Restore and cloning Postgres clusters (incl. major version upgrade) * Additionally logical backups to S3 bucket can be configured @@ -23,10 +23,12 @@ pipelines with no access to Kubernetes API directly, promoting infrastructure as * Basic credential and user management on K8s, eases application deployments * UI to create and edit Postgres cluster manifests * Works well on Amazon AWS, Google Cloud, OpenShift and locally on Kind +* Support for custom TLS certificates +* Base support for AWS EBS gp3 migration (iops, throughput pending) ### PostgreSQL features -* Supports PostgreSQL 12, starting from 9.6+ +* Supports PostgreSQL 13, starting from 9.6+ * Streaming replication cluster via Patroni * Point-In-Time-Recovery with [pg_basebackup](https://www.postgresql.org/docs/11/app-pgbasebackup.html) / @@ -48,7 +50,25 @@ pipelines with no access to Kubernetes API directly, promoting infrastructure as [timescaledb](https://github.com/timescale/timescaledb) The Postgres Operator has been developed at Zalando and is being used in -production for over two years. +production for over three years. + +## Notes on Postgres 13 support + +If you are new to the operator, you can skip this and just start using the Postgres operator as is, Postgres 13 is ready to go. + +The Postgres operator supports Postgres 13 with the new Spilo Image that includes also the recent Patroni version to support PG13 settings. +More work on optimizing restarts and rolling upgrades is pending. + +If you are already using the Postgres operator in older version with a Spilo 12 Docker Image you need to be aware of the changes for the backup path. +We introduce the major version into the backup path to smooth the major version upgrade that is now supported manually. + +The new operator configuration, sets a compatilibty flag *enable_spilo_wal_path_compat* to make Spilo look in current path but also old format paths for wal segments. +This comes at potential perf. costs, and should be disabled after a few days. + +The new Spilo 13 image is: `registry.opensource.zalan.do/acid/spilo-13:2.0-p1` + +The last Spilo 12 image is: `registry.opensource.zalan.do/acid/spilo-12:1.6-p5` + ## Getting started diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 2cbc2cfce..2ac9ca7fc 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -319,6 +319,10 @@ spec: properties: logical_backup_docker_image: type: string + logical_backup_google_application_credentials: + type: string + logical_backup_provider: + type: string logical_backup_s3_access_key_id: type: string logical_backup_s3_bucket: diff --git a/charts/postgres-operator/templates/clusterrole.yaml b/charts/postgres-operator/templates/clusterrole.yaml index 46113c4f1..165cce7c6 100644 --- a/charts/postgres-operator/templates/clusterrole.yaml +++ b/charts/postgres-operator/templates/clusterrole.yaml @@ -44,13 +44,6 @@ rules: - get - patch - update -# to read configuration from ConfigMaps -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get # to send events to the CRs - apiGroups: - "" @@ -64,14 +57,11 @@ rules: - update - watch # to manage endpoints/configmaps which are also used by Patroni +{{- if toString .Values.configGeneral.kubernetes_use_configmaps | eq "true" }} - apiGroups: - "" resources: -{{- if toString .Values.configGeneral.kubernetes_use_configmaps | eq "true" }} - configmaps -{{- else }} - - endpoints -{{- end }} verbs: - create - delete @@ -81,6 +71,34 @@ rules: - patch - update - watch +- apiGroups: + - "" + resources: + - endpoints + verbs: + - get +{{- else }} +# to read configuration from ConfigMaps +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get +- apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +{{- end }} # to CRUD secrets for database access - apiGroups: - "" diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index fc8e36ec9..b52f021c8 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -22,7 +22,7 @@ configGeneral: # update only the statefulsets without immediately doing the rolling update enable_lazy_spilo_upgrade: false # set the PGVERSION env var instead of providing the version via postgresql.bin_dir in SPILO_CONFIGURATION - enable_pgversion_env_var: false + enable_pgversion_env_var: true # start any new database pod without limitations on shm memory enable_shm_volume: true # enables backwards compatible path between Spilo 12 and Spilo 13 images diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 65340d1cd..e17d78840 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -25,7 +25,7 @@ configGeneral: # update only the statefulsets without immediately doing the rolling update enable_lazy_spilo_upgrade: "false" # set the PGVERSION env var instead of providing the version via postgresql.bin_dir in SPILO_CONFIGURATION - enable_pgversion_env_var: "false" + enable_pgversion_env_var: "true" # start any new database pod without limitations on shm memory enable_shm_volume: "true" # enables backwards compatible path between Spilo 12 and Spilo 13 images diff --git a/delivery.yaml b/delivery.yaml index fca5af7ea..d8b8d724a 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -16,7 +16,7 @@ pipeline: - desc: 'Install go' cmd: | cd /tmp - wget -q https://storage.googleapis.com/golang/go1.15.5.linux-amd64.tar.gz -O go.tar.gz + wget -q https://storage.googleapis.com/golang/go1.15.6.linux-amd64.tar.gz -O go.tar.gz tar -xf go.tar.gz mv go /usr/local ln -s /usr/local/go/bin/go /usr/bin/go diff --git a/docs/administrator.md b/docs/administrator.md index 64bb10b68..1e18c31f4 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -11,15 +11,29 @@ switchover (planned failover) of the master to the Pod with new minor version. The switch should usually take less than 5 seconds, still clients have to reconnect. -Major version upgrades are supported either via [cloning](user.md#how-to-clone-an-existing-postgresql-cluster)or in-place. +Major version upgrades are supported either via [cloning](user.md#how-to-clone-an-existing-postgresql-cluster) +or in-place. -With cloning, the new cluster manifest must have a higher `version` string than the source -cluster and will be created from a basebackup. Depending of the cluster size, -downtime in this case can be significant as writes to the database should be -stopped and all WAL files should be archived first before cloning is started. +With cloning, the new cluster manifest must have a higher `version` string than +the source cluster and will be created from a basebackup. Depending of the +cluster size, downtime in this case can be significant as writes to the database +should be stopped and all WAL files should be archived first before cloning is +started. -Starting with Spilo 13, Postgres Operator can do in-place major version upgrade, which should be faster than cloning. To trigger the upgrade, simply increase the version in the cluster manifest. As the very last step of -processing the manifest update event, the operator will call the `inplace_upgrade.py` script in Spilo. The upgrade is usually fast, well under one minute for most DBs. Note the changes become irrevertible once `pg_upgrade` is called. To understand the upgrade procedure, refer to the [corresponding PR in Spilo](https://github.com/zalando/spilo/pull/488). +Starting with Spilo 13, Postgres Operator can do in-place major version upgrade, +which should be faster than cloning. However, it is not fully automatic yet. +First, you need to make sure, that setting the PG_VERSION environment variable +is enabled in the configuration. Since `v1.6.0`, `enable_pgversion_env_var` is +enabled by default. + +To trigger the upgrade, increase the version in the cluster manifest. After +Pods are rotated `configure_spilo` will notice the version mismatch and start +the old version again. You can then exec into the Postgres container of the +master instance and call `python3 /scripts/inplace_upgrade.py N` where `N` +is the number of members of your cluster (see `number_of_instances`). The +upgrade is usually fast, well under one minute for most DBs. Note, that changes +become irrevertible once `pg_upgrade` is called. To understand the upgrade +procedure, refer to the [corresponding PR in Spilo](https://github.com/zalando/spilo/pull/488). ## CRD Validation diff --git a/docs/developer.md b/docs/developer.md index 59fbe09a2..3316ac4cc 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -286,7 +286,7 @@ manifest files: Postgres manifest parameters are defined in the [api package](../pkg/apis/acid.zalan.do/v1/postgresql_type.go). The operator behavior has to be implemented at least in [k8sres.go](../pkg/cluster/k8sres.go). -Validation of CRD parameters is controlled in [crd.go](../pkg/apis/acid.zalan.do/v1/crds.go). +Validation of CRD parameters is controlled in [crds.go](../pkg/apis/acid.zalan.do/v1/crds.go). Please, reflect your changes in tests, for example in: * [config_test.go](../pkg/util/config/config_test.go) * [k8sres_test.go](../pkg/cluster/k8sres_test.go) diff --git a/docs/gsoc-2019/ideas.md b/docs/gsoc-2019/ideas.md deleted file mode 100644 index 456a5a0ff..000000000 --- a/docs/gsoc-2019/ideas.md +++ /dev/null @@ -1,63 +0,0 @@ -

Google Summer of Code 2019

- -## Applications steps - -1. Please carefully read the official [Google Summer of Code Student Guide](https://google.github.io/gsocguides/student/) -2. Join the #postgres-operator slack channel under [Postgres Slack](https://postgres-slack.herokuapp.com) to introduce yourself to the community and get quick feedback on your application. -3. Select a project from the list of ideas below or propose your own. -4. Write a proposal draft. Please open an issue with the label `gsoc2019_application` in the [operator repository](https://github.com/zalando/postgres-operator/issues) so that the community members can publicly review it. See proposal instructions below for details. -5. Submit proposal and the proof of enrollment before April 9 2019 18:00 UTC through the web site of the Program. - -## Project ideas - - -### Place database pods into the "Guaranteed" Quality-of-Service class - -* **Description**: Kubernetes runtime does not kill pods in this class on condition they stay within their resource limits, which is desirable for the DB pods serving production workloads. To be assigned to that class, pod's resources must equal its limits. The task is to add the `enableGuaranteedQoSClass` or the like option to the Postgres manifest and the operator configmap that forcibly re-write pod resources to match the limits. -* **Recommended skills**: golang, basic Kubernetes abstractions -* **Difficulty**: moderate -* **Mentor(s)**: Felix Kunde [@FxKu](https://github.com/fxku), Sergey Dudoladov [@sdudoladov](https://github.com/sdudoladov) - -### Implement the kubectl plugin for the Postgres CustomResourceDefinition - -* **Description**: [kubectl plugins](https://kubernetes.io/docs/tasks/extend-kubectl/kubectl-plugins/) enable extending the Kubernetes command-line client `kubectl` with commands to manage custom resources. The task is to design and implement a plugin for the `kubectl postgres` command, -that can enable, for example, correct deletion or major version upgrade of Postgres clusters. -* **Recommended skills**: golang, shell scripting, operational experience with Kubernetes -* **Difficulty**: moderate to medium, depending on the plugin design -* **Mentor(s)**: Felix Kunde [@FxKu](https://github.com/fxku), Sergey Dudoladov [@sdudoladov](https://github.com/sdudoladov) - -### Implement the openAPIV3Schema for the Postgres CRD - -* **Description**: at present the operator validates a database manifest on its own. -It will be helpful to reject erroneous manifests before they reach the operator using the [native Kubernetes CRD validation](https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#validation). It is up to the student to decide whether to write the schema manually or to adopt existing [schema generator developed for the Prometheus project](https://github.com/ant31/crd-validation). -* **Recommended skills**: golang, JSON schema -* **Difficulty**: medium -* **Mentor(s)**: Sergey Dudoladov [@sdudoladov](https://github.com/sdudoladov) -* **Issue**: [#388](https://github.com/zalando/postgres-operator/issues/388) - -### Design a solution for the local testing of the operator - -* **Description**: The current way of testing is to run minikube, either manually or with some tooling around it like `/run-operator_locally.sh` or Vagrant. This has at least three problems: -First, minikube is a single node cluster, so it is unsuitable for testing vital functions such as pod migration between nodes. Second, minikube starts slowly; that prolongs local testing. -Third, every contributor needs to come up with their own solution for local testing. The task is to come up with a better option which will enable us to conveniently and uniformly run e2e tests locally / potentially in Travis CI. -A promising option is the Kubernetes own [kind](https://github.com/kubernetes-sigs/kind) -* **Recommended skills**: Docker, shell scripting, basic Kubernetes abstractions -* **Difficulty**: medium to hard depending on the selected desing -* **Mentor(s)**: Dmitry Dolgov [@erthalion](https://github.com/erthalion), Sergey Dudoladov [@sdudoladov](https://github.com/sdudoladov) -* **Issue**: [#475](https://github.com/zalando/postgres-operator/issues/475) - -### Detach a Postgres cluster from the operator for maintenance - -* **Description**: sometimes a Postgres cluster requires manual maintenance. During such maintenance the operator should ignore all the changes manually applied to the cluster. - Currently the only way to achieve this behavior is to shutdown the operator altogether, for instance by scaling down the operator's own deployment to zero pods. That approach evidently affects all Postgres databases under the operator control and thus is highly undesirable in production Kubernetes clusters. It would be much better to be able to detach only the desired Postgres cluster from the operator for the time being and re-attach it again after maintenance. -* **Recommended skills**: golang, architecture of a Kubernetes operator -* **Difficulty**: hard - requires significant modification of the operator's internals and careful consideration of the corner cases. -* **Mentor(s)**: Dmitry Dolgov [@erthalion](https://github.com/erthalion), Sergey Dudoladov [@sdudoladov](https://github.com/sdudoladov) -* **Issue**: [#421](https://github.com/zalando/postgres-operator/issues/421) - -### Propose your own idea - -Feel free to come up with your own ideas. For inspiration, -see [our bug tracker](https://github.com/zalando/postgres-operator/issues), -the [official `CustomResouceDefinition` docs](https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/) -and [other operators](https://github.com/operator-framework/awesome-operators). diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index cd1cf8782..01e5c4039 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -80,7 +80,7 @@ Those are top-level keys, containing both leaf keys and groups. The default is `false`. * **enable_pgversion_env_var** - With newer versions of Spilo, it is preferable to use `PGVERSION` pod environment variable instead of the setting `postgresql.bin_dir` in the `SPILO_CONFIGURATION` env variable. When this option is true, the operator sets `PGVERSION` and omits `postgresql.bin_dir` from `SPILO_CONFIGURATION`. When false, the `postgresql.bin_dir` is set. This setting takes precedence over `PGVERSION`; see PR 222 in Spilo. The default is `false`. + With newer versions of Spilo, it is preferable to use `PGVERSION` pod environment variable instead of the setting `postgresql.bin_dir` in the `SPILO_CONFIGURATION` env variable. When this option is true, the operator sets `PGVERSION` and omits `postgresql.bin_dir` from `SPILO_CONFIGURATION`. When false, the `postgresql.bin_dir` is set. This setting takes precedence over `PGVERSION`; see PR 222 in Spilo. The default is `true`. * **enable_spilo_wal_path_compat** enables backwards compatible path between Spilo 12 and Spilo 13 images. The default is `false`. @@ -375,7 +375,7 @@ configuration they are grouped under the `kubernetes` key. * **storage_resize_mode** defines how operator handels the difference between requested volume size and actual size. Available options are: ebs - tries to resize EBS volume, pvc - - changes PVC definition, off - disables resize of the volumes. Default is "ebs". + changes PVC definition, off - disables resize of the volumes. Default is "pvc". When using OpenShift please use one of the other available options. ## Kubernetes resource requests diff --git a/docs/user.md b/docs/user.md index 7552dca9d..3463983f7 100644 --- a/docs/user.md +++ b/docs/user.md @@ -275,9 +275,18 @@ Postgres clusters are associated with one team by providing the `teamID` in the manifest. Additional superuser teams can be configured as mentioned in the previous paragraph. However, this is a global setting. To assign additional teams, superuser teams and single users to clusters of a given -team, use the [PostgresTeam CRD](../manifests/postgresteam.yaml). It provides -a simple mapping structure. +team, use the [PostgresTeam CRD](../manifests/postgresteam.yaml). +Note, by default the `PostgresTeam` support is disabled in the configuration. +Switch `enable_postgres_team_crd` flag to `true` and the operator will start to +watch for this CRD. Make sure, the cluster role is up to date and contains a +section for [PostgresTeam](../manifests/operator-service-account-rbac.yaml#L30). + +#### Additional teams + +To assign additional teams and single users to clusters of a given team, +define a mapping with the `PostgresTeam` Kubernetes resource. The Postgres +Operator will read such team mappings each time it syncs all Postgres clusters. ```yaml apiVersion: "acid.zalan.do/v1" @@ -285,55 +294,118 @@ kind: PostgresTeam metadata: name: custom-team-membership spec: - additionalSuperuserTeams: - acid: - - "postgres_superusers" additionalTeams: - acid: [] - additionalMembers: - acid: - - "elephant" + a-team: + - "b-team" ``` -One `PostgresTeam` resource could contain mappings of multiple teams but you -can choose to create separate CRDs, alternatively. On each CRD creation or -update the operator will gather all mappings to create additional human users -in databases the next time they are synced. Additional teams are resolved -transitively, meaning you will also add users for their `additionalTeams` -or (not and) `additionalSuperuserTeams`. +With the example above the operator will create login roles for all members +of `b-team` in every cluster owned by `a-team`. It's possible to do vice versa +for clusters of `b-team` in one manifest: -For each additional team the Teams API would be queried. Additional members -will be added either way. There can be "virtual teams" that do not exists in -your Teams API but users of associated teams as well as members will get -created. With `PostgresTeams` it's also easy to cover team name changes. Just -add the mapping between old and new team name and the rest can stay the same. +```yaml +spec: + additionalTeams: + a-team: + - "b-team" + b-team: + - "a-team" +``` + +You see, the `PostgresTeam` CRD is a global team mapping and independent from +the Postgres manifests. It is possible to define multiple mappings, even with +redundant content - the Postgres operator will create one internal cache from +it. Additional teams are resolved transitively, meaning you will also add +users for their `additionalTeams`, e.g.: + +```yaml +spec: + additionalTeams: + a-team: + - "b-team" + - "c-team" + b-team: + - "a-team" +``` + +This creates roles for members of the `c-team` team not only in all clusters +owned by `a-team`, but as well in cluster owned by `b-team`, as `a-team` is +an `additionalTeam` to `b-team` + +Not, you can also define `additionalSuperuserTeams` in the `PostgresTeam` +manifest. By default, this option is disabled and must be configured with +`enable_postgres_team_crd_superusers` to make it work. + +#### Virtual teams + +There can be "virtual teams" that do not exist in the Teams API. It can make +it easier to map a group of teams to many other teams: + +```yaml +spec: + additionalTeams: + a-team: + - "virtual-team" + b-team: + - "virtual-team" + virtual-team: + - "c-team" + - "d-team" +``` + +This example would create roles for members of `c-team` and `d-team` plus +additional `virtual-team` members in clusters owned by `a-team` or `b-team`. + +#### Teams changing their names + +With `PostgresTeams` it is also easy to cover team name changes. Just add +the mapping between old and new team name and the rest can stay the same. +E.g. if team `a-team`'s name would change to `f-team` in the teams API it +could be reflected in a `PostgresTeam` mapping with just two lines: + +```yaml +spec: + additionalTeams: + a-team: + - "f-team" +``` + +This is helpful, because Postgres cluster names are immutable and can not +be changed. Only via cloning it could get a different name starting with the +new `teamID`. + +#### Additional members + +Single members might be excluded from teams although they continue to work +with the same people. However, the teams API would not reflect this anymore. +To still add a database role for former team members list their role under +the `additionalMembers` section of the `PostgresTeam` resource: ```yaml apiVersion: "acid.zalan.do/v1" kind: PostgresTeam metadata: - name: virtualteam-membership + name: custom-team-membership spec: - additionalSuperuserTeams: - acid: - - "virtual_superusers" - virtual_superusers: - - "real_teamA" - - "real_teamB" - real_teamA: - - "real_teamA_renamed" - additionalTeams: - real_teamA: - - "real_teamA_renamed" additionalMembers: - virtual_superusers: - - "foo" + a-team: + - "tia" ``` -Note, by default the `PostgresTeam` support is disabled in the configuration. -Switch `enable_postgres_team_crd` flag to `true` and the operator will start to -watch for this CRD. Make sure, the cluster role is up to date and contains a -section for [PostgresTeam](../manifests/operator-service-account-rbac.yaml#L30). +This will create the login role `tia` in every cluster owned by `a-team`. +The user can connect to databases like the other team members. + +The `additionalMembers` map can also be used to define users of virtual +teams, e.g. for `virtual-team` we used above: + +```yaml +spec: + additionalMembers: + virtual-team: + - "flynch" + - "rdecker" + - "briggs" +``` ## Prepared databases with roles and default privileges diff --git a/e2e/run.sh b/e2e/run.sh index 0024a2569..0f9e6c170 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -8,7 +8,7 @@ IFS=$'\n\t' readonly cluster_name="postgres-operator-e2e-tests" readonly kubeconfig_path="/tmp/kind-config-${cluster_name}" -readonly spilo_image="registry.opensource.zalan.do/acid/spilo-12:1.6-p5" +readonly spilo_image="registry.opensource.zalan.do/acid/spilo-13:2.0-p1" readonly e2e_test_runner_image="registry.opensource.zalan.do/acid/postgres-operator-e2e-tests-runner:0.3" export GOPATH=${GOPATH-~/go} diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index fa889047c..892efa926 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -11,8 +11,8 @@ from kubernetes import client from tests.k8s_api import K8s from kubernetes.client.rest import ApiException -SPILO_CURRENT = "registry.opensource.zalan.do/acid/spilo-12:1.6-p5" -SPILO_LAZY = "registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p114" +SPILO_CURRENT = "registry.opensource.zalan.do/acid/spilo-13:2.0-p1" +SPILO_LAZY = "registry.opensource.zalan.do/acid/spilo-cdp-13:2.0-p145" def to_selector(labels): diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 2555ed450..1dcc20de7 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -36,7 +36,7 @@ spec: defaultRoles: true defaultUsers: false postgresql: - version: "12" + version: "13" parameters: # Expert section shared_buffers: "32MB" max_connections: "10" @@ -93,9 +93,9 @@ spec: encoding: "UTF8" locale: "en_US.UTF-8" data-checksums: "true" - pg_hba: - - hostssl all all 0.0.0.0/0 md5 - - host all all 0.0.0.0/0 md5 +# pg_hba: +# - hostssl all all 0.0.0.0/0 md5 +# - host all all 0.0.0.0/0 md5 # slots: # permanent_physical_1: # type: physical diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 293404e99..d203ef83e 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -31,7 +31,7 @@ data: # default_memory_request: 100Mi # delete_annotation_date_key: delete-date # delete_annotation_name_key: delete-clustername - docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p5 + docker_image: registry.opensource.zalan.do/acid/spilo-13:2.0-p1 # downscaler_annotations: "deployment-time,downscaler/*" # enable_admin_role_for_users: "true" # enable_crd_validation: "true" @@ -41,16 +41,16 @@ data: # enable_init_containers: "true" # enable_lazy_spilo_upgrade: "false" enable_master_load_balancer: "false" - # enable_pgversion_env_var: "false" + enable_pgversion_env_var: "true" # enable_pod_antiaffinity: "false" # enable_pod_disruption_budget: "true" # enable_postgres_team_crd: "false" # enable_postgres_team_crd_superusers: "false" enable_replica_load_balancer: "false" # enable_shm_volume: "true" - # enable_pgversion_env_var: "false" + enable_pgversion_env_var: "true" # enable_sidecars: "true" - enable_spilo_wal_path_compat: "false" + enable_spilo_wal_path_compat: "true" # enable_team_superuser: "false" enable_teams_api: "false" # etcd_host: "" @@ -122,4 +122,4 @@ data: # wal_gs_bucket: "" # wal_s3_bucket: "" watched_namespace: "*" # listen to all namespaces - workers: "8" + workers: "16" diff --git a/manifests/minimal-postgres-manifest.yaml b/manifests/minimal-postgres-manifest.yaml index 4dd6b7ee4..ff96e392b 100644 --- a/manifests/minimal-postgres-manifest.yaml +++ b/manifests/minimal-postgres-manifest.yaml @@ -18,4 +18,4 @@ spec: preparedDatabases: bar: {} postgresql: - version: "12" + version: "13" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 5ac955913..50405a8cc 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -315,6 +315,10 @@ spec: properties: logical_backup_docker_image: type: string + logical_backup_google_application_credentials: + type: string + logical_backup_provider: + type: string logical_backup_s3_access_key_id: type: string logical_backup_s3_bucket: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 7ab6c35a9..170fc50ff 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -6,7 +6,7 @@ configuration: docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p3 # enable_crd_validation: true # enable_lazy_spilo_upgrade: false - # enable_pgversion_env_var: false + enable_pgversion_env_var: true # enable_shm_volume: true enable_spilo_wal_path_compat: false etcd_host: "" diff --git a/mkdocs.yml b/mkdocs.yml index 34f55fac8..b8e8c3e04 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -13,4 +13,3 @@ nav: - Config parameters: 'reference/operator_parameters.md' - Manifest parameters: 'reference/cluster_manifest.md' - CLI options and environment: 'reference/command_line_and_environment.md' - - Google Summer of Code 2019: 'gsoc-2019/ideas.md' diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 852a2f961..3d4ff09bc 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -1291,6 +1291,12 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{ "logical_backup_docker_image": { Type: "string", }, + "logical_backup_google_application_credentials": { + Type: "string", + }, + "logical_backup_provider": { + Type: "string", + }, "logical_backup_s3_access_key_id": { Type: "string", }, diff --git a/pkg/cluster/connection_pooler_test.go b/pkg/cluster/connection_pooler_test.go index 39e0ba9ba..54be0f5bd 100644 --- a/pkg/cluster/connection_pooler_test.go +++ b/pkg/cluster/connection_pooler_test.go @@ -778,7 +778,7 @@ func TestConnectionPoolerDeploymentSpec(t *testing.T) { }, expected: nil, cluster: cluster, - check: testDeploymentOwnwerReference, + check: testDeploymentOwnerReference, }, { subTest: "selector", @@ -931,7 +931,7 @@ func TestConnectionPoolerServiceSpec(t *testing.T) { ConnectionPooler: &acidv1.ConnectionPooler{}, }, cluster: cluster, - check: testServiceOwnwerReference, + check: testServiceOwnerReference, }, { subTest: "selector", diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 8a5103cbe..ec2898d81 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -939,7 +939,7 @@ func testCustomPodTemplate(cluster *Cluster, podSpec *v1.PodTemplateSpec) error return nil } -func testDeploymentOwnwerReference(cluster *Cluster, deployment *appsv1.Deployment) error { +func testDeploymentOwnerReference(cluster *Cluster, deployment *appsv1.Deployment) error { owner := deployment.ObjectMeta.OwnerReferences[0] if owner.Name != cluster.Statefulset.ObjectMeta.Name { @@ -950,7 +950,7 @@ func testDeploymentOwnwerReference(cluster *Cluster, deployment *appsv1.Deployme return nil } -func testServiceOwnwerReference(cluster *Cluster, service *v1.Service, role PostgresRole) error { +func testServiceOwnerReference(cluster *Cluster, service *v1.Service, role PostgresRole) error { owner := service.ObjectMeta.OwnerReferences[0] if owner.Name != cluster.Statefulset.ObjectMeta.Name { diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 250705656..17f13351d 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -70,7 +70,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.WatchedNamespace = fromCRD.Kubernetes.WatchedNamespace result.PDBNameFormat = fromCRD.Kubernetes.PDBNameFormat result.EnablePodDisruptionBudget = util.CoalesceBool(fromCRD.Kubernetes.EnablePodDisruptionBudget, util.True()) - result.StorageResizeMode = util.Coalesce(fromCRD.Kubernetes.StorageResizeMode, "ebs") + result.StorageResizeMode = util.Coalesce(fromCRD.Kubernetes.StorageResizeMode, "pvc") result.EnableInitContainers = util.CoalesceBool(fromCRD.Kubernetes.EnableInitContainers, util.True()) result.EnableSidecars = util.CoalesceBool(fromCRD.Kubernetes.EnableSidecars, util.True()) result.SecretNameTemplate = fromCRD.Kubernetes.SecretNameTemplate diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index acc00b2e8..622dee0fb 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -182,7 +182,7 @@ type Config struct { CustomPodAnnotations map[string]string `name:"custom_pod_annotations"` EnablePodAntiAffinity bool `name:"enable_pod_antiaffinity" default:"false"` PodAntiAffinityTopologyKey string `name:"pod_antiaffinity_topology_key" default:"kubernetes.io/hostname"` - StorageResizeMode string `name:"storage_resize_mode" default:"ebs"` + StorageResizeMode string `name:"storage_resize_mode" default:"pvc"` EnableLoadBalancer *bool `name:"enable_load_balancer"` // deprecated and kept for backward compatibility ExternalTrafficPolicy string `name:"external_traffic_policy" default:"Cluster"` MasterDNSNameFormat StringTemplate `name:"master_dns_name_format" default:"{cluster}.{team}.{hostedzone}"` @@ -202,7 +202,7 @@ type Config struct { PostgresSuperuserTeams []string `name:"postgres_superuser_teams" default:""` SetMemoryRequestToLimit bool `name:"set_memory_request_to_limit" default:"false"` EnableLazySpiloUpgrade bool `name:"enable_lazy_spilo_upgrade" default:"false"` - EnablePgVersionEnvVar bool `name:"enable_pgversion_env_var" default:"false"` + EnablePgVersionEnvVar bool `name:"enable_pgversion_env_var" default:"true"` EnableSpiloWalPathCompat bool `name:"enable_spilo_wal_path_compat" default:"false"` } diff --git a/ui/manifests/deployment.yaml b/ui/manifests/deployment.yaml index 4161b4fc1..976da53f2 100644 --- a/ui/manifests/deployment.yaml +++ b/ui/manifests/deployment.yaml @@ -68,10 +68,8 @@ spec: "cost_core": 0.0575, "cost_memory": 0.014375, "postgresql_versions": [ + "13", "12", - "11", - "10", - "9.6", - "9.5" + "11" ] } diff --git a/ui/operator_ui/main.py b/ui/operator_ui/main.py index 893e45ef0..5fbb6d24e 100644 --- a/ui/operator_ui/main.py +++ b/ui/operator_ui/main.py @@ -303,7 +303,7 @@ DEFAULT_UI_CONFIG = { 'users_visible': True, 'databases_visible': True, 'resources_visible': True, - 'postgresql_versions': ['9.6', '10', '11'], + 'postgresql_versions': ['11','12','13'], 'dns_format_string': '{0}.{1}.{2}', 'pgui_link': '', 'static_network_whitelist': {}, From 07c4f52eded02dd140e0c45e032fcc22582e149d Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 18 Dec 2020 12:07:59 +0100 Subject: [PATCH 145/168] use pointer type for nodeAffinity (#1263) * use pointer type for nodeAffinity --- pkg/apis/acid.zalan.do/v1/postgresql_type.go | 2 +- pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go | 6 +++++- pkg/cluster/cluster.go | 6 +++--- pkg/cluster/connection_pooler.go | 2 +- pkg/cluster/k8sres.go | 2 +- pkg/cluster/k8sres_test.go | 2 +- 6 files changed, 12 insertions(+), 8 deletions(-) diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 0c87f96f8..bdae22a7c 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -61,7 +61,7 @@ type PostgresSpec struct { Databases map[string]string `json:"databases,omitempty"` PreparedDatabases map[string]PreparedDatabase `json:"preparedDatabases,omitempty"` SchedulerName *string `json:"schedulerName,omitempty"` - NodeAffinity v1.NodeAffinity `json:"nodeAffinity,omitempty"` + NodeAffinity *v1.NodeAffinity `json:"nodeAffinity,omitempty"` Tolerations []v1.Toleration `json:"tolerations,omitempty"` Sidecars []Sidecar `json:"sidecars,omitempty"` InitContainers []v1.Container `json:"initContainers,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 83cd18c40..2f4104ce9 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -633,7 +633,11 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { *out = new(string) **out = **in } - in.NodeAffinity.DeepCopyInto(&out.NodeAffinity) + if in.NodeAffinity != nil { + in, out := &in.NodeAffinity, &out.NodeAffinity + *out = new(corev1.NodeAffinity) + (*in).DeepCopyInto(*out) + } if in.Tolerations != nil { in, out := &in.Tolerations, &out.Tolerations *out = make([]corev1.Toleration, len(*in)) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 050aeb5b5..42515a7c0 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -113,9 +113,9 @@ func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgres return fmt.Sprintf("%s-%s", e.PodName, e.ResourceVersion), nil }) - password_encryption, ok := pgSpec.Spec.PostgresqlParam.Parameters["password_encryption"] + passwordEncryption, ok := pgSpec.Spec.PostgresqlParam.Parameters["password_encryption"] if !ok { - password_encryption = "md5" + passwordEncryption = "md5" } cluster := &Cluster{ @@ -128,7 +128,7 @@ func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgres Secrets: make(map[types.UID]*v1.Secret), Services: make(map[PostgresRole]*v1.Service), Endpoints: make(map[PostgresRole]*v1.Endpoints)}, - userSyncStrategy: users.DefaultUserSyncStrategy{PasswordEncryption: password_encryption}, + userSyncStrategy: users.DefaultUserSyncStrategy{PasswordEncryption: passwordEncryption}, deleteOptions: metav1.DeleteOptions{PropagationPolicy: &deletePropagationPolicy}, podEventsQueue: podEventsQueue, KubeClient: kubeClient, diff --git a/pkg/cluster/connection_pooler.go b/pkg/cluster/connection_pooler.go index 82b855bf2..1d1d609e4 100644 --- a/pkg/cluster/connection_pooler.go +++ b/pkg/cluster/connection_pooler.go @@ -22,7 +22,7 @@ import ( "github.com/zalando/postgres-operator/pkg/util/k8sutil" ) -// K8S objects that are belong to connection pooler +// ConnectionPoolerObjects K8s objects that are belong to connection pooler type ConnectionPoolerObjects struct { Deployment *appsv1.Deployment Service *v1.Service diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 1650d38d3..6b47b37f6 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1223,7 +1223,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef effectiveRunAsUser, effectiveRunAsGroup, effectiveFSGroup, - nodeAffinity(c.OpConfig.NodeReadinessLabel, &spec.NodeAffinity), + nodeAffinity(c.OpConfig.NodeReadinessLabel, spec.NodeAffinity), spec.SchedulerName, int64(c.OpConfig.PodTerminateGracePeriod.Seconds()), c.OpConfig.PodServiceAccountName, diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index ec2898d81..e880fcc3b 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -882,7 +882,7 @@ func TestNodeAffinity(t *testing.T) { Volume: acidv1.Volume{ Size: "1G", }, - NodeAffinity: *nodeAffinity, + NodeAffinity: nodeAffinity, } } From 102178409b794288906f7292481160ae1c6a3bfd Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 18 Dec 2020 13:10:35 +0100 Subject: [PATCH 146/168] bump tp v1.6.0 (#1265) * bump tp v1.6.0 * update logical-backup image * Using smaller image for e2e test. * fix env var name in docs * add postgresql-client-13 to logical backup image Co-authored-by: Jan Mussler --- .../postgres-operator-issue-template.md | 2 +- README.md | 16 +++--- charts/postgres-operator-ui/Chart.yaml | 4 +- charts/postgres-operator-ui/index.yaml | 52 +++++++++--------- .../postgres-operator-ui-1.4.0.tgz | Bin 3517 -> 0 bytes .../postgres-operator-ui-1.6.0.tgz | Bin 0 -> 3900 bytes .../templates/deployment.yaml | 6 +- charts/postgres-operator-ui/values.yaml | 2 +- charts/postgres-operator/Chart.yaml | 4 +- charts/postgres-operator/index.yaml | 48 ++++++++-------- .../postgres-operator-1.4.0.tgz | Bin 14223 -> 0 bytes .../postgres-operator-1.6.0.tgz | Bin 0 -> 18415 bytes charts/postgres-operator/values-crd.yaml | 6 +- charts/postgres-operator/values.yaml | 6 +- delivery.yaml | 4 +- docker/logical-backup/Dockerfile | 1 + docs/administrator.md | 2 +- e2e/run.sh | 2 +- e2e/tests/test_e2e.py | 6 +- manifests/complete-postgres-manifest.yaml | 2 +- manifests/configmap.yaml | 5 +- manifests/postgres-operator.yaml | 2 +- ...gresql-operator-default-configuration.yaml | 4 +- manifests/standby-manifest.yaml | 2 +- pkg/controller/operator_config.go | 2 +- pkg/util/config/config.go | 2 +- ui/manifests/deployment.yaml | 2 +- ui/run_local.sh | 6 +- 28 files changed, 91 insertions(+), 97 deletions(-) delete mode 100644 charts/postgres-operator-ui/postgres-operator-ui-1.4.0.tgz create mode 100644 charts/postgres-operator-ui/postgres-operator-ui-1.6.0.tgz delete mode 100644 charts/postgres-operator/postgres-operator-1.4.0.tgz create mode 100644 charts/postgres-operator/postgres-operator-1.6.0.tgz diff --git a/.github/ISSUE_TEMPLATE/postgres-operator-issue-template.md b/.github/ISSUE_TEMPLATE/postgres-operator-issue-template.md index ff7567d2d..a4dec9409 100644 --- a/.github/ISSUE_TEMPLATE/postgres-operator-issue-template.md +++ b/.github/ISSUE_TEMPLATE/postgres-operator-issue-template.md @@ -9,7 +9,7 @@ assignees: '' Please, answer some short questions which should help us to understand your problem / question better? -- **Which image of the operator are you using?** e.g. registry.opensource.zalan.do/acid/postgres-operator:v1.5.0 +- **Which image of the operator are you using?** e.g. registry.opensource.zalan.do/acid/postgres-operator:v1.6.0 - **Where do you run it - cloud or metal? Kubernetes or OpenShift?** [AWS K8s | GCP ... | Bare Metal K8s] - **Are you running Postgres Operator in production?** [yes | no] - **Type of issue?** [Bug report, question, feature request, etc.] diff --git a/README.md b/README.md index 465e726d4..7edb60d84 100644 --- a/README.md +++ b/README.md @@ -14,21 +14,21 @@ pipelines with no access to Kubernetes API directly, promoting infrastructure as ### Operator features * Rolling updates on Postgres cluster changes, incl. quick minor version updates -* Live volume resize without pod restarts (AWS EBS, PvC) +* Live volume resize without pod restarts (AWS EBS, PVC) * Database connection pooler with PGBouncer * Restore and cloning Postgres clusters (incl. major version upgrade) * Additionally logical backups to S3 bucket can be configured * Standby cluster from S3 WAL archive * Configurable for non-cloud environments * Basic credential and user management on K8s, eases application deployments +* Support for custom TLS certificates * UI to create and edit Postgres cluster manifests * Works well on Amazon AWS, Google Cloud, OpenShift and locally on Kind -* Support for custom TLS certificates * Base support for AWS EBS gp3 migration (iops, throughput pending) ### PostgreSQL features -* Supports PostgreSQL 13, starting from 9.6+ +* Supports PostgreSQL 13, starting from 9.5+ * Streaming replication cluster via Patroni * Point-In-Time-Recovery with [pg_basebackup](https://www.postgresql.org/docs/11/app-pgbasebackup.html) / @@ -59,13 +59,13 @@ If you are new to the operator, you can skip this and just start using the Postg The Postgres operator supports Postgres 13 with the new Spilo Image that includes also the recent Patroni version to support PG13 settings. More work on optimizing restarts and rolling upgrades is pending. -If you are already using the Postgres operator in older version with a Spilo 12 Docker Image you need to be aware of the changes for the backup path. -We introduce the major version into the backup path to smooth the major version upgrade that is now supported manually. +If you are already using the Postgres operator in older version with a Spilo 12 Docker image you need to be aware of the changes for the backup path. +We introduce the major version into the backup path to smoothen the [major version upgrade](docs/administrator.md#minor-and-major-version-upgrade) that is now supported manually. -The new operator configuration, sets a compatilibty flag *enable_spilo_wal_path_compat* to make Spilo look in current path but also old format paths for wal segments. -This comes at potential perf. costs, and should be disabled after a few days. +The new operator configuration can set a compatibility flag *enable_spilo_wal_path_compat* to make Spilo look for wal segments in the current path but also old format paths. +This comes at potential performance costs and should be disabled after a few days. -The new Spilo 13 image is: `registry.opensource.zalan.do/acid/spilo-13:2.0-p1` +The new Spilo 13 image is: `registry.opensource.zalan.do/acid/spilo-13:2.0-p2` The last Spilo 12 image is: `registry.opensource.zalan.do/acid/spilo-12:1.6-p5` diff --git a/charts/postgres-operator-ui/Chart.yaml b/charts/postgres-operator-ui/Chart.yaml index 13550d67e..9be6c84dd 100644 --- a/charts/postgres-operator-ui/Chart.yaml +++ b/charts/postgres-operator-ui/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 name: postgres-operator-ui -version: 1.5.0 -appVersion: 1.5.0 +version: 1.6.0 +appVersion: 1.6.0 home: https://github.com/zalando/postgres-operator description: Postgres Operator UI provides a graphical interface for a convenient database-as-a-service user experience keywords: diff --git a/charts/postgres-operator-ui/index.yaml b/charts/postgres-operator-ui/index.yaml index 5a7b42d80..7d1f5fc1c 100644 --- a/charts/postgres-operator-ui/index.yaml +++ b/charts/postgres-operator-ui/index.yaml @@ -1,9 +1,32 @@ apiVersion: v1 entries: postgres-operator-ui: + - apiVersion: v1 + appVersion: 1.6.0 + created: "2020-12-17T15:49:56.570324588+01:00" + description: Postgres Operator UI provides a graphical interface for a convenient + database-as-a-service user experience + digest: 9ce86d53b4e79dc405aea5fe2feadd163dfbbde43205782c20206ac0ba9d5e4d + home: https://github.com/zalando/postgres-operator + keywords: + - postgres + - operator + - ui + - cloud-native + - patroni + - spilo + maintainers: + - email: opensource@zalando.de + name: Zalando + name: postgres-operator-ui + sources: + - https://github.com/zalando/postgres-operator + urls: + - postgres-operator-ui-1.6.0.tgz + version: 1.6.0 - apiVersion: v1 appVersion: 1.5.0 - created: "2020-06-04T17:06:37.153522579+02:00" + created: "2020-12-17T15:49:56.569780943+01:00" description: Postgres Operator UI provides a graphical interface for a convenient database-as-a-service user experience digest: c91ea39e6d51d57f4048fb1b6ec53b40823f2690eb88e4e4f1a036367b9fdd61 @@ -24,29 +47,4 @@ entries: urls: - postgres-operator-ui-1.5.0.tgz version: 1.5.0 - - apiVersion: v1 - appVersion: 1.4.0 - created: "2020-06-04T17:06:37.15302073+02:00" - description: Postgres Operator UI provides a graphical interface for a convenient - database-as-a-service user experience - digest: 00e0eff7056d56467cd5c975657fbb76c8d01accd25a4b7aca81bc42aeac961d - home: https://github.com/zalando/postgres-operator - keywords: - - postgres - - operator - - ui - - cloud-native - - patroni - - spilo - maintainers: - - email: opensource@zalando.de - name: Zalando - - email: sk@sik-net.de - name: siku4 - name: postgres-operator-ui - sources: - - https://github.com/zalando/postgres-operator - urls: - - postgres-operator-ui-1.4.0.tgz - version: 1.4.0 -generated: "2020-06-04T17:06:37.152369987+02:00" +generated: "2020-12-17T15:49:56.569108956+01:00" diff --git a/charts/postgres-operator-ui/postgres-operator-ui-1.4.0.tgz b/charts/postgres-operator-ui/postgres-operator-ui-1.4.0.tgz deleted file mode 100644 index 8d1276dd16ab28ad5d176c810c86ad617c682359..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3517 zcmV;u4MOrCiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PH;Na@#nP`OT;3rEjZdvLQuE@<&)Tck6Y0Dc4RMmF;9}Z)?g0 zku3>h2;cyqY{$_(`xU?sk(6x7iN~4T3O^)}Xf%LEqr1^)A{9!HB`Qy(D2W!*%V_6l zLZtR@$&|f#iWCGvu-EU~{~!q3|G|ED|3%o_>-P6|_q$>EA_#YT-S7njPty9DlqpT* zi{Pu9B0;v=;nxFy# zV@c8pjR^zFHOdi*5k^7+fmrYva*A9#0144#NECWRdBjsFXEa906iUFG6eUF-qcagx zj9{X5s-jM3O!Xui`mvaHJ`qNEB05XF>s;aOwUCL591qG;$AhZegN!;J#7ty~$BCvh zG>}9~!A*fmDHG0=P_7B(C@mtK63QZjge#HB82?&yz)#Qt;KX$EfAUugaCP-pMXx-L zx(=r3Rn~X%{HmqrbJjMV|CQ^1Mp%aG$ppZf_21p?_uA`!cXxNY{+}c5 zz#ANqjA_Ux-7eQvOyN5_a513@s9EU$Jbd@FHxhD6w8jKRl%WB5gE1o#VMZh+Lxu{p zfFVLkl)}Ua1p&7rlb~^$GNMuWjw3N;G$uzP<2r)SadwQf7}0Si5i$yNO2+8e-^c-w zIHpR=+o;O2w?72+qt0;WPyEet4$X|lX(%z^{ZWNc)gT{i$}#@IkG8sA3Xo{Ua~ z9!ONkwd*)?NMh549XL-hrXz|9t|zD`DBSihhqB4ddedn}P?OdKe35onpA z;}l}=FpllO??Fr8yaBf!vB7gOK`U@Tj6hGYjYA6~Y<&xQG6F6%DAYiBQlK)EY>>SI zlH;KY#>frTFo8@_KDKyA8DqA*oX=9wH7C(136U(Q<5W2_Y(}S4+Z;fgW)bv)U}~SH zIJL9a3xePswNv_AhN=M^wt+(n+iw5(^rLc+&s0w6lXrvnXBUU37ZLnpLK#*n0%BwP zSWc;8B~w%>i7^5<$=A5MM3$l#B{Lf0 z=-bp(Ibxa8C{G3kV=APMz|9+IV5kW0=5{i7;3^xUgFv2Ohm;oo_&ZQ zqdwk{Y0A(Sa%_vhm;nI(GN-Aj?P;d+k{bG*9F3P_pOO;$ zw-)garj`1?lFpse_ZP?KzP{04)5bdczq{9O>Hqy8+~4Z|=SUX_C7W9PZWz7#0 zqQH=s^|Hq4L|3^68|LQrCg@b^k)<%1tlWQnq-p|xKzF;=xT z3x62iTEe_JJvXq%R#qv-Kl;we2xKlHb0o2_eg;+8nYpE{;hHjL4E)UOnTXkLna&rz z-HgtqVZI;_9aW?)3oG)LF!us%M1Q6hnEu0#CHiUeDX#UiB1?{?p;Z=rZM=PS2UAJ8 z9)bI3<^5T?Eo^zak72s|@-1d8jeb&W#2z!H&HdtOZE`Cj|3G=nvSd9|_&-*Y&_rwJ z1Rfehja7{_DEv-_O)a|?yq`lcGDNoN_-qyT-U+JnPo6IvmGIf@^qDa>+`qWsUb>H( zxo<6izj0cl|KofYCi_3RcXxHoAcrLO3m4FQ?$i$l zY!S)yL`eF{?$-UQ18cy~Lg$L|B!a?aa3&Z!Q`96OnnacB+qCqccv>akjJ@wbE=YX<{lYb-;C3wVJe=TS9zo5pRU8V2Xz@E|Voq)e20SXdGosp?(y$RlTZqy$@l! zyR%B>kF8@im;Xb`6UxV5pa#~-|6b7Y|8%=yzrU6L&yjv7`Tw_^S+;86N!5U0c!nc0 z{!&IgFfR_kvReN7Sy0*VSB$k)M>+oH;nI^#G+&%u+~-#OKE&%EKCO}e38qZkPLY32 zIM5pT-`{Wh|GGiH*WJqh=SX!C?;69YGYj29p1W_V*>@X~{NG#-*CgDUF4jCQR(7Ts z=OGL>;z30aIsgh8##+dno>OBR|6C*R6O#B#hFZ2z^sdGQCb33GpR}uA(%C#|BvZr9 zF-=_9fdP?ZiW(*Tg#TrM%6YK)B`x%Is>=o@h(-KZ*(tAJ8n>wHRwa-}xJ2NZK^2ym zedU)CE-;^WRuq=;Eij))OU0UuVLo4KW>AZ7S*o)LeyP(lKmV_dB_B^U%7KjzI0z2H z1_VGVg%+`35x5segL+pal7w;o#%}oVBS(K2v@( z%B0!v8up9Bv$w|=m#2sCj?V{&N5>mtZ$+fskN)`Z-T6H@KYOkB0RLKg1I4mmD)?m$ zwLY9&9=$*P?~}Kiw|=*(^+d$#l2Lx;Ms69vxg5ltqB~q>~ z^I*G6r6uKK*9ePn?%f4*|E@c~r&bYVt1f3$(IG=OkH%REJSAq2aLEKoE{B8}AuBhj z$BDii6OGs8w%(47?O9o&(0z|@saj#10A~v2Mt}<+*=2E*q10BWfnVqIV8A5d zW4s@-j1{dczgivpzHY;KcHr~Pw4Zjxlp{qTT$C3 zH;kc5(&gLcCHKn0t#WZRSaKkKiP!9jCN%$aWW}&dfdO&0a`_LIHk1EFyy6$Cfi?0! z*z0#&=RaY0Z~Ohnv!v$ve=8VtbtFJrO_T?}+q3>B)Hly{y1jD!`-V<*%!S0KXk*>_ z-|g-#egECve*f_-X$J;GYn0r6s-4d{Tu+e0FrzG?d<-dxugDlx^#!=fQftkCnjm8^ zW@4BhTT(uL1rnLPSi$ZRYwrn9oE_jew)=bdA(c3yH<;wEfj|A|!+XwdL2z5sbO}W=-*yYKkJPqJfw%NLVz$NZF!YQ{SY zL}t&LrS{tw<-GLOb?UtIhvdq6>Fa6gy!>Bh2R;%>MW*27&9QR)REl3Q){ajTB%K^i zieDXnrecv`=R2};R<3`_6TEr0`(MKT{?h#qy}j-J?^#kyu_M=#=B}nyoKeQtHtH1W zt#xmVd0X625P{nbx`7vTy>4(3?nm8T6zuy!(0ko|y>}4)B?zM+$PYYzkLzYE-i@xD zv5<6(N=Fcm@F02J>-O*<814<@(QY_8IOz4eqc|S+Uz6ZSYlV=+D+ElJOd$&#gl;3XyHq3-J zV-(ijb7NBTuA4@+_I)=C@cho3#q&MaZ~!#U+8=rUjj57JwnVXdBcA0uiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PK8iZ`(Mw@O;*DcCkCRLJhnx?+L{esksCE1D7Hj{SlgdY+~JUk>1&ntP%R46@_s63IPBw9!>qn-U3 zk=nl|bN0hSlpqL#VZU$x2SL#O9}I`xAHv>fI2?|GAPjy8!a*+_`~bm&^uDEJN)!1* z@YQ{_C-(;_B&F|ADk^vci_jryTD^w;&<~shRV-<$?WdE1Tj03h7I=FIsT2#EpaKF@ zNzxgO2?NSC$_a@PCPD&%SnvgMid;JY3DIOs6naE?#8W62G)Bl2O27{(N{T#2XC~$t z!A$E^MV-!+>RC4SV=?c1B#iJxbk=y+xy0)$Arloj9+b6?2UWcX8Ff5}naC246HOOr zAc>ZOn+lauCY(8;TocMsT0}S}ltl&!S0a-!{=FE0pP&Q4i5cd9=bxPXZIe_vd1H(6 zvnI7@F37{K|3>*=5SF2OFa@wh{(FOdzb*fR!IS(yO4)&zI3XF+kT1Fw*G){}J3DYb zqY9`I^ndTYdF@SvoD;1vfeB@30A6Cuh(uTrNy(U@0xe*SkP@XZF-k$ewa6rBoTrRv zRKDX#Oc{;IzR0+aAatA^qb(+Mnn{F=0-cj7I`(hm07#rtrR8;0J^3rq^sN~AiLijF zP*e*8N@ALcjVB#d2yN-?IB zYakqml;j%3Bp>QY#@HFg616fkDIQ^-DQ##mCp@zVVu?gss(_xM)gvk;u_dI$$x8KI zW@D5b4YjD~BqCOwSnwEAZ669%)v=wc&Q38f%BP9{F+Xsn-Mguhy8B5IMia9w@JVl^ohK^I{_`Nu` zt9%Pu0%r}l?T8JY^MZJ00SqBqIJ7X9#kZh`6W~IFLJfo`1uDbQ2HDR*ay(SQ6uDs$ zOdwN~Pc7a_#+a=ynYUE_O^!7dQ6wwsIMtJxHluT@?Gr$pW)XCQU~b>$IJd0Y4T3k+ zvhVv0RRcC`!-f{N-TlGQJLMo>sGQD+Z%&R+&-aebBly>hGOWxY#KzCFlz4kszDNX` zAWtwgf4Ke`bRqSI1}n2ZMtz0IZ4Yk`Es_fF-pQd4hg_l5wpD52>#I};h8aryiv!ii zy_CL{ECSbvK-5{6+-Nu04cuCbeU>SW@`%jwH5sF68|R-q)2nL*4L?NQjFp7#9&N?)DgJ5!EWf5yl3WZU`ra5 zbfF54M1m*6EP!nZG2w>oRUspFKIA3ngd$7oxSKTwt>yL-$Il=n&0h#ChQ)Q$k$IgYeXd~!#2iF5PcUv;RwFi~L{AkfkVx!cxn^qm6Eg9Im(-z~-73e<+4e!k&ovg()raxkui<@0Snf}e zG_}+4oqu7D-NI|rhnbRsF~%fQ2ujQm{#og)eDc=AQr2CzE`I*v2^%J5u1L_k~Tx=quSzDME;TTm}SX!rtp8N7NLpOt_j>Vi5jaKZBT4Z z8aB1;TJU}j#Y`l2-}%`l?yU<{=b!w!*rL~nf_ve9*2;Zr1^kW67W-es z&{8n`^6lU)_CFj92Tl9m4Tjy|$^Jh^xw)xZ2013NU+ju4mrng~$X1a|&xE8OZLscN z?pgnF=7f_eCf@kk;;XDmu_bwXL&}vVJjQ0Tqe2B&DyU z$WluR&+ckBcf4ymjjI!|_ja#hfpT0_)0B|4tjrz}lOgS7#; z?xtRoR(ETNuPowwVJn#8E{yABNmI3sDZ!M9>p60*a?F}&oMcR){w(a9Mp;F{cVW7@ zu}0_nvN`wH|6|G%%BNpo2Da({UeMbA>2||@_euXhM){re|1UYSJeh$9H3Nd-DNfA% zOPzGbx;OyqX8G$^L1p9LFxJ)_<@lTDZ4a{0e030W+qUBOsZRfJWsCk_8O^WA2HB$j z2fabi_W%36r~SW2DRmWZ{HB#Qe_1VkLyP~(b#P6)Z5d*1x5ets6yrR7#bto%y zA;VY;nbUJ_{NmR&0zV*$zhtUq55?$eTwn@o4D>;V`Xz(Si$*e!z%`MP3p;Q^B$=Z| zNk8I$S)g(q?DLuh`Z}{^N^7sRon^BET+hEL5r7u|CUbT>j!!^sZU*c&`z`EY{$Z6F zv|6GvyT>gi;Z`-pcp`8Ob?Y7bWttT%mz@ojWs(&vm(g0aCR120*Sa~WC0VVtS!H0= z7dFqRsLdsxPc_RtL18!84I2;usT5kof<@q-@1N9%B9SDNqf#eQjIoJ%rgi!Xb<{>i z^eh6mGeg4k%x%|h9s}i6Q^H;%BiCmb3!bP51`QBR=O{8=)rO5>)?&?7F$>;elvPI7 zC*TE(gxw|;kuTabEO=t?Ff5SY>vn(6^DtKFq=eCrnYQAb$oJgx_5qjda-wM zcn8kf(j#9eznT8i9QP*d`>)@gogbWD9PPb1xOlzy%faV%)Xo~Zwae3kv*Wj?`v+$i z?+(une|ddy8y~wTdw8*beDsgQSNHGzW>fEph}8w7{L+owGC6WhNBc&s^$ZQLu^Y%c z)|b^C_vejBxw^yS$}Uk;1Mn7Wu-ud8X4w;5ji|cEJQm zF2;lzD<<#LjuU+`B^s~Db-f>(H?^@r;aq%vPZgWq2RKtG?*+KpfV?QqiIf{{WGnzT z>}{m9sp;NY6RtC3n;lJgiV<%sTjIYm?e4373a~Bzqt^@8@;}1iQ~dWBrR}a3>s{n% zuFmsKG;>s(>EHX5dDUDkVw;#6y=Wgi`!?OQcKWYyzCgJz<|&Z~?*D%g`eEM-T=4fv zO2(8?O;P#(%*I$muKp{k%fn!=wQb2!UkQ0h`E<2iIYv#wW=cT434YMXt+Hyu=FW8< z93SR%+PuHHBG%KcrD!IEAZ=l z9SoQxe2TY2mZ`$Eu&iohzt>$w&rTdZ!TU^Tydx&7ClQ+_tM|1DHQE*jfOn0&F{m=l z+4Nza)`w=j`Dy}r=5zI0>&4bI-D0nZg6y1I4)d7Am&lWL*i^{+(Jd0P0W=Ti3}~r3 zYLE@uY^K?_)~kP%vc>+FO8WP-|6xC9|NeJ03VKiW|1nDI_rHx3ou_@@2i*6q?Ou7s zMo}%2Z#YAhrprV7wVkcP$`PKxkNAEKhy8_-TmGQPT}$=ztDgx~9hF};j%giy&91HX z*v|J)*0#03WH0xDSSLjIxCCDWoL6BvVw?4G=0ztv3T|hK#|KiE(~(P5Ja^kjU)C zDmF^2eJ4C|c7Ws5#`o}JDse(TV3KbQ{N*Pfjybyq!EH-3BuG&Lqa1z5e|dIsriDaj z2lmB$E;zi~KZAry<@i&oJN9ore8(SulpXuG{4kq#%%AdAEqG^z$i%E!YJdBroM*ne zN}Xr^m|Qx~d_7N{XaD2uz&j$T$P^sDJW!6GO7R=U+VN?Eq?5x*@tfl>R4fwg{DFLm zjq*=#8D)NTqfKGnYWGGB>`f}=RUJtL zZa3%#UNH26{&_fxf??Df`Qe~9=ynIA=YI>jQ4r)OIlsqsQx@;NuAH)vbc#wx5I+x* z^q%7|8HCATH0cNZ-N`r{4>2C}#{FQo+naQU&jTFq?qVPNlaK_xVQ)AZKc6JYFsI8G z++k&!t?scZsRa$JRS|>-6RM}94FlY>5oSwCcEJ{?so~2`1vqQ{v5noHB*;= zSM<{3sN3L*US|F8gt7aZ8Qbt{bv@Co+}iC8pFiInmF)Vh&CFAIDo^D*EdLt-0RR6v K46+OWRsaC)$fq&@ literal 0 HcmV?d00001 diff --git a/charts/postgres-operator-ui/templates/deployment.yaml b/charts/postgres-operator-ui/templates/deployment.yaml index 5733a1c35..29bf2e670 100644 --- a/charts/postgres-operator-ui/templates/deployment.yaml +++ b/charts/postgres-operator-ui/templates/deployment.yaml @@ -68,10 +68,8 @@ spec: "resources_visible": true, "users_visible": true, "postgresql_versions": [ + "13", "12", - "11", - "10", - "9.6", - "9.5" + "11" ] } diff --git a/charts/postgres-operator-ui/values.yaml b/charts/postgres-operator-ui/values.yaml index 55c1d2452..4dc6388ac 100644 --- a/charts/postgres-operator-ui/values.yaml +++ b/charts/postgres-operator-ui/values.yaml @@ -8,7 +8,7 @@ replicaCount: 1 image: registry: registry.opensource.zalan.do repository: acid/postgres-operator-ui - tag: v1.5.0-dirty + tag: v1.6.0 pullPolicy: "IfNotPresent" # Optionally specify an array of imagePullSecrets. diff --git a/charts/postgres-operator/Chart.yaml b/charts/postgres-operator/Chart.yaml index cd9f75586..e5a66b6e3 100644 --- a/charts/postgres-operator/Chart.yaml +++ b/charts/postgres-operator/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 name: postgres-operator -version: 1.5.0 -appVersion: 1.5.0 +version: 1.6.0 +appVersion: 1.6.0 home: https://github.com/zalando/postgres-operator description: Postgres Operator creates and manages PostgreSQL clusters running in Kubernetes keywords: diff --git a/charts/postgres-operator/index.yaml b/charts/postgres-operator/index.yaml index 3c62625a1..6b64fd705 100644 --- a/charts/postgres-operator/index.yaml +++ b/charts/postgres-operator/index.yaml @@ -1,9 +1,31 @@ apiVersion: v1 entries: postgres-operator: + - apiVersion: v1 + appVersion: 1.6.0 + created: "2020-12-17T16:16:25.639708821+01:00" + description: Postgres Operator creates and manages PostgreSQL clusters running + in Kubernetes + digest: 2f5f527bae0a22b02f2f7b1e2352665cecf489a990e18212444fa34450b97604 + home: https://github.com/zalando/postgres-operator + keywords: + - postgres + - operator + - cloud-native + - patroni + - spilo + maintainers: + - email: opensource@zalando.de + name: Zalando + name: postgres-operator + sources: + - https://github.com/zalando/postgres-operator + urls: + - postgres-operator-1.6.0.tgz + version: 1.6.0 - apiVersion: v1 appVersion: 1.5.0 - created: "2020-06-04T17:06:49.41741489+02:00" + created: "2020-12-17T16:16:25.637262877+01:00" description: Postgres Operator creates and manages PostgreSQL clusters running in Kubernetes digest: 198351d5db52e65cdf383d6f3e1745d91ac1e2a01121f8476f8b1be728b09531 @@ -23,26 +45,4 @@ entries: urls: - postgres-operator-1.5.0.tgz version: 1.5.0 - - apiVersion: v1 - appVersion: 1.4.0 - created: "2020-06-04T17:06:49.416001109+02:00" - description: Postgres Operator creates and manages PostgreSQL clusters running - in Kubernetes - digest: f8b90fecfc3cb825b94ed17edd9d5cefc36ae61801d4568597b4a79bcd73b2e9 - home: https://github.com/zalando/postgres-operator - keywords: - - postgres - - operator - - cloud-native - - patroni - - spilo - maintainers: - - email: opensource@zalando.de - name: Zalando - name: postgres-operator - sources: - - https://github.com/zalando/postgres-operator - urls: - - postgres-operator-1.4.0.tgz - version: 1.4.0 -generated: "2020-06-04T17:06:49.414521538+02:00" +generated: "2020-12-17T16:16:25.635647131+01:00" diff --git a/charts/postgres-operator/postgres-operator-1.4.0.tgz b/charts/postgres-operator/postgres-operator-1.4.0.tgz deleted file mode 100644 index 88a187374ae3f834013d39371358266e7edc2815..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14223 zcmZYF18*i!*Dm1NwrxJOJ#DAP)V6Kgp4zr;+cu}3+QxI{{oZ`Z$vHn@uk7sXWM$p! zB94Xu$)T>-0|CGCwRFelYW({(tsf{kBk&uIOP!H(ZzAJ2Wo(|kji$OvB3shY9s7n{ zSUBMRqSsT&Sg z>hm-(9{RmmtLAV>ED$(I4P@@N$%}A%X;SjAm zOHN;Q2n60VssQ`;B7P$u!h?vXPoM7kkac^?MGz+z;Lqxot7k#6wNaNkc1L*4oD4^o zP1Li%na%9k5*t9xi02Ah-|QUgqVOu!nZZ6`^`5_AHRDmlakLrVtd5Af5kNU3_lm>M zAzQI+WsTiJ--_ZeeLj0@_!B`&aXSIYw(Pi>Nch@Nhx#6h@PKq=BOI{1YB?POQJFtfgGgfDv_TCd2z4N$dyr=6l8EJGeNnj6xj3EUo@ikGm*b$Em^ai4I!PU zTCoDb<&GRj0(a0;k4T%2TL?CZw<*)wL$F)$R0MP8g4ag_BfJg{&3zt6i8=*=m7vnb z_D;s(=v-ke{6%^8SCEK*h9Pxq^&JW*C;Z^VuBRFLX>@Z~N9>)9a@Io51 z2Amjd_EhlU>F*m)o)p&sv7~)@d3rtH6(31g-bfrE2HPD1k2?c2E>LanJo(t+k=kWu z4vr&$92pk(8wia0a87pk9`be(H2A$N&&E@*`M6Rx7H4x|JxlOg8wdPXSfoPG?N~`9 zH?Qls@-aXojnODcYHj*4*YJo9DziA6h`pZ&kyHlB?A$vd1TqNJFA{x_DUuHw5{QH$ zZ?2<_kg-qaei_r_dyB;&5?bRHz8<&3buAZ|+v#tTlL>@~I14W%aPVa;(gLdjx=>Rw z4coh(bZCBdMcumxX|Ca8m^lZmf+ATWGLQ|} zK}@gU;i;pOQE)%L1Y3jy17U$MqBvWPrg|d}K`a=$djfZp@M)SK**WhKhEBf&r65sJ z5qjjugdnV-YEe;7mAX+hGJUzI=c54uzvEnP9Cak5aKP>b(fDXMZj>ldpTx9Q&%m&% zLlA#=J`ID(U6gzo|L8=B?Rg{W3e_jj!T@OY2WRk=cPwD^Csphst6UK=?RT*0m<3A0 zJ|a)ej~m_uz|)`b>40f4yM-x+!{N<04^puxsESDJIgYsKg0H3aT7)3@1y%;x?y-G@#G`s^k@9Kf*A4#(T_wZ z?=>0Er>Jr~Z;*&gIJ1S)+T?Nmy4V}aJP?3h=i@RD#)^^(04(qxn z3)&p80TF_3rPVvn7ugb6e7H2f^&Q^}Pb|zhlVA@w*TOJj1I&}$cf1wAQyfc&!6pf3 z@470N#;!nD(9AeobEnJJ!zP12^FN0O5E;KwN`nyK$Q>(^NgFz;8!2EaV7I_Noqtjq z!B2w#JuhYy>_vvcNii8gF_vZrox5pde?sCSpobq{RQ@?7(LcWj; zx(=`v6ypSs$3ZX>=Q~u2(gj)t)s#3|1lLPRID!QeLPfwm(6P5;3EP^$p_be1gJKc^ zrbXQDzLwe=LmBC&mu`Yx#A1%_Bg))lFLA==vW7Mv68y#t0aMjj^Znf{4jhq@4C<{W zxg8JM$RTVjwz*vPCxui2{B+V1;}m0&$cwgQs<=mQoE*VN{jt;!Fcm^~jy58-Jhev` z%~1|U<|HIgXl@EF!i_Yzzi)DDW_}T<7T;<^!EXLcctt|e38~h;=gId^pF$Z0O>G*g zMvX?krH%^tS!2LRsV2`R=GBvQ&i| zZB(fR7`e9D2S+yIfK{9P#$*_D?tvMW(Sm@MH7oDt6El-mG&DOrpN9iV@_ghm(hNVw zi1G}Npg>3WBU8zMB7|+yRlA1dXwQMGT7$GCD4909EH9%Z9ZW8~1FWl{LL;^L9z9?W z^P%$FjZX{m@^u2x3Op!{j8yRJUBDPgbHq%u4T9Hk-P{$);aTl|7Jx8akUdFQ_o8k>9OH`7vYOT^2O9u)>`4(#>)VC2a7lCh})!K^3S7#oe zcVLyHC4AES%qRXqLqS|1b*KO3o;K5$)9YKrZcXLCNF6q%f~&|cd~Zc03+qK9f~-70 zqWW9XxXEo&YdQx)d{d+sOtkVjjAqA2{OPGoofu$^aC*hb48!oL`&jrLK*$U5Gl+B z2ckha8TqI(crDoR$GjoQR!uE|j3NZ+SHgS0aI=G&5B|Qlg_K8I?u~r@0xmpUSf=0{ z9hz20cat#+>+d6dr}JbcYRE5$kP-eT_Ixshmh3=3sQJDc`i;rUo#(G+`H=J8#@&T! zqbAlCjQX!atznL3HiN^pS~oExYjbs>#+Ed&pKFP$#`FEACEfrnicO8gRg%F33z$W; zHW12-rypIEGrX1^wxP+*rxbdUHLKclAuaGun&`i8lZ4CM{wn4f{`PD#p^~+N?1UH$ zSdp^Q4{VYV+rb&0{e8(5m;!Ty#KVUMLXc2v`2>q7lefxo-a9Z||GRj$w)v zfK?{=F<4(BQ*vRpVejAe-l-@-%ehy0SM%yjD{b42JL$jUs*%)1m&!3YuB@;bekuWw zcTm~bHtnp~;5ZWfod*1kSF}4YJzAQRx&QYoL2gr;5qI=LPwRmrfXSxa4Za-MpSN*V> zU>j~YsSp?9i_FUzh%mILr{%(85qBzlFCdTb@4)eo5WOZOD%K7V`F&{fSXc^IfZl*p z*XnXG;>1#6-*I8lpXa;v z2D{kfIXXnYJ6cItXTkA;bvl(*@iKFfHUaemYS+{}kJxBqvNoCO6)TuJZ4Rw&IW>4~ z0seF+_W;BXuJ7#ITpI1q_d@^Ru)5`JgKo@=eAfKi;WGGm$pS?+zu(+mG$W$RMOUF* z{dEBo533Jq0;gk;C7h2sHG8z25Fkd%+$e2^V1G01_}<(|=#vUJ=>nc3&nXPRQSkA{ zJ&ar`Ej?k_A|S$6A$*7xU7>X6?zIj+7)~_5BuqPpfj^HN-kpiR4C-cUk1TR`#cu}H ziy-3&x_8?tH^T*i=sXJV33w#Idnv}BGPFU^$jpeWFDnqdY%vj)aFx8I1| z))p|6L{KEKlwfF|ILS{IuwraWvOg=_KNfY$Ej;T3pz`0STK8PK6`>{G46m~TASxWOGgooTO{qlk(+{UE#AXbKZ(GVB1yh=NM+ibuo1 zWe#jDSa@Oy=rs9N3T4sFjS1Tu+!4X{1ra{7iTQv`kK=F0CpC+F8*e4iL`$di;EhM$Dy-bgrUIttw>QSHoUBrrBL?h=| z5yXZ-ilKzP?@c}}=Ny_3oXDBRGSDQY9X|kP5*)wYUsHLZOK?t+kM zTSth2b>?v~K;)<1(O?hHl#+FWD22C4EXPPC+S^z>xKTrU$15z>!6EU}gcDn^jf%KO zfejd;LrNf<7LwAkX}s zQ)FDr46XPcs5?{t`&I)g4J0uXRHcnli}8-}Tk^=7I)A+2k9s9P(j`0S{c#Vc9{479 z)iSlI>b^5$bp3L_+q_lLir;Bq5Au0EJrB@ia`TI#*^p?x4!Jro+1X@4y{=hOfc2Ra zA)(xKzBiLp@fT|DpuGFJA{>~OML1FepB1xo6$}?HSCh?eufo71H>H%`gdgoj$D%6+ z(gdZq z(51NZ6!utcsnjD5xutc5URdxOEFEki)8H|dkDop#Nc$|Dz8(gO;f!{cETV=e!BGdi zXCx%7KW-a6&r>$-&CYidOP!}!q4!+yK(Ep_D=Y~X^G{AUK#(^nT}dSaUC{>H1p?4o zpoht_#L1W$2uE}&SPY|pbTDBOxs`cv_^1jd&;6CX6jbmk((sfotP!-0h+Q>Nj z+v6Glj#(a@X(K$9$EWmS(rdIt>nP!Zh4}tlkq?M5wy(boJW?WZK6$Vzc)vMtg|w;| zMVSLu8u%ZsXgZH<;Yb4JT{ar^G5SP^v0y#<%paq5{VP6xd4|O{UioNVk?^Z#U3lTg zwI*4BV@;Pb4jw3b*|8#Q8#FjvGj)1z z?tjGqRg3XCnD=&c9`WzOUvq7%qk%UB?oM}oV>?AfWv_ckn4R>TTI>5VWx1{)uVR$qvPqDDF9dl0i9l?5>KgushpHWHeA65680$-8na# z+Ppv`p*-P!fak9O*Qd_(ubZsCkV|Wk6^bOf+t-u=I4=5@7^{IvvkSjIfki%oel~ve zc0HQ$>_jg0`*!^=I2!Q_p#pcd4-cn!sg=4u8VNZmOC=I6deu}@nu~3EJyA8y1ma$t zul=}<2n(-~$aw}6?sSQN(JS>HesfLK&_qGRVWr-AgQ_VV zybb%SLH`A@e_`BX@1BnzBL7o5AUTF#%5jE&2Tw$C`1DR=8OYZ7%~pAu@7T4asI&4y zKzSmT&6zt_t?6PpT}`y_YmbDL*Azy82krIR1hC{Brh7z$6GtMMt0r}=B%z&w(Wu%j zH&^M^<6)-SxOMkrMLmINTT<*Q_gBY$67j8q{n#XUiGR|h1eV}~=HRB{Dz#m0^mE+z z`T95|rvP(i9lM?=fW(L3Hp2&e^jmFr`9!AF702wA>ZK ze8%Zml+>&osM;-35IIq5b|P|0o3h%g*LLcX?s9y?BC>KBNqdFRD!hi`V?tLUgK5x_ ztrWL)Ldw6;#rgKk=tJUaO$_DQ6Gq_oSd>DOjp3-K`1e3>LTLRVU_A0x$HQKZ+Q->< z_}Cji5pXQE@@bisL;e1QS}G7fdKltX+o6n%(6dY2`+Tu}9!y~{BJ2|Pn?PH6ph4-| z9o@Ye&97Q(l6!|(X5D~>0i6nptfPUym}#!u7zNyucHxga7A1N8t@0=XZ5uZVfAM)i zgTLyA)Ji1SH3L8x&dq=k!yVSo+*D)+~JmttBN4= zaCp6NniJUe@Joj0{0{aD(RGlOHcvY~1N?bhR(#;&iJ4FYbWjLnv!zg`H)H+52@Tf{ z@^!fuXzEnblZl|>+1tG zeaf{ur;|Pr#kLjWPjyGg7q+yL;R*gjEtu)H$=L?1`HJQHXbmIl%7!Nzr}k7%Yhbw6 z63{CT@yefPp{oOQ_06Ae%oE702l8S96dFB+-vBaa=;dyMAn!v3+zop)XU{sT`tNoK^XD@$2>No}>Olbm#2nql`GvFkIDQ=4D*r_5>WeF3Crh z^`ed!$W%jW_rQ721n>N{TcBE*Yj!+Pkk=;)=##9V1`NmLTJ|qi18faw0nv%_{HVU! zb;yeT`ZDABi*uPArVt~D6P^{nF)B&xGDQhhfRTumvkJB`OzD97caPlKziPI8FJ%?n zj6tqe1BZsxNWVXhj%dFxlmts9!u76~GlkZg@)j<>Z8@5ERJ0JjPcEL$7;Kh{F3ibY z1!Dji1Y1I;JSX{A}D8(l) zuEiQaYZQAdFSy1F9Gpo^qsxNozQ)uWdk#VBF~#d(U9`loD(>KvJ=%V|`dY0z2X1st z_4WF3czAnpwDbFV7wp0J1~__0_gU3cLe=d)+3V@*KSJNsGSw>B@9HAsgc9$MUAU(O zyE6kX4lYkX?*MDR%%&=S)}K9>QLVdGCB8pQh%5$;G?R|B!nJT|kc;lM=s+p$b?8CS zI!ARBH@@{^1bCiz7WdwD1~p{(ko&s{+Lj(^^fXY=(F<-6-8V@C>r%b8S{fvPHyDpV z`LZS7Mj-KVlRRGe4|{Ve#BAL>?CpUk*%;b%R98#MWKj<%%p1E1lbe){MRSH%rmAx* zp3^&0b?Li9(7I`n`W*(l!<5wYzgv-itv*y2JKnu6!R9seBfC4cwAE&I-8xH%bq4z{ zAcGwM1Nb;xFHzPI)9|xuhKlFZX^I5BaxP-w9eA9+v1?dyw-~lARfia(RU#@pe<<@| zcw2N>?tQ)Ts-7u;$L1-YX%dS)Z6K>xz}EXaArIitug|$6A8B`&k^ry!kDa}n@HWf8 z8rf^=6#au5&;X;cJEQ}=EmjeN7_dHSyV=^mQnFg>DVhuyyxSv;&cKb2%^x3+p7nE3 zpZZYnH%w6f3KO|+y*=cO<&=H&5gjM(0B3X&l!;(7vC-*ZZfRE^z6Vbhj0vJI`U1s{ zVSkTae1FY@%IJV1i~Eq46yuHKFK!e9)hMa+5saGJoYT%&xpH@ zp)XrSXyjxhnEJ9loks#has`iUv!WrIHZ+&yzIJmgV$a(-<5OAK1$@tf?b$F|pLM}u z9^qv#x37PKlm>$XT!fs8R=IDG-#x5(1GwSM*g$TVH1)ala8~fpne7($V*F}YJ_T$R z_s$1g7Ki(m=l;6w%7RW<51$$6UTsOZv<38qKYPFO{up!PT=rXH z2^YMpX-MPrlA9fZB>H5Z^gn6*F$bs$dxeWz5`TttqTMZ#c(lWq`dSn-dWN_Ls zkOF62oiLTb(%Ja$`MBgYo`vK1R@3=b1nz0u+x2jwHPc@@G*a7X{J5!^W=uCDSo}A%rl%HzX#_IpCE1^Zt6Sul z(Mad2&-E*9=NBkj=6?`?1agaq+Xg&6ZV(at3uVoa+~EdZ;^rYYMR&M=wnAUsddF>l z<3Jp_49pViO~<7STp} zZ8%JZ5fLfGG##&rkX3HRGmXoUwTtNE=gt%xIeATW9B3X^SGn~%L9ss3T`~Nt3>;KV zU|hkf_Za}wuacRj8eiH3y609ezT$DD=+z(Vn6p8=xT2^xrIRKhu9qxe*m4&K_xjgGp7AJ$DfTO0 z*R48l z52@EZEjisSDB}u~yMME{{QT{8EcSMA&djBeg{Y#n!B?F-kH!j`a-cPSk8;9TX^t3U zh%DM*Qmp|r`~%b5(qK~nq=d^8;gv2vr$9=uNvyB&Er#YnAsIGuJr$(;ZFyw-Y*x#v zb!c8MwL5gONw-wXOl9PE4rE)c{8g7NGJw+||7J?74XKwMR6VrPW~(ZcZMxWfr_(R) z9<)UK4|Edc@$eR8CSTvvV1V)}f{Vtde8PH`T)^u=c1=ODWzM;Aq!$XWT|`F@fWxh) z5`${{-gk`|OPE3xaB2{;kj*D=MDh@QHJUwT)B2d&4-MMp+;$&Oo^`*Q`auS31Nw*u zc!WwqVQ7U@bzr11enV_Lm2CwL%8eS}Hqw_jy>hyeRaLNu*vvgFE11%cuekG7o+atw zGR`urjw;w{r>;`r$w)Ey7q}j8es9jgwt&u~?|trl>F3;-<^8Gq(#P-PCEl9r%tQY6 zxh=C-Yo$kwleE*+;zM3O^Tqz=eQlBe>7A17)6v=K_WBsI-=$jTUst@0Le^@DmxEtF zS<8!MuWYV9uEEA)QbW@Y=EG0E(`{i>&b~9^Q^~r4j11ukCV6?N>FN2wVQ?7-#a0hW zc+-VUhs)iSp9BxZRMpaBISo~Sb>k0FgrZ{jm=g%{E|6}|JXo?oc@>)yoI0pwJRnsz z?pq+0o$cwuY^rbQOw4Vj;-Q&RLra_EV+u2)Qn^yYs}5=&fd9P(9;`QsalEA_&b-{8 zWSP`fD|hDM8uesry=QppUYQfW#$464m!c4N{1ZpeIhE8c<&~jslHC+hOYK>UJ$M8Wgy2c}ezU@-*g#i8CWuyY`c_ zJ1dTg-rtv45;ddl+9C08{y+@=wl)9`*I8u>o1P)`>5>5)XbtUI`>HYw#KX};i6&^0*YUF+;2 z*zxmg(w=Hm&*;$w$j+d_hk^G@#_p#5ryE&|0SW{~P*$pSghk?PAWp7DXny|~KsnCL zBG%%7Me)cZq5@s7hn&Mj36j+a9{xy^@*x4jhuvo3CEd1SP8#WT=f3~Ix>+{G z+e4sOrZWSIkUmq~MmKW3QOKlZ_gAVZ);u+Te6Ae9AB~!>u$9UBjfsMbbx{n zu>h-ptjo=bo?1K1D!00`6)~<;g{R3^`Rp?hiL(l z;ML&Jti7k4|54f{e@(s46g82MnuHgQShJv>t9REGk>G_l-vRM352A3uhcN;QU|0l$ z+5MIg6k%Iv!?KS?%?FqZMTN2wz1PcE|E&EZCU3K^c~jW*{e}59dXnU;#h>r>nCjb> zxbIG^c;nh0y2rOr!z=ae?yTJvM6Jr^c<$q)@AKx zpk)F77Clp8QKGMBlm$dA$!&pwVbv;2pkjR6nS4m$2h_Q9$0T)7Rd?GxDP^}cd)w{0 zwlaoT?{fDcM|`_HFQt|R-Y;Bno5yv*?4T0y`#{j@?sGF!_K-RJ09Lbd0^`H$>=!lq zjEtQ|f;w-)C~{dM12qCV{h*#W+qaCSuPbc6=z4ed{PCxSb||jP@^nv^X4u~z*XBQ# z+YS}JOpruFOb4kk(EPtwl$0U+{Iu?nDAv*f;c-sG>hkH+;&(G2lr%{j{tO=>H)96E z590du_*RaX00=i%u7n=V^pBuN07K04JJP~i0a|5$M(7EiRkIBIP#9>U9KyfYxRBB= zioPl<=Pr?d&VldcA_Tn|*m(OnbWO6-fG78y0qt6xA{oLR zrUcUHC#Hl^PX2)^)-2t>a+);6bfwido1ou}o$1rS?UuxJ!#S{mz(sfaNbOl5TK_c1 z%zXt-;B$2SzKnDR_?WdapSZ>F(HM+3Xd^Jcqzj;nL|`eXMoO2pGRwUYbIh84kSAg0BMGCo4GEjRZJ;Ed6d9f77-@+T z)~sYQH)UfH1}6&pc7Vn_uN{L zJiO(!e7Ckc)6>+IxfR}6borW%nLGLbMyX0GxjSlAb}MUw)b4?KY|sW-q^=Abtjt!> zOb>27TD}E6j<)KBWg7@eXzP7+JO72}DTpVS1D?UPpZgHV8jO#)7PA#OoEd-1hCU!j zs#HLvb5hQ!=Dxr**P(9ZtBnMe+(L6PXOl@3IZ5LckT3l)AV2@B6Ks|&+=5a?m3m-D zwUCJlo8*}WmmC7X%A6ZnWqxpy;qfHf?Mt=Gg~6I)U@HU{qQwGFAfJ8s%7ikmX2`Vd z851B7fZ%WUXO8~$0qgto&(~cCX`raJiupGTMHR(u(zZ`xmV(oRl%3spNa?Og zrvNq?=9iR>VB}042`6!k(Msr+{?L+&>%c6ry>k###H%$>b>j(*p6tvx0cr>ge|@3L zz$U?dUvk}5JI7d?^Lg2n^&@3v!)vS9Xi%-f66@MCRd7xRHPinrZp@UiW_6ctkYbh9 zJ#<1oMV$w;ff^eK;am281zk7IQ_^UB9fX3sgMBt->ToweNa>i87)H6wX=wVpYM!ub ze=>;n_*pjrYF66cWP=X6;RR)O+?E-9JM1OZp0UQHwByhCyKBJ0P_FGliOpNwm~t&B@Nsc>t^flRG)6n^T(R~dx`z&!1-p{joMn; z3ktG~ey*;5_O4ELU$up&>CTipy|uG0e|THr@rAoRlUl2;rhbbatjO+9_4n(iV_-+S zpYN}Q`^wCJ=T}Bh0$B-{(d>77ynHnm^jvGE+*h7rKGbu@FE2@O=>fC?WK$~`3eC~J1 z)w5evIe(yxn|QQD=02Cq0Ws$M4Ks(4UxXe_eO8bBd2C!C-t>_acz4Pk^yBX)DBW1) zNwH~y=O~WpNCq~eL;aTR2spVLC8hc2sd;PUr>tD76=^V<(atviIV z+t-nN|CRuD@!8iC3m(gMJi4cEw+`&*tqO4xH4PW-%t**vv$vsN2gQd3TRHo8r0&jm z2Hhb}zht{TG7<2L3|5Q9;N<5S_{JnSM|4r>5ZMK!N&L+wjRL$9xL32uoK+!_(i8i} zElO@Mw#=Zwf)>E%xG_hveM$X)w{tXWCl?8R&&FGyB{?0jX`ih==O~Ew%{%H2lA&1* zC+$@X+vYZMk_=pjQLOWL`fYpden(Mw9OUx*3qn;o+qkbQn{X+<&26+WZW*!E))bm1EtyPlAOhnchNV$S6&y8GKM;IeZa{Ty8+U$({bFI|<$nH?8& z7aBX=>GDYDJzXqBHA=leyNZ<5qgd30{6Z^>xUL~V5Egpuj4%QWpt}0!; z{_C{jvHIFPp6@<)3QE9JMipxeUSv~<>ItBmi;G9%91!MwKj|BYL+ty7uXk z$DyodxtoR3Da=N+t2yjWmwaEbo||uwOt(dL_eI~UuJ?~L4&3(!-5XdhTek=#*j#Qs zrY%D0A9%9g8!JQ658uL(Jf2;Wl@uiFKhe@A1jB+VRtz3l()tQ;s=@6z+An?mpykWV zb|~$jMja9m^9m6-XiA>H>M+%g25Hhh`Hxs!8e1}-vgTjkz68)(2<3^ z{Et#JQU)n;W={$g>x5>#SUaa%%4D za$lTL@S{hbgolfWxB{JSLiZ$UCdIl`sw%_Cd2Xp(u(P{kOgeu~?NRLzd89&=G4y@6{O<>1SlQbS0OX>Ubn(muoL%=aGp zbVmdFgEzf=9F8P>+(YBxMUKnR!rO0`LK$r%NR@kY-8NklWxgd_z+1{ve){rr%zRq3 zw>)iUz=F!4D^=9oGB{inls2Tip~Viz`50BH{gg?`gg|>BX00fDUUq%?#|N3Yy)lp& zs1lM*;x=N$j9mnl*e8#37G;KzC7QcB%`bZvsyySEK^3EtQ;sLIE9Vm1j(bx=-kSDP1*O znZ2yL+m*$K-7h3a`C z?Z4y|Em;#O2n;Tn@E=%1P(3b4-0lzYbp1wS88N=Qozu4W^MWrWMrgD!w2<#=rNT`i zO+Et9e6!Lah5pwCf*BL<8|}Z1n^#H~%$X`HR|;E&T7j(jbBd(oL#QSW6u^UW{g{BN zbDUn!v{4)7VWihyFC65*@^Hs8qEJWJ5eOh42$L^sLpubdn_{6@IPRv~Q*6@91-D4% zVmntM=HXZ-w7Yty#%pLK&#PkN1oAvkDT9mt7NK5=q`~=Z!esw*RH+n0O&O4UI*dk$ zA}c+7)MZ7FBt{;i;x|QdhX+8Q4i$+>sFUH0;Br&)qmU2pPtyAR1AZ-|Qz$~lUKsrK zamNx(fiDQn@V89^E~t;>OC{X=A&i>pD1-Ag2ePuPMl?&xdblc+hUkZSJ?GrhISR9- zD**s;H7c^@hpsjTF{PzXYd#9SG0l;f4mF)GmTp*pEvAOBt?nYD(p)BJ`4cJC(_lhl z;TKjj#G)5sBDfp2S9hnQB6vMQ{R7USjE*YH(s_t(xfv^^)I~TreTsWL-hQ=Cabf>< z*Uwp0JYg4HEw!3xD^;Ez=|zVW*nF@xjE|qAtX=@CN_d;XU@g)DEE!IjQM^mcdx|p8 zazGsB&5G6c)#DX@NeO;NS@@KSry_!A<86cL#Y{1joz_UlV2#J63tyGLIa(2%o;Lh!)tNXvePg?) zaPh-}nWilui)eU0460RMC9gfq3(DtflV7b%GMx97y$ULUiRmG;-zX8FWWUCN5E< zy-twa>5W4ryBWL4z71C~T{C%5zT$O}QSTLl8!@HwK^gjx3 z&;5t!#%dhF|E=%@{vU;>HNdYZdK{=6lHR9g6~7JT0*_}@{vUsL{y+ZiLFGUG&MLH7 z#D@Cs~WhT`e$hyhFh#}A;O~~eoSCpsOj4-Q_OOCLN zQ`6=1BO318&`H=T;e1lfqdZrgR5`_T`hT(A)lb?5XYP*K5c%lXCMGGA+*+a3s=qqN z4^n05vN@!7_$UwvyRpg-yNbZZ4euyDf}LINjQDla&{_i4%92O43a2#GE@|u1^w6)EpH$EV1jI(^AdRI<%z3;JEm8ai(r|_Scn0h5!lUR#7jjc#=^a|6 zWiVi~PFe@=3`d%G5*(ew&Qd@A$JLqs$JKHF*VSdO*(qd~UQ6v_fO|hM4dNv&h~lXA zgC59KoT!8+JL;xv$S*5}Xnsx`E#=No;5)3&qa+I;2qQ^DP`t9xoG+k3nU&(kD_=jC z6x6d~=azFde3c%CHtG_<&9bLv1w?<#m{u>dO5KHIX^>|dZnbUFCajg&Yi5bSz)GP; zHk;4kJx_;;BPr74|F9B}mNm(S+AN->T3UKG>7(b?#^($v6O>|3_lkyGOl)}P?-y2{ z(_?De4eR$4gaqHn$sZH7V;V%i2)e43?GymBNF?w05c|6+6jb1ng;Ar>6V~*iJ*lf* zLM~xdc}y7^^iYO+<(c3iI{2SE6<5hJW{3j_5SFt|>s<^ZV8ji&@> zOcPyb`}b5w2eRSC&1QE|#^4atp`=3m`6Yvwllwoo9m}x4W*1o2(!k4E&n(SiL&g%@ zAQC!vl$X007px-f?}W(o`9L|4MtYQw91Y4~6Y9E;Q&t1qX6RC+HdJ19WD0aznakl; z1!;!vwZQFJO?UjzHUY@p&>+=;D*wfDF9NJ?43kNVQX-jf*BDNnGvSNKY{~(!Jj1i) z0t_`++c<)!_tmbHR>naul;xO}q{-ZwaIlvM28&AMo;>TMLVA5HJZPPLN=M^4bG|(+ zQJtV2BN;E)zqT#q)83sO$Q-f|cbG>5#(5NZ!MqzR+CZiz;VGoN*6uTILl97oBE87l zw9@6TOLr|Q$9%2}BL)aT1sH!2&Rg=fs2e?%X}6k2K0L?d4?Xs5KPsh<=S~O^dxk*m zJpauWGha>s_gRe1Hs@z8H^<_tvRcV_WDt4Gc-LRqWko+!qe!1iGY{a@zNLPAM>enH{n9a6X%PfAS7l}|T+ zK2;jIBy+M2)3To5n^@i0`pkc_?F0!D%I>T=W}IPdj~P=qc2~3SxQ(KiGIV!E7k-Fm zfXCvt8A=lG5%vC#*wi}+UbGeJ?&BTnsxA5G+z-kG8?&^m2|CS1E#T~IM79? zYP+5fn|{0sF2T*kN~Bm*L4a@qM5KH7lEs6&zD#UN5Zd0E`MnGeZ6?OWf=siD%E!(64?T1LkpdXu~u zM{JuJjM^n`QS7(^H`FfxfN!hH^0vxa2{LSQ*jrijZm`UU9Az7HDQ^48Pa{vWXlx}U zaJ5&R;jR|(Uq2_LcfI+~&qWDZC-%Qv|MPQ`k31*!|CgUT`M2tQ=l9JO44nf)r-TPg Jflz>g{2%o}nxOyy diff --git a/charts/postgres-operator/postgres-operator-1.6.0.tgz b/charts/postgres-operator/postgres-operator-1.6.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..ce19d588233b71430c42cf82a76cc994d1123136 GIT binary patch literal 18415 zcmZtNV{j#5*Dw6o$;1<5$F`G+GqG*kwrx#3$z)>Nwv!#(wvBz}zMpf>TW{6rZ(aSR zt9o@`SFiv25y!w`g8kXZnzYg3%w1SDl(aGcL!*$(eD$Lc9hH6A_C>lFNh+WR)2X zu4Y4}H#2A1U*`n`Htkz{Za#QETGlV8K`lN&pW70B&`@7rUyT3rP+wkq*D=$RcK~4X zUBqxvg<9?~uieAmrrQNX`EBY|L6(-W9z63&n|)?N(C2rEd`g8eqRF zur!D2?=R(Ct8q3;a2OdH1(;(0Vlh&(ryjfY-x*DCW2JpJpuGz0uck=MB`D99v&qIE zYu|o=V$K7kQ-aIhO~ERb1A0JvJFgpW+t*zuKdM0yo}SVpwm^VbJ&T6+G|XX*d=r3I zsQ0&~P@gV!br_8_84(vM2WeJ3vLJXCB{@^_fe1N&7P92Kt1IP0m}#_G`=<3}qKhFT zyOg=uA^lzfZa!|l@*FN283c|#rj+v#V65@1(LM50YEHz8)inJ6Uzw~b$$CmWg!Zke z@x*16k*}8Su`g2~o;lXx$4FjE0Ae(&cO%TcbpTt}HtQm9Z3%km=E=e&p#X@Mi7~BB z4S^N7D}?BpM9I^Pm%y)HbIrZj;>yKwao-Fx5fwI6YoQ;0EHztmq%t=(>g4rk#62a> zlMjK72|F_$>=ED}Ba5%NQiGnhI2V$msb|;|C(K45MSxEXF^s7g#*pkQ%S1*Yyzrl} zP;)7H3b{<9rRF9sO{8~JxlqN}8?-2`+(@l)*2@f#ebNXCR6Jt>R@<((@qwtjr_-AX zh$`y_2hS2MSTZarcEIY@UuaDmyApYdgd9+B3WHI~zG$mTK0etTM2{fnnN^>mGL?)Q z6?WQ4Lo1$~u?_dvnjxRqf|U8Ic620p*j8wb8GEJB81KxW^Y!%L?(E?sw$@gB{`;*ATYFK104|!S1Bd6kRHofy(yp4iW^JH z=wuo)-T(`C?&K|eTr)|skY~|1gPDwq%5?Mr*#x(Zl}y6hhh+7V*pJ-tkOOkxTQmiz zih`!;=bIUCENYD#6F8lkmL@+2jc|m*BjqAM7G@m%aigp?CKsPj&09=EIJ7H94f$PI zzb;gK4Yk0yeY6fRv8Q_@OLs?ZPNq%%cFJbVj!ANqdK7JG3=jGJ;Vq;~o@?|O%u!k}u~ITZ z7GirMwODH~HNQmpwjMz(0!rXcU<_(bBqlM{!br1&uY6lMmq&6alSmG&EpT^M2p-E`xTpwD@xt zgDL0Ms$Bb>glO>!*dv z+EdDU?@!3Q7l`X-qS~NH?i1kPPVxZ-WbqB;JGPc z&^21=$IpXRp+MqxFa1&TbLsscy(=5E8AQTpR{cwy;SuRqB!ybe5l6X#UNBooDes7e zB^|>)DNdkl0BvQ^fJ+$2Ae5)qu!8#>k&RXPrxRdS)F=J?DPlbmv9E>8GsyuwNq z8#f;fz+*~Anu%qgoBy|OPdX#}%&6#N2@{rK*knFbNz}5KC%+(xMvx3b?K1iJ0Putf zMNU7ZciT?wrpOd;Fo4Q}Q|7_Xm!oVCm*W}-cfGhfq&o%?8}6t%Sc=9}&K+;}GX^p_ z($@AzM_n%wwWwMvV~%?W$r+i_8YatJK8}$c@L130^&KVOF_L+JSK~tf{wxfVx-*u0Dp<+gO>tlzeuh%^E?M2r|LA1#uj#}hyz+)A zJxX|kcgX1KRelu=%v-FP3ZpoT4fJVv$s~1uIBY3^72u*uPJU=&Gl_#7D8d#_Pe3p5{43aT_%GW zRNF6a09zxumMnPz3xc!?KYIXPm=KS@{uxqsa;BG$zvjsp(d=H9*&hi-+cn?$*y3mc z{Tv8QVMGD={2k>xT8j9T=Az-hT3J-XLXelAFC1RZq{hy=)j#j>miAQ!w=}=wWNJUxovP7`u&K zWjgRE!hd=UlN^j{PQFBUN&*!rY{w5OQ%hiIYP8M7z^Gum%(#9-Q@cZcXy+f?2pxlY z8MA9ESc4BMTqASkfWWOZ;CAU`f~WCKm{~Cbn=u+QLbzZXP3V^3GER&e4b`am)YLAS zJ-SIqjg)$6Gn&#%B@ySs_*r7)S$Gc7R62@wpBI`n*=XpnAjb1@btjIV?y)CL+sw`(uJd}BSp^+dN{b>~m` zm`lqCHUE-eKP!+b(pq6X#HD2xkuVoY!ObU|ml4<%hyVR+DFj zuAHBI`;)b(o4^!?vJluP$b+~mQ*uD~Te3a_mol+*n(Ft2P41Av&a!4S0Hplt`NvC? z`vzPQG8}v6M_voHYqr8SXN+vBW*;Ml7^`w3L+rNwQ~Cq8-Dm9+e+ER<4S zR2nXI_g6Z>;C7((*+`Pp+Z@8X+6;?i(rP0$)#ht|LGz=<3XIN7eY1+;p686c`|B#D zY0b(^kGE95N|0K`MbcI6ma7RGtxBxd$B0?>Z+VY@uWc5tzKR6@pa;VLVtL>>#8rq% zeUa%~(UGNOQ$7AJ7Vj3}71G+`@Q!TA(7!HPoiMo7E!?hFUKA~v2Ij!V87C>A5ZW}9 zBeo8e7M#o^&OgI9EM%VWFlpk$pj;_{1^6m1EB%JrX+TvOgIS*}!2l0HLZKlbrsNi0-b?JfL!sQbZYyQm4e3I}JwuWQYA-(0;bst+(2CB~5FQRogAp5A0Z7q>>|ThkE-+zmC3&Usjmotjs5A6sbyp! z?t)cpqqMr0oY^dE*6K3VIx*jYC?x|KaigK>Uh}o0$#`tEddft_R*@{7CPg$KM}#B_ zr((Qo?{xNtM|R`OA%v^O)=AYAt-Z4?H$P@TT3p$xh{Ne>;Vt@XBOBEw6?v(J=EQOk zKw{ZqVs|ioIJW5H?CE6v$Vw?Q3}wccITyh*qNGV}6QD`ZmXtK7Ny~nm|H?yhnZB96 z`Lv;rG|_laxj;milI>WKG!2J3R*MstaqZn=AgY7wdT|AD0#!N<@dTF713PUk4BbUkCZ%@gX2Gbgr0*#1|ePF)0uYD3P@r*>V;_v-6mu&aOqYED( zEsHht0*!{)?v6NUSf?-24P;VVGfczGjcZzzpjIyT@z(6|q@6kOx?Ke~ezX*z( z?wPuI1Ox7$$oV^4J$<~PW|4Mh565m-$KK}$S~TLisT-XO_#bxEiQw2YOCqinZ`o6n z0()ETXROxpX+Ei0y$eVJr)f)zRMwx_e(7hN8;^>}t2hR#+UY%x)kao3Ipiyj-YkY5 zx1HvW06keMv$7%HUvC(cY;9rsVfsIOUpTpi5jPUg0KHs4H9Ug2)j1-6 z)C3|c(Xiezu{3sfh&#xPnR|()EVH*W(t897*jkWKx1eCq!`+gJqa@~Os!eOkMjz}m zGtT8uMroBadqxfocqfYa_@#zc5gUH?HA149*>Pb+P-poYvN?NquH>BQC}HY1XpQdE z`VZ!b9Y8jVCYJ$%kT=CO54lSkzye^yWThk1x+0UpHsJ zb+yXnco2;&RxyFLyWm#O9fj)@+nFG6Tph{@X`5-Ji$Y1OaAD+D?C~z-5@Ei9wN551 z7!}&WI+l%~gpC_=QVd=*b^Zh@%bhrism9Hc7%_yWOrw+U4&zhhcpg zW$)uGj3MMa?P7kYADjLM;O@x%<(W!!K&UF+vqOJs2*mu9q}XF*Bo+yj80_Y45v;r* zJ0pYR#QXOSpt@$TqDwwosiAtvSrf&dwt5Z5wR}~>?d7Km+j*cd{J*;qM=;Y&J5;5} zfPrm*RSjc5c0rMIjnMuTqIkD|sFTMvFQO1D)S$$wFiwKUrd8VzPAhE>Gxn{n@~?sy zCwe!0C*r!ZPlw{V??<*?{>`3`vf~Q3?A*0S+#yY3nB|473$6L#Le=Df zZ=kq{#^&YGR8rW8AqggWSWg(g)gIr+hRY{OUlHEvK7*w|T`q%bv3A8XJc$Z9w~IJ>Y~UC%M^Dc*^6?4!iA<5&yE%^4oU=4(aTJ-)F}qUN57dm-B=p%X%dWA4py| zg$FRuoN!TMX;RaTdG-CnlDxPR#{#~sn2c6DAs>qta{6; zc34{J4bEXFxq?crAUy>X>eNn76oVs6&atzJr81IjLl=pRi_pZ1oc`E!CkcvHf`%YZ zf@cq|wJn4BLqyuHY2hM3P5ez9T%O<|9ID3Fk?Pa%rc#K`CKtRtElExcovYcwHpmml zy9u_?4`{a>d?Bkv27jrq2#L8;Yh@$F7M6C}$zeNtI{^2+&+t|;)*8IT$Ia9?;2aP+ zOqk%eajU8Cy^1|({n9j*+0<)i1+|$hibuMt--p6yQNEGa;p^wA>NYQDG$k6}12cGn zTa||PDr1;0+IUBzq0=`N=rKAY0TSDE=}&G1A%%s%g19k~hd^pDL-`vsv`%dF3!ilO!Yzu%8#OLcqa9y^rZISC@SOsVx#a#1> zzmAg{H$W*y@?|!33f25v?Q-n4NS&+qq*vc?p)|FJ5ZPb2xiLrMbKB%Yza{Tu5~-QS z!(=Y(^_q51o{YQdyI$)L(>C%ERp<*Mh33N*>j|B`V#tJlVQz}!m|q0T4%&K*gpokA z$3SLT5v4jwtL^NUFu~+MhYPO6#lH`Zos{FPEt?^BRGqcpB^vm%!R16{v}rb1p)&tv z#QDx|-ckG<6g-au%z4HCW&x)~q&uff4xdsoGNIwG7M(-ngGl^oF|S2qboKW?w_eN2tS3lZ)11c@3(zmeO!(O zzvVvb^XGzmBagdCR47@-6V8T0rc6a9ByTOmuvdFC5X0%jcu9p7C8m8rrx>TY^`S~b zycSmsuXkawnG(dtiHV$3_wv6>aNvraX`0s%C0z|+gIq_`nJyy)0iVR5mD-@)Pzs)u z@`8IlLTH1AulKI+_vaw-p5*{t(6?LPx5jtE%sW>wW%EIdVhf)4tx*L$xs^cQmiI%M ziB#*~zN38u98BFL29gtLh`OeIb7};thrPw;hW%&*tdhr6vOf%^b2|)gW+XPu(0EP; z6ms>0SY_6Pp<~ThJ|tulDr*{SAqTVqm@JLi6aFoc5kFkD1=&JT4x6k79)2c~#+0%+ zF)x&?84eZv%%R|P#9mPUUNVX0W!#0$xz-D3@~h#*s>F>KfWArwQ5PN658n7!w}5y? z3q}5#MxFavk00K??X&2@e7F%*SF{O!`m~>Bv{e1he>DA~eWYz-s|4H9iS{$te9lHY39b4=~@?71dNqNv_H2=Pl z;EPu_1GygWGyLYhNpDoV6Yc3MRCYc4x!BHjxm*R&{Dtzf{B#R{D#8J{;YaCfd4BuI z9kx5jhzzb)M4WFrYPoB-M5fZ+u_0vh(9?DN+4dpm<5T=n@bwHfQ=-uAPZGg?rpYys&cq(iy?E~?HaZB?BxVlF z%T`R9+OBeDRlT1WOq<%jo(E}^v~`U_E2vD%U4Q&8(vlyw==X2gMTCh61m^T4{T(%e zfr!N{X42R(k=J-CI%&Vn-ToX#tOkyUq=CsD_SWFrj&C-#-$f(V8)mS_TGgn zzZkio$aAbdy) zIlqG)rm~6mRu#XtX$W_$U$@WQzrDHK^>i$+2c-VzoiKUrnVa-eg{VXqN`Y!ieWkG9 zc%|LgViI^%a~y=2ue>zmxRvuDSC|5IRj@BYce^tzauN|sFnb5tC*muz;#FwZ?mcyd zpA>O8&tXKld%;;ruP#Jz-tUH*wQ2-=!QnHHdr|aUyjcOu42?Hsn5<1s75RL>OE|*L z0L@r}_^l3EatkJq-QX&ln|I@|qP>dTmgfa8%V5Uf^nbKV5uHNg?Ml^@BY_ae=w3*aAG~vW>bG= zd@R=5=Ek+)|58hF`&Q5!UQPf_mpF=oAU%0n`ts8!w~#-XvqucrI=DcRflV1BMSnDR zw)I;d+jh=up)QKOj_2pmG}7JVHLRprZl)VT9m`f``;^j_8L?*LhI7*gG_tO2{~*h- z!;g#2nBcCC#ZJ?gtR}EtGzhc`o(913l}M>~aTGFhvDcPi_`6g#Lp)>ayZUDaSxEW! zD^>iv*|*F_82Siz{v)$2%uH}+*wmvD&&kGgnPOp%h)&N8OAAqT1G8c+LsDSj=5iai ze--b+Pu-6va%xtVlfmlCC6R1T|DL3i3ig-`GpGHayTyaIv>e)mkkrreJl~e%Li;Ap z3;zRjqH`vdn3TLQn_WX6V5*f3`0t{n8$eny$MT9d=B<#_X`Z)hjKB|r$5kN60^XC4 zYbWcoXKSnPD+g55(gSJ*J(J96+=03zyuDl@w<7Qr$uE-QU@~S8kc*#mC<3v&>O2U1 z)eN-g*=5A8hzOTnA-5iQg}4p`^skX;<9xC?~sc)u8q`ahId7~T3h)&r%MYu-OXiS&)Ia$U0enE9#o}Ad#*@cBgv}Dj-{X47_d}xvR0!e+NgbHFVBSa z)K|89KXb<>uloiiS_){A56>zKqUs;bmuElmyGf%yAXPHNn8r3 z%DNFXy4tWbIasHJi+wfl!<}o z5th9gwPQi2fvBP$*Zg?W{vwOzkOjC7#b#qT=YjlNX zgfrH2yUaUhfQh%7<1ORkP&2l*Loc_aSA;Ic;GOmarDW5C1oE%X>o~9vS6Sr8t{Iuu zt{K^E0iPo)$)!Ud=yWw-_R+6*fr^WsL+zQ%HO&^#{hpV@*M~#Tx=$mBXitkOZH57v z;FaCc)vL3XkUgEkZp}uAuO6*`Uw!r|+)|+6n!!UCu2{i9-_HL0kVn0}TCdMP#;&}> zUX>y8`hBC4^(IX}(udExbLwirrRCC`O?TjTQH5&x;W8p4*y7tax)wFdR(j*^a3+2;E7k`C=OArk+@@)c4+G zdhwCfU&0J3yWLu+nkP3cu5~aj_gx%jU_Q3BSJz{L+-?nEKfOQ(&pSITp!X2|$CsM6 z!P+URprfHsKBzm-rQ@sqqrRC|`f5--E^X)4aqhpz`t|H{|F-S#aLw{{FH9uKR+>4u z-r%&xV%A@~BZb6uFn5v_pZPBc9({+`aMY?%GLFouB_|zD9>k%{jmtwDLoa_#MT2{< z+?uDizf6pcyKIVTGbQ^98uCmz23=c4t&#o& z$can%^R@$6(#scdUAhoKfVZmYn}Ut7fX$dO&*hK(FV%V576?-Z#xN-Ks%zdeN(aAX z+`02555nI^Ro7qJODL~V(u@e=@(Bv66)7^eOxUOD5hJlz#ym7%kJqK&8L9&!kL=_h zn~D%>y+5A~9^UV7^Zsr4x<(#Bs0spKE-ZYR9_`=jliPkr?6cBog=`pz2;9G}a3@D_JcrKNSfyv0vXr}qs0`SI1}7&n#;suCRP)w(!4q^X;@Om=7TDU9}zZb)k3CH z4sfris;NW=$wB(qn5%A-_Vr*)1U9yJ|5D@TP7E8XDz4|T@Q6)>&qLdc#JdxW!6uv& zPxw>gw#5coq@4Bhh+g-&68bBye{TW^MY_degSm4lBK(PH@^Dw0*hGm7ryQPX2WYc=4-Ce;m1pNQ^|zt4KK7D$A)z z>_|;0wU7?DY?zqRRisivDdfb{2qRXT&%@oeghd_hZq>P_H}FUK+?yT6%o zq|nU7f-|SF8_W2S;RoI?`L}9^RzgCdaA}#U;#zee46|bNz(BS1T!ERndZ4O#-(EhP zha)m~PxD%cHd(tV6kr()SuQ*@@vePdhG9{zCJfM?UfKP0TAgjud9-fQx91pO9*;5m zF^t?UtxG=^_;CpGy0iIMez>o>HOTu|d*%3A@L2-gJvbE5KEjBXpzNz>7hZ=u2fYxY${Q)W_NV*P5cl}Nzi8AfYkrZax`?ogF`WK@ z9s1sx4lW8OqMR5d=EPgZU>I4hzpkAw^wKF~rWdH|R>7eROIRI0v-+pNi#X9|#Xv>_)gr=V5E4 zYsBjI@5YU|=*^GvlUh)D0iRc=DSC}i{%QfgeI6g?@0k#f0;huJHty1nUTSe7I2Xr$ zd-lk{8d+RF)pqJKJg*4H5XQx6!4eH8^=hVcTzTqfL<#f79;9{Gm;ps1quThgz-mI> zEEHvqEHJ3sPFL)W%!4S&QM$}EB znVsVaS3T2+hrQ`fIq45B6GD-Fju0 z;|Sm|a_t~R#o05*z)AVY&D%`T77=HwJn-mj!H%4ChI)fvIPJx2ZXI^Lnx`CUGIYQK zFlSBx(gCO=P2va;euwJxC8P$rE|-@Ce)m984v(!v_5Egy4pkL9sO1AzZCjV$oX#Ky zdxh5~O1-7$_))F!ADd@8h;g2s*$EMUNB_#*N}onB6NN8EFyOW(9143h6!)yjR~Sn> zB~>{xSxd2rga1BK-pZ(<=~!Xm55SI|Ef>W%Ef8jaJbPX;Tz&9EYjEiJ)WTofGBbqVD$^0@uFSRYw)26Vu8eStZx_sXfyDc>5;!(LfFg z@%i*|{>BAXJq@bezI@J`IR>qt|2PWpOgyeQ2)HRb-UghChB0?mvI?jYW6$w<{tkUs zlaJuBjEN$+3s}cQxyF^#k%5-HjP0S3CRb@2#U!SjJhHEaeNWh8GcH=z4!xPsa( zcL}k%tVD0AA~j6gDd3=TUA{)Kb7gm(qMU;{`8O$sxA@CKwq-gaFo8B)=ilmw?lfS0 z(MvI-7fner({~SF!CMC;03i7Z4|S{W*u6n=SbK6mSaj-XrkYbYC0h zZkI1`-wcxBT6wpd)kCF-bIPVGGs4n=JHljgsO20xsonex{@7w8MHVF>UTFIt2LP@t;C0`d@v{U0!~we}2k1Tys&I zz%@Ou&l_x=Z|7zhyv(6`fLcmIsD##Ts-H%VL$2njdfaX40(|Zc(tBX!9=pzD1S}3d zL(#^DFGS&5r2zCR%AS`gr*o4_@@ z8AEo6Eg!ih+fh7_r9=FjY(cR}Q{t2JBOMdkV1rLz6lHc}QR*3=651%PM8|=&=waaY zyr9GW_9N)o20L}RrhdDAPZDoctV1)s2S73n(A5UxM9&heQIHJ!wS1~cw^$&T>=+o> z82(g$ojcT_XE2FzI8;7^4>VidO7jtYF|_DLq4iiLA*B!ynD!F&0R_4EM`%V@i2%lB zR_tl6u1II}i@;hZ?aCJpmql51nSkMvNmQ^Td zmlWgr18r4;B@{%YMIg7V&g1y?`z`sU(Yxt^dQdvv>6r|bhGWgd-15O9cWCd(zbiG{ z)5xDNL0(Pim#>^93+i1JcHxGKvV5$PM-d5fRijhy@aQ8c~!SecN=EIyJd|&hEmz| zucA!>oyN<;(Vx)bF3Snb4DFg&A^Y3{-NA?oRxn??P3P0YyX;b!2@*!#3Lm;%wafa= z3NoMeS{3VT9Uj;D;?GHB-k=#g3t)wOKp5GH+VSgn%{yHWQRU0pdG*^0PW9Q3tE#t^ zmFyhpEY_|g-_7Okv~8wsbs_P@gNZt8d2Y=;aStC(uR(*4IdCevrq*qc54*U{4J*WW z`5a%lQ@SkKveDZoT{2yrHp--xH&4n-CY0yI?*Cw2!S>!c&}do*&A5^khcA`%cD^J9(@a~5Kf=uAv?5cc%0$m&TL z43W8#!_tu~#4c4wpJGI*SU0!ffE%wm5 z6({&lL{0;@yim;_mW7B}sFKl@aKB=OM5rG^!~y!y^CL>EaMBbPAuPLckOvZ&QnKN( zve7JL>@j0vlJKNi>^58SqOF-giTsQtc$qt;BplUpK4-WTWyxgLqLYj=Q2*tRB(2Hx z%T%0RfQ)%y)#gNFZr8tQr^&Uv^{KW088<%{9jctM zYe_4<{dV_ImSD?`gJeNa_nN!jYp!mV#hvTz9KqZuFc%4r^)*$__{C;A2n@LQ#}!$` zh_+vll#HDUqx->-pu;(Ou6u%C{TUD)EXMBAcoNK721q@CKg?ufm9^Mm&_%@)q8~zN ztya&MH9#671*efvWnpDr2+ZLp`#v-JZgDWhc6Wu_cE#7xLZNPOX&>=HI`sb#SX53q@XChkXH6H-! zH5e!G;?Zrkm#KS4vv>i^xsE3$cCGkm{Z2Y6Hi?q1pY9q_m8@HjmXbU)5@xk@wQy_e zi?Cd{zDz}|sah!mbSe1^;oBs~OvE1^Nikc5T$f4XfO$$m)}zKsRGW^E!gSXd0)cPjKA&^V)dRr;mlYHaoILQ%TC_tuPT&o_TFoyp)^NP#D`!-uA zNn%QQbJOg4$=n94dBbp)rQm4Nt(j?acq2r=M8V}AnL2Q34c_3VAim;cc_h(%P0BVRZyw>Fr%y-&_Yh&RxcG}=mUSTSPFIUF@}7>GkxbnM32I-d^5 zFZ#oBOU0O_MeJ4v(V>O;@A!=HX`ss0RZSO@S;(U0&Pzt0d6YIeJXn?)9XBg8kzW*U#3=#( z=N58b3auEs)wn_$$DoTJ*N^S@uY%7VpCcc6|6;Jkk32~M5H_xm9-x!8&+HfE0;rK4 zk=hljoBjxV5yf1E5_q>yOpAXK;QrJTHr5Prl#mK?NO|zlkoWW6_GOusHY>IF=qlhb z9U0`th(susj*7&m>$9m>e!J<;5$`z9&WJPD6Im$fi?S3Br$?u$NcK}-Hr>oC(tLaX z9|0xsopE90<49@onkpyZ=aSr_E|as`Ml2@--84LZvVdj>-!JxpM&Q0Cpz%#gIE<; zb1H-sqnHdHd0`1B?+pcF|ILe?U%8R$?8H2GefHgl^y$~!pS@0e_1$-UZW(lR2`>|! z*h%kJkG9!ui>UTS-L8b(VRY#*7Dv+8SE8fv-=kO?vH4D>5+!ClzHLIbZzt@h`@qhe zZpGLK47M|U(?`mCwyF`lsQZdZzaa5n#V5RQ>!6Ruy%c9!JyGji-V}9n2gKf8PTR${ zmlYgZHOE-*BK57S%FI%QT)iEeGu#R($DS|0<|( znA+D6h`A$%gb$V+elCKlOapy>eSK}^d~65&>pGABvqR(SD4Tv^ko9LrBGeg(Lru)$ zw?qTs(8Z@KH|7)d_W-GX!#xG7fsG13Ekq0?i#3batCmoBW@e?O&E2<+B`m|A66;LC z24VnGH%MXsWkOn%@Um9wU|JW`@B#Sq1t{ z7-&(oghF%&tA0c4PW4`;61BU!TSYvZ;tDQZlRg>If@7tozpE=`!B0e!!{jW@kVtu+QlLuLC^@zW zHB(_NO1_b`k}zrI7LTS$&=XBNPoXK%m(QC{v@n{K6L5BOdVAvM=M{W^Kg}j)+LSNr zDD$%g52eozSE22?!cQxJpE4&m;Zi%A&>b{8-X*7-gZz)^Y2=l#)=^AhZ(RR_W%*G& znkcc%r8*of2AC=f-~Z%U#9%dj3$KgS&8!1-2>{J5i2Vu;4>=&n&f=@W;3kQ+iVIdI z%IrSDJ?Bx*U}ui|VDJcz(2>WrVw8Mqn5<=DXty8B^Dh3J{)*)4=>G%V^<`Kyk9^maCih^df(xeNGWLK(IQ4GCOMa4cEk&7#pq(f( z(~+MFB>*+nz!z9Zj3;y?>x`Dpjvgf|4EL6b+pW;{FrUAZje3qW>feq6rK_q6X}FaN z_NEt#IvabCEEjaoY~{%kRZ0rM;w%L@m^vm|NV6O++{W?4T{pmY;#HZ-I6Fz>$(a&g+`^n;KkH5*KIa{VcUqKSN&^-rn|h?Fa=H@2jexYSbgKY&=^yImdmR7JPNlx2 zB#*~NhtcReQTrOCM{hGq=Ncf8vKQY#`e=9Z$kG;#{zRCL0X4?UjGgq&eK%#lfeszf z#5^PDbJtAj=m2JqaD4W#o8qXtZzL`X8jDb;#;=`3F+VOYzv9 zV^b4Zy2BFIXi3N$*OXo?AjenaR-4r%Z0eUE+{Nb4CPvTqqGAR-(G(=s))!XTWrB;V z@V3Z-Hoql8cC1q{6B)}#YfA&6=(iZdgmEPwP-P!`4!L*5tr9j9ITrtP=MFg9-AIUXi%eNy7&xKOpC zLJ+b=1v0&$*;e_T%BtkHj5&eJF>nwDP!=?ERx?<^>&k~X@l31is%y%!E7><|{8 zk%j@9Lv!%N>RB*vH%k_cdax7#0XECY1marK{^#tTA~^s*d&V9U)yJ$9aXDHgR#!d( z!Q0AiP%@5Dn8)(1nqi>Wi!)i(N=vLg@y{Hq*oCTfnBZ zA`^kot{J&?T3o$NC|>b(@IM3hhBTv8B|es>Lp}j^*D})QsP2I6wJp%7dAmyJ2>lim zdxShutQ(EAgsO~oln1@_2cZ?&>VIg?H{2qK`T|HR06^TFEk2_y{N7I84GhJ0lpae( z{ivD3XR}ca53jnTxng0AWJ4rY2#b{@DocpAT9?|4yay}wpUa{6uge)%|F6qI7SXzL zSnfdF?44^V5W9c+s0F47{2k($5JJn3jCl>`>xlSX0;UcWQ4pbSO@)vNIwW^5ySGd` zieh>1<>dC4%hip{AV{D1)fs;HhFRx;mOLVQ(0v6o>u}}ptyjIH7~Ln;MwTE;HQ%*P ziV=&oAiT7g2_Nc`mX}Xj#>8U?f}B~ z%hD!Pp~%YmP2*zVtB~z8_PmIS>aZ_OB4fKjxzKlmmQ%T&kg<7N`1af&$hm6NehqbU zUJaDuY3D6xOBPrS)RP_cFEr+l-J^~6&A%6_YE1(hJjtPsAJoehkMb=M+K{=XWT(vFIQD;K~6j&!2&sI zYjxQebg)jQ3@5-~6gr?GmMBjMEUxpUtJ_5LzSlu|Gty^E75iT$VmaEW{xq5y#d$PV zR*bAkIKoVQ5ptg=~9YmpLVXJjHO_}8?JBPEbNQW}v0Xg?oeTj*en!+ws&O45C zI6X67uKTBm<65$~8rB5)`ATFk-QVyCdN^Y+V{j;NoSDQBXW>ejQ>cE z^#3L~Scy{MOuS{`(KY`+k|R=%eEyH*blvIS|Ci*z6R7@wBnLTAi>y!P7%emCq}sd& zl8)utLygdd0jy_H00?ov{rA)8=piD#Tv~0jd`>+=(VpinZXOZ-qaH3*P}^JPlX4Ve z3D;xvm-*q!$(a4*U}ko9QWIi(71}}nBiqJ&0V$mig-;=az@C_ z2aNf{b4Ci@sv(x6Dabk`66ETH<1~w9QNoPjg2;_4X+be1G6ToVXh6x3h8kOke*(w* z_dkKNJTNYyd|Jeje}(xTy|;nyW71?--MuJGDg$QvCSEA?q?@bTMfJY|{{{g0hZm<- zH)FD7PEh8ODSeo89D#+h_J_rg8#yVBr9yKanx6ymviq_-x0hqXB%S`~*xZC0opeTk zJch++#C*TK{TjL$OQW;Q15SY2ELaZ=nsVSV7P4y4%$y-CW2+@%971&fT<7B;Q&5FK zo55HB4Vw=@1cyIg5h2N&qYFEZhd1h6zKL=i!NY2Y!Fd|!jWRXHQBFLQ>M3lhK=Te&>OQ674gE9Zh4jnl(- zs8^cNEuxIU4=p+ zA6FBe=Vsk`oQfwix6*dZk5KhPtXTnjb9A)|+Ruk?iDKiFWj4UYya`UwzNSeyRtpeU zqqh#=9Ev_Ofc;V_GV(7mv~IaYM@q>->(`Fj5GSYqE4R?>osST9Qse;niPhAML8M4N z4&x%tX_A2Uy<@pv&K0 zN=!q4sQxqna(9N3cE3xyUGkmzA5gJnqO7#l%)m~HlxFV4lUWMou7%Tzo!M;Om#Tm% z71G$4<-kUOc}?ilUYx?m6Exq(l4ja&(nTT#>&r0gi#O#SRgHgEAdP|-V(J~M%`A0I z`A1U8(Ix75;KyD#{D;Bk_L!Y9FDqAhxebcILg->7iaJc3yF~)An-?HNUS5 zErr!V%UaM4s*KfedMycPtn85sfG8Ex_ud|qZ5CQ9K{S~|g)(f)a;FkPgTL5=s6v|a znuij-n^ub;l>B)4*5}5w&=VuVY}m|`GOEVR5Nwm7){&dB@nbM6E?*G?9Mg+qa4UKP zVhPV_r|9~HjYLz{-t7r`V~apTKzO(?q@rdVq5 zW>xV%)kNSZy1m)O?{+KNVPbn`w}Wkd5;0}+MslrLW@d|K{vPi$92sNbEN-E<;Lx;Y zk!Eo<3igaVEXryd#RVoIG2V&<=h();1MRykM||vCF1~95hID?4OUOA#x72)l7o z15gj&T&%OXrjd__q5$6qx)Fj`Z5s*j!QfeeR@TA+bS>kUh$L=o-ze1aF!X8<$U)a> z^AO`^N*YmPH&Pm4zt3q(#P$;#&lR#X~Xjqe1s-sQAvxMF5T41^c*N zu)Op#|NUzCwVjQ%z-?&+-I7BZA<16Z>`1EI+PZvwcvN-MO<&EujK7>5H!^hzEc7N; z?$>G$wHfdZ`Pry6B0*db9 z8{F4jhAkymwQ}qGPT)2?HWzYGgMq8H14U0qpt-sTH>-1S1;CyWig38Usf$Aa7JZQ< z$f~Wc&83)uGZR-0JzS63Mtkx;*0;kDdf5(g5x4E_V`W;LDtyI!1=p6nX_&y(#lY}N zg&-GkD_^*Y*jjT975$J-)oQ%P7%JCNzxY>CMtM;#-16?Ha<{I2gBFzS&6h3rv)p|p zRrn34vJjNDLoRP|5XZae3}KkD3o;S2-5nRZ6VNOUXr_5k2KZFrgrB|`gx*(~XYUm! zqZevuzsmTz+w$|=l3#u^2%L$Sxu~$M5HMmc7#BQdyQcSnf Date: Fri, 8 Jan 2021 12:00:23 +0100 Subject: [PATCH 152/168] [UI] on read_pods filter only spilo pods (#1297) --- ui/operator_ui/spiloutils.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ui/operator_ui/spiloutils.py b/ui/operator_ui/spiloutils.py index 34ca42718..26113bd54 100644 --- a/ui/operator_ui/spiloutils.py +++ b/ui/operator_ui/spiloutils.py @@ -107,6 +107,12 @@ def encode_labels(label_selector): ]) +def cluster_labels(spilo_cluster): + labels = COMMON_CLUSTER_LABEL + labels[OPERATOR_CLUSTER_NAME_LABEL] = spilo_cluster + return labels + + def kubernetes_url( resource_type, namespace='default', @@ -151,7 +157,7 @@ def read_pods(cluster, namespace, spilo_cluster): cluster=cluster, resource_type='pods', namespace=namespace, - label_selector={OPERATOR_CLUSTER_NAME_LABEL: spilo_cluster}, + label_selector=cluster_labels(spilo_cluster), ) From 9d94e018ffc903ba50959ad140a2345bf3c30bd5 Mon Sep 17 00:00:00 2001 From: Pavel Tumik Date: Fri, 8 Jan 2021 03:30:28 -0800 Subject: [PATCH 153/168] fix incorrect tag for logical backup docker image (#1295) --- charts/postgres-operator/values.yaml | 2 +- manifests/postgresql-operator-default-configuration.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 5683013e5..ebfd49252 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -239,7 +239,7 @@ configAwsOrGcp: # configure K8s cron job managed by the operator configLogicalBackup: # image for pods of the logical backup job (example runs pg_dumpall) - logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:v.1.6.0" + logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:v1.6.0" # path of google cloud service account json file # logical_backup_google_application_credentials: "" # prefix for the backup job name diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index f011feca7..96394976d 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -115,7 +115,7 @@ configuration: # wal_gs_bucket: "" # wal_s3_bucket: "" logical_backup: - logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:v.1.6.0" + logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:v1.6.0" # logical_backup_google_application_credentials: "" logical_backup_job_prefix: "logical-backup-" logical_backup_provider: "s3" From b7f4cde541ba959d1519adf544bad39672a303bf Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 8 Jan 2021 15:08:44 +0100 Subject: [PATCH 154/168] wrap getting Patroni state into retry (#1293) Retry calls to Patorni API to get cluster state Co-authored-by: Sergey Dudoladov --- pkg/cluster/pod.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/pkg/cluster/pod.go b/pkg/cluster/pod.go index a13eb479c..cf43de9a7 100644 --- a/pkg/cluster/pod.go +++ b/pkg/cluster/pod.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "math/rand" + "time" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -11,6 +12,7 @@ import ( "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util" + "github.com/zalando/postgres-operator/pkg/util/retryutil" ) func (c *Cluster) listPods() ([]v1.Pod, error) { @@ -309,7 +311,23 @@ func (c *Cluster) isSafeToRecreatePods(pods *v1.PodList) bool { } for _, pod := range pods.Items { - state, err := c.patroni.GetPatroniMemberState(&pod) + + var state string + + err := retryutil.Retry(1*time.Second, 5*time.Second, + func() (bool, error) { + + var err error + + state, err = c.patroni.GetPatroniMemberState(&pod) + + if err != nil { + return false, err + } + return true, nil + }, + ) + if err != nil { c.logger.Errorf("failed to get Patroni state for pod: %s", err) return false From 50cb5898ea715a1db7e634de928b2d16dc8cd969 Mon Sep 17 00:00:00 2001 From: polarclair Date: Fri, 8 Jan 2021 08:46:38 -0600 Subject: [PATCH 155/168] Update workers to the new default (#1284) --- docs/reference/operator_parameters.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 76a3cefd4..04d5fe23d 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -126,7 +126,7 @@ Those are top-level keys, containing both leaf keys and groups. * **workers** number of working routines the operator spawns to process requests to - create/update/delete/sync clusters concurrently. The default is `4`. + create/update/delete/sync clusters concurrently. The default is `8`. * **max_instances** operator will cap the number of instances in any managed Postgres cluster up From 2eac36d003cac6c1a8610fa3a0e9e488fd44ea35 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 8 Jan 2021 17:07:28 +0100 Subject: [PATCH 156/168] update year in generated code and maintainer info in Dockerfiles (#1298) * update year in generated code and maintainer info in Dockerfiles * minor update in admin docs --- LICENSE | 2 +- docker/DebugDockerfile | 2 +- docs/administrator.md | 8 ++++---- pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go | 2 +- pkg/generated/clientset/versioned/clientset.go | 2 +- pkg/generated/clientset/versioned/doc.go | 2 +- .../clientset/versioned/fake/clientset_generated.go | 2 +- pkg/generated/clientset/versioned/fake/doc.go | 2 +- pkg/generated/clientset/versioned/fake/register.go | 2 +- pkg/generated/clientset/versioned/scheme/doc.go | 2 +- pkg/generated/clientset/versioned/scheme/register.go | 2 +- .../typed/acid.zalan.do/v1/acid.zalan.do_client.go | 2 +- .../clientset/versioned/typed/acid.zalan.do/v1/doc.go | 2 +- .../versioned/typed/acid.zalan.do/v1/fake/doc.go | 2 +- .../acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go | 2 +- .../acid.zalan.do/v1/fake/fake_operatorconfiguration.go | 2 +- .../typed/acid.zalan.do/v1/fake/fake_postgresql.go | 2 +- .../typed/acid.zalan.do/v1/fake/fake_postgresteam.go | 2 +- .../typed/acid.zalan.do/v1/generated_expansion.go | 2 +- .../typed/acid.zalan.do/v1/operatorconfiguration.go | 2 +- .../versioned/typed/acid.zalan.do/v1/postgresql.go | 2 +- .../versioned/typed/acid.zalan.do/v1/postgresteam.go | 2 +- .../informers/externalversions/acid.zalan.do/interface.go | 2 +- .../externalversions/acid.zalan.do/v1/interface.go | 2 +- .../externalversions/acid.zalan.do/v1/postgresql.go | 2 +- .../externalversions/acid.zalan.do/v1/postgresteam.go | 2 +- pkg/generated/informers/externalversions/factory.go | 2 +- pkg/generated/informers/externalversions/generic.go | 2 +- .../internalinterfaces/factory_interfaces.go | 2 +- .../listers/acid.zalan.do/v1/expansion_generated.go | 2 +- pkg/generated/listers/acid.zalan.do/v1/postgresql.go | 2 +- pkg/generated/listers/acid.zalan.do/v1/postgresteam.go | 2 +- ui/Dockerfile | 2 +- 33 files changed, 36 insertions(+), 36 deletions(-) diff --git a/LICENSE b/LICENSE index da62089ec..7c0f459a5 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2020 Zalando SE +Copyright (c) 2021 Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/docker/DebugDockerfile b/docker/DebugDockerfile index 0c11fe3b4..0bfe9a277 100644 --- a/docker/DebugDockerfile +++ b/docker/DebugDockerfile @@ -1,5 +1,5 @@ FROM alpine -MAINTAINER Team ACID @ Zalando +LABEL maintainer="Team ACID @ Zalando " # We need root certificates to deal with teams api over https RUN apk --no-cache add ca-certificates go git musl-dev diff --git a/docs/administrator.md b/docs/administrator.md index 396629b1a..30b612ded 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -30,10 +30,10 @@ To trigger the upgrade, increase the version in the cluster manifest. After Pods are rotated `configure_spilo` will notice the version mismatch and start the old version again. You can then exec into the Postgres container of the master instance and call `python3 /scripts/inplace_upgrade.py N` where `N` -is the number of members of your cluster (see `number_of_instances`). The -upgrade is usually fast, well under one minute for most DBs. Note, that changes -become irrevertible once `pg_upgrade` is called. To understand the upgrade -procedure, refer to the [corresponding PR in Spilo](https://github.com/zalando/spilo/pull/488). +is the number of members of your cluster (see [`numberOfInstances`](https://github.com/zalando/postgres-operator/blob/50cb5898ea715a1db7e634de928b2d16dc8cd969/manifests/minimal-postgres-manifest.yaml#L10)). +The upgrade is usually fast, well under one minute for most DBs. Note, that +changes become irrevertible once `pg_upgrade` is called. To understand the +upgrade procedure, refer to the [corresponding PR in Spilo](https://github.com/zalando/spilo/pull/488). ## CRD Validation diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 2f4104ce9..4bcbd2f5e 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -1,7 +1,7 @@ // +build !ignore_autogenerated /* -Copyright 2020 Compose, Zalando SE +Copyright 2021 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/clientset.go b/pkg/generated/clientset/versioned/clientset.go index 5f1e5880a..ab4a88735 100644 --- a/pkg/generated/clientset/versioned/clientset.go +++ b/pkg/generated/clientset/versioned/clientset.go @@ -1,5 +1,5 @@ /* -Copyright 2020 Compose, Zalando SE +Copyright 2021 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/doc.go b/pkg/generated/clientset/versioned/doc.go index 9ec677ac7..ae87609f6 100644 --- a/pkg/generated/clientset/versioned/doc.go +++ b/pkg/generated/clientset/versioned/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2020 Compose, Zalando SE +Copyright 2021 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/fake/clientset_generated.go b/pkg/generated/clientset/versioned/fake/clientset_generated.go index 55771905f..6ae5db2d3 100644 --- a/pkg/generated/clientset/versioned/fake/clientset_generated.go +++ b/pkg/generated/clientset/versioned/fake/clientset_generated.go @@ -1,5 +1,5 @@ /* -Copyright 2020 Compose, Zalando SE +Copyright 2021 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/fake/doc.go b/pkg/generated/clientset/versioned/fake/doc.go index 7c9574952..bc1c91a11 100644 --- a/pkg/generated/clientset/versioned/fake/doc.go +++ b/pkg/generated/clientset/versioned/fake/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2020 Compose, Zalando SE +Copyright 2021 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/fake/register.go b/pkg/generated/clientset/versioned/fake/register.go index c5a24f7da..c4d383aab 100644 --- a/pkg/generated/clientset/versioned/fake/register.go +++ b/pkg/generated/clientset/versioned/fake/register.go @@ -1,5 +1,5 @@ /* -Copyright 2020 Compose, Zalando SE +Copyright 2021 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/scheme/doc.go b/pkg/generated/clientset/versioned/scheme/doc.go index 02fd3d592..cd594164b 100644 --- a/pkg/generated/clientset/versioned/scheme/doc.go +++ b/pkg/generated/clientset/versioned/scheme/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2020 Compose, Zalando SE +Copyright 2021 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/scheme/register.go b/pkg/generated/clientset/versioned/scheme/register.go index 381948a4a..8be969eb5 100644 --- a/pkg/generated/clientset/versioned/scheme/register.go +++ b/pkg/generated/clientset/versioned/scheme/register.go @@ -1,5 +1,5 @@ /* -Copyright 2020 Compose, Zalando SE +Copyright 2021 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/acid.zalan.do_client.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/acid.zalan.do_client.go index e48e2d2a7..5666201d4 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/acid.zalan.do_client.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/acid.zalan.do_client.go @@ -1,5 +1,5 @@ /* -Copyright 2020 Compose, Zalando SE +Copyright 2021 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/doc.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/doc.go index 55338c4de..eb8fcf1f4 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/doc.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2020 Compose, Zalando SE +Copyright 2021 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/doc.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/doc.go index 1ae436a9b..c5fd1c04b 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/doc.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2020 Compose, Zalando SE +Copyright 2021 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go index 9e31f5192..03e7dda94 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go @@ -1,5 +1,5 @@ /* -Copyright 2020 Compose, Zalando SE +Copyright 2021 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go index d515a0080..c03ea7d94 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go @@ -1,5 +1,5 @@ /* -Copyright 2020 Compose, Zalando SE +Copyright 2021 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go index e4d72f882..01a0ed7a4 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go @@ -1,5 +1,5 @@ /* -Copyright 2020 Compose, Zalando SE +Copyright 2021 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresteam.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresteam.go index 20c8ec809..b333ae046 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresteam.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresteam.go @@ -1,5 +1,5 @@ /* -Copyright 2020 Compose, Zalando SE +Copyright 2021 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/generated_expansion.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/generated_expansion.go index bcd80f922..b4e99cbc8 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/generated_expansion.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/generated_expansion.go @@ -1,5 +1,5 @@ /* -Copyright 2020 Compose, Zalando SE +Copyright 2021 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go index 80ef6d6f3..be22e075d 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go @@ -1,5 +1,5 @@ /* -Copyright 2020 Compose, Zalando SE +Copyright 2021 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go index ca8c6d7ee..5241cfb54 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go @@ -1,5 +1,5 @@ /* -Copyright 2020 Compose, Zalando SE +Copyright 2021 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresteam.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresteam.go index 82157dceb..96fbb882a 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresteam.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresteam.go @@ -1,5 +1,5 @@ /* -Copyright 2020 Compose, Zalando SE +Copyright 2021 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/informers/externalversions/acid.zalan.do/interface.go b/pkg/generated/informers/externalversions/acid.zalan.do/interface.go index 4ff4a3d06..6f77564fa 100644 --- a/pkg/generated/informers/externalversions/acid.zalan.do/interface.go +++ b/pkg/generated/informers/externalversions/acid.zalan.do/interface.go @@ -1,5 +1,5 @@ /* -Copyright 2020 Compose, Zalando SE +Copyright 2021 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/informers/externalversions/acid.zalan.do/v1/interface.go b/pkg/generated/informers/externalversions/acid.zalan.do/v1/interface.go index b83d4d0f0..5c05e6d68 100644 --- a/pkg/generated/informers/externalversions/acid.zalan.do/v1/interface.go +++ b/pkg/generated/informers/externalversions/acid.zalan.do/v1/interface.go @@ -1,5 +1,5 @@ /* -Copyright 2020 Compose, Zalando SE +Copyright 2021 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go b/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go index be09839d3..1453af276 100644 --- a/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go +++ b/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go @@ -1,5 +1,5 @@ /* -Copyright 2020 Compose, Zalando SE +Copyright 2021 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresteam.go b/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresteam.go index 7ae532cbd..a19e4726f 100644 --- a/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresteam.go +++ b/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresteam.go @@ -1,5 +1,5 @@ /* -Copyright 2020 Compose, Zalando SE +Copyright 2021 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/informers/externalversions/factory.go b/pkg/generated/informers/externalversions/factory.go index 4e6b36614..e4b1efdc6 100644 --- a/pkg/generated/informers/externalversions/factory.go +++ b/pkg/generated/informers/externalversions/factory.go @@ -1,5 +1,5 @@ /* -Copyright 2020 Compose, Zalando SE +Copyright 2021 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/informers/externalversions/generic.go b/pkg/generated/informers/externalversions/generic.go index 7dff3e4e5..5fd693558 100644 --- a/pkg/generated/informers/externalversions/generic.go +++ b/pkg/generated/informers/externalversions/generic.go @@ -1,5 +1,5 @@ /* -Copyright 2020 Compose, Zalando SE +Copyright 2021 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go b/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go index 9f4e14a1a..6d1b334bf 100644 --- a/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go +++ b/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go @@ -1,5 +1,5 @@ /* -Copyright 2020 Compose, Zalando SE +Copyright 2021 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/listers/acid.zalan.do/v1/expansion_generated.go b/pkg/generated/listers/acid.zalan.do/v1/expansion_generated.go index 81e829926..cc3e578b2 100644 --- a/pkg/generated/listers/acid.zalan.do/v1/expansion_generated.go +++ b/pkg/generated/listers/acid.zalan.do/v1/expansion_generated.go @@ -1,5 +1,5 @@ /* -Copyright 2020 Compose, Zalando SE +Copyright 2021 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/listers/acid.zalan.do/v1/postgresql.go b/pkg/generated/listers/acid.zalan.do/v1/postgresql.go index ee3efbdfe..d2258bd01 100644 --- a/pkg/generated/listers/acid.zalan.do/v1/postgresql.go +++ b/pkg/generated/listers/acid.zalan.do/v1/postgresql.go @@ -1,5 +1,5 @@ /* -Copyright 2020 Compose, Zalando SE +Copyright 2021 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/listers/acid.zalan.do/v1/postgresteam.go b/pkg/generated/listers/acid.zalan.do/v1/postgresteam.go index 102dae832..38073e92d 100644 --- a/pkg/generated/listers/acid.zalan.do/v1/postgresteam.go +++ b/pkg/generated/listers/acid.zalan.do/v1/postgresteam.go @@ -1,5 +1,5 @@ /* -Copyright 2020 Compose, Zalando SE +Copyright 2021 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/ui/Dockerfile b/ui/Dockerfile index 5ea912dbc..9384f90db 100644 --- a/ui/Dockerfile +++ b/ui/Dockerfile @@ -1,5 +1,5 @@ FROM alpine:3.6 -MAINTAINER team-acid@zalando.de +LABEL maintainer="Team ACID @ Zalando " EXPOSE 8081 From f927d6616c186de0aa1789d94a537551049da8d1 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 11 Jan 2021 17:24:24 +0100 Subject: [PATCH 157/168] add default values to operatorconfiguration crd (#1283) * add default values to operatorconfiguration crd * leave default for enable_master_load_balancer to true * add missing bits for new logical backup option * fix wrong lb tag and update chart package --- .../crds/operatorconfigurations.yaml | 110 ++++++++++++++++-- .../postgres-operator-1.6.0.tgz | Bin 18549 -> 19074 bytes charts/postgres-operator/values-crd.yaml | 2 + charts/postgres-operator/values.yaml | 1 + docs/reference/operator_parameters.md | 2 +- manifests/configmap.yaml | 1 + manifests/operatorconfiguration.crd.yaml | 108 +++++++++++++++-- pkg/apis/acid.zalan.do/v1/crds.go | 8 +- pkg/controller/operator_config.go | 6 +- pkg/util/config/config.go | 2 +- pkg/util/util.go | 8 ++ 11 files changed, 218 insertions(+), 30 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 2ac9ca7fc..09c29002c 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -65,32 +65,45 @@ spec: properties: docker_image: type: string + default: "registry.opensource.zalan.do/acid/spilo-13:2.0-p2" enable_crd_validation: type: boolean + default: true enable_lazy_spilo_upgrade: type: boolean + default: false enable_pgversion_env_var: type: boolean + default: true enable_shm_volume: type: boolean + default: true enable_spilo_wal_path_compat: type: boolean + default: false etcd_host: type: string + default: "" kubernetes_use_configmaps: type: boolean + default: false max_instances: type: integer minimum: -1 # -1 = disabled + default: -1 min_instances: type: integer minimum: -1 # -1 = disabled + default: -1 resync_period: type: string + default: "30m" repair_period: type: string + default: "5m" set_memory_request_to_limit: type: boolean + default: false sidecar_docker_images: type: object additionalProperties: @@ -104,24 +117,31 @@ spec: workers: type: integer minimum: 1 + default: 8 users: type: object properties: replication_username: type: string + default: standby super_username: type: string + default: postgres kubernetes: type: object properties: cluster_domain: type: string + default: "cluster.local" cluster_labels: type: object additionalProperties: type: string + default: + application: spilo cluster_name_label: type: string + default: "cluster-name" custom_pod_annotations: type: object additionalProperties: @@ -136,12 +156,16 @@ spec: type: string enable_init_containers: type: boolean + default: true enable_pod_antiaffinity: type: boolean + default: false enable_pod_disruption_budget: type: boolean + default: true enable_sidecars: type: boolean + default: true infrastructure_roles_secret_name: type: string infrastructure_roles_secrets: @@ -180,16 +204,20 @@ spec: type: string master_pod_move_timeout: type: string + default: "20m" node_readiness_label: type: object additionalProperties: type: string oauth_token_secret_name: type: string + default: "postgresql-operator" pdb_name_format: type: string + default: "postgres-{cluster}-pdb" pod_antiaffinity_topology_key: type: string + default: "kubernetes.io/hostname" pod_environment_configmap: type: string pod_environment_secret: @@ -199,20 +227,27 @@ spec: enum: - "ordered_ready" - "parallel" + default: "ordered_ready" pod_priority_class_name: type: string pod_role_label: type: string + default: "spilo-role" pod_service_account_definition: type: string + default: "" pod_service_account_name: type: string + default: "postgres-pod" pod_service_account_role_binding_definition: type: string + default: "" pod_terminate_grace_period: type: string + default: "5m" secret_name_template: type: string + default: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}" spilo_runasuser: type: integer spilo_runasgroup: @@ -221,12 +256,14 @@ spec: type: integer spilo_privileged: type: boolean + default: false storage_resize_mode: type: string enum: - "ebs" - "pvc" - "off" + default: "pvc" toleration: type: object additionalProperties: @@ -239,36 +276,48 @@ spec: default_cpu_limit: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + default: "1" default_cpu_request: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + default: "100m" default_memory_limit: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + default: "500Mi" default_memory_request: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + default: "100Mi" min_cpu_limit: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + default: "250m" min_memory_limit: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + default: "250Mi" timeouts: type: object properties: pod_label_wait_timeout: type: string + default: "10m" pod_deletion_wait_timeout: type: string + default: "10m" ready_wait_interval: type: string + default: "4s" ready_wait_timeout: type: string + default: "30s" resource_check_interval: type: string + default: "3s" resource_check_timeout: type: string + default: "10m" load_balancer: type: object properties: @@ -278,19 +327,25 @@ spec: type: string db_hosted_zone: type: string + default: "db.example.com" enable_master_load_balancer: type: boolean + default: true enable_replica_load_balancer: type: boolean + default: false external_traffic_policy: type: string enum: - "Cluster" - "Local" + default: "Cluster" master_dns_name_format: type: string + default: "{cluster}.{team}.{hostedzone}" replica_dns_name_format: type: string + default: "{cluster}-repl.{team}.{hostedzone}" aws_or_gcp: type: object properties: @@ -298,12 +353,16 @@ spec: type: string additional_secret_mount_path: type: string + default: "/meta/credentials" aws_region: type: string + default: "eu-central-1" enable_ebs_gp3_migration: type: boolean + default: false enable_ebs_gp3_migration_max_size: type: integer + default: 1000 gcp_credentials: type: string kube_iam_role: @@ -319,10 +378,15 @@ spec: properties: logical_backup_docker_image: type: string + default: "registry.opensource.zalan.do/acid/logical-backup:v1.6.0" logical_backup_google_application_credentials: type: string + logical_backup_job_prefix: + type: string + default: "logical-backup-" logical_backup_provider: type: string + default: "s3" logical_backup_s3_access_key_id: type: string logical_backup_s3_bucket: @@ -338,30 +402,40 @@ spec: logical_backup_schedule: type: string pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' + default: "30 00 * * *" debug: type: object properties: debug_logging: type: boolean + default: true enable_database_access: type: boolean + default: true teams_api: type: object properties: enable_admin_role_for_users: type: boolean + default: true enable_postgres_team_crd: type: boolean + default: true enable_postgres_team_crd_superusers: type: boolean + default: false enable_team_superuser: type: boolean + default: false enable_teams_api: type: boolean + default: true pam_configuration: type: string + default: "https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees" pam_role_name: type: string + default: "zalandos" postgres_superuser_teams: type: array items: @@ -370,23 +444,32 @@ spec: type: array items: type: string + default: + - admin team_admin_role: type: string + default: "admin" team_api_role_configuration: type: object additionalProperties: type: string + default: + log_statement: all teams_api_url: type: string + defaults: "https://teams.example.com/api/" logging_rest_api: type: object properties: api_port: type: integer + default: 8080 cluster_history_entries: type: integer + default: 1000 ring_log_lines: type: integer + default: 100 scalyr: # deprecated type: object properties: @@ -395,60 +478,65 @@ spec: scalyr_cpu_limit: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + default: "1" scalyr_cpu_request: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + default: "100m" scalyr_image: type: string scalyr_memory_limit: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + default: "500Mi" scalyr_memory_request: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + default: "50Mi" scalyr_server_url: type: string + default: "https://upload.eu.scalyr.com" connection_pooler: type: object properties: connection_pooler_schema: type: string - #default: "pooler" + default: "pooler" connection_pooler_user: type: string - #default: "pooler" + default: "pooler" connection_pooler_image: type: string - #default: "registry.opensource.zalan.do/acid/pgbouncer" + default: "registry.opensource.zalan.do/acid/pgbouncer:master-12" connection_pooler_max_db_connections: type: integer - #default: 60 + default: 60 connection_pooler_mode: type: string enum: - "session" - "transaction" - #default: "transaction" + default: "transaction" connection_pooler_number_of_instances: type: integer - minimum: 2 - #default: 2 + minimum: 1 + default: 2 connection_pooler_default_cpu_limit: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - #default: "1" + default: "1" connection_pooler_default_cpu_request: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - #default: "500m" + default: "500m" connection_pooler_default_memory_limit: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - #default: "100Mi" + default: "100Mi" connection_pooler_default_memory_request: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - #default: "100Mi" + default: "100Mi" status: type: object additionalProperties: diff --git a/charts/postgres-operator/postgres-operator-1.6.0.tgz b/charts/postgres-operator/postgres-operator-1.6.0.tgz index b57f99027d13b696e638eb10a52b7ab03e24ca4c..bf98cd8183d93920720fcfca1bfee716dabe6960 100644 GIT binary patch delta 19050 zcmZ^qRa70()@C6DcMlNUJ$MMN!AWp;cXue<-QC^Y-GX~?IJmp}>D>E|9^L)WFI5k< z$EeybwbowWocFPi{z;Ir0s!X?P2?rW1N&*rbDC=7SMu4i3&k&C}nGqqBW_KmuMbSRbIvM_e#S;4AOzydHR^Vc?^o3m6#a61azz&q2=q z2)nxs_f?k9V^0E3K=?}ryC1Sd>9;v)N?JxDtOQPG*(m^uM@W|swFzB3@6y%Dx&tD2 z!@4lpXsJo~3d6hblj^qu&14~2{RnHeEZp>uz&EwX)QMD+ayD#Qg7~t< zHvEeVxz=o)0v(DuU;~vsPo2RfgCz{H9LP}Q1Mv{G(2@@7S12osWQk~&OOY^S4I~6D&O)^-EpWV1D zCMq}A9{E4UZvpWH|FRA&h>An-$NamgJnjLwZ_DoL8~Ab;*rv^@)ek`%rP4CzCQk10 z_#1fy3H4|I3GJBIic?Gz<+k3t5NP&;<>s?ndTggeykp#z89&9w{IzD@G^HZL{Dz!q zNT#1%o+K+Zohdr;H2fr7Qv6id7tGwNf2*Z&B#V`68br?I<8h=C(AnZkARuU>4f@&g zOOOt#$IJ%-8OqV&q{22jv0pI}8)~8!To~$u4#)zPZp+Yn7OsCVh8rXyYGfj?FwjF8 za)p?7rlv5eiVa;Q^o%8b^wes(TtL<3}YUiSGT97~D~ z{6;?yO9U>#GMGeA{wSmpN?%65XC_^S6jm7bwfn-5qQ_A>ei1AxGub8kZPJKr^xLd| zK5-}$ut=VeWRNM4{Y@J`_$H2q-5ltX8bb0kK$atc-#~gStV<#_#J(T^+~1BzjL$c; z1RArk7Q|o4n2Q?@1h!$Z5%DOgm?hwq>?>T=6mspTe{ocd(aO&0=0}W~-u!sK`zw$8 z^HOelQw3`u6OqJdg(35|`spX~81}rCOrx`OU{GluT{&`(jERS2o3>>_SX6w0osw2M zC{y+O0z0xP87&gXz4y9;j2vHETM>^QBaFQyJqE*Mag-L1#d}gsoJlq&u8*kl>>0k0sg(41$=+Ao10wgd0}nZbA*|=?wt9LlZtm_g z08i;h_(0h&7$*NtBlw6$eeofuI76}Ip|-6wmA&5+!;KPmyVQebG`$nFli0KUCo~lL z>?ZNNKe)T!wolqTko4%&&~U(T0wt(AyD(F@G?9IYWK) zcqait_0oFF*OL*lz&kK2rkkKE2Ioi(0poN^d>BY0s1%qZh=jrl*+Irhq{lE&g`E2< zjJ|V^QG$+CZK2XSVWP!qO1-&~$%;&2Mqs+sIRsp!G2>9@;5`y;SmDWrQk33KTFA3U zbYW4pcxg^3$-zSO6^i^|?7`(JSTYkCaWdI9&QIf$k7zx?I6q6p;25iiSE7H~0D`nc z4WUw6GDxV;A~k4KIPl&~zj~{$+D-skgh@6ilQF5Nq|!_sQ-~iaWCl`6Rb1~F*b4PB z)Fv*vi0;M z%`pNkWD95w4A|2blQAi-V5ND`Wt@!&vJ768U{yK z={F}$z)B1DCvmOak}ZCEG>$DlCtdlP(+ta)%p@zu9fRP zmaPQs+S5SMro1sxx~t&MAz>Vcj!zm#&WRJhvL8XO#IB~rcu?%2M%%+NBor|^nETBF zZjznaiI;Q#P&Bt(dn{9HFx%fZvZ#$-M^}n);WhzxMhwa01bI5z1V|)+q4sVFL_N7O zs5XKW7x_+Sun?M}{FS=7U>`+Nl#?Y#TUQS;ahom^SFS?{?LM2vI{eG?ugE&iqgxb$ z_8-IXbc_)axEObe#b;y&!vZBze#B6@i-;k=zKf>nYtwzu!=D5}(X0<=n)K7~@RY_K z2bwBUxsO(K^E$p+NCNfyCrGr_Y&ft5?u4XL%{;w-Nd$9ht%i$6^r}ffSBZFH%OeoV zq9&tXap5JO2l3g&p>lnPYstx&S&fbK<5KG=(?Zj<_l1h_hoy!hgK!>V&+g~t%t>3R zRu5+9@{EJ$CTj8gDJQbv86 zyQhoN(J|5NPPlQd_%BiGZgX)(vFIsz$jaQH?BHLuNnK;uf2x{`hV#|asfPuj&foW- z-JO^W92BO0-h4XPH7wjz`?!V+qDwClnPcpNCT1k}Lb-7ZR2(<6oSTqlbchQb{aV=7 z*F@ohbOt?((GF;+_%F$N84BiN>CSc(X-Clq|8X5A*`raD0f%==%>1IX8s6*Fl1Kin z(=-+eA&KHKZu0}xZUywp(a;_=Ha;eVxEsJ{e;b(_UPfQ}`%eOzJ8l za?c-}gt!o!A9-2Eo6BWLNpMkQzrQdBLSBAZEJFB7+X3eFxLw67`|c^kO*BbAxfm6u zeNt=6cr@dWLO=7jkkcs^eY!WnPGtR>?7Av7?;S^)9PfU!a8AZ5?FI?^3n@8U?r7#r zoq18V!#)L2eZF1q_C1c>Afj1YRwj{-I`z|{xrz6$NsDUg6k%8kiFV?wpsP03v1lC% z^@&I{38l+|seSC}o#`t}GO;+I+KbnB^Bam6#GGKRKf*6M_`Numu2#`Sw$)E3a7uA9 zMt)erJmlwfSrokJc?_)ONWrAXC@+gr5B!jFPt7p^C^mE+Gjn3eYDsjI|D9M%ffuvhCQh`(^vE@|1)kW45nFNpko{+2&OWw)vH&}x{`-X6iR-rlkX&{X2bf80dKK<z=^e%5aEP7U?F?M6lgnk&DSnqs$OaB6wyWv0yLjz$VrLv5>YuDAiQ zgs-}p3md{)dimF1T8l0e7;eNre`pfrcMZcR%_kff~2PfT~_?q*MRV@(9DGn zD`sdHP8f(hj+s`bC9X`t@Co@(;9YB_PL6#~_W%uGi7U)QVj45JGm;`h{E#^T9u>cpdx4Y;4R_IG~fz+U|oAlh1v zHtfU5w)S(Y$JE|T5Ji&rr0#JcQkxm$IDTmiZl72F;0Qx7%M_jb%kmih35Tjm88Tyg z!)5{E=>X<*ldy4yM0b7jw}*Itce7`dY@_E!PG#&_w5 zo+??hv5LNUMRe`s*1A+^@^EB$rorV?y?BI>-gro}SJ40uX}>{J6_0Ka1*^(*`k+9%0DQRNG4`Iz8o{hbGZpwpdl*gj6| z=;pv9nhB+YE?$bhit$jcs-|_S3%*V~thQTL=#L=|q(}#$Wy6J5XQ$WCe?Q79ona%| z)Z7VZ#TS~xWkhM?b5v<^F3UppWQ0j~GanSmM|ZG$Pe(2804pOmebY~XdarohtkCh3 z3@hER;X%jd`O=IE3}Hvl=O2*O4ZeJWW>bD_Y9k0mctQhu!#dJ~6oDsj?V2fjwdJ`O ze#rrKQ1}mMzJ$>}Z4|=ss(%ifxnipSrg>rSFUS1+%NcUZHDhUPA(Nl!7W~(6`8%}d z3g@UhR(Tf2p^U$Q$Q*|>Pu-;kcQ94_DUL}BIzk;~+f7m>Q6f=zTLDG6>L zi_*6^HZCZbdf!DNg)7-8Harmh+s8i_gr`zK>VI8l({2KBV6^gtDY@b`R(uQek0y<= z3;Kwg96cU&04SSmRrjvKCq;-<_1XEE3%)l-CCFgIpII^w(5*0L!9MZO7%2!lZK^4Z zd~SM>F;k!!nE2VF#p&@`-|h+`;iXedTqQDNaiTGn1zm#xiOZgVa4h05UHp$>kW zjC;C@V+GP)AxAkcgj>-6|q2%-k10qulrS?2}= zQWcALBaIT~!F_}@m}7iITu_+-eVxqM)QWV4_G4_Zv3u#%@M&{+4HjcY;lUT8b24l1 zV-kgWCFw!uZ^~`vs+z`hvEr1$f++H@md?A}kT#(de{f5213;jd-riy%H*U5Q?DU<# zi^6TpKxD75nGwaVmQO-46z7yiGOQt72@FYxpqHxGRcB496H^qa*0Rl=UjTt z{d|5Yu@7x)&{RH5BxS6)Q8?BZ94>vG^f%;a7t~S&^C{e~9Y;9L<+yo!B@s&>_+p5B z9Q||RhL`t!i`@;YBDKuI%>GyByF$#Lc4s=PfwSt4xMp$~&9bF< zz;cq6?TWp^y43K`7imXs@xE$z;RcC@)|1a7NE1!~^SGD`sVgY91b1)tlNZRI_W%s%<~PmFsqY8Xkqa^}*Z=7tkuaFgA=Az7oT=ojr{5eHBomty($Qh)G4$yP zYbNut66h!~`CUV#r=mF9S-Y$i8aTQS9G!3`owrtcpm125t-~c}C#F~XwRTaEhM}3r zI&gNgkOi@rOT&?pq1MrT%X;oO75s!-KnEw9k7o8+e@FpYkWay-ZR+4fj~_4WQ>OY6 zHbQrcQ1x9`Uc;xV$~1&M4Ixf!$@NgsvgkF;5|P=n$ElP(&^SMW>E%6|C^wPZLf#DS zR2I}r7$hHaw(|8qvW$f5pL)gy07w&#GZ?@29e1=*HD{r;?j3jAFj0E3=$Y^41ut@-tfpFEmsLYHZ|J>3P%`G;L&w z+UzD-(-GLM;mX!Xak;+ZD_7-rzzy7K$A+bzU$p+J^fj>;WSb0N)u}*$cz;?!yPFz; z?bvoqVaL-}hVs!iSUy+k0sh9my>yUnRfVQS9Iu5#G%P?T`2Vq5YUKZAM+ZHV)^i;j z9=i|8#3&a3&D~>;|F%M5?`|;T=?FJcnmDjzAS2j}%{rklozHlk(Y}%g+~eBXV)yZQb*y)}{dVcid-*wy zCG2;$5;L(CwW3BgmYc9q9CB2eQK5bzCdOyuy#-Waf%r{m=`YHakBeQDIRXjr^G6~m z7;{DLn-${A#P(rP8r zof^%E+d2}zsQy)Hzc=|@zaNjBRpQcAhF@PQyKMj>XM#0H4A`?H_PM#kNOYS+rqg4H z*yd93u=4JtL_sqBP!+-~upbIM;BL%YdJZCWqNzbfnI{qxXUHFJ9QajZ5kMG|ZJ)f{ zYyN9VJM)L?UfliSvKeMUU}>gUjAzz7OB@XmdZ_XUmP@X(KKl!EmY67Zo?vH@OMslq ziQHc0uV6}E3Ho??CUzS&Bc|r3=?XPmQFFFZjv5}|0`_bKF!&qq?>yrBB@9x6{mbV7 z1q7o;D=y18T99=X@yFc|h*JdX`7^1#`1t8ZlXScOqd4ZFWsMi=bk5=tun0>n*6+Vs z*{yF=%JBl)oPQ5`llSxtd7UP@>v?tK_qn@=;?Dm)-LA?HTs)0#Z(%atDM@@l@``s= z9P0G~lQ3EI*+U`*BEHO`{Xb~u*^ARrq;)I$8Yu|Z7U2v}oI{EUh%Wb7EeGe3lCU4v6w)|+@q-XCvY0yQ z9j~4zMN2u`gu?8T=PllOkDTzwUCB-Nl_6=R_NCEzip!wtuVGD5^gZ8P7=n?f1cM<%tnG3G{0R6d5S^_7TTm)d~B|)T9&=gHD+n-M@>8ptx zm(}#^~(YJdINTWmQ*t*g|j zyC`SnP)TKJGhXIQx-;vM9;Lpa8bgV3*DNyg0QgRHU(_DwB2>)JEq-1qpk8d@jWJNg zQrGYfJA2jTq4@n>B->QWQ&ZA#<$IscwupVe*T=8P!B3xQOf|Y8J*;Q|%$@BW1&UuFz+vXvxn!(f+j793gdDu`I zF>VPZ;HD*59(|R5*|Le)jE)yD6?%v#>mK;WDRSgs#i&C^okPua2GOc;Os5tRW2G+$ z!_kB8qc6LMHim*p4xOEeg#)P=ddvX8hzM#NYdK7z%xH|_;Z&Ekno0{|E8x9dgREek zp5P5iCApN@KcKj*e`K&|(W>$ig2GL(vyZgaLVA462W)LQTAXwna3I))Axt<-Tt=HZ z(8{yTjiAEKv$;~YedcVW3lJ@Oe)Hfg=UN2u{0((z^r5Qr8;d)S8RC!)pMU_`pd}-` zdG%MvDHn3nMRY`E8XNOUYgjQE!_WlHADySf>OR<;*q_yxlQ9TB%cdaj~ep8)2vr6HoY9LKtR_(05ll1wRJr30qWw_9RWXn zdfx9tcEhz1jH^ZDBr*`sP)%L6|NE)&EUL@+u;SDF7!r>kw2tngtw5`e|8Vp7oe=CJ zXSC$MQqK*(#)#eUr;>_ZK750nnaK8U@kl>|^S0AZUaKU5f% z<6=I7v_?8?G~t%5f3SzrIG~4IOz6Xzq72k~E-pQ28%A*#wnVM4ywiV3&KIum$@8OU z4>8Oe*(#g2R+tRb>)1V*U~4V@MUHrFW8tdYxv8{{>>KZH)F&K4Xyx`|4%;m5GE)3w zrRK##*r(96rGNc_lLI83_C#Lp5zRCmf?q-+4#6nde3{_3loA`T80*7?0Gm1GBvs@a zcZtVK;F~);mx0R@)bV>3wamQ^rGaRBu)IqTl-#if-nFT!uX!aBPqok%qCaff6*6zd z0kGfPlZBxqr20fX=-U zzu(woY(Kcyh*0V0{E|-3rt-AS{K!o|cE-;_e0SeNO68Us8xtXnSHW+AbPA_NrD^zX zWRY)ek5hAVoJ2(HiWUpgfOjfU_{(0WH0Nq?TslFN)#j?st9x6Sbd)nDq7SorL5rQ(rvjxEl2tAt zj@NfgdmYp^29xY}X8%Hz-RL<_N*o=auSe@ibk9>WY* ztd@DHq;Z2@|7um}cJp~$zZgHoj5oAwpZ+`g%7MD+fN`|WtD$jb&`A1P82nF&%|gO% z?jNWiqAvHg$hYF?VovmJwCo)1Zcpdy8&+N(z}^1t;w;}M&DHH;hnP7Av_VRU?ETkG zqETU*;N@szXytT?nORr@%8PC6kQbT>V^@yjYf35AH(u%?U2g>wat#TYc?sTxl@P4yG{fR;{ht2@rY! zU8);{n=JXb2I_3Uhb!}aJI|mlQ+eXEeys9QvFoi)V3XNqL$A06(i5p9TU(YjfxEXrvnH^|nNfJzbCEb-vzE6#(9i{% zK3)@O>=eKI(t43I>S@2}f^&OnKQ1o`d7psG{gq#_uk>AA)q$p%%%8sfC$9dp_X0di z)}1_s@mLm>TS>cB*K`gq)oU-|o^3vf_YBOS({LP*lbMzI|iL)frPM0M~ zCeF^?H>k|SSmTV16dOY9a`bVVqYT`DPGnn9Al2x>xVt^D|C&S8YtZ8jyZD{D{ym9@ zZO>)NAve8Yo&2=Wc}hEPsv-VIC%7}FI0{QclngK4_0QpzcmTG~1#X2cp-ls$V@Ye*oid108@r`d%F0z^|U>ppdD zk8Diiw>_rq2sPa8?LDs_CV}-Fi{x_^-?OBuSi$^TZF%j(tV<7akH62=CG(xP8{oASQUS$L zvAmJvQR% zJkBgFVmf@?ma^#J8+)$HCe=Oy1dC(xYY9#tj;H<}z{HF}abe>C zmSriYOzc%q!bPa&449?2Rpd|kH-aXLD3oZ3XBTz1u!Vpg$2l{Arj0qPNJqrOeA=SC z+r;*BvvsPpN)kvMnJUXhk*pyq+yb%d7W8TCKz_!zkw8oYpce%ARehLJPW)A3cvR>1 z>slwPAv&V;72kNl=z zA2z|oyK5fj;aZR^8LT&J+e+b1IZHb&tV7-(=i1n7&l$1siJiLN<0ARqmDgBmp8XG= z@8iy@2bt*#+3fT2^lM7f5=ww?!;MbmdJh7VI`19X%@vmSxfo@LWlc)L3>hg9Aw;%$ z7*rc1AKaoaG>j|K|D6&>H#0=An42 z-Jk7_@+{K>hdJTN;|`QS)`0vm4N8e{ld$%4uT~;}d9ViqIljytCx|;iYfv+0+ebkAtN5tH*uFI+F+-oS z)%C%}FGSDj2W{5S9hQhvB=5G>9}>v%=?rFJ)()PsrfRGjEQ++pUcj-Y3PspQ%D_*rO+FV5&Ff=&?esQk*3XMKHVpyf_IyR z`UmBt*c%FhKw7*bx0DNqHfJsEk~NuEw?2Bob}vg(~ZiKgROEc^tjPJ`R3b|bv6ch`$sgU-cd>EGJSYdEB>>ZKTL1w)jd z+=iW7y;5u7mDb877E6I&a8s0(M>k>GDPf zGHnSQskm>-5#yV|^r{dW@mx!0SVn9Q+N3P$gs}I2j0>5)Sto-g=G=$U*Ze3Q$`_hq z?LEu>MttSRW;r+JR4=PqoTKx%f%s#&T&0g!-G=zx1tyispmm=RlKpN2TTs?L1tJO#(GWOq^Omg9^oAXqerkEK_KzhS*1n1 zn|6!`8~q>sBa1Hbb8X@Han^3qXpGGeMUEm$t)k^R_$aw6x=`uy7g4p&v90e(!R@Wr zMG|WrfGdf^M*fEJ_qEi~eW>Ay?ag2IeL8BvuR<3GPZ?&9u77pinuFvEza1jY?pi=; zwds>Vvhe_+h^}~OeVhj657XK38UtP(YDG;5C3w#6Zl(a$tU>*xm$zcV4Hw@h@hT1L zl@%i7f=;tFrRChRL9sjJQQeQ6pFKl>tiU@C0KT8K?E>w@1W>=`5QERJbHMc`qOh{N z#|fq_0^LM@?EK-!b4lZ zr>uv|jaYs&a=!f)uHJf2*vYT@rLQV&`j@$R(iMfDCPi&1H6mv|QL=XyYk|`Ec+7wc z$E&3JK);eZel&m1m{bxn8*$qjYXNI5Ha;2Gzs2u9h*VmJ!V-mUuIHwekaz2&Y2512erSoTh}ZTcVX+DxecYYD;}yEgS;0ai*r1Q z;|P%1IMbbM(0aa*`t{YqbND1@m<(#dM_AbK9GcLDTXn!R_xks?Xg)66jInRrPuvu5 zvu07O{41)2m2hOq=8_4yvq-fKnNdpLM;`|g8-|zs{<)&6aLc$PD?uT6q%)K!pqsgLrRL1$}QB*NgT-rDfIJjqZ}>yHG8i@z67})TrO_>vNfs| zy4^y!bh}UQNgMD4psXwe1_96g0eVi}gR;$1(b=pO%EBenn z?9QHo8IC!QDkB&{38)OKK!FR*4z~SfO4sIeDg^Z_%G28SUot66aC^^%k&HqWU5wu^ z8T?@7tcWQ@78ul7gsbG@HNZH}H{6UhMMT&|V!~mR3@9g<%GxjxM1#CqZGNeR zzP^7P!uS48B-+^2VJ z2eA3kdJ_*90JSf6ulonJ_R!kHZ$th~Sc>5h;*8VDGw~+jpSVw5(igF`ZE&gJ;fLJ* z9^2LbPO1Jbq)F~zOWxmoy;b`6GjJ~UkZT-gUGk!-&8b-syU4jupe31tZ^*{yoGfXn z*2GyiV{!hndOZEHCcyX%E{9@qwS#&)c>`eI-``6jfM*cwGz!rTc;nBl*bVr3{nj?_ zu7>Q3j!N1`IjZ#Wd#8B+D_sfbQ{%p<&vTD8LXT@?an|B}S@P~A7G->ymQnaW*3a>A zWzgnNjV4A%THdL-`22ipa9cK^HK9Yq;Xl~bg>T8f48I~?+Mw0fchL%>7aC!!nn4H_M&>JCN{dxisQ5(#b}igtdh zqrsup<~SMc_jT16-x`1YzA>4$5G}{u7mLc}%4M;Z#iY|}p+?+Zpy^0J3NT*$nVLwz zdD$8pZ=7E0j(U}sNf838?jy*xM-7F0=^SU4z z*iQc%@=>%;|58(hitf|UcqQ$&<7i!Qttcz8HGkndA*r1P z5F4MKSw58-G#Z!8Q=l&~48v&6Sc#E!GRH+%dmFYNpg!H5l01Dpf_CHsKs*1go;?o% zB?r@80lKe!eU~x7>4t5dwx-#1=LWcm`0Ty=EGjg9I025nO+QHu4hM_5rh#*GwRwYT zn$hs|g2L@;%KvD==Q|FoK`M*#@iTA|>-TVl+oVgSvk{JVepkeEmHfVn+6{iMi3#}W zy|pU;?>zxspo$)V@a!GM7^wio2Y5T>d4DCpMbyPV*aW{{U!m7>c2>M!AJ>APW^MT* z_HAQ`-d~n-%eqy&CRtiUHf8bRtok<(-BMr_bJqwM*}i!TUtxWAce;Kw1nPTwz`BFp{JdBT9c{v9?&ULw660zsDEo~>tS`lNWvN!A z(a!l|1Lmi2Wfpg(V~1>yQmqcPuUkUeU+tLT&%Rw>1BN|L}L7IHMMb$jAmx>&QwC*oG{+xDhVb~AEiU-b=p;VIZO zfQ6-eJigq*c!B*4eInGNstlOMq+4xU*EfWwE~FssM!l8F;s%X?F^VIyXhE{-C}in5 zVRKdk3X|W$lo9aLg4ce%4h};HQ{(1>X+541j5iMaGJ6;+?EVDi;;{?RgOj#J(NI2$+<`|R)T5j;dV4bL`M*3Xx5=$q`;Nl=HN zGRCm!ij*qTA{8khaxEuLs%(~sZJ?m%iUDWI^|WYx5b>7a`C_(!i|qBTS}(^(|GLC= zerFy;33igaFBvEEQ+^yq`%}l zrz$dvxwQD5nve|@)BwFC=wu<844N8tcU{K~-|`ZE8EpfpH=+~GvW6-0{l{iS)z`*T zr4q&aUJ0Fy^cWyY1LyDy9>U3_v3uFxD%Xn11~WP_DugCZ8^<7dYe*@xD>2Pt-z5)Y zC#Tyl`(*qYN%IHNr^(i3zqu5US4pxpegD90t_^o(B|)q+t!4eFH%vCCBd`A%R#SVdV+mdJwz86yqm)h4aZJA- zXQpd2YO4#7yBLbKw4VO41`?0|SnzpsO-QI${<7LaR+}~C1NvNb{b+R%zGC#ttFR)t z80K1-qJpI+n1*0tG>XENp!HlKw;HFDp0}$?XOjaS&p^e^^|F^^BjJOI0I2KTaAHkQ zxPwQJA3Ud2>5U4i?xS7vDQrpicsntgv&AZ3OKflQO>!FwotWNa;wUE}%ul!;G{ThRvC!!6dU2tueRt-G zNd5}&p0+Y>0lr!hWMQMBaez6CexM0U8Uih;XoAV)nmHU7<|+-ABdf6_KNE(ClyeoW zz5;@zLNQj>w|m{z4Qf_t_cggNNQQbNl!etUnbVbwE4=Z__qL7Fwt0_xlHLD<>z5}S zcesTJqJ^tP)RWJgI!MwFcEYOdoc(Lf@#Hm;& z2a#V`-awtY!N2Yk^9eXGFbo|Hvt=_QEl>BHneLDeKLo^t$kMrWoP^Sp(4}qyZzk~Y zidn8)CO{3}ebvIT?X;_e)0)3Ti@+#EmKs>;m6EzY2WAS8US`Lt%8z7F@0A~ps6(bw zNuhBiILrf0CKLBfB;s(>X8o+zg16Lr|5^Hkc@r)`YWwd-4(7yj*;t&?zA(3*9#-T} zvl;kPThD2KeRAFeJ`;2GfjMuegh{ zM-~l=BHURr!=?$atI+%g9*)o1XexRnT=_g!hHceAX{`}?afb2sQnGpZmmPf7ETO~Pw^6t;*N&hRu0`#53SgZ9WT=;McQBARN=S~_YtWul3W)8qa9J-)m z&!;2=6aMI4D;!--{M3|4wSgG-jaBZ*=r~fMO&KdpMjHQC0#m$z7Tsetlg}w=xn1l? zQ&9Xila4asz#T0X7tAZr-r4Eqy;AN{6L9rlMTjIpKCtt9{BSoS<>$XqT&G!)p6q*z zsU?Y?H?77K=(nBy!^=KCz6#K5r&Q_N-$0wtss0Q%a=Zoxk4MbAR$*0EzyE81-$iBc z70)tkYo>Lwx~+#g$`95Wl%1_d5^WHmhcIxNr*(6Ph`V_BEdTTnjjc7@+l^Ix(dgFw zaQe8}DH%_}t#YdT_c@iw!&dvxdFKyYS{Mvq^@fJJgYuTw(M|VVZQ!Ap&>@2z#UQ)I zVs0#uj?tMoKEmhRRQ5x@Gkhl*Pf+y`d?rAM^b+wbki*a8H3D`gp2nR7hyAB*gj}HP zr^nP%;TC+CuTmGxE4wo0MlE&umcifn*CvpD8^0Jp52E-B`HkyQU-pb%%7P4PvL#~}Czj-epGrd0SVCCnbOg1vyq zT)&F0v&1*q;d`a^c|k90zJu+R3-v{?J5Ix6i~P$apXD7G=n5!_cn1I5+Bz`v&waYS z2e0JrZVa>x;+0x4DUp;8rqpb8+b@{T4t%HB==#thu7~FLe)T|Hlb@JRk6J z^lBRNUIJl77kP6&Wi9hh7b92E= zL^WLt2!*o01$R>W;HUZ4Cf*oHly_8rv=mjT5|INUm^6oO8kpKH%YuzJ6=#_3#xo;E9BY@T_hifQIN&02G7z z=hf=!>{#gS-B%o{`R8ju-=^N%+i}Cd_3fp5@qLfLGBI!M7Op{GfC&zJCWn3+ zZ8P9sgMyO7PjHr!e{MPcq&$7|665z-h+?-x;gd=Y_*EV@#1rup+}F_66>|TNG`Zmy zbEVvgt!)2_j&LAs&nWY|ldewR-~0zl{Yv=R|IbC+H>0F_&e&c zl0dUKSon;(zy-NT4?J0hWmP~8GAydNKlR#6#vqy6AiG|s!`QZE#8B-cnprL~sHVik zSafM?dSB>D&fU<5Vz>4OiH=4s&*yEW!|v_kbNe_Nwo;gC8uo4TF#|&@YDWyjY}qTp z4hf7~0wJYVj2C^PetayPXVJM-W!#rbv?2EN&Ke&dcYAk_hvZr-aT7oZBb#>83<1|zMl_Tqiwp|``^mgF$XC0tE?F( zHC{(GtL_0A69Fox}4UZ zNZ1`GuFkIb1JRs*qo21Pbyu7Z`g#U80T6~~&DxiVo^LeJ@005gA!gwqJ%o66`NUp2 zTjj%Mj6npZOBb6yna!fkllS?)39|ZW&G=z?Z0?s(d?As@-uOh}4T4}bjgM&Ei<2-y z5S}re81Wt}%0Yn!1*(}!pv*awY~|fO%TUP2*syD0}NBx z1t(G?MbjOuKPP7FBH~&GYb>7%pTn)ijUbbF(%>$^4e)&A(g%i$2?KA}i26*}K1(0M z2ep9Nk7KfW3$H>N_GzmZpB5#Ir1?{%nL`CpnX-C7W?IZ`K+(YDy(v~T_WrI7{U!;m zg=&0cqTOM+@7;9BGws2&B+<|qpwhV|4<0=U3QRQo(?t;6d6`$i=LBnY+FQZ+Nf2`L z$-$Ut(mHw=rO%(Jaqd%HqN+G%ye&09x4WmLf&K*2e4PM4^Gc?cNTD1&ng69HXT1g>50&`kP#)GmG|a{A8C|l>SP;JJu0EE!0QgIdVsz z)jQ~DjPvuuMS%w9KPogJV%Rrc2nH9J?Bj;*=+I}{vD43ShLsijF}RGO(S^UFN^W#d z&-n+E>r4ri4Zd>4la zPqs1mpY?T7{Xf;V%d2)tG0ws*5&?!$9C~N9P#ih|;^-Ov2=)nW?Y32uUlGzo(PrL|kpbEl z!eEnhOp6!>%(%Z?1P+}Fvt^RKcIh2E9~x(R**TrQQwB4pfUTPgd%?O4#2goXZ!QYs zffSCOLSO+IjJLZak3+qH#znZO5$5g6{U?PQB-d>%`|!B&<6lC|RZ(BsW+`c4WkK7l z2I;1Tl1a&8g^2KUjEXLVtXi>(MZIkEeP^6L+h_23X~bK#88;)50G?kX51bomUN0aFDNS8_v5#O`)43Qj>}gq3MMlI@0UVz7XcBy_@_# z+sVg6cjpviWi;~E>#cPpmU14 z|E0r(UL>0FcA0ngwRHB?rH}tgc%)7e6S;WCXe)TV%Ts{-2assT7J3?E3Y$s_C~L@U z`TFf(3*i&tdNuGF325bRv1y^L8_nJ+nDOA$zEAh6vE5eFk=F(&{3rR zyqrM7M*!@vtZe}QAc}Jtz09dxy_ka}m$Tu)>VV}2949lO5NxyB@Vv~tfXMGA?d_x& zZgEd46Ei*$MRAcHwhHD{%`)!6>Cv%qcr!+FN6U>E@emp;=ANw#ME_!v<@|RwOw6Ke zgoD8Q*0?z+P%fpX1FchTaN+B$I$n-v(bB z{L&Hd!S5qNCMR#ls+$qS((Czwq;m)VZ$5#?rq$ zIdI4hTK!M44MN2uN6uFmDvVJqa4*uK!X(u#bzr_nN~*osP@)tkg;im8tQQU+sR8dr zqF)s0;ve>t1DwfbLMz~B{#fd z!Y91NyZ_i-rmcNHOe?|r;-cy`kwvGuAmD!mnF(h0!+luW5Lb8f0DTtdluEpV+}=Au z{AqwumB}1+{igfbQ=a*qxOZUN4PY(7+e5DR#NL4+x>M??3h!Y=fJ&sEC&QX(Zhs@d zmLJwM-0LaJ$Tj8KIbfF;KU^2kj*YE6hQExtWU5vw?Z7s?6}nkfU`;Y|(XDEz_*%&D zM+;b=4)ZA7TV@)P|ilxqR>t!{x9nEw{-4@5JjKR#Dgs_IUawmO7z?Zh_@FC_5GB2fNl=`Hn9 zkVmXH>5)sVo*1qB@iNBR*wn<>h)wB?3-Dhig2rTI&!Rd;T)dO|KOBFLU}u2Ep#eE0 z@u+7#ti4DeA1!*h@;sTNw*d8>MT}s7D{IBqd|#y|)ad_uZ=GEqnp|QvcZI{O=Q&$w zE2x@QKQp6O*_}>#ui9??4A$f{aKce}5_s83as#Z^mAewHEz^z{Ej!56FCi{j8dEBA z%Dv;y8YLZ4mJeVlsi8$K(300@sQqV>?GrOtE2& zS!PU*gklJRBx0f}|BI(f!{;y&fTY3MSK=ENS%h9^uIK3Wuxo!Ksr1s?qV5#?isRfH zdtz{UTAY_FTNYd`FJXPpxjoWGS4W%mq{WDCgJ%OJ(g3OCdMwq%@Wa z&3R~k4$1TW^ZvqKjt!G^`lAzbQ*Lz9IRW+<72`4U{r2{2=wd95&awzN0dBKkJuqm> zfyY?L3ZWS}!&raDR!hVrgz5meF2`X_K@|dR24ev*U8^(s zD$213kE$I8=V_qV%G8)dIq^)Y$FLy_ma=V-^CeW96wZwV#fGG77D($y2t5D#In~n& zngK;eVph63Ech``n3^li(o(EB)$x(Faau5APFB+0HTZvbV=gUT(-o3UOt;ks;N|5D zXJ;8+xmF<-8xmFokr`o{1OwmqPH3S;N;Qvs*JB1KpxY<;o`FBf_dUpVZm^LGC;5I@ z)?Ajvs=P`7Ys1-O(=)GLkRsDOA=gtLO@R!Lf+@{&mMM?WWn^JW--mG28CezzzdAnm z2OB3noHc*&91k_2v|G76CQ%-Gl`H3h>5bE)H>g*d-Y#H0w*$;++M_A`AToNbAjr?^ zpQTINcOjM7u8I*0IZeU0QQ^AWC!->|!d-viA;v|F$wY`r!Z2)g$#eII5*dgKH>5r_STppt7MgiAmu$@6m#e?mbi13l z0QY|nF$%HHkaX)U2)Z?o2nHfCNdm$+_+GkdziUKncM`!#qXp2e!Vn5N_Bq0(_eVA- zrzgvm)V^J6!z(rqd0fTc_?(CG&NfamRV1+Oy$q{lw{#xh@1T%L3DJsCF}AQ9VK^aL z5#s!_q>u^f@|n=~riY_nE*qh^IzAk`QdECk(pb~3301fFAmC)JguO$;vPjNzYb-Xv zQHGC?US94U9AbUjfi91IsX3T=h-zR1Aa{2v>F)PQzfZn1|G;dviwP@jR@1T5BBhx- z`DH{xxqIU#fO8+1_oYf?N`*8|c?JR!h4_@t>;*D>GDS0XB59`WW@RK&utLqkb^d>< z+^egtBNa%a=!Ter$ZB=1E;|2cE;+tHT^IcMONSsa2;UwzG-h_SWl(MhBd`#f7|FP9 zIDvpL6~TwF;*hIt=K^VdTbD3})q&Gm@DZxcRd;%r1RyJ0 zSy(6|P^^?LAvE~SJ%}o#Ij?zY(aV2pwJAc$PZzKKRf!gQYQ&=rA9_?q7nu=)?ZU@a zHm6}N7PS(l6*Itb#W)7{s@EZw@tl^9E^ydTHD&D`r=UM~2p|H+!|h3X6aYtT@zAPK zid>&(%qV$N<_!x>37%o9sRplJ757vP1oooaqh9`Qx8xlqwqh)q^!1+T!0obEx^k3z5ZkR0}%RRghprleVF?1oYU4A}SSYDqJNCT@Rs!Kk*x_|c5Z zV6b>7#(q@nUX2vrX}b)daogkow@unhx9wxBhF{y^Sqt2jM$jEOq7ky}rOgth$}PT& z7e~icGu`mj-0b+vt>b1*ZGnYe)yfTB?Ikw@-XXsj^$u0LDNEA*Cn)O8znB&KzshKR zv+{IDY>kbTiiz+j(;U* zlo#d1E$_f8cP;BTi9y-HV%c(E&)wTng&~0|3&CkSoA`SG(M=v|e8_fBatdZ7mVRp!&(j-T(B{QA2g;H`+6i;mlh0JF>m zlY+-=*R(!Ra9Do?(D`ZT%VSeoiX>t3^rU^}SEb15i8t}lu@;vWzNal?%DpF`q?WU% zjO>MDX+|Y3Rc5^0GD*ZWI<2b^WtfO#QDoK!!&Mzmx1LU@qP06D|FillI6Z;838$=# w!24gG#Hw`pMZr*S{5?pluYKb2;dyu-o`>h*G0*=O000XA|9@>f)c~FW0My@*asU7T literal 18549 zcmYh?V{j(X)+pd46I&DWjqPM&+qP|66HIK|$;7s8+qUiGy>rf~uj=-X?&{jD>aOm+ zd#&{lMna>5{AU7EfzTLAC@~sK$g<0LabF#~;sjI1)&swyX$;DF@Piz41YIVlWhJt@2%#GAU$~Dr|C9T$G9617o55P2m#w4v-l6|{FcaMb;a8Q9JYS)&N_C(Cx&hudjq{z6% zLT-c>Ea;aQ-fQ&Pp|GWcTZUwX4iBSE?*pNhWyAd$&Rl3q6e)z3Q$l;J(qJNHWQb!M zAr()0%n?{xOZGTG3*JYF@uo4K_9Th>4yH(5lK*{_nJZTVX^wdkY^uy$oNy~+DP}wn z+=i}9z^$TYkx2FTNa@yl9{Zj)s;x{MGS%Fkq5mNAp(nTN?e6^a{c^vvfRa!fgm|K4 zFKJv06U(y*X3C7zxP`PN1F_7A6w`4H6h^+y6a&CqKu#s8Nqtz^pHFW2>!+mQ>-3Kp z{h!$Lr^hMd8u1cEoJ&4wj16*fqj6{2!_3T#ykcJ7we=hP&+&c9p^)!SDdhQOcz{|j zAT$0@$PzOKXf72UU2cN3P8#(o;edIZ(|fV8Q)os$#KDV7RX#6Y+LBz+)WZLsJ% zW1&&|XdPf;U*}ea=AP7yM2qzOjKzo*-QYOsIKslv6+M9whea|c3lFKg$FnR4Y*mUp z_>YmHYTY)`Wt4^phm28ZW7rpqh-14_@O9v9a15N+$5)pe$LI@)gVau9r9`Mq;O0b9 ziRNI^-e0A=dRWy^aNc{~G4Ok#$hahPL(O9ObwQ*uSx+~lyhmS>*(^!={RDMDJQc=I zMvH`_Bx%6f4TQ4RUZ7wbe4Sh_UeSd>sV2q}Xc`CE?=O0xFbgrKu@a*2G*&>S$YO0E z*4mrEVA7aG5!H&ALE9U_{>tn=bQ5&`(9*AsyHNznA!7{1@W}8zV*k^h{eyt~83@?b zpM2_H45Im-X1y{RWp${@3mfnPCrTnRW%W-b`UOH#DQAO|*V@GPQR`Cc41$m9LYa95 ze5}qDc&kFH`=8BhJCQ-6%qp}7_LV!jnx1~WDtD>35 zru`;?8fLv)F(vnPMWl2ASZQ`e7a>3GN)(j$Yu|i2(&OS*MO1gPQ zOh*adpGCd{W{pVAAiiz`X2PGmjd=gof0c)rmq&=mmkyH+IS5Fx@EJ*=l}2U+9GMwm zTC!xoEKCj%WXSI!u#5xz;3qRbjGKeho)-aTr>i(-+G9So;G{a11NMIWCrXDd@P+IN z06N8F6N|kH=Gqc9FLYEAJC>hw!DDR!rQ;A8e=jvv9bUMl6SSA6qPXaj)q_+pu=7z+ zIZa52({;OBg+zmfmW=GmLzC~dmGK&}Vi8~qLuO^1g~Ume{lvh_*NZ2|L@%kJM>R4! z_MeIu1-p1W5p)Oh{%+r0T1d4SI?S_4d(q}lOTyxjhM=v3_-<ln2cFLU@hRe0rvo9F$|1w zZ0&^bv^7-%lXAmK6&Xzn!pq1zH# z`X1Ss5db^dF`sov_~EZ2bW-9;@pLU!W_k;z_@A6h6goQ%NGovB#p6sfR@KAPd-z^h zty4*8h}3%Zw*A(n=0y!w>4!lqi@dJfPI7sAQ5m`MHwnD~AD@Ut+zEby0tL~AM)~#P zmfb?r;2A*zBu6W3TiFcHGEsU|rksFAj#iApm;hc1Eg|&@soVB&dwA%ieHyQp#tO%w z+t+EVU_6M(GyQl>7@YE8+4QXBi!$Qz3N>sLqA@}HHCb~|`VLUPDhVf-Jm=p$OgL37 zR-i-jgi-8xKt4-#@I9J|z-e4#O6DKGo6!s=`!av5*Xb@pr=1!#8mLlotEgTvx_1*1 z8!GkErZuIOO2AdXc^Z*t8##@Wl({NO9Oh@rS(vADxKRs}2^C79i@_Q8a)-E< zfr*b_^T%#?xr$Lw>nX&9Hb^zN8XKi@U24K{HRcV#y0DcY;G2xH+>&mhxbw(=DoxJ% zMHa_LdD$yjk~hnEhQLO}#beK!h@Fl$D#^Dxp{_hTX!77mRFjiSC8NoJUa&Us zm71}wBueN1SfaR!ofdlCq%sw!NT4+ngC;G1P&~O0F~2i^WlJd-q=4Z!0FUq0jgehQPt3nFrHhSw?lX>TYQ21<& z*h>tq!N?tJH#|e2V=xC{@^V;78;QBP+#b30KL#5nLOIJeOQ!_PL`zQn{lUi78AaYomwHcwi1PHC*d z-9y<3lf{wDd%apxeQkvBa;-F|Ub%_Pyc)ycFOn+-k%vlN?6(e>ZbT^Tq|`-C$%KcO z4r#Jthcdl*lB}`LrdZ~8q9~|b8Ju^Wz3M*TtF#TP;5Pcr2j?PW3U4dtIlKTM`&MCm z#z-84UwZpPHqt)ODYwa8dj(mWvU^3m;%#)(i@Zg>q$ElzEzpdeDOQw2(Z6mqoohew zVl8!b6>NH#;&ioH6bVOEjbKqQtVTYdKTOW7N<-Idj@ymiuBmCD%UnNKWm9^gHF`UO zTFbK_BWF2(I$F3MW?yxD%PE?Or=PvJCA4c$a0egHx$0YAgR$txNG6hzNne^aDFCj~ zy0;5TL$z~L&UK^(;#;Ae2N6zA+b)^h9kJbRu#*O^OiR2eNR8-=hIEY}8tWCE~|3Hdj=Re6KlfyL=*zKO$@pN2XN z8g(Gxu4ZHql>NiLNQ6>N9bBhBG5JjVdZUSl6X%PH#C^%jX1o$E(!SfoEW|mJh}{mZ zaFRX(4cS~D;x(15Y9;(|UHNj_|9(5^Y`eBJ_ePAz&O5QD^C2+Q=#`FLH{R(Y<-MeL z!B?A79Dv64YwA0ed$$R3H95>ST{Ug4oaB=CR|{%EF;gMn?<#k9<(v`n>U}<)e%bwq zgot;a8EyLTSyQ*u>61y6v~GzE;vXaE08*n$>X(SW z;sYm)jvqY;+nuNfrK12BPiJ0X%uwB*lEul(Db9bF)pbs`gEb08G6bYw`bA>1CZLLg_%`b#RYjiK0umiR!t(64c2}Io z(5i%0S{mgfX@p{VUNaE8Isc=m%lh_aSVya~goYG1)2;%k?`*fJp&5Q;vm80{35F(c zG>Ta^Fz3dMqHz{boFU%b!}J4v_UgcArkC*KgTRDad{yDkfe$~eHM^v#sM0$Dt8%4$ zZ|y#|=R*d1`Y}vf_N@h2um`J*MNWf^XAyk?obo##>#o%BpMConhA&WYvQW#b&q#ZE z`t43$gjjlpog92&R$&hxS1%_&FJHGux7pK3AsUWxVI;ew0LE-rw@1XE@+?>ow6AWT zoU+`k#e^ba_CLXAH6fZ@tUmlo=HZ-`;WbAsVG-kCR}6CDBTpe;Fkt_H>C1ZLWFC8h zMN_bF{B)m;Rk#Y(a8@?pC+s4Nh?W&!6-djC-G=;Y5rC_v3;ufS;XoK>tQvGW0lSKv z3?Kob3SV?+KQ<>BrHHe7$`)q!OF^v;xIqRwJuvnKBJ(R9g1m*R9z_imN$?>a0H!Px z(mbsqLp69Rp(Gnu#3fhXL6>jjxseskjFF=#QAn6EjItP`-gItnzNEw92g7ldbMY#% zy|@vkCeL~Bu)qgNVW1CoWi2bdno-iO9E;W?uUvn=E6m=_pwO4{*J5*WYgQILUxDFl z*J_R)P@fN(^Cs!JVUa3}f^E!3)=FJVn1g;J!lox7S1F*Az;&R6poRY=o<5O+P0zE- zfsutZ)p$NV5hOuZ9|<)+@>?Q{W~&OWX&-_z49SvQy(XcJJfv+I*8hQLx_W);QtR#z z_GAmS9!;!v4raE#D$y<~(Cm=g_|j3dLAH|)>^p3b{y}4_uuzu>D8>#k+J&cgH^J-R za{ukAU(Q*gB5{ypPBb%0Cy&aUuWXhaaR)v%Ussgg+z}Zs(O1qoF?*Eh6j{|wu0j+I z24{$wQ{B9Hc05=`oPejFj{&FW#jgjJ3E8(hNhxgcOC1;Vh@Kt{ESPL#HeW*h z3gH8Ecub+P~yo^XQqpv)bLcY`b0>66ys#TU4 z(lIthq#vzGyoP+x+mK&h3E4i}*}H}GN8U*G8FsKp_Uy1Vr=&5p$m%+#8Cv0t?EZQ3 zTD@;jKAqD>)UGP2)HEd*hP85&WOZP@HeGk{j=ZeeDiHG%TpEiu9*$!EWw@;{U~+&x zJi+=3EQkJUW#OpMxg{GrvhGT4GjK`J7M~#uiJRM!U_Hgma?4s_>u)@Q)#1QY00NUB z)F9E&dJZlOKk4FsGB#vL?7kEK_vCO6&u3?fd*66x|6z-Bw(Q~HGGK<2-7x0IMqadK zxgoGtK5?*sO{SaCHA}_Sz_2;^k4#+hM@I*V*8tcn`fQeaB}aQAhqO+{_b|r4Q;>y32#iz>xE|a$ z0>J6Th7iI;h-abuBZ_f;d+A&lWluG=`0$`ZwC2o)qIK32mpyjnH-J@DD#9FU39_R| zZiek>Fy1jfTD?eq)N11>Amdn!XC%*F8?Xrk*p@V>uqs=~&urYL;PJmYqQGMy4V7Y$ zcc@bgzo0S%T5SI1?Dfq!Do(Y{stnQJP|||iOc9%<<19j{zXI6@`=xwTy}xu(cu<~h z2W{==`?*9RHe=B=na~4{(e6mohSrdWJtlw8d*%qfWBax)=$IGfr>tA!*&B*ZsCzz(%98$Dui8p;FjkLct z=8OCkgHS;yB?Wz*chnMl*@ocr`1A2Z3U0VKeQSM`FPJPY*sj~?5Yty1bu_-;W(!7x&Ge#hf64ys*Nmw?-Piqw2inJW=0|1w1P< ztqr=f3Yt+`myzCM(oOc_2ZXrt|LrxDom(49&pzcPd5Q?W9JcK`F;KjYOrruhKpLgC-B~f=llzM_(rwk?Xyy znb1o(n@G6N*wyTRwl0e<1kG!ZG2GmR&n4+UMB2u&n{&4kg`cQbWH`9!-21?EL% zy?-Bw&$k)D6h^pCZ)xAZv0$@A^Bocq+h&|(x4W<-%N!|zS}-8b1>lp;6X)``5FYgg z@l_oA|1y-EE;>K`)GGs7o&sZ?BFo+{{Oogv%Ew@@Y9om)?cp zvljjDhaFopx!NTv7mvP(-Tj`TAooL!Hf_D4DYV%`f~ts>2Z-cZOn@L4zy3qClEXjT z@V8@>h?*jZFMAa7PzD%rZVI5rn(#gG^Tro!^o|C<9&R9HqYyFCGEQbg`785D(5Ft_< zQeF*EqnbZOab?F)4Fum*>vx>9e)TZ5-FYl;A)joIrtp3S zn=2{}TCe6mG+usl$zUPWXMBbnKv`$TMS75&e}SeqbJ)vrV>)(NEd~l3C%P?FrbQ1A za3I|9qw+OAJ$fbdOBkhw1eVW}D8rgGCUaQFQ|;(+@qeC0_Sl9{-ops(N2RQMTW2;7 zJZFDNj<{DHU|5x8iK8t0QT(!OpcNN0 zu?%f|-W2Bgnm^IF<^R{5fohk8^QvjW`*WPrKLA_ zg}GFby+I~`0i*o293vh719u5Fmip9Uzj&kKe)|`hR3xa_FYid1gf|yy>9km%1b0{3Gg{exSNf>(gwcy$8Aq{4<|>$NE>Llb9U~=$L$8nT5j;; z!Dk7Gn32NdQNu9GM|FRuN*zyJak-sF@EsoI2-!K3x>jhF9m2}%`e><1S6F_f_F6+Z z_fna{rD&WlqDNr6P9yfUq)h1_tT}x3sB?`jUHI2*@NVX613SFCzDAya);v2NU;38E zeAJ+7FnN-JO5>lXE!RHykWxuG@6?kff)>lqV3~&{+zDmJz-`rxb1;2wnI^i;ng&KM z;-ENOC1#uot<{s~s*qo`Y!^i>r0ZwIMO3EB`ss zz77<-x-z{YsecyU>Ex(G-_W9M*^EL?y!R1(6<*9~#T`@-%8xISG#L;^X>6+mL4ag7 z_M%+K3-j>jgs+M%3b&Us1JUT6Gf}`8*;6%o!As@@2n}!Az+hb87V6I9jNMfX$EmPa zZTNy@N7y*t1LkQLJk;o}!fC6YRASlGJ;$Z&$0UAuRuR1j8+_$Gi*_adXHY%*u)IucdGvCkcO0 z29K7OK?t%8s81U-qOZFDdNb`K*B!>stR!bYncPfQ%z3-|t(tCbmLrrD?_T2MoZ!CZ zjX~FA8DajQfYCZ`J^t|L)4wKOpJ3}Z(jFu_Oove+ zpIM`4P#L~Doj~OFs<&G=HDi@b&rglrC)2O*pbIh!9Srf$Rxsai^Wsl;xm<5O$Y%QQ z!WLv8aKHBDQ}NqBdc=!x^~Tu`s?%z`$I{AAuynASW}fy+eN2Uc{JN(t&V7p=vv$J; z9!2pGfVf39j4a0#V){~9bJa17<1As1+hF{z{+U=J+2Eby1K^3&EE(G_pLbB73Nh{4 zy_}+Mt*pSscyTti(-=C^+`tM-^|c(}iu>u{{HllACF?y#Ubt5K>Ju3B$ExpO|4ZCQ zts2Pe7w0Xok^{UTn9c=qU-IPu->Bucz9UU8)BVlnr81(i-<;*14zb@J9Lxr7fZp?O z-12GXosxrbjR?7yeo$HS&CL5V)gTKR*ls#Oy_n#!%zH?;l^MX~jc-3CgQ-vc)9bFZ zSm60J|2a_q=#I@R?KmMNBkS-m<*x-|NOhoP+9*Odbz`3R}K6RA{*b4Z0uYWLG@9ShuavZx&$Gnik`3^bXd@_E(JkGCUSO=oVVJ?P$lJ(Rn6>fYO2$AajgyDKXl8Vcgx z(a`Su)Y2JIXDd$^S9^DBx&AS}Ug!77+1=BR`MFv3=ui=wr_TJ?>Kgj(Zr(0f;+oqn z%8FcTT~S?Ie@{mY^*Y2O+H2R;X4~)K&+^@U-S@~=3@dx0UlN|7255u#S_|Y7O(Kd2 z$zM0oAfJP32v1!z(nVbiGMT*I$CeV_$KDXBsy?iv-yggcmpw;XGg}Uo8$i#yP7eQX z_nVsiVxy=k?_no8a44)y6Jjt(j<_5q!+R@Y7WF z%ndKkm}EOE_1hi8%=2zsRk(>4VLLT!4`~LGZe0FtQ}^?>4Y!82{=MmSRqCbt3o^{# z=l4>5Q!PCB;hnLfh)YH z(^bF^Z>ZSsotk?a%F(an?QQ@5hB{18ES8^=ImYeZJ)fl-uAhV1O-icGWIQYCdI#2; zHwBGqMc!sZ&o^+F`v$2V%Kh zfNwL@Z}v|m-k~6s$-sxZoi5Ceu94T9ntN8UM@_>gO0S!qo4@9qG{$u^^nzEZ?L2A5 zIN$Xj^^qmf=vpFxv&4%7X^+A_*q+D8r541NwUjpCsjqd3uMffHn!vXAnOWmEZ0h*!E~CCTB#9Jky!G#0 zox1Rc>iTyzGd1@D8G=)!+c_}kMP;L-@*_V`oy)YcnN~V(84`BR;rT~(Ust-)qTHwWujSdM z+-L-z5j7-;=H9Cd?H-ohvx&PbeOKlzx8JXUwpVevrH2IyhnI&cO2`^6TsEb!PkF>-yjmh$GMY?aaiT?%s}BpU{>a zn#fF}Indq{?0x)DXInqcaBE&c3|)7n_7aZFam5#Ig136XDlKdibz*j6IjZyg!`-IY zvt!;zzIC3Tmf-SiR$2tHn$Fxe zrF$6`Puc{)l+--(QF@m$I<0nXG6Pewh)LHLT#C_Stx_Z^`Vi7mVry_Z- zCn5-7$r}#BGAa_$tb?#t9&O(DUh%zin~Hk3*JfJ+ee95}8^o$Dd}0Egc&RFB(0SMs zc~QEjCo(WySSBTv0*S%qO3{x0k(YPVux*$3^RD7-$2s@ zO}LyV#yt*7OlKiyOoL|8aULhivg1p${zDR#>;j%S$abUkMR=B}c&a)76_QmAJ|w95 zQI|&B4jjGa<_qBd7Txzslq}S`26HIT75zxYYB~w&=V9uLJ4ws#Ks}%e6G07{&!;ol z$eZ**5U_pHRvRLU?u_5jNiul=_xsJB#)95nGu8x8&a(di(uo>cm>5;8!3-JZixDn4 z-qX(w#||fqlXo_P_0$zf*V$>h!NTX=EsxM`Dv@PClk2?4e}q$O;BSR5$RHr z=5Rap0LjTP57-2t!_@rGU|LHA;+vp|g&-Z~*@-|v$Jn4P#_k{EXiED(orV**btx-e z&bc-otJ$Oa^dT@*v3!C$kpVF)ZS*PHyjtL_bDMfvJ@`>?l+939c?*X|@k-mZYBsaX zP)6S%GsEZ!+>_zlv&vL|H`w58#GoCle{nuvWrV`O@n#4WVuow8zmV0SwDAl-m(lDnZ)Gq$@T1M$L2@j0)Isd7nm17wE6Xj)laQcWRH_ z48pUhU1>}@I~T3?qUA*=@0qPbVF{)0yY@T~Iyh(t*pNU+ww*09!t#xfz-MP)6hPRH zCe&uRd&>xe^dKyfdD0li*fu=wKA4R<0mfZ!cczpQRFr69dVh_^g>Z|ylX)gZCi|JK zzrD5&FDTI`#0%!{|ZfLq(24o z40CM}VBHTuQOO&(VU-dm%DuqmF__H~hI;)Pp6{JFlUXG-=2cAA9Mfzw?L z?OhKhg919$(M|#c6K%!zLy6Q_((fm+K#V0xnio0hL4Vm_hgPJ>-zn`tV*2w!^ed73E@@@b> zAMNFVpQ&uCRSgfByDi_#P?x_cF8{L$P%YtZ<*$E3jUQ)Mr{BYM=+3JdE3ZY1jlAl3 zgZ&^0A z9{O+M!_;;LNj8m*1UC1SmbPQJX*-v*6s|SG@ha|M1TlR#<>tMECcz zCjB3?cqZ#{4BsErzlF=s>Qr+9QCY^q>;y+O#=I#$xES}>!yiT%rQA)X(E;%f$O#ZO z#rPct)iCN>qne}4bg6C&Tm~g)X86!(IkrE9MOo9w_=vel&Du;*mf+^9Ue0MmKn_}~ zj0^;Qa9avOKeat_=-#h>I>YV8?qlnco+0hGHj4=6)oe-~vK~fSS9fI>+7oz{9#3~{ zmhHG1rCOCo?`G(TMgCf`BteRG^q#KCchTJ82AE^NyHo@*u+if;xy zu+#{f(RcZhL${_B{OhvShLZLH+IxD731$&soKwY}eEQ06b%q|_wCIJZq3ok#*;>J^~pGeVr{@2Y1 z`}oIv0G6IwIzc%S+!=nSSpNt0-vPYFArZv49$VN@Yq5ppd59vn;T?3cgfbnI2qZK! zXO7jNuMzuHyh{$@mAR-Ab4LW{FPBfQLyVb!6^^dFE#S&*YWO_{o(Kxb)&-Ifl_z?X ziP&jcxW`N&00o3DtZcRt##fjaTl>{pP#_2l(PJ=*vEHD`xq9wM6d`wnf0}fnggD|{DQ-&7*$Obit$Js!5e|%wG8sbuxojY{xZd9B5d;UN&H1GDBgn&;o%e~aw!IO2dgTEPZz2V(vT*o}v1 z!LwM?`uF&m{I@3&z)Ehx*7MC>E$)gUTWjbvvR98On60M{VO)F!-G=Z+&^ABw8gkbo z?1d^8^4-FSy_9UZM3K#UEL;cChM!|J-6F8zKecnZ`@YXo^={8w^bh2_^bh{cy?b1O zPL5}~g7n+_`t0TXuY9(II+|y5JUf8W9IGF;bGpc6(TQv=Ym)dbgUERB6i5^&dQFc$1*41&U;4(hsicE#ZCj)VoS_Bg*B*{7?Zl3Ke}B_33;ZSG zJjLT9^Ach!#rGLZ2cW;-1&w0mN6bcr%}FrnlG!c=etP(J2B$2%uues9o_BeY_euUm2)iyIb8mL+}gQ zDiR6UWnlYpW}3H!q|QDPS4Lj8_TPTotk6={=$rJI4#l(9hr89$`-=)>iMBR^*pRaN z>*dCQ(bO!|sg;XolJ5P6n8#V4==R3AakFKTPr|I=v+qqM>!y1Nx$PVF!B)C%5EPQ~ z^TK<8^Z^DK`^9QRR0#pbrB|<8H+cnex}-oI(l=Df;s;IoNlyVW0sy;c1nGGpOJ);N zGese?NSGOcy8s1srvbC+2}^j2C%A3%My5j2pu{#QmxmDBDgcN5v2r)%va<)Bo!Baj4{ zif!H4%;V8KXK4S~uq!Fk!_bdxhu8C7&Lp=Q?)l6d!@oTTJ@=db?Jww3E~MylBM@A+ zPkxyFbEBrN0WdWqPt(nd(2cyQKjsdH-xe_3t) z1rVBeq^;%f*mc&xn17&PvKMwQBGV%KBQbVh%%QmY&QuOwqU6XXk>rvFb%fN&HDdq@ ze=2$WNw>Gky<)1~0w78b+r-}P^he$nQOe?0RJ){i(aRLv=>h8qz^9cAx+{H=BF&Jx zBf_Qu>A0u0#Z9NTM86TlRsR=`yBK3@ygHNp%G_=i+GqJ!ke=(a`maDe*PNP9uiE>I zy@;Bw>+;y>z4GgW?SIRvv9cnn1?%Y*-#1aRR9X(V*)esd!<1(8>??Z`UVj}wBF8S$ zess)qt!t34cd!l3Oa06|w(p!N9i~k9h*+1U*+n9~a1*xUC1lee&bIhNJxIF_5t9mt z8(8@pC5D>}6=<5y4y4@-z3VOQvo-A*DH>ic+_qV_QJ6>%DYOJdU5Sji_}(oS=B)Tf zY?Oq-3F*q;xNy2bs)r(Q1r!N&s78|`Iu)M$@ld7^KCQ2DG|VO_bm%{37Lba4;hF4X zzD9$o6Yfd(A{17uv_V&>-^Dt`Hzhg|1L*j1&H~B7696h#(r@+Is6RAJ#^6hvywjH(IF6N5G(Vg>6f-TdE55Joct?L>#IHw- zVASah#Ga$qcdf96iDY1AKom>BNcAjS%Z*zMH#icB^-;^e`))2TrBDeIq`jmCdN0C{ z#>FGaz_K`pL}7i~Z!-v&dI`-%*p7z^Kkgb(f#?SyHVs|L-E=2v7;!Te{G+v0yPWXeX@^-@q zD;^l^Oy<8LvBr+d)?`9z<~Z@q)gb>9ZStHkvRtJpKZAxUL6JuSRj3VjL2@Iv#>e&| zZ=2-9gy^&+T0Saxtq$dphC6JXL_7W5Zga&tdcm2lXQXS_L;;-N)OVM(wWo_Ka zsr;F4lzi6Lg5^2UL4mi9hkhu#r%yHXNlXEmEpB3^^Ha5@wIYwTXE$i8f1?Y+;Kcq;bqENZ?evv5I{S*Z(Bn`Dq`dzxWb7U+FfqTTZs?^_z$SYTtV{JmGI0Xn2-*Lirc_67t#o?m)VK~z}h z>Q`3hhB4W1^C6CIL~@v)NZ(bLd-~OtS0kj)%jV~4?4Lj0U#;EEJ*^EneO}~e=~sp- zlMn0P``#WLoh`)%EmRb{R0cWt)xi(D9KGdVJ#*-w(gHOb8tRUR3u=<|9;!PoWw{U8 zE%63XMvw;mCohuj9CqSOlXMzWRzvz?5W1z_O zTgxcOIqF_A41EP&-uK^PO@aKfTT>>aQa8p-=71DJpu(B}u%Q7+2j8q~@GbEMWS8vW za5!IZeHap!^TMC@{e}KY5#Ojy_Mr`y1N40+%1!G zyZo~lRAEpTrto;Xi@p8o7VcHF9=x}WJ<#Ch_Lj(K?(}U!{c)*+{-SI1;)>V*+ehRj zm!JO>QNQcgB_EUlnqiW?yPfgmQxTRMApSZK5-W0xTp98?!0t~Jjc{*Dz5nt!B&n1L z3KVvch09My&yX)Tp`tyGL*BcEQSjj6Bnn?8U6H3AI@|1fEbI(qAf|pxImguCicRS5 zbu@!X{p@YM2ng-s+yRn;5!Z|+ZtZSg2<@xUjl<1RB<4;Wgm;bJHR0oakcUU@2qIE} zMaKzy+RG-^&AY2>`}FAFeJxBK*pBO3_2#_n=32est>48o>I*W% zV9jDPOQ&iMtU*M~?K!OJk&Q{!K3M6~v%%}7Spe`X7l6e>u#Spuu50Ekxm?Pfu#rZ7bf!?OJ2m2Q;RD&9nhI0!g z$~xa{nOA90jHhHNg{-Dz-Lo7wGPa>$L50hSz7yz>Kz+a!RB1(hH^k`2LH{2S8Ve(b zMWHU4JM?{6I9uAi+|R1Dkt$I^&7qpIfXDnR!E{t7Y*NgZ?sR_icn0r@F}%4*5;W1! zg1L7$^tLmz{poS!CbcDdWei93-+<&9+fEpstOYv>c1IPn{xN_a7l}eMo-k}QETd?N zNhwlLc7E0_NG`>vmv`R8>OT||lpG4~wTQ0` zhmy9_AR}Ivs=V?3k1tRL`+GmwR!*HKm`+JHKS;m zK8I9AkpdWUmWX!zi(?Y~7L%l&|Mq$PJ}&lxHK?E>#2DY=v!i~W>Wk^!%wNT(OO;yO zhVM|~+n=hQwG_1P=~+30eAAmC_o^*}7btBr@d}AO{$j*OeH?uuMH_{nlgK~WQjiKH z2s~Wl6_|fPAaW*e3zH@Y7bzhE_Z^Sg{HybRrf@48=K6j}qzeT`PhAemXfFlwp&N`m z7j>8*D|pXnX~GmCh2*pEOPC!hO+Y3ZM?*@c)|{P0bX`cNmwXvWS1q;{b& z4bsOE!XD&frM;Fb1{1`O(2@goK*g|lgwVre$&I@-nIdP`6o@aq=wK4{$H=2jFb5Br z+42so=MPPJeH>za7)R@R_^C>oQ2b4Qwa!#ae>W*kHM4y-extNXJqZa;x6Ka2(GP<5 zbzk@1X2i~Q00{bSG=uu_-sG{x|GWb$aRJmts9hKeFMc@T56FeED_l)QgX>6FgJb1I-lu-K+6yQ2OzProjh#uldsi48Y* zLy@b1sb#t#1kriV|E3Q;TR-OP)=hFq1vL|&UWHe}7 z0?1%oW2FpG$EyY_+fIFQ>#<&14D`w|6b__i4qRcQWd5)3`OzlC3&EaM#axr`UA7S~vvGsT|$IOe<|`m;q17(7kQu)3rjd%K_w1YF%kM`}j19Rd6?4 zi3{EZ9M4G8Kip#%PxoVVhX86o6Q6Fp8>&}hYz^N5;yobh7~y}$2iXbbI2U3q54IJ- zQiYWC$k-YQyBMs#@P{>kSk#OqCao^y!B))`1jc5S_NTJiaWk3sR--BoPIX&z*mRHDY7+rewg45KHihxZPq%#Z^gX)&@g7^Y|L~qUqNL<+x9*Nv1X^DXIaYgy7+Bsl z60OZzcyyGA#x(oZ25%QnNCODEN7CRxY0g@m+RX^vbDoT1Poqh`S_C=0M2b`ZImM9+ zS&d;kpXj-E@ZlYDaSgZ28oe$8Azi8u;=MZBxvwlwD$a_5tnhQDDp}2}5@-a;nIT`O;)ST8v#Bz3k2tc%3 z;Rg%h-^|i((N4lIFLz7h7Eg#{ls3?sT$g68{8Jpt!_+Wq2BFkZtogQR#0*%lRJRye z19`$%51*w({iCQpj`d6E=i5Jy>hqE-ezVPmhV_?)a&TfbYg6JZzn5S|xd0)Q(*UQhRwzarfWhc=B@ zglK7xbF0~~p7F0|in`KsC+s0cF{k*IZa>wJwae=d$?zV$m@*iJ*Z!{w>hr(MCkVD_YCcz&PxQDZ2eFrXqj^)c&g2vlUrm*>yN&>Ds11l7ck44xwqk)@Oy{ z>SkRozt1&+TvAmWyKwVOQ!^Q^vGEvxV)4nKa&KEV;&mKpax(p(>k#L!Pa0#DRAQyrwAN7W) zu#-`#! zXYy(cqmlCuo3L8Cpexymw5U11j(bnV*2taPkvS;Ld4!DOUGlA|!?A;yMT0YBkC;Ff zbYyfPRu=j6RGcSSgFaT-1z@BBF|LqUO87ffRLjrRlAF1LohHKV@$@8iQ$_PW=jOG~ z3yI`4%r}XDROMh-eVhS?*H;$@SL0+)li+Uk$peQ?EM^M0#Ikx|te||H9ePUhXC41| z!cbE;=0k=%s1iqgX_+-68Jhnd$_HJexT`hht<07nBAoJWyXQ&IZvlDA_KY&{r?DPd z@zF0PUpxhR*=Oq#X6P8Px&IlcH0R8;rYX%(<7X*UU(1&BtXa4Ff?qylb?7(rMD>`H zbb_i}PVPXhqHDlo&~l3_(-+dIXDv}VO!6MdWs}k>q9t6vPv-L1fyiIJ8cy)cvb&%O z(_>i1y&BWK?E9tcylheo?Z@hNeKIX>!aiXA{{puMNcf+(pu_d+GD2LRuAn3^Wtz!p z?Ud?};d367m<@~Z7;+wu>bnYDE0&Dx96d3``ZZ>mGC35AAq0|$v8wzpo-Pfa!$bg* z24`Q1Z&YLvdY!qRq1VH%jik~`Ym2&5>?@9QZ|sfL7F~U-gF==7$n=Xu!P;|mI=CiB zmuFTtW3m)ZQ09s$eUx+TfrYa6hoz94by6Bjh2}goKl|im_hol(FUN*SI{nd!xd}Hq z>5Kq-42#i-`F?x*HFPnSMrWA^oB+33upSsR<-lVsWQEX-oM9|ut0iI_LUjOKm*cRe zpbCLDgRuY_HXncqj(#2xA<3KLOWTh}x9UQ^iE`}0qiTo2c^c@gGBw6gPCS$9DQw7s zrEDAIdxf8u|8>=1=9Kv0?)sGN%f?HW%3x32Cre;dBv=nPjb$nuN zoF$ksCo5_18vMI4mlm(-3dzQ%+v)>wI(Y5uEW<0eD#T)a!ipd=BTSQ^=lk9XEwo6f z=8^Au%m4*+`#9e-@W=VS2ieXIHdNs_-w(^0i;`HCR|#NkIGb#G=FMwTWSS@BcEY0x zkl|4Trjr5{8_ZxsajS^cwgY5Oju65CBNWFe<1_%* zC=|+ZHQ{+~=A9?0crtS{ZO8lw)qaRID`0Pq2dkj{eDszmHcnY)16<6T;1um^nuKGu z0C6>W>j2J?=nDhbFO?!A{}My%mRoeBloGUl?WhfLa{9k=6V2ZF2w^8h4v?Q%P0biY zisa)kF4CMPiJ6GDc!ImD;X?TXn~}4V z#Y$@5E{x#~n}s~CqGfzOLFr^iK{8b&ut~iLt7MLJ7U1uokVy&A3OzBhup41GkyjDo zys4y+32NS%(Dq)2qn`(jP+T1!j+N_Gmo(P261?gb9|W9(m9W!CSQg1yZcV!eILdI{ z(dl6C;1KKE4s==P%XQw&8dTaE0J)n^NqfIbx?S>}`5(+ZEBaY!VVaJe6e-QzIW04R z%e@xIuFlvu?@I;7lnQB_@(jcoirfjE+Us8Uc!EafSkg?}t;I;BU^AL!=JHLsvsRP( z6-c9&hL|wO>R?M9YW_i5a(soFDR|!(jx1uxygeLfjN;0{UvAzaun?LU$+&Ljz;reh z!Qx+WB-OTajx@im3z)*{cxWwD2o>F`J3UMSkd-ZRVH>3yeeLa>S+szy1kq%~70L({ zD^p7d4PI^!q6%rwYkpPq$y(iqQ1au|TYn9qg`OB;W`lj6l(9f&gkW>;sgbUVt6-|Ysvqr|q%ZerW4Dq_myt>jv>%#0Sz{GI7% z*fYj1Tip0>!BKC`BF*Be7wpM?n3UD#j|-l0QxE8}(zEII^&JuqZHfcaGMh7kBtjUe?~GYvH3?(Cl@P)lK!i3s0%_ylZ3hxq z5LF}d-cGtk5O%w#2B02vx>#p({U{#~MFAcgbbbV{+BSaSy`i%Lt*l)JV|nRiyAG@2*Y;!90=K0RbW09tge*I-Ih|CwA$Rrq=(uX88@`%T z9DliW+^nffV4)ARavE1Vzs-Pm$j?T-L)C7=l63zWihA>Z%whdsWo*4!dD=msZg?I< zGFEL+Rd0CSSs>9}*Mo}sGHNLq)XJ@IJApg=*j%7Q4F=BE4ir5bf#&Ms@~qCm6##oi zD8k|XRy1}6_#uiUK~Zfzgf1}-ob(tpv~WFT8$}ZMRNsw4Xk|OdMclS`rj^Njs-PS5 z6&!N*eq;g{E`z`;6@pyEt$gABWNU*qRP+N;RjcvxWvE$#8R6#?a%0ejF4!OF+@geViHiTjNF33d8c6VIFQ9!depqb`D8J1K9 zL4N#f2)eHFz?`g}JAnzF{spae`<8a{+nNf*Ll^L(1 zOcHU6F6Jtr7$)LK6q)sya8<|Ct|$MgXl?h&|E&HB&Q9Pa!3itl<^Go^u_|4DUNF=f k{}m+Gw?3cv_&h$3&*Ss>e3|F}3jhHB|2RXXr2vot0B!1O$p8QV diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 773f311d8..3593dd276 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -252,6 +252,8 @@ configLogicalBackup: # path of google cloud service account json file # logical_backup_google_application_credentials: "" + # prefix for the backup job name + logical_backup_job_prefix: "logical-backup-" # storage provider - either "s3" or "gcs" logical_backup_provider: "s3" # S3 Access Key ID diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index ebfd49252..15f13df7e 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -242,6 +242,7 @@ configLogicalBackup: logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:v1.6.0" # path of google cloud service account json file # logical_backup_google_application_credentials: "" + # prefix for the backup job name logical_backup_job_prefix: "logical-backup-" # storage provider - either "s3" or "gcs" diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 04d5fe23d..212515dc1 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -556,7 +556,7 @@ grouped under the `logical_backup` key. runs `pg_dumpall` on a replica if possible and uploads compressed results to an S3 bucket under the key `/spilo/pg_cluster_name/cluster_k8s_uuid/logical_backups`. The default image is the same image built with the Zalando-internal CI - pipeline. Default: "registry.opensource.zalan.do/acid/logical-backup" + pipeline. Default: "registry.opensource.zalan.do/acid/logical-backup:v1.6.0" * **logical_backup_google_application_credentials** Specifies the path of the google cloud service account json file. Default is empty. diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 848389d63..3788d8b32 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -64,6 +64,7 @@ data: # log_s3_bucket: "" logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:v1.6.0" # logical_backup_google_application_credentials: "" + logical_backup_job_prefix: "logical-backup-" logical_backup_provider: "s3" # logical_backup_s3_access_key_id: "" logical_backup_s3_bucket: "my-bucket-url" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index d52608c15..cc8dbb6cc 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -61,32 +61,45 @@ spec: properties: docker_image: type: string + default: "registry.opensource.zalan.do/acid/spilo-13:2.0-p2" enable_crd_validation: type: boolean + default: true enable_lazy_spilo_upgrade: type: boolean + default: false enable_pgversion_env_var: type: boolean + default: true enable_shm_volume: type: boolean + default: true enable_spilo_wal_path_compat: type: boolean + default: false etcd_host: type: string + default: "" kubernetes_use_configmaps: type: boolean + default: false max_instances: type: integer minimum: -1 # -1 = disabled + default: -1 min_instances: type: integer minimum: -1 # -1 = disabled + default: -1 resync_period: type: string + default: "30m" repair_period: type: string + default: "5m" set_memory_request_to_limit: type: boolean + default: false sidecar_docker_images: type: object additionalProperties: @@ -100,24 +113,31 @@ spec: workers: type: integer minimum: 1 + default: 8 users: type: object properties: replication_username: type: string + default: standby super_username: type: string + default: postgres kubernetes: type: object properties: cluster_domain: type: string + default: "cluster.local" cluster_labels: type: object additionalProperties: type: string + default: + application: spilo cluster_name_label: type: string + default: "cluster-name" custom_pod_annotations: type: object additionalProperties: @@ -132,12 +152,16 @@ spec: type: string enable_init_containers: type: boolean + default: true enable_pod_antiaffinity: type: boolean + default: false enable_pod_disruption_budget: type: boolean + default: true enable_sidecars: type: boolean + default: true infrastructure_roles_secret_name: type: string infrastructure_roles_secrets: @@ -176,16 +200,20 @@ spec: type: string master_pod_move_timeout: type: string + default: "20m" node_readiness_label: type: object additionalProperties: type: string oauth_token_secret_name: type: string + default: "postgresql-operator" pdb_name_format: type: string + default: "postgres-{cluster}-pdb" pod_antiaffinity_topology_key: type: string + default: "kubernetes.io/hostname" pod_environment_configmap: type: string pod_environment_secret: @@ -195,20 +223,27 @@ spec: enum: - "ordered_ready" - "parallel" + default: "ordered_ready" pod_priority_class_name: type: string pod_role_label: type: string + default: "spilo-role" pod_service_account_definition: type: string + default: "" pod_service_account_name: type: string + default: "postgres-pod" pod_service_account_role_binding_definition: type: string + default: "" pod_terminate_grace_period: type: string + default: "5m" secret_name_template: type: string + default: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}" spilo_runasuser: type: integer spilo_runasgroup: @@ -217,12 +252,14 @@ spec: type: integer spilo_privileged: type: boolean + default: false storage_resize_mode: type: string enum: - "ebs" - "pvc" - "off" + default: "pvc" toleration: type: object additionalProperties: @@ -235,36 +272,48 @@ spec: default_cpu_limit: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + default: "1" default_cpu_request: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + default: "100m" default_memory_limit: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + default: "500Mi" default_memory_request: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + default: "100Mi" min_cpu_limit: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + default: "250m" min_memory_limit: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + default: "250Mi" timeouts: type: object properties: pod_label_wait_timeout: type: string + default: "10m" pod_deletion_wait_timeout: type: string + default: "10m" ready_wait_interval: type: string + default: "4s" ready_wait_timeout: type: string + default: "30s" resource_check_interval: type: string + default: "3s" resource_check_timeout: type: string + default: "10m" load_balancer: type: object properties: @@ -274,19 +323,25 @@ spec: type: string db_hosted_zone: type: string + default: "db.example.com" enable_master_load_balancer: type: boolean + default: true enable_replica_load_balancer: type: boolean + default: false external_traffic_policy: type: string enum: - "Cluster" - "Local" + default: "Cluster" master_dns_name_format: type: string + default: "{cluster}.{team}.{hostedzone}" replica_dns_name_format: type: string + default: "{cluster}-repl.{team}.{hostedzone}" aws_or_gcp: type: object properties: @@ -294,12 +349,16 @@ spec: type: string additional_secret_mount_path: type: string + default: "/meta/credentials" aws_region: type: string + default: "eu-central-1" enable_ebs_gp3_migration: type: boolean + default: false enable_ebs_gp3_migration_max_size: type: integer + default: 1000 gcp_credentials: type: string kube_iam_role: @@ -315,12 +374,15 @@ spec: properties: logical_backup_docker_image: type: string + default: "registry.opensource.zalan.do/acid/logical-backup:v1.6.0" logical_backup_google_application_credentials: type: string logical_backup_job_prefix: type: string + default: "logical-backup-" logical_backup_provider: type: string + default: "s3" logical_backup_s3_access_key_id: type: string logical_backup_s3_bucket: @@ -336,30 +398,40 @@ spec: logical_backup_schedule: type: string pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' + default: "30 00 * * *" debug: type: object properties: debug_logging: type: boolean + default: true enable_database_access: type: boolean + default: true teams_api: type: object properties: enable_admin_role_for_users: type: boolean + default: true enable_postgres_team_crd: type: boolean + default: true enable_postgres_team_crd_superusers: type: boolean + default: false enable_team_superuser: type: boolean + default: false enable_teams_api: type: boolean + default: true pam_configuration: type: string + default: "https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees" pam_role_name: type: string + default: "zalandos" postgres_superuser_teams: type: array items: @@ -368,23 +440,32 @@ spec: type: array items: type: string + default: + - admin team_admin_role: type: string + default: "admin" team_api_role_configuration: type: object additionalProperties: type: string + default: + log_statement: all teams_api_url: type: string + defaults: "https://teams.example.com/api/" logging_rest_api: type: object properties: api_port: type: integer + default: 8080 cluster_history_entries: type: integer + default: 1000 ring_log_lines: type: integer + default: 100 scalyr: # deprecated type: object properties: @@ -393,60 +474,65 @@ spec: scalyr_cpu_limit: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + default: "1" scalyr_cpu_request: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + default: "100m" scalyr_image: type: string scalyr_memory_limit: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + default: "500Mi" scalyr_memory_request: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + default: "50Mi" scalyr_server_url: type: string + default: "https://upload.eu.scalyr.com" connection_pooler: type: object properties: connection_pooler_schema: type: string - #default: "pooler" + default: "pooler" connection_pooler_user: type: string - #default: "pooler" + default: "pooler" connection_pooler_image: type: string - #default: "registry.opensource.zalan.do/acid/pgbouncer" + default: "registry.opensource.zalan.do/acid/pgbouncer:master-12" connection_pooler_max_db_connections: type: integer - #default: 60 + default: 60 connection_pooler_mode: type: string enum: - "session" - "transaction" - #default: "transaction" + default: "transaction" connection_pooler_number_of_instances: type: integer - minimum: 2 - #default: 2 + minimum: 1 + default: 2 connection_pooler_default_cpu_limit: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - #default: "1" + default: "1" connection_pooler_default_cpu_request: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - #default: "500m" + default: "500m" connection_pooler_default_memory_limit: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - #default: "100Mi" + default: "100Mi" connection_pooler_default_memory_request: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - #default: "100Mi" + default: "100Mi" status: type: object additionalProperties: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 3d4ff09bc..f03b4c2ab 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -106,7 +106,6 @@ var OperatorConfigCRDResourceColumns = []apiextv1.CustomResourceColumnDefinition var min0 = 0.0 var min1 = 1.0 -var min2 = 2.0 var minDisable = -1.0 // PostgresCRDResourceValidation to check applied manifest parameters @@ -232,7 +231,7 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ }, "numberOfInstances": { Type: "integer", - Minimum: &min2, + Minimum: &min1, }, "resources": { Type: "object", @@ -1294,6 +1293,9 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{ "logical_backup_google_application_credentials": { Type: "string", }, + "logical_backup_job_prefix": { + Type: "string", + }, "logical_backup_provider": { Type: "string", }, @@ -1470,7 +1472,7 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{ }, "connection_pooler_number_of_instances": { Type: "integer", - Minimum: &min2, + Minimum: &min1, }, "connection_pooler_schema": { Type: "string", diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index faf1a4908..16fb05004 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -141,11 +141,11 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.AdditionalSecretMount = fromCRD.AWSGCP.AdditionalSecretMount result.AdditionalSecretMountPath = util.Coalesce(fromCRD.AWSGCP.AdditionalSecretMountPath, "/meta/credentials") result.EnableEBSGp3Migration = fromCRD.AWSGCP.EnableEBSGp3Migration - result.EnableEBSGp3MigrationMaxSize = fromCRD.AWSGCP.EnableEBSGp3MigrationMaxSize + result.EnableEBSGp3MigrationMaxSize = util.CoalesceInt64(fromCRD.AWSGCP.EnableEBSGp3MigrationMaxSize, 1000) // logical backup config result.LogicalBackupSchedule = util.Coalesce(fromCRD.LogicalBackup.Schedule, "30 00 * * *") - result.LogicalBackupDockerImage = util.Coalesce(fromCRD.LogicalBackup.DockerImage, "registry.opensource.zalan.do/acid/logical-backup") + result.LogicalBackupDockerImage = util.Coalesce(fromCRD.LogicalBackup.DockerImage, "registry.opensource.zalan.do/acid/logical-backup:v1.6.0") result.LogicalBackupProvider = util.Coalesce(fromCRD.LogicalBackup.BackupProvider, "s3") result.LogicalBackupS3Bucket = fromCRD.LogicalBackup.S3Bucket result.LogicalBackupS3Region = fromCRD.LogicalBackup.S3Region @@ -154,7 +154,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.LogicalBackupS3SecretAccessKey = fromCRD.LogicalBackup.S3SecretAccessKey result.LogicalBackupS3SSE = fromCRD.LogicalBackup.S3SSE result.LogicalBackupGoogleApplicationCredentials = fromCRD.LogicalBackup.GoogleApplicationCredentials - result.LogicalBackupJobPrefix = fromCRD.LogicalBackup.JobPrefix + result.LogicalBackupJobPrefix = util.Coalesce(fromCRD.LogicalBackup.JobPrefix, "logical-backup-") // debug config result.DebugLogging = fromCRD.OperatorDebug.DebugLogging diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 142aa2be9..1d8e37bd2 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -112,7 +112,7 @@ type Scalyr struct { // LogicalBackup defines configuration for logical backup type LogicalBackup struct { LogicalBackupSchedule string `name:"logical_backup_schedule" default:"30 00 * * *"` - LogicalBackupDockerImage string `name:"logical_backup_docker_image" default:"registry.opensource.zalan.do/acid/logical-backup"` + LogicalBackupDockerImage string `name:"logical_backup_docker_image" default:"registry.opensource.zalan.do/acid/logical-backup:v1.6.0"` LogicalBackupProvider string `name:"logical_backup_provider" default:"s3"` LogicalBackupS3Bucket string `name:"logical_backup_s3_bucket" default:""` LogicalBackupS3Region string `name:"logical_backup_s3_region" default:""` diff --git a/pkg/util/util.go b/pkg/util/util.go index 20e2915ba..bebb9f8da 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -271,6 +271,14 @@ func CoalesceUInt32(val, defaultVal uint32) uint32 { return val } +// CoalesceInt64 works like coalesce but for int64 +func CoalesceInt64(val, defaultVal int64) int64 { + if val == 0 { + return defaultVal + } + return val +} + // CoalesceBool works like coalesce but for *bool func CoalesceBool(val, defaultVal *bool) *bool { if val == nil { From 010865f5d9fb901f5bf6fac29d856cf9cfac79d0 Mon Sep 17 00:00:00 2001 From: dervoeti Date: Tue, 12 Jan 2021 15:39:54 +0100 Subject: [PATCH 158/168] Fix typo in operatorconfigurations CRD (#1305) * fix typo in operatorconfigurations crd * fix typo in operatorconfigurations manifest --- charts/postgres-operator/crds/operatorconfigurations.yaml | 2 +- manifests/operatorconfiguration.crd.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 09c29002c..a360da0c6 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -457,7 +457,7 @@ spec: log_statement: all teams_api_url: type: string - defaults: "https://teams.example.com/api/" + default: "https://teams.example.com/api/" logging_rest_api: type: object properties: diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index cc8dbb6cc..7add1b8c6 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -453,7 +453,7 @@ spec: log_statement: all teams_api_url: type: string - defaults: "https://teams.example.com/api/" + default: "https://teams.example.com/api/" logging_rest_api: type: object properties: From ff46bb069bbe9efe7bb9956f2953e64c0ab5f02b Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 13 Jan 2021 10:40:55 +0100 Subject: [PATCH 159/168] update docker base images and UI dependencies (#1302) * update docker base images and UI dependencies * use latest compliant base image --- docker/DebugDockerfile | 2 +- docker/Dockerfile | 2 +- docker/logical-backup/Dockerfile | 2 +- ui/Dockerfile | 2 +- ui/requirements.txt | 16 ++++++++-------- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docker/DebugDockerfile b/docker/DebugDockerfile index 0bfe9a277..e8f51badd 100644 --- a/docker/DebugDockerfile +++ b/docker/DebugDockerfile @@ -1,4 +1,4 @@ -FROM alpine +FROM registry.opensource.zalan.do/library/alpine-3.12:latest LABEL maintainer="Team ACID @ Zalando " # We need root certificates to deal with teams api over https diff --git a/docker/Dockerfile b/docker/Dockerfile index bf844850f..c1b87caf7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine +FROM registry.opensource.zalan.do/library/alpine-3.12:latest LABEL maintainer="Team ACID @ Zalando " # We need root certificates to deal with teams api over https diff --git a/docker/logical-backup/Dockerfile b/docker/logical-backup/Dockerfile index c8bbe6f28..b84ea2b22 100644 --- a/docker/logical-backup/Dockerfile +++ b/docker/logical-backup/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM registry.opensource.zalan.do/library/ubuntu-18.04:latest LABEL maintainer="Team ACID @ Zalando " SHELL ["/bin/bash", "-o", "pipefail", "-c"] diff --git a/ui/Dockerfile b/ui/Dockerfile index 9384f90db..ad775ece2 100644 --- a/ui/Dockerfile +++ b/ui/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.6 +FROM registry.opensource.zalan.do/library/alpine-3.12:latest LABEL maintainer="Team ACID @ Zalando " EXPOSE 8081 diff --git a/ui/requirements.txt b/ui/requirements.txt index 7dc49eb3d..8f612d554 100644 --- a/ui/requirements.txt +++ b/ui/requirements.txt @@ -1,15 +1,15 @@ Flask-OAuthlib==0.9.5 Flask==1.1.2 -backoff==1.8.1 -boto3==1.10.4 +backoff==1.10.0 +boto3==1.16.52 boto==2.49.0 -click==6.7 -furl==1.0.2 -gevent==1.2.2 -jq==0.1.6 +click==7.1.2 +furl==2.1.0 +gevent==20.12.1 +jq==1.1.1 json_delta>=2.0 kubernetes==3.0.0 -requests==2.22.0 +requests==2.25.1 stups-tokens>=1.1.19 -wal_e==1.1.0 +wal_e==1.1.1 werkzeug==0.16.1 From e398cf8c7e7553d00a796cb278e2d22542ba2345 Mon Sep 17 00:00:00 2001 From: Rafia Sabih Date: Thu, 14 Jan 2021 09:53:09 +0100 Subject: [PATCH 160/168] Avoid syncing when possible (#1274) Avoid extra syncing in case there are no changes in pooler requirements. Add pooler specific labels to pooler secrets. Add test case to check for pooler secret creation and deletion. Co-authored-by: Rafia Sabih --- e2e/README.md | 2 +- e2e/tests/test_e2e.py | 12 ++++++++-- pkg/cluster/connection_pooler.go | 40 ++++++++++++++++++++++++++++---- pkg/cluster/k8sres.go | 9 ++++++- 4 files changed, 55 insertions(+), 8 deletions(-) diff --git a/e2e/README.md b/e2e/README.md index 3bba6ccc3..5aa987593 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -44,7 +44,7 @@ To run the end 2 end test and keep the kind state execute: NOCLEANUP=True ./run.sh main ``` -## Run indidual test +## Run individual test After having executed a normal E2E run with `NOCLEANUP=True` Kind still continues to run, allowing you subsequent test runs. diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 9e2035652..ecc0b2327 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -160,7 +160,7 @@ class EndToEndTestCase(unittest.TestCase): self.k8s.create_with_kubectl("manifests/minimal-fake-pooler-deployment.yaml") self.eventuallyEqual(lambda: self.k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") self.eventuallyEqual(lambda: self.k8s.get_deployment_replica_count(name="acid-minimal-cluster-pooler"), 1, - "Initial broken deplyment not rolled out") + "Initial broken deployment not rolled out") self.k8s.api.custom_objects_api.patch_namespaced_custom_object( 'acid.zalan.do', 'v1', 'default', @@ -221,6 +221,8 @@ class EndToEndTestCase(unittest.TestCase): self.eventuallyEqual(lambda: k8s.count_services_with_label( 'application=db-connection-pooler,cluster-name=acid-minimal-cluster'), 2, "No pooler service found") + self.eventuallyEqual(lambda: k8s.count_secrets_with_label('application=db-connection-pooler,cluster-name=acid-minimal-cluster'), + 1, "Pooler secret not created") # Turn off only master connection pooler k8s.api.custom_objects_api.patch_namespaced_custom_object( @@ -246,6 +248,8 @@ class EndToEndTestCase(unittest.TestCase): self.eventuallyEqual(lambda: k8s.count_services_with_label( 'application=db-connection-pooler,cluster-name=acid-minimal-cluster'), 1, "No pooler service found") + self.eventuallyEqual(lambda: k8s.count_secrets_with_label('application=db-connection-pooler,cluster-name=acid-minimal-cluster'), + 1, "Secret not created") # Turn off only replica connection pooler k8s.api.custom_objects_api.patch_namespaced_custom_object( @@ -268,6 +272,8 @@ class EndToEndTestCase(unittest.TestCase): 0, "Pooler replica pods not deleted") self.eventuallyEqual(lambda: k8s.count_services_with_label('application=db-connection-pooler,cluster-name=acid-minimal-cluster'), 1, "No pooler service found") + self.eventuallyEqual(lambda: k8s.count_secrets_with_label('application=db-connection-pooler,cluster-name=acid-minimal-cluster'), + 1, "Secret not created") # scale up connection pooler deployment k8s.api.custom_objects_api.patch_namespaced_custom_object( @@ -301,6 +307,8 @@ class EndToEndTestCase(unittest.TestCase): 0, "Pooler pods not scaled down") self.eventuallyEqual(lambda: k8s.count_services_with_label('application=db-connection-pooler,cluster-name=acid-minimal-cluster'), 0, "Pooler service not removed") + self.eventuallyEqual(lambda: k8s.count_secrets_with_label('application=spilo,cluster-name=acid-minimal-cluster'), + 4, "Secrets not deleted") # Verify that all the databases have pooler schema installed. # Do this via psql, since otherwise we need to deal with @@ -1034,7 +1042,7 @@ class EndToEndTestCase(unittest.TestCase): except timeout_decorator.TimeoutError: print('Operator log: {}'.format(k8s.get_operator_log())) raise - + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_zzzz_cluster_deletion(self): ''' diff --git a/pkg/cluster/connection_pooler.go b/pkg/cluster/connection_pooler.go index 1d1d609e4..e6cc60cb2 100644 --- a/pkg/cluster/connection_pooler.go +++ b/pkg/cluster/connection_pooler.go @@ -539,13 +539,13 @@ func updateConnectionPoolerAnnotations(KubeClient k8sutil.KubernetesClient, depl // Test if two connection pooler configuration needs to be synced. For simplicity // compare not the actual K8S objects, but the configuration itself and request // sync if there is any difference. -func needSyncConnectionPoolerSpecs(oldSpec, newSpec *acidv1.ConnectionPooler) (sync bool, reasons []string) { +func needSyncConnectionPoolerSpecs(oldSpec, newSpec *acidv1.ConnectionPooler, logger *logrus.Entry) (sync bool, reasons []string) { reasons = []string{} sync = false changelog, err := diff.Diff(oldSpec, newSpec) if err != nil { - //c.logger.Infof("Cannot get diff, do not do anything, %+v", err) + logger.Infof("cannot get diff, do not do anything, %+v", err) return false, reasons } @@ -681,13 +681,45 @@ func logPoolerEssentials(log *logrus.Entry, oldSpec, newSpec *acidv1.Postgresql) } func (c *Cluster) syncConnectionPooler(oldSpec, newSpec *acidv1.Postgresql, LookupFunction InstallFunction) (SyncReason, error) { - logPoolerEssentials(c.logger, oldSpec, newSpec) var reason SyncReason var err error var newNeedConnectionPooler, oldNeedConnectionPooler bool oldNeedConnectionPooler = false + if oldSpec == nil { + oldSpec = &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + }, + } + } + + needSync, _ := needSyncConnectionPoolerSpecs(oldSpec.Spec.ConnectionPooler, newSpec.Spec.ConnectionPooler, c.logger) + masterChanges, err := diff.Diff(oldSpec.Spec.EnableConnectionPooler, newSpec.Spec.EnableConnectionPooler) + if err != nil { + c.logger.Error("Error in getting diff of master connection pooler changes") + } + replicaChanges, err := diff.Diff(oldSpec.Spec.EnableReplicaConnectionPooler, newSpec.Spec.EnableReplicaConnectionPooler) + if err != nil { + c.logger.Error("Error in getting diff of replica connection pooler changes") + } + + // skip pooler sync only + // 1. if there is no diff in spec, AND + // 2. if connection pooler is already there and is also required as per newSpec + // + // Handling the case when connectionPooler is not there but it is required + // as per spec, hence do not skip syncing in that case, even though there + // is no diff in specs + if (!needSync && len(masterChanges) <= 0 && len(replicaChanges) <= 0) && + (c.ConnectionPooler != nil && *newSpec.Spec.EnableConnectionPooler) { + c.logger.Debugln("syncing pooler is not required") + return nil, nil + } + + logPoolerEssentials(c.logger, oldSpec, newSpec) + // Check and perform the sync requirements for each of the roles. for _, role := range [2]PostgresRole{Master, Replica} { @@ -841,7 +873,7 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql var specReason []string if oldSpec != nil { - specSync, specReason = needSyncConnectionPoolerSpecs(oldConnectionPooler, newConnectionPooler) + specSync, specReason = needSyncConnectionPoolerSpecs(oldConnectionPooler, newConnectionPooler, c.logger) } defaultsSync, defaultsReason := needSyncConnectionPoolerDefaults(&c.Config, newConnectionPooler, deployment) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 6b1af045f..06b074b4c 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1561,11 +1561,17 @@ func (c *Cluster) generateSingleUserSecret(namespace string, pgUser spec.PgUser) } username := pgUser.Name + lbls := c.labelsSet(true) + + if username == constants.ConnectionPoolerUserName { + lbls = c.connectionPoolerLabels("", false).MatchLabels + } + secret := v1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: c.credentialSecretName(username), Namespace: namespace, - Labels: c.labelsSet(true), + Labels: lbls, Annotations: c.annotationsSet(nil), }, Type: v1.SecretTypeOpaque, @@ -1574,6 +1580,7 @@ func (c *Cluster) generateSingleUserSecret(namespace string, pgUser spec.PgUser) "password": []byte(pgUser.Password), }, } + return &secret } From 258799b420e4c2193b8fa3afd6f3f9113fa952a1 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 15 Jan 2021 15:11:02 +0100 Subject: [PATCH 161/168] allow additional members from other teams (#1314) --- pkg/cluster/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go index d5e887656..393a490bd 100644 --- a/pkg/cluster/util.go +++ b/pkg/cluster/util.go @@ -240,7 +240,7 @@ func (c *Cluster) getTeamMembers(teamID string) ([]string, error) { c.logger.Debugf("fetching possible additional team members for team %q", teamID) members := []string{} - additionalMembers := c.PgTeamMap[c.Spec.TeamID].AdditionalMembers + additionalMembers := c.PgTeamMap[teamID].AdditionalMembers for _, member := range additionalMembers { members = append(members, member) } From 2b45478f3ac00e74036b7a43e5de3ab83aff0c23 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 19 Jan 2021 10:47:32 +0100 Subject: [PATCH 162/168] add host info to connection docs (#1319) --- docs/user.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/user.md b/docs/user.md index 3463983f7..ec5941d9e 100644 --- a/docs/user.md +++ b/docs/user.md @@ -71,26 +71,26 @@ kubectl describe postgresql acid-minimal-cluster ## Connect to PostgreSQL With a `port-forward` on one of the database pods (e.g. the master) you can -connect to the PostgreSQL database. Use labels to filter for the master pod of -our test cluster. +connect to the PostgreSQL database from your machine. Use labels to filter for +the master pod of our test cluster. ```bash # get name of master pod of acid-minimal-cluster -export PGMASTER=$(kubectl get pods -o jsonpath={.items..metadata.name} -l application=spilo,cluster-name=acid-minimal-cluster,spilo-role=master) +export PGMASTER=$(kubectl get pods -o jsonpath={.items..metadata.name} -l application=spilo,cluster-name=acid-minimal-cluster,spilo-role=master -n default) # set up port forward -kubectl port-forward $PGMASTER 6432:5432 +kubectl port-forward $PGMASTER 6432:5432 -n default ``` -Open another CLI and connect to the database. Use the generated secret of the -`postgres` robot user to connect to our `acid-minimal-cluster` master running -in Minikube. As non-encrypted connections are rejected by default set the SSL -mode to require: +Open another CLI and connect to the database using e.g. the psql client. +When connecting with the `postgres` user read its password from the K8s secret +which was generated when creating the `acid-minimal-cluster`. As non-encrypted +connections are rejected by default set the SSL mode to `require`: ```bash export PGPASSWORD=$(kubectl get secret postgres.acid-minimal-cluster.credentials -o 'jsonpath={.data.password}' | base64 -d) export PGSSLMODE=require -psql -U postgres -p 6432 +psql -U postgres -h localhost -p 6432 ``` ## Defining database roles in the operator From a9b677c957be09de5e02f3b0eec4221c551eb2ca Mon Sep 17 00:00:00 2001 From: Rafia Sabih Date: Tue, 19 Jan 2021 17:40:20 +0100 Subject: [PATCH 163/168] Use fake client for connection pooler (#1301) Connection pooler creation, deletion, and synchronization now tested using fake client API. Co-authored-by: Rafia Sabih --- pkg/cluster/connection_pooler.go | 2 +- pkg/cluster/connection_pooler_test.go | 540 ++++++++++++++------------ 2 files changed, 297 insertions(+), 245 deletions(-) diff --git a/pkg/cluster/connection_pooler.go b/pkg/cluster/connection_pooler.go index e6cc60cb2..2e3f04876 100644 --- a/pkg/cluster/connection_pooler.go +++ b/pkg/cluster/connection_pooler.go @@ -713,7 +713,7 @@ func (c *Cluster) syncConnectionPooler(oldSpec, newSpec *acidv1.Postgresql, Look // as per spec, hence do not skip syncing in that case, even though there // is no diff in specs if (!needSync && len(masterChanges) <= 0 && len(replicaChanges) <= 0) && - (c.ConnectionPooler != nil && *newSpec.Spec.EnableConnectionPooler) { + (c.ConnectionPooler != nil && (needConnectionPooler(&newSpec.Spec))) { c.logger.Debugln("syncing pooler is not required") return nil, nil } diff --git a/pkg/cluster/connection_pooler_test.go b/pkg/cluster/connection_pooler_test.go index 54be0f5bd..280adb101 100644 --- a/pkg/cluster/connection_pooler_test.go +++ b/pkg/cluster/connection_pooler_test.go @@ -6,13 +6,17 @@ import ( "strings" "testing" + "github.com/stretchr/testify/assert" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + fakeacidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/fake" + "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/k8sutil" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" ) func mockInstallLookupFunction(schema string, user string, role PostgresRole) error { @@ -27,79 +31,122 @@ func int32ToPointer(value int32) *int32 { return &value } -func TestConnectionPoolerCreationAndDeletion(t *testing.T) { - testName := "Test connection pooler creation" - var cluster = New( - Config{ - OpConfig: config.Config{ - ProtectedRoles: []string{"admin"}, - Auth: config.Auth{ - SuperUsername: superUserName, - ReplicationUsername: replicationUserName, - }, - ConnectionPooler: config.ConnectionPooler{ - ConnectionPoolerDefaultCPURequest: "100m", - ConnectionPoolerDefaultCPULimit: "100m", - ConnectionPoolerDefaultMemoryRequest: "100Mi", - ConnectionPoolerDefaultMemoryLimit: "100Mi", - NumberOfInstances: int32ToPointer(1), - }, - }, - }, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger, eventRecorder) - - cluster.Statefulset = &appsv1.StatefulSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-sts", - }, - } - - cluster.Spec = acidv1.PostgresSpec{ - ConnectionPooler: &acidv1.ConnectionPooler{}, - EnableReplicaConnectionPooler: boolToPointer(true), - } - - reason, err := cluster.createConnectionPooler(mockInstallLookupFunction) - - if err != nil { - t.Errorf("%s: Cannot create connection pooler, %s, %+v", - testName, err, reason) - } +func deploymentUpdated(cluster *Cluster, err error, reason SyncReason) error { for _, role := range [2]PostgresRole{Master, Replica} { - if cluster.ConnectionPooler[role] != nil { - if cluster.ConnectionPooler[role].Deployment == nil { - t.Errorf("%s: Connection pooler deployment is empty for role %s", testName, role) - } - if cluster.ConnectionPooler[role].Service == nil { - t.Errorf("%s: Connection pooler service is empty for role %s", testName, role) - } + poolerLabels := cluster.labelsSet(false) + poolerLabels["application"] = "db-connection-pooler" + poolerLabels["connection-pooler"] = cluster.connectionPoolerName(role) + + if cluster.ConnectionPooler[role] != nil && cluster.ConnectionPooler[role].Deployment != nil && + util.MapContains(cluster.ConnectionPooler[role].Deployment.Labels, poolerLabels) && + (cluster.ConnectionPooler[role].Deployment.Spec.Replicas == nil || + *cluster.ConnectionPooler[role].Deployment.Spec.Replicas != 2) { + return fmt.Errorf("Wrong number of instances") } } - oldSpec := &acidv1.Postgresql{ - Spec: acidv1.PostgresSpec{ - EnableConnectionPooler: boolToPointer(true), - EnableReplicaConnectionPooler: boolToPointer(true), - }, - } - newSpec := &acidv1.Postgresql{ - Spec: acidv1.PostgresSpec{ - EnableConnectionPooler: boolToPointer(false), - EnableReplicaConnectionPooler: boolToPointer(false), - }, + return nil +} + +func objectsAreSaved(cluster *Cluster, err error, reason SyncReason) error { + if cluster.ConnectionPooler == nil { + return fmt.Errorf("Connection pooler resources are empty") } - // Delete connection pooler via sync - _, err = cluster.syncConnectionPooler(oldSpec, newSpec, mockInstallLookupFunction) - if err != nil { - t.Errorf("%s: Cannot sync connection pooler, %s", testName, err) - } + for _, role := range []PostgresRole{Master, Replica} { + poolerLabels := cluster.labelsSet(false) + poolerLabels["application"] = "db-connection-pooler" + poolerLabels["connection-pooler"] = cluster.connectionPoolerName(role) - for _, role := range [2]PostgresRole{Master, Replica} { - err = cluster.deleteConnectionPooler(role) - if err != nil { - t.Errorf("%s: Cannot delete connection pooler, %s", testName, err) + if cluster.ConnectionPooler[role].Deployment == nil || !util.MapContains(cluster.ConnectionPooler[role].Deployment.Labels, poolerLabels) { + return fmt.Errorf("Deployment was not saved or labels not attached %s %s", role, cluster.ConnectionPooler[role].Deployment.Labels) + } + + if cluster.ConnectionPooler[role].Service == nil || !util.MapContains(cluster.ConnectionPooler[role].Service.Labels, poolerLabels) { + return fmt.Errorf("Service was not saved or labels not attached %s %s", role, cluster.ConnectionPooler[role].Service.Labels) } } + + return nil +} + +func MasterObjectsAreSaved(cluster *Cluster, err error, reason SyncReason) error { + if cluster.ConnectionPooler == nil { + return fmt.Errorf("Connection pooler resources are empty") + } + + poolerLabels := cluster.labelsSet(false) + poolerLabels["application"] = "db-connection-pooler" + poolerLabels["connection-pooler"] = cluster.connectionPoolerName(Master) + + if cluster.ConnectionPooler[Master].Deployment == nil || !util.MapContains(cluster.ConnectionPooler[Master].Deployment.Labels, poolerLabels) { + return fmt.Errorf("Deployment was not saved or labels not attached %s", cluster.ConnectionPooler[Master].Deployment.Labels) + } + + if cluster.ConnectionPooler[Master].Service == nil || !util.MapContains(cluster.ConnectionPooler[Master].Service.Labels, poolerLabels) { + return fmt.Errorf("Service was not saved or labels not attached %s", cluster.ConnectionPooler[Master].Service.Labels) + } + + return nil +} + +func ReplicaObjectsAreSaved(cluster *Cluster, err error, reason SyncReason) error { + if cluster.ConnectionPooler == nil { + return fmt.Errorf("Connection pooler resources are empty") + } + + poolerLabels := cluster.labelsSet(false) + poolerLabels["application"] = "db-connection-pooler" + poolerLabels["connection-pooler"] = cluster.connectionPoolerName(Replica) + + if cluster.ConnectionPooler[Replica].Deployment == nil || !util.MapContains(cluster.ConnectionPooler[Replica].Deployment.Labels, poolerLabels) { + return fmt.Errorf("Deployment was not saved or labels not attached %s", cluster.ConnectionPooler[Replica].Deployment.Labels) + } + + if cluster.ConnectionPooler[Replica].Service == nil || !util.MapContains(cluster.ConnectionPooler[Replica].Service.Labels, poolerLabels) { + return fmt.Errorf("Service was not saved or labels not attached %s", cluster.ConnectionPooler[Replica].Service.Labels) + } + + return nil +} + +func objectsAreDeleted(cluster *Cluster, err error, reason SyncReason) error { + for _, role := range [2]PostgresRole{Master, Replica} { + if cluster.ConnectionPooler[role] != nil && + (cluster.ConnectionPooler[role].Deployment != nil || cluster.ConnectionPooler[role].Service != nil) { + return fmt.Errorf("Connection pooler was not deleted for role %v", role) + } + } + + return nil +} + +func OnlyMasterDeleted(cluster *Cluster, err error, reason SyncReason) error { + + if cluster.ConnectionPooler[Master] != nil && + (cluster.ConnectionPooler[Master].Deployment != nil || cluster.ConnectionPooler[Master].Service != nil) { + return fmt.Errorf("Connection pooler master was not deleted") + } + return nil +} + +func OnlyReplicaDeleted(cluster *Cluster, err error, reason SyncReason) error { + + if cluster.ConnectionPooler[Replica] != nil && + (cluster.ConnectionPooler[Replica].Deployment != nil || cluster.ConnectionPooler[Replica].Service != nil) { + return fmt.Errorf("Connection pooler replica was not deleted") + } + return nil +} + +func noEmptySync(cluster *Cluster, err error, reason SyncReason) error { + for _, msg := range reason { + if strings.HasPrefix(msg, "update [] from '' to '") { + return fmt.Errorf("There is an empty reason, %s", msg) + } + } + + return nil } func TestNeedConnectionPooler(t *testing.T) { @@ -210,133 +257,178 @@ func TestNeedConnectionPooler(t *testing.T) { } } -func deploymentUpdated(cluster *Cluster, err error, reason SyncReason) error { - for _, role := range [2]PostgresRole{Master, Replica} { - if cluster.ConnectionPooler[role] != nil && cluster.ConnectionPooler[role].Deployment != nil && - (cluster.ConnectionPooler[role].Deployment.Spec.Replicas == nil || - *cluster.ConnectionPooler[role].Deployment.Spec.Replicas != 2) { - return fmt.Errorf("Wrong number of instances") - } - } - return nil -} +func TestConnectionPoolerCreateDeletion(t *testing.T) { -func objectsAreSaved(cluster *Cluster, err error, reason SyncReason) error { - if cluster.ConnectionPooler == nil { - return fmt.Errorf("Connection pooler resources are empty") + testName := "test connection pooler creation and deletion" + clientSet := fake.NewSimpleClientset() + acidClientSet := fakeacidv1.NewSimpleClientset() + namespace := "default" + + client := k8sutil.KubernetesClient{ + StatefulSetsGetter: clientSet.AppsV1(), + ServicesGetter: clientSet.CoreV1(), + DeploymentsGetter: clientSet.AppsV1(), + PostgresqlsGetter: acidClientSet.AcidV1(), + SecretsGetter: clientSet.CoreV1(), } - for _, role := range []PostgresRole{Master, Replica} { - if cluster.ConnectionPooler[role].Deployment == nil { - return fmt.Errorf("Deployment was not saved %s", role) - } - - if cluster.ConnectionPooler[role].Service == nil { - return fmt.Errorf("Service was not saved %s", role) - } - } - - return nil -} - -func MasterobjectsAreSaved(cluster *Cluster, err error, reason SyncReason) error { - if cluster.ConnectionPooler == nil { - return fmt.Errorf("Connection pooler resources are empty") - } - - if cluster.ConnectionPooler[Master].Deployment == nil { - return fmt.Errorf("Deployment was not saved") - } - - if cluster.ConnectionPooler[Master].Service == nil { - return fmt.Errorf("Service was not saved") - } - - return nil -} - -func ReplicaobjectsAreSaved(cluster *Cluster, err error, reason SyncReason) error { - if cluster.ConnectionPooler == nil { - return fmt.Errorf("Connection pooler resources are empty") - } - - if cluster.ConnectionPooler[Replica].Deployment == nil { - return fmt.Errorf("Deployment was not saved") - } - - if cluster.ConnectionPooler[Replica].Service == nil { - return fmt.Errorf("Service was not saved") - } - - return nil -} - -func objectsAreDeleted(cluster *Cluster, err error, reason SyncReason) error { - for _, role := range [2]PostgresRole{Master, Replica} { - if cluster.ConnectionPooler[role] != nil && - (cluster.ConnectionPooler[role].Deployment != nil || cluster.ConnectionPooler[role].Service != nil) { - return fmt.Errorf("Connection pooler was not deleted for role %v", role) - } - } - - return nil -} - -func OnlyMasterDeleted(cluster *Cluster, err error, reason SyncReason) error { - - if cluster.ConnectionPooler[Master] != nil && - (cluster.ConnectionPooler[Master].Deployment != nil || cluster.ConnectionPooler[Master].Service != nil) { - return fmt.Errorf("Connection pooler master was not deleted") - } - return nil -} - -func OnlyReplicaDeleted(cluster *Cluster, err error, reason SyncReason) error { - - if cluster.ConnectionPooler[Replica] != nil && - (cluster.ConnectionPooler[Replica].Deployment != nil || cluster.ConnectionPooler[Replica].Service != nil) { - return fmt.Errorf("Connection pooler replica was not deleted") - } - return nil -} - -func noEmptySync(cluster *Cluster, err error, reason SyncReason) error { - for _, msg := range reason { - if strings.HasPrefix(msg, "update [] from '' to '") { - return fmt.Errorf("There is an empty reason, %s", msg) - } - } - - return nil -} - -func TestConnectionPoolerSynchronization(t *testing.T) { - testName := "Test connection pooler synchronization" - newCluster := func(client k8sutil.KubernetesClient) *Cluster { - return New( - Config{ - OpConfig: config.Config{ - ProtectedRoles: []string{"admin"}, - Auth: config.Auth{ - SuperUsername: superUserName, - ReplicationUsername: replicationUserName, - }, - ConnectionPooler: config.ConnectionPooler{ - ConnectionPoolerDefaultCPURequest: "100m", - ConnectionPoolerDefaultCPULimit: "100m", - ConnectionPoolerDefaultMemoryRequest: "100Mi", - ConnectionPoolerDefaultMemoryLimit: "100Mi", - NumberOfInstances: int32ToPointer(1), - }, - }, - }, client, acidv1.Postgresql{}, logger, eventRecorder) - } - cluster := newCluster(k8sutil.KubernetesClient{}) - - cluster.Statefulset = &appsv1.StatefulSet{ + pg := acidv1.Postgresql{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-sts", + Name: "acid-fake-cluster", + Namespace: namespace, }, + Spec: acidv1.PostgresSpec{ + EnableConnectionPooler: boolToPointer(true), + EnableReplicaConnectionPooler: boolToPointer(true), + Volume: acidv1.Volume{ + Size: "1Gi", + }, + }, + } + + var cluster = New( + Config{ + OpConfig: config.Config{ + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", + NumberOfInstances: int32ToPointer(1), + }, + PodManagementPolicy: "ordered_ready", + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: "cluster-name", + DefaultCPURequest: "300m", + DefaultCPULimit: "300m", + DefaultMemoryRequest: "300Mi", + DefaultMemoryLimit: "300Mi", + PodRoleLabel: "spilo-role", + }, + }, + }, client, pg, logger, eventRecorder) + + cluster.Name = "acid-fake-cluster" + cluster.Namespace = "default" + + _, err := cluster.createService(Master) + assert.NoError(t, err) + _, err = cluster.createStatefulSet() + assert.NoError(t, err) + + reason, err := cluster.createConnectionPooler(mockInstallLookupFunction) + + if err != nil { + t.Errorf("%s: Cannot create connection pooler, %s, %+v", + testName, err, reason) + } + for _, role := range [2]PostgresRole{Master, Replica} { + poolerLabels := cluster.labelsSet(false) + poolerLabels["application"] = "db-connection-pooler" + poolerLabels["connection-pooler"] = cluster.connectionPoolerName(role) + + if cluster.ConnectionPooler[role] != nil { + if cluster.ConnectionPooler[role].Deployment == nil && util.MapContains(cluster.ConnectionPooler[role].Deployment.Labels, poolerLabels) { + t.Errorf("%s: Connection pooler deployment is empty for role %s", testName, role) + } + + if cluster.ConnectionPooler[role].Service == nil && util.MapContains(cluster.ConnectionPooler[role].Service.Labels, poolerLabels) { + t.Errorf("%s: Connection pooler service is empty for role %s", testName, role) + } + } + } + + oldSpec := &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + EnableConnectionPooler: boolToPointer(true), + EnableReplicaConnectionPooler: boolToPointer(true), + }, + } + newSpec := &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + EnableConnectionPooler: boolToPointer(false), + EnableReplicaConnectionPooler: boolToPointer(false), + }, + } + + // Delete connection pooler via sync + _, err = cluster.syncConnectionPooler(oldSpec, newSpec, mockInstallLookupFunction) + if err != nil { + t.Errorf("%s: Cannot sync connection pooler, %s", testName, err) + } + + for _, role := range [2]PostgresRole{Master, Replica} { + err = cluster.deleteConnectionPooler(role) + if err != nil { + t.Errorf("%s: Cannot delete connection pooler, %s", testName, err) + } + } +} + +func TestConnectionPoolerSync(t *testing.T) { + + testName := "test connection pooler synchronization" + clientSet := fake.NewSimpleClientset() + acidClientSet := fakeacidv1.NewSimpleClientset() + namespace := "default" + + client := k8sutil.KubernetesClient{ + StatefulSetsGetter: clientSet.AppsV1(), + ServicesGetter: clientSet.CoreV1(), + DeploymentsGetter: clientSet.AppsV1(), + PostgresqlsGetter: acidClientSet.AcidV1(), + SecretsGetter: clientSet.CoreV1(), + } + + pg := acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: "acid-fake-cluster", + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Volume: acidv1.Volume{ + Size: "1Gi", + }, + }, + } + + var cluster = New( + Config{ + OpConfig: config.Config{ + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", + NumberOfInstances: int32ToPointer(1), + }, + PodManagementPolicy: "ordered_ready", + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: "cluster-name", + DefaultCPURequest: "300m", + DefaultCPULimit: "300m", + DefaultMemoryRequest: "300Mi", + DefaultMemoryLimit: "300Mi", + PodRoleLabel: "spilo-role", + }, + }, + }, client, pg, logger, eventRecorder) + + cluster.Name = "acid-fake-cluster" + cluster.Namespace = "default" + + _, err := cluster.createService(Master) + assert.NoError(t, err) + _, err = cluster.createStatefulSet() + assert.NoError(t, err) + + reason, err := cluster.createConnectionPooler(mockInstallLookupFunction) + + if err != nil { + t.Errorf("%s: Cannot create connection pooler, %s, %+v", + testName, err, reason) } tests := []struct { @@ -358,10 +450,10 @@ func TestConnectionPoolerSynchronization(t *testing.T) { ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, - cluster: newCluster(k8sutil.ClientMissingObjects()), + cluster: cluster, defaultImage: "pooler:1.0", defaultInstances: 1, - check: MasterobjectsAreSaved, + check: MasterObjectsAreSaved, }, { subTest: "create if doesn't exist", @@ -375,10 +467,10 @@ func TestConnectionPoolerSynchronization(t *testing.T) { ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, - cluster: newCluster(k8sutil.ClientMissingObjects()), + cluster: cluster, defaultImage: "pooler:1.0", defaultInstances: 1, - check: MasterobjectsAreSaved, + check: MasterObjectsAreSaved, }, { subTest: "create if doesn't exist with a flag", @@ -390,10 +482,10 @@ func TestConnectionPoolerSynchronization(t *testing.T) { EnableConnectionPooler: boolToPointer(true), }, }, - cluster: newCluster(k8sutil.ClientMissingObjects()), + cluster: cluster, defaultImage: "pooler:1.0", defaultInstances: 1, - check: MasterobjectsAreSaved, + check: MasterObjectsAreSaved, }, { subTest: "create no replica with flag", @@ -405,7 +497,7 @@ func TestConnectionPoolerSynchronization(t *testing.T) { EnableReplicaConnectionPooler: boolToPointer(false), }, }, - cluster: newCluster(k8sutil.NewMockKubernetesClient()), + cluster: cluster, defaultImage: "pooler:1.0", defaultInstances: 1, check: objectsAreDeleted, @@ -421,10 +513,10 @@ func TestConnectionPoolerSynchronization(t *testing.T) { EnableReplicaConnectionPooler: boolToPointer(true), }, }, - cluster: newCluster(k8sutil.NewMockKubernetesClient()), + cluster: cluster, defaultImage: "pooler:1.0", defaultInstances: 1, - check: ReplicaobjectsAreSaved, + check: ReplicaObjectsAreSaved, }, { subTest: "create both master and replica", @@ -438,7 +530,7 @@ func TestConnectionPoolerSynchronization(t *testing.T) { EnableConnectionPooler: boolToPointer(true), }, }, - cluster: newCluster(k8sutil.ClientMissingObjects()), + cluster: cluster, defaultImage: "pooler:1.0", defaultInstances: 1, check: objectsAreSaved, @@ -456,7 +548,7 @@ func TestConnectionPoolerSynchronization(t *testing.T) { ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, - cluster: newCluster(k8sutil.NewMockKubernetesClient()), + cluster: cluster, defaultImage: "pooler:1.0", defaultInstances: 1, check: OnlyReplicaDeleted, @@ -474,7 +566,7 @@ func TestConnectionPoolerSynchronization(t *testing.T) { EnableReplicaConnectionPooler: boolToPointer(true), }, }, - cluster: newCluster(k8sutil.NewMockKubernetesClient()), + cluster: cluster, defaultImage: "pooler:1.0", defaultInstances: 1, check: OnlyMasterDeleted, @@ -489,7 +581,7 @@ func TestConnectionPoolerSynchronization(t *testing.T) { newSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{}, }, - cluster: newCluster(k8sutil.NewMockKubernetesClient()), + cluster: cluster, defaultImage: "pooler:1.0", defaultInstances: 1, check: objectsAreDeleted, @@ -502,53 +594,11 @@ func TestConnectionPoolerSynchronization(t *testing.T) { newSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{}, }, - cluster: newCluster(k8sutil.NewMockKubernetesClient()), + cluster: cluster, defaultImage: "pooler:1.0", defaultInstances: 1, check: objectsAreDeleted, }, - { - subTest: "update deployment", - oldSpec: &acidv1.Postgresql{ - Spec: acidv1.PostgresSpec{ - ConnectionPooler: &acidv1.ConnectionPooler{ - NumberOfInstances: int32ToPointer(1), - }, - }, - }, - newSpec: &acidv1.Postgresql{ - Spec: acidv1.PostgresSpec{ - ConnectionPooler: &acidv1.ConnectionPooler{ - NumberOfInstances: int32ToPointer(2), - }, - }, - }, - cluster: newCluster(k8sutil.NewMockKubernetesClient()), - defaultImage: "pooler:1.0", - defaultInstances: 1, - check: deploymentUpdated, - }, - { - subTest: "update deployment", - oldSpec: &acidv1.Postgresql{ - Spec: acidv1.PostgresSpec{ - ConnectionPooler: &acidv1.ConnectionPooler{ - NumberOfInstances: int32ToPointer(1), - }, - }, - }, - newSpec: &acidv1.Postgresql{ - Spec: acidv1.PostgresSpec{ - ConnectionPooler: &acidv1.ConnectionPooler{ - NumberOfInstances: int32ToPointer(2), - }, - }, - }, - cluster: newCluster(k8sutil.NewMockKubernetesClient()), - defaultImage: "pooler:1.0", - defaultInstances: 1, - check: deploymentUpdated, - }, { subTest: "update image from changed defaults", oldSpec: &acidv1.Postgresql{ @@ -561,7 +611,7 @@ func TestConnectionPoolerSynchronization(t *testing.T) { ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, - cluster: newCluster(k8sutil.NewMockKubernetesClient()), + cluster: cluster, defaultImage: "pooler:2.0", defaultInstances: 2, check: deploymentUpdated, @@ -580,7 +630,7 @@ func TestConnectionPoolerSynchronization(t *testing.T) { ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, - cluster: newCluster(k8sutil.NewMockKubernetesClient()), + cluster: cluster, defaultImage: "pooler:1.0", defaultInstances: 1, check: noEmptySync, @@ -591,6 +641,8 @@ func TestConnectionPoolerSynchronization(t *testing.T) { tt.cluster.OpConfig.ConnectionPooler.NumberOfInstances = int32ToPointer(tt.defaultInstances) + t.Logf("running test for %s [%s]", testName, tt.subTest) + reason, err := tt.cluster.syncConnectionPooler(tt.oldSpec, tt.newSpec, mockInstallLookupFunction) From 4ea0b5f432e3d1ff084d2904a81f12ad84d709dc Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 22 Jan 2021 14:06:19 +0100 Subject: [PATCH 164/168] set AllowPrivilegeEscalation on container securityContext (#1326) --- .../templates/clusterrole-postgres-pod.yaml | 2 ++ .../templates/clusterrole.yaml | 4 ++- manifests/operator-service-account-rbac.yaml | 36 +++++++++---------- pkg/cluster/k8sres.go | 5 +-- 4 files changed, 26 insertions(+), 21 deletions(-) diff --git a/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml b/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml index b3f9f08f5..33c43822f 100644 --- a/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml +++ b/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml @@ -63,6 +63,7 @@ rules: - services verbs: - create +{{- if toString .Values.configKubernetes.spilo_privileged | eq "true" }} # to run privileged pods - apiGroups: - extensions @@ -72,4 +73,5 @@ rules: - privileged verbs: - use +{{- end }} {{ end }} diff --git a/charts/postgres-operator/templates/clusterrole.yaml b/charts/postgres-operator/templates/clusterrole.yaml index 165cce7c6..885bad3f7 100644 --- a/charts/postgres-operator/templates/clusterrole.yaml +++ b/charts/postgres-operator/templates/clusterrole.yaml @@ -228,7 +228,8 @@ rules: verbs: - get - create -# to grant privilege to run privileged pods +{{- if toString .Values.configKubernetes.spilo_privileged | eq "true" }} +# to run privileged pods - apiGroups: - extensions resources: @@ -237,4 +238,5 @@ rules: - privileged verbs: - use +{{- end }} {{ end }} diff --git a/manifests/operator-service-account-rbac.yaml b/manifests/operator-service-account-rbac.yaml index 1ba5b4d23..f0307f6a0 100644 --- a/manifests/operator-service-account-rbac.yaml +++ b/manifests/operator-service-account-rbac.yaml @@ -203,15 +203,15 @@ rules: verbs: - get - create -# to grant privilege to run privileged pods -- apiGroups: - - extensions - resources: - - podsecuritypolicies - resourceNames: - - privileged - verbs: - - use +# to grant privilege to run privileged pods (not needed by default) +#- apiGroups: +# - extensions +# resources: +# - podsecuritypolicies +# resourceNames: +# - privileged +# verbs: +# - use --- apiVersion: rbac.authorization.k8s.io/v1 @@ -265,12 +265,12 @@ rules: - services verbs: - create -# to run privileged pods -- apiGroups: - - extensions - resources: - - podsecuritypolicies - resourceNames: - - privileged - verbs: - - use +# to grant privilege to run privileged pods (not needed by default) +#- apiGroups: +# - extensions +# resources: +# - podsecuritypolicies +# resourceNames: +# - privileged +# verbs: +# - use diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 06b074b4c..83098b8a9 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -453,8 +453,9 @@ func generateContainer( VolumeMounts: volumeMounts, Env: envVars, SecurityContext: &v1.SecurityContext{ - Privileged: &privilegedMode, - ReadOnlyRootFilesystem: util.False(), + AllowPrivilegeEscalation: &privilegedMode, + Privileged: &privilegedMode, + ReadOnlyRootFilesystem: util.False(), }, } } From 4a88f00a3f8e56fde9bd31b9a7cd704cd6b949cc Mon Sep 17 00:00:00 2001 From: Jan Mussler Date: Mon, 25 Jan 2021 10:07:18 +0100 Subject: [PATCH 165/168] Full AWS gp3 support for iops and througput config. (#1261) Support new AWS EBS volume type `gp3` with `iops` and `throughput` in the manifest. Co-authored-by: Felix Kunde --- .../postgres-operator/crds/postgresqls.yaml | 4 + docs/developer.md | 18 ++ docs/reference/cluster_manifest.md | 12 +- docs/reference/operator_parameters.md | 11 +- go.mod | 2 +- go.sum | 8 +- manifests/complete-postgres-manifest.yaml | 2 + manifests/postgresql.crd.yaml | 4 + pkg/apis/acid.zalan.do/v1/crds.go | 6 + pkg/apis/acid.zalan.do/v1/postgresql_type.go | 1 + pkg/cluster/sync.go | 72 +---- pkg/cluster/volumes.go | 240 ++++++++++++-- pkg/cluster/volumes_test.go | 305 +++++++++++++++--- pkg/util/volumes/ebs.go | 13 +- pkg/util/volumes/volumes.go | 2 +- 15 files changed, 535 insertions(+), 165 deletions(-) diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 13811936d..ad11f6407 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -557,6 +557,8 @@ spec: required: - size properties: + iops: + type: integer size: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' @@ -565,6 +567,8 @@ spec: type: string subPath: type: string + throughput: + type: integer status: type: object additionalProperties: diff --git a/docs/developer.md b/docs/developer.md index 3316ac4cc..8ab1e60bc 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -235,6 +235,24 @@ Then you can for example check the Patroni logs: kubectl logs acid-minimal-cluster-0 ``` +## Unit tests with Mocks and K8s Fake API + +Whenever possible you should rely on leveraging proper mocks and K8s fake client that allows full fledged testing of K8s objects in your unit tests. + +To enable mocks, a code annotation is needed: +[Mock code gen annotation](https://github.com/zalando/postgres-operator/blob/master/pkg/util/volumes/volumes.go#L3) + +To generate mocks run: +```bash +make mocks +``` + +Examples for mocks can be found in: +[Example mock usage](https://github.com/zalando/postgres-operator/blob/master/pkg/cluster/volumes_test.go#L248) + +Examples for fake K8s objects can be found in: +[Example fake K8s client usage](https://github.com/zalando/postgres-operator/blob/master/pkg/cluster/volumes_test.go#L166) + ## End-to-end tests The operator provides reference end-to-end (e2e) tests to diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 589921bc5..1b2d71a66 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -338,13 +338,13 @@ archive is supported. the url to S3 bucket containing the WAL archive of the remote primary. Required when the `standby` section is present. -## EBS volume resizing +## Volume properties Those parameters are grouped under the `volume` top-level key and define the properties of the persistent storage that stores Postgres data. * **size** - the size of the target EBS volume. Usual Kubernetes size modifiers, i.e. `Gi` + the size of the target volume. Usual Kubernetes size modifiers, i.e. `Gi` or `Mi`, apply. Required. * **storageClass** @@ -356,6 +356,14 @@ properties of the persistent storage that stores Postgres data. * **subPath** Subpath to use when mounting volume into Spilo container. Optional. +* **iops** + When running the operator on AWS the latest generation of EBS volumes (`gp3`) + allows for configuring the number of IOPS. Maximum is 16000. Optional. + +* **throughput** + When running the operator on AWS the latest generation of EBS volumes (`gp3`) + allows for configuring the throughput in MB/s. Maximum is 1000. Optional. + ## Sidecar definitions Those parameters are defined under the `sidecars` key. They consist of a list diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 212515dc1..5c5850505 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -373,10 +373,13 @@ configuration they are grouped under the `kubernetes` key. possible value is `parallel`. * **storage_resize_mode** - defines how operator handels the difference between requested volume size and - actual size. Available options are: ebs - tries to resize EBS volume, pvc - - changes PVC definition, off - disables resize of the volumes. Default is "pvc". - When using OpenShift please use one of the other available options. + defines how operator handles the difference between the requested volume size and + the actual size. Available options are: + 1. `ebs` : operator resizes EBS volumes directly and executes `resizefs` within a pod + 2. `pvc` : operator only changes PVC definition + 3. `off` : disables resize of the volumes. + 4. `mixed` :operator uses AWS API to adjust size, throughput, and IOPS, and calls pvc change for file system resize + Default is "pvc". ## Kubernetes resource requests diff --git a/go.mod b/go.mod index 4e9c8a742..bbe5140b7 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/zalando/postgres-operator go 1.15 require ( - github.com/aws/aws-sdk-go v1.36.3 + github.com/aws/aws-sdk-go v1.36.29 github.com/golang/mock v1.4.4 github.com/lib/pq v1.9.0 github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d diff --git a/go.sum b/go.sum index d8df3fda4..fa8d2b135 100644 --- a/go.sum +++ b/go.sum @@ -35,6 +35,8 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/Storytel/gomock-matchers v1.2.0 h1:VPsbL6c/9/eCa4rH13LOEXPsIsnA1z+INamGIx1lWQo= +github.com/Storytel/gomock-matchers v1.2.0/go.mod h1:7HEuwyU/eq/W3mrSqPSYETGXiTyU2um0Rrb+dh5KmKM= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -45,8 +47,8 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.36.3 h1:KYpG5OegwW3xgOsMxy01nj/Td281yxi1Ha2lJQJs4tI= -github.com/aws/aws-sdk-go v1.36.3/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go v1.36.29 h1:lM1G3AF1+7vzFm0n7hfH8r2+750BTo+6Lo6FtPB7kzk= +github.com/aws/aws-sdk-go v1.36.29/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -182,6 +184,7 @@ github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 h1:5ZkaAPbicIKTF github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.1-0.20190311213431-837231f7bb37/go.mod h1:L3bP22mxdfCUHSUVMs+SPJMx55FrxQew7MSXT11Q86g= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= @@ -525,6 +528,7 @@ golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190221204921-83362c3779f5/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 412bac29b..f721f0ccb 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -44,6 +44,8 @@ spec: volume: size: 1Gi # storageClass: my-sc +# iops: 1000 # for EBS gp3 + # throughput: 250 # in MB/s for EBS gp3 additionalVolumes: - name: empty mountPath: /opt/empty diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index d5170e9d4..61a04144c 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -553,6 +553,8 @@ spec: required: - size properties: + iops: + type: integer size: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' @@ -561,6 +563,8 @@ spec: type: string subPath: type: string + throughput: + type: integer status: type: object additionalProperties: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index f03b4c2ab..02d40342f 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -835,6 +835,9 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ Type: "object", Required: []string{"size"}, Properties: map[string]apiextv1.JSONSchemaProps{ + "iops": { + Type: "integer", + }, "size": { Type: "string", Description: "Value must not be zero", @@ -846,6 +849,9 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ "subPath": { Type: "string", }, + "throughput": { + Type: "integer", + }, }, }, }, diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index bdae22a7c..7346fb0e5 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -118,6 +118,7 @@ type Volume struct { SubPath string `json:"subPath,omitempty"` Iops *int64 `json:"iops,omitempty"` Throughput *int64 `json:"throughput,omitempty"` + VolumeType string `json:"type,omitempty"` } // AdditionalVolume specs additional optional volumes for statefulset diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index dc54ae8ee..5c0f6ce84 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -53,8 +53,6 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { return err } - c.logger.Debugf("syncing volumes using %q storage resize mode", c.OpConfig.StorageResizeMode) - if c.OpConfig.EnableEBSGp3Migration { err = c.executeEBSMigration() if nil != err { @@ -62,32 +60,8 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { } } - if c.OpConfig.StorageResizeMode == "mixed" { - // mixed op uses AWS API to adjust size,throughput,iops and calls pvc chance for file system resize - - // resize pvc to adjust filesystem size until better K8s support - if err = c.syncVolumeClaims(); err != nil { - err = fmt.Errorf("could not sync persistent volume claims: %v", err) - return err - } - } else if c.OpConfig.StorageResizeMode == "pvc" { - if err = c.syncVolumeClaims(); err != nil { - err = fmt.Errorf("could not sync persistent volume claims: %v", err) - return err - } - } else if c.OpConfig.StorageResizeMode == "ebs" { - // potentially enlarge volumes before changing the statefulset. By doing that - // in this order we make sure the operator is not stuck waiting for a pod that - // cannot start because it ran out of disk space. - // TODO: handle the case of the cluster that is downsized and enlarged again - // (there will be a volume from the old pod for which we can't act before the - // the statefulset modification is concluded) - if err = c.syncVolumes(); err != nil { - err = fmt.Errorf("could not sync persistent volumes: %v", err) - return err - } - } else { - c.logger.Infof("Storage resize is disabled (storage_resize_mode is off). Skipping volume sync.") + if err = c.syncVolumes(); err != nil { + return err } if err = c.enforceMinResourceLimits(&c.Spec); err != nil { @@ -590,48 +564,6 @@ func (c *Cluster) syncRoles() (err error) { return nil } -// syncVolumeClaims reads all persistent volume claims and checks that their size matches the one declared in the statefulset. -func (c *Cluster) syncVolumeClaims() error { - c.setProcessName("syncing volume claims") - - act, err := c.volumeClaimsNeedResizing(c.Spec.Volume) - if err != nil { - return fmt.Errorf("could not compare size of the volume claims: %v", err) - } - if !act { - c.logger.Infof("volume claims do not require changes") - return nil - } - if err := c.resizeVolumeClaims(c.Spec.Volume); err != nil { - return fmt.Errorf("could not sync volume claims: %v", err) - } - - c.logger.Infof("volume claims have been synced successfully") - - return nil -} - -// syncVolumes reads all persistent volumes and checks that their size matches the one declared in the statefulset. -func (c *Cluster) syncVolumes() error { - c.setProcessName("syncing volumes") - - act, err := c.volumesNeedResizing(c.Spec.Volume) - if err != nil { - return fmt.Errorf("could not compare size of the volumes: %v", err) - } - if !act { - return nil - } - - if err := c.resizeVolumes(); err != nil { - return fmt.Errorf("could not sync volumes: %v", err) - } - - c.logger.Infof("volumes have been synced successfully") - - return nil -} - func (c *Cluster) syncDatabases() error { c.setProcessName("syncing databases") diff --git a/pkg/cluster/volumes.go b/pkg/cluster/volumes.go index 162d24075..e07d453ec 100644 --- a/pkg/cluster/volumes.go +++ b/pkg/cluster/volumes.go @@ -10,13 +10,215 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/aws/aws-sdk-go/aws" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/filesystems" + "github.com/zalando/postgres-operator/pkg/util/volumes" ) +func (c *Cluster) syncVolumes() error { + c.logger.Debugf("syncing volumes using %q storage resize mode", c.OpConfig.StorageResizeMode) + var err error + + // check quantity string once, and do not bother with it anymore anywhere else + _, err = resource.ParseQuantity(c.Spec.Volume.Size) + if err != nil { + return fmt.Errorf("could not parse volume size from the manifest: %v", err) + } + + if c.OpConfig.StorageResizeMode == "mixed" { + // mixed op uses AWS API to adjust size, throughput, iops, and calls pvc change for file system resize + // in case of errors we proceed to let K8s do its work, favoring disk space increase of other adjustments + + err = c.populateVolumeMetaData() + if err != nil { + c.logger.Errorf("populating EBS meta data failed, skipping potential adjustements: %v", err) + } else { + err = c.syncUnderlyingEBSVolume() + if err != nil { + c.logger.Errorf("errors occured during EBS volume adjustments: %v", err) + } + } + + // resize pvc to adjust filesystem size until better K8s support + if err = c.syncVolumeClaims(); err != nil { + err = fmt.Errorf("could not sync persistent volume claims: %v", err) + return err + } + } else if c.OpConfig.StorageResizeMode == "pvc" { + if err = c.syncVolumeClaims(); err != nil { + err = fmt.Errorf("could not sync persistent volume claims: %v", err) + return err + } + } else if c.OpConfig.StorageResizeMode == "ebs" { + // potentially enlarge volumes before changing the statefulset. By doing that + // in this order we make sure the operator is not stuck waiting for a pod that + // cannot start because it ran out of disk space. + // TODO: handle the case of the cluster that is downsized and enlarged again + // (there will be a volume from the old pod for which we can't act before the + // the statefulset modification is concluded) + if err = c.syncEbsVolumes(); err != nil { + err = fmt.Errorf("could not sync persistent volumes: %v", err) + return err + } + } else { + c.logger.Infof("Storage resize is disabled (storage_resize_mode is off). Skipping volume sync.") + } + + return nil +} + +func (c *Cluster) syncUnderlyingEBSVolume() error { + c.logger.Infof("starting to sync EBS volumes: type, iops, throughput, and size") + + var err error + + targetValue := c.Spec.Volume + newSize, err := resource.ParseQuantity(targetValue.Size) + targetSize := quantityToGigabyte(newSize) + + awsGp3 := aws.String("gp3") + awsIo2 := aws.String("io2") + + errors := []string{} + + for _, volume := range c.EBSVolumes { + var modifyIops *int64 + var modifyThroughput *int64 + var modifySize *int64 + var modifyType *string + + if targetValue.Iops != nil { + if volume.Iops != *targetValue.Iops { + modifyIops = targetValue.Iops + } + } + + if targetValue.Throughput != nil { + if volume.Throughput != *targetValue.Throughput { + modifyThroughput = targetValue.Throughput + } + } + + if targetSize > volume.Size { + modifySize = &targetSize + } + + if modifyIops != nil || modifyThroughput != nil || modifySize != nil { + if modifyIops != nil || modifyThroughput != nil { + // we default to gp3 if iops and throughput are configured + modifyType = awsGp3 + if targetValue.VolumeType == "io2" { + modifyType = awsIo2 + } + } else if targetValue.VolumeType == "gp3" && volume.VolumeType != "gp3" { + modifyType = awsGp3 + } else { + // do not touch type + modifyType = nil + } + + err = c.VolumeResizer.ModifyVolume(volume.VolumeID, modifyType, modifySize, modifyIops, modifyThroughput) + if err != nil { + errors = append(errors, fmt.Sprintf("modify volume failed: volume=%s size=%d iops=%d throughput=%d", volume.VolumeID, volume.Size, volume.Iops, volume.Throughput)) + } + } + } + + if len(errors) > 0 { + for _, s := range errors { + c.logger.Warningf(s) + } + // c.logger.Errorf("failed to modify %d of %d volumes", len(c.EBSVolumes), len(errors)) + } + return nil +} + +func (c *Cluster) populateVolumeMetaData() error { + c.logger.Infof("starting reading ebs meta data") + + pvs, err := c.listPersistentVolumes() + if err != nil { + return fmt.Errorf("could not list persistent volumes: %v", err) + } + c.logger.Debugf("found %d volumes, size of known volumes %d", len(pvs), len(c.EBSVolumes)) + + volumeIds := []string{} + var volumeID string + for _, pv := range pvs { + volumeID, err = c.VolumeResizer.ExtractVolumeID(pv.Spec.AWSElasticBlockStore.VolumeID) + if err != nil { + continue + } + + volumeIds = append(volumeIds, volumeID) + } + + currentVolumes, err := c.VolumeResizer.DescribeVolumes(volumeIds) + if nil != err { + return err + } + + if len(currentVolumes) != len(c.EBSVolumes) { + c.logger.Debugf("number of ebs volumes (%d) discovered differs from already known volumes (%d)", len(currentVolumes), len(c.EBSVolumes)) + } + + // reset map, operator is not responsible for dangling ebs volumes + c.EBSVolumes = make(map[string]volumes.VolumeProperties) + for _, volume := range currentVolumes { + c.EBSVolumes[volume.VolumeID] = volume + } + + return nil +} + +// syncVolumeClaims reads all persistent volume claims and checks that their size matches the one declared in the statefulset. +func (c *Cluster) syncVolumeClaims() error { + c.setProcessName("syncing volume claims") + + needsResizing, err := c.volumeClaimsNeedResizing(c.Spec.Volume) + if err != nil { + return fmt.Errorf("could not compare size of the volume claims: %v", err) + } + + if !needsResizing { + c.logger.Infof("volume claims do not require changes") + return nil + } + + if err := c.resizeVolumeClaims(c.Spec.Volume); err != nil { + return fmt.Errorf("could not sync volume claims: %v", err) + } + + c.logger.Infof("volume claims have been synced successfully") + + return nil +} + +// syncVolumes reads all persistent volumes and checks that their size matches the one declared in the statefulset. +func (c *Cluster) syncEbsVolumes() error { + c.setProcessName("syncing EBS and Claims volumes") + + act, err := c.volumesNeedResizing() + if err != nil { + return fmt.Errorf("could not compare size of the volumes: %v", err) + } + if !act { + return nil + } + + if err := c.resizeVolumes(); err != nil { + return fmt.Errorf("could not sync volumes: %v", err) + } + + c.logger.Infof("volumes have been synced successfully") + + return nil +} + func (c *Cluster) listPersistentVolumeClaims() ([]v1.PersistentVolumeClaim, error) { ns := c.Namespace listOptions := metav1.ListOptions{ @@ -125,15 +327,16 @@ func (c *Cluster) resizeVolumes() error { c.setProcessName("resizing EBS volumes") - resizer := c.VolumeResizer - var totalIncompatible int - newQuantity, err := resource.ParseQuantity(c.Spec.Volume.Size) if err != nil { return fmt.Errorf("could not parse volume size: %v", err) } - pvs, newSize, err := c.listVolumesWithManifestSize(c.Spec.Volume) + newSize := quantityToGigabyte(newQuantity) + resizer := c.VolumeResizer + var totalIncompatible int + + pvs, err := c.listPersistentVolumes() if err != nil { return fmt.Errorf("could not list persistent volumes: %v", err) } @@ -214,33 +417,23 @@ func (c *Cluster) volumeClaimsNeedResizing(newVolume acidv1.Volume) (bool, error return false, nil } -func (c *Cluster) volumesNeedResizing(newVolume acidv1.Volume) (bool, error) { - vols, manifestSize, err := c.listVolumesWithManifestSize(newVolume) +func (c *Cluster) volumesNeedResizing() (bool, error) { + newQuantity, _ := resource.ParseQuantity(c.Spec.Volume.Size) + newSize := quantityToGigabyte(newQuantity) + + vols, err := c.listPersistentVolumes() if err != nil { return false, err } for _, pv := range vols { currentSize := quantityToGigabyte(pv.Spec.Capacity[v1.ResourceStorage]) - if currentSize != manifestSize { + if currentSize != newSize { return true, nil } } return false, nil } -func (c *Cluster) listVolumesWithManifestSize(newVolume acidv1.Volume) ([]*v1.PersistentVolume, int64, error) { - newSize, err := resource.ParseQuantity(newVolume.Size) - if err != nil { - return nil, 0, fmt.Errorf("could not parse volume size from the manifest: %v", err) - } - manifestSize := quantityToGigabyte(newSize) - vols, err := c.listPersistentVolumes() - if err != nil { - return nil, 0, fmt.Errorf("could not list persistent volumes: %v", err) - } - return vols, manifestSize, nil -} - // getPodNameFromPersistentVolume returns a pod name that it extracts from the volume claim ref. func getPodNameFromPersistentVolume(pv *v1.PersistentVolume) *spec.NamespacedName { namespace := pv.Spec.ClaimRef.Namespace @@ -258,7 +451,7 @@ func (c *Cluster) executeEBSMigration() error { } c.logger.Infof("starting EBS gp2 to gp3 migration") - pvs, _, err := c.listVolumesWithManifestSize(c.Spec.Volume) + pvs, err := c.listPersistentVolumes() if err != nil { return fmt.Errorf("could not list persistent volumes: %v", err) } @@ -294,10 +487,13 @@ func (c *Cluster) executeEBSMigration() error { return err } + var i3000 int64 = 3000 + var i125 int64 = 125 + for _, volume := range awsVolumes { if volume.VolumeType == "gp2" && volume.Size < c.OpConfig.EnableEBSGp3MigrationMaxSize { c.logger.Infof("modifying EBS volume %s to type gp3 migration (%d)", volume.VolumeID, volume.Size) - err = c.VolumeResizer.ModifyVolume(volume.VolumeID, "gp3", volume.Size, 3000, 125) + err = c.VolumeResizer.ModifyVolume(volume.VolumeID, aws.String("gp3"), &volume.Size, &i3000, &i125) if nil != err { c.logger.Warningf("modifying volume %s failed: %v", volume.VolumeID, err) } diff --git a/pkg/cluster/volumes_test.go b/pkg/cluster/volumes_test.go index 17fb8a4af..aea7711af 100644 --- a/pkg/cluster/volumes_test.go +++ b/pkg/cluster/volumes_test.go @@ -11,7 +11,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "github.com/aws/aws-sdk-go/aws" "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" "github.com/zalando/postgres-operator/mocks" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" @@ -187,60 +189,16 @@ func TestMigrateEBS(t *testing.T) { cluster.Namespace = namespace filterLabels := cluster.labelsSet(false) - pvcList := CreatePVCs(namespace, clusterName, filterLabels, 2, "1Gi") - - ps := v1.PersistentVolumeSpec{} - ps.AWSElasticBlockStore = &v1.AWSElasticBlockStoreVolumeSource{} - ps.AWSElasticBlockStore.VolumeID = "aws://eu-central-1b/ebs-volume-1" - - ps2 := v1.PersistentVolumeSpec{} - ps2.AWSElasticBlockStore = &v1.AWSElasticBlockStoreVolumeSource{} - ps2.AWSElasticBlockStore.VolumeID = "aws://eu-central-1b/ebs-volume-2" - - pvList := &v1.PersistentVolumeList{ - Items: []v1.PersistentVolume{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "persistent-volume-0", - }, - Spec: ps, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "persistent-volume-1", - }, - Spec: ps2, - }, + testVolumes := []testVolume{ + { + size: 100, + }, + { + size: 100, }, } - for _, pvc := range pvcList.Items { - cluster.KubeClient.PersistentVolumeClaims(namespace).Create(context.TODO(), &pvc, metav1.CreateOptions{}) - } - - for _, pv := range pvList.Items { - cluster.KubeClient.PersistentVolumes().Create(context.TODO(), &pv, metav1.CreateOptions{}) - } - - pod := v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: clusterName + "-0", - Labels: filterLabels, - }, - Spec: v1.PodSpec{}, - } - - cluster.KubeClient.Pods(namespace).Create(context.TODO(), &pod, metav1.CreateOptions{}) - - pod = v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: clusterName + "-1", - Labels: filterLabels, - }, - Spec: v1.PodSpec{}, - } - - cluster.KubeClient.Pods(namespace).Create(context.TODO(), &pod, metav1.CreateOptions{}) + initTestVolumesAndPods(cluster.KubeClient, namespace, clusterName, filterLabels, testVolumes) ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -256,8 +214,251 @@ func TestMigrateEBS(t *testing.T) { {VolumeID: "ebs-volume-2", VolumeType: "gp3", Size: 100}}, nil) // expect only gp2 volume to be modified - resizer.EXPECT().ModifyVolume(gomock.Eq("ebs-volume-1"), gomock.Eq("gp3"), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + resizer.EXPECT().ModifyVolume(gomock.Eq("ebs-volume-1"), gomock.Eq(aws.String("gp3")), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) cluster.VolumeResizer = resizer cluster.executeEBSMigration() } + +type testVolume struct { + iops int64 + throughtput int64 + size int64 + volType string +} + +func initTestVolumesAndPods(client k8sutil.KubernetesClient, namespace, clustername string, labels labels.Set, volumes []testVolume) { + i := 0 + for _, v := range volumes { + storage1Gi, _ := resource.ParseQuantity(fmt.Sprintf("%d", v.size)) + + ps := v1.PersistentVolumeSpec{} + ps.AWSElasticBlockStore = &v1.AWSElasticBlockStoreVolumeSource{} + ps.AWSElasticBlockStore.VolumeID = fmt.Sprintf("aws://eu-central-1b/ebs-volume-%d", i+1) + + pv := v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("persistent-volume-%d", i), + }, + Spec: ps, + } + + client.PersistentVolumes().Create(context.TODO(), &pv, metav1.CreateOptions{}) + + pvc := v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s-%d", constants.DataVolumeName, clustername, i), + Namespace: namespace, + Labels: labels, + }, + Spec: v1.PersistentVolumeClaimSpec{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: storage1Gi, + }, + }, + VolumeName: fmt.Sprintf("persistent-volume-%d", i), + }, + } + + client.PersistentVolumeClaims(namespace).Create(context.TODO(), &pvc, metav1.CreateOptions{}) + + pod := v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%d", clustername, i), + Labels: labels, + }, + Spec: v1.PodSpec{}, + } + + client.Pods(namespace).Create(context.TODO(), &pod, metav1.CreateOptions{}) + + i = i + 1 + } +} + +func TestMigrateGp3Support(t *testing.T) { + client, _ := newFakeK8sPVCclient() + clusterName := "acid-test-cluster" + namespace := "default" + + // new cluster with pvc storage resize mode and configured labels + var cluster = New( + Config{ + OpConfig: config.Config{ + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: "cluster-name", + }, + StorageResizeMode: "mixed", + EnableEBSGp3Migration: false, + EnableEBSGp3MigrationMaxSize: 1000, + }, + }, client, acidv1.Postgresql{}, logger, eventRecorder) + + cluster.Spec.Volume.Size = "150Gi" + cluster.Spec.Volume.Iops = aws.Int64(6000) + cluster.Spec.Volume.Throughput = aws.Int64(275) + + // set metadata, so that labels will get correct values + cluster.Name = clusterName + cluster.Namespace = namespace + filterLabels := cluster.labelsSet(false) + + testVolumes := []testVolume{ + { + size: 100, + }, + { + size: 100, + }, + { + size: 100, + }, + } + + initTestVolumesAndPods(cluster.KubeClient, namespace, clusterName, filterLabels, testVolumes) + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resizer := mocks.NewMockVolumeResizer(ctrl) + + resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-1")).Return("ebs-volume-1", nil) + resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-2")).Return("ebs-volume-2", nil) + resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-3")).Return("ebs-volume-3", nil) + + resizer.EXPECT().DescribeVolumes(gomock.Eq([]string{"ebs-volume-1", "ebs-volume-2", "ebs-volume-3"})).Return( + []volumes.VolumeProperties{ + {VolumeID: "ebs-volume-1", VolumeType: "gp3", Size: 100, Iops: 3000}, + {VolumeID: "ebs-volume-2", VolumeType: "gp3", Size: 105, Iops: 4000}, + {VolumeID: "ebs-volume-3", VolumeType: "gp3", Size: 151, Iops: 6000, Throughput: 275}}, nil) + + // expect only gp2 volume to be modified + resizer.EXPECT().ModifyVolume(gomock.Eq("ebs-volume-1"), gomock.Eq(aws.String("gp3")), gomock.Eq(aws.Int64(150)), gomock.Eq(aws.Int64(6000)), gomock.Eq(aws.Int64(275))).Return(nil) + resizer.EXPECT().ModifyVolume(gomock.Eq("ebs-volume-2"), gomock.Eq(aws.String("gp3")), gomock.Eq(aws.Int64(150)), gomock.Eq(aws.Int64(6000)), gomock.Eq(aws.Int64(275))).Return(nil) + // resizer.EXPECT().ModifyVolume(gomock.Eq("ebs-volume-3"), gomock.Eq(aws.String("gp3")), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + + cluster.VolumeResizer = resizer + cluster.syncVolumes() +} + +func TestManualGp2Gp3Support(t *testing.T) { + client, _ := newFakeK8sPVCclient() + clusterName := "acid-test-cluster" + namespace := "default" + + // new cluster with pvc storage resize mode and configured labels + var cluster = New( + Config{ + OpConfig: config.Config{ + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: "cluster-name", + }, + StorageResizeMode: "mixed", + EnableEBSGp3Migration: false, + EnableEBSGp3MigrationMaxSize: 1000, + }, + }, client, acidv1.Postgresql{}, logger, eventRecorder) + + cluster.Spec.Volume.Size = "150Gi" + cluster.Spec.Volume.Iops = aws.Int64(6000) + cluster.Spec.Volume.Throughput = aws.Int64(275) + + // set metadata, so that labels will get correct values + cluster.Name = clusterName + cluster.Namespace = namespace + filterLabels := cluster.labelsSet(false) + + testVolumes := []testVolume{ + { + size: 100, + }, + { + size: 100, + }, + } + + initTestVolumesAndPods(cluster.KubeClient, namespace, clusterName, filterLabels, testVolumes) + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resizer := mocks.NewMockVolumeResizer(ctrl) + + resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-1")).Return("ebs-volume-1", nil) + resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-2")).Return("ebs-volume-2", nil) + + resizer.EXPECT().DescribeVolumes(gomock.Eq([]string{"ebs-volume-1", "ebs-volume-2"})).Return( + []volumes.VolumeProperties{ + {VolumeID: "ebs-volume-1", VolumeType: "gp2", Size: 150, Iops: 3000}, + {VolumeID: "ebs-volume-2", VolumeType: "gp2", Size: 150, Iops: 4000}, + }, nil) + + // expect only gp2 volume to be modified + resizer.EXPECT().ModifyVolume(gomock.Eq("ebs-volume-1"), gomock.Eq(aws.String("gp3")), gomock.Nil(), gomock.Eq(aws.Int64(6000)), gomock.Eq(aws.Int64(275))).Return(nil) + resizer.EXPECT().ModifyVolume(gomock.Eq("ebs-volume-2"), gomock.Eq(aws.String("gp3")), gomock.Nil(), gomock.Eq(aws.Int64(6000)), gomock.Eq(aws.Int64(275))).Return(nil) + + cluster.VolumeResizer = resizer + cluster.syncVolumes() +} + +func TestDontTouchType(t *testing.T) { + client, _ := newFakeK8sPVCclient() + clusterName := "acid-test-cluster" + namespace := "default" + + // new cluster with pvc storage resize mode and configured labels + var cluster = New( + Config{ + OpConfig: config.Config{ + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: "cluster-name", + }, + StorageResizeMode: "mixed", + EnableEBSGp3Migration: false, + EnableEBSGp3MigrationMaxSize: 1000, + }, + }, client, acidv1.Postgresql{}, logger, eventRecorder) + + cluster.Spec.Volume.Size = "177Gi" + + // set metadata, so that labels will get correct values + cluster.Name = clusterName + cluster.Namespace = namespace + filterLabels := cluster.labelsSet(false) + + testVolumes := []testVolume{ + { + size: 150, + }, + { + size: 150, + }, + } + + initTestVolumesAndPods(cluster.KubeClient, namespace, clusterName, filterLabels, testVolumes) + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resizer := mocks.NewMockVolumeResizer(ctrl) + + resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-1")).Return("ebs-volume-1", nil) + resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-2")).Return("ebs-volume-2", nil) + + resizer.EXPECT().DescribeVolumes(gomock.Eq([]string{"ebs-volume-1", "ebs-volume-2"})).Return( + []volumes.VolumeProperties{ + {VolumeID: "ebs-volume-1", VolumeType: "gp2", Size: 150, Iops: 3000}, + {VolumeID: "ebs-volume-2", VolumeType: "gp2", Size: 150, Iops: 4000}, + }, nil) + + // expect only gp2 volume to be modified + resizer.EXPECT().ModifyVolume(gomock.Eq("ebs-volume-1"), gomock.Nil(), gomock.Eq(aws.Int64(177)), gomock.Nil(), gomock.Nil()).Return(nil) + resizer.EXPECT().ModifyVolume(gomock.Eq("ebs-volume-2"), gomock.Nil(), gomock.Eq(aws.Int64(177)), gomock.Nil(), gomock.Nil()).Return(nil) + + cluster.VolumeResizer = resizer + cluster.syncVolumes() +} diff --git a/pkg/util/volumes/ebs.go b/pkg/util/volumes/ebs.go index 17016fb09..8f998b4cb 100644 --- a/pkg/util/volumes/ebs.go +++ b/pkg/util/volumes/ebs.go @@ -141,18 +141,9 @@ func (r *EBSVolumeResizer) ResizeVolume(volumeID string, newSize int64) error { } // ModifyVolume Modify EBS volume -func (r *EBSVolumeResizer) ModifyVolume(volumeID string, newType string, newSize int64, iops int64, throughput int64) error { +func (r *EBSVolumeResizer) ModifyVolume(volumeID string, newType *string, newSize *int64, iops *int64, throughput *int64) error { /* first check if the volume is already of a requested size */ - volumeOutput, err := r.connection.DescribeVolumes(&ec2.DescribeVolumesInput{VolumeIds: []*string{&volumeID}}) - if err != nil { - return fmt.Errorf("could not get information about the volume: %v", err) - } - vol := volumeOutput.Volumes[0] - if *vol.VolumeId != volumeID { - return fmt.Errorf("describe volume %q returned information about a non-matching volume %q", volumeID, *vol.VolumeId) - } - - input := ec2.ModifyVolumeInput{Size: &newSize, VolumeId: &volumeID, VolumeType: &newType, Iops: &iops, Throughput: &throughput} + input := ec2.ModifyVolumeInput{Size: newSize, VolumeId: &volumeID, VolumeType: newType, Iops: iops, Throughput: throughput} output, err := r.connection.ModifyVolume(&input) if err != nil { return fmt.Errorf("could not modify persistent volume: %v", err) diff --git a/pkg/util/volumes/volumes.go b/pkg/util/volumes/volumes.go index 9b44c0d00..556729dc4 100644 --- a/pkg/util/volumes/volumes.go +++ b/pkg/util/volumes/volumes.go @@ -21,7 +21,7 @@ type VolumeResizer interface { GetProviderVolumeID(pv *v1.PersistentVolume) (string, error) ExtractVolumeID(volumeID string) (string, error) ResizeVolume(providerVolumeID string, newSize int64) error - ModifyVolume(providerVolumeID string, newType string, newSize int64, iops int64, throughput int64) error + ModifyVolume(providerVolumeID string, newType *string, newSize *int64, iops *int64, throughput *int64) error DisconnectFromProvider() error DescribeVolumes(providerVolumesID []string) ([]VolumeProperties, error) } From 5ecb7b42e0f84c0fcbf48ae1b6d5c32af49061ec Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Mon, 25 Jan 2021 17:00:14 +0100 Subject: [PATCH 166/168] persist modified go.sum (#1329) Co-authored-by: Sergey Dudoladov --- go.sum | 4 ---- 1 file changed, 4 deletions(-) diff --git a/go.sum b/go.sum index fa8d2b135..64434e2e0 100644 --- a/go.sum +++ b/go.sum @@ -35,8 +35,6 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/Storytel/gomock-matchers v1.2.0 h1:VPsbL6c/9/eCa4rH13LOEXPsIsnA1z+INamGIx1lWQo= -github.com/Storytel/gomock-matchers v1.2.0/go.mod h1:7HEuwyU/eq/W3mrSqPSYETGXiTyU2um0Rrb+dh5KmKM= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -184,7 +182,6 @@ github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 h1:5ZkaAPbicIKTF github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.1-0.20190311213431-837231f7bb37/go.mod h1:L3bP22mxdfCUHSUVMs+SPJMx55FrxQew7MSXT11Q86g= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= @@ -528,7 +525,6 @@ golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190221204921-83362c3779f5/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= From ac2a00c45eadf1d99775b2ed77ebdf36f104116a Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 25 Jan 2021 18:23:29 +0100 Subject: [PATCH 167/168] set allowPrivilegeEscalation for deployment templates (#1328) * set allowPrivilegeEscalation for deployment templates * securityContext of container, not pod * aligning * default service account for pooler --- charts/postgres-operator/templates/deployment.yaml | 2 ++ charts/postgres-operator/values-crd.yaml | 14 ++++++++++---- charts/postgres-operator/values.yaml | 14 ++++++++++---- manifests/complete-postgres-manifest.yaml | 2 +- manifests/postgres-operator.yaml | 1 + pkg/cluster/connection_pooler.go | 4 +++- 6 files changed, 27 insertions(+), 10 deletions(-) diff --git a/charts/postgres-operator/templates/deployment.yaml b/charts/postgres-operator/templates/deployment.yaml index 9841bf1bc..89500ae94 100644 --- a/charts/postgres-operator/templates/deployment.yaml +++ b/charts/postgres-operator/templates/deployment.yaml @@ -54,6 +54,8 @@ spec: {{- end }} resources: {{ toYaml .Values.resources | indent 10 }} + securityContext: +{{ toYaml .Values.securityContext | indent 10 }} {{- if .Values.imagePullSecrets }} imagePullSecrets: {{ toYaml .Values.imagePullSecrets | indent 8 }} diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 3593dd276..f3115dc8e 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -359,18 +359,24 @@ resources: cpu: 100m memory: 250Mi +securityContext: + runAsUser: 1000 + runAsNonRoot: true + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + # Affinity for pod assignment # Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity affinity: {} -# Tolerations for pod assignment -# Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ -tolerations: [] - # Node labels for pod assignment # Ref: https://kubernetes.io/docs/user-guide/node-selection/ nodeSelector: {} +# Tolerations for pod assignment +# Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +tolerations: [] + controllerID: # Specifies whether a controller ID should be defined for the operator # Note, all postgres manifest must then contain the following annotation to be found by this operator diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 15f13df7e..e8a330d4b 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -354,18 +354,24 @@ resources: cpu: 100m memory: 250Mi +securityContext: + runAsUser: 1000 + runAsNonRoot: true + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + # Affinity for pod assignment # Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity affinity: {} -# Tolerations for pod assignment -# Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ -tolerations: [] - # Node labels for pod assignment # Ref: https://kubernetes.io/docs/user-guide/node-selection/ nodeSelector: {} +# Tolerations for pod assignment +# Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +tolerations: [] + controllerID: # Specifies whether a controller ID should be defined for the operator # Note, all postgres manifest must then contain the following annotation to be found by this operator diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index f721f0ccb..9f2d19639 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -45,7 +45,7 @@ spec: size: 1Gi # storageClass: my-sc # iops: 1000 # for EBS gp3 - # throughput: 250 # in MB/s for EBS gp3 +# throughput: 250 # in MB/s for EBS gp3 additionalVolumes: - name: empty mountPath: /opt/empty diff --git a/manifests/postgres-operator.yaml b/manifests/postgres-operator.yaml index da4ca7fc6..a03959805 100644 --- a/manifests/postgres-operator.yaml +++ b/manifests/postgres-operator.yaml @@ -32,6 +32,7 @@ spec: runAsUser: 1000 runAsNonRoot: true readOnlyRootFilesystem: true + allowPrivilegeEscalation: false env: # provided additional ENV vars can overwrite individual config map entries - name: CONFIG_MAP_NAME diff --git a/pkg/cluster/connection_pooler.go b/pkg/cluster/connection_pooler.go index 2e3f04876..db4f1f56d 100644 --- a/pkg/cluster/connection_pooler.go +++ b/pkg/cluster/connection_pooler.go @@ -280,6 +280,9 @@ func (c *Cluster) generateConnectionPoolerPodTemplate(role PostgresRole) ( }, }, }, + SecurityContext: &v1.SecurityContext{ + AllowPrivilegeEscalation: util.False(), + }, } podTemplate := &v1.PodTemplateSpec{ @@ -289,7 +292,6 @@ func (c *Cluster) generateConnectionPoolerPodTemplate(role PostgresRole) ( Annotations: c.annotationsSet(c.generatePodAnnotations(spec)), }, Spec: v1.PodSpec{ - ServiceAccountName: c.OpConfig.PodServiceAccountName, TerminationGracePeriodSeconds: &gracePeriod, Containers: []v1.Container{poolerContainer}, // TODO: add tolerations to scheduler pooler on the same node From 43168ca62201d81a2c5d57b5b107b5481c0af0f7 Mon Sep 17 00:00:00 2001 From: Jan Mussler Date: Mon, 25 Jan 2021 20:28:37 +0100 Subject: [PATCH 168/168] Also sync volumes on updates. (#1330) --- pkg/cluster/cluster.go | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 42515a7c0..16d399865 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -659,20 +659,8 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { } // Volume - if oldSpec.Spec.Size != newSpec.Spec.Size { - c.logVolumeChanges(oldSpec.Spec.Volume, newSpec.Spec.Volume) - c.logger.Debugf("syncing volumes using %q storage resize mode", c.OpConfig.StorageResizeMode) - if c.OpConfig.StorageResizeMode == "pvc" { - if err := c.syncVolumeClaims(); err != nil { - c.logger.Errorf("could not sync persistent volume claims: %v", err) - updateFailed = true - } - } else if c.OpConfig.StorageResizeMode == "ebs" { - if err := c.syncVolumes(); err != nil { - c.logger.Errorf("could not sync persistent volumes: %v", err) - updateFailed = true - } - } + if c.OpConfig.StorageResizeMode != "off" { + c.syncVolumes() } else { c.logger.Infof("Storage resize is disabled (storage_resize_mode is off). Skipping volume sync.") }

=V_#SNrHH=t|kuC zT;z)v@ra20w5)2>@)-Y=+R^?u*op|8{L5-l4*%{{kO zSWs~K{A}wSmV42@i*L1BWubvQjf=3P#goW>kr2ziUwMxA>9d!Z1~OSd|7% z57{bTyud$kl#~N$>yCQ;qg-KFocxWvZ zn@J@;xRFM6ppi3VP6EOdppvcqU=aqT2u;g@IT1cSLKt}tczIERon!;B==@+P4c_e7 zkdZ3i?YQ9n{@tW(NQF<&z{!aR(ESaJhJQcV^3N!mWtMrq3Fv0F-RbbQ1O^596cli4 zmYN40?=9(0y>RCL9Zx@8ghRY~fi8eY47soT;R}vJQwk85eRuD*FUV8>L~Vk$6QRSQ z*O_qRLw2?VXs4=<<0qn!ki_Nq67c2%N5ysg?VSJ`E7<~Km8`+cyCg|s9%RPdZvfBy zo+2jYcyJ4ptPyj=9k5>4wQv^u0~(?#i0hPPQn!ICAW$AdH9Hqzp=HpteCdtb75FvT z19mQd(f}gz)JY-l>Z}G(d)7f)_uccw{;RmjT`02!L7V}6O9F#`9~-Q2M4hX3jmbS7 zLpFs3@2jLpP7s7`J#e2oQ6m8Ch!qjU6}%WTm=mg92zkL<;VKC0GB&LiGRe!!Z-aXY z4J#`KcnRDAx3@xtPJ=4ov4a)G!4FdMEBic3i;5IA^p7>oV{I??+bFJKUw7YUM{ReU8e><=6I z`tZRqq!~mT{guRyC8?{xnq~*w9+nSh?R3=D-%Z`6*B?P*65fpC)I!Y)UIRr{HRX14 zExc2Z6-%{p{PXtpXkdUIH1HwR-ZK|;fVN=Hfa-$7Pgpl?taMo0|Lz9j5GObPgCdkW zA_M!lGN=OP(YP`+Ba%rlK;8tx&g~&F+RCE9wC_QRkVRPkL=G+hy^ZDnYzVa?V2Y&$?0fC1C_#M)V zWs|NU_P?_?1UDet4QCq83ix`jKdC)lO#I$Y_bN}La8`xPps^g>2N5k96B84x>)m&g zuDE*WMvLwJ6fI6friD;x7opTYHKmI~(jk|X7HtR)gMNTGuXJbPB!;p!=r}u<2Yb?W z1+`XysVE{*LQTDD)2DsZJquZ-8Mqz1ru{5|8;pNn*>$85EPA46_>gBC0`ch2`s}?< zR+vj7t6}AwHfH(X9g5E=9>Pg-@>bLjQo3@mPscc1=~UO0ya)(wu<`#cTAHlta6G+X;t+cI$X4s&*6fJTeEL8t`)S)jRAkjZEQT$|y9vH}I5W8LtY_7gDa}`P^ zfpaRzY$cY2m@IRo~d8+6rLz9Y% zP&{6&SY(t=cy#H*MN&Tdz267o-aK-9Vc{(*G z*+kY4g_)_Ld@&h@_HrI-u!7?)*~T~6pUlirB+-fM@9A7OlQ>BQ_(Y-BRD$gzvqHiP zUti?YC=HzUMLfLEPZ|UF6=J9|ZYZm$_=3BqgsxR{DK3 zdq%Nxm97GSymMAlyd%V@r* zYH2|Q!fwwOp^8WCkV#F(Q8Lbb(0|Zw^ED1;E<)D{dc5& zSI~$LgMeB~$*4$AS{TBdcMb57l|Jrn=itx`7=Qyw?Ec;8FQr!~Z)&H3<>_02^LgnR;5F|r(YdxOG(sU} zHjwZ_Lo}RRVDqwTUx0$;Y&-UCj6(NUc5*iolGZdO*v8Mk7kJHp#^Mm0K>I^W6M3)K z0aUKCd0dEK-*Q`A60NWt92MD`(hfA~goU-FP%rNeNrbfiR6~ZL00ORi-(i+g{q74Y z!Ajm>g`IJU5^=A>*;Lj8W>H0s#07`_-N3E;x`bD^x{BM%MB)Sf$XVB@I_W*Qo(gBXOasZ9&b33SjHB*7f5OCh>BZ=wJV2 zzrS}N30>vg=15hf$9HAb@&fJQNLR%+x?o}aWJFdeL^wpQ*-5cz6&f^W+B7BD9Pci` zAT?F}S;v|IWZsg|^+O5Qtl+#)5T;)ZJu5WJcbOmr$lji=b$@jL6R5=B^dW0R}$9AO;nQ6h+U>}Q$57pB>bNKUlm7ncQ&VSvBsOS0 z==+-V0#bz>XU%96@ih*PuNSTv1tX4>BG9hLL0Ng%?V8(bpiugA=u1j zU}Sv8n5g~fKatZGXcMM09BH4bW4)jtB?TW}wNE6NCL>QC5CA-NLrcM`tOG65)0R%9 z7N)<@_*JY9i~I|P`PUWMAg)MoFzQpeBE*VOP?h^o^C&|G^RIjgE%l1Sz-JFP1-srf zDK>d+&WxpW)t~+QC6%K)aEs2^knjeLx{|Uy1|=^L+-*(3osl<>$!tw%g7mRHYF3ct z%?}dx!((Hha(V@dq?1!HBJxFmy6MtkeP$jYfN-M8d5-Y)@T&x4p+Ko@bSF`_gBQaZ zI-E_mw$Qvqy7$tSjQ-&Qd_Gxa7Y%TS>Zw3Hm|> z{(hk@U?7Aw#>mde?ED>M^&qansu(6hARNYrgEMUuP;=J}qX!bbLBC+&#jNfG-odv4 zeDQpKo6Y@CUTyY83>(xYfX-i#WotB)|9BKrWfpob(@#z%xb=5Vw3W@P$XLbBoG@h7 z5>?COy7>x)p{mOdR`25mz!E&2tXBy8=Inv85E;g;qR&OEW~_oO04`*Cn$eL%OEz;Y zkE0Uy^}+GtDlCF#-|_;W=ezcMrIbOt(??0SymVT1VMjUGmF^@ckhzw*j~tLRM^i6W z+SwEe_5sJf2sO6xKPbx|sPEU?7nTA+wxzZuNuLrQw}U0?;3G(FK>rC{?5ef>6<`PH zaiGkv^W6W1>i%`c$!!v}ZxCdUmn8!`9kyn0ipyjc^HWxh1NgUI+#R(X@p9tQJscYlFQiTxzKkfKGhgiH3 zP|3WReC+FHsVKlL{BBe}P9ea{`z-a*HBNjQAAfloA)%|Do}LMBG{%CSzS(zp`Ky^& zeS|kLe@^Et=MJrese+SBph9w>jS4OEX@1k7CEShCJM!x+nJTg`L<++QX$xBLRZiqw zTwG|`+1CUD=Z6s^$+0pVaJ*LbQR3jZ%5Yy7C4=prYe`TYIWe)i?)#bY|0T4;xw~Hg z-Q57Y&Xqq6@y$N06f~Bbma!4Vem&1Dysanlt8<_sxO0s#J@oYUb8_C zStU+XjD8ZNk?LK56b!zv4zQ8_=a7Jn@}oyW92{JlqksN?-F-x*)*oXy`RjU6<)&dB zya6h)tcnl~VgdrYtcMAzDRM+(fNo81Zsw~!qjSyEtvwU43;TDjc^GvsmjO=aann1c z29VXddi5&X$H$zWKX+s{gua6p^=!$IPSxlPB5q#ufgqO(W&!IsM{yXtLI2#z`{&@B z!mej&c@U>$&#=kx^`1g69p90FVH1_;6`LuO5fK!yr+z5%M)y8Cnp>6G%`HDn##)!N z8=_HYQNJRdI3*B))n#aeuznvXZA_#UAEp5VmD}p7su`yvlKn4NKcP`MUjT2U(QbuY zs%9OPlcJ5u8ZGu7Qwa_RnNU(6F!jxQ@&j(-HaI%ZHTYr1xyl4(&ck9hNQ|Y|Og3dJ zo!kqf5ZcXtCSOoC(-<%Zy-O<~Sq=kpE*CJMgzDqIkD|9i$#RgTuKsXR?>J{}*X5t< z6h|&$j!b%qs%zEF{>FK6UFgZuJYBUEdM*d2ms3we8r|^7=-CVgLG(+T>1|%;%Br_o z?;RgfIEkDQUi)OL`W5(AU7NFl*A{0Cnf3K8{jXSPov%FU%6vETq~YrD=xD0pn7a9~ z@{i1=-_Lo~8)D#1?>5@s$!}T{Bk&_X0(APsgdDGNb5Ig|2vo;kQkmg{>HW7a*Y4iE zivUc3bp>3JJLcxOp68$&HVv!U%^#H0KbrK?aXc!L%sZN&YL8~lxeo<;5MZYttcQR- z78tfLU;y+nQUObfmI&$y$iiEu0pMqTq=oXvc^0E4)@ftZ590S(Ug6P`f6WAZatJ2<8or#vneRG^3ea>;C|Df-v_8o1j zkbpzwO$MqzGp`bu$K=Jtq^Ov`_j@{(bKjxIWV0T!Uz2qT;iXByf4^MA76`c$p%1`80- zEJO~54ZVyj-MKKdW5mJCS7#1p8Vg+WIeoK8|D3MvLyV;EHi@rq=|6B3c*nR9B;Aa; zP?)6z%aHBSJ8fUSNP^c?2C#B=qq~Vs`cMW2>H=k`>US49?mvDU1~ZKb5thWC46yX} z-R;=c+Pa6Ai)3+LXqS=raQBhSXA*tI6E)$RH&vhVV@8#I}iJyuBq-n$E=ivOM?G%DJAGk z>}mL~oULb$0LR9HhrYQtx8#DGY~0C}fp5=N?KSRvtx!a==7FG3E-@YRP^TWj&yHqR zqGMplP_&K2$yV_zb>1;J6&?rQUBJcy1Izh5ZEScqYtKD7G(KK9J3Di*)wv?^8?*j> zw|cfQ)+0oH8+_fLBr7$-4r;NL{k^Y9PnMq0^YEZkj+Q;odrmMaSu7pkXrZjCa^>z_ zyWug7oQWFq(3jY5g#lOQ?y2B^k>{cj6@4sq?U=*B$jJYlLh{Zsfl@7`vZ_zCW)iDz z0vjgM$slMT%8K)EBpSZwxr{6`t4LSZjYcjYlU)wkh+k6L^0vA9{$9 zBdkSUE~r}!EvS6}FhVKB9Fa|siDo6+F!vAZ2S1384&_y5lTnVA_?Ki}@|-NbBoj_~ z$I|i2R|TKlA{8PuZuOO9mBOxpPV7xfOHD@L{)OkRBbcVCMMalA&(7F*N`Wrr_5=oS zWcvF0xc=(uu_nZA3c=DlcN^spGlf5CCd`Gj(971wruVZ%Uq~Bn1FjuEN?CM=6nR;p?~}`R>g`o`eaojIu=!&? zQHq`8b)>^dvrrDebhd(_WMZkwP(G)~xF{|@eyHW+O1+7eypNwg-7Zty(0}&KXMgW? zpePyI1A+%no-U={eP4>c{r$F%ClbD5in8*?{>T&U^83hU?CeS1U;C)6`{?tltC1() zY$9|b#s>8@XVkUbk*4xB)78xJ@-%7as{IVw+B@&`Z=Obq!P+0vhwOZ5J#X%=B+>VV z#!nVf&00&hzaKPxNg2Fj&imv0dNA?Z`ZyjP7S{R*GtZXZ^j1~v>r$7}(hsMf{KQaw zs75PN&A3iUp}#eqD)jDM#x%4Kkw2t38yfI%NnvT?WbQz}%v9~437NFF!Po%$&Q1<~ zcFox-_`w&U>&ZVCHi>$MGQWEyk~CH02SD|}yLEO%?|HN%^RwEs#`E*?P{A#> zg!r7ypMJ9v>*1sRV5uW1B{kEZ^^J`IWQ+~x+muOnwl4;Xx@uqIF=m!wU`oDo-@D*I zrK7W3#^sTpdSXm$EN!*a_4RSei-4@yuk~NqdEZ6Dbbzku2kVumB1y?fJ#Q`8Wp#@^ z(5{%pBo@Sx_+zbyh31;ChX`fRJuj`Pj9WatkM5?tuYsYXQfU;@tEV(6n2gR>!T;{o_u-53K_2S z_Pt?v570l}dx?S-T6eu$;eWU6%t?u{yN>Y&Xd!2PPiDE;o4WBm8Ys=DfC*GZlsP_j>!3VD`^=L2Ha>UE{ISs@ThCB!kxLDrs^H) zd-tx1Vjm#dYJ%P}i z3fK=QJ8!GfRr$_OXOZBNG(Ca*9TRgm$WPJwI{t#x9OO5NT71?w>&wgGlgoq4yPH|7 zB+azB z_OS2q`e;l(fDOmB{k&XJ{E{slZOu6Y05LjOPo9bq7!QTCe&+k=p9x2EAb?a`2nQGU z+w0zwaJsa&2tbnn1yG#f+4kgSCC@XMHjNer646%U*8N}U=vJY)K_LmwdW8jLLk%p` zPXC$@u~CB_Fbuf%Iq%%fn_=$s+@DB9d2>^ z%7@yw)YNGKa%$3_KE<`o4d@opbq%TJ&*f~gGE<35if+87gU!@(JyH>Hii#_vKWfFE8opc`?YW`bGJJpk%yVqk7? zdvSYil3M-=tc^udz&E)M_l14eY8c8lMW zy#4iA!*OfJJp&B8rkgll?oc^A<;Pj@ZCjshY57SV4vwtwF*RoP4W^V7;wpZ*Aj%CD z+NRNuAJy2tI$+%RZ7=gmF8x}Dq%?X1tp!Vis`4?8Dr;aDhEvR|Pduxp3aXFxs^TlGlPB)2v87k_-m2Rif08~VS1AIJn^V4E=&GeCsquUo2 z&;d>P0i85b>bqBSMf0_i!9tiz5{0vV`^fqCKgEA$q$-;o82sK^oGP$C@6HG?_E7*7 z187Es5yE%G0OQaOgi∋|Y%bz{0o zVFI8zpes-+YRCp6D0X2%LBVeG#}l&{e|AjJ&8CiH(t4w3M}5a)AU<%hH>8#3PQ?o%;|U`&8awP~VILWZ&F}C1SFS|J5ARvbC(z+)gHdPe;V(u*Yx9uzWKRR|)2@)X zC3gKQoYi=zXDK;3k<=&Sv}xSVr)B2^OKZZVR8^p>zfdrE@Img-)`>V>2lEeq6L8yH zT3$Y~UyH`AG=-^OrLHxo~*Q-8kgs$m_^~x0&QB1r> z#&A`2^(z~hvgl8IvRryKLp!d!-!@>Few&d$>j#@j}cQ54FGY=;^a&0HFn~ zJ)}wXIAj#`AFDa-EPOP$BtI*~@=ooJf82ST{bT#H^8ToRduzwM0*_jKZS4|pn+R@c z@e^S;419-sfyUV#5hIQRMTg<6{ENTcE0F1R8xm zU8dmGKrx!ehHQQa z@5&jEfuU{lr!z3%%u^^nIyyo#XMqRl)|-djzRE<+B2T|Z<#)Drh0k`B@KbOqV2JxgzIuloIMXn;@5=hO(masmZMNNl!Hz58y) z86NlEbnq&jA_X=5y?-uUTGpUtZXN15X~NO>49E~4TvGHd;?MBaO;?@yYlD`ZdZt74 zkT!|?vjZmM=fZ zFGVc!9VoJ;8i)A!_%`uxK%xv>&X&Rx!+EPgBZ$$L*6a}O>KD#&*u4Vud)nXzfd2me z!nt5pMEjVS7br9rbtv%6=k(y=;{zbo({sR}CHFW#KVLT)GoTg}Y*hbYMmGf(Ws$Ky zFjfT=yUO-u&~=>BQ;r9kuR6!(m!PA6qM!uVIT{hiSO!`J!N6TISKN;k*57jhTCi{z zUMu1H)>e@sXe>KG2At7Yko^sU^|eEh|B-o$h|`n;l-Hc+R*`44 z8NmO&HQRg@Np^HM>v~=3TcOQ)x;cqi)Sk5UJSU&&jaQ|5poeE|f#urPPA*>5m@xB( zZg{w*?+g8}U-N%hGk^02AWqEf&T{r0Z`wJ%R26mo;xnvHU)4^Qja7BEU$n0c8GUv0 zj16Gun7yp@s>zbyEE$2^gu;zi!8whIg#BXfYx9Y(Qm|9IN^QmbK2U*=5=XG&bhHl; zC0+?jE`AUeA?i%_&rnLn8{|tIRmB;$UJ2}GuHod^Y1>nGzH#ROBgaRcRu>#f(V~ED#Ls~Go zyL>->1#(6T9#MCIoydxK6%$JEIVL7!7*kY!0XfW{PQYscz&UYCti?KcJyut?x4kzM z=6|(o4i4=Y(2)l&1L(x7-dLW`<>OnzcNftR8VWo_%1c=|BnJSRQy;lvwG3Sxmfamh z`>pe0QBHsU^lAMpUSl^b_1*E;tBM$CXA4xzx$#vRN;GtkwV5yOZa7syBUnr;yVq%hq9eTBY9ykJRh)WmD+8mE^WaIYJ!x-lXOXxo0AWZy3oRv}y5j?BYHCuKpPZ(WkxVW08|42Hx$)~)?v(1@Let$RA~NcLOLFp- zoRVkyUH-c3daqLzukC#PgVP>jG5>T7?e{gU zkol`u+gPYl7y9;Lp=ibxB7Rj7pL{Jq`6}w`OQHS#eS6(EZ0|B^@mfhEO>3dp}onF0HJ5^+6GFaA96x;|3jbVns0}urBf2X&`ohf-xwQ zx95o7EzI+O8(eGa=oxv&gmZ;XDhjHAAjB(^1)YkM7ooO+@n}1LYMg*>1%ODT(q9BF z+0tEjC^JuKd`DHyj=h{tx3zGyOp#)PLS@G~c(aa#$#}Xr{l61z`Q+#ql20^VxAmGJHG#xo7z#W}bZ z92g$ghba$5Mo#{P1<1WDqRqM71USG6liX-EfKli2Tc((Frre#7h(gvl@|(G!X^6|I zS1c6BB^>VP5GTV6*aV)Mli_T9w3L)$T_TtxK_$nH0swxx>xNa9Pb)$G+4?!3ISrpLc7-{bL~Feuz=c2n#B?)yt*(9T&9bU5i*9b&VMzgrP4@>^d=+&jMnJ77dF@I0 zaRCm{Z1HU>ZeX(lM0tsk&s#4g^b$)#lbf5HwFl_~H^JoSxQ@e5>uP)378klk{98It z)s?mdySziqcZ!6B7q`bSz~_hP z>j_XW(joQ1#wvM|ku4|-yV74uDC(1Xx^5NXeV|)W%CyNlwcY|~yl6K&ZR?2?WGmwU zhC+T<%YTR%pFxEx!fzmg9Y|+t)I)+KxIfQZAFZB<{Se3T9C7)DQm5uhe?}UlUYCT=J zI7{|Sm)VsLi~0U%GGN&V1~ps@RTGJ_xbKn8RXyw0&3*bYl*Snw zRnCZ6EcqB8OGRxmyE{-df5cRuQvYQrTJAJoxCpo3t#{Qtj*%@epp1Vu(+N@YZjqeJ z|E62mC)>&L?duf~lRQCd2uFmXpWBkM z8zXiq>37Os_=2?GT7aQF1KBqIeW({@LTC(KAdu&|jr7;Gp#0jum}T((ShE20%7!$y zMYz09q?rBU9#sq3;gvgAW-ge9U7mrUx;bW5JOMe?R(AwBNP$Bu^2m9p^G9Y)W-@7P{*+xT+aUq~lQI-YuXqV+i>Ro*N%`~nh zFTSU0XjQw&oRv&RpB0b>vV0_HkT5sQm~+c756>$+419H?g3G7x_kTgIQ(vo+N&3Wj zD|}zwi|}@$^}i0SrMt2-(k0i|7L#|XdrkfM)n>RnTGcdX;i$|w;_I^HpOo7lVryNYNw_R2uawVkfK^ zXBWLl?9i%m+F~K*|7cP6zHq$zB;qoJ7Tym(Fs-QTxW4foal5SSVo0;Ls)}mBDPP~@ zSJ6kIqqA2u!7*#<;#60kN^tB-_zu)x*GQQq4h~eYw~eJFjE0WC#p%C#trjt{#hqET z^W5CjEP<`o_lTRf4W&;uyU<{g-(T=MxqP^;pVGzR#wL-sv4h0AGdnFvNCKvC#gU6ecM1Xgo#)Ki>K2*vXBF z>e-b2bl}hsOD80>vRQKqvN9bWJ(<;*^(X>5%%O7lRyGmEMz_NV$8_?O%W6BXY(2;y zl@j_P$^9Ep8*A%sE)Aw`=uU$IqN$*nvdBGJh+J9og38xMsP8 z0-}TnqJ*I(R?IZ_^S!ObfpC=o-O4Ym;%)EPgLP$c*VmrR(@Utt7-#p2_h1pVgpLYz zvD&%N9H2f9&af%2>z2Uh_EpI^?@p3#ZBN}xh zzYs;au6TQU%a^X!GOY7%hWJb}JBJ41U@=6d&gbXnKOrTxM{faV=-poTz^c|&5NbIN z>ad&7K|eK#*_>6kl=MCP^g0>&yq~bTPF@YjAVu});92tGHFP|*OJKLQ15vPkt2fY~ z+(U>X8Uo=qiiL;*kq8r$Y?2<(58~9%mcv?~dVI1>RdM z@y6gyk-ZjNvA9nrf6gz^Fv=_!aAUjd3qV}Zz-+JMBf`b;>y#C*J?~XZ4?(>`@AO0G z4|`~*zn+|*>wgjd)N^br7*bYt-DoVZxIUj|vg@H|PmqzIr=nuC>hwc9qqHXqg&2Rn z^14vy%4hC~e@^RVTtI*Ysrj%gDU6V7wu1H6J~LG6p%9qB?x#fpc*;1L-$J8Tng$r_ z1AMGEVcym`4?jg+!0NW!3-%_T1MQX=o9{apXFrCfPTo??A+7Mi1RUywkXH15B*0ez zc-=c?8X6qFzTH{dj{Hnk^kp86%N30_=(YNUW+Ua2sSS!tAub7Pg=RM&Cx^oJx5E7H zbrqD*u<**avfSN#yTWg$iFLu)H{m(OgfZn)cbWT=!;_J z=i9Jx+B{PID3w)ROseCy9S~80t*fhtDM0zN^JVyl$if^$X>?2@ir#+Vhe}Eq5z)to zf=Aafz|JoQQ(PZ;VLb0;NS$Nt1%rUDG!36vzZdyXgE*d`) zjpiF$Z|m|d^)dFhOOD0!6-Z>Y7+0seqK=L+MMYo3{E^Mi!gu>Ejx?OZl=^L_m0`wFwTyV&%E{EtN0+n#|TZ8rQzrJAC zfAb1=hY^i?QIz~z5R|qb#yk~xuQ*!ar_{G|(K0kQ>^a!CGn2e0p(poyzlk7vri1=% zzthBPjTz_rQx*#b2lpRAaFcu0we+Iazsd}xbD>H0$N@A;$XGeHqpK6wazRBwe`n$M z@79%-)yB(niBOkS+GHOj=Mk4fobc#)&711Jja3Et*+vtLXU_@-a=9QY%G%|8wF?Gy zKRa3YE9eQ!*q@cx*5XiMuNHP|`?sVQp|a;YO;$2MpJexTt;>cRx%A-Tk@S31V_x6f z?B4Z3+O~+)H)UijX-zuOk@qL$^vsz^%eQ$kiI69xIBL~OFeXH z8Fs)9bUVfE^xmBi5n`E65hinV6MMK^E2Q)Hbv~DpKlGfRS0qU0_-Q@G4k^#ALc8zl zd;a!@7W<^vb-a3>_9gg7;}sgB)VE|M^Oh?x;4wwrovwAXcha~K0fX(fpz<*xr zhC@gk>4StZTicto`nC9*Mz!!HA!7@e8`mKAzYWY*|11wlVtv^%=g67dMNl8dLVW zK!khv?sdI`mrm(Zp5b;ld9_Q5qOlSE#H_=4s8MHOCnIl}A4NgdU$Q(qN;_zFOfhW$ z)u;0S8^=entZeqN52%AeHlqimoctes0IPR6i`%Pv%gHHC#knxJhu=#mS-jWOj=Q^| z%m)wuHf5Ku{^y$QRk=p>?M*+j@&IgWkm99UZ)T{M+cjlL2cbWX>U3>@KP3SVzVh=G z7@lwm{h```IkQ}k1%7ni>StrDMKu_v6<=Mi{G;+T)+g1v_v(J$*T{g@&%XU$qkkqZH1j?>sn)ze$~6cg&`Oq+?#|u86Y= zADx#fl|^7g!8kDZeTHeQPq%M9ASNC6{HB!Y%~osc*!|BU(ibfj8E@|@Orl7fY9a@6 zEY}T(h8OpI_PkD&w;T}+WaRhao6N5c7k{0)R4r=ZR9;-JS==W%fzZ~#@`rsc(2!~@ z44The>_Bb_f7KEoE_iM+7hn!skocZKFOjy2?dHlmI`gJ6=k~WDIUmCCCNz~@=7Q3k ztomqM`2++gX!*y8OT9pM-g=d1*oW!tgNGJ4jM@@bjLyra`&F&UW*NsBME(e2w- zYCCw2r2WFWc3dk2l6}+i9GM9xr{U2|{73aAde*`wv-Qrb=)cw|nV$dXmbxEl*8wHz zW#lyvG6z&I9S(E_A5EQtw3bcD__+8fG;sk%D*=eB%opm1Q9(th2;;(Y$ zz4()2T!#2}RyQy68y3G@4r*Ag+n3A33?}4frC%7q?ebgA!ks78wap!o$a{dVNfJeg zi(kqxR;uzO>nc2A@H%TrGvkZDI1J}5=SinuHe)1eW1*E44~k?GP^(@W_jE9L2;2cv z(OkZ$yuHloK5#NhGbYUbQJXRN^<}-ux_O3UZ?8mkMbCiA`p`<9?x!!I)5(Kn$bvwR zLkAJs&WiuRD_qs3cVwf-Lv<;OxApR+E-%#hDSp?*uVh7G(w;h1>ZPVEwU8AWcW)<( zv>~wthj2&qsi*jJ3^J?LV;RBsiW&AxjOmDooJw{)B1 z0+sJWj`D;EjQ`qMWgU^EU=olljL*lz;Qcvaq2w^BdBpkG>9x|;P4^ztwO-HEPKn=* z4O>#8g8gt!REMLpm;U;dEbLFh6WS_5)w;K-n+~#*bhQP~E0TzioTG3AbV+&)3?&@Z zj6=s7Pg5lHUs;7&SeVO3GDMt&2W1xD<#6IZEw8#JKOw}jMVRb8*SxAYmU!@DrgCsj zUxP`@&=`}5#M;mENGRe>BlNI~R=o`J)Fw)WEjxq9?l94FJo2L{lQHAG(QPxq=LADY z^+w`T=$F-Dt)s$%3e@>^?s(z{&!hx75Mud&={j(*rC{bm+2V2N0W{tm$WtqHjz?+$ zDEM~k=5y#HjCX*3BWYf)PV{u7$(_38Ntjbv0+&xXi5ETHaEg!OkYYC@rwJv{`YSUT z_Kk?jc_NEMmd)jnI+1Q$%?=cW6E$ls7Aag@T`5ggc&{$#l54cCsk-DQeY5)^Q#vhX zNB|v%iHjNTWW|6dTxY!_m11ordblou&ErwR`~C)G&0c%=YwH?JM&}Z{(`|U?z78Ja z_V8oygaSpo-)SW?JguC(xo((IPFO~u4kqSi5j#R9#t*Gi?(3%KWC3hbS`tLCKZXs%gqAjv4f*cL8pKuA9j`s z3+fsCsHi}SV3wiaAz?_gUkebLmcMp+&hM}u=mF-Z1HnD4bA?r(Vqy%BG7=`#kk_1C zR#hA`B7$uh(#DLWjAEl$?y;l@Aii4gW2qg#Sg}MGjiOweu_y7pza~a&KqP!MGkn{v zFycJqLL4KU{c@u?)4zZgp-Iwzh#_18r}mt$`|OcHiA!bFQ)k+hpc2<+KVk$Rx_)he zt#|I5AtKGgfDiMke}3V+bQObecF{fRJL8p>{D8o#XZSO#HH@ff)_I_jMc50d(l|AO zVcF@02tLyTrf`3kP?aCky$G+E?-iWP)Z(scfhoQHr{0Y-B*Y1s5~a6?5c-A z!J@py+4)|NsCz}n{M*w;58Yu+6KP@&zq{keHo*p^TgZA;!oq^5UHX>m+Nq#;cuNF5 z|F{`T4eXBJxzC)pPH6bVCwuj8eP+n^WXbt?O9$xoP;#@N4+~y7$b?3B( zUpjhbXDQih_7>h}KadQKj%L4ach^RY%_dFLota-hnMG}5y)_UpHM#)#0#yS@>gUx0 zHFn)+WFFJpMu#5nX&%&0(w5n=H41P^!BYcXVX(i3qNg$hlTg)gh7(Xy^{!RR96$0hKX(b=i!9-!x9c{YwKMG+n` zHs>QT^$>r5t)shuL>{SaZcWn2^Tno&~2Bg zv4B1)l~e=)s#n82kkbe#)bTtmqGHE8+uaK?bKtvbuj#9J;G7V3O<$fqU;DwIh@Pcx z4eKhJi)JI?rg88Aj3fFeEJ?^qa62BOv(eo}#rX({OpxK&3EQ9Q<=M8^uUbX0^y|qP zzHqz%ryn4_oe$`AE~?=Zj(cU6z!vzG?1wq2_W**w@q8uX)JqF|Iiy>?ymq=>boQy3 z>Ua81j_^;wJ)Tr2PL_1(^XuDFFGj~-W^%Tck>1y)N2f|uXmyE+n6WN#QSy<}YU8rl z3z}DvD>G%QxljsJR20$I3TT9cO4_Mh;f`vwQPTb(wlk?BuC`oUzBRht-mtV0uyY+J zOiEhXUU{tKju9!R(7vqOeIl?7Z@4LNeo-tp(eSREv70U@>FNK@&gBz`;%kf3pzmi9%r{BEuwjY&c|ucGWwZ-- zb*!75mu)1WD)oik!H3A<$H!+qn`p@+lRwQdKjX}=_x7g6*}dNkd3CTf8~19MsMqF&u;Cl_7Mm-B#q z!XE^Gf{y2Vkg+W+^RAb)BTMnmvPjqx9mnpm-X%ZEr(-(UM- zIpXJq)Lxp9x%52eqfi_yuZq-u!67>9YS{F-qoQbxF`cl6N4i%vz_b?FEIF#D3ptR^ za!w|oVnixJZ33gFa^6Kj44U-`_WGnK4#iA7y<&GXn&RftBu^JsEQ03J?qK8bkQ+7+e7yLh#W7BrCKl>0RHafdkf>;5u zZ-hQ7c-QUL{ki(P#1|S=3n~ikZbY9!h0nF7 zK>9(H3q(>n#Khs0!jz0Zu(kDGXOD!*!1@PXbr)hV7b^|zbz+cJx3KpHO@K3y<$th* zG?%HHFarvP^x-3>+;@x1uz}v^A7Y3i7xc>>DbOXeQ*t0Y#qRle)&{Z|G2LzCACnx_ zSkQBm)vAVuhRCPE?d6A$u>WC*tE=T*U0Tkzd@=D@gGZzGmg_0K$FWR7U$xDmXizYs z(ZIUIz6CS0-Fb(N!_0+2%VoKef!Ko{h)GYHk9q~fcqPb*A?{(91<8UEt&EczdAd`r z)2Cr7K%M&M7tL2E&@=9_{Z@t_MvyDpgK6`9^l1lcMEb9sr$OUCVvz3Le?h?UXoMog zQN~52>|k-amMEt9S^exe-75LhFuCPO)cwd8IWjOuo5SU3VH08+Jw00&DfYDJbRV3( zAvzEj+4zEl!iVO{&duy6j=~m%yapm8b96pqWJb!E4t^iDE`&+Ab6y+}ro+s)Ja+2_ zR=u;#f$mUIG0q-Isx9%m4wQZ`g=L5^%!1iwW3+7D(qcGSSXf{(0b{x4yo+@oz>IWm zn$x06P^7F?QyNDS7H++5dUsH;r{3Pt7oJ5H$ITv9wRogEvcI<hz;f794nff9uk^?5}-ah?)etST~HAB@%b1frp*@7@~>s1_KD*GS*Wtf z50kbsV~LRWiP`qwb5OoISc>T1vSas%y?YnqRXLL4PrDCYG@NA>XJXnH7uR}^A`tT>IaJMpiG`@I z5j$wsGLgojb4Xc$2@N?vM|%)QxTb*HPS3{axC~jkxl>GGpA^f16BV257~)p)$f9296FYtmFay$Klf=MLHR|&VO(fKl<*Vy*HVgBUffCT z?Bpmh5YVt6*%o%Vo$niAkDOn0NuD3swcPADFe}FIa)5S%`4zys*f!t;aB$l~Bhw2T z0g8FVe_;UzB9*A=c5Tf_q^0Op{g=e~$)L3!h5(I{KKaHQxUSg3w(BaL6MHVC20x-~ z3A{mimMz9m?xQ7;?$uRmYM*2%#3!|OsAP{vPBpnNICZMU-URcscOavyu}GBUb5?nW zt~mL`^?99;@WrPtM>nt2Q!m9S?7s^vLegJl6X;oTm+b(!OI4I4hR|`$gfH~W?8P0E z_8SK1kU?ky<|veK)_tjT0?|Z)y_-1g#k{>S_EsogQba|>1XuQW6iZQduq5Dy1=mF5 z$FU@s5Fu#w@0cDjNJs=_mG0M0yhTMr>CU8*F57FR;1%FAfmZgZ!**g7W{s)-dd>u@F%~a$bnC=<77&msQss6|>T#=G8+-T>kp5eJthJ=N0C!0ivBj35) zmNYfBced*kw>+}Tn{`?m!2~{l9pfD&sd{>rd+Tf}AH_&uJr)d){9Vda{AazEo?xvQ z$HaXM8k}`YoR$u7TG+ONmS*pRW-cnw;i#>vCFkNpF{A0Ku8Soi)(Q+#lzgF7HZe6t z#m+vO)?X|7G5N*BpHnvk4@5ojEnoe8+jYcq;Gc&vc~rL?Ld0^;&#waS9kb8|xNc2q zk+h-W&ayx?ugX~;Jy8M~B2Kry-7zdG)+kXVD6e($RKcAgp`uuHm1y*ssiYKz8F&|8OS04T>BIYp)dUb+KxHe9oCeJ{}I7#sk zOK%bfN}K3?ONYHyq=Ly62{Rh3Rbw_MT|!E#_JPFHW$7(tWfx*p=s;`j56hJ7f3%1YG1g%bfu9##GT1f{SbVLh_ z_JzHXPv=L$?*5`3Reh4|o{?Sgj}w(tYuCh#bwihgw9#4fJ9-92X+Y3=+0R(SO%zwCN1(zjJqh6}>|f%{ORl7M8wx`PAY+1CaFg~i#A(F9~HBBBVOaWMl`iF>2+XSWe( zgH|2Va2RxInSEVqjoT%>QQJnZ)fNNE0*Qm41>`Q3Th zPM5X?J?%zQjdXf$Uz?Sk{vl}2w0G}V{%py;y-%%z&0<$~&TmghMqWn5Re|JDc=+g? z_xBWCwXq(Z9ZvrC(`|v!kT0}|NDo(D*)Zzr>0**RHBTO07s<}(qL-Nri6FrYt|F?| zc3BJkiflVmHrx4}l#|OOux2I4qvv0-8lyq6h!cyD!NxDcR(=X56Y+t z-AINDzUK3(Me}jZbk+?)SR!T%9c(4^^xjR_u3oDbYdPErJHOE0 zp~GoI_D)NVq=0#9VIr&WLha~c`apFJikm>+v-hft#!Q_E0%;G-uG6f!qyj2rgoSqs zHH30@Y|XdZXvUG_7C!S+`d))65z-Lir{}DSQOdt3Jxh(qGPVM&y5eVsP#3_<(~07M zTK(%;I<9o0NJvX3bJkcC&KVLkbR%Cib=12RKFyyLKm=WZ4i)=hRzD#$w;EXdu(I@@ zyePDFiM%yRJklp`+6@?ryd!!vY_FT=aq%l&)Cdw)^=($5!Kv}<$R>9W9kt-FsG$@3i@fQ7T%#wE6A!6O zrj1Q9ircK)Zj}jxxyqc0i4cFc^caIQ{`~YOY8O{`DmorA1v;gbrDZ|q8fJYqqOF?k z)(?#cM9$WS^bNGL3)|xA21Q?MaW+=fK74%sXy$!%!Z^fr5|nPn-*cQoA7j9MTbZOd zZC=)gnY$d03d!&gONvc8m5BUk-T`Z09<1_ z+TI~4*fsz+94Bz6{Gh1OXE8m8Gccf5^l)qYzxUK$Rv6hocVE^8!q5gA{QO$K125iv zdz}5Vvl?FX$x2lsUBq+p@7g$BU0t+E`97`)Z@g}q#p{GxE^S_F{24BQkLx-JcP9pS zW(beFxF-#uKL_x+q!e$5@2)Z6Dr;;E#<$lG*0(pM+}-0Rm|uK!CWzO46UMhpe=j&l zI%!(spl`GsgS5=k?!bv%hA`1Ts21~7>3XJE2&r)Q=Uigqk`vd-3G(q-4RTNA+9>TC zyVpD8>xA>o-!?Cp<#%y)=ET?4(MdkgtF-)pVhcp{4Bjrjy~)TRFA%@paqye9 zmixU|z!d#&yYq)F{G2qL#$4(^iwJd?ug3*K!yWdw?>yRpC}z$?=38i@#Xe9nM2CKR zoKV)k!)IvU^#D!6ei*g=OYEOBR>DvjJHsT3atQ9NJZs;82K0XgU>CM?wU~R&HN-kK z_HC2QI&4Pm67s41U7K~2KYk=+z@>qJmJ%fw*M#*FM@ag~ux14_Tk9gRgKaL|_cGTW zefdFx=PhZOcNfFaYb_}6(b<_KXrKN3skr(*7gc^B z>!>jCLqt!%<7SELVq;TNsaX#xcI`*0?&Ni;Jd$i$_wBywP#FU<2it|_SzXhKpzM5`ipk@0&T1nY|BP-ZdZM<4#kGC)KlT6Ahq^8?B0q_j`=pO`Ci)H0rh|?Ei7> zO+-RYzu)kJEe@$b$oKqbuhQrx)lA-X3E<)4W|o(aeN3;di)98Njp@Mz_J6QxujMSHcvL}{AqlE5=Psan@deWW0li( zZaU5#k&?5|#X<dwTnw3yLU5_}TIx*0x`xSkz8W2wqx8fM)R2qSk$Ue0+HsIZ;cQ zQodpS2L2*a!Uyd9%bM|a6G>6fLOD=W6n40xlUS7sK z=#`4sQ&Wq**&uu3|FxGM7_(9c36TK)Oc}jb95rb5@`#pO^s_QmpZ6_Td)wy%fWVHn zAOQ6n5`RtlQ=^Sgl9$<8&MOT{{XUi9AtV7iX4ZMX`QF+oapRdH)xSDDs5Grn?{FY^ ztoo5Xq+Z)u6Itf3jDdPx-Qwxqa5vDQ{s7G3fkxPK9-d^>p@|7JU*BeRn}MDw463Yb zOgubDXXo1#YoVz6%ZKs=@#aHZ?(Xi`dyvG^$)Lz%r;JvY#+=8XDF}vM-%}iL+LIL1 zBoud5#PP_m?TpDx@Q4Tr9nWQoB@HgGE;Dj+aH683EUoy=K0sVD%F4*F^O@4 zf`Z-xntT!?a9joLYgJ{qIH!XA>hMFc?DeyX^qa^1@vJ=WlmCCN-mwLK^>b%R+5fuw zBN|6gN@f9v*_O+R633g9spS31#iGnIl)PDO=FON7pIoAU!SFM(vgSH@TTymNccBna zcvt>3bFp*%2m0yA(Y>bLs&IAE?eaO`s|@OO?|s7=ICrier(ajGT;+pkN-MFlB0hMS zgw0GziDmVT0r@eysJAylvs;%LZO#Ed5;{yfs;2XN`nKD3=mwBF@Zl^;zE`kNO0+vksI5a6oe_6HE4O2i6yQ*NUvcKF{*3mXy0d@IpM zG20wn-NekyXb3d=kb3~YhBnBsv6%;>rTUWc(8=zmX z7q4%TLAJE95eAo|1N5#hH+i(blYh2Kkd_i@pQ%+~{;BZ)$43HcM7SwZSwklkYY$fv2i%I|qIdL%U#;d5Z%rj6xyIiCLTSd)wQ&jlXAdm`6~8;^f~ z`9FR+c#7xW5!3u`J%E8-(zF{$$*6dE&KM2U@CgzSHCG{LUO@2x>Ztag716hu|M{%{ zhD$7jVWBf(#S~s6MCp-8O9sokv!2#BB z4z=0*e?5R3=_UM@^9KN;>Ii#=Y4mzC=HZ{^gTCnU+l=? z|Jgvb=U{2-c|$jUm+F7B4G?Vp?X?evk;GhFT(s4|;uiVh;v$Pf0tFEXY4w9NsVG>I zz8C!l=%ZTJ+VFSA=8Xa#Q}dZV=id$bKknqZG%!>?dQ1qy1~3sa0fJn>5P7|4^y^kL z^%FwEaJU8ZKHb>ZIA1txp88+i z%K1HC{Q3F$w?N#LrW)-aUR+m~7;Y*Y8Toj>`G4lOhRRr=N7g=;E5{B@as{upD}YWf z(Wj%449S06M}Kd_k~i}88l7+eF!tE3T}2-N>03`UnQ;2J>oz%MOhN+kBYSXAw)x${ z{Lr&L*!)e{U}7T1vvta2?BMzPDE?O30bR*>nL z)B@@>pl8HSPp0#%P#y~bfpX_6J+~P%x0e6b$Fk<8Wc!LzSzWyXc&S|g@(4Gc5EdU6=< zQqI$SfF_^JjRt%W@rxW>U7S!NiFk%4IepJN$lxNQ_%+Taf5QI=4BWO8K4Vu9V@8M& zeXnL0R|1eoMcmza;6ZuLebJ}MNlQ=s|FZEeqwWRc3kwTl&z6^$gI_Y6x_$+rQ%tdB z>CQ{@d<4e3>sRO#@BXA+k~H8L$XZdi0M}&@ECS*sspmQcaHE2&PlI=Ld(T(v_1_-z zzlXMfHkqZe9#U};5r3L`5CJ3I_vd;9t;VQ9A9h)zCy??8AGeB+P$K%O!Xi-mj7Q2L~Knf(QfEKyW_~Bzq6L<@KGqa0CCX1|IbzS`>qk}Bf_~@z>aAN-p}YF2($a= zHcoHu*8hGje?Mo7K@q&(?+6VKZwD-P>M}d>#jpAVvw#K(cX27yYrq4tHM_x#n^=E- z;GZxsLJn}@d5(wqB%nkhqoAPw&G6lEp70F5M*UnZr}7^f%6|r$hm-+uKqmnpR6&Q+ zqXqUFv4Ot|VQ9yX1kZq0(0Ow_URSXN-ozj{H#th`iTz%rhUA!l$`Z31(oG4LG$SI= zbT=CPc#YLRcLj2v-2bnz`u8;uD&l}u3w-CE07VCotC0awWR*x$hjtRfsqf#viva;e zbg~zPhxq?&FIe)zFA3}wu;d>~!gq$D2p0TQ4S@^rfHLJRkTP}x`7%bYleO~yVe2cP zs!qGF5kW#45tMGELAtv`x;v!1Q>42?Qb0huyBkE1?gjy+rTcrX&iuc5-+9+!xnvE) z%)R&boadaq_u2d3?_GC%hcHN~2FkESyKZCQbgI{uppKg~X#MtEqMO_ZO&@mxljQk)85`M(EBLl?57nLg50d z-KhLE9yZTbz~sYpJJfzVB)X%b^B-^7pEkQ=8}l;Q_YyARCF_?s^-U|>`SJu1bCL}P z2OvuEudlxt%`7pcgq)u481PaI+$LB}&~Zj!AdCV;=K&(o`hZDx@Qc@`NN@2{oBwq} z1^6-CjymCD6B3LKwSdaNknHAI_tbd?q@Q;JZ=3)Sf7^B-HS?!0+JOVp^7W&pCU8D? z^q^@}OCa?<3T&W4AiU>vKeo1}+x-;}|HtPN)#2KHqXW`{H=1_Kx<5(A15h^9pkKd1 z#toEANEXTb{Cv<*0;@daHz#eLp@l+3%VG zo|b=nVMjtusDa4zN~<~MP;2Z+j|V4_O6%?*WHTF^gRDoN)&3kyK2R@9<&S@^gMasN zhJd9djZNE?DJ=+N1C58~g2=)H;V01bB-h)K%L&(D8&lAZadO}UN$pF5v$@zZdC z-ibl;->#t^Lwa8Y1qCxw-_mQ zqz>XLmy@$ztSz?v=O#pY0|7~PBF8Gg{eB%SASsFP90WCiB4h{}cJQ%pY)tq*R{Zww zS0fzvd$0##`yCFVgv<#iZI;vES8NNcLdLny zTd!$nNa`D%uHyfyUmoEdycdi9;3C-ox`nO)1k6p3pL7+EkGLQmrH&3h$a`Rc5-%YB za{|3oz-wru7R+D(?fxc$*9PuN$HQIw!%UbeXhU{EP~?kWj?*yzZvCz^JHSvG3EXE> zXZz-@!*KRIx9+AMXIKM=>w$0I4$Fx@>;3gqk%6ZQb?xUkcoD+E9l)W;4%B-R&}Bee z^nMRUuQ`WB{`E+KZs+KVnTF=66EB^10~$!bxMN0fp1*)m&(8r_V89>8d*y=%I2GWl zVFw%x9jDEo;NsF}#B>Aqhc4$Qy&(HfFK-{mTDQ7}5NENEvOHdB`Xz-s5sZ!cstf=;?y`r2PtpdW&1- z(#qc6UfaHe+Fu0)8&XhUn<0%I4kApZ_qhho!|e%J@9gD&IN{$o?k6NfoEXRZuf7s| zOON3P*}i&C><$ch3_*lZSZs^L;!Q4Ym^Fw>m<16~zE0(Qe;tt&$Psx`G6Xpy-q665 z0}A5z4-yg9BH5QKc2Ix!xPRVYG}Ljr|MLa-`-NRH0~;#P3&BpGd;txHq~u_7MvgDx zCm3)HjMJQf12e0{+m8m|*Ml1F1wm~+ydl+GfB_8>L=TXWAaJNwgK%v+fJAm%x1orh z_&@3vJ?`hfA9JDki;(({n-@wN zb@Z48G=i8*AZi}gBm_fWmJd*hy%H#6H^t_^%SjJg>l4T`xk`d&W?|6CruBWw>t$04pt&cfof3D?~yP2@~<#z<*7*$CcY?%BwBic&=X8fhJQCj+jHdt_sx%_r-n=M;gRF1WL`|@ANW0g5 zt{XnA0H1E1r6qV2#P`wBsJFMbXgD|_fMUYq)_vm3uh-KnqD5HidG%%o`Xy761u)9mgMt8JS<3l?{lSJeh(j_0Is+VZe7}&2QThJ(G`)}pbOkeW^G6d} z=X?C`uCK*lU@yD+7lwRBBJ;!@qH_uxUu59#1A$A2mTlXynrmvjPFla4o5#A77VhHG z(%AB{U(^VS)5BEn^w=1L@2IJ-Ps78%0`(7zj?FcbAO%oMXz#CuIG*8#Dc^$2Z?^xJY za}4p`oadIFJu50MdZVZrHlM1=!p7FIu&}U>2AmL(8EkExTwS{n4t2BxGefcgAw}+jn+emo)_S zQNUbYWKkujrdsR)b4_O^o676I?z#cUU1z-d^Eb3GLIFu#Tm{cdJ;M@1>nTQAzS%!jqC?zST!AIEC&Y%`Y$EYLvs3O^R)xZ?Szz4()K%}f_NNu z9U}B#RYqnw-NI1S*_lo*!$M5Qnl3fqH8eD)Nr^s4c zkaShLt?|4WUGQau%QN{7sOs+VKu0T^8;v;~1( ze?o|BKFF@M>d*S~ga3f;Lpnyu?arpujDT^adXrP3bqkNzrCpZHIWTaVZLlYuww-;a z!|keR&C%4>MVgDYO89Op7sI~l>Y48Su=Mk#l=SrXB_-xZ#}~oA`ivA&xNvIn3_bl3 zKn|-hk>7Dw{;lK;ITAwb36@kBKb_HT!yzCz0HUrA%Cx#Q5qI*qc(JaD?#Hi;#E5TJ z(H*PDU}u()8ycEA5SBRR7T=FV$-%%}8URskYAljJ@V)h*SX|Ps05Neb51qlYE$&Z?Xoh05e4hb=RRCasi%YzajdH*!1OIe( zd%N2JgcM%Oq+T~uW8+f<&wkO@uU`X@Aqa~`^%PjTHT=H%1Rl_0W~ccVr;2~y=@fp* zxNPsZacO91Ak*gts_&)0#Q!L(3efe74&jN|DJL&>Lgu-J&{ z?CcEuk*1~sM@Hl@)4jaC(Oy0eTKz}_WE{XTX;k@fYf z2W*zL!c{!HL0LC4s&Kof-Ss^=*8vuQLT(ah`6i~Tt83cfc{~YeaD)u<6~Op=38NAe5DjxGdKU#-m~2h5StA z^12wLQHgNs&t%gYqw zk+S$kmGM`BqArn}W4Wt)d&czaV$wE1Y!JW+>>&dJVM&d;I53Zwuk;fmU9G;Pyq=Mi zWENivE}mJq#2(48k!yKx)T{dVVEDlu7i2JYg^n9yVsr}$qb0|foo%P4rrI3|u*tEp z8h`*KRi$VIV3y3Z?P+o@7a-i6qgm_+@tExD-@+baoO5)MHbIY3Y0@`1lzV0!+VRbFeKl9xXwo%Safd zlRmUo>~MT3E&|f{uAYX+!BIr*6|-d-$$BAUp~pDMNmjyo?e5KYS|TD;%&#d(SU3iH zs=;~@6}JL#X--iH-2p^*Vbb&B{XV1HyIZiT{|Uh)t$trf#X2K06g^=>F}~7Z2nF2J zM)&DI*zSk9T}a6|;OPo9SNAsp6+(|eDVAqsCZ1k&0>miLt)`1=D>j5jMhQu4;of?+ zpraE6`ExdM=xRoV#0a=Gbg{J%3=a1Kk4QigTdk)x7eD3V=7v}%I<`XKc~zrm*42@E zg*PTn-vO}Z*jDhCIOk+-}ABd%kK^%W8+g zKkcnt$H1J714cg0#b5^-C~Rdvr^bFTK&jjl7T3|Ju_iD=vv1xCL9n{R&Kx- zpLqex)HI=99ch$3tE#E%S5{`z6B)uRSYFaH4pRxmuZKrRE2eG50xZ}9Md;2e?f<>T= zAK*(dI^d5gs43|(r0$O(Fo)J2AAkv8za9Fba%uKG*v#5mP|$}D01H-oqN1Yb7a-LH z9iqw@E_I=|2Epp^sHpwGa`D@DJN!TEPK*N(de)d-rW-PUw!z@gU4i*v&S3*&vOOgv z%vyUY{4Yk(k67T{djObd8?7T}U7a7UJbbA3xM;mNdbr!9oUML~N;R+Td54_B=SYvS zE=B~_Qn`ak7Q}bix@P*}*y)u=hDrroKhfK_^GqA6gEzLAeR03 z!}zt!-g0nLkQmJ6Rp;>Va1U$dw}HN%2As_;LjUA~V9vGFRJoki+=()Arik}bfDSL7 zVKa3ip)R(pFq+|GVPhpV6?xVrpwK8 zgXJbntVFPs@_uVeiS4DkE{;L4wY|EvLs(BN2FjtUt?lSxZ`1Za+8nhVH|BY^fU zQ>gZqE2m)OrhT=59iEi5G#&K>am(E*96zIJ){Gk-zP)Yh*)*ElcMHTn!@yh2nA!Z9 z0LcA|L;?-;o}Iv;&ssF&5Wr)%3+;CcL@!%|sQ)!bkH!P@Scq_1B;_yB*$u7$@QbB~ zOvrfLE=Yyu=9k^Y<(l60iI&rlwXNk$-RH#;Yi0{&-c-B_DA;e`77O;l_irf`$T0po zJ!J$j*W(f#+|8?Y2MP_%O(ZNVJtV}hq7;PpGEE*9CRf}VuBc~$U8XH z?%uiHd%5HAgV*Q2dv1lH{*jTg=eB~69zFN{;8+7C_wn_}QVw-%D^I2MjIQH%*>7Jf zUOL&fwqt+|ppXc7CP7;f%&4+B+1J1 z<6!jF!|1%Hrriq*Ln8%@DTz$z%@>3$^+dfqy3y7{=Q?iRFWSX@meE1R`jnL55#M}s^a8(zvvt#rD%>4!! zbzjV5PVfEqjiL{ap`TF)T=@Njz5=j9wivsw52qW#M{MPUMMd=~$@BZi98qD$MI~nn zm^ia;jij6A%pJ!uhkKTk6!osdR*%lj1=20JcF*}+$jnwkr5v=zg?4r0poI3S2^>ohzvvb*)d2ZB2$ zALH?Qu#8Vk90SSiWZubX_U;1xb4aJopJTjAzEI ziyV569|MCmc*r&wUd@(jHlr8F)e<>=5s8qO74sG*b#OQVyi*jgu5+rXCrztNChq)f zYbBul6&|?9F>hxC7r<$1Tv(ev2XMy7?bjt_h{6wwd1M^mA)p6gyTA|R2zwvZckSxb zP^r2E7QbD1mfl4ikfn7%`GS@^bz&bb`VLJ+MFswdAG3eZQ)=4~5HWz4x?r1$09Z#5 z0t6l36thePbx}AOO0*6_)*&M!Lm2#{yF*{!9XZw4|4r}u5sSPt#C^bJwRqMBcAwHt z%mJ;nmFjU`!^N&Z%KoDtQ5Q(n}Qwe_y6{?XH^ZrFslk+Uggj^X>ut{v$%P8m9tPT9SD1U%`Mu?Z1x_sk9j zL_}~D`y+$|$Ly6=r#|jG;R35AgZ0^Gm?gkUGwk!x8_Zs+#i{A(v>0(g8XlgW9u7qF zeXujsbXODVz!Pn4doF0T)CjbJBz9NpS6j@vR2SFWG+#~nvlL)*LnH*F_s6BTh?#jN z@9&Q()81&R`gK6@8_VDX&!$jKPEHfw9HE2BKqG<--`(jDAnx;b`lVdrcO26-#U>dI z>t5h7zo>)+^#P9P(W6Hx+^+P%c{!5!6meSlPjSNoV8xHx-zb)FKnFzf41k;$5cMR( zFWOYFoBQWd`~QD3+98Zi^-)RkwJRhTq#FZG(z4IC4okWfllW{y0vXH`SpDz)SwzJ2 z&wbjKx5RRvhL2E(sgh!@+mV$+c~+rnlo+WvBX13-ns$x0Jr1t)zVLDU02X5U(+!_x z?d(}_xnoF5k}^53yJ1Go`dsw0gT%=ou(N}phK+qQcRbDv#LjTQ{Tv<@p>kJsYC5N_ z4c(q;Y>O=yGkPOCjh>kVsrC#RWC)6REy5R{_00|jG~nQIYQC%bCFSPsk)2nH<#+G1 z^K(zgOd>{9KGTXX5VR=aHG4ZdH69j?SAjcwdo0O#4>WW%{lF_R^4r%EqzI{qdV58? zMOPv?;7Itn)LKBj=20@Tkq`DwE-ghCrM_%vcGowZHZFdlrlS>} z_jG%nDz>kf2u!r79vBR8TckyRuW&T1C9k+}?ItxA4>jGIp zclX9AyOYz?9*_oO#X5^c%4xR}{qn=_i5?5W2O;ui&_F(z#}NVm(gb25GtP(lWe*x% zKxaN;6G9gM?x$h0)JX2&$G5^z2Pv)Qcl)N5E=`VOiPx z9K(bJ)!D}6uW9BeESd7ep=B(6QX?Nem;*PFrp>X?tpsj|8nh|aT8jL<2bEjuSG<0p zm;Wj&E(p9IWN=YQNnH1b>ogsoZjW4^_MNCt>pw+B5EFREpD*{)wrlDas#pK;>mvv@950B5#2^GD#IrvCK7{GYy!5)k?0N(gdIS5y9(EL0Hi62ye)%7 zTB;B~yuf^;wxUlmIXg+_=wiLlh>f3-Vdv^7?&^v?-e2$`E!=rvc02Toly;?C8?Q8o zds)%el#>&fPZ@&@t429PRuj`Y6!!`VPmxgo8qPj#yKKW0ITpJ{p~|Eh_nv4qAb-m= zLN22F6N1s0R}m~63V!8fzfo@<&THyNFMe#4@^kG|e9F@w8sQCzg0HW10R}hb#yOk| zQTPnKKPL8p*MZP{3K>~Xzy44Z?^_!S*ZL`wp)V_o&(oRRr%FEO25ok=Y%nnL}ZT4P@a$^qsC^P8!$oS2UA<8u?jpz`wi zP!FeBCf#bFC9!CUlnD_9ZgZkmib&cn-!U3ExQVH#;K8l9YYtQ{I=Ua;csXBlLA=T6IR>+Wb>^c_%EYDiw@9 zZthgf4$b=4-NDdYTv*y4M7I#jMKQQe@?#Zf9)*Fzw{dSgPHfH2rY+PiBJ=e4oXr&v zR%oPm*^M=E}@iXTR8a%uwWT98oF}E;~}seH1Ya1A3lFBp<0P1 zSW%TXmd&rgdRx1=tkG%jvRWm*MZ!fr-X}k!0T#S3Z*JC|7R)EvGel&RKTQQYx;nmQ zvg!lv#IwGuj5#D-B{l@TRgJH!tgLp~tg9Pj!tBt|vsDSBBfFM73B4kK zWu&(E6-n_@lJ6b|ShI_+h#|tnc3&as>i*j5`g5|-?|6NJ>d$HONCf8?grsB;N!$IB$<_CyV^j-0Iep-auKF10P^?`w3^6!1Q zf1t~!oR7E`)IB2&NBU&bjvr>ZQ5%YWUPSXF2_}Uhl7vxBM0y@(RMVDPnRV0B+$EYP zaVp}@kCxPYO_Ck*wMg=`UGyZug?VK1ifG`joex!~EAb=MvM2wy-QK4GV5M{0o58JZ z`-CU!+bsKXLO~GEtA4rkq>2HvVVvEm(s()5eO^0}a{?X`xv0GFe!5F42|}Ig$FqZI zmGv~_OWd^+Jhe*MYYvwK2$FKr@h)_!Vdc^Fasi!f=$IyG$w%lr+qrcSXmwwj8q82| zDf*6hWt=+=tGpT=QuDSBH|Pvvnb!Be7h>8|$%I<3)T&W8G)dhxvG!gN9$Vfe?QsqEHg#lnyMz<*6r91er+PVa5gW8);E?Z z*_(HKLkUMr@ZNk$^AVTqOoIXM@$q>au%nYy{`$h*X22*eGm{A2o{gO&S7kfg1K z8(>hrUvnGqsK3JuYNXtr3ykZfD6R!=TRq7LsiyH|o5GS{H zvWwBVLH${S+mjO!LHkN;TEhbNQqoPiW{=-Z284uBv9`3bTp-08q0Lp4+9KPU&r}ui zvFG@bR8{Ry1h0`c!Xy=zJ8AiT}4RfXg_5?6uSJd$Vi&O$&(2OPqd; z8@^SO+J7k@6cAcl3lgk-gi1z|xiNQyk8Sd)@&WHuq^WL4Q`w1mC> z(j>d^sinABT}xx|Dna?ja$#wh`9Ku654AGoD3K3ww9Ug>E(ZDw(kM9;Lt8DW_dll; zkI|tuCDa?{d4*yur%Ga#i+>hsSdUfOuW#;r-1a{toag_!-gHCN z-oXxyy*|vsKU9)~=lR}C_RqyKNu|plmE2dCs?57@FOc?VJ(p*6#wQfNYb%c9yEE7q z{E6`;mj*oj;IHlKcqoLBfkww|V<4_z_B^+0)n}UqL3uR7K}e17#{go?;)}$F#ea7i z4^nF+>N_W=XOp(}NvIrQ$wng2Yb>{!Lu3f(GliB5@@gq>2#rlIeBgMvo-S$3M4C zo}P>y3c}0C$;B3BCZexUU}+CYT6w(}5fLl?JX2yH19Le&>RmEdp;FNO{^#^00_(E_ zOj`AT#U@&Juhlz4Tib$_{>FpZM)@yEG`U^6ck_O6nLY}X`xVhenFQ3Cf}{Rt`H8i) zKB&_%m+V%A)y&mSB;|2hqQlMH9_uxkhu%8eAsMBma(O1ECc7e5+^$;;P(&EAS$sNl z&AhCs!q#)KZ1^rRUFbFm!{r=^<#$n}s8dMM4>^*~eg#a`(^kk&*x!Q_U#(eGOukYGe z!Sp6?huR*C_jXp+oE}Uoj8ISv-V67WF};$c?A2torkZLgk`Nl_oU61f|CFyYT38Il zPk4N4H)BoQBih?1`6ffaeDPx@fwF{Iz&xvv0GHG0Cqz$_Jn7v}>cMPu#xrgHT(GKT z5#gp)&ikWc^rog?rtzD}nSP{i<<(jyQu1a0s+v$OZZ(OtP?<8VtO{ z>M0MZNAyV{xv@TG7UY6CU)xs^E$BfZv0k&fRR7ji zC#wZTL&FW@i~H_zs?TL0SKiD<2P}Vw1#-Mqf79iG*@c%sLG;mpy3|G1msd5!3>f}C zM|EZYu}~fEgQ}Xo;JQD<)0mIE`$|b>wp6=XlHO!n;*QI2{ZLDX-qcv?=-^Iqxwb^R zAtt+v04I|>D@O=-cAwu#1hC)I)f9plEjRCEJa&;0mMdj}K(gHaaP1KOcPi04caUXj zN>EPv?LCZ4lmH0k1a9YwM~$kbn!f<)J3WurtT34opd!`+2qxh}6ptwmx(b93_M z+EF7U@%A5IG6@~u_#bG2qHtPFYSWS*EECO>g$`0rMpa zV7JXg)G-2B5BkCr_4MRHOSJ*ALPUVeM@0qJW}X-Y&U*kGi_@DCc)rfinV{0wl1 zv-X_MT-pxwda49y&xkuf@S6NSIi(B^gWhlkO*HR$iWksUTa*s|xu6;O!6^0hkoEhE z*bZKKeGtYm-e7OkDnZCgR$Odf)*S65~L4RiY5x6m(w;EOoh zZ*Pu7qVc&YKvpr=i+<{tejtlI1B1uo^z&B3a0Yk18Tab+U(xY@0_z2!pUXlC;Re8j zz?;SJxc~ZO8Y9SPsOq$~@^XN|chwJ$h2RmUO~@MoI_)ZL`tQcO%Gf1n#Eg0Q`O=M5 zCZD8CP_aHRVyPD7XPMSaA)dyMhEO3OASa(}E3tT%$c9ri@m3AhJvod`s9`C@++l!p zQ`;e&nB{ZkQ^g)qlONn_?Vao$lIS{{Ck@e~{Z|9;UOG7DiB5=7Q+(GRiA|Elm2@*s z2oMw5J;m)|pQQ~F@B4g5Lb1x&wWXWgg2(5hSUsmNI+rW$nZf7pz0oGz|4CZ|`!2JZ zv}zow%4Z0L)8jUw#(GNm{m;!%i;9ffAN9Sds52LB^wc8;7^v~?XkTNn-dY-w%HqQQ z_=KB3H9qnBlLZ#4eeoMzYlaJao+1+CrTItHQlJHt9HGdJFT-C5iLOuPL+xpBqbygpxR$vP51<(5GHPC;!-2L z-8j2g2SC-U+cZC|KgBu)4{*H{7jeh$Bs)$K()qkKK@u4?&`>Ki0|Npcw>QmK`kc_= z-N3n1VV`(S>j7+JOW&OD%RX!#uD$Lsv;hgXO%*r4h8RvLpB}CWZ)L}VjQ4+hB{2Gd zm8|1mzc9RlcHw{Z$Fw73Ba6TQGbT84uYmlsQ?Ou`033%!fHskZI$Jfvqso3)E`x7q za2Ore61Nr@**V47;4l0f>_lW{=29&sX?C@t%p^!urseYaRr4udZ*-AS#@v7z1$r{9 zHjRveP1geUx+u?B&Lyg3z5*0y#25j7LaYO#7Gld1=juL1&|*eqA;u$X)rE#vS0j0S zOimaNnzKS!uGU*0q1bt*Q&30ee2s*a74tdgc@LFLiQr_nF&@PP(K~n5qE93*M;@=Q z%zbj?O59pgRVXE}PY3IxgSKC-C#nh$JkNsY*5AzQT3K05vQ-o<%En<9bocbmNoo%zPNB}Kk}w=NsRx8bfwJ`Detdj_)st@9p)Ab1;&V@CY+YBnG{*=M>-vb9+UP^}p->vv^B%&?0nD{{l4jL|-e=Ykavr+x;F& znF-e7aIS!qoBzj3d%H&FMFtO;6XeW|s6|pATCY=Oa2}GjJ|@=HCokQgfTpTtb}lIn z^mop!Pj)5L<+_@VwH?CT)JjR_IOpQhEZ;Ca1^sOS7|J9}P4h+i(qoHMs4Wj(oTsHG z7LU=A+p6~*}x-Rs2}jlhzV z!B?r-TRKxpARw7IDk{oQ?TadsD#@!?h1B_?qP;~}#*Gg2adC0^@vhtPO61h%5gl$U z$6i&Y<*b&^K5^QJ%LhrAb937}IV8~UYyEIMoUN9Z3l#Zmw)5$mf|Qq~a{OeClSaLJ zv)sm1F(wsV8cV{_XPi0LqYy;%fl%RTtt*G`##gD47Tt3>=E8Gzj^$(-+zhNmo`aPe z#N*{^YLXH>l$Pw)Q%IaLeXvu-Ur@6taQlT&Jb4^bUzvX#_f4dTYYqiLi`_7+iOy&ZL|;y zFbceK%71BfGlqol6cU-ajp?^f>`-et`8j(+J;hE~(9Zv;qsbY%kJT2ab6}pwS9k<` ze3A`6VO%_I8>`61HeRRl8Y`5sw3gg0Ivpi4hS?-N&%E11ajLp1J(RyVDGDu$aWHVN z%0h)#W%1ha$|8eg*HI6eim5l*=zpBP!41h}uug2c4_f|TY|ClZ;m8&M+Ep1Z6 zD%2~;zisvK4^tUXke1+q)cg2>o!5#}6T&%$BReAa`CsY|7D=%gOwDz*Ha<2{!_e!s z3U=k@7;u+E3h%+#eCam9Awm4_mmZ zsmcZ3`3tNlJbrTQ*UYR%BtNvP26T*r}kjwv4+fXy#4Az@*Y z#RmH-kVx=jvT&lb9pO^q&w-971_#<$mzsbIAx1zD8k<4)8Q8e2rSfAmrgi#3j0=T* zU6IwI(0Rkh%BpgIe`EXp4i&n*?Yf8_r%0{5EF5Om>%RvynPdSgs}kixMNxUE-9iv| z6Oxs+A}enOvQ*!nJ<=9keK%HXspbuQTbx0hsy6P6Sh&cp8UH89O{Q5XF2Z6gJzU1> zl{Rvat(PyDLZ~EoZeb5jm89bNzhs=(}2h`-sZaGl07R?siH)cHz$Q#dGQt%Ujm3+Zzs*o z)!XZg-)`?TeiZ5^>Ft}={L#=FH`-+Ky5}IwWnS-J;!) zlFgQovU74#Shn!i*wp-9N6w|r*x1CpO#1^fy$2o|VWpnTL8lES$;ubjUE`%xl`8W) zO>chS8lsJFC>DW3?0LQy3rq!qEO*k{dmCXP2Rj8B+|i~dioZl8I;!A_z9DDC0fj?Z z3lPp(ltUQ${c~YT+%GFDde~7wxYz<6#RO+_dwI-k&;>ID?fUd>_@JQ7OCU&8BJesA zyaH^MtpS7ApLyW_Zt9L!0jTGGAe4K2*K_tJl7Lqe8u}OQqo>o^!oizB@{Z0zZP-=h zezJ9q^DKXM^*OX4TvtYTcsLL--v$)~v(AHEKZe7WlPJJ#{YYKU4kl>AwoCcS)RA)n zB6Jjd?he}Z#9Dr(ZWRj>aoJNV(_tGBhwia>;rp^F6wHkDWT*(z(DFz@FcrfyYrQie0tY`Pg4Vf2CL+(wRSdgjn<9N`sfr(!wLw*uLHvsxku%Ddz6R` z;?h&|uUt4xWo!*|7A@pnP33K&jEU|KaD+EwN2}SSRiN0lAUJP0Y$${0g}O^lJeH0(JSQQS(@usL*FoBU3}q4@VGtlYgb?sY zFv$fBh9HfycA`jy25Z=KB>fesc+@0qk@wBHxhk%Yi?j7ym1qz$ZvMJ9$nyRUE|DbQ z1(v+};lsHSDefW2Xnr%~%MpG0V@9u+K*rKU=gaeQY66Qi99&n`kqr4H0bfE$dyHt= zHdP!7khzR7t$O?B%PZAnf{cY%G1Z}xzqWtXDz<5B9ABIzef=sOm*gV|eQWWVoJ>R$ z-Vm*xUia(hx6)L0MZpm%kwAvclulC&Oxp^b9_0)i5is>63ADfDG6>$dwCvT?X`WNa2SV?WdZ(@Lylf8o2^mJNa*a^XTux{E86Y;xHwr2EvRfax* z#vwF0d8XCDj0femUeF4q`XWMAbM9czK?^d5+qsyqTar)fsv)6v>cEWv$%QU4_X(*~wkZph( zEhwn}JNEHy$pR%*S$u&^Y^&$_&rKEmVNHvrtG?Np6W6FcrdfhaMC6%4THA8!xI|~k zY3up?^N@Q_p^19Ojv#EF1v?)8gR_{e3hRvnkTE7x9{t zZvQ=^ty~mJs)<@(uSUV$wDmGYsTND3g$i}Cj)~Q+Mxpkgj$8(ck}9Y^N&FAG6x{y) zCJWQ04m+!lA}e+l8>=-7Tz02rOG-+~DJWE*Gh7RA4W|{n_yzX6s_sWB{yO)`kaO?7 z@4iK*KMvD&{gdX$SZ**P?|>O#x+nE*)iWkbG+sX}t`0@P;&V|kg$4$aK7X(0bCt() zvFO-40>wsiWYV9I!8sF4D*nm$;U27RkT3ug5bJ$DXBPXn$p0t;GJ#M5Rhhh=MNm+l z-jg6^cQTA{ZVC;Vs3$uX<@_2=qt0fjNRBvHEK(7@b8t{H<~XS6P4;^^C4;3}pohgG zj3E!&XS8MosTLVNO-+nfLED1V@2<2kDF=(b4}ISmBmMd7{nj|36Qs+Ry+5^h znj9CczdGC9ERC-R+PTq0z}8K3nQS&YJZQ`9?~DgOV-J>QjDJ zwA`UVD#V!2L;Kvt1!G4|$KuH)1ex3%IL7o?j(qnB$R}HqwOn&l7AAPtjw@I#IzR+wBT;V%UEFl1M#nVDaVB?bZrhH5h<;ybRXhuzs7b9MvRZMeHYLOG4C|cI(lDou>jWz z3efDgtfv#fbk4)9^b28hj-zerH&+_=A_8ZEu_&3_J&(X}3m89ORddg;k$r>Ng;_!e zGJegTYd>|wcsr7g9+H8}W;J>eGzy<4|Mw&J7g!@Oh~WlyU7U-)s-txRWg^8?sr@-$^>srMlislJT#;_TuNCX_ zk>3mf3KP~6jpS|;-5Jk5K3?xabCZ+RiWnX~emTdmHTOkP9%&DQ@yF~KSXV9*#h25? zxs5hx3v6(QNe(93PH91ByAtj*RaN$u6(V9;W3O0qM}ztgIqlTt0s|yhGIe^!ItA(I znao|+vRHY0Uc=|5SanZoySPNLSS#yUQ!;3OeKTge-)BwD(!?<@eN@IR{qX!nxBA5qLrF`h?+^kQ4^x0iu$DzrJr;8Rmd(Z0TVrJuptoo#S&+>#ER5*HXJTT7Nd86e8mzR-jxbDq3?N zBWk8|tDn`>3fB@<)A6SyBa?EHi3sT=db_?(#!M|$()+Po(#$NSjDKQcg2i`qY*1t7 zd&+#bC(e1aIri(`xx93z-APpd#*y*z>a2WA=~%2WDJG*J8z_vYBqt%!UO7D5otZKJ z3NBu00Cze2{{0S8A%T%AFuTVZ+`=+p9rt%Pd0?6Mr6ST(WIa8-VN@!P57RZ~N-vm= zLv%U1>0y^_2JHPAgF9?x8VwYVf!jkdDz{LZL9;`y5CI;SNaaUXm(bK_HHJLuM=;Wx zFannWP=6us|3{5CaKq?F!m*>u_+rtjKQ$EE|Jmh~oRyW;Q3*YLIWXA8$~F8{$3 zEISP9`QGK(P@&B+9tU_?ghZpIuj(=z^&u51`RnQZalIYlP z$ZU3WkKnxYzdT(x5);dpgWm!W_Yfy1b%FsGMiz0CK`L4nQ(}<0pjXf+thDq-0&)H% z$WEPFM_<69L^uZMhDI;H*Xq@AX3sT2d@|Z~451r6r!0hvN=uUcKAvYa=erZa!JU*e z#OdB>0VtenA3k?#kt*ibsDf90L3ltl#^}8WS!1;v+9rfNw3}KD%yOdtyQjx zxB_?jB^52v=x~rtBhY|dfz;EJYWOlfLLTh=mFv?VCt2?oGuG`G4zOjOAmMz}sew0s z0CeR+Z}%qCe<0QGG|)Rd;AL&mUJ%)|T+ToCOZi(1upT~Mtj5p?Ms6qX>bKuAcQY9% z<}*ncvm-q5L+?yk-1*cMQYWr4+L$t&$fU~GmC4vVxH8shL7TEc+YcyLOc-=b=rYkvlbt_N7U7yIsKfJq+Db?3wjD%@!T07xq{Gz+G687 zY5kFOPG-}uIN!Mezn?*2{2y;EfF@r+qfzSn21}C5SiGqIGzMP}fUU+cC)}tfLfSy7 zTweL1<+Zq`gJ&QS!bM~}rM%R|!IFycV?ubFJ`cbxngEvJ%OUX5`J^R|u1CMwDfq1W z6E0s&Vlb-L!{y)vM4=<3*%`qDZ=k=(~&{+u7#C0$gUE_4@m>&C@mN?&r-l4#4>QvOh zH()6LqYH7#3vKsJ!)IS$t~iwn%Z6J~@GfHjSjZ}T-ma;MtVjtIK(KsHn?A~mX!&0^ ztU9%yQGZln%DX3B57Z6rIHfmXU5GQQE-o%ts2c4)t!$`(`3Ii$TPG8!EafnF&?D2&ff;=f)C-f}t>0-z zZL-G}(9YF_O#CieUq{{U*IxSwoRZamHRy0Q2jfrJMKAE7lPYmU)u^5~piQ&cEK&h; zpHV9|7%AaR|1+|4r3%b(Wt`>V4Vc%P1i%1m$1*UGCj37p1=>icmvEKS9m=Ry1O*`Y zeok2pn>pA1E~12y1#`Ms?J!BfU^uvz;Mbx)9DFgKW7)uf#PBAy)C+4;;s)XZN2(VHZl3JuXrf|Ba|}C zz=|6f;lEGEpSDLG@sUh1GElU*94x*9#6T+Q5}0}{<=Vl_Jbx4{^yCcS+^};>^_uGZ zos}D=6HfI_4{<(tRjpv(H=j@)*9UmA1oDxwK;Hw`e@qsKu6qJb*MGh=MDWr?vwN*2 zJCG1MbYU{$Dl03;tKZ6B0lZB1C+(Ow@X_==vw8pw+7wFFXV~)l_fO@VQgSR$!P-1^ zm-Ei(TB=UY>t{JH1NdZWiQ#N|LeQH&cCTB3`&^V4DSV!fnp%p$`&6N|^Bo>cNLsz) zMn}%-vGG4S_pfmRGBOdGv+aLD423$)bv0&VLLft)g%rGiELDRjXYxlOF=DaSfZMwk zE+vk3x=h(V#f?$!wnF}pG;2-ZNcv>n%pew5Sjj@V=nL#Q7 zGJ7PjanwcOZ5qPx&2k(0i8|pn#Op9U_fNcPc$JBHaxVO1BCqHFS3>Qqr9Q(ji^%J?DGQ*`D9KKl=~&-hb?w zdG6=B*R|HV)`gB{Sp0y4@sKLj7LSrw7Mv4Su|*JFYb4_FZ*$JS|HmeK5O_WqB~)u6 z72mlrJ$~0CVG)~K4n-4)1z$-KM7CCs5J=Q^1rp;>(H#c3kx?-Lg|9Rw4tgI)9|ND} zc`ja}V+yk@?A~8*C=AerHM60fHmXI+s)iCY#+IlTWL5y| zryLC64}6oM7b5!(;zY>2ysq;0LTCT4#QwXZ@c#z&f3#LPBwjQ<+Q-XZ%IAmzbmh_C z{N|=&{!LbEbiiveV=-7>hhL?T$ik*p6)}w!#F%J=T8G#Qn8&3b@dXkLC%#iRiLP#x> zXCPva$ifE1xhQIBC3yqSgpGp^JdQAihWE)IT|MB6&<1RbhDFeUJNqkF_rufEr0Gga zTbR&a4>B?cJjj)tKA$-M2qMX6Yujb5rGZc~T431}&OO=4_WOE}^!~kT$O%J!)$1QA z3?P8%J$N9BV1<#Ue23X^!q$Mm;(IU=uEssTv9gjU^=r}hssvjWjH#a*24m@>^f9-YL05?L=MKD|rph*bX60V_yVU7SRg&sMa zA`G840T7=GfcbOh~v4T4rpH;0OkO>vfuNZ<<+v8^Fdo(9R#cJv>U=Uni-|^rf7LBzi2U zcUT-)o93o`tPQ%%TT`5a!ZAeDIMy3of163~H|dLLG~GFNw$INSKJ69vxtyDcJ%8}e z%Z0d)(NW-|e&DcwHth_3S)fV_(2g-A#u>2J8dv9e{Iitvj{y_NR-#wU7RuCA`zGME zaJVgWI_^bG;C+bh_xv9Pg!KHtJ7WixPDPZw}V z0w{4yc5W9_l_gLEL{u}tnFE^F#!c62o$>puaBy)+fDv-K)$fk95kzag^m^L#8jLWQ zSz5kfS2KP2jz?Ado|vabh*^_glqYfLC)E@TC6s^sNsdEAZE1zx7e^@DG9=Dp2-3UR z^Q7UkxttMC20J}wZu<`dTfm4_cpRFMvJ&b)FVW*e7#~DWr_5_JI^CJHtX|RxyB78^ zn;0r3gm>nmv5QVv5j7k7joXxOBuWq;85yA`bPxbB< zs4tpb?X9fJDfB5GLGaO!&j?79qBU#mbTfrJP`;Reopawo&X5Jb&p*q}dmDXY-yCD{ zAcq^ETkzw8E2o6XLEGesyU7Zl)a1_i35XWne$OB*_2?lztsM_vI5mxjaSH$hZ;n5& zyZnvUQ;-85>DO4vET5#IJGgtoC&G?V@`s=I-F8g&xx6-1S^}|4TPP?2yM6IYY5nML zNG+a!bUIcgF_46FcG{i{itJ;uVElE*IkF;d)LXPLqvE9|x zkBc(slqd~=Sa{^?>l^r6FxcJvJDx-k=_@IvPBrlhS(logcXV9fVe|8VJzNGBby8MA zf>@z&p(evzy!I{7O0H~eeO0%Ia+Yz<(Ob>dl)RQfBw&O{m^_~16%qoAOgz!&;UFDn zp!O+W(CF&Je_v1k=b4Z~sS6E$B*ZcbSFL}6&wEq?sPk6?YFYgDX{zOTKt^G$9Ja$q zjwmkXAe|nz2=}WjWj3*ccD-3vMUKZ)$#UGUQxEBRS^Vm7#v351t*XCYV-XcV_hhpC zxJmTKBJ>xdBTL-;5gZq=6A*|tD~@S*@G}g)zPVW_A(wLk9CfTP7;NSXSo@xS2L}nB z@`{R)swS!CrpRF-G{p+T^O48R;45cf&v-McBaMcE&a7!>lmbWH5G1UQD2Hns`;5~Z zF8s;RDl;l625zdV$7IAwufD&p|GioVnw|eDz5c(5B*pn46v_q+1)e~?o6K|!66pnE z9*G99ZD??GyeGj{SGV#AN(n-L>g@{xwm8VQ@CIq3P0O(2@#x-sJkV=PARfpDA$LvG zYTq{g%V5wW=T7wqfd7?bE~cc$5pa0}Pf);+Z-HuNsRs>NA+VcF7}OT+GMD+HK*Ahm zsuakJ^J!Fl4p*N3o-E7Gaj6h2IOFBz)u`*bWtcOy8P)+y*~T*}0Pi-?8MHe93`MRe zg9$ihoDHu>G(|zD>0YLHlpe65`p=tNE045?)n_F4n4AhKCMr5+ic%9+ciN*d3*Hdd zX4L*SB+l(iRsM2ah;}Y5Lh}f6x;x7XRCA6C)Mdi|{zy6q{)g=N<*}^7(kbSTEHV9E z&A`CGv55oE5IcSjdDBXiA_Wc@V3ta!Nqld>Jar+5VS@&muzK;IKd!$4;F__LUTc4^ z>qP2&Xafu~X3QKg`g;g>b$4yxI8arqZ)7JRN>83aiBmoN0>6PfryY%dk>Z~cB#9Ej z68HZ7CsiUo+X;=cNU)>Ht4lUuUWB&@X9Bdu9IZE}>98}$88R$f(u>DgJ^ zp7e6~b71o>%bA5sfyXZx>18>Z?RTpyqQ~IDAhPRT%*68n?Wz;p>DE28*h0`g9&Ww2 za}f9@yX zId1O}tPrvw$5RIwV7ks!l&T@A``3*e0wKjZbk;K4CqJ|2e}FbJM@hQ^Q3e%v* zE*0cFDrJlNR9(Qe%Z!o&12z=?`4R6Fz}WX9JvcQXEGFC(FyZzPwBp9bMtXf*+FW>o z?85ALi0TWAd-U)IkiRflozC$mAoV;Hp7(Yw{+R_B@bYBQsbjLEbmMY zFkMgqC^Ggz$~|g$U>G&ZQ6WGbdB#I&d`cyZjwuhNgD9q1`Ats$DjSA;ywYHOckv^S234Cu; zo&;*HfLirxd!oc9*8>o5l>BxnKT;nTfNwkIj_%|P>{`4&m#$J99L|ixQjusdpx(b{ zXF+!&M$+Qk{2Fo}e@A$vI?7t`nmUiQW15`PpwF(>Zvg^e5!=>^7)AiO&|82d9#_|5 zzDxd#>{()gfDXH0dKfxEPsU)0J7CwNsHUb}W z?vt^cx1nz1T1@=gyz+nVCYb-|=5LV!{}A~BDZd!dI92+8wzhzr&8cJRwzEE+#8Z!%`7VNKN0@7e?>V=h)KA@}hS&#zo zPQ&_~MI4vf+|E#`yFmM>*m8ZORGlfCi%L4`ob7x01>H&CEK|%&i{D|Pcp@G^9Z#kl z;JD)j>A92uC#tr$bNgs9=d-2?AIpYxm*^)FZ@ ziE<{6{6q~wUV+fVBZ7(1QT4K@cvf^J`8DZ}V9i1I#KdW2q9+Jeb)82QP(HD*pbjhn_=^Xt5>0S)U_C>AUB(k@=>|%M}cQ zO^ULzOkbYubBVm!BM}i1sa7a9YHnC(&apiKp0AhsvJdRMXqjy{_X zz|%Ywx3i<8w2dEZ-$;F8oiNi*&qRP@zXM{^jb=`46*8OJ+uLm(VaY8ay{HBD6z}xE zA?e&l$Z+`j_BOlKJsH#)aOi)5u_Im}khyW5uO+Hk08Y%NHg=BxNeN|T516f}R=`uS zbqx293cgRWGq%hfNKi})bj0xmztf%RB*#n9XaDp&b?k`ig7k+s5qOy3T~{i>H=V|? z?F=L(Cubc<;~P~*bIn}Y=zA07!^~ks#cx+Ap`@(*9(Z5KMSlGFp}cDmGG%6i);P#x z-Xs6=<;&aLp!>p}roXX_(ELJUqTRD5MYQ3@m9JyInAYvCb_Y|#n%Y7!iD5ub;H-QI zOJ{rYpLqyS#kyELJ$qpGTE?K#J!A4Rbx{K2zHpv&ry<&BJC zZy<@J0BPWhChHOF^J?2kEp`QtDq0$vBQ;xqCK23bbG<*OHl{LIZJY$Er=Gn#h%<)S zU`GD(h^IaWi<`cd;7@@dS{Vnqzj}Rt$Q~_9iU|;?Q|>`!A1hZ-+MXXQ^MYtzMEB{c z!UKXZQ@Kq8jaZrJs8lRLDCkeV0l%}s>3N`%I^nNSiPLj1_EKu4LZ`OIi{4zRwTb-~ z&VfG|AdECEJm8BfzDKI4G%-hULN#401T&8M_V_pxfJ=ALO%+0t~B7O(>k zKk7zE(I%`fU9y*x_aIK`1spyl$ATt)2f~H#yM>>v9e3zqs0TnFg5Z@;{E2;S=6|2s z_o4wR>HG?SQ9Y4#+>Q0F0S*%z7A}_BgX0qYTE!x*5?5-2QPZz@sjzjsmVUu=>g+0iNcZ*E0QhwRGJB4G!4%f5GY@P+HDgs z1sr6*fs9k0jn2{p^skRNF5wSBaO}g=jR)Uo&1Ud)N$aozzKMmW^#sDP`sFQ>+9Qv? z6lAfT%J_i%@;@gsXXOvc8gfioBLW|YIIYO4i{jmT{Ld$tbAiM)hT0x{|G-XuF}36# zaDqBV`yA{I_*tSa0O&(Fwo20bZg)P>sa*2}Jl!7%sLQJ_{@3)E{L%!XA9Qo~W(J)C z51}N+764r1TNZ$P>aLo+TfoaI1MMJHUM#OB&bRdTK)wyY<&HD*($hQ>RW zRhIwQ+5vrIg&wrp0m%K6=O!C1AnWA|sK)X5CE$U1#$Y9**AYqj%Pc&e!&>k@Rsod# z!=JZxJpcR*h`PuLo<*%v+&iGg)Y+qer};9Sn4{v&-rU>XMs9?({`Pd*lCKf@)mGqh zQ)7S7V{8VS0C1%-C%Y_%;k@POVg}}{R@T?uyZ5*a!@mhYiVE`cw}zaE&3C5C1)3rP zo_k`xsY?y~CZs%>>N$`upe7eX_1V3kugL=>`9IN)d-@L$WPUD)(nr+e#t_V7z|N?K zyC@^aSvnN;60R8bO1R=*1$$&<#M3^s$oJiPGzx3R z(pdoIU?5MOfFD@Z-URH-zWat)Vz9Tj*X`*HGIP?dLX`yp#R3+FyeR*T*bL_y4_5gu z4h;=)G@4dcR(=LxTljOxZ+>t3B7AiRC>7It6!49JdDae) z!gqbV-6CG6i*j|ZDHO9%Ai0B?UidRPhfWT0JOG<5s?Gm{Bw(sYcuGLBisNcImj;b1 zi>ayU3g8k{aRB4maDk6)M@UUwG@0-Cm0N(_W7Rru1P?IF2*n0QqEqP&)(pI?wi#zU zTePeSRlr0R!+%)^zi(?0nV$&l{to=Xz1u$+rUr4 zgI6R_+ri9|Kj(PcM=52=m+}{_*7g!>>GBYi+MjZd!mXekja?|*vSP6xgLXp~C|jBqu66mnZj|{mk&OyveHiM|1W0riN}XmBjEvHt&8>QfpOosV z^+0ZTj`W{1Px2TX_-@?yjZEOci=lvc;v=vt;rEI&SUbk%pGzNVDkbK3>rL8+tq+d` zlVlCxH+hOiZ;{drY{fP)=RY0~^?@?$HiH-Pr4$^F->%e013#)AT~@*P(DQy(5=bQg z3g7zr;sDQR#Yt%t1a8y^7|L~~@pk~Z|E@`~5pmmkkC;(wjK`S(0WqJ59l(WMgjk=m z9{GG|8!Q1IIU`_UZYRFuf95n&Y%H5a^PkV(W>4I9!cSEjHyzc%Cm;?WQx`BW0{-0s z7~{8$Ktkddd}K#O$z>|y6{5LEk$S+8Ra;5`^dG6wf$9Hv0rDqH@(rxQJ1;Tl>IQq( zsHj#D(wn{a|J41te+4L9l@Y#+gK#7vk*%c2c6QKqP?K##oxnl_>~6@Y4#FQWs5^%Q z768=f$>u?O@EfVt?r9WhskDY!b6!)EJxY3BCxuK+w^k9rCmaJ%Qi&N$Lm~(-Chf%G zYRZ`<<(Qk2jMZlZ)^_ynTFXH4e{ zUoa_2x4GQ^PnCy1NAbD*^8!Z!(*l?rS~$_d?Leie#5TXXAIYFZsfq<^*Cpk|{+(uB z5@O)=kMO;u&HPPX|84oH85s*1U)sM>f7=sqZ9N{CjCrPl!t6+k0{z}{bE;pR_2=v? z8LTqm3$h|0X1$ZAu7~pb9W@^wU;_Dbt z`st@s@){}20`8B4CCE=)paN+J_k!ARY!$+r|LT6ZU!lNtd;d`651Ke(vZAiaQc;!H z*iAFSrMo^&pV)xAVkR0w?hlmO-82>>c+mcAZ5fAd z#oFui!Ah9$u>C~$u{Ansn0M`oZ2CM4o{9Hmu!cN5iTZc8%C3L8zWkHDN zXtrq}PVSc5)>u5STgwB)F%IPT{uIM71-`Y0_KuBK?6)-V4O#!ja(ZqcEXshLTsBTC zzd(%bNwdf4jxM-m6K0h0DY)ByE^3MT{TGE3GyB_G=|rwlHJE^+m=q5@(9XW2R4i4) zj&99?&n-z-xNkz`Y5}n82mL%6|9hb7ebh^-wHa45WPvfy7&LkpNDWBp3EcRvyx_PGl{BI%9RZt*q20MX?htUAm?{8DOdOa@G!G;IxItAEyqhW4b!Mw4;>EuQB#CN`5h6svcc`9 z?GIe+QHM;-%(Cf7PAcKwME*86_+J1E>FbuVM?=xT|04{c_W+CQRR2B7p&^AwYye!N zHtZ3DGeE`qibszv!{(ssws^%T1<_BCG}0=G(Q11oU#g$76JWIm{*N5Lr>mL25;v*!< z2*huY$3;T`hFx+tA@qe(%pEaEkiw7ZE2F@|z2nf5P(j(VZ zGwE&cIwu3s;Fo#ZK=N{DSU-r$|DXm|=K(9Rs%p0QBkG}t1fwv7QkV=+5+y?F&Jix#K zXYdEO>dO*P)`SO?WpY!uwzkO($X70RffBDB|9d%%j?7?RQYX;L5-TM1iNQuk;0+J~ z#?*TtCV(G&k7LLCKh0i}3<&@D)ifchP6YQrf%gYMgSp>$$L#Qji}k&gJQr^821_l= z1SoOQgWUJ?X8FNi*|bNC+`mH)PJ@FXhO#onLHA>Y$XVu1?pOKVcs5@U(a;pr{(%0V zrx!zHHheaycYc*g6KoRkrIa<>`=u@uR~bO4!Ybj^ynR!T$tsT9XwBY#nMJ>u^g_br z`Na=GrrCReHdec@js@ zwyalg0#-n|c&ao3F2hYiGM^d(hM>}ZpZ#bac14fJJ3a1;>Z#_EtpR{A?dr?%HF(GW z`!`Gy!IM&fgGu%j$?V+=dL>20pT*d~i=SzRgN^;&l8~0RM4EC2T!GaO#T$?F!1zf=@YwOF4zy&z` zO-OWtD8Je97<>P7csOP?mk$WU_|;8rTs3uuua@zjkF>N$4IANEQT@^GZt8%|9zH4Y_bZ;Va=Fy@k!$CA0IXV&&>kB zW`o984TYxWX1<{kKp?DpHi&Zlr=ABiN+Ow&r**CmW)=xGI^KJr13U>6+XQvQIiS0} zZzbvCx8U961?<28O=lIg^8$T~X#$?W12}EPcN2E&y=n8>%Z8jQ z$_rQ@n~XqJx#PzI;9O@LytumBSUj&E-v}6B6+cKzk#5iXgDBg$Qgdydz*0S?M^`sGE$YyT^^rpfAA#ywA4p~iguy>@s5Al+mDww@}f)hI|Ux z@3C~S>s^)26CEAh=iLDA`RtlO(nImv zCT$Th3M~3rJQXSECafyPdZgjh3b;MfxhepAt1jR_z;Y}8S8s!Gfp7zcjx?007L29C z1yP`)CauQ^D^FVxb!^?+fFWpD%lQh9Zlin2%pNdEJl*whTqO*l;a$^~r0*CQ03Fw+ zFMNJ?44@EAt3b(Dk?ui@-p4UB&Vi}rad<~yoiSF$ACngv2bN3M!1wh%L&ya}N#7G_ z682l#Rg=6Jz&CJeHC_F*^;zce&l)eL00!v?y)!-NhCPqBfw(nBL;A0~mxS5BHKLBE z#qZA7p1Tdt#o}MKnZ*}iw`=>N>~(V&1}xUN$>T)`&qx2zp$FF`e76!|`-V3G3S>Gx zQWi2YNY09;nYg+5y#aSz_V)HC4uW=0hi~cta()Rc1_QEBU7mtVNE`zk%%wpnprItj z)(Kma>(;NqpUd7dUhnyWT$in%{ynVbxe>UD1}Rlxl9mc+I35{SAjJ!En)U#o=6_}V zOn#I28!sSXktBfzxFcc39Z@Egkcf4GyDSde&`MEck8TrDyO@<9E{8HZ_rc{Ct8J5W zn9jkH#(3}+{7MpBIRxIzL{KRQWM%Nv9+JU^&7|=Jg22cA;LwO=iX{;3l+`oW$RHD1 zXca--y*Z421=?dQ4)Jmo;7qJ*XjnIwB%^^5Nkw%XFhcTOr(gtgLF^A-ZP0_^ zmpIiDn~2?CkaG5I*3XK`r0n{vH`M8wnU8d_MD90MwHkcq-)!Z2^{VglPXFZz(1lLw zc^^MMLG7TGJF#;5zI;^xO35St}pb1A@~HE*XHs5_ler} zNlSkpT;a4+=kc?EeOW$t#2CJJU_Dt`>v0|DSKZ2d*t-l<<v9v9!*J(&vJnjzVckwOpfc;YhO%%bakovQz z+Z=Q`4+v2%JtB9)rFkcO3W+gzlvvI+_s-3Au>H0Pz)phyx zF1un#2{vbJ_G?DE>$@B-Piy3d4H+}RM#bXqwPZ!kc%&>f@pE?Nr`K<0?h9OMm$Mn_ z7PHl~5_zb84c~Q>tx!^6bSKp?2!i@6D_)Dq1);HYn)XMx?w(WL8Uv%fEYg$wB^Cjy zm)wNCjL0s`NM74uTe4z0{GgGLh$v5bb{wtT`(Ua4@~*Y~qOT4wUzZ_r`49xUtWcd? zY-DTKh5%ClHV74RHZFs=f@SAA1?xDfcBU(McuLUO;U}*L5vG}Gz>T!s6M6W!cj`ARmqbLFb2qjm5KfYibVJMEpN)mDM5TA6_y&|rgLS#gU^wG0%X>oin~DyrmOE+J_vY=^#i4y(_`@~&y)yC{>{tNZnqNs*e&3EMrb)*TXXq>bH956+YUZ%%x4Z2R0qX0e9p^ zHlr_k80vMFWK|G1I^n&L-54PoXd}Z;`wA&13bfYL`W`9i{XT}-2Dc)k&W^M>A{;+8 z(`meTJUo5SwwBA{idy~$r64Ah2n+={cEF*pa?RHFH1fYR8(w$?#+5#fe=f6BqGVWVcKV9W!bcz1Iy<2 z+T{2vd5Qb=tc&DDfSzlZiDUQmzUbR#;nOU~5wed1{`+sUv!-bPIX z!){@?n{|1`8*=tDn3?he<8Y#q(L2SSbu_F#2JjxelzYZ2vP~d}4A7d2$tH{*&|o*~||mRWD*(!ok`-NaTbUjj#;2m|dpWJ!&n69!%? zCqviDZ7KuuTuLlFyTm6XlhafTQ_4uiNmi` z7l6==s`sUOc{uP!RHCIfcwwO?9lO9Q+Y(5`vhIr*dW)lnEWZwS;)Q{YXuPhQ&*y!? z$~v1H^M@hnDs{huBLoikRm%;1ZsFNoF{~QX6|d0TePrvItK=Ai5Nc0MIzyTtWJu&WVEknqOyHtbe_ZSo>6@`BCe2hJm>Fi{cE_ zt{I0hQY2@!CZv*({hKIX|_m`ZT-R}y$gJ2X{(M*rO90$^z9IA*4+@$ zP3bKfsDv}4q|7*s=EoRKf~?QYh64O}%EZi+Q?8UD98)}%XV6S1vEWt$fDT6$s)2%n zLj%PoE5$oF`v~$qXS^XJ_ycBjgI#U}zGXiSXOKrdmx5YTq5S{Tn==1px0ghDErlc? zZsLa+A(!gAjHMO%gO1KJ%bgmM&eo?Hw7UkfLtTY)!H5#@IX=?`&C)`QYLHZx@1?bm za(l<0@sW?u-29-L9i5$gz37WUR+bOZc!)FS;n(f&sbQjq`|DWwxa~bUOXyL27A;a$ zT^LGu}Cp76uW#m4s$$4{47zK63!HtT)kC7D}m2^2W zXgI5>5If;RsTZ|ni;Huw6<}-c7^nNJU5IVVdcm^iqlhnz$%9{pS<&sze6r-NjHQZ; z)+2C3yYS5V!nM2<*^|@m?s@g$7J8WNCQnKR+}Vy;1V#v4@9s^Mr1LALtSrJteJqdq zqJoeXde56fJ#b$Pfcd^MTBnl0vkNe7%IAj{D8f67ZA85yTnH?9lQifpVG7SJ8p$}; zTH1d$nY7a)lT|?v4l~wY3aW*NL8ndHBiO!11=quu-u|t*z83|2+xW>u6 zhY9POlY*f`7+iN{p0vx-RP@k8m-DflryW-jwo>8o(DVmDHlFw*pu`?-j#+XCG$*&0 zj=Ies@>ddR@6fOJIqKX4FccrsGSz#UU{vLtIKiGj*4|@Mf>ue4F1M?)6h+=?tNN z;zULIc-_WPe$H$wkZzmYw+_FQ>Joa+%&JmcB=%*6Hh;(j)mmPRS?d&L_G{w4gD1Xp zdQ9Tf`8%?QA6U0GdahX#D7nlG+0@!9o%*g?UThXgD77Y2U->bnmNRNi!NS8;ta+IC zzxBryO&Q^qM|FMO`e^oREm7cpGF-J7wI}j}OU|0y;E+g-ANC3=Z`r{az6wVxFq8PI ze)!$IB5J0@E>PF`xfjwLX#*7%XIrnS79Ea6~+R9wh`ny0Na>{KTF!BNWzldjq z6C5!^s2C`m4Q2@27_pXEu$+Yf>GIjIHpQe!r#x?GvDLWpTQn&peTxb(iO`QaAGG$4 z^r~$7DQt>{Cu5NbymsqYI)uhi3{HtUlNp7KAIeSotuV7$`P~-CgBYHLu!Fa&JFZS# zt*WAoIl)cq=_98DvTu?7S1lz+xB9M7~ zJWYwAmVy%lIu-W{HvrE)1-i&X`s05tO+c1_QNd5f$l?+Jx_3h{v9RLN@u)U~$zc9V zJ#FKZuoQ1##AcayWFk~|NEvI$4AeOT)+yUkUC`w?SZd$D)R}T$kt1nnFl9nd+Qh|B za<%I;BX*BZp=oNEC^ak+dQb?G!!~0mUD%qQIUZAv`atI$OZ)723exS?|9Bo6lrkgf zPcAK)Oapf*jZl?#2h$M3;(RN923n2+i67rie|VgN@s+W;&o)N)j=IwaKFbvB-L2=| zR1zLd;FjO9@}EeMJv7wBwM>$S?J$;~`e}lqIfRj#UiH-ZUF#$MYL;L2$Rn(OduBmaP+x%Jg_s_8ugS*hqdV9v?O7OlS0Z2?Nvrfq)Ip(tc zNn&UyNNJk2zITz-cAvrQ*N77CA$wO`B8yh3iW|z%_icr8=RYm$ljwP9c4ypp;a>#S zOv~k$j98Ff(5?ux48--LvpZWHn<=tCe5gP`n>3s<9QgE!ow}(AR-=OQo_QMH-J)@( zDbBXDa@X?ygPn(9%{ns(V`|!af7kqDL$3D}7+GyAsXQpr_@9uPMwTo=avo+;zf1iO z(+CKZwkAY>QZ|`rDFaBfcf|gD%(4Q2k{B?T0j>n)M~qILl(da?%P;elDF;?NhK9d* z7lnMm3V#WI6u9|XHQ;d5Ce-i~Ph$b_Wq5Yl03E=lL<3I6XTuDn{!O0AUI}a{`5l&q~&Jq_=*H#3~@--pQL66uUlRA_E=sl8keIiT#2`Y_3sS+ zsDHzMu`aMdGf(kama~<7ddQ>g_A2k~f!A16Yh_6hEpC1d z?8UxO=CQ&MuK5vaM{3l8H$myb%+8TUw-^W73plmCw2o;fVa9*%H}Sp0?jzKADmP68 zAveY-xWTOZ=0%yT_tS)Ud};*=^%v&QglZ5mZCMLH>kaUE3bzg2L47sFGvp`_`X8%| zeWEV0zZqC;xA$uB92dL}R9i}Z6t6y@a6z6Si+Y7Q-t=R3*|qT=>`44TiM(_1oPPeD-=64;@)z;!gT4ELTb);rsuxeCLHCOAwzFZS%n1?Nf_;*O~EvoeEyFaI_BH%Xc0r0&Cs>~KF};6>p31qa-MPZ&u$w&P z_;5Src$ara2##|qcJSa2ouCdUexQ)t7J?D=9Eugj!~?p`Y53z`G342DL+53X4rn4u zZRbV2`mqx8$CL>O&cx8MD1Hf0~gCd^xxgmoxP z8}FY*wM^7Rr!aH7r0rYBR&$K{x43s!i%BdRi=c%o`X|;Yt=;SxyR6Kl+z9OAvc9Fu zVWVsR{BbSKMZ_4{?^qQ1Y3+cM^S;mk2K!51QB3q-TedDV8C||xDJ~-K#w13Um01b= z_M9?@cbl4{f!*@qWr{@NW*a^P#kq8HVgmas4|b|w0aHXP!9f6xIE75uXv@_eJGm$w zMY;LLbN|*v9`tlopB}xtxt#4?U?2P0V4A_nz(JR_az#OH=y$Lo%0P03OIcta_OC|#OleC5CX<8b0`wBRS1C{+RB1i|z7;@K0~qHlQzg|~|rKaB}(WT2jU z!$(6$-rof~fqx9QjFsF83hs6V8Fz!4~FhbAB#&$9y~GK!Rs?a^|K5yx$=FR62a*uwg+Iz}03`=hZGl=sSVn^%a*FWOe zLk5$cvBW&4wXFW^nCKofnzx{G^xX}UlLD(?;LveYEwt4w6WQULSDU>t#@do5li(er z)Fb!Ix7)^n#H0kR`4pn6n|JV7Hoh1rQ&#ovTl z(0%Ku{*HGs?7oB*wR>ZEZ46xXV*^q)9Xq zC6i1|E9E`BvQI@yN*a0m@nhOjG>qo9*p!Ce`~}uDFfen(A+|W&oHaQ}$T(S7`wBdG z@L>4t4cIOQI;wFMx2-P)YxTb7(8?-V659zY4P3vjh?^4TUhi`$#2SU(sow(3Uqm!^9!S1gp29C46 zTLOyV4$6Y27$I^Egk+fUsW85JfGzx z4NezH9faX5G2;c#E~Dh+L)t$JbVk^s)o$bAxngnBV6pO3!X~Mg9XttqRj{DpsMqv! z6TMdYexp|hML2)zcf{1MEym?Mrq%t|n*BS5mOh0Dk5j1E-@XzYa1Tf~r3sGC_#@DS z%*u)9OOFpre=b`__auj_1!GP%**Y6ykEw++f))7s^&MTxP2-3^HFd-{+|oMgsS2uq zd$pYeumH6U>h_iO{Uup}B2>@o=$}syv?P#Nt>4t`hIZ=TF=lK+MisPcAgbPKIX1+B z(}{$e9t>HjhinK{FI;NsX~D6v*!giV=O^HQ{hA#GH60R@fwkSaac7VV6DOYatjs~0 zj(-M)tMJC!$;A?B9Wy|0MK|`*n(}W|Q_i(*8y@ezBf#TkYy!i!&b+tkzRz%m82khn zOjhZ7{D|3j2LeoZt{Z0upDdml{6DAYRtQ^X>U1d8)+--zFX4Yeawx2}pVbm5dYkC3 z-(&b@{yxU&i)XYU+D32hA3ga~XhM zazj@_FyRI%`@KZJBL;L&r#}p+BM)EWXVh4Jo$S|l7kO@o%X;$(lR0wk{i9pCLAKgm zU1!5bX#&*SNBrhn)ctU0H-0kC{-;1T@fW1y%qep7qHw)W7z%w4MXykcUy8veb(qIB zQQ{x|L_AlONJUeMZ1oVm_~Kq*YAs4THpDX;9Yuw47Cm`^@lA?>=~Lr&E`5%ab!}L1 zx>H(>x$8$pwvTCTrp4w(odHW)tJtS+Jg)=+bQ{M-NWQ8?aoI|}>9srf^;2_$$L>!cvZyF7#sP`$2Ac+AkcOC_7q827LrptX zyYue%la}`mN|biDnn6Etr=>{rQd%nrSXzuFp}tV@^Q35hgkIHs1~vGU9P$^(`Mh6K@;dHpf7!5DmN1G?fqgC2?t^fMzEw6I_pgg6>( zvRsWZ1(mP$8M5#=W-%DaU=oVNe2-ZP7VF+^&8)^~NW;3HRT}71gs2L2 zKI3!Oj(*0&m*IZ3IvXe&%<-1E zP5<<3x9J~qLqCK|7AX0rSmf{$7!A52OKGfA4U>^N#M(~p9u!3pZkn$|+^AIkp)7n9 zGTq|^o%nC5k9#$GY<+Slm~Z@f;NBTc!lv)gW-K59h;w@B4CBQobk4i zaWV0w@ynB;b0Yb+{rXMeyq&`jLg-QSXJvh&W|Eb+1VZJe&W^#_v!{5%PnmRy4o^&ftcOf;Ou}C{09YWlCYvy-uiL zrT#!`0k!s~J`($6;zrwFfwS&sw;Zv+AAbc$oWTBG1|djalg}EaV>h0@F&;^F^ok7`E%uvlB5q20^fYIvSu!Wy z=sc&N6+M*w z^!c;>sO5A_$^Q>hMIs>8gU7GljRHso$G(=p4i;GeCi-%laW-m#x|pI6&KkH&Oq;)t z{W1?o_By#8E@{JT%6x^$DR!h;C2O|=3=S1_! zgu!iLmdE}2VyA!rubG4`!q;vLV zktt5gcwP?gdTYuoC0r%Yr_Fi=&RK8CN9|*i_}~=mQlh`tBu|;7e%1H%K_RgfF6hAP zm!X-3I?@6s4@{KE0;3~}AC9{3yi@$KbnxD@bgdLt*<4`n z51zew?bDJi_SBy=)?89QeHBUg$k{k#_K;C<0G%T9Sfbcvx>{WC;9U@OD<=p)p;8EB z95{Uh(&U{?QWt_~O);0<2cR{?dNluL&E*Dfzpc!`lAd15DKR#8^#;>-9DUWX_)$pq z8G6R)UBIlTRVLYXR#5g?T1IrZz4b!v+pHD&%zVS;8WC;oM1m*-?iG$Ev18P+G4%0g zikqOLTiSo=zf=p9>B!A>R26`e|KCdHj;LhQmQz{Th)VYExe4$n*MU^RagiQszgZNa z%|9!2;%~3jXcm3WXaNoRX-a4zJX65o9?;AKK`e)SY*A^0;z&j?Zgk?@G`iFZF!o0v zqoVgUKU;F2vCcpKU5hWLUQrhA3B&>BFIL2><5+Ep^0#ujGBV>+{Z-=&#O!l z@g-do=9E=e@t;&%Q|P>Lw6N{QA`*s9a0~P31Z>Mm1iow43q|Y4X~*veZRfR`W`e5t zX*ZsTDdx*G)}@-pdkJ2yAev=;E~+)8_ju{BR-_SGXKnj-D-2w#_~HAXeT5ymA*ii- z_w?RmrHlL5dJ@gS1UeO{tiZ-brSo5@ED4EJp5y3MBMgO`y>X*44J5uO1uxLAO-Sed zB+AUH1{k*xaR%tQxCdPxOsGC~3MDOCezM4fQZCw&$M>#~2_LIjd5(dJ@MC&s#6D*- z&D8{CW==2%tOv}BRv`R?!`Y%W?+EKv!i5&iJh1vY$d_enklt7CIjz>tm4RBjUG2E? z6Vy(QFe>EFlKNZFoO}|a3F64iu@)KOt8U{|x1Zia=#oiGTy<+Kq8Ae^1|I#k^77%m z^&7X(IBORlpSuKqdKvkHU(TmWAIf<(jL5FJi!%_zKQ&BFxeU`a_<5%G++=KxPo2J=C^-# zu|Lw=GWoBn_5U2b>qHVbd^h{a?ElBsTLr}xw%fY61`80}9fAdSha@-zNzkAHf;aAN zjRgV(cXt9b?(Xi;xLa^PlYgyUwa;Gbva5?KE_#0Rd&fJ*Gp==k1J5r_(+_YA9Gda< z`5}`RqSUbIfl1bPk=7S+9D#tS8I$gk+Sf9D~WsUg9PA(1Z$rT4+YTcPp5I zNX@A7Axmv9Pqxx>awS;Bsx-K3wKW*&LhCT=+8l(_aeD}m@6cPF(Zn=YU{dB^55%4R zW6~E@&<^;=&VZ1AKkE>AArLw`AO-+S0!l>AZ|Ipgk~c!?G|iPV zD4!Ef8OJv$qeF-HozkR@KRwdFX$+_222O{g+rtb3!!s7gNceE2qRa12BwE%V(hQcd z)|C~j_Me#_Ng0-_F)!Yk403i~P`